Merge pull request #256 from multinet-app/standardize_validation_resp…

…onse

Standardize validation response

multinet/__init__.py

@@ -6,9 +6,9 @@
 
 from typing import Optional, MutableMapping, Any, Tuple, Union
 
-from . import api
-from . import uploaders, downloaders
-from .errors import ServerError
+from multinet import api
+from multinet import uploaders, downloaders
+from multinet.errors import ServerError
 
 
 def create_app(config: Optional[MutableMapping] = None) -> Flask:

multinet/api.py

@@ -4,11 +4,12 @@
 from webargs import fields
 from webargs.flaskparser import use_kwargs
 
-from typing import Any, Optional
-from .types import EdgeDirection, TableType
+from typing import Any, Optional, List
+from multinet.types import EdgeDirection, TableType
+from multinet.validation import ValidationFailure, UndefinedKeys, UndefinedTable
 
-from . import db, util
-from .errors import (
+from multinet import db, util
+from multinet.errors import (
     ValidationFailed,
     BadQueryArgument,
     MalformedRequestBody,
@@ -156,21 +157,17 @@ def create_graph(workspace: str, graph: str, edge_table: Optional[str] = None) -
     from_tables = edge_table_properties["from_tables"]
     to_tables = edge_table_properties["to_tables"]
 
-    errors = []
+    errors: List[ValidationFailure] = []
     for table, keys in referenced_tables.items():
         if not loaded_workspace.has_collection(table):
-            errors.append(f"Reference to undefined table: {table}")
+            errors.append(UndefinedTable(table=table))
         else:
             table_keys = set(loaded_workspace.collection(table).keys())
             undefined = keys - table_keys
 
             if undefined:
-                errors.append(
-                    f"Nonexistent keys {', '.join(undefined)} "
-                    f"referenced in table: {table}"
-                )
+                errors.append(UndefinedKeys(table=table, keys=list(undefined)))
 
-    # TODO: Update this with the proper JSON schema
     if errors:
         raise ValidationFailed(errors)
 

multinet/db.py

@@ -8,9 +8,9 @@
 
 from typing import Callable, Any, Optional, Sequence, List, Set, Generator, Tuple
 from mypy_extensions import TypedDict
-from .types import EdgeDirection, TableType
+from multinet.types import EdgeDirection, TableType
 
-from .errors import (
+from multinet.errors import (
     BadQueryArgument,
     WorkspaceNotFound,
     TableNotFound,

multinet/errors.py

@@ -1,7 +1,8 @@
 """Exception objects representing Multinet-specific HTTP error conditions."""
+from typing import Tuple, Any, Union, List, Sequence
+from typing_extensions import TypedDict
 
-from typing import Tuple, Any, Union, List
-from mypy_extensions import TypedDict
+from multinet.validation import ValidationFailure
 
 
 FlaskTuple = Tuple[Any, Union[int, str]]
@@ -146,9 +147,9 @@ def flask_response(self) -> FlaskTuple:
 class ValidationFailed(ServerError):
     """Exception for reporting validation errors."""
 
-    def __init__(self, errors: List[Any]):
+    def __init__(self, errors: Sequence[ValidationFailure]):
         """Initialize the exception."""
-        self.errors = errors
+        self.errors = [error.asdict() for error in errors]
 
     def flask_response(self) -> FlaskTuple:
         """Generate a 400 error."""

multinet/types.py

@@ -2,7 +2,6 @@
 from typing import Dict, Set
 from typing_extensions import Literal, TypedDict
 
-
 EdgeDirection = Literal["all", "incoming", "outgoing"]
 TableType = Literal["all", "node", "edge"]
 

multinet/uploaders/csv.py

@@ -1,64 +1,67 @@
 """Multinet uploader for CSV files."""
 import csv
+import re
 from flasgger import swag_from
 from io import StringIO
-import re
+from dataclasses import dataclass
 
-from .. import db, util
-from ..errors import ValidationFailed
-from ..util import decode_data
+from multinet import db, util
+from multinet.errors import ValidationFailed
+from multinet.util import decode_data
+from multinet.validation import ValidationFailure, DuplicateKey, UnsupportedTable
 
 from flask import Blueprint, request
 from flask import current_app as app
 
 # Import types
-from typing import Set, MutableMapping, Sequence, Any
+from typing import Set, MutableMapping, Sequence, Any, List
 
 
 bp = Blueprint("csv", __name__)
 bp.before_request(util.require_db)
 
 
+@dataclass
+class InvalidRow(ValidationFailure):
+    """Invalid syntax in a CSV file."""
+
+    row: int
+    fields: List[str]
+
+
 def validate_csv(rows: Sequence[MutableMapping]) -> None:
     """Perform any necessary CSV validation, and return appropriate errors."""
-    data_errors = []
+    data_errors: List[ValidationFailure] = []
 
     fieldnames = rows[0].keys()
     if "_key" in fieldnames:
         # Node Table, check for key uniqueness
         keys = [row["_key"] for row in rows]
         unique_keys: Set[str] = set()
-        duplicates = set()
         for key in keys:
             if key in unique_keys:
-                duplicates.add(key)
+                data_errors.append(DuplicateKey(key=key))
             else:
                 unique_keys.add(key)
 
-        if len(duplicates) > 0:
-            data_errors.append({"error": "duplicate", "detail": list(duplicates)})
     elif "_from" in fieldnames and "_to" in fieldnames:
         # Edge Table, check that each cell has the correct format
         valid_cell = re.compile("[^/]+/[^/]+")
 
-        detail = []
-
         for i, row in enumerate(rows):
-            fields = []
+            fields: List[str] = []
             if not valid_cell.match(row["_from"]):
                 fields.append("_from")
             if not valid_cell.match(row["_to"]):
                 fields.append("_to")
 
             if fields:
                 # i+2 -> +1 for index offset, +1 due to header row
-                detail.append({"fields": fields, "row": i + 2})
+                data_errors.append(InvalidRow(fields=fields, row=i + 2))
 
-        if detail:
-            data_errors.append({"error": "syntax", "detail": detail})
     else:
         # Unsupported Table, error since we don't know what's coming in
-        data_errors.append({"error": "unsupported"})
+        data_errors.append(UnsupportedTable())
 
     if len(data_errors) > 0:
         raise ValidationFailed(data_errors)

multinet/uploaders/d3_json.py

@@ -1,44 +1,66 @@
 """Multinet uploader for nested JSON files."""
-from flasgger import swag_from
 import json
 from io import StringIO
+from flasgger import swag_from
+from dataclasses import dataclass
 from collections import OrderedDict
 
-from .. import db, util
-from ..errors import ValidationFailed
-from ..util import decode_data
+from multinet import db, util
+from multinet.errors import ValidationFailed
+from multinet.util import decode_data
+from multinet.validation import ValidationFailure
 
 from flask import Blueprint, request
 
 # Import types
-from typing import Any, List
+from typing import Any, List, Sequence
 
 bp = Blueprint("d3_json", __name__)
 bp.before_request(util.require_db)
 
 
-def validate_d3_json(data: dict) -> List[dict]:
+@dataclass
+class InvalidStructure(ValidationFailure):
+    """Invalid structure in a D3 JSON file."""
+
+
+@dataclass
+class InvalidLinkKeys(ValidationFailure):
+    """Invalid link keys in a D3 JSON file."""
+
+
+@dataclass
+class InconsistentLinkKeys(ValidationFailure):
+    """Inconsistent link keys in a D3 JSON file."""
+
+
+@dataclass
+class NodeDuplicates(ValidationFailure):
+    """Duplicate nodes in a D3 JSON file."""
+
+
+def validate_d3_json(data: dict) -> Sequence[ValidationFailure]:
     """Perform any necessary d3 json validation, and return appropriate errors."""
-    data_errors = []
+    data_errors: List[ValidationFailure] = []
 
     # Check the structure of the uploaded file is what we expect
     if "nodes" not in data.keys() or "links" not in data.keys():
-        data_errors.append({"error": "structure"})
+        data_errors.append(InvalidStructure())
 
     # Check that links are in source -> target form
     if not all(
         "source" in row.keys() and "target" in row.keys() for row in data["links"]
     ):
-        data_errors.append({"error": "invalid_link_keys"})
+        data_errors.append(InvalidLinkKeys())
 
     # Check that the keys for each dictionary match
     if not all(data["links"][0].keys() == row.keys() for row in data["links"]):
-        data_errors.append({"error": "inconsistent_link_keys"})
+        data_errors.append(InconsistentLinkKeys())
 
     # Check for duplicated nodes
     ids = set(row["id"] for row in data["nodes"])
     if len(data["nodes"]) != len(ids):
-        data_errors.append({"error": "node_duplicates"})
+        data_errors.append(NodeDuplicates())
 
     return data_errors
 

multinet/uploaders/nested_json.py

@@ -3,7 +3,7 @@
 import itertools
 import json
 
-from .. import db, util
+from multinet import db, util
 
 from flask import Blueprint, request
 

multinet/uploaders/newick.py

@@ -3,62 +3,60 @@
 import uuid
 import newick
 
-from .. import db, util
-from ..errors import ValidationFailed
-from ..util import decode_data
+from multinet import db, util
+from multinet.errors import ValidationFailed
+from multinet.util import decode_data
+from multinet.validation import ValidationFailure, DuplicateKey
 
+from dataclasses import dataclass
 from flask import Blueprint, request
 from flask import current_app as app
 
-from typing import Any, Optional, List, Dict, Set, FrozenSet, Tuple
+from typing import Any, Optional, List, Set, Tuple
 
 bp = Blueprint("newick", __name__)
 bp.before_request(util.require_db)
 
 
+@dataclass
+class DuplicateEdge(ValidationFailure):
+    """The edge which is duplicated."""
+
+    _from: str
+    _to: str
+    length: int
+
+
 def validate_newick(tree: List[newick.Node]) -> None:
     """Validate newick tree."""
-    data_errors: List[Dict[str, Any]] = []
+    data_errors: List[ValidationFailure] = []
     unique_keys: Set[str] = set()
-    duplicate_keys: Set[str] = set()
-    unique_edges: Set[FrozenSet[Tuple[str, object]]] = set()
-    duplicate_edges: Set[FrozenSet[Tuple[str, object]]] = set()
+    unique_edges: Set[Tuple[str, str, float]] = set()
 
     def read_tree(parent: Optional[str], node: newick.Node) -> None:
         key = node.name or uuid.uuid4().hex
 
         if key in unique_keys:
-            duplicate_keys.add(key)
+            data_errors.append(DuplicateKey(key=key))
         else:
             unique_keys.add(key)
 
         for desc in node.descendants:
             read_tree(key, desc)
 
         if parent:
-            edge = frozenset(
-                {
-                    "_from": f"table/{parent}",
-                    "_to": f"table/{key}",
-                    "length": node.length,
-                }.items()
-            )
-
-            if edge in unique_edges:
-                duplicate_edges.add(edge)
+            unique = (parent, key, node.length)
+            if unique in unique_edges:
+                data_errors.append(
+                    DuplicateEdge(
+                        _from=f"table/{parent}", _to=f"table/{key}", length=node.length
+                    )
+                )
             else:
-                unique_edges.add(edge)
+                unique_edges.add(unique)
 
     read_tree(None, tree[0])
 
-    if len(duplicate_keys) > 0:
-        data_errors.append({"error": "duplicate", "detail": list(duplicate_keys)})
-
-    if len(duplicate_edges) > 0:
-        data_errors.append(
-            {"error": "duplicate", "detail": [dict(x) for x in duplicate_edges]}
-        )
-
     if len(data_errors) > 0:
         raise ValidationFailed(data_errors)
     else:

multinet/util.py

@@ -3,12 +3,11 @@
 import os
 
 from flask import Response
-
 from typing import Sequence, Any, Generator, Dict, Set
 
-from . import db
-from .errors import DatabaseNotLive, DecodeFailed
-from .types import EdgeTableProperties
+from multinet import db
+from multinet.types import EdgeTableProperties
+from multinet.errors import DatabaseNotLive, DecodeFailed
 
 TEST_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test/data"))