diff --git a/README.md b/README.md index 9b1320c..ddf8bde 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Other papers related to SeqScore include: * [If You Build Your Own NER Scorer, Non-replicable Results Will Come](https://aclanthology.org/2020.insights-1.15.pdf) * [Toward More Meaningful Resources for Lower-resourced Languages](https://aclanthology.org/2022.findings-acl.44/) * [CoNLL#: Fine-grained Error Analysis and a Corrected Test Set for CoNLL-03 English](https://aclanthology.org/2024.lrec-main.330/) +* [OpenNER 1.0: Standardized Open-Access Named Entity Recognition Datasets in 50+ Languages](https://aclanthology.org/2025.emnlp-main.1708/) # Usage @@ -86,7 +87,7 @@ For a list of commands, run `seqscore --help`: $ seqscore --help Usage: seqscore [OPTIONS] COMMAND [ARGS]... - Provides scoring and analysis tools for NER/chunking files (version 0.6.0) + Provides scoring and analysis tools for NER/chunking files (version 0.9.0) Options: --version Show the version and exit. @@ -209,7 +210,7 @@ scoring will fail: ``` seqscore.encoding.EncodingError: Stopping due to validation errors in invalid.bio: -Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 +Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 of invalid.bio ``` To score output with invalid transitions, we need to specify a repair method which can @@ -218,8 +219,8 @@ we refer to as "begin" repair in our paper): `seqscore score --labels BIO --repair-method conlleval --reference samples/reference.bio samples/invalid.bio`: ``` -Validation errors in sequence at line 7 of invalid.bio: -Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 +Validation errors in sequence beginning at line 7 of invalid.bio: +Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 of invalid.bio Used method conlleval to repair: Old: ('I-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'B-LOC', 'O') New: ('B-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'B-LOC', 'O') @@ -271,14 +272,14 @@ To check if a file has any invalid transitions, we can run `seqscore validate --labels BIO samples/reference.bio`: ``` -No errors found in 0 tokens, 2 sequences, and 1 documents in reference.bio +No errors found in 15 tokens, 2 sequences, and 1 document(s) in reference.bio ``` For the example of the [samples/invalid.bio](samples/invalid.bio), we can run `seqscore validate --labels BIO samples/invalid.bio`: ``` -Encountered 1 errors in 1 tokens, 2 sequences, and 1 documents in invalid.bio +Encountered 1 errors in 15 tokens, 2 sequences, and 1 document(s) in invalid.bio Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 ``` @@ -313,11 +314,21 @@ We can get a list of available chunk encodings by running `seqscore convert --he ``` Usage: seqscore convert [OPTIONS] FILE OUTPUT_FILE + convert between mention encodings + Options: --file-encoding TEXT [default: UTF-8] - --ignore-comment-lines - --ignore-document-boundaries / --use-document-boundaries - --output-delim TEXT [default: space] + --allow-comment-lines allow comment lines starting with # before + sequences + --ignore-document-boundaries ignore any document boundaries in the input + --token-index INTEGER index of the input field to use for the + token [default: 0] + --label-index INTEGER index of the input field to use for the + label [default: -1] + --output-delim TEXT the delimiter to be used for output (has no + effect on input) [default: tab] + --discard-extra-fields discard any fields other than the token and + label when writing output files --input-labels [BIO|BIOES|BILOU|BMES|BMEOW|IO|IOB] [required] --output-labels [BIO|BIOES|BILOU|BMES|BMEOW|IO|IOB] @@ -387,11 +398,13 @@ For example, if we run `seqscore summarize --labels BIO samples/reference.bio` w the following output: ``` -File 'samples/reference.bio' contains 1 document(s) with the following mentions: +File 'samples/reference.bio' contains 1 document(s) and 2 sentences | Entity Type | Count | |---------------|---------| | LOC | 2 | | ORG | 1 | +|---------------|---------| +| TOTAL | 3 | ``` If the quiet (`-q`) flag is provided, the first line giving the filename and document @@ -574,10 +587,13 @@ For development, check out the `dev` branch (latest, but less tested than `main` ## Setting up an environment for development +Python 3.10 (the minimum version SeqScore supports) is used for development to avoid +accidentally using newer Python features. + ### Using uv -1. Create an environment: `uv venv --python 3.10 .venv` -2. Install seqscore and development dependencies: `uv pip install -e ".[dev]"` +Create an environment and install seqscore with development dependencies: +`uv sync --python 3.10 --extra dev` ### Using conda diff --git a/pyproject.toml b/pyproject.toml index 38cb8e9..0c8d9d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,8 @@ authors = [ ] requires-python = ">=3.10" dependencies = [ - "attrs>=19.2.0", "click", + "pydantic>=2.0", "tabulate", ] classifiers = [ diff --git a/seqscore/__init__.py b/seqscore/__init__.py index 777f190..3e2f46a 100644 --- a/seqscore/__init__.py +++ b/seqscore/__init__.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.9.0" diff --git a/seqscore/conll.py b/seqscore/conll.py index 6c75ecc..ab2ae7e 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -1,6 +1,7 @@ import sys from collections import Counter, defaultdict from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field from itertools import chain from math import sqrt from statistics import mean, stdev @@ -11,11 +12,10 @@ TextIO, ) -from attr import attrib, attrs from tabulate import tabulate from seqscore.encoding import Encoding, EncodingError, get_encoding -from seqscore.model import LabeledSequence, SequenceProvenance +from seqscore.model import AnnotatedSequence, LabeledSequence, SequenceProvenance from seqscore.scoring import ( AccuracyScore, ClassificationScore, @@ -46,17 +46,17 @@ class CoNLLFormatError(Exception): pass -@attrs(frozen=True) +@dataclass(frozen=True) class LineSpec: """Defines the fields and delimiters for a CoNLL-format line""" - token_index: int = attrib() - ner_label_index: int = attrib() + token_index: int + ner_label_index: int - def __attrs_post_init__(self) -> None: + def __post_init__(self) -> None: # This will only catch cases where the indices are identical, not - # when they refer to the same position, such as 1 and -1 in a - # sequence of length two + # when they refer to the same position (such as 1 and -1 in a + # sequence of length two). if self.token_index == self.ner_label_index: raise ValueError( f"Token index ({self.token_index}) and " @@ -64,13 +64,13 @@ def __attrs_post_init__(self) -> None: ) -@attrs(frozen=True) +@dataclass(frozen=True, slots=True) class _CoNLLToken: - text: str = attrib() - label: str = attrib() - is_docstart: bool = attrib() - line_num: int = attrib() - orig_fields: tuple[str, ...] = attrib() + text: str + label: str + is_docstart: bool + line_num: int + orig_fields: tuple[str, ...] @classmethod def from_line( @@ -87,7 +87,7 @@ def from_line( if line.startswith("#"): raise CoNLLFormatError( f"Line {line_num} of {source_name} does not appear to be delimited " - "and begins with #. Perhaps you want to use the --parse-comment-lines " + "and begins with #. Perhaps you want to use the --allow-comment-lines " f"flag? Line contents: {repr(line)}" ) else: @@ -102,12 +102,12 @@ def from_line( return cls(text, label, is_docstart, line_num, orig_fields) -@attrs(frozen=True) +@dataclass(frozen=True) class CoNLLIngester: - encoding: Encoding = attrib() - line_spec: LineSpec = attrib() - parse_comment_lines: bool = attrib(default=False, kw_only=True) - ignore_document_boundaries: bool = attrib(default=False, kw_only=True) + encoding: Encoding + line_spec: LineSpec + allow_comment_lines: bool = field(default=False, kw_only=True) + ignore_document_boundaries: bool = field(default=False, kw_only=True) def ingest( self, @@ -116,12 +116,12 @@ def ingest( repair: Optional[str], *, quiet: bool = False, - ) -> list[list[LabeledSequence]]: - all_documents: list[list[LabeledSequence]] = [] - document: list[LabeledSequence] = [] + ) -> list[list[AnnotatedSequence]]: + all_documents: list[list[AnnotatedSequence]] = [] + document: list[AnnotatedSequence] = [] for source_sequence, comment in self._parse_file( - source, source_name, parse_comments=self.parse_comment_lines + source, source_name, parse_comments=self.allow_comment_lines ): if source_sequence[0].is_docstart: # We can ony receive DOCSTART in a sequence by itself, see _parse_file. @@ -157,7 +157,7 @@ def ingest( err.label, str(err) + f" The first token {repr(tokens[0])} of this sentence starts with '#'." - + " If it's a comment, consider enabling --parse-comment-lines.", + + " If it's a comment, consider enabling --allow-comment-lines.", ) from err else: raise err @@ -206,11 +206,11 @@ def ingest( ) from e try: - sequences = LabeledSequence( - tokens, - labels, - mentions, - orig_fields=orig_fields, + sequences = AnnotatedSequence( + tokens=tokens, + labels=labels, + mentions=tuple(mentions), + token_fields=orig_fields, provenance=SequenceProvenance(line_nums[0], source_name), comment=comment, ) @@ -238,7 +238,7 @@ def validate( for source_sequence, _ in self._parse_file( source, source_name, - parse_comments=self.parse_comment_lines, + parse_comments=self.allow_comment_lines, ): if source_sequence[0].is_docstart: # We can ony receive DOCSTART in a sequence by itself, see _parse_file. @@ -356,9 +356,9 @@ def ingest_conll_file( *, repair: Optional[str] = None, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, quiet: bool = False, -) -> list[list[LabeledSequence]]: +) -> list[list[AnnotatedSequence]]: mention_encoding = get_encoding(mention_encoding_name) if repair and repair not in mention_encoding.supported_repair_methods(): @@ -370,7 +370,7 @@ def ingest_conll_file( ingester = CoNLLIngester( mention_encoding, line_spec, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ignore_document_boundaries=ignore_document_boundaries, ) with open(input_path, encoding=file_encoding) as input_file: @@ -385,13 +385,13 @@ def validate_conll_file( line_spec: LineSpec, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, ) -> ValidationResult: encoding = get_encoding(mention_encoding_name) ingester = CoNLLIngester( encoding, line_spec, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ignore_document_boundaries=ignore_document_boundaries, ) with open(input_path, encoding=file_encoding) as input_file: @@ -401,7 +401,7 @@ def validate_conll_file( n_sequences = sum(len(doc_results) for doc_results in results) n_tokens = sum(sent.n_tokens for doc_results in results for sent in doc_results) - errors = list( + errors = tuple( chain.from_iterable( result.errors for doc_results in results for result in doc_results ) @@ -411,42 +411,40 @@ def validate_conll_file( def repair_conll_file( input_file: PathType, - output_file: PathType, mention_encoding_name: str, repair: Optional[str], file_encoding: str, line_spec: LineSpec, - output_delim: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, quiet: bool, -) -> None: - docs = ingest_conll_file( +) -> list[list[AnnotatedSequence]]: + return ingest_conll_file( input_file, mention_encoding_name, file_encoding, line_spec, repair=repair, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) - write_docs_using_encoding( - docs, mention_encoding_name, file_encoding, output_delim, line_spec, output_file - ) def write_docs_using_encoding( - docs: Sequence[Sequence[LabeledSequence]], + docs: Sequence[Sequence[AnnotatedSequence]], mention_encoding_name: str, file_encoding: str, delim: str, line_spec: LineSpec, output_path: PathType, + *, + discard_extra_fields: bool = False, + always_write_docstart: bool = False, ) -> None: mention_encoding = get_encoding(mention_encoding_name) - output_docstart = len(docs) > 1 + output_docstart = len(docs) > 1 or always_write_docstart with open(output_path, "w", encoding=file_encoding) as file: for doc in docs: @@ -457,41 +455,67 @@ def write_docs_using_encoding( file, line_spec, output_docstart=output_docstart, + discard_extra_fields=discard_extra_fields, ) -def write_doc_using_encoding( +def write_docs_raw( + docs: Sequence[Sequence[LabeledSequence]], + file_encoding: str, + delim: str, + line_spec: LineSpec, + output_path: PathType, + *, + outside_label: str = "O", + discard_extra_fields: bool = False, + always_write_docstart: bool = False, +) -> None: + output_docstart = len(docs) > 1 or always_write_docstart + + with open(output_path, "w", encoding=file_encoding) as file: + for doc in docs: + write_doc_raw( + doc, + delim, + file, + line_spec, + output_docstart=output_docstart, + outside_label=outside_label, + discard_extra_fields=discard_extra_fields, + ) + + +def write_doc_raw( doc: Sequence[LabeledSequence], - encoding: Encoding, delim: str, file: TextIO, line_spec: LineSpec, *, output_docstart: bool, + outside_label: str = "O", + discard_extra_fields: bool = False, ) -> None: if output_docstart: # Get the fields of the first token of the first sentence - if doc[0].orig_fields: - # to figure out how many fields there are - sequence_orig_fields = doc[0].orig_fields[0] - # Create the write number of fields + if doc[0].token_fields and not discard_extra_fields: + # Figure out how many fields there are + sequence_orig_fields = doc[0].token_fields[0] + # Create the right number of fields fields = [EMPTY_OTHER_FIELD] * len(sequence_orig_fields) # Fill in the token and label fields[line_spec.token_index] = DOCSTART - fields[line_spec.ner_label_index] = encoding.dialect.outside + fields[line_spec.ner_label_index] = outside_label else: - fields = [DOCSTART, encoding.dialect.outside] + fields = [DOCSTART, outside_label] # Write output print(delim.join(fields), file=file) print(file=file) for sequence in doc: - labels = encoding.encode_sequence(sequence) - # Lengths of labels and orig_fields have previously been checked to match tokens for (token, orig_fields), label in zip( - sequence.tokens_with_orig_fields(), labels + sequence.tokens_with_fields(), sequence.labels ): - if orig_fields: + if orig_fields and not discard_extra_fields: fields = list(orig_fields) fields[line_spec.token_index] = token fields[line_spec.ner_label_index] = label @@ -500,10 +524,44 @@ def write_doc_using_encoding( # Write output print(delim.join(fields), file=file) - # Print an emtpy line after each sequence + # Print an empty line after each sequence print(file=file) +def write_doc_using_encoding( + doc: Sequence[AnnotatedSequence], + encoding: Encoding, + delim: str, + file: TextIO, + line_spec: LineSpec, + *, + output_docstart: bool, + discard_extra_fields: bool = False, +) -> None: + # Re-encode mentions -> labels for each sequence, then defer to write_doc_raw + # for the actual DOCSTART/token/blank-line writing so the two writers share + # one implementation. + raw_doc = [ + LabeledSequence( + tokens=sequence.tokens, + labels=tuple(encoding.encode_sequence(sequence)), + token_fields=sequence.token_fields, + provenance=sequence.provenance, + comment=sequence.comment, + ) + for sequence in doc + ] + write_doc_raw( + raw_doc, + delim, + file, + line_spec, + output_docstart=output_docstart, + outside_label=encoding.dialect.outside, + discard_extra_fields=discard_extra_fields, + ) + + # TODO: Refactor to remove CoNLL-specific file loading so that this can move to the scoring module def score_conll_files( pred_files: Sequence[PathType], @@ -514,7 +572,7 @@ def score_conll_files( line_spec: LineSpec, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, output_format: str, delim: str, error_counts: bool = False, @@ -530,7 +588,7 @@ def score_conll_files( line_spec, repair=repair, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) @@ -555,7 +613,7 @@ def score_conll_files( line_spec, repair=repair, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) @@ -589,13 +647,19 @@ def score_conll_files( (error_type, item.type, " ".join(item.tokens)) ] = count + # Sort by count descending (the negative reverses the default + # ascending sort), breaking ties on the token string + # (item[0] is the (error_type, mention_type, token_str) key, + # so item[0][2] is the token string; item[1] is the count). rows = [ [count, error_type, mention_type, token_str] for ( error_type, mention_type, token_str, - ), count in combined_counts.most_common() + ), count in sorted( + combined_counts.items(), key=lambda item: (-item[1], item[0][2]) + ) ] if output_format == FORMAT_PRETTY: diff --git a/seqscore/encoding.py b/seqscore/encoding.py index 9efae76..fceba2a 100644 --- a/seqscore/encoding.py +++ b/seqscore/encoding.py @@ -1,5 +1,6 @@ from abc import abstractmethod from collections.abc import Sequence +from dataclasses import dataclass, field from functools import lru_cache from typing import ( AbstractSet, @@ -7,9 +8,7 @@ Protocol, ) -from attr import Factory, attrib, attrs - -from seqscore.model import LabeledSequence, Mention, Span +from seqscore.model import AnnotatedSequence, LabeledSequence, Mention, Span REPAIR_CONLL = "conlleval" REPAIR_DISCARD = "discard" @@ -134,7 +133,7 @@ def encode_mentions( def encode_sequence( self, - sequence: LabeledSequence, + sequence: AnnotatedSequence, ) -> Sequence[str]: labels = self.encode_mentions(sequence.mentions, len(sequence)) assert len(labels) == len(sequence) @@ -660,11 +659,11 @@ def get_encoding(name: str) -> Encoding: raise ValueError(f"Unknown encoder {repr(name)}") -@attrs +@dataclass class _MentionBuilder: - start_idx: Optional[int] = attrib(default=None, init=False) - entity_type: Optional[str] = attrib(default=None, init=False) - mentions: list[Mention] = attrib(default=Factory(list), init=False) + start_idx: Optional[int] = field(default=None, init=False) + entity_type: Optional[str] = field(default=None, init=False) + mentions: list[Mention] = field(default_factory=list, init=False) def start_mention(self, start_idx: int, entity_type: str) -> None: # Check arguments diff --git a/seqscore/model.py b/seqscore/model.py index 2921b50..74f69a8 100644 --- a/seqscore/model.py +++ b/seqscore/model.py @@ -1,36 +1,50 @@ -from collections.abc import Iterable, Iterator, Sequence +from collections.abc import Iterator, Sequence +from dataclasses import dataclass from itertools import repeat -from typing import TYPE_CHECKING, Any, Optional, Union, overload - -from attr import Attribute, attrib, attrs - -from seqscore.util import ( - tuplify_optional_nested_strs, - tuplify_strs, - validator_nonempty_str, +from typing import ( + TYPE_CHECKING, + Any, + Optional, + Protocol, + Union, + overload, + runtime_checkable, ) +from pydantic import BaseModel, ConfigDict, model_validator + if TYPE_CHECKING: from seqscore.encoding import Encoding # pragma: no cover +__all__ = [ + "AnnotatedSequence", + "LabeledSequence", + "Mention", + "SequenceProvenance", + "Span", + "TokenSequence", +] -def _validator_nonnegative(_inst: Any, _attr: Attribute, value: Any) -> None: - if value < 0: - raise ValueError(f"Negative value: {repr(value)}") +@dataclass(frozen=True, slots=True) +class Span: + """A token index range [start, end) within a sequence. -def _tuplify_mentions( - mentions: Iterable["Mention"], -) -> tuple["Mention", ...]: - return tuple(mentions) + The start index is inclusive, and the end index is exclusive. + Both start and end must be non-negative, and end must be + greater than start. Zero-length spans are not allowed. + """ + start: int + end: int -@attrs(frozen=True, slots=True) -class Span: - start: int = attrib(validator=_validator_nonnegative) - end: int = attrib(validator=_validator_nonnegative) + def __post_init__(self) -> None: + if self.start < 0: + raise ValueError(f"Start ({self.start}) cannot be negative") + + if self.end < 0: + raise ValueError(f"End ({self.end}) cannot be negative") - def __attrs_post_init__(self) -> None: if not self.end > self.start: raise ValueError( f"End of span ({self.end}) must be greater than start ({self.start}" @@ -40,40 +54,86 @@ def __len__(self) -> int: return self.end - self.start -@attrs(frozen=True, slots=True) +@dataclass(frozen=True, slots=True) class Mention: - span: Span = attrib() - type: str = attrib(validator=validator_nonempty_str) + """A typed span representing a named entity or chunk. + + Combines a Span with a string entity type. + """ + + span: Span + type: str + + def __post_init__(self) -> None: + if not isinstance(self.type, str): + raise TypeError(f"Expected str for type, got {type(self.type).__name__}") + if not self.type: + raise ValueError(f"Empty string for type: {repr(self.type)}") def __len__(self) -> int: return len(self.span) def with_type(self, new_type: str) -> "Mention": + """Return a new Mention with the same span and the provided type.""" return Mention(self.span, new_type) -@attrs(frozen=True, slots=True) +@dataclass(frozen=True, slots=True) class SequenceProvenance: - starting_line: int = attrib() - source: Optional[str] = attrib() - - -@attrs(frozen=True, slots=True) -class LabeledSequence(Sequence[str]): - tokens: tuple[str, ...] = attrib(converter=tuplify_strs) - labels: tuple[str, ...] = attrib(converter=tuplify_strs) - mentions: tuple[Mention, ...] = attrib(default=(), converter=_tuplify_mentions) - orig_fields: Optional[tuple[tuple[str, ...], ...]] = attrib( - default=None, kw_only=True, converter=tuplify_optional_nested_strs - ) - provenance: Optional[SequenceProvenance] = attrib( - default=None, eq=False, kw_only=True - ) - comment: Optional[str] = attrib(default=None, eq=False, kw_only=True) - - def __attrs_post_init__(self) -> None: - # TODO: Check for overlapping mentions + """Origin of a sequence, with a starting line and optional source name.""" + + starting_line: int + source: Optional[str] + + +@runtime_checkable +class TokenSequence(Protocol): + """Protocol for a sequence of tokens with optional metadata.""" + + @property + def tokens(self) -> tuple[str, ...]: ... + + @property + def token_fields(self) -> Optional[tuple[tuple[str, ...], ...]]: ... + + @property + def provenance(self) -> Optional[SequenceProvenance]: ... + + @property + def comment(self) -> Optional[str]: ... + + @overload + def __getitem__(self, index: int) -> str: ... + + @overload + def __getitem__(self, index: slice) -> tuple[str, ...]: ... + + def __getitem__(self, i: Union[int, slice]) -> Union[str, tuple[str, ...]]: ... + + def __iter__(self) -> Iterator[str]: ... + + def __len__(self) -> int: ... + def span_tokens(self, span: Span) -> tuple[str, ...]: ... + + def tokens_with_fields( + self, + ) -> tuple[tuple[str, Optional[tuple[str, ...]]], ...]: ... + + +class _SequenceBase(BaseModel, Sequence[str]): + """Shared base for token sequences with labels.""" + + model_config = ConfigDict(frozen=True) + + tokens: tuple[str, ...] + labels: tuple[str, ...] + token_fields: Optional[tuple[tuple[str, ...], ...]] = None + provenance: Optional[SequenceProvenance] = None + comment: Optional[str] = None + + @model_validator(mode="after") + def _validate_fields(self) -> "_SequenceBase": if len(self.tokens) != len(self.labels): raise ValueError( f"Tokens ({len(self.tokens)}) and labels ({len(self.labels)}) " @@ -82,43 +142,35 @@ def __attrs_post_init__(self) -> None: if not self.tokens: raise ValueError("Tokens and labels must be non-empty") - if self.orig_fields and len(self.tokens) != len(self.orig_fields): + if self.token_fields and len(self.tokens) != len(self.token_fields): raise ValueError( - f"Tokens ({len(self.tokens)}) and orig_fields ({len(self.orig_fields)}) " + f"Tokens ({len(self.tokens)}) and token_fields ({len(self.token_fields)}) " "must be of the same length" ) for idx, label in enumerate(self.labels): - # Labels cannot be None or an empty string if not label: raise ValueError(f"Invalid label at sequence index {idx}: {repr(label)}") for idx, token in enumerate(self.tokens): - # Tokens cannot be None or an empty string if not token: raise ValueError(f"Invalid token at sequence index {idx}: {repr(token)}") - def with_mentions(self, mentions: Sequence[Mention]) -> "LabeledSequence": - return LabeledSequence( - self.tokens, self.labels, mentions, provenance=self.provenance - ) + return self @overload - def __getitem__(self, index: int) -> str: - raise NotImplementedError + def __getitem__(self, index: int) -> str: ... @overload - def __getitem__(self, index: slice) -> tuple[str, ...]: - raise NotImplementedError + def __getitem__(self, index: slice) -> tuple[str, ...]: ... def __getitem__(self, i: Union[int, slice]) -> Union[str, tuple[str, ...]]: return self.tokens[i] - def __iter__(self) -> Iterator[str]: + def __iter__(self) -> Iterator[str]: # type: ignore[override] return iter(self.tokens) def __len__(self) -> int: - # Guaranteed that labels and tokens are same length by construction return len(self.tokens) def __str__(self) -> str: @@ -127,20 +179,74 @@ def __str__(self) -> str: ) def tokens_with_labels(self) -> tuple[tuple[str, str], ...]: + """Return a tuple of (token, label) tuples.""" return tuple(zip(self.tokens, self.labels)) - def tokens_with_orig_fields( + def tokens_with_fields( self, ) -> tuple[tuple[str, Optional[tuple[str, ...]]], ...]: - if self.orig_fields: - return tuple(zip(self.tokens, self.orig_fields)) + """Return a tuple of (token, token_fields) pairs, with None if token_fields is absent.""" + if self.token_fields: + return tuple(zip(self.tokens, self.token_fields)) else: return tuple(zip(self.tokens, repeat(None))) def span_tokens(self, span: Span) -> tuple[str, ...]: + """Return the tokens included in the given span.""" return self.tokens[span.start : span.end] + +class LabeledSequence(_SequenceBase): + """A sequence of tokens with their labels. + + This class only contains labels and tokens. For mentions, use + AnnotatedSequence. + + Equality and hashing are defined using only the tokens and labels. + """ + + def __eq__(self, other: object) -> bool: + if not isinstance(other, LabeledSequence): + return False + return self.tokens == other.tokens and self.labels == other.labels + + def __hash__(self) -> int: + return hash((self.tokens, self.labels)) + + +class AnnotatedSequence(_SequenceBase): + """A sequence of tokens with labels and decoded mentions. + + Equality and hashing are defined using only the tokens, labels, and mentions. + """ + + mentions: tuple[Mention, ...] + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AnnotatedSequence): + return False + return ( + self.tokens == other.tokens + and self.labels == other.labels + and self.mentions == other.mentions + ) + + def __hash__(self) -> int: + return hash((self.tokens, self.labels, self.mentions)) + + def with_mentions(self, mentions: Sequence[Mention]) -> "AnnotatedSequence": + """Return a copy of this sequence with different mentions.""" + return AnnotatedSequence( + tokens=self.tokens, + labels=self.labels, + mentions=tuple(mentions), + token_fields=self.token_fields, + provenance=self.provenance, + comment=self.comment, + ) + def mention_tokens(self, mention: Mention) -> tuple[str, ...]: + """Return the tokens included in the given mention.""" return self.span_tokens(mention.span) @classmethod @@ -150,6 +256,9 @@ def from_tokens_and_labels( labels: Sequence[str], encoding: "Encoding", **kwargs: Any, - ) -> "LabeledSequence": + ) -> "AnnotatedSequence": + """Create an AnnotatedSequence by decoding mentions from labels using the given encoding.""" mentions = encoding.decode_labels(labels) - return cls(tokens, labels, mentions, **kwargs) + return cls( + tokens=tuple(tokens), labels=tuple(labels), mentions=tuple(mentions), **kwargs + ) diff --git a/seqscore/processing.py b/seqscore/processing.py index 581a826..3a7ede5 100644 --- a/seqscore/processing.py +++ b/seqscore/processing.py @@ -1,6 +1,6 @@ from collections.abc import Iterable -from seqscore.model import LabeledSequence, Mention +from seqscore.model import AnnotatedSequence, Mention class TypeMapper: @@ -30,7 +30,7 @@ def __init__( else: self.type_map[from_type] = to_type - def map_types(self, sequence: LabeledSequence) -> LabeledSequence: + def map_types(self, sequence: AnnotatedSequence) -> AnnotatedSequence: new_mentions: list[Mention] = [] for mention in sequence.mentions: if mention.type in self.type_map: @@ -47,13 +47,13 @@ def map_types(self, sequence: LabeledSequence) -> LabeledSequence: def modify_types( - docs: list[list[LabeledSequence]], + docs: list[list[AnnotatedSequence]], keep_types: set[str], remove_types: set[str], type_map: dict[str, list[str]], -) -> list[list[LabeledSequence]]: +) -> list[list[AnnotatedSequence]]: mapper = TypeMapper(keep_types, remove_types, type_map) - mapped_docs: list[list[LabeledSequence]] = [] + mapped_docs: list[list[AnnotatedSequence]] = [] for doc in docs: mapped_docs.append([mapper.map_types(sequence) for sequence in doc]) diff --git a/seqscore/scoring.py b/seqscore/scoring.py index c0c8079..5b50876 100644 --- a/seqscore/scoring.py +++ b/seqscore/scoring.py @@ -1,13 +1,11 @@ from collections import Counter, defaultdict from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal from typing import DefaultDict, Optional, Union -from attr import Factory, attrib, attrs - from seqscore.encoding import Encoding, EncodingError, get_encoding -from seqscore.model import LabeledSequence, Mention -from seqscore.util import tuplify_strs, validator_nonempty_str +from seqscore.model import AnnotatedSequence, Mention from seqscore.validation import validate_labels @@ -15,10 +13,10 @@ def _defaultdict_classification_score() -> DefaultDict[str, "ClassificationScore return defaultdict(ClassificationScore) -@attrs(frozen=True, slots=True) +@dataclass(frozen=True, slots=True) class TokensWithType: - tokens: tuple[str, ...] = attrib(converter=tuplify_strs) - type: str = attrib(validator=validator_nonempty_str) + tokens: tuple[str, ...] + type: str class TokenCountError(ValueError): @@ -48,7 +46,7 @@ def __init__( @classmethod def from_predicted_sequence( - cls, reference_token_count: int, pred_sequence: LabeledSequence + cls, reference_token_count: int, pred_sequence: AnnotatedSequence ) -> "TokenCountError": if pred_sequence.provenance is None: raise ValueError( @@ -62,16 +60,16 @@ def from_predicted_sequence( ) -@attrs +@dataclass class ClassificationScore: - true_pos: int = attrib(default=0, kw_only=True) - false_pos: int = attrib(default=0, kw_only=True) - false_neg: int = attrib(default=0, kw_only=True) - type_scores: DefaultDict[str, "ClassificationScore"] = attrib( - default=Factory(_defaultdict_classification_score), kw_only=True + true_pos: int = field(default=0, kw_only=True) + false_pos: int = field(default=0, kw_only=True) + false_neg: int = field(default=0, kw_only=True) + type_scores: DefaultDict[str, "ClassificationScore"] = field( + default_factory=_defaultdict_classification_score, kw_only=True ) - false_pos_examples: Counter[TokensWithType] = attrib(default=Factory(Counter)) - false_neg_examples: Counter[TokensWithType] = attrib(default=Factory(Counter)) + false_pos_examples: Counter[TokensWithType] = field(default_factory=Counter) + false_neg_examples: Counter[TokensWithType] = field(default_factory=Counter) def count_false_positive(self, tokens: Iterable[str], type_: str) -> None: self.false_pos_examples[TokensWithType(tuple(tokens), type_)] += 1 @@ -117,10 +115,10 @@ def f1(self) -> float: return 2 * (precision * recall) / (precision + recall) -@attrs +@dataclass class AccuracyScore: - hits: int = attrib(default=0, kw_only=True) - total: int = attrib(default=0, kw_only=True) + hits: int = field(default=0, kw_only=True) + total: int = field(default=0, kw_only=True) @property def accuracy(self) -> float: @@ -130,18 +128,18 @@ def accuracy(self) -> float: def compute_scores( - pred_docs: Sequence[Sequence[LabeledSequence]], - ref_docs: Sequence[Sequence[LabeledSequence]], + pred_docs: Sequence[Sequence[AnnotatedSequence]], + ref_docs: Sequence[Sequence[AnnotatedSequence]], *, count_fp_fn_examples: bool = False, ) -> tuple[ClassificationScore, AccuracyScore]: accuracy = AccuracyScore() classification = ClassificationScore() - # TODO: Recommend use of ignore_document_boundaries if this error is encountered if len(pred_docs) != len(ref_docs): raise ValueError( - f"Prediction has {len(pred_docs)} documents, reference has {len(ref_docs)}" + f"Prediction has {len(pred_docs)} documents, reference has {len(ref_docs)}. " + "Consider setting --ignore-document-boundaries/ignore_document_boundaries." ) for pred_doc, ref_doc in zip(pred_docs, ref_docs): diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index 053cab2..00f52d4 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -26,13 +26,16 @@ ) from seqscore.processing import modify_types +# TODO: For each command, reorder click decorators so that --help has the most important arguments first + # Set up a click command group @click.group( help=f"Provides scoring and analysis tools for NER/chunking files (version {seqscore.__version__})" ) @click.version_option(seqscore.__version__) -# This is tested by a subprocess call in test_seqscore_main so coverage will miss it +# Tests invoke subcommands directly (e.g. runner.invoke(validate, [...])) rather +# than through this group, so tests never actually call this function. def cli() -> None: # pragma: no cover pass @@ -41,23 +44,29 @@ def cli() -> None: # pragma: no cover def _input_file_options() -> list[Callable]: return [ click.option("--file-encoding", default="UTF-8", show_default=True), - click.option("--parse-comment-lines", is_flag=True), click.option( - "--ignore-document-boundaries/--use-document-boundaries", default=False + "--allow-comment-lines", + is_flag=True, + help="allow comment lines starting with # before sequences", + ), + click.option( + "--ignore-document-boundaries", + is_flag=True, + help="ignore any document boundaries in the input", ), click.option( "--token-index", default=0, show_default=True, type=int, - help="Index of the input field to use for the token", + help="index of the input field to use for the token", ), click.option( "--label-index", default=-1, show_default=True, type=int, - help="Index of the input field to use for the label", + help="index of the input field to use for the label", ), ] @@ -121,6 +130,14 @@ def _output_delim_option() -> Callable: ) +def _discard_extra_fields_option() -> Callable: + return click.option( + "--discard-extra-fields", + is_flag=True, + help="discard any fields other than the token and label when writing output files", + ) + + def _quiet_option() -> Callable: return click.option( "--quiet", @@ -140,7 +157,7 @@ def validate( file_encoding: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, token_index: int, label_index: int, quiet: bool, @@ -154,14 +171,15 @@ def validate( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) if result.errors: print( f"Encountered {len(result.errors)} errors in {result.n_tokens} tokens, " + f"{result.n_sequences} sequences, and {result.n_docs} document(s) in {each_file}" ) - print("\n".join(err.msg for err in result.errors)) + if not quiet: + print("\n".join(err.msg for err in result.errors)) error = True elif not quiet: print( @@ -179,6 +197,7 @@ def validate( @_repair_required_option() @_labels_option() @_output_delim_option() +@_discard_extra_fields_option() @_quiet_option() def repair( file: str, @@ -191,7 +210,8 @@ def repair( output_delim: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, + discard_extra_fields: bool, quiet: bool, ) -> None: output_delim = _normalize_tab(output_delim) @@ -201,24 +221,32 @@ def repair( ) line_spec = LineSpec(token_index, label_index) - repair_conll_file( + docs = repair_conll_file( file, - output_file, labels, repair_method, file_encoding, line_spec, - output_delim, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) + write_docs_using_encoding( + docs, + labels, + file_encoding, + output_delim, + line_spec, + output_file, + discard_extra_fields=discard_extra_fields, + ) @cli.command(help="convert between mention encodings") @_single_input_file_arguments @click.argument("output_file") @_output_delim_option() +@_discard_extra_fields_option() @click.option("--input-labels", required=True, type=click.Choice(SUPPORTED_ENCODINGS)) @click.option("--output-labels", required=True, type=click.Choice(SUPPORTED_ENCODINGS)) def convert( @@ -232,7 +260,8 @@ def convert( output_labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, + discard_extra_fields: bool, ) -> None: output_delim = _normalize_tab(output_delim) line_spec = LineSpec(token_index, label_index) @@ -243,11 +272,17 @@ def convert( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) write_docs_using_encoding( - docs, output_labels, file_encoding, output_delim, line_spec, output_file + docs, + output_labels, + file_encoding, + output_delim, + line_spec, + output_file, + discard_extra_fields=discard_extra_fields, ) @@ -271,6 +306,7 @@ def convert( help="a JSON file containing types to be modified, in the format of a dict with keys as the target type and values as the source type [example file: {'MISC': ['WorkOfArt', 'Event']}]", ) @_output_delim_option() +@_discard_extra_fields_option() def process( file: str, output_file: str, @@ -284,7 +320,8 @@ def process( type_map: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, + discard_extra_fields: bool, ) -> None: output_delim = _normalize_tab(output_delim) line_spec = LineSpec(token_index, label_index) @@ -306,7 +343,7 @@ def process( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) try: @@ -315,7 +352,13 @@ def process( raise click.UsageError(str(err)) from err write_docs_using_encoding( - mod_docs, labels, file_encoding, output_delim, line_spec, output_file + mod_docs, + labels, + file_encoding, + output_delim, + line_spec, + output_file, + discard_extra_fields=discard_extra_fields, ) @@ -339,7 +382,7 @@ def count( labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, output_delim: str, repair_method: str, quiet: bool, @@ -363,7 +406,7 @@ def count( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, repair=repair_method, quiet=quiet, ) @@ -401,7 +444,7 @@ def summarize( labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, repair_method: str, quiet: bool, ) -> None: @@ -419,7 +462,7 @@ def summarize( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, repair=repair_method, quiet=quiet, ) @@ -445,7 +488,7 @@ def summarize( rows: list[Union[tuple[str, int], str]] = sorted(type_counts.items()) rows.append(SEPARATING_LINE) rows.append(("TOTAL", sum(type_counts.values()))) - print(tabulate(rows, header, intfmt=",")) + print(tabulate(rows, header, tablefmt="github", intfmt=",")) @cli.command(help="score a file and report performance or an error count table") @@ -481,7 +524,7 @@ def score( labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, reference: str, score_format: str, delim: str, @@ -515,7 +558,7 @@ def score( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, output_format=score_format, delim=delim, error_counts=error_counts, @@ -537,7 +580,7 @@ def extract_text( output_file: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, ) -> None: line_spec = LineSpec(token_index, label_index) all_docs = [] @@ -548,7 +591,7 @@ def extract_text( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) all_docs.extend(docs) diff --git a/seqscore/util.py b/seqscore/util.py index 666ead2..2eb54cc 100644 --- a/seqscore/util.py +++ b/seqscore/util.py @@ -1,32 +1,14 @@ import os -from collections.abc import Iterable from itertools import zip_longest from os import PathLike from pathlib import Path -from typing import Any, Optional, Union - -from attr import Attribute, validators +from typing import Union # Union[str, Path] isn't enough to appease PyCharm's type checker, so adding Path here # avoids warnings. PathType = Union[str, Path, PathLike] -# Type-specific implementations to work around type checker limitations. No, writing these as -# generic functions with type variables does not satisfy all type checkers. -def tuplify_strs(strs: Iterable[str]) -> tuple[str, ...]: - return tuple(strs) - - -def tuplify_optional_nested_strs( - items: Optional[Iterable[Iterable[str]]], -) -> Optional[tuple[tuple[str, ...], ...]]: - if items is not None: - return tuple(tuple(item) for item in items) - else: - return None - - def file_fields_match(path1: PathType, path2: PathType, *, debug: bool = False) -> bool: """Return whether the whitespace-delimited fields of two files are identical.""" with open(path1, encoding="utf8") as f1, open(path2, encoding="utf8") as f2: @@ -58,15 +40,3 @@ def file_lines_match(path1: PathType, path2: PathType, debug: bool = False) -> b def normalize_str_with_path(s: str) -> str: """Normalize the OS path separator to '/'.""" return s.replace(os.path.sep, "/") - - -# Instantiate in advance for _validator_nonempty_str -_instance_of_str = validators.instance_of(str) - - -def validator_nonempty_str(_inst: Any, attr: Attribute, value: Any) -> None: - # Check type - _instance_of_str(value, attr, value) - # Check string isn't empty - if not value: - raise ValueError(f"Empty string: {repr(value)}") diff --git a/seqscore/validation.py b/seqscore/validation.py index 9593713..bc2a332 100644 --- a/seqscore/validation.py +++ b/seqscore/validation.py @@ -1,24 +1,22 @@ -from collections.abc import Iterable, Sequence +from collections.abc import Sequence +from dataclasses import dataclass from typing import Any, Optional -from attr import attrib, attrs - from seqscore.encoding import _ENCODING_NAMES, Encoding, EncodingError -from seqscore.util import tuplify_strs # All encodings can be validated VALIDATION_SUPPORTED_ENCODINGS: Sequence[str] = tuple(_ENCODING_NAMES) -@attrs +@dataclass class ValidationError: - msg: str = attrib() - label: str = attrib() - type: str = attrib() - state: str = attrib() - token: Optional[str] = attrib(default=None) - line_num: Optional[int] = attrib(default=None) - source_name: Optional[str] = attrib(default=None) + msg: str + label: str + type: str + state: str + token: Optional[str] = None + line_num: Optional[int] = None + source_name: Optional[str] = None class InvalidStateError(ValidationError): @@ -35,17 +33,11 @@ def __init__(self, label: str, *args: Any, **kwargs: Any) -> None: self.label: str = label -def tuplify_errors(errors: Iterable[ValidationError]) -> tuple[ValidationError, ...]: - return tuple(errors) - - -@attrs +@dataclass class SequenceValidationResult: - errors: Sequence[ValidationError] = attrib(converter=tuplify_errors) - n_tokens: int = attrib() - repaired_labels: Optional[tuple[str, ...]] = attrib( - converter=tuplify_strs, default=() - ) + errors: tuple[ValidationError, ...] + n_tokens: int + repaired_labels: tuple[str, ...] = () def is_valid(self) -> bool: return not self.errors @@ -57,12 +49,12 @@ def __len__(self) -> int: return len(self.errors) -@attrs(frozen=True) +@dataclass(frozen=True) class ValidationResult: - errors: Sequence[ValidationError] = attrib(converter=tuplify_errors) - n_tokens: int = attrib() - n_sequences: int = attrib() - n_docs: int = attrib() + errors: tuple[ValidationError, ...] + n_tokens: int + n_sequences: int + n_docs: int def validate_labels( @@ -190,6 +182,8 @@ def validate_labels( if errors and repair: repaired_labels = encoding.repair_labels(labels, repair) - return SequenceValidationResult(errors, len(labels), repaired_labels) + return SequenceValidationResult( + tuple(errors), len(labels), tuple(repaired_labels) + ) else: - return SequenceValidationResult(errors, len(labels)) + return SequenceValidationResult(tuple(errors), len(labels)) diff --git a/tests/test_conll_format.py b/tests/test_conll_format.py index 2b19a71..477a8ca 100644 --- a/tests/test_conll_format.py +++ b/tests/test_conll_format.py @@ -9,10 +9,11 @@ LineSpec, _CoNLLToken, ingest_conll_file, + write_docs_raw, write_docs_using_encoding, ) from seqscore.encoding import REPAIR_NONE, get_encoding -from seqscore.model import LabeledSequence +from seqscore.model import AnnotatedSequence, LabeledSequence from seqscore.util import file_fields_match from seqscore.validation import InvalidLabelError @@ -21,7 +22,7 @@ def test_parse_comments_true() -> None: - ingester = CoNLLIngester(BIO, LINE_SPEC, parse_comment_lines=True) + ingester = CoNLLIngester(BIO, LINE_SPEC, allow_comment_lines=True) comments_path = Path("tests") / "test_files" / "minimal_comments.bio" with comments_path.open(encoding="utf8") as file: documents = ingester.ingest(file, "test", REPAIR_NONE) @@ -51,7 +52,7 @@ def test_parse_comments_false() -> None: ingester.ingest(file, "test", REPAIR_NONE) assert ( str(err1.value) - == "Line 1 of test does not appear to be delimited and begins with #. Perhaps you want to use the --parse-comment-lines flag? Line contents: '#'" + == "Line 1 of test does not appear to be delimited and begins with #. Perhaps you want to use the --allow-comment-lines flag? Line contents: '#'" ) comments_path = Path("tests") / "test_files" / "minimal_comments_2.bio" @@ -138,7 +139,7 @@ def test_repair_bad_name() -> None: LINE_SPEC, repair="conlleval", ignore_document_boundaries=False, - parse_comment_lines=False, + allow_comment_lines=False, ) assert str(err.value).startswith( @@ -158,13 +159,25 @@ def test_bad_label1() -> None: ) +def test_bad_label2() -> None: + ingester = CoNLLIngester(BIO, LINE_SPEC) + path = Path("tests") / "test_files" / "bad_label2.bio" + with path.open(encoding="utf8") as file: + with pytest.raises(InvalidLabelError) as err: + ingester.ingest(file, str(path), repair=REPAIR_NONE) + + assert str(err.value).startswith( + "Could not parse label 'OUT' on line 1 of tests/test_files/bad_label2.bio during validation" + ) + + def test_write_docs_no_orig_fields(tmp_path: Path) -> None: - sent1 = LabeledSequence( + sent1 = AnnotatedSequence( tokens=("This", "is", "a", "sentence", "."), labels=("O", "O", "O", "O", "O"), mentions=(), ) - sent2 = LabeledSequence.from_tokens_and_labels( + sent2 = AnnotatedSequence.from_tokens_and_labels( ( "University", "of", @@ -188,13 +201,119 @@ def test_write_docs_no_orig_fields(tmp_path: Path) -> None: ) -def test_bad_label2() -> None: - ingester = CoNLLIngester(BIO, LINE_SPEC) - path = Path("tests") / "test_files" / "bad_label2.bio" - with path.open(encoding="utf8") as file: - with pytest.raises(InvalidLabelError) as err: - ingester.ingest(file, str(path), repair=REPAIR_NONE) +def test_write_docs_using_encoding_single_doc_no_docstart_by_default( + tmp_path: Path, +) -> None: + sent = AnnotatedSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + mentions=(), + ) + output_file = tmp_path / "out.bio" + write_docs_using_encoding([[sent]], "BIO", "utf-8", " ", LINE_SPEC, output_file) + assert DOCSTART not in output_file.read_text() - assert str(err.value).startswith( - "Could not parse label 'OUT' on line 1 of tests/test_files/bad_label2.bio during validation" + +def test_write_docs_using_encoding_single_doc_always_write_docstart( + tmp_path: Path, +) -> None: + sent = AnnotatedSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + mentions=(), + ) + output_file = tmp_path / "out.bio" + write_docs_using_encoding( + [[sent]], + "BIO", + "utf-8", + " ", + LINE_SPEC, + output_file, + always_write_docstart=True, + ) + text = output_file.read_text() + assert text.count(DOCSTART) == 1 + assert text == "-DOCSTART- O\n\nThis O\nis O\na O\nsentence O\n. O\n\n" + + +def test_write_docs_raw_single_doc_no_docstart_by_default(tmp_path: Path) -> None: + sent = LabeledSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + ) + output_file = tmp_path / "out.bio" + write_docs_raw([[sent]], "utf-8", " ", LINE_SPEC, output_file) + assert DOCSTART not in output_file.read_text() + + +def test_write_docs_raw_single_doc_always_write_docstart(tmp_path: Path) -> None: + sent = LabeledSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + ) + output_file = tmp_path / "out.bio" + write_docs_raw( + [[sent]], "utf-8", " ", LINE_SPEC, output_file, always_write_docstart=True + ) + text = output_file.read_text() + assert text.count(DOCSTART) == 1 + assert text == "-DOCSTART- O\n\nThis O\nis O\na O\nsentence O\n. O\n\n" + + +def test_write_docs_raw_outside_label( + tmp_path: Path, +) -> None: + sent = LabeledSequence(tokens=("A",), labels=("O",)) + output_file = tmp_path / "out.bio" + write_docs_raw( + [[sent]], + "utf-8", + " ", + LINE_SPEC, + output_file, + outside_label="NONE", + always_write_docstart=True, + ) + assert output_file.read_text().startswith("-DOCSTART- NONE\n") + + +def test_write_docs_raw_invalid_labels( + tmp_path: Path, +) -> None: + # Completely invalid labels are written without error + sent = LabeledSequence(tokens=("A", "B", "C"), labels=("B-", "B-AAA", "I-YYY")) + output_file = tmp_path / "out.bio" + write_docs_raw( + [[sent]], + "utf-8", + " ", + LINE_SPEC, + output_file, + ) + assert output_file.read_text().startswith("A B-\nB B-AAA\nC I-YYY") + + +def test_write_docs_using_encoding_multi_doc_always_write_docstart( + tmp_path: Path, +) -> None: + sent1 = AnnotatedSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + mentions=(), + ) + sent2 = AnnotatedSequence( + tokens=("Another", "sentence", "."), + labels=("O", "O", "O"), + mentions=(), + ) + docs = [[sent1], [sent2]] + default_file = tmp_path / "default.bio" + forced_file = tmp_path / "forced.bio" + write_docs_using_encoding(docs, "BIO", "utf-8", " ", LINE_SPEC, default_file) + write_docs_using_encoding( + docs, "BIO", "utf-8", " ", LINE_SPEC, forced_file, always_write_docstart=True ) + # Setting always_write_docstart=True has no effect. + # DOCSTART is always written once per document. + assert default_file.read_text() == forced_file.read_text() diff --git a/tests/test_conll_scoring.py b/tests/test_conll_scoring.py index 1aaabf1..2242666 100644 --- a/tests/test_conll_scoring.py +++ b/tests/test_conll_scoring.py @@ -29,7 +29,7 @@ def _score( file_encoding="utf-8", line_spec=LineSpec(0, -1), ignore_document_boundaries=False, - parse_comment_lines=False, + allow_comment_lines=False, delim="\t", output_format=output_format, error_counts=error_counts, diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 97c8ae6..1224d3e 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass + import pytest -from attr import attrs from seqscore.encoding import ( _ENCODING_NAMES, @@ -15,7 +16,7 @@ EncodingError, get_encoding, ) -from seqscore.model import LabeledSequence, Mention, Span +from seqscore.model import AnnotatedSequence, LabeledSequence, Mention, Span FULL_SENTENCE_LABELS = { "IO": ["I-PER", "O", "I-ORG", "I-ORG", "I-ORG", "I-ORG", "I-ORG", "I-LOC"], @@ -51,7 +52,7 @@ } -@attrs(auto_attribs=True) +@dataclass class EdgeTestSentence: name: str mentions: list[Mention] @@ -159,7 +160,11 @@ def test_basic_encoding() -> None: assert encoding.encode_mentions(mentions, len(labels)) == labels # Also test encoding sentence object, intentionally putting no mentions in the # sentence labels to make sure encoding using the mentions, not the labels - sentence = LabeledSequence(["a"] * len(labels), ["O"] * len(labels), mentions) + sentence = AnnotatedSequence( + tokens=tuple("a" for _ in labels), + labels=tuple("O" for _ in labels), + mentions=tuple(mentions), + ) assert encoding.encode_sequence(sentence) == labels @@ -267,21 +272,21 @@ def test_labeled_sequence() -> None: # Test length mismatch with pytest.raises(ValueError): LabeledSequence( - ["a"] * 10, - ["O"] * 9, + tokens=("a",) * 10, + labels=("O",) * 9, ) def test_decode_bio_invalid_continue() -> None: decoder = get_encoding("BIO") - sent1 = LabeledSequence(("a", "b"), ("B-PER", "I-LOC")) + sent1 = LabeledSequence(tokens=("a", "b"), labels=("B-PER", "I-LOC")) with pytest.raises(AssertionError): assert decoder.decode_sequence(sent1) def test_decode_iob_invalid_begin() -> None: decoder = get_encoding("IOB") - sent = LabeledSequence(("a", "b"), ("I-PER", "B-LOC")) + sent = LabeledSequence(tokens=("a", "b"), labels=("I-PER", "B-LOC")) with pytest.raises(AssertionError): assert decoder.decode_sequence(sent) @@ -289,8 +294,8 @@ def test_decode_iob_invalid_begin() -> None: def test_decode_bioes_invalid_start() -> None: decoder = get_encoding("BIOES") sents = [ - LabeledSequence(("a",), ("I-PER",)), - LabeledSequence(("a",), ("E-PER",)), + LabeledSequence(tokens=("a",), labels=("I-PER",)), + LabeledSequence(tokens=("a",), labels=("E-PER",)), ] for sent in sents: with pytest.raises(AssertionError): @@ -301,14 +306,14 @@ def test_decode_bioes_invalid_end() -> None: decoder = get_encoding("BIOES") sents = [ # Single-token mentions must start (and end) with S - LabeledSequence(("a", "b"), ("B-PER", "S-PER")), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "S-PER")), # Multi-token mentions must end in E - LabeledSequence(("a",), ("B-PER",)), - LabeledSequence(("a", "b"), ("B-PER", "I-PER")), + LabeledSequence(tokens=("a",), labels=("B-PER",)), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "I-PER")), # Ends with wrong type - LabeledSequence(("a", "b", "c"), ("B-PER", "I-PER", "E-ORG")), + LabeledSequence(tokens=("a", "b", "c"), labels=("B-PER", "I-PER", "E-ORG")), # Multi-token mentions cannot end in S - LabeledSequence(("a", "b", "c"), ("B-PER", "I-PER", "S-PER")), + LabeledSequence(tokens=("a", "b", "c"), labels=("B-PER", "I-PER", "S-PER")), ] for sent in sents: with pytest.raises(AssertionError): @@ -319,10 +324,10 @@ def test_decode_bioes_invalid_continue() -> None: decoder = get_encoding("BIOES") sents = [ # B must be followed by I or E of the same type - LabeledSequence(("a", "b"), ("B-PER", "B-PER")), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "B-PER")), # Cannot change types mid-mention - LabeledSequence(("a", "b"), ("B-PER", "E-ORG")), - LabeledSequence(("a", "b", "c"), ("B-PER", "I-PER", "E-ORG")), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "E-ORG")), + LabeledSequence(tokens=("a", "b", "c"), labels=("B-PER", "I-PER", "E-ORG")), ] for sent in sents: with pytest.raises(AssertionError): diff --git a/tests/test_model.py b/tests/test_model.py index 0853d16..6a3ce3f 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,12 @@ import pytest -from seqscore.model import LabeledSequence, Mention, SequenceProvenance, Span +from seqscore.model import ( + AnnotatedSequence, + LabeledSequence, + Mention, + SequenceProvenance, + Span, +) def test_span() -> None: @@ -8,12 +14,21 @@ def test_span() -> None: assert len(Span(1, 2)) == 1 assert len(Span(0, 2)) == 2 + with pytest.raises(ValueError): + Span(-2, -1) + with pytest.raises(ValueError): Span(-1, 0) + with pytest.raises(ValueError): + Span(0, -1) + with pytest.raises(ValueError): Span(0, 0) + with pytest.raises(ValueError): + Span(3, 1) + def test_mention() -> None: m1 = Mention(Span(0, 1), "PER") @@ -29,11 +44,13 @@ def test_mention() -> None: Mention(Span(0, 1), None) # type: ignore -def test_labeled_sentence() -> None: +def test_labeled_sequence() -> None: s1 = LabeledSequence( - ["a", "b"], - ["B-PER", "I-PER"], + tokens=("a", "b"), + labels=("B-PER", "I-PER"), provenance=SequenceProvenance(7, "test"), + token_fields=(("NOUN",), ("VERB",)), + comment="Comment", ) assert s1.tokens == ("a", "b") assert s1[0] == "a" @@ -44,33 +61,68 @@ def test_labeled_sentence() -> None: assert str(s1) == "a/B-PER b/I-PER" assert s1.tokens_with_labels() == (("a", "B-PER"), ("b", "I-PER")) assert s1.span_tokens(Span(0, 1)) == ("a",) - assert s1.mention_tokens(Mention(Span(0, 1), "PER")) == ("a",) - s2 = LabeledSequence(s1.tokens, s1.labels) - # Provenance not included in equality + s2 = LabeledSequence(tokens=s1.tokens, labels=s1.labels) + # Attributes other than tokens and labels not included in equality assert s1 == s2 + # Hashes identical for equal objects + assert hash(s1) == hash(s2) + # Equality fails for objects of other types + assert s1 != "" with pytest.raises(ValueError): # Mismatched length - LabeledSequence(["a", "b"], ["B-PER"]) + LabeledSequence(tokens=("a", "b"), labels=("B-PER",)) with pytest.raises(ValueError): # Empty - LabeledSequence([], []) + LabeledSequence(tokens=(), labels=()) with pytest.raises(ValueError) as err: # Bad label - LabeledSequence(["a"], [""]) - assert str(err.value) == "Invalid label at sequence index 0: ''" + LabeledSequence(tokens=("a",), labels=("",)) + assert "Invalid label at sequence index 0: ''" in str(err.value) with pytest.raises(ValueError) as err: # Bad token - LabeledSequence([""], ["B-PER"]) - assert str(err.value) == "Invalid token at sequence index 0: ''" - - s2 = s1.with_mentions([Mention(Span(0, 2), "PER")]) - assert s2.mentions == (Mention(Span(0, 2), "PER"),) + LabeledSequence(tokens=("",), labels=("B-PER",)) + assert "Invalid token at sequence index 0: ''" in str(err.value) with pytest.raises(ValueError): - # Mismatched length between tokens and other_fields - LabeledSequence(["a", "b"], ["B-PER", "I-PER"], orig_fields=[["DT"]]) + # Mismatched length between tokens and token_fields + LabeledSequence( + tokens=("a", "b"), labels=("B-PER", "I-PER"), token_fields=(("DT",),) + ) + + +def test_annotated_sequence() -> None: + s1 = AnnotatedSequence( + tokens=("a", "b"), + labels=("B-PER", "I-PER"), + mentions=(Mention(Span(0, 2), "PER"),), + provenance=SequenceProvenance(7, "test"), + token_fields=(("NOUN",), ("VERB",)), + comment="Comment", + ) + assert s1.tokens == ("a", "b") + assert s1[0] == "a" + assert s1[0:2] == ("a", "b") + assert list(s1) == ["a", "b"] + assert s1.labels == ("B-PER", "I-PER") + assert s1.mentions == (Mention(Span(0, 2), "PER"),) + assert s1.provenance == SequenceProvenance(7, "test") + assert str(s1) == "a/B-PER b/I-PER" + assert s1.tokens_with_labels() == (("a", "B-PER"), ("b", "I-PER")) + assert s1.span_tokens(Span(0, 1)) == ("a",) + assert s1.mention_tokens(Mention(Span(0, 1), "PER")) == ("a",) + + s2 = AnnotatedSequence(tokens=s1.tokens, labels=s1.labels, mentions=s1.mentions) + # Attributes other than tokens, labels, and mentions not included in equality + assert s1 == s2 + # Hashes identical for equal objects + assert hash(s1) == hash(s2) + # Equality fails for objects of other types + assert s1 != "" + + s3 = s1.with_mentions([Mention(Span(0, 1), "ORG")]) + assert s3.mentions == (Mention(Span(0, 1), "ORG"),) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index a74558d..c08e307 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -4,7 +4,7 @@ import pytest from seqscore.encoding import EncodingError -from seqscore.model import LabeledSequence, Mention, SequenceProvenance, Span +from seqscore.model import AnnotatedSequence, Mention, SequenceProvenance, Span from seqscore.scoring import ( AccuracyScore, ClassificationScore, @@ -234,8 +234,12 @@ def test_compute_scores() -> None: Mention(Span(4, 5), "ORG"), ) tokens = ("a", "b", "c", "d", "e") - ref_sequence = LabeledSequence(tokens, ref_labels, ref_mentions) - pred_sequence = LabeledSequence(tokens, pred_labels, pred_mentions) + ref_sequence = AnnotatedSequence( + tokens=tokens, labels=ref_labels, mentions=ref_mentions + ) + pred_sequence = AnnotatedSequence( + tokens=tokens, labels=pred_labels, mentions=pred_mentions + ) class_score, acc_score = compute_scores([[pred_sequence]], [[ref_sequence]]) assert acc_score.accuracy == 4 / 5 print(class_score) @@ -247,11 +251,17 @@ def test_compute_scores() -> None: def test_token_count_error() -> None: ref_labels = ("O", "B-ORG", "I-ORG", "O") pred_labels = ("O", "B-ORG", "I-ORG", "O", "O") - ref_sequence = LabeledSequence( - ("a", "b", "c", "d"), ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=("a", "b", "c", "d"), + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - ("a", "b", "c", "d", "e"), pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=("a", "b", "c", "d", "e"), + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(TokenCountError): compute_scores([[pred_sequence]], [[ref_sequence]]) @@ -259,7 +269,9 @@ def test_token_count_error() -> None: def test_token_count_error_provenance_none_raises_error() -> None: labels = ("O", "B-ORG") - sequence = LabeledSequence(("a", "b"), labels, provenance=None) + sequence = AnnotatedSequence( + tokens=("a", "b"), labels=labels, mentions=(), provenance=None + ) with pytest.raises(ValueError): TokenCountError.from_predicted_sequence(2, sequence) @@ -268,11 +280,17 @@ def test_differing_num_docs() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") tokens = ("a", "b") - ref_sequence = LabeledSequence( - tokens, ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=tokens, + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - tokens, pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=tokens, + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence], [ref_sequence]]) @@ -282,11 +300,17 @@ def test_differing_doc_length() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") tokens = ("a", "b") - ref_sequence = LabeledSequence( - tokens, ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=tokens, + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - tokens, pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=tokens, + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence, ref_sequence]]) @@ -295,11 +319,17 @@ def test_differing_doc_length() -> None: def test_differing_pred_and_ref_tokens() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") - ref_sequence = LabeledSequence( - ("a", "b"), ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=("a", "b"), + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - ("a", "c"), pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=("a", "c"), + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence]]) diff --git a/tests/test_scoring_click.py b/tests/test_scoring_click.py index 9a97ab4..0f3ff45 100644 --- a/tests/test_scoring_click.py +++ b/tests/test_scoring_click.py @@ -301,10 +301,17 @@ def test_score_error_counts_single_file() -> None: ], ) assert result.exit_code == 0 - assert "| Count | Error | Type | Tokens |" in result.output - assert "| 1 | FP | LOC | West |" in result.output - assert "| 1 | FP | LOC | Philadelphia |" in result.output - assert "| 1 | FN | LOC | West Philadelphia |" in result.output + # Ordering within the same count is deterministic and determined by + # the token string + assert ( + result.output + == """| Count | Error | Type | Tokens | +|---------|---------|--------|-------------------| +| 1 | FP | LOC | Philadelphia | +| 1 | FP | LOC | West | +| 1 | FN | LOC | West Philadelphia | +""" + ) def test_score_error_counts_delim_format() -> None: @@ -323,10 +330,14 @@ def test_score_error_counts_delim_format() -> None: ], ) assert result.exit_code == 0 - assert "Count\tError\tType\tTokens" in result.output - assert "1\tFP\tLOC\tWest" in result.output - assert "1\tFP\tLOC\tPhiladelphia" in result.output - assert "1\tFN\tLOC\tWest Philadelphia" in result.output + assert ( + result.output + == """Count\tError\tType\tTokens +1\tFP\tLOC\tPhiladelphia +1\tFP\tLOC\tWest +1\tFN\tLOC\tWest Philadelphia +""" + ) def test_score_error_counts_multiple_files() -> None: diff --git a/tests/test_seqscore_main.py b/tests/test_seqscore_main.py index 7251c62..8515043 100644 --- a/tests/test_seqscore_main.py +++ b/tests/test_seqscore_main.py @@ -1,19 +1,20 @@ -import subprocess +from click.testing import CliRunner import seqscore +from seqscore.scripts.seqscore import cli HELP_OUTPUT = "Usage: seqscore [OPTIONS] COMMAND [ARGS]..." def test_seqscore_help() -> None: - result = subprocess.run(["seqscore", "--help"], capture_output=True, encoding="UTF-8") - assert result.returncode == 0 - assert result.stdout.startswith(HELP_OUTPUT) + runner = CliRunner() + result = runner.invoke(cli, ["--help"], prog_name="seqscore") + assert result.exit_code == 0 + assert result.output.startswith(HELP_OUTPUT) def test_seqscore_version() -> None: - result = subprocess.run( - ["seqscore", "--version"], capture_output=True, encoding="UTF-8" - ) - assert result.returncode == 0 - assert result.stdout == f"seqscore, version {seqscore.__version__}\n" + runner = CliRunner() + result = runner.invoke(cli, ["--version"], prog_name="seqscore") + assert result.exit_code == 0 + assert result.output == f"seqscore, version {seqscore.__version__}\n" diff --git a/tests/test_summarize_click.py b/tests/test_summarize_click.py index 8a9dec8..bb5ee04 100644 --- a/tests/test_summarize_click.py +++ b/tests/test_summarize_click.py @@ -19,12 +19,12 @@ def test_summarize_bio_onedoc() -> None: assert ( result.output == """File 'tests/conll_annotation/minimal.bio' contains 1 document(s) and 2 sentences -Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 +| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -43,12 +43,12 @@ def test_summarize_bio_onedoc_quiet() -> None: assert result.exit_code == 0 assert ( result.output - == """Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 + == """| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -67,12 +67,12 @@ def test_summarize_iob_twodoc() -> None: assert ( result.output == """File 'tests/conll_annotation/minimal_fields.iob' contains 2 document(s) and 2 sentences -Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 +| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -92,12 +92,12 @@ def test_summarize_iob_twodoc_ignore_doc_boundaries() -> None: assert ( result.output == """File 'tests/conll_annotation/minimal_fields.iob' contains 1 document(s) and 2 sentences -Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 +| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -119,11 +119,11 @@ def test_summarize_bio_twofiles() -> None: == """File 'tests/conll_annotation/minimal.bio' contains 1 document(s) and 2 sentences File 'tests/conll_annotation/minimal2.bio' contains 1 document(s) and 2 sentences Total 2 document(s) and 4 sentences -Entity Type Count -------------- ------- -LOC 5 -ORG 2 -------------- ------- -TOTAL 7 +| Entity Type | Count | +|---------------|---------| +| LOC | 5 | +| ORG | 2 | +|---------------|---------| +| TOTAL | 7 | """ ) diff --git a/tests/test_utils.py b/tests/test_utils.py index f5b6721..e87420d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,12 +1,9 @@ import os -from seqscore.util import file_fields_match, file_lines_match, tuplify_strs - - -def test_tuplify_strs() -> None: - strs = ["a", "b", "c"] - tup = tuplify_strs(strs) - assert tup == ("a", "b", "c") +from seqscore.util import ( + file_fields_match, + file_lines_match, +) def test_identical_files() -> None: diff --git a/tests/test_validation.py b/tests/test_validation.py index 9fbde19..3f0ecc0 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,11 +1,12 @@ +from dataclasses import dataclass + import pytest -from attr import attrs from seqscore.encoding import REPAIR_NONE, EncodingError, get_encoding from seqscore.validation import validate_labels -@attrs(auto_attribs=True) +@dataclass class RepairTest: original_labels: list[str] n_errors: int diff --git a/tests/test_validation_click.py b/tests/test_validation_click.py index 6f72a7d..c84fc5f 100644 --- a/tests/test_validation_click.py +++ b/tests/test_validation_click.py @@ -64,12 +64,11 @@ def test_mixed_valid_bio_twofiles_quiet() -> None: os.path.join("tests", "conll_annotation", "invalid1.bio"), ], ) + # Only summary line, no individual invalid transitions printed out assert result.output == ( "Encountered 3 errors in 15 tokens, 2 sequences, and 1 document(s) in tests/conll_annotation/invalid1.bio\n" - "Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7\n" - "Invalid transition 'O' -> 'I-LOC' for token 'West' on line 12\n" - "Invalid transition 'O' -> 'I-LOC' for token 'Pennsylvania' on line 15\n" ) + assert "Invalid transition" not in result.output assert result.exit_code != 0 @@ -133,6 +132,27 @@ def test_invalid_bio() -> None: ) +def test_invalid_bio_quiet() -> None: + runner = CliRunner() + result = runner.invoke( + validate, + [ + "--labels", + "BIO", + "-q", + os.path.join("tests", "conll_annotation", "invalid1.bio"), + ], + ) + assert result.exit_code != 0 + assert ( + normalize_str_with_path( + "Encountered 3 errors in 15 tokens, 2 sequences, and 1 document(s) in tests/conll_annotation/invalid1.bio\n" + ) + == result.output + ) + assert "Invalid transition" not in result.output + + def test_invalid_bioes() -> None: runner = CliRunner() result = runner.invoke(