From 570cb361bfb56fdaa464e65ce4bae3e0746d7100 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Fri, 5 Jun 2026 11:30:26 -0400 Subject: [PATCH 01/15] Increment version to 0.9.0 --- seqscore/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqscore/__init__.py b/seqscore/__init__.py index 777f190..3e2f46a 100644 --- a/seqscore/__init__.py +++ b/seqscore/__init__.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.9.0" From 60869fd589ed3383cc3d007af75753e38ed9e7da Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Fri, 5 Jun 2026 12:22:46 -0400 Subject: [PATCH 02/15] Add --discard-extra-fields flag and fix handling of sequence metadata in process --- seqscore/conll.py | 17 +++++++-------- seqscore/model.py | 7 +++++- seqscore/scripts/seqscore.py | 41 +++++++++++++++++++++++++++++++----- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/seqscore/conll.py b/seqscore/conll.py index 6c75ecc..8f2ad95 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -411,18 +411,16 @@ def validate_conll_file( def repair_conll_file( input_file: PathType, - output_file: PathType, mention_encoding_name: str, repair: Optional[str], file_encoding: str, line_spec: LineSpec, - output_delim: str, *, ignore_document_boundaries: bool, parse_comment_lines: bool, quiet: bool, -) -> None: - docs = ingest_conll_file( +) -> list[list[LabeledSequence]]: + return ingest_conll_file( input_file, mention_encoding_name, file_encoding, @@ -432,9 +430,6 @@ def repair_conll_file( parse_comment_lines=parse_comment_lines, quiet=quiet, ) - write_docs_using_encoding( - docs, mention_encoding_name, file_encoding, output_delim, line_spec, output_file - ) def write_docs_using_encoding( @@ -444,6 +439,8 @@ def write_docs_using_encoding( delim: str, line_spec: LineSpec, output_path: PathType, + *, + discard_extra_fields: bool = False, ) -> None: mention_encoding = get_encoding(mention_encoding_name) output_docstart = len(docs) > 1 @@ -457,6 +454,7 @@ def write_docs_using_encoding( file, line_spec, output_docstart=output_docstart, + discard_extra_fields=discard_extra_fields, ) @@ -468,10 +466,11 @@ def write_doc_using_encoding( line_spec: LineSpec, *, output_docstart: bool, + discard_extra_fields: bool = False, ) -> None: if output_docstart: # Get the fields of the first token of the first sentence - if doc[0].orig_fields: + if doc[0].orig_fields and not discard_extra_fields: # to figure out how many fields there are sequence_orig_fields = doc[0].orig_fields[0] # Create the write number of fields @@ -491,7 +490,7 @@ def write_doc_using_encoding( for (token, orig_fields), label in zip( sequence.tokens_with_orig_fields(), labels ): - if orig_fields: + if orig_fields and not discard_extra_fields: fields = list(orig_fields) fields[line_spec.token_index] = token fields[line_spec.ner_label_index] = label diff --git a/seqscore/model.py b/seqscore/model.py index 2921b50..23cae15 100644 --- a/seqscore/model.py +++ b/seqscore/model.py @@ -100,7 +100,12 @@ def __attrs_post_init__(self) -> None: def with_mentions(self, mentions: Sequence[Mention]) -> "LabeledSequence": return LabeledSequence( - self.tokens, self.labels, mentions, provenance=self.provenance + self.tokens, + self.labels, + mentions, + orig_fields=self.orig_fields, + provenance=self.provenance, + comment=self.comment, ) @overload diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index 053cab2..eae1f13 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -26,6 +26,8 @@ ) from seqscore.processing import modify_types +# TODO: For each command, reorder click decorators so that --help has the most important arguments first + # Set up a click command group @click.group( @@ -121,6 +123,10 @@ def _output_delim_option() -> Callable: ) +def _discard_extra_fields_option() -> Callable: + return click.option("--discard-extra-fields", is_flag=True) + + def _quiet_option() -> Callable: return click.option( "--quiet", @@ -179,6 +185,7 @@ def validate( @_repair_required_option() @_labels_option() @_output_delim_option() +@_discard_extra_fields_option() @_quiet_option() def repair( file: str, @@ -192,6 +199,7 @@ def repair( *, ignore_document_boundaries: bool, parse_comment_lines: bool, + discard_extra_fields: bool, quiet: bool, ) -> None: output_delim = _normalize_tab(output_delim) @@ -201,24 +209,32 @@ def repair( ) line_spec = LineSpec(token_index, label_index) - repair_conll_file( + docs = repair_conll_file( file, - output_file, labels, repair_method, file_encoding, line_spec, - output_delim, ignore_document_boundaries=ignore_document_boundaries, parse_comment_lines=parse_comment_lines, quiet=quiet, ) + write_docs_using_encoding( + docs, + labels, + file_encoding, + output_delim, + line_spec, + output_file, + discard_extra_fields=discard_extra_fields, + ) @cli.command(help="convert between mention encodings") @_single_input_file_arguments @click.argument("output_file") @_output_delim_option() +@_discard_extra_fields_option() @click.option("--input-labels", required=True, type=click.Choice(SUPPORTED_ENCODINGS)) @click.option("--output-labels", required=True, type=click.Choice(SUPPORTED_ENCODINGS)) def convert( @@ -233,6 +249,7 @@ def convert( *, ignore_document_boundaries: bool, parse_comment_lines: bool, + discard_extra_fields: bool, ) -> None: output_delim = _normalize_tab(output_delim) line_spec = LineSpec(token_index, label_index) @@ -247,7 +264,13 @@ def convert( ) write_docs_using_encoding( - docs, output_labels, file_encoding, output_delim, line_spec, output_file + docs, + output_labels, + file_encoding, + output_delim, + line_spec, + output_file, + discard_extra_fields=discard_extra_fields, ) @@ -271,6 +294,7 @@ def convert( help="a JSON file containing types to be modified, in the format of a dict with keys as the target type and values as the source type [example file: {'MISC': ['WorkOfArt', 'Event']}]", ) @_output_delim_option() +@_discard_extra_fields_option() def process( file: str, output_file: str, @@ -285,6 +309,7 @@ def process( *, ignore_document_boundaries: bool, parse_comment_lines: bool, + discard_extra_fields: bool, ) -> None: output_delim = _normalize_tab(output_delim) line_spec = LineSpec(token_index, label_index) @@ -315,7 +340,13 @@ def process( raise click.UsageError(str(err)) from err write_docs_using_encoding( - mod_docs, labels, file_encoding, output_delim, line_spec, output_file + mod_docs, + labels, + file_encoding, + output_delim, + line_spec, + output_file, + discard_extra_fields=discard_extra_fields, ) From 408e752a095d258faeb9c6e862adbc72d4f88905 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Tue, 9 Jun 2026 13:16:01 -0400 Subject: [PATCH 03/15] Migrate model.py to Pydantic --- pyproject.toml | 1 + seqscore/conll.py | 16 ++-- seqscore/model.py | 206 ++++++++++++++++++++++++++++------------- seqscore/util.py | 11 +-- tests/test_encoding.py | 34 ++++--- tests/test_model.py | 35 +++++-- tests/test_scoring.py | 30 +++--- tests/test_utils.py | 6 +- 8 files changed, 221 insertions(+), 118 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 38cb8e9..fd5454d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ requires-python = ">=3.10" dependencies = [ "attrs>=19.2.0", "click", + "pydantic>=2.0", "tabulate", ] classifiers = [ diff --git a/seqscore/conll.py b/seqscore/conll.py index 8f2ad95..2810872 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -207,10 +207,10 @@ def ingest( try: sequences = LabeledSequence( - tokens, - labels, - mentions, - orig_fields=orig_fields, + tokens=tokens, + labels=labels, + mentions=tuple(mentions), + token_fields=orig_fields, provenance=SequenceProvenance(line_nums[0], source_name), comment=comment, ) @@ -470,9 +470,9 @@ def write_doc_using_encoding( ) -> None: if output_docstart: # Get the fields of the first token of the first sentence - if doc[0].orig_fields and not discard_extra_fields: + if doc[0].token_fields and not discard_extra_fields: # to figure out how many fields there are - sequence_orig_fields = doc[0].orig_fields[0] + sequence_orig_fields = doc[0].token_fields[0] # Create the write number of fields fields = [EMPTY_OTHER_FIELD] * len(sequence_orig_fields) # Fill in the token and label @@ -487,9 +487,7 @@ def write_doc_using_encoding( for sequence in doc: labels = encoding.encode_sequence(sequence) # Lengths of labels and orig_fields have previously been checked to match tokens - for (token, orig_fields), label in zip( - sequence.tokens_with_orig_fields(), labels - ): + for (token, orig_fields), label in zip(sequence.tokens_with_fields(), labels): if orig_fields and not discard_extra_fields: fields = list(orig_fields) fields[line_spec.token_index] = token diff --git a/seqscore/model.py b/seqscore/model.py index 23cae15..0e943f4 100644 --- a/seqscore/model.py +++ b/seqscore/model.py @@ -1,36 +1,49 @@ -from collections.abc import Iterable, Iterator, Sequence +from collections.abc import Iterator, Sequence +from dataclasses import dataclass from itertools import repeat -from typing import TYPE_CHECKING, Any, Optional, Union, overload - -from attr import Attribute, attrib, attrs - -from seqscore.util import ( - tuplify_optional_nested_strs, - tuplify_strs, - validator_nonempty_str, +from typing import ( + TYPE_CHECKING, + Any, + Optional, + Protocol, + Union, + overload, + runtime_checkable, ) +from pydantic import BaseModel, ConfigDict, model_validator + if TYPE_CHECKING: from seqscore.encoding import Encoding # pragma: no cover +__all__ = [ + "LabeledSequence", + "Mention", + "SequenceProvenance", + "Span", + "TokenSequence", +] -def _validator_nonnegative(_inst: Any, _attr: Attribute, value: Any) -> None: - if value < 0: - raise ValueError(f"Negative value: {repr(value)}") +@dataclass(frozen=True, slots=True) +class Span: + """A token index range [start, end) within a sequence. -def _tuplify_mentions( - mentions: Iterable["Mention"], -) -> tuple["Mention", ...]: - return tuple(mentions) + The start index is inclusive, and the end index is exclusive. + Both start and end must be non-negative, and end must be + greater than start. Zero-length spans are not allowed. + """ + start: int + end: int -@attrs(frozen=True, slots=True) -class Span: - start: int = attrib(validator=_validator_nonnegative) - end: int = attrib(validator=_validator_nonnegative) + def __post_init__(self) -> None: + if self.start < 0: + raise ValueError(f"Start ({self.start}) cannot be negative") + + if self.end < 0: + raise ValueError(f"End ({self.end}) cannot be negative") - def __attrs_post_init__(self) -> None: if not self.end > self.start: raise ValueError( f"End of span ({self.end}) must be greater than start ({self.start}" @@ -40,40 +53,104 @@ def __len__(self) -> int: return self.end - self.start -@attrs(frozen=True, slots=True) +@dataclass(frozen=True, slots=True) class Mention: - span: Span = attrib() - type: str = attrib(validator=validator_nonempty_str) + """A typed span representing a named entity or chunk. + + Combines a Span with a string entity type. + """ + + span: Span + type: str + + def __post_init__(self) -> None: + if not isinstance(self.type, str): + raise TypeError(f"Expected str for type, got {type(self.type).__name__}") + if not self.type: + raise ValueError(f"Empty string for type: {repr(self.type)}") def __len__(self) -> int: return len(self.span) def with_type(self, new_type: str) -> "Mention": + """Return a new Mention with the same span and the provided type.""" return Mention(self.span, new_type) -@attrs(frozen=True, slots=True) +@dataclass(frozen=True, slots=True) class SequenceProvenance: - starting_line: int = attrib() - source: Optional[str] = attrib() - - -@attrs(frozen=True, slots=True) -class LabeledSequence(Sequence[str]): - tokens: tuple[str, ...] = attrib(converter=tuplify_strs) - labels: tuple[str, ...] = attrib(converter=tuplify_strs) - mentions: tuple[Mention, ...] = attrib(default=(), converter=_tuplify_mentions) - orig_fields: Optional[tuple[tuple[str, ...], ...]] = attrib( - default=None, kw_only=True, converter=tuplify_optional_nested_strs - ) - provenance: Optional[SequenceProvenance] = attrib( - default=None, eq=False, kw_only=True - ) - comment: Optional[str] = attrib(default=None, eq=False, kw_only=True) - - def __attrs_post_init__(self) -> None: - # TODO: Check for overlapping mentions + """Origin of a sequence, with a starting line and optional source name.""" + + starting_line: int + source: Optional[str] + + +@runtime_checkable +class TokenSequence(Protocol): + """Protocol for a sequence of tokens with optional metadata.""" + + @property + def tokens(self) -> tuple[str, ...]: ... + + @property + def token_fields(self) -> Optional[tuple[tuple[str, ...], ...]]: ... + + @property + def provenance(self) -> Optional[SequenceProvenance]: ... + + @property + def comment(self) -> Optional[str]: ... + + @overload + def __getitem__(self, index: int) -> str: ... + @overload + def __getitem__(self, index: slice) -> tuple[str, ...]: ... + + def __getitem__(self, i: Union[int, slice]) -> Union[str, tuple[str, ...]]: ... + + def __iter__(self) -> Iterator[str]: ... + + def __len__(self) -> int: ... + + def span_tokens(self, span: Span) -> tuple[str, ...]: ... + + def tokens_with_fields( + self, + ) -> tuple[tuple[str, Optional[tuple[str, ...]]], ...]: ... + + +class LabeledSequence(BaseModel, Sequence[str]): + """A sequence of tokens with associated labels and optional mentions. + + Tokens and labels must be non-empty and of equal length. Iterating + over a LabeledSequence yields its tokens. + """ + + model_config = ConfigDict(frozen=True) + + tokens: tuple[str, ...] + labels: tuple[str, ...] + mentions: tuple[Mention, ...] = () + token_fields: Optional[tuple[tuple[str, ...], ...]] = None + provenance: Optional[SequenceProvenance] = None + comment: Optional[str] = None + + def __eq__(self, other: object) -> bool: + if not isinstance(other, LabeledSequence): + return NotImplemented + return ( + self.tokens == other.tokens + and self.labels == other.labels + and self.mentions == other.mentions + and self.token_fields == other.token_fields + ) + + def __hash__(self) -> int: + return hash((self.tokens, self.labels, self.mentions, self.token_fields)) + + @model_validator(mode="after") + def _validate_fields(self) -> "LabeledSequence": if len(self.tokens) != len(self.labels): raise ValueError( f"Tokens ({len(self.tokens)}) and labels ({len(self.labels)}) " @@ -82,48 +159,46 @@ def __attrs_post_init__(self) -> None: if not self.tokens: raise ValueError("Tokens and labels must be non-empty") - if self.orig_fields and len(self.tokens) != len(self.orig_fields): + if self.token_fields and len(self.tokens) != len(self.token_fields): raise ValueError( - f"Tokens ({len(self.tokens)}) and orig_fields ({len(self.orig_fields)}) " + f"Tokens ({len(self.tokens)}) and token_fields ({len(self.token_fields)}) " "must be of the same length" ) for idx, label in enumerate(self.labels): - # Labels cannot be None or an empty string if not label: raise ValueError(f"Invalid label at sequence index {idx}: {repr(label)}") for idx, token in enumerate(self.tokens): - # Tokens cannot be None or an empty string if not token: raise ValueError(f"Invalid token at sequence index {idx}: {repr(token)}") + return self + def with_mentions(self, mentions: Sequence[Mention]) -> "LabeledSequence": + """Return a copy of this sequence with different mentions.""" return LabeledSequence( - self.tokens, - self.labels, - mentions, - orig_fields=self.orig_fields, + tokens=self.tokens, + labels=self.labels, + mentions=tuple(mentions), + token_fields=self.token_fields, provenance=self.provenance, comment=self.comment, ) @overload - def __getitem__(self, index: int) -> str: - raise NotImplementedError + def __getitem__(self, index: int) -> str: ... @overload - def __getitem__(self, index: slice) -> tuple[str, ...]: - raise NotImplementedError + def __getitem__(self, index: slice) -> tuple[str, ...]: ... def __getitem__(self, i: Union[int, slice]) -> Union[str, tuple[str, ...]]: return self.tokens[i] - def __iter__(self) -> Iterator[str]: + def __iter__(self) -> Iterator[str]: # type: ignore[override] return iter(self.tokens) def __len__(self) -> int: - # Guaranteed that labels and tokens are same length by construction return len(self.tokens) def __str__(self) -> str: @@ -132,20 +207,24 @@ def __str__(self) -> str: ) def tokens_with_labels(self) -> tuple[tuple[str, str], ...]: + """Return a tuple of (token, label) tuples.""" return tuple(zip(self.tokens, self.labels)) - def tokens_with_orig_fields( + def tokens_with_fields( self, ) -> tuple[tuple[str, Optional[tuple[str, ...]]], ...]: - if self.orig_fields: - return tuple(zip(self.tokens, self.orig_fields)) + """Return a tuple of (token, token_fields) pairs, with None if token_fields is absent.""" + if self.token_fields: + return tuple(zip(self.tokens, self.token_fields)) else: return tuple(zip(self.tokens, repeat(None))) def span_tokens(self, span: Span) -> tuple[str, ...]: + """Return the tokens included in the given span.""" return self.tokens[span.start : span.end] def mention_tokens(self, mention: Mention) -> tuple[str, ...]: + """Return the tokens included in the given mention.""" return self.span_tokens(mention.span) @classmethod @@ -156,5 +235,8 @@ def from_tokens_and_labels( encoding: "Encoding", **kwargs: Any, ) -> "LabeledSequence": + """Create a LabeledSequence by decoding mentions from the labels using the given encoding.""" mentions = encoding.decode_labels(labels) - return cls(tokens, labels, mentions, **kwargs) + return cls( + tokens=tuple(tokens), labels=tuple(labels), mentions=tuple(mentions), **kwargs + ) diff --git a/seqscore/util.py b/seqscore/util.py index 666ead2..5c6d5a6 100644 --- a/seqscore/util.py +++ b/seqscore/util.py @@ -3,7 +3,7 @@ from itertools import zip_longest from os import PathLike from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Union from attr import Attribute, validators @@ -18,15 +18,6 @@ def tuplify_strs(strs: Iterable[str]) -> tuple[str, ...]: return tuple(strs) -def tuplify_optional_nested_strs( - items: Optional[Iterable[Iterable[str]]], -) -> Optional[tuple[tuple[str, ...], ...]]: - if items is not None: - return tuple(tuple(item) for item in items) - else: - return None - - def file_fields_match(path1: PathType, path2: PathType, *, debug: bool = False) -> bool: """Return whether the whitespace-delimited fields of two files are identical.""" with open(path1, encoding="utf8") as f1, open(path2, encoding="utf8") as f2: diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 97c8ae6..427c614 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -159,7 +159,11 @@ def test_basic_encoding() -> None: assert encoding.encode_mentions(mentions, len(labels)) == labels # Also test encoding sentence object, intentionally putting no mentions in the # sentence labels to make sure encoding using the mentions, not the labels - sentence = LabeledSequence(["a"] * len(labels), ["O"] * len(labels), mentions) + sentence = LabeledSequence( + tokens=tuple("a" for _ in labels), + labels=tuple("O" for _ in labels), + mentions=tuple(mentions), + ) assert encoding.encode_sequence(sentence) == labels @@ -267,21 +271,21 @@ def test_labeled_sequence() -> None: # Test length mismatch with pytest.raises(ValueError): LabeledSequence( - ["a"] * 10, - ["O"] * 9, + tokens=("a",) * 10, + labels=("O",) * 9, ) def test_decode_bio_invalid_continue() -> None: decoder = get_encoding("BIO") - sent1 = LabeledSequence(("a", "b"), ("B-PER", "I-LOC")) + sent1 = LabeledSequence(tokens=("a", "b"), labels=("B-PER", "I-LOC")) with pytest.raises(AssertionError): assert decoder.decode_sequence(sent1) def test_decode_iob_invalid_begin() -> None: decoder = get_encoding("IOB") - sent = LabeledSequence(("a", "b"), ("I-PER", "B-LOC")) + sent = LabeledSequence(tokens=("a", "b"), labels=("I-PER", "B-LOC")) with pytest.raises(AssertionError): assert decoder.decode_sequence(sent) @@ -289,8 +293,8 @@ def test_decode_iob_invalid_begin() -> None: def test_decode_bioes_invalid_start() -> None: decoder = get_encoding("BIOES") sents = [ - LabeledSequence(("a",), ("I-PER",)), - LabeledSequence(("a",), ("E-PER",)), + LabeledSequence(tokens=("a",), labels=("I-PER",)), + LabeledSequence(tokens=("a",), labels=("E-PER",)), ] for sent in sents: with pytest.raises(AssertionError): @@ -301,14 +305,14 @@ def test_decode_bioes_invalid_end() -> None: decoder = get_encoding("BIOES") sents = [ # Single-token mentions must start (and end) with S - LabeledSequence(("a", "b"), ("B-PER", "S-PER")), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "S-PER")), # Multi-token mentions must end in E - LabeledSequence(("a",), ("B-PER",)), - LabeledSequence(("a", "b"), ("B-PER", "I-PER")), + LabeledSequence(tokens=("a",), labels=("B-PER",)), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "I-PER")), # Ends with wrong type - LabeledSequence(("a", "b", "c"), ("B-PER", "I-PER", "E-ORG")), + LabeledSequence(tokens=("a", "b", "c"), labels=("B-PER", "I-PER", "E-ORG")), # Multi-token mentions cannot end in S - LabeledSequence(("a", "b", "c"), ("B-PER", "I-PER", "S-PER")), + LabeledSequence(tokens=("a", "b", "c"), labels=("B-PER", "I-PER", "S-PER")), ] for sent in sents: with pytest.raises(AssertionError): @@ -319,10 +323,10 @@ def test_decode_bioes_invalid_continue() -> None: decoder = get_encoding("BIOES") sents = [ # B must be followed by I or E of the same type - LabeledSequence(("a", "b"), ("B-PER", "B-PER")), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "B-PER")), # Cannot change types mid-mention - LabeledSequence(("a", "b"), ("B-PER", "E-ORG")), - LabeledSequence(("a", "b", "c"), ("B-PER", "I-PER", "E-ORG")), + LabeledSequence(tokens=("a", "b"), labels=("B-PER", "E-ORG")), + LabeledSequence(tokens=("a", "b", "c"), labels=("B-PER", "I-PER", "E-ORG")), ] for sent in sents: with pytest.raises(AssertionError): diff --git a/tests/test_model.py b/tests/test_model.py index 0853d16..80954cc 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -8,12 +8,21 @@ def test_span() -> None: assert len(Span(1, 2)) == 1 assert len(Span(0, 2)) == 2 + with pytest.raises(ValueError): + Span(-2, -1) + with pytest.raises(ValueError): Span(-1, 0) + with pytest.raises(ValueError): + Span(0, -1) + with pytest.raises(ValueError): Span(0, 0) + with pytest.raises(ValueError): + Span(3, 1) + def test_mention() -> None: m1 = Mention(Span(0, 1), "PER") @@ -31,8 +40,8 @@ def test_mention() -> None: def test_labeled_sentence() -> None: s1 = LabeledSequence( - ["a", "b"], - ["B-PER", "I-PER"], + tokens=("a", "b"), + labels=("B-PER", "I-PER"), provenance=SequenceProvenance(7, "test"), ) assert s1.tokens == ("a", "b") @@ -46,31 +55,37 @@ def test_labeled_sentence() -> None: assert s1.span_tokens(Span(0, 1)) == ("a",) assert s1.mention_tokens(Mention(Span(0, 1), "PER")) == ("a",) - s2 = LabeledSequence(s1.tokens, s1.labels) + s2 = LabeledSequence(tokens=s1.tokens, labels=s1.labels) # Provenance not included in equality assert s1 == s2 + # Hashes identical for equal objects + assert hash(s1) == hash(s2) + # Equality fails for objects of other types + assert s1 != "" with pytest.raises(ValueError): # Mismatched length - LabeledSequence(["a", "b"], ["B-PER"]) + LabeledSequence(tokens=("a", "b"), labels=("B-PER",)) with pytest.raises(ValueError): # Empty - LabeledSequence([], []) + LabeledSequence(tokens=(), labels=()) with pytest.raises(ValueError) as err: # Bad label - LabeledSequence(["a"], [""]) - assert str(err.value) == "Invalid label at sequence index 0: ''" + LabeledSequence(tokens=("a",), labels=("",)) + assert "Invalid label at sequence index 0: ''" in str(err.value) with pytest.raises(ValueError) as err: # Bad token - LabeledSequence([""], ["B-PER"]) - assert str(err.value) == "Invalid token at sequence index 0: ''" + LabeledSequence(tokens=("",), labels=("B-PER",)) + assert "Invalid token at sequence index 0: ''" in str(err.value) s2 = s1.with_mentions([Mention(Span(0, 2), "PER")]) assert s2.mentions == (Mention(Span(0, 2), "PER"),) with pytest.raises(ValueError): # Mismatched length between tokens and other_fields - LabeledSequence(["a", "b"], ["B-PER", "I-PER"], orig_fields=[["DT"]]) + LabeledSequence( + tokens=("a", "b"), labels=("B-PER", "I-PER"), token_fields=(("DT",),) + ) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index a74558d..66f3e9e 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -234,8 +234,12 @@ def test_compute_scores() -> None: Mention(Span(4, 5), "ORG"), ) tokens = ("a", "b", "c", "d", "e") - ref_sequence = LabeledSequence(tokens, ref_labels, ref_mentions) - pred_sequence = LabeledSequence(tokens, pred_labels, pred_mentions) + ref_sequence = LabeledSequence( + tokens=tokens, labels=ref_labels, mentions=ref_mentions + ) + pred_sequence = LabeledSequence( + tokens=tokens, labels=pred_labels, mentions=pred_mentions + ) class_score, acc_score = compute_scores([[pred_sequence]], [[ref_sequence]]) assert acc_score.accuracy == 4 / 5 print(class_score) @@ -248,10 +252,14 @@ def test_token_count_error() -> None: ref_labels = ("O", "B-ORG", "I-ORG", "O") pred_labels = ("O", "B-ORG", "I-ORG", "O", "O") ref_sequence = LabeledSequence( - ("a", "b", "c", "d"), ref_labels, provenance=SequenceProvenance(0, "test") + tokens=("a", "b", "c", "d"), + labels=ref_labels, + provenance=SequenceProvenance(0, "test"), ) pred_sequence = LabeledSequence( - ("a", "b", "c", "d", "e"), pred_labels, provenance=SequenceProvenance(0, "test") + tokens=("a", "b", "c", "d", "e"), + labels=pred_labels, + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(TokenCountError): compute_scores([[pred_sequence]], [[ref_sequence]]) @@ -259,7 +267,7 @@ def test_token_count_error() -> None: def test_token_count_error_provenance_none_raises_error() -> None: labels = ("O", "B-ORG") - sequence = LabeledSequence(("a", "b"), labels, provenance=None) + sequence = LabeledSequence(tokens=("a", "b"), labels=labels, provenance=None) with pytest.raises(ValueError): TokenCountError.from_predicted_sequence(2, sequence) @@ -269,10 +277,10 @@ def test_differing_num_docs() -> None: pred_labels = ("O", "B-LOC") tokens = ("a", "b") ref_sequence = LabeledSequence( - tokens, ref_labels, provenance=SequenceProvenance(0, "test") + tokens=tokens, labels=ref_labels, provenance=SequenceProvenance(0, "test") ) pred_sequence = LabeledSequence( - tokens, pred_labels, provenance=SequenceProvenance(0, "test") + tokens=tokens, labels=pred_labels, provenance=SequenceProvenance(0, "test") ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence], [ref_sequence]]) @@ -283,10 +291,10 @@ def test_differing_doc_length() -> None: pred_labels = ("O", "B-LOC") tokens = ("a", "b") ref_sequence = LabeledSequence( - tokens, ref_labels, provenance=SequenceProvenance(0, "test") + tokens=tokens, labels=ref_labels, provenance=SequenceProvenance(0, "test") ) pred_sequence = LabeledSequence( - tokens, pred_labels, provenance=SequenceProvenance(0, "test") + tokens=tokens, labels=pred_labels, provenance=SequenceProvenance(0, "test") ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence, ref_sequence]]) @@ -296,10 +304,10 @@ def test_differing_pred_and_ref_tokens() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") ref_sequence = LabeledSequence( - ("a", "b"), ref_labels, provenance=SequenceProvenance(0, "test") + tokens=("a", "b"), labels=ref_labels, provenance=SequenceProvenance(0, "test") ) pred_sequence = LabeledSequence( - ("a", "c"), pred_labels, provenance=SequenceProvenance(0, "test") + tokens=("a", "c"), labels=pred_labels, provenance=SequenceProvenance(0, "test") ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence]]) diff --git a/tests/test_utils.py b/tests/test_utils.py index f5b6721..70fb344 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,10 @@ import os -from seqscore.util import file_fields_match, file_lines_match, tuplify_strs +from seqscore.util import ( + file_fields_match, + file_lines_match, + tuplify_strs, +) def test_tuplify_strs() -> None: From bfe5d3bc13902205a3ff77693e9b075f05aab2ed Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Tue, 9 Jun 2026 13:58:23 -0400 Subject: [PATCH 04/15] Replace some LabeledSequence uses with AnnotatedSequence --- seqscore/conll.py | 18 ++++---- seqscore/encoding.py | 4 +- seqscore/model.py | 90 ++++++++++++++++++++++++-------------- seqscore/processing.py | 10 ++--- seqscore/scoring.py | 8 ++-- tests/test_conll_format.py | 6 +-- tests/test_encoding.py | 4 +- tests/test_model.py | 53 ++++++++++++++++++---- tests/test_scoring.py | 58 ++++++++++++++++-------- 9 files changed, 166 insertions(+), 85 deletions(-) diff --git a/seqscore/conll.py b/seqscore/conll.py index 2810872..cfe800b 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -15,7 +15,7 @@ from tabulate import tabulate from seqscore.encoding import Encoding, EncodingError, get_encoding -from seqscore.model import LabeledSequence, SequenceProvenance +from seqscore.model import AnnotatedSequence, SequenceProvenance from seqscore.scoring import ( AccuracyScore, ClassificationScore, @@ -116,9 +116,9 @@ def ingest( repair: Optional[str], *, quiet: bool = False, - ) -> list[list[LabeledSequence]]: - all_documents: list[list[LabeledSequence]] = [] - document: list[LabeledSequence] = [] + ) -> list[list[AnnotatedSequence]]: + all_documents: list[list[AnnotatedSequence]] = [] + document: list[AnnotatedSequence] = [] for source_sequence, comment in self._parse_file( source, source_name, parse_comments=self.parse_comment_lines @@ -206,7 +206,7 @@ def ingest( ) from e try: - sequences = LabeledSequence( + sequences = AnnotatedSequence( tokens=tokens, labels=labels, mentions=tuple(mentions), @@ -358,7 +358,7 @@ def ingest_conll_file( ignore_document_boundaries: bool, parse_comment_lines: bool, quiet: bool = False, -) -> list[list[LabeledSequence]]: +) -> list[list[AnnotatedSequence]]: mention_encoding = get_encoding(mention_encoding_name) if repair and repair not in mention_encoding.supported_repair_methods(): @@ -419,7 +419,7 @@ def repair_conll_file( ignore_document_boundaries: bool, parse_comment_lines: bool, quiet: bool, -) -> list[list[LabeledSequence]]: +) -> list[list[AnnotatedSequence]]: return ingest_conll_file( input_file, mention_encoding_name, @@ -433,7 +433,7 @@ def repair_conll_file( def write_docs_using_encoding( - docs: Sequence[Sequence[LabeledSequence]], + docs: Sequence[Sequence[AnnotatedSequence]], mention_encoding_name: str, file_encoding: str, delim: str, @@ -459,7 +459,7 @@ def write_docs_using_encoding( def write_doc_using_encoding( - doc: Sequence[LabeledSequence], + doc: Sequence[AnnotatedSequence], encoding: Encoding, delim: str, file: TextIO, diff --git a/seqscore/encoding.py b/seqscore/encoding.py index 9efae76..9ea6316 100644 --- a/seqscore/encoding.py +++ b/seqscore/encoding.py @@ -9,7 +9,7 @@ from attr import Factory, attrib, attrs -from seqscore.model import LabeledSequence, Mention, Span +from seqscore.model import AnnotatedSequence, LabeledSequence, Mention, Span REPAIR_CONLL = "conlleval" REPAIR_DISCARD = "discard" @@ -134,7 +134,7 @@ def encode_mentions( def encode_sequence( self, - sequence: LabeledSequence, + sequence: AnnotatedSequence, ) -> Sequence[str]: labels = self.encode_mentions(sequence.mentions, len(sequence)) assert len(labels) == len(sequence) diff --git a/seqscore/model.py b/seqscore/model.py index 0e943f4..74f69a8 100644 --- a/seqscore/model.py +++ b/seqscore/model.py @@ -17,6 +17,7 @@ from seqscore.encoding import Encoding # pragma: no cover __all__ = [ + "AnnotatedSequence", "LabeledSequence", "Mention", "SequenceProvenance", @@ -120,37 +121,19 @@ def tokens_with_fields( ) -> tuple[tuple[str, Optional[tuple[str, ...]]], ...]: ... -class LabeledSequence(BaseModel, Sequence[str]): - """A sequence of tokens with associated labels and optional mentions. - - Tokens and labels must be non-empty and of equal length. Iterating - over a LabeledSequence yields its tokens. - """ +class _SequenceBase(BaseModel, Sequence[str]): + """Shared base for token sequences with labels.""" model_config = ConfigDict(frozen=True) tokens: tuple[str, ...] labels: tuple[str, ...] - mentions: tuple[Mention, ...] = () token_fields: Optional[tuple[tuple[str, ...], ...]] = None provenance: Optional[SequenceProvenance] = None comment: Optional[str] = None - def __eq__(self, other: object) -> bool: - if not isinstance(other, LabeledSequence): - return NotImplemented - return ( - self.tokens == other.tokens - and self.labels == other.labels - and self.mentions == other.mentions - and self.token_fields == other.token_fields - ) - - def __hash__(self) -> int: - return hash((self.tokens, self.labels, self.mentions, self.token_fields)) - @model_validator(mode="after") - def _validate_fields(self) -> "LabeledSequence": + def _validate_fields(self) -> "_SequenceBase": if len(self.tokens) != len(self.labels): raise ValueError( f"Tokens ({len(self.tokens)}) and labels ({len(self.labels)}) " @@ -175,17 +158,6 @@ def _validate_fields(self) -> "LabeledSequence": return self - def with_mentions(self, mentions: Sequence[Mention]) -> "LabeledSequence": - """Return a copy of this sequence with different mentions.""" - return LabeledSequence( - tokens=self.tokens, - labels=self.labels, - mentions=tuple(mentions), - token_fields=self.token_fields, - provenance=self.provenance, - comment=self.comment, - ) - @overload def __getitem__(self, index: int) -> str: ... @@ -223,6 +195,56 @@ def span_tokens(self, span: Span) -> tuple[str, ...]: """Return the tokens included in the given span.""" return self.tokens[span.start : span.end] + +class LabeledSequence(_SequenceBase): + """A sequence of tokens with their labels. + + This class only contains labels and tokens. For mentions, use + AnnotatedSequence. + + Equality and hashing are defined using only the tokens and labels. + """ + + def __eq__(self, other: object) -> bool: + if not isinstance(other, LabeledSequence): + return False + return self.tokens == other.tokens and self.labels == other.labels + + def __hash__(self) -> int: + return hash((self.tokens, self.labels)) + + +class AnnotatedSequence(_SequenceBase): + """A sequence of tokens with labels and decoded mentions. + + Equality and hashing are defined using only the tokens, labels, and mentions. + """ + + mentions: tuple[Mention, ...] + + def __eq__(self, other: object) -> bool: + if not isinstance(other, AnnotatedSequence): + return False + return ( + self.tokens == other.tokens + and self.labels == other.labels + and self.mentions == other.mentions + ) + + def __hash__(self) -> int: + return hash((self.tokens, self.labels, self.mentions)) + + def with_mentions(self, mentions: Sequence[Mention]) -> "AnnotatedSequence": + """Return a copy of this sequence with different mentions.""" + return AnnotatedSequence( + tokens=self.tokens, + labels=self.labels, + mentions=tuple(mentions), + token_fields=self.token_fields, + provenance=self.provenance, + comment=self.comment, + ) + def mention_tokens(self, mention: Mention) -> tuple[str, ...]: """Return the tokens included in the given mention.""" return self.span_tokens(mention.span) @@ -234,8 +256,8 @@ def from_tokens_and_labels( labels: Sequence[str], encoding: "Encoding", **kwargs: Any, - ) -> "LabeledSequence": - """Create a LabeledSequence by decoding mentions from the labels using the given encoding.""" + ) -> "AnnotatedSequence": + """Create an AnnotatedSequence by decoding mentions from labels using the given encoding.""" mentions = encoding.decode_labels(labels) return cls( tokens=tuple(tokens), labels=tuple(labels), mentions=tuple(mentions), **kwargs diff --git a/seqscore/processing.py b/seqscore/processing.py index 581a826..3a7ede5 100644 --- a/seqscore/processing.py +++ b/seqscore/processing.py @@ -1,6 +1,6 @@ from collections.abc import Iterable -from seqscore.model import LabeledSequence, Mention +from seqscore.model import AnnotatedSequence, Mention class TypeMapper: @@ -30,7 +30,7 @@ def __init__( else: self.type_map[from_type] = to_type - def map_types(self, sequence: LabeledSequence) -> LabeledSequence: + def map_types(self, sequence: AnnotatedSequence) -> AnnotatedSequence: new_mentions: list[Mention] = [] for mention in sequence.mentions: if mention.type in self.type_map: @@ -47,13 +47,13 @@ def map_types(self, sequence: LabeledSequence) -> LabeledSequence: def modify_types( - docs: list[list[LabeledSequence]], + docs: list[list[AnnotatedSequence]], keep_types: set[str], remove_types: set[str], type_map: dict[str, list[str]], -) -> list[list[LabeledSequence]]: +) -> list[list[AnnotatedSequence]]: mapper = TypeMapper(keep_types, remove_types, type_map) - mapped_docs: list[list[LabeledSequence]] = [] + mapped_docs: list[list[AnnotatedSequence]] = [] for doc in docs: mapped_docs.append([mapper.map_types(sequence) for sequence in doc]) diff --git a/seqscore/scoring.py b/seqscore/scoring.py index c0c8079..46983db 100644 --- a/seqscore/scoring.py +++ b/seqscore/scoring.py @@ -6,7 +6,7 @@ from attr import Factory, attrib, attrs from seqscore.encoding import Encoding, EncodingError, get_encoding -from seqscore.model import LabeledSequence, Mention +from seqscore.model import AnnotatedSequence, Mention from seqscore.util import tuplify_strs, validator_nonempty_str from seqscore.validation import validate_labels @@ -48,7 +48,7 @@ def __init__( @classmethod def from_predicted_sequence( - cls, reference_token_count: int, pred_sequence: LabeledSequence + cls, reference_token_count: int, pred_sequence: AnnotatedSequence ) -> "TokenCountError": if pred_sequence.provenance is None: raise ValueError( @@ -130,8 +130,8 @@ def accuracy(self) -> float: def compute_scores( - pred_docs: Sequence[Sequence[LabeledSequence]], - ref_docs: Sequence[Sequence[LabeledSequence]], + pred_docs: Sequence[Sequence[AnnotatedSequence]], + ref_docs: Sequence[Sequence[AnnotatedSequence]], *, count_fp_fn_examples: bool = False, ) -> tuple[ClassificationScore, AccuracyScore]: diff --git a/tests/test_conll_format.py b/tests/test_conll_format.py index 2b19a71..e1d456e 100644 --- a/tests/test_conll_format.py +++ b/tests/test_conll_format.py @@ -12,7 +12,7 @@ write_docs_using_encoding, ) from seqscore.encoding import REPAIR_NONE, get_encoding -from seqscore.model import LabeledSequence +from seqscore.model import AnnotatedSequence from seqscore.util import file_fields_match from seqscore.validation import InvalidLabelError @@ -159,12 +159,12 @@ def test_bad_label1() -> None: def test_write_docs_no_orig_fields(tmp_path: Path) -> None: - sent1 = LabeledSequence( + sent1 = AnnotatedSequence( tokens=("This", "is", "a", "sentence", "."), labels=("O", "O", "O", "O", "O"), mentions=(), ) - sent2 = LabeledSequence.from_tokens_and_labels( + sent2 = AnnotatedSequence.from_tokens_and_labels( ( "University", "of", diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 427c614..377b72f 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -15,7 +15,7 @@ EncodingError, get_encoding, ) -from seqscore.model import LabeledSequence, Mention, Span +from seqscore.model import AnnotatedSequence, LabeledSequence, Mention, Span FULL_SENTENCE_LABELS = { "IO": ["I-PER", "O", "I-ORG", "I-ORG", "I-ORG", "I-ORG", "I-ORG", "I-LOC"], @@ -159,7 +159,7 @@ def test_basic_encoding() -> None: assert encoding.encode_mentions(mentions, len(labels)) == labels # Also test encoding sentence object, intentionally putting no mentions in the # sentence labels to make sure encoding using the mentions, not the labels - sentence = LabeledSequence( + sentence = AnnotatedSequence( tokens=tuple("a" for _ in labels), labels=tuple("O" for _ in labels), mentions=tuple(mentions), diff --git a/tests/test_model.py b/tests/test_model.py index 80954cc..6a3ce3f 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,12 @@ import pytest -from seqscore.model import LabeledSequence, Mention, SequenceProvenance, Span +from seqscore.model import ( + AnnotatedSequence, + LabeledSequence, + Mention, + SequenceProvenance, + Span, +) def test_span() -> None: @@ -38,11 +44,13 @@ def test_mention() -> None: Mention(Span(0, 1), None) # type: ignore -def test_labeled_sentence() -> None: +def test_labeled_sequence() -> None: s1 = LabeledSequence( tokens=("a", "b"), labels=("B-PER", "I-PER"), provenance=SequenceProvenance(7, "test"), + token_fields=(("NOUN",), ("VERB",)), + comment="Comment", ) assert s1.tokens == ("a", "b") assert s1[0] == "a" @@ -53,10 +61,9 @@ def test_labeled_sentence() -> None: assert str(s1) == "a/B-PER b/I-PER" assert s1.tokens_with_labels() == (("a", "B-PER"), ("b", "I-PER")) assert s1.span_tokens(Span(0, 1)) == ("a",) - assert s1.mention_tokens(Mention(Span(0, 1), "PER")) == ("a",) s2 = LabeledSequence(tokens=s1.tokens, labels=s1.labels) - # Provenance not included in equality + # Attributes other than tokens and labels not included in equality assert s1 == s2 # Hashes identical for equal objects assert hash(s1) == hash(s2) @@ -81,11 +88,41 @@ def test_labeled_sentence() -> None: LabeledSequence(tokens=("",), labels=("B-PER",)) assert "Invalid token at sequence index 0: ''" in str(err.value) - s2 = s1.with_mentions([Mention(Span(0, 2), "PER")]) - assert s2.mentions == (Mention(Span(0, 2), "PER"),) - with pytest.raises(ValueError): - # Mismatched length between tokens and other_fields + # Mismatched length between tokens and token_fields LabeledSequence( tokens=("a", "b"), labels=("B-PER", "I-PER"), token_fields=(("DT",),) ) + + +def test_annotated_sequence() -> None: + s1 = AnnotatedSequence( + tokens=("a", "b"), + labels=("B-PER", "I-PER"), + mentions=(Mention(Span(0, 2), "PER"),), + provenance=SequenceProvenance(7, "test"), + token_fields=(("NOUN",), ("VERB",)), + comment="Comment", + ) + assert s1.tokens == ("a", "b") + assert s1[0] == "a" + assert s1[0:2] == ("a", "b") + assert list(s1) == ["a", "b"] + assert s1.labels == ("B-PER", "I-PER") + assert s1.mentions == (Mention(Span(0, 2), "PER"),) + assert s1.provenance == SequenceProvenance(7, "test") + assert str(s1) == "a/B-PER b/I-PER" + assert s1.tokens_with_labels() == (("a", "B-PER"), ("b", "I-PER")) + assert s1.span_tokens(Span(0, 1)) == ("a",) + assert s1.mention_tokens(Mention(Span(0, 1), "PER")) == ("a",) + + s2 = AnnotatedSequence(tokens=s1.tokens, labels=s1.labels, mentions=s1.mentions) + # Attributes other than tokens, labels, and mentions not included in equality + assert s1 == s2 + # Hashes identical for equal objects + assert hash(s1) == hash(s2) + # Equality fails for objects of other types + assert s1 != "" + + s3 = s1.with_mentions([Mention(Span(0, 1), "ORG")]) + assert s3.mentions == (Mention(Span(0, 1), "ORG"),) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 66f3e9e..c08e307 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -4,7 +4,7 @@ import pytest from seqscore.encoding import EncodingError -from seqscore.model import LabeledSequence, Mention, SequenceProvenance, Span +from seqscore.model import AnnotatedSequence, Mention, SequenceProvenance, Span from seqscore.scoring import ( AccuracyScore, ClassificationScore, @@ -234,10 +234,10 @@ def test_compute_scores() -> None: Mention(Span(4, 5), "ORG"), ) tokens = ("a", "b", "c", "d", "e") - ref_sequence = LabeledSequence( + ref_sequence = AnnotatedSequence( tokens=tokens, labels=ref_labels, mentions=ref_mentions ) - pred_sequence = LabeledSequence( + pred_sequence = AnnotatedSequence( tokens=tokens, labels=pred_labels, mentions=pred_mentions ) class_score, acc_score = compute_scores([[pred_sequence]], [[ref_sequence]]) @@ -251,14 +251,16 @@ def test_compute_scores() -> None: def test_token_count_error() -> None: ref_labels = ("O", "B-ORG", "I-ORG", "O") pred_labels = ("O", "B-ORG", "I-ORG", "O", "O") - ref_sequence = LabeledSequence( + ref_sequence = AnnotatedSequence( tokens=("a", "b", "c", "d"), labels=ref_labels, + mentions=(), provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( + pred_sequence = AnnotatedSequence( tokens=("a", "b", "c", "d", "e"), labels=pred_labels, + mentions=(), provenance=SequenceProvenance(0, "test"), ) with pytest.raises(TokenCountError): @@ -267,7 +269,9 @@ def test_token_count_error() -> None: def test_token_count_error_provenance_none_raises_error() -> None: labels = ("O", "B-ORG") - sequence = LabeledSequence(tokens=("a", "b"), labels=labels, provenance=None) + sequence = AnnotatedSequence( + tokens=("a", "b"), labels=labels, mentions=(), provenance=None + ) with pytest.raises(ValueError): TokenCountError.from_predicted_sequence(2, sequence) @@ -276,11 +280,17 @@ def test_differing_num_docs() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") tokens = ("a", "b") - ref_sequence = LabeledSequence( - tokens=tokens, labels=ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=tokens, + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - tokens=tokens, labels=pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=tokens, + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence], [ref_sequence]]) @@ -290,11 +300,17 @@ def test_differing_doc_length() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") tokens = ("a", "b") - ref_sequence = LabeledSequence( - tokens=tokens, labels=ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=tokens, + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - tokens=tokens, labels=pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=tokens, + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence, ref_sequence]]) @@ -303,11 +319,17 @@ def test_differing_doc_length() -> None: def test_differing_pred_and_ref_tokens() -> None: ref_labels = ("O", "B-ORG") pred_labels = ("O", "B-LOC") - ref_sequence = LabeledSequence( - tokens=("a", "b"), labels=ref_labels, provenance=SequenceProvenance(0, "test") + ref_sequence = AnnotatedSequence( + tokens=("a", "b"), + labels=ref_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) - pred_sequence = LabeledSequence( - tokens=("a", "c"), labels=pred_labels, provenance=SequenceProvenance(0, "test") + pred_sequence = AnnotatedSequence( + tokens=("a", "c"), + labels=pred_labels, + mentions=(), + provenance=SequenceProvenance(0, "test"), ) with pytest.raises(ValueError): compute_scores([[pred_sequence]], [[ref_sequence]]) From bd78dd4866d6dea501fd12c41217d69e74fabdf1 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Tue, 9 Jun 2026 14:38:34 -0400 Subject: [PATCH 05/15] Migrate remaining attrs classes to dataclasses --- pyproject.toml | 1 - seqscore/conll.py | 38 ++++++++++++++--------------- seqscore/encoding.py | 11 ++++----- seqscore/scoring.py | 32 ++++++++++++------------- seqscore/util.py | 23 +----------------- seqscore/validation.py | 52 ++++++++++++++++++---------------------- tests/test_encoding.py | 5 ++-- tests/test_utils.py | 7 ------ tests/test_validation.py | 5 ++-- 9 files changed, 69 insertions(+), 105 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fd5454d..0c8d9d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ authors = [ ] requires-python = ">=3.10" dependencies = [ - "attrs>=19.2.0", "click", "pydantic>=2.0", "tabulate", diff --git a/seqscore/conll.py b/seqscore/conll.py index cfe800b..97e7ccc 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -1,6 +1,7 @@ import sys from collections import Counter, defaultdict from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field from itertools import chain from math import sqrt from statistics import mean, stdev @@ -11,7 +12,6 @@ TextIO, ) -from attr import attrib, attrs from tabulate import tabulate from seqscore.encoding import Encoding, EncodingError, get_encoding @@ -46,17 +46,17 @@ class CoNLLFormatError(Exception): pass -@attrs(frozen=True) +@dataclass(frozen=True) class LineSpec: """Defines the fields and delimiters for a CoNLL-format line""" - token_index: int = attrib() - ner_label_index: int = attrib() + token_index: int + ner_label_index: int - def __attrs_post_init__(self) -> None: + def __post_init__(self) -> None: # This will only catch cases where the indices are identical, not - # when they refer to the same position, such as 1 and -1 in a - # sequence of length two + # when they refer to the same position (such as 1 and -1 in a + # sequence of length two). if self.token_index == self.ner_label_index: raise ValueError( f"Token index ({self.token_index}) and " @@ -64,13 +64,13 @@ def __attrs_post_init__(self) -> None: ) -@attrs(frozen=True) +@dataclass(frozen=True, slots=True) class _CoNLLToken: - text: str = attrib() - label: str = attrib() - is_docstart: bool = attrib() - line_num: int = attrib() - orig_fields: tuple[str, ...] = attrib() + text: str + label: str + is_docstart: bool + line_num: int + orig_fields: tuple[str, ...] @classmethod def from_line( @@ -102,12 +102,12 @@ def from_line( return cls(text, label, is_docstart, line_num, orig_fields) -@attrs(frozen=True) +@dataclass(frozen=True) class CoNLLIngester: - encoding: Encoding = attrib() - line_spec: LineSpec = attrib() - parse_comment_lines: bool = attrib(default=False, kw_only=True) - ignore_document_boundaries: bool = attrib(default=False, kw_only=True) + encoding: Encoding + line_spec: LineSpec + parse_comment_lines: bool = field(default=False, kw_only=True) + ignore_document_boundaries: bool = field(default=False, kw_only=True) def ingest( self, @@ -401,7 +401,7 @@ def validate_conll_file( n_sequences = sum(len(doc_results) for doc_results in results) n_tokens = sum(sent.n_tokens for doc_results in results for sent in doc_results) - errors = list( + errors = tuple( chain.from_iterable( result.errors for doc_results in results for result in doc_results ) diff --git a/seqscore/encoding.py b/seqscore/encoding.py index 9ea6316..fceba2a 100644 --- a/seqscore/encoding.py +++ b/seqscore/encoding.py @@ -1,5 +1,6 @@ from abc import abstractmethod from collections.abc import Sequence +from dataclasses import dataclass, field from functools import lru_cache from typing import ( AbstractSet, @@ -7,8 +8,6 @@ Protocol, ) -from attr import Factory, attrib, attrs - from seqscore.model import AnnotatedSequence, LabeledSequence, Mention, Span REPAIR_CONLL = "conlleval" @@ -660,11 +659,11 @@ def get_encoding(name: str) -> Encoding: raise ValueError(f"Unknown encoder {repr(name)}") -@attrs +@dataclass class _MentionBuilder: - start_idx: Optional[int] = attrib(default=None, init=False) - entity_type: Optional[str] = attrib(default=None, init=False) - mentions: list[Mention] = attrib(default=Factory(list), init=False) + start_idx: Optional[int] = field(default=None, init=False) + entity_type: Optional[str] = field(default=None, init=False) + mentions: list[Mention] = field(default_factory=list, init=False) def start_mention(self, start_idx: int, entity_type: str) -> None: # Check arguments diff --git a/seqscore/scoring.py b/seqscore/scoring.py index 46983db..ff9dac8 100644 --- a/seqscore/scoring.py +++ b/seqscore/scoring.py @@ -1,13 +1,11 @@ from collections import Counter, defaultdict from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field from decimal import ROUND_HALF_UP, Decimal from typing import DefaultDict, Optional, Union -from attr import Factory, attrib, attrs - from seqscore.encoding import Encoding, EncodingError, get_encoding from seqscore.model import AnnotatedSequence, Mention -from seqscore.util import tuplify_strs, validator_nonempty_str from seqscore.validation import validate_labels @@ -15,10 +13,10 @@ def _defaultdict_classification_score() -> DefaultDict[str, "ClassificationScore return defaultdict(ClassificationScore) -@attrs(frozen=True, slots=True) +@dataclass(frozen=True, slots=True) class TokensWithType: - tokens: tuple[str, ...] = attrib(converter=tuplify_strs) - type: str = attrib(validator=validator_nonempty_str) + tokens: tuple[str, ...] + type: str class TokenCountError(ValueError): @@ -62,16 +60,16 @@ def from_predicted_sequence( ) -@attrs +@dataclass class ClassificationScore: - true_pos: int = attrib(default=0, kw_only=True) - false_pos: int = attrib(default=0, kw_only=True) - false_neg: int = attrib(default=0, kw_only=True) - type_scores: DefaultDict[str, "ClassificationScore"] = attrib( - default=Factory(_defaultdict_classification_score), kw_only=True + true_pos: int = field(default=0, kw_only=True) + false_pos: int = field(default=0, kw_only=True) + false_neg: int = field(default=0, kw_only=True) + type_scores: DefaultDict[str, "ClassificationScore"] = field( + default_factory=_defaultdict_classification_score, kw_only=True ) - false_pos_examples: Counter[TokensWithType] = attrib(default=Factory(Counter)) - false_neg_examples: Counter[TokensWithType] = attrib(default=Factory(Counter)) + false_pos_examples: Counter[TokensWithType] = field(default_factory=Counter) + false_neg_examples: Counter[TokensWithType] = field(default_factory=Counter) def count_false_positive(self, tokens: Iterable[str], type_: str) -> None: self.false_pos_examples[TokensWithType(tuple(tokens), type_)] += 1 @@ -117,10 +115,10 @@ def f1(self) -> float: return 2 * (precision * recall) / (precision + recall) -@attrs +@dataclass class AccuracyScore: - hits: int = attrib(default=0, kw_only=True) - total: int = attrib(default=0, kw_only=True) + hits: int = field(default=0, kw_only=True) + total: int = field(default=0, kw_only=True) @property def accuracy(self) -> float: diff --git a/seqscore/util.py b/seqscore/util.py index 5c6d5a6..2eb54cc 100644 --- a/seqscore/util.py +++ b/seqscore/util.py @@ -1,23 +1,14 @@ import os -from collections.abc import Iterable from itertools import zip_longest from os import PathLike from pathlib import Path -from typing import Any, Union - -from attr import Attribute, validators +from typing import Union # Union[str, Path] isn't enough to appease PyCharm's type checker, so adding Path here # avoids warnings. PathType = Union[str, Path, PathLike] -# Type-specific implementations to work around type checker limitations. No, writing these as -# generic functions with type variables does not satisfy all type checkers. -def tuplify_strs(strs: Iterable[str]) -> tuple[str, ...]: - return tuple(strs) - - def file_fields_match(path1: PathType, path2: PathType, *, debug: bool = False) -> bool: """Return whether the whitespace-delimited fields of two files are identical.""" with open(path1, encoding="utf8") as f1, open(path2, encoding="utf8") as f2: @@ -49,15 +40,3 @@ def file_lines_match(path1: PathType, path2: PathType, debug: bool = False) -> b def normalize_str_with_path(s: str) -> str: """Normalize the OS path separator to '/'.""" return s.replace(os.path.sep, "/") - - -# Instantiate in advance for _validator_nonempty_str -_instance_of_str = validators.instance_of(str) - - -def validator_nonempty_str(_inst: Any, attr: Attribute, value: Any) -> None: - # Check type - _instance_of_str(value, attr, value) - # Check string isn't empty - if not value: - raise ValueError(f"Empty string: {repr(value)}") diff --git a/seqscore/validation.py b/seqscore/validation.py index 9593713..bc2a332 100644 --- a/seqscore/validation.py +++ b/seqscore/validation.py @@ -1,24 +1,22 @@ -from collections.abc import Iterable, Sequence +from collections.abc import Sequence +from dataclasses import dataclass from typing import Any, Optional -from attr import attrib, attrs - from seqscore.encoding import _ENCODING_NAMES, Encoding, EncodingError -from seqscore.util import tuplify_strs # All encodings can be validated VALIDATION_SUPPORTED_ENCODINGS: Sequence[str] = tuple(_ENCODING_NAMES) -@attrs +@dataclass class ValidationError: - msg: str = attrib() - label: str = attrib() - type: str = attrib() - state: str = attrib() - token: Optional[str] = attrib(default=None) - line_num: Optional[int] = attrib(default=None) - source_name: Optional[str] = attrib(default=None) + msg: str + label: str + type: str + state: str + token: Optional[str] = None + line_num: Optional[int] = None + source_name: Optional[str] = None class InvalidStateError(ValidationError): @@ -35,17 +33,11 @@ def __init__(self, label: str, *args: Any, **kwargs: Any) -> None: self.label: str = label -def tuplify_errors(errors: Iterable[ValidationError]) -> tuple[ValidationError, ...]: - return tuple(errors) - - -@attrs +@dataclass class SequenceValidationResult: - errors: Sequence[ValidationError] = attrib(converter=tuplify_errors) - n_tokens: int = attrib() - repaired_labels: Optional[tuple[str, ...]] = attrib( - converter=tuplify_strs, default=() - ) + errors: tuple[ValidationError, ...] + n_tokens: int + repaired_labels: tuple[str, ...] = () def is_valid(self) -> bool: return not self.errors @@ -57,12 +49,12 @@ def __len__(self) -> int: return len(self.errors) -@attrs(frozen=True) +@dataclass(frozen=True) class ValidationResult: - errors: Sequence[ValidationError] = attrib(converter=tuplify_errors) - n_tokens: int = attrib() - n_sequences: int = attrib() - n_docs: int = attrib() + errors: tuple[ValidationError, ...] + n_tokens: int + n_sequences: int + n_docs: int def validate_labels( @@ -190,6 +182,8 @@ def validate_labels( if errors and repair: repaired_labels = encoding.repair_labels(labels, repair) - return SequenceValidationResult(errors, len(labels), repaired_labels) + return SequenceValidationResult( + tuple(errors), len(labels), tuple(repaired_labels) + ) else: - return SequenceValidationResult(errors, len(labels)) + return SequenceValidationResult(tuple(errors), len(labels)) diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 377b72f..1224d3e 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass + import pytest -from attr import attrs from seqscore.encoding import ( _ENCODING_NAMES, @@ -51,7 +52,7 @@ } -@attrs(auto_attribs=True) +@dataclass class EdgeTestSentence: name: str mentions: list[Mention] diff --git a/tests/test_utils.py b/tests/test_utils.py index 70fb344..e87420d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,16 +3,9 @@ from seqscore.util import ( file_fields_match, file_lines_match, - tuplify_strs, ) -def test_tuplify_strs() -> None: - strs = ["a", "b", "c"] - tup = tuplify_strs(strs) - assert tup == ("a", "b", "c") - - def test_identical_files() -> None: assert file_fields_match( os.path.join("tests", "test_files", "minimal_bio_copy.txt"), diff --git a/tests/test_validation.py b/tests/test_validation.py index 9fbde19..3f0ecc0 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,11 +1,12 @@ +from dataclasses import dataclass + import pytest -from attr import attrs from seqscore.encoding import REPAIR_NONE, EncodingError, get_encoding from seqscore.validation import validate_labels -@attrs(auto_attribs=True) +@dataclass class RepairTest: original_labels: list[str] n_errors: int From fb3365b7a79d96536773c52d0b637a6728e8cecb Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 11:13:36 -0400 Subject: [PATCH 06/15] Move seqscore main tests to click CliRunner --- seqscore/scripts/seqscore.py | 3 ++- tests/test_seqscore_main.py | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index eae1f13..3176bfb 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -34,7 +34,8 @@ help=f"Provides scoring and analysis tools for NER/chunking files (version {seqscore.__version__})" ) @click.version_option(seqscore.__version__) -# This is tested by a subprocess call in test_seqscore_main so coverage will miss it +# Tests invoke subcommands directly (e.g. runner.invoke(validate, [...])) rather +# than through this group, so tests never actually call this function. def cli() -> None: # pragma: no cover pass diff --git a/tests/test_seqscore_main.py b/tests/test_seqscore_main.py index 7251c62..8515043 100644 --- a/tests/test_seqscore_main.py +++ b/tests/test_seqscore_main.py @@ -1,19 +1,20 @@ -import subprocess +from click.testing import CliRunner import seqscore +from seqscore.scripts.seqscore import cli HELP_OUTPUT = "Usage: seqscore [OPTIONS] COMMAND [ARGS]..." def test_seqscore_help() -> None: - result = subprocess.run(["seqscore", "--help"], capture_output=True, encoding="UTF-8") - assert result.returncode == 0 - assert result.stdout.startswith(HELP_OUTPUT) + runner = CliRunner() + result = runner.invoke(cli, ["--help"], prog_name="seqscore") + assert result.exit_code == 0 + assert result.output.startswith(HELP_OUTPUT) def test_seqscore_version() -> None: - result = subprocess.run( - ["seqscore", "--version"], capture_output=True, encoding="UTF-8" - ) - assert result.returncode == 0 - assert result.stdout == f"seqscore, version {seqscore.__version__}\n" + runner = CliRunner() + result = runner.invoke(cli, ["--version"], prog_name="seqscore") + assert result.exit_code == 0 + assert result.output == f"seqscore, version {seqscore.__version__}\n" From dd5f93e981303b8513dca325d078dcfae2758b01 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 11:14:22 -0400 Subject: [PATCH 07/15] Suggest ignore_document_boundaries in common error case --- seqscore/scoring.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seqscore/scoring.py b/seqscore/scoring.py index ff9dac8..5b50876 100644 --- a/seqscore/scoring.py +++ b/seqscore/scoring.py @@ -136,10 +136,10 @@ def compute_scores( accuracy = AccuracyScore() classification = ClassificationScore() - # TODO: Recommend use of ignore_document_boundaries if this error is encountered if len(pred_docs) != len(ref_docs): raise ValueError( - f"Prediction has {len(pred_docs)} documents, reference has {len(ref_docs)}" + f"Prediction has {len(pred_docs)} documents, reference has {len(ref_docs)}. " + "Consider setting --ignore-document-boundaries/ignore_document_boundaries." ) for pred_doc, ref_doc in zip(pred_docs, ref_docs): From bb2acfb7d058233304d9390927d0d851d14e962c Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 11:19:31 -0400 Subject: [PATCH 08/15] Fix --quiet for validation --- seqscore/scripts/seqscore.py | 3 ++- tests/test_validation_click.py | 26 +++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index 3176bfb..44fa2af 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -168,7 +168,8 @@ def validate( f"Encountered {len(result.errors)} errors in {result.n_tokens} tokens, " + f"{result.n_sequences} sequences, and {result.n_docs} document(s) in {each_file}" ) - print("\n".join(err.msg for err in result.errors)) + if not quiet: + print("\n".join(err.msg for err in result.errors)) error = True elif not quiet: print( diff --git a/tests/test_validation_click.py b/tests/test_validation_click.py index 6f72a7d..c84fc5f 100644 --- a/tests/test_validation_click.py +++ b/tests/test_validation_click.py @@ -64,12 +64,11 @@ def test_mixed_valid_bio_twofiles_quiet() -> None: os.path.join("tests", "conll_annotation", "invalid1.bio"), ], ) + # Only summary line, no individual invalid transitions printed out assert result.output == ( "Encountered 3 errors in 15 tokens, 2 sequences, and 1 document(s) in tests/conll_annotation/invalid1.bio\n" - "Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7\n" - "Invalid transition 'O' -> 'I-LOC' for token 'West' on line 12\n" - "Invalid transition 'O' -> 'I-LOC' for token 'Pennsylvania' on line 15\n" ) + assert "Invalid transition" not in result.output assert result.exit_code != 0 @@ -133,6 +132,27 @@ def test_invalid_bio() -> None: ) +def test_invalid_bio_quiet() -> None: + runner = CliRunner() + result = runner.invoke( + validate, + [ + "--labels", + "BIO", + "-q", + os.path.join("tests", "conll_annotation", "invalid1.bio"), + ], + ) + assert result.exit_code != 0 + assert ( + normalize_str_with_path( + "Encountered 3 errors in 15 tokens, 2 sequences, and 1 document(s) in tests/conll_annotation/invalid1.bio\n" + ) + == result.output + ) + assert "Invalid transition" not in result.output + + def test_invalid_bioes() -> None: runner = CliRunner() result = runner.invoke( From 957a0e9ed54f853fde180a758c003881d8b8cbc2 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 11:27:57 -0400 Subject: [PATCH 09/15] Allow writing raw labels for CoNLL files --- seqscore/conll.py | 87 +++++++++++++++++++---- tests/test_conll_format.py | 137 ++++++++++++++++++++++++++++++++++--- 2 files changed, 202 insertions(+), 22 deletions(-) diff --git a/seqscore/conll.py b/seqscore/conll.py index 97e7ccc..fdf375c 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -15,7 +15,7 @@ from tabulate import tabulate from seqscore.encoding import Encoding, EncodingError, get_encoding -from seqscore.model import AnnotatedSequence, SequenceProvenance +from seqscore.model import AnnotatedSequence, LabeledSequence, SequenceProvenance from seqscore.scoring import ( AccuracyScore, ClassificationScore, @@ -441,9 +441,10 @@ def write_docs_using_encoding( output_path: PathType, *, discard_extra_fields: bool = False, + always_write_docstart: bool = False, ) -> None: mention_encoding = get_encoding(mention_encoding_name) - output_docstart = len(docs) > 1 + output_docstart = len(docs) > 1 or always_write_docstart with open(output_path, "w", encoding=file_encoding) as file: for doc in docs: @@ -458,36 +459,62 @@ def write_docs_using_encoding( ) -def write_doc_using_encoding( - doc: Sequence[AnnotatedSequence], - encoding: Encoding, +def write_docs_raw( + docs: Sequence[Sequence[LabeledSequence]], + file_encoding: str, + delim: str, + line_spec: LineSpec, + output_path: PathType, + *, + outside_label: str = "O", + discard_extra_fields: bool = False, + always_write_docstart: bool = False, +) -> None: + output_docstart = len(docs) > 1 or always_write_docstart + + with open(output_path, "w", encoding=file_encoding) as file: + for doc in docs: + write_doc_raw( + doc, + delim, + file, + line_spec, + output_docstart=output_docstart, + outside_label=outside_label, + discard_extra_fields=discard_extra_fields, + ) + + +def write_doc_raw( + doc: Sequence[LabeledSequence], delim: str, file: TextIO, line_spec: LineSpec, *, output_docstart: bool, + outside_label: str = "O", discard_extra_fields: bool = False, ) -> None: if output_docstart: # Get the fields of the first token of the first sentence if doc[0].token_fields and not discard_extra_fields: - # to figure out how many fields there are + # Figure out how many fields there are sequence_orig_fields = doc[0].token_fields[0] - # Create the write number of fields + # Create the right number of fields fields = [EMPTY_OTHER_FIELD] * len(sequence_orig_fields) # Fill in the token and label fields[line_spec.token_index] = DOCSTART - fields[line_spec.ner_label_index] = encoding.dialect.outside + fields[line_spec.ner_label_index] = outside_label else: - fields = [DOCSTART, encoding.dialect.outside] + fields = [DOCSTART, outside_label] # Write output print(delim.join(fields), file=file) print(file=file) for sequence in doc: - labels = encoding.encode_sequence(sequence) - # Lengths of labels and orig_fields have previously been checked to match tokens - for (token, orig_fields), label in zip(sequence.tokens_with_fields(), labels): + for (token, orig_fields), label in zip( + sequence.tokens_with_fields(), sequence.labels + ): if orig_fields and not discard_extra_fields: fields = list(orig_fields) fields[line_spec.token_index] = token @@ -497,10 +524,44 @@ def write_doc_using_encoding( # Write output print(delim.join(fields), file=file) - # Print an emtpy line after each sequence + # Print an empty line after each sequence print(file=file) +def write_doc_using_encoding( + doc: Sequence[AnnotatedSequence], + encoding: Encoding, + delim: str, + file: TextIO, + line_spec: LineSpec, + *, + output_docstart: bool, + discard_extra_fields: bool = False, +) -> None: + # Re-encode mentions -> labels for each sequence, then defer to write_doc_raw + # for the actual DOCSTART/token/blank-line writing so the two writers share + # one implementation. + raw_doc = [ + LabeledSequence( + tokens=sequence.tokens, + labels=tuple(encoding.encode_sequence(sequence)), + token_fields=sequence.token_fields, + provenance=sequence.provenance, + comment=sequence.comment, + ) + for sequence in doc + ] + write_doc_raw( + raw_doc, + delim, + file, + line_spec, + output_docstart=output_docstart, + outside_label=encoding.dialect.outside, + discard_extra_fields=discard_extra_fields, + ) + + # TODO: Refactor to remove CoNLL-specific file loading so that this can move to the scoring module def score_conll_files( pred_files: Sequence[PathType], diff --git a/tests/test_conll_format.py b/tests/test_conll_format.py index e1d456e..ebf3bd3 100644 --- a/tests/test_conll_format.py +++ b/tests/test_conll_format.py @@ -9,10 +9,11 @@ LineSpec, _CoNLLToken, ingest_conll_file, + write_docs_raw, write_docs_using_encoding, ) from seqscore.encoding import REPAIR_NONE, get_encoding -from seqscore.model import AnnotatedSequence +from seqscore.model import AnnotatedSequence, LabeledSequence from seqscore.util import file_fields_match from seqscore.validation import InvalidLabelError @@ -158,6 +159,18 @@ def test_bad_label1() -> None: ) +def test_bad_label2() -> None: + ingester = CoNLLIngester(BIO, LINE_SPEC) + path = Path("tests") / "test_files" / "bad_label2.bio" + with path.open(encoding="utf8") as file: + with pytest.raises(InvalidLabelError) as err: + ingester.ingest(file, str(path), repair=REPAIR_NONE) + + assert str(err.value).startswith( + "Could not parse label 'OUT' on line 1 of tests/test_files/bad_label2.bio during validation" + ) + + def test_write_docs_no_orig_fields(tmp_path: Path) -> None: sent1 = AnnotatedSequence( tokens=("This", "is", "a", "sentence", "."), @@ -188,13 +201,119 @@ def test_write_docs_no_orig_fields(tmp_path: Path) -> None: ) -def test_bad_label2() -> None: - ingester = CoNLLIngester(BIO, LINE_SPEC) - path = Path("tests") / "test_files" / "bad_label2.bio" - with path.open(encoding="utf8") as file: - with pytest.raises(InvalidLabelError) as err: - ingester.ingest(file, str(path), repair=REPAIR_NONE) +def test_write_docs_using_encoding_single_doc_no_docstart_by_default( + tmp_path: Path, +) -> None: + sent = AnnotatedSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + mentions=(), + ) + output_file = tmp_path / "out.bio" + write_docs_using_encoding([[sent]], "BIO", "utf-8", " ", LINE_SPEC, output_file) + assert DOCSTART not in output_file.read_text() - assert str(err.value).startswith( - "Could not parse label 'OUT' on line 1 of tests/test_files/bad_label2.bio during validation" + +def test_write_docs_using_encoding_single_doc_always_write_docstart( + tmp_path: Path, +) -> None: + sent = AnnotatedSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + mentions=(), + ) + output_file = tmp_path / "out.bio" + write_docs_using_encoding( + [[sent]], + "BIO", + "utf-8", + " ", + LINE_SPEC, + output_file, + always_write_docstart=True, + ) + text = output_file.read_text() + assert text.count(DOCSTART) == 1 + assert text == "-DOCSTART- O\n\nThis O\nis O\na O\nsentence O\n. O\n\n" + + +def test_write_docs_raw_single_doc_no_docstart_by_default(tmp_path: Path) -> None: + sent = LabeledSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + ) + output_file = tmp_path / "out.bio" + write_docs_raw([[sent]], "utf-8", " ", LINE_SPEC, output_file) + assert DOCSTART not in output_file.read_text() + + +def test_write_docs_raw_single_doc_always_write_docstart(tmp_path: Path) -> None: + sent = LabeledSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + ) + output_file = tmp_path / "out.bio" + write_docs_raw( + [[sent]], "utf-8", " ", LINE_SPEC, output_file, always_write_docstart=True + ) + text = output_file.read_text() + assert text.count(DOCSTART) == 1 + assert text == "-DOCSTART- O\n\nThis O\nis O\na O\nsentence O\n. O\n\n" + + +def test_write_docs_raw_outside_label( + tmp_path: Path, +) -> None: + sent = LabeledSequence(tokens=("A",), labels=("O",)) + output_file = tmp_path / "out.bio" + write_docs_raw( + [[sent]], + "utf-8", + " ", + LINE_SPEC, + output_file, + outside_label="NONE", + always_write_docstart=True, + ) + assert output_file.read_text().startswith("-DOCSTART- NONE\n") + + +def test_write_docs_raw_invalid_labels( + tmp_path: Path, +) -> None: + # Completely invalid labels are written without error + sent = LabeledSequence(tokens=("A", "B", "C"), labels=("B-", "B-AAA", "I-YYY")) + output_file = tmp_path / "out.bio" + write_docs_raw( + [[sent]], + "utf-8", + " ", + LINE_SPEC, + output_file, + ) + assert output_file.read_text().startswith("A B-\nB B-AAA\nC I-YYY") + + +def test_write_docs_using_encoding_multi_doc_always_write_docstart( + tmp_path: Path, +) -> None: + sent1 = AnnotatedSequence( + tokens=("This", "is", "a", "sentence", "."), + labels=("O", "O", "O", "O", "O"), + mentions=(), + ) + sent2 = AnnotatedSequence( + tokens=("Another", "sentence", "."), + labels=("O", "O", "O"), + mentions=(), + ) + docs = [[sent1], [sent2]] + default_file = tmp_path / "default.bio" + forced_file = tmp_path / "forced.bio" + write_docs_using_encoding(docs, "BIO", "utf-8", " ", LINE_SPEC, default_file) + write_docs_using_encoding( + docs, "BIO", "utf-8", " ", LINE_SPEC, forced_file, always_write_docstart=True ) + # Setting always_write_docstart=True has no effect. + # DOCSTART is always written once per document. + assert default_file.read_text() == forced_file.read_text() From da48c46531e9f862ebddad65da0aa54ce2894438 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 11:44:15 -0400 Subject: [PATCH 10/15] Add OpenNER to related papers in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9b1320c..4284f6a 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Other papers related to SeqScore include: * [If You Build Your Own NER Scorer, Non-replicable Results Will Come](https://aclanthology.org/2020.insights-1.15.pdf) * [Toward More Meaningful Resources for Lower-resourced Languages](https://aclanthology.org/2022.findings-acl.44/) * [CoNLL#: Fine-grained Error Analysis and a Corrected Test Set for CoNLL-03 English](https://aclanthology.org/2024.lrec-main.330/) +* [OpenNER 1.0: Standardized Open-Access Named Entity Recognition Datasets in 50+ Languages](https://aclanthology.org/2025.emnlp-main.1708/) # Usage From be5adf6486c8d1e0d9100ba7a2095dda7674f459 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 13:04:21 -0400 Subject: [PATCH 11/15] Rename --parse-comment-lines and drop --use-document-boundaries The parse-comment-lines flag becomes allow-comment-lines. The ignore-document-boundaries remains, but use-document-boundaries is removed to avoid confusion as it is the default. --- seqscore/conll.py | 28 +++++++++++----------- seqscore/scripts/seqscore.py | 46 ++++++++++++++++++++---------------- tests/test_conll_format.py | 6 ++--- tests/test_conll_scoring.py | 2 +- 4 files changed, 44 insertions(+), 38 deletions(-) diff --git a/seqscore/conll.py b/seqscore/conll.py index fdf375c..e36d937 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -87,7 +87,7 @@ def from_line( if line.startswith("#"): raise CoNLLFormatError( f"Line {line_num} of {source_name} does not appear to be delimited " - "and begins with #. Perhaps you want to use the --parse-comment-lines " + "and begins with #. Perhaps you want to use the --allow-comment-lines " f"flag? Line contents: {repr(line)}" ) else: @@ -106,7 +106,7 @@ def from_line( class CoNLLIngester: encoding: Encoding line_spec: LineSpec - parse_comment_lines: bool = field(default=False, kw_only=True) + allow_comment_lines: bool = field(default=False, kw_only=True) ignore_document_boundaries: bool = field(default=False, kw_only=True) def ingest( @@ -121,7 +121,7 @@ def ingest( document: list[AnnotatedSequence] = [] for source_sequence, comment in self._parse_file( - source, source_name, parse_comments=self.parse_comment_lines + source, source_name, parse_comments=self.allow_comment_lines ): if source_sequence[0].is_docstart: # We can ony receive DOCSTART in a sequence by itself, see _parse_file. @@ -157,7 +157,7 @@ def ingest( err.label, str(err) + f" The first token {repr(tokens[0])} of this sentence starts with '#'." - + " If it's a comment, consider enabling --parse-comment-lines.", + + " If it's a comment, consider enabling --allow-comment-lines.", ) from err else: raise err @@ -238,7 +238,7 @@ def validate( for source_sequence, _ in self._parse_file( source, source_name, - parse_comments=self.parse_comment_lines, + parse_comments=self.allow_comment_lines, ): if source_sequence[0].is_docstart: # We can ony receive DOCSTART in a sequence by itself, see _parse_file. @@ -356,7 +356,7 @@ def ingest_conll_file( *, repair: Optional[str] = None, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, quiet: bool = False, ) -> list[list[AnnotatedSequence]]: mention_encoding = get_encoding(mention_encoding_name) @@ -370,7 +370,7 @@ def ingest_conll_file( ingester = CoNLLIngester( mention_encoding, line_spec, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ignore_document_boundaries=ignore_document_boundaries, ) with open(input_path, encoding=file_encoding) as input_file: @@ -385,13 +385,13 @@ def validate_conll_file( line_spec: LineSpec, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, ) -> ValidationResult: encoding = get_encoding(mention_encoding_name) ingester = CoNLLIngester( encoding, line_spec, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ignore_document_boundaries=ignore_document_boundaries, ) with open(input_path, encoding=file_encoding) as input_file: @@ -417,7 +417,7 @@ def repair_conll_file( line_spec: LineSpec, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, quiet: bool, ) -> list[list[AnnotatedSequence]]: return ingest_conll_file( @@ -427,7 +427,7 @@ def repair_conll_file( line_spec, repair=repair, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) @@ -572,7 +572,7 @@ def score_conll_files( line_spec: LineSpec, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, output_format: str, delim: str, error_counts: bool = False, @@ -588,7 +588,7 @@ def score_conll_files( line_spec, repair=repair, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) @@ -613,7 +613,7 @@ def score_conll_files( line_spec, repair=repair, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index 44fa2af..9668a20 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -44,23 +44,29 @@ def cli() -> None: # pragma: no cover def _input_file_options() -> list[Callable]: return [ click.option("--file-encoding", default="UTF-8", show_default=True), - click.option("--parse-comment-lines", is_flag=True), click.option( - "--ignore-document-boundaries/--use-document-boundaries", default=False + "--allow-comment-lines", + is_flag=True, + help="allow comment lines starting with # before sequences", + ), + click.option( + "--ignore-document-boundaries", + is_flag=True, + help="ignore any document boundaries in the input", ), click.option( "--token-index", default=0, show_default=True, type=int, - help="Index of the input field to use for the token", + help="index of the input field to use for the token", ), click.option( "--label-index", default=-1, show_default=True, type=int, - help="Index of the input field to use for the label", + help="index of the input field to use for the label", ), ] @@ -147,7 +153,7 @@ def validate( file_encoding: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, token_index: int, label_index: int, quiet: bool, @@ -161,7 +167,7 @@ def validate( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) if result.errors: print( @@ -200,7 +206,7 @@ def repair( output_delim: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, discard_extra_fields: bool, quiet: bool, ) -> None: @@ -218,7 +224,7 @@ def repair( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, quiet=quiet, ) write_docs_using_encoding( @@ -250,7 +256,7 @@ def convert( output_labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, discard_extra_fields: bool, ) -> None: output_delim = _normalize_tab(output_delim) @@ -262,7 +268,7 @@ def convert( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) write_docs_using_encoding( @@ -310,7 +316,7 @@ def process( type_map: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, discard_extra_fields: bool, ) -> None: output_delim = _normalize_tab(output_delim) @@ -333,7 +339,7 @@ def process( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) try: @@ -372,7 +378,7 @@ def count( labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, output_delim: str, repair_method: str, quiet: bool, @@ -396,7 +402,7 @@ def count( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, repair=repair_method, quiet=quiet, ) @@ -434,7 +440,7 @@ def summarize( labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, repair_method: str, quiet: bool, ) -> None: @@ -452,7 +458,7 @@ def summarize( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, repair=repair_method, quiet=quiet, ) @@ -514,7 +520,7 @@ def score( labels: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, reference: str, score_format: str, delim: str, @@ -548,7 +554,7 @@ def score( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, output_format=score_format, delim=delim, error_counts=error_counts, @@ -570,7 +576,7 @@ def extract_text( output_file: str, *, ignore_document_boundaries: bool, - parse_comment_lines: bool, + allow_comment_lines: bool, ) -> None: line_spec = LineSpec(token_index, label_index) all_docs = [] @@ -581,7 +587,7 @@ def extract_text( file_encoding, line_spec, ignore_document_boundaries=ignore_document_boundaries, - parse_comment_lines=parse_comment_lines, + allow_comment_lines=allow_comment_lines, ) all_docs.extend(docs) diff --git a/tests/test_conll_format.py b/tests/test_conll_format.py index ebf3bd3..477a8ca 100644 --- a/tests/test_conll_format.py +++ b/tests/test_conll_format.py @@ -22,7 +22,7 @@ def test_parse_comments_true() -> None: - ingester = CoNLLIngester(BIO, LINE_SPEC, parse_comment_lines=True) + ingester = CoNLLIngester(BIO, LINE_SPEC, allow_comment_lines=True) comments_path = Path("tests") / "test_files" / "minimal_comments.bio" with comments_path.open(encoding="utf8") as file: documents = ingester.ingest(file, "test", REPAIR_NONE) @@ -52,7 +52,7 @@ def test_parse_comments_false() -> None: ingester.ingest(file, "test", REPAIR_NONE) assert ( str(err1.value) - == "Line 1 of test does not appear to be delimited and begins with #. Perhaps you want to use the --parse-comment-lines flag? Line contents: '#'" + == "Line 1 of test does not appear to be delimited and begins with #. Perhaps you want to use the --allow-comment-lines flag? Line contents: '#'" ) comments_path = Path("tests") / "test_files" / "minimal_comments_2.bio" @@ -139,7 +139,7 @@ def test_repair_bad_name() -> None: LINE_SPEC, repair="conlleval", ignore_document_boundaries=False, - parse_comment_lines=False, + allow_comment_lines=False, ) assert str(err.value).startswith( diff --git a/tests/test_conll_scoring.py b/tests/test_conll_scoring.py index 1aaabf1..2242666 100644 --- a/tests/test_conll_scoring.py +++ b/tests/test_conll_scoring.py @@ -29,7 +29,7 @@ def _score( file_encoding="utf-8", line_spec=LineSpec(0, -1), ignore_document_boundaries=False, - parse_comment_lines=False, + allow_comment_lines=False, delim="\t", output_format=output_format, error_counts=error_counts, From f5c162714c20b818b5cd83ebaa5138d398d37372 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 13:14:50 -0400 Subject: [PATCH 12/15] Switch summarize output to the GitHub table style The summarize table used tabulate's plain style. This change makes it consistent with other seqscore output. --- seqscore/scripts/seqscore.py | 2 +- tests/test_summarize_click.py | 60 +++++++++++++++++------------------ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index 9668a20..09f8e4c 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -484,7 +484,7 @@ def summarize( rows: list[Union[tuple[str, int], str]] = sorted(type_counts.items()) rows.append(SEPARATING_LINE) rows.append(("TOTAL", sum(type_counts.values()))) - print(tabulate(rows, header, intfmt=",")) + print(tabulate(rows, header, tablefmt="github", intfmt=",")) @cli.command(help="score a file and report performance or an error count table") diff --git a/tests/test_summarize_click.py b/tests/test_summarize_click.py index 8a9dec8..bb5ee04 100644 --- a/tests/test_summarize_click.py +++ b/tests/test_summarize_click.py @@ -19,12 +19,12 @@ def test_summarize_bio_onedoc() -> None: assert ( result.output == """File 'tests/conll_annotation/minimal.bio' contains 1 document(s) and 2 sentences -Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 +| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -43,12 +43,12 @@ def test_summarize_bio_onedoc_quiet() -> None: assert result.exit_code == 0 assert ( result.output - == """Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 + == """| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -67,12 +67,12 @@ def test_summarize_iob_twodoc() -> None: assert ( result.output == """File 'tests/conll_annotation/minimal_fields.iob' contains 2 document(s) and 2 sentences -Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 +| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -92,12 +92,12 @@ def test_summarize_iob_twodoc_ignore_doc_boundaries() -> None: assert ( result.output == """File 'tests/conll_annotation/minimal_fields.iob' contains 1 document(s) and 2 sentences -Entity Type Count -------------- ------- -LOC 2 -ORG 1 -------------- ------- -TOTAL 3 +| Entity Type | Count | +|---------------|---------| +| LOC | 2 | +| ORG | 1 | +|---------------|---------| +| TOTAL | 3 | """ ) @@ -119,11 +119,11 @@ def test_summarize_bio_twofiles() -> None: == """File 'tests/conll_annotation/minimal.bio' contains 1 document(s) and 2 sentences File 'tests/conll_annotation/minimal2.bio' contains 1 document(s) and 2 sentences Total 2 document(s) and 4 sentences -Entity Type Count -------------- ------- -LOC 5 -ORG 2 -------------- ------- -TOTAL 7 +| Entity Type | Count | +|---------------|---------| +| LOC | 5 | +| ORG | 2 | +|---------------|---------| +| TOTAL | 7 | """ ) From b6a99532f434edecde252bca64661de56420d1dc Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 13:27:15 -0400 Subject: [PATCH 13/15] Make error_counts ordering deterministic --- seqscore/conll.py | 8 +++++++- tests/test_scoring_click.py | 27 +++++++++++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/seqscore/conll.py b/seqscore/conll.py index e36d937..ab2ae7e 100644 --- a/seqscore/conll.py +++ b/seqscore/conll.py @@ -647,13 +647,19 @@ def score_conll_files( (error_type, item.type, " ".join(item.tokens)) ] = count + # Sort by count descending (the negative reverses the default + # ascending sort), breaking ties on the token string + # (item[0] is the (error_type, mention_type, token_str) key, + # so item[0][2] is the token string; item[1] is the count). rows = [ [count, error_type, mention_type, token_str] for ( error_type, mention_type, token_str, - ), count in combined_counts.most_common() + ), count in sorted( + combined_counts.items(), key=lambda item: (-item[1], item[0][2]) + ) ] if output_format == FORMAT_PRETTY: diff --git a/tests/test_scoring_click.py b/tests/test_scoring_click.py index 9a97ab4..0f3ff45 100644 --- a/tests/test_scoring_click.py +++ b/tests/test_scoring_click.py @@ -301,10 +301,17 @@ def test_score_error_counts_single_file() -> None: ], ) assert result.exit_code == 0 - assert "| Count | Error | Type | Tokens |" in result.output - assert "| 1 | FP | LOC | West |" in result.output - assert "| 1 | FP | LOC | Philadelphia |" in result.output - assert "| 1 | FN | LOC | West Philadelphia |" in result.output + # Ordering within the same count is deterministic and determined by + # the token string + assert ( + result.output + == """| Count | Error | Type | Tokens | +|---------|---------|--------|-------------------| +| 1 | FP | LOC | Philadelphia | +| 1 | FP | LOC | West | +| 1 | FN | LOC | West Philadelphia | +""" + ) def test_score_error_counts_delim_format() -> None: @@ -323,10 +330,14 @@ def test_score_error_counts_delim_format() -> None: ], ) assert result.exit_code == 0 - assert "Count\tError\tType\tTokens" in result.output - assert "1\tFP\tLOC\tWest" in result.output - assert "1\tFP\tLOC\tPhiladelphia" in result.output - assert "1\tFN\tLOC\tWest Philadelphia" in result.output + assert ( + result.output + == """Count\tError\tType\tTokens +1\tFP\tLOC\tPhiladelphia +1\tFP\tLOC\tWest +1\tFN\tLOC\tWest Philadelphia +""" + ) def test_score_error_counts_multiple_files() -> None: From 846fc74268ac2127164ff5dd588f845964899f13 Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 13:27:56 -0400 Subject: [PATCH 14/15] Update README and discard-extra-fields help string --- README.md | 32 ++++++++++++++++++++++---------- seqscore/scripts/seqscore.py | 6 +++++- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 4284f6a..4d447b4 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ For a list of commands, run `seqscore --help`: $ seqscore --help Usage: seqscore [OPTIONS] COMMAND [ARGS]... - Provides scoring and analysis tools for NER/chunking files (version 0.6.0) + Provides scoring and analysis tools for NER/chunking files (version 0.9.0) Options: --version Show the version and exit. @@ -210,7 +210,7 @@ scoring will fail: ``` seqscore.encoding.EncodingError: Stopping due to validation errors in invalid.bio: -Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 +Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 of invalid.bio ``` To score output with invalid transitions, we need to specify a repair method which can @@ -219,8 +219,8 @@ we refer to as "begin" repair in our paper): `seqscore score --labels BIO --repair-method conlleval --reference samples/reference.bio samples/invalid.bio`: ``` -Validation errors in sequence at line 7 of invalid.bio: -Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 +Validation errors in sequence beginning at line 7 of invalid.bio: +Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 of invalid.bio Used method conlleval to repair: Old: ('I-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'B-LOC', 'O') New: ('B-ORG', 'I-ORG', 'I-ORG', 'O', 'O', 'B-LOC', 'I-LOC', 'O', 'B-LOC', 'O') @@ -272,14 +272,14 @@ To check if a file has any invalid transitions, we can run `seqscore validate --labels BIO samples/reference.bio`: ``` -No errors found in 0 tokens, 2 sequences, and 1 documents in reference.bio +No errors found in 15 tokens, 2 sequences, and 1 document(s) in reference.bio ``` For the example of the [samples/invalid.bio](samples/invalid.bio), we can run `seqscore validate --labels BIO samples/invalid.bio`: ``` -Encountered 1 errors in 1 tokens, 2 sequences, and 1 documents in invalid.bio +Encountered 1 errors in 15 tokens, 2 sequences, and 1 document(s) in invalid.bio Invalid transition 'O' -> 'I-ORG' for token 'University' on line 7 ``` @@ -314,11 +314,21 @@ We can get a list of available chunk encodings by running `seqscore convert --he ``` Usage: seqscore convert [OPTIONS] FILE OUTPUT_FILE + convert between mention encodings + Options: --file-encoding TEXT [default: UTF-8] - --ignore-comment-lines - --ignore-document-boundaries / --use-document-boundaries - --output-delim TEXT [default: space] + --allow-comment-lines allow comment lines starting with # before + sequences + --ignore-document-boundaries ignore any document boundaries in the input + --token-index INTEGER index of the input field to use for the + token [default: 0] + --label-index INTEGER index of the input field to use for the + label [default: -1] + --output-delim TEXT the delimiter to be used for output (has no + effect on input) [default: tab] + --discard-extra-fields discard any fields other than the token and + label when writing output files --input-labels [BIO|BIOES|BILOU|BMES|BMEOW|IO|IOB] [required] --output-labels [BIO|BIOES|BILOU|BMES|BMEOW|IO|IOB] @@ -388,11 +398,13 @@ For example, if we run `seqscore summarize --labels BIO samples/reference.bio` w the following output: ``` -File 'samples/reference.bio' contains 1 document(s) with the following mentions: +File 'samples/reference.bio' contains 1 document(s) and 2 sentences | Entity Type | Count | |---------------|---------| | LOC | 2 | | ORG | 1 | +|---------------|---------| +| TOTAL | 3 | ``` If the quiet (`-q`) flag is provided, the first line giving the filename and document diff --git a/seqscore/scripts/seqscore.py b/seqscore/scripts/seqscore.py index 09f8e4c..00f52d4 100644 --- a/seqscore/scripts/seqscore.py +++ b/seqscore/scripts/seqscore.py @@ -131,7 +131,11 @@ def _output_delim_option() -> Callable: def _discard_extra_fields_option() -> Callable: - return click.option("--discard-extra-fields", is_flag=True) + return click.option( + "--discard-extra-fields", + is_flag=True, + help="discard any fields other than the token and label when writing output files", + ) def _quiet_option() -> Callable: From 7e545de75633640f92f3b0381b8fc282fe7bf81b Mon Sep 17 00:00:00 2001 From: Constantine Lignos Date: Mon, 13 Jul 2026 14:18:43 -0400 Subject: [PATCH 15/15] Update uv development instructions to use uv sync --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d447b4..ddf8bde 100644 --- a/README.md +++ b/README.md @@ -587,10 +587,13 @@ For development, check out the `dev` branch (latest, but less tested than `main` ## Setting up an environment for development +Python 3.10 (the minimum version SeqScore supports) is used for development to avoid +accidentally using newer Python features. + ### Using uv -1. Create an environment: `uv venv --python 3.10 .venv` -2. Install seqscore and development dependencies: `uv pip install -e ".[dev]"` +Create an environment and install seqscore with development dependencies: +`uv sync --python 3.10 --extra dev` ### Using conda