Merge pull request #212 from allenai/gopher-fix

Fix gopher tagger bug
allenai · Jun 12, 2023 · ccb3869 · ccb3869
2 parents a80cdc1 + 955fb39
commit ccb3869
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pretrain_data/filters/src/ai2_llm_filters/taggers/gopher.py b/pretrain_data/filters/src/ai2_llm_filters/taggers/gopher.py
@@ -133,7 +133,7 @@ def get_attributes(text: str) -> GopherAttributes:
             if line.endswith("\u2026"):
                 attrs.fraction_of_lines_ending_with_ellipsis += 1
         attrs.fraction_of_lines_starting_with_bullet_point /= line_count
-        attrs.fraction_of_words_with_alpha_character /= line_count
+        attrs.fraction_of_lines_ending_with_ellipsis /= line_count
 
         line_counts = Counter(lines)
         attrs.fraction_of_duplicate_lines = (