reduce-eud-literals

pybart/constants/constants.py

@@ -0,0 +1,10 @@
+import json
+
+with open("./eud_literal_allowed_list_en.json") as f:
+    en = json.load(f)
+
+with open("./eud_literal_allowed_list_he.json") as f:
+    he = json.load(f)
+
+# not sure we need to put these literals as well but to be on the safe sie since they are after `:` we included them
+EUD_LITERAL_ALLOWED_LIST = ["cite", "preconj", "qmod", "prt", "predet", "nor", "negcc", "relcl"] + en + he

pybart/constants/eud_literal_allowed_list_en.json

@@ -0,0 +1 @@
+["and", "of", "in", "with", "to", "for", "relcl", "poss", "on", "from", "as", "or", "at", "by", "between", "agent", "such_as", "after", "including", "during", "into", "than", "among", "through", "but", "due_to", "within", "while", "under", "based_on", "over", "if", "against", "according_to", "although", "without", "about", "tmod", "per", "before", "because", "via", "compared_to", "across", "where", "since", "following", "whereas", "like", "compared_with", "such", "negcc", "regarding", "despite", "because_of", "versus", "upon", "based", "until", "given", "around", "along", "in_order", "towards", "vs", "throughout", "toward", "depending", "in_order_to", "above", "compared", "below", "with_respect_to", "though", "plus", "onto", "along_with", "except", "not", "prior_to", "due", "beyond", "nor", "that", "up", "so_that", "near", "in_accordance_with", "when", "in_addition_to", "except_for", "inside", "out_of", "regardless_of", "according", "unlike", "together_with", "close_to", "owing_to", "outside", "with_regard_to", "besides", "post", "once", "concerning", "instead_of", "relative", "or", "amongst", "whether", "behind", "relative_to", "considering", "unless", "whilst", "away_from", "combined", "out", "and", "et", "excluding", "as_to", "alongside", "outside_of", "in_case", "rather_than", "yet", "in_case_of", "independently_of", "as_per", "by_means_of", "so", "as_for", "und", "downstream_of", "similar_to", "albeit", "till", "off", "apart_from", "why", "together", "consistent_with", "starting", "prior", "upstream_of", "rather", "minus", "so_as", "on_top_of", "as_of", "up_to", "in_spite_of", "next_to", "down_to", "provided", "in_front_of", "contrary_to", "far_from", "beneath", "also", "beside", "less_than", "similarly_to", "followed_by", "how", "down", "beginning", "first_of", "regardless", "closer_to", "and", "past", "comprising", "viz", "assuming", "independent_of", "owing", "pre", "underneath", "notwithstanding", "close", "here", "respectively", "other", "greater_than", "on_behalf_of", "significantly", "previously", "comparing", "building", "away", "ahead_of", "on_account_of", "either", "once_per", "different_from", "parallel_to", "instead", "perpendicular_to", "en", "near_to", "aside_from", "amid", "in_place_of", "opposite", "such_that", "etc", "form", "first", "as_if", "both", "then", "differently_from", "plus", "is", "independently", "amidst", "independently_from", "still", "often", "as_as", "already", "pending", "downstream_from", "only", "as_in", "contrary", "downstream", "worth", "ou", "north_of", "despite_of", "currently", "less", "but", "als", "south_of", "neither", "greater", "involving", "well", "in_lieu_of", "followed", "now", "divided", "highly", "directly", "far", "apart", "vis", "very", "basing", "before_after", "however", "usually", "more_to", "excepting", "upstream_from", "recently", "aside", "times", "coupled", "dans", "upstream", "twice_per", "relatively", "upto", "posterior_to", "be", "using", "next", "atop", "con", "therefore", "more_on", "thus", "strongly", "with_respect", "thanks_to", "for", "more_from", "west_of", "are", "inside_of", "fully", "as_from", "more_than", "further_from", "was", "concurrently_with", "in_between", "re", "typically", "much", "the", "further_to", "completely", "just", "particularly", "similarly", "generally", "further", "trough", "in_to", "mainly", "likely", "better_with", "para", "secondary_to", "aboard", "proportionally_to", "potentially", "most_of", "always", "positively_with", "ago", "separately_for", "fold", "anterior_to", "free_of", "twice", "closely", "alone", "up_from", "short_of", "initially", "actually", "once_for", "clearly", "all", "frequently", "bei", "better_to", "ever", "down_from", "largely", "de", "off_of", "according_with", "more_with", "side_by", "specifically", "counter_to", "opposite_to", "east_of", "nearby", "onboard", "separately_from", "such_us", "farther_from", "more", "commonly", "normally", "nur", "ont", "too", "negatively_with", "ein", "were", "vor", "later", "whereby", "easily", "an", "successfully", "back", "corresponding", "otherwise", "directly_from", "por", "downstream_to", "nor", "more_for", "closer", "simultaneously_with", "even_with", "statistically", "more_by", "widely", "distal_to", "f\u00fcr", "primarily", "round", "pursuant_to", "rapidly", "subsequent", "proximal_to", "accordingly_to", "back_from", "back_on", "analogously_to", "of_mm", "ca", "subsequently", "higher_on", "verses", "irrespectively_of", "positively", "nach", "subsequent_to", "aber", "negatively", "better_on", "probably", "enough", "simultaneously", "enough_for", "through_to", "less_to", "partially", "by_means", "relatively_to", "really", "perpendicularly_to", "else", "even", "ahead", "slightly", "base", "not", "resp", "later_on", "voor", "mostly", "am", "alternatively", "separately", "poorly", "readily", "then_with", "providing", "clinically", "facing", "forward_to", "or", "focusing", "at_for", "once_with", "evidenced", "randomly", "twice_for", "comparably_to", "positive_for", "closest_to", "ofthe", "approximately", "better_for", "quite", "enough_to", "twice_with", "effectively", "again", "respectively_for", "circumventing", "earlier_by", "consistently", "independent", "greatly", "nearest", "almost", "early", "originally", "let", "of_mg", "save", "extremely", "substantially", "predominantly", "on_to", "auf", "differently", "immediately", "differently_to", "up_of", "drawing", "fold_for", "years_of", "nearer", "correctly", "upstream_to", "then_for", "especially", "deep_to", "of_health", "fits", "anywhere_on", "directly_by", "respectively_of", "gradually", "experimentally", "markedly", "on_from", "elsewhere", "suggestive_of", "fo", "oft", "exclusively", "ultimately", "sufficiently", "northwest_of", "circa", "akin_to", "daily", "thru", "unto", "indeed", "directly_to", "elsewhere_on", "possibly", "as_regards", "sometimes", "moderately_to", "at_to", "identically_to", "supported", "ist", "barring", "hence", "analogous_to", "lateral_to", "forward", "extensively", "differentially", "vom", "directly_on", "considerably", "south_to", "equally", "earlier_on", "entirely", "andthe", "lower_on", "consistent", "cause", "continuously", "lest", "na", "avec", "first_with", "there", "herein", "consistently_with", "entre", "actively", "early_on", "coincident_with", "inthe", "examining", "dramatically", "cf", "proximally_to", "distally_to", "faster_to", "laterally_to", "being", "with_mm", "because_that", "less_from", "rarely", "eventually", "additionally", "accurately", "adjacent_to", "online", "regularly", "immediately_to", "south", "secondary", "regarding_to", "compare", "irrespective_of", "measured", "routinely", "long", "with_egfr", "inversely", "less_on", "individually", "expressed", "severely", "earlier_for", "northeast_of", "like_as", "concurrent_with", "accordingly", "parallel", "selectively", "southwest_of", "irrespective", "preceding", "furthermore", "physically", "necessarily", "fue", "nearer_to", "but_with", "lower", "oder", "down_on", "accompanying", "tightly", "anteriorly_to", "medial_to", "for_health", "associated", "increasingly", "then_to", "separate_from", "assessed", "functionally", "less_for", "confirmed", "quickly", "in_addition", "better", "perpendicular", "fold_of", "simply", "north", "combining", "parallel_with", "indirectly", "nearly", "strictly", "linearly", "determined", "respectively_from", "faster_with", "contrarily_to", "right_from", "precluding", "finally", "worldwide", "once_on", "sich", "namely", "judging", "paralleling", "downwind_of", "essentially", "first_by", "second", "ser", "wie", "similar", "back_for", "secondly", "auch", "sobre", "earlier", "additional_to", "adequately", "outward_from", "explicitly", "naturally", "preferentially", "up_for", "ain", "een", "thorough", "ot", "sur", "efficiently", "as_bmi", "al", "between_control", "north_to", "contrasting", "interestingly", "most", "southeast_of", "apparently", "precisely", "partly", "foram", "posteriorly_to", "1and", "inversely_with", "aan", "solely", "down_for", "od", "with_type", "moreover", "spontaneously", "non", "systematically", "whit", "last", "whenever", "medially_to", "anywhere_from", "at_with", "controlling", "directly_with", "between_baseline", "with_bmi", "publicly", "comparatively", "ad", "globally", "of_type", "heavily", "presumably", "exactly", "adding", "sie", "with_post", "faster_on", "earlier_with", "personally", "genetically", "later_for", "such_like", "longer", "first_for", "board", "thereby", "om", "other_than", "locally", "whiles", "in_with", "respectively_with", "met", "anti", "outwith", "do", "og", "right_to", "worse_on", "automatically", "subsequently_to", "twice_of", "furthest_from", "opposed", "critically", "traditionally", "first_to", "right_of", "high_on", "inter", "collectively", "faster_for", "overall", "iff", "less_with", "up_on", "daily_for", "often_with", "reliably", "consequently", "importantly", "other_for", "secondarily_to", "of_protein", "upwards_of", "fold_to", "only_with", "continuing", "repeatedly", "down_by", "than_without", "aus", "km_from", "rostral_to", "progressively", "greater_to", "higher", "of_acid", "adjusting", "concomitant_with", "totally", "akin", "had", "concurrently", "similarly_for", "notably", "slowly", "inspite_of", "uniquely", "caudal_to", "constantly", "equally_to", "to_target", "of_factor", "with_stage", "freely", "earlier_to", "ex", "down_of", "defined", "due_of", "in_accordance", "short", "somewhat", "aka", "to_from", "inspite", "farthest_from", "of_cm", "reminiscent_of", "then_from", "separately_by", "early_from", "cross", "simultaneously_to", "presently", "been", "ofa", "in_conjunction", "manually", "second_to", "yet_with", "clockwise_from", "late_to", "mutually", "properly", "outwards_from", "conditional", "as", "west", "relative_for", "weekly_for", "cos", "as_egfr", "obviously", "there_for", "se", "offshore_of", "and10", "different_to", "respectively_by", "afor", "early_to", "rather", "truly", "alongwith", "later_with", "most_from", "only_for", "complementary_to", "disregarding", "broadly", "spatially", "longer_on", "side", "then_by", "perhaps", "cat", "que", "inversely_to", "twice_on", "later_from", "contralateral_to", "remarkably", "first_on", "cfor", "home_to", "surgically", "such_the", "structurally", "carefully", "anda", "within_of", "theoretically", "wit", "previously_to", "uniformly", "earlier_from", "minimum_of", "briefly", "remotely_from", "similar_with", "au", "moderately", "back_of", "and1", "of_control", "better_by", "absent", "higher_for", "distant_from", "once_from", "como", "af", "reportedly", "faster_from", "nearest_to", "as_above", "over_to", "annually", "va", "wird", "referring", "simultaneous_with", "of_genus", "owning", "of_pi3k", "reported", "untill", "reasonably", "hourly_for", "wtih", "roughly", "soon", "nevertheless", "dynamically", "incl", "also_for", "virtually", "much_to", "concluding", "that_of", "kind_of", "in_concert", "seriously", "immer", "least_to", "more_of", "free", "up_by", "comparatively_to", "eine", "initially_with", "superiorly_to", "added", "with_change", "em", "commercially", "which", "corroborating", "having", "downward_to", "expect", "safely", "immediately_on", "most_with", "constitutively", "sufficiently_for", "ap\u00f3s", "single", "versa", "orthogonally_to", "visually", "of_erk1", "concordant_with", "proportionally", "chez", "up_with", "ao", "historically", "free_from", "for_medicare", "hardly", "dorsal_to", "adversely", "coinciding", "owning_to", "superior_to", "thanks", "drastically", "perfectly", "excepted", "inferior_to", "with_mg", "exponentially", "jointly", "where_for", "originally_from", "additionally_to", "only_from", "counter", "inferiorly_to", "from_control", "fro", "late_for", "after_that", "caudally_to", "thoroughly", "sequentially", "somewhere_on", "late", "newly", "of_grade", "wih", "straight_to", "concomitantly_with", "intra", "less_by", "later_to", "by_et", "equivalently_to", "independently_to", "west_to", "in_place", "in_common", "quantitatively", "definitely"]

pybart/constants/eud_literal_allowed_list_he.json

@@ -0,0 +1 @@
+["and", "\u05d1", "poss", "gen", "\u05dc", "relcl", "acc", "\u05e2\u05dc", "\u05d5", "\u05de", "\u05d0\u05dc", "\u05db", "\u05db\u05d9", "\u05de\u05df", "\u05e2\u05dd", "\u05e2\u05d3", "\u05d0\u05dd", "\u05d1\u05d9\u05df", "\u05d0\u05d7\u05e8\u05d9", "\u05dc\u05e4\u05e0\u05d9", "\u05e9", ";", "\u05db\u05de\u05d5", "\u05db\u05e9", "\u05d1\u05dc\u05d9", "\u05de\u05ea\u05d5\u05da", "\u05dc\u05e4\u05d9", "\u05db\u05d0\u05e9\u05e8", "\u05d0\u05e6\u05dc", "\u05de\u05e4\u05e0\u05d9", "\u05d0\u05da", "\u05d0\u05d1\u05dc", "\u05d1\u05ea\u05d5\u05da", "\u05d0\u05e9\u05e8", "\u05e2\u05dc_\u05d9\u05d3\u05d9", "\u05dc\u05de\u05e2\u05df", "\u05ea\u05d7\u05ea", "\u05db\u05d3\u05d9", "\u05de\u05e2\u05dc", "\u05d0\u05d7\u05e8", "\u05d1\u05e9\u05d1\u05d9\u05dc", "\u05d1\u05e2\u05d3", "\u05dc\u05d0", "\u05e2\u05dc_\u05e4\u05d9", "\u05e0\u05d2\u05d3", "\u05de\u05d0\u05ea", "q", "\u05de\u05d1\u05dc\u05d9", "\u05dc\u05d0\u05d7\u05e8", "\u05d1\u05de\u05e9\u05da", "\u05d0\u05d5\u05dc\u05dd", "\u05d1\u05d9\u05d3\u05d9", "\u05de\u05dc\u05d1\u05d3", "\u05d1\u05ea\u05d5\u05e8", "\u05d1\u05e4\u05e0\u05d9", "\u05d0\u05dc\u05d0", "\u05d0\u05d5", "\u05db\u05e4\u05d9", "\u05dc\u05ea\u05d5\u05da", "\u05de\u05e9\u05d5\u05dd", "\u05d1\u05e2\u05d5\u05d3", "\u05db\u05db\u05dc", "\u05dc\u05e9\u05dd", "\u05d1\u05e7\u05e8\u05d1", "\u05d9\u05d7\u05d3", "\u05d4", "\u05db\u05e0\u05d2\u05d3", "\u05db\u05dc\u05e4\u05d9", "\u05de_\u05e4\u05d9", "\u05d1\u05de\u05e7\u05d5\u05dd", "\u05dc\u05d9\u05d3\u05d9", "\u05d1\u05dc\u05d0", "\u05db\u05d0\u05d9\u05dc\u05d5", "\u05d3\u05e8\u05da", "\u05de\u05e6\u05d3", "\u05d1\u05d8\u05e8\u05dd", "\u05dc\u05de\u05e8\u05d5\u05ea", "\u05dc\u05dc\u05d0", "\u05de\u05ea\u05d7\u05ea", "\u05db\u05d2\u05d5\u05df", "\u05de\u05d0\u05e9\u05e8", "\u05e9\u05dd", "\u05e1\u05d1\u05d9\u05d1", "\u05d0\u05ea\u05d5", "\u05dc\u05e7\u05e8\u05d0\u05ea", "\u05d1\u05d2\u05dc\u05dc", "\u05d0\u05d9\u05dc\u05d5", "\u05dc\u05d5", "\u05dc_\u05d8\u05d5\u05d1\u05ea", "\u05d0\u05d9\u05df", "\u05dc\u05d9\u05d3", "\u05d1\u05d0\u05e9\u05e8", "\u05e2\"\u05d9", "\u05dc\u05d2\u05d1\u05d9", "\u05e2\u05dc_\u05e4\u05e0\u05d9", "\u05dc\u05e2\u05d5\u05de\u05ea", "\u05de_\u05d7\u05d5\u05e5", "\u05d7\u05d5\u05e5", "\u05de\u05e2\u05d1\u05e8", "\u05e4\u05df", "\u05db\u05e9\u05dd", "\u05e2\u201d\u05d9", "\u05de\u05e1\u05d1\u05d9\u05d1", "\u05e2\u201c\u05d9", "\u05d0\u05e3", "\u05de\u05d1\u05d9\u05df", "\u05db\u05e2\u05d1\u05d5\u05e8", "\u05de\u05d7\u05de\u05ea", "\u05de\u05d5\u05dc", "\u05d1\u05db\u05d3\u05d9", "\u05ea\u05d5\u05da", "\u05de\u05d0\u05d7\u05d5\u05e8\u05d9", "\u05e2\u05dc_\u05d2\u05d1\u05d9", "\u05d1\u05e2\u05d1\u05d5\u05e8", "\u05db\u05d1\u05e8", "\u05dc\u05d1\u05d9\u05df", "\u05dc\u05d5\u05dc\u05d0", "\u05d1_\u05d9\u05d7\u05e1", "\u05d0\u05e3_\u05e4\u05d9", "\u05de\u05db\u05dc", "\u05db\u05d9\u05d5\u05df", "\u05d0\u05d6", "\u05dc\u05db\u05d1\u05d5\u05d3", "\u05d0\u05ea\u05d9", "\u05e2\u05d5\u05d3", "\u05d1\u05e9\u05dc", "\u05d1\u05e2\u05ea", "\u05dc\u05d0\u05d5\u05e8", "\u05d0\u05ea\u05e0\u05d5", "\u05dc\u05e2\u05d1\u05e8", "\u05dc\u05de\u05df", "\u05dc\u05e8\u05d2\u05dc", "\u05e2\u05ea\u05d4", "\u05de\u05db\u05d9\u05d5\u05df", "\u05d9\u05d5\u05ea\u05e8", "\u05de\u05d1\u05e2\u05d3", "\u05d1\u05d9\u05d7\u05d3", "\u05d1\u05e6\u05dc", "\u05ea\u05de\u05d9\u05d3", "\u05e2\u05d1\u05d5\u05e8", "\u05de\u05d0\u05d7\u05e8\u05d9", "\u05dc\u05d0\u05d5\u05e8\u05da", "\u05e7\u05d5\u05d3\u05dd", "\u05d4\u05d9\u05d5\u05ea", "\u05d8\u05e8\u05dd", "\u05d4\u05d5\u05d0\u05d9\u05dc", "\u05d0\u05d5\u05d3\u05d5\u05ea", "\u05d0\u05dc\u05de\u05dc\u05d0", "\u05d1\u05dc\u05e2\u05d3\u05d9", "\u05d6\u05d5\u05dc\u05ea", "\u05de\u05d0\u05d7\u05e8", "\u05d0\u05ea\u05d4", "\u05e8\u05e7", "\u05d4\u05d5\u05d3\u05d5\u05ea", "\u05e0\u05d5\u05e1\u05e3", "\u05e9\u05db\u05df", "\u05e9\u05de\u05d0", "\u05d4\u05e8\u05d9", "\u05d1\u05e2\u05e7\u05d1\u05d5\u05ea", "\u05dc\u05e4\u05e0\u05d5\u05ea", "\u05de\u05d0\u05d3", "\u05de\u05e4\u05d0\u05ea", "\u05d0\u05ea\u05dd", "\u05e2\u05dc_\u05d9\u05d3", "\u05d2\u05dd", "\u05de\u05d4", "\u05d1\u05d6\u05db\u05d5\u05ea", "\u05db\u05d0\u05df", "\u05e2\u05e7\u05d1", "\u05dc\u05d1\u05d3", "\u05d1\u05d0\u05de\u05e6\u05e2\u05d5\u05ea", "\u05d1\u05e0\u05d9\u05d2\u05d5\u05d3", "\u05de\u05e2\u05ea", "\u05d0\u05d2\u05d1", "\u05de\u05d0\u05dc", "\u05d1\u05d0\u05de\u05ea", "\u05e2\u05e4\"\u05d9", "\u05de\u05e9", "\u05de_\u05d9\u05d3\u05d9", "\u05d4\u05d9\u05d5\u05dd", "\u05e2\u05e4\u201c\u05d9", "\u05e4\u05d5\u05df", "\u05e2\u201d\u05d3", "\u05e2\"\u05d3", "\u05d1\u05de\u05d5", "\u05e4\u05d4", "\u05e2\u05d3\u05d9\u05d9\u05df", "\u05de\u05e4\u05d9", "\u05d1_\u05e4\u05d9", "\u05de\u05d4\u05dc\u05da", "\u05de\u05d9\u05df", "\u05d0\u05ea", "\u05de\u05e8\u05d5\u05d1", "\u05d4\u05d7\u05dc", "\u05e2\u201c\u05d3", "\u05de\u05e9\u05dc", "\u05d1\u05dc\u05d1", "\u05e2\u05e4\u201d\u05d9", "\u05db\u05dc\u05dc", "\u05d0\u05d5\u05dc\u05d9", "\u05d1\u05e2\u05d9\u05e0\u05d9", "\u05d1_\u05d4\u05ea\u05d0\u05dd", "\u05ea\u05de\u05d5\u05e8\u05ea", "\u05e2\u201c\u05e4", "\u05e0\u05d5\u05db\u05d7", "\u05d0\u05de\u05e0\u05dd", "\u05d4\u05e0\u05d4", "\u05dc\u05e4\u05e2\u05de\u05d9\u05dd", "\u05d1\u05d9\u05d5\u05ea\u05e8", "\u05dc\u05e8\u05d1\u05d5\u05ea", "\u05e2\"\u05e4", "\u05dc\u05e6\u05d3", "\u05d1\u05d0\u05d9\u05df", "\u05de\u05db\u05d9\u05d5\u05d5\u05df", "\u05e2\u201d\u05e4", "\u05d1\u05d4\u05d9\u05d5\u05ea", "\u05de\u05d7\u05d5\u05e6\u05d4", "\u05e2\u05db\u05e9\u05d9\u05d5", "in", "\u05db\u05df", "\u05db\u05d9\u05d5\u05d5\u05df", "\u05d1\u05d2\u05d3\u05e8", "\u05dc_\u05de\u05e9\u05da", "\u05de\u05d9\u05d5\u05dd", "\u05de\u05d9\u05d8", "\u05dc\u05e0\u05d5\u05db\u05d7", "\u05d1\u05d0\u05d7\u05d3", "\u05d1\u05dc\u05ea\u05d9", "\u05d4\u05e8\u05d1\u05d4", "\u05dc\u05de\u05e2\u05dc\u05d4", "\u05e6\u05d5", "\u05db\u05de\u05e2\u05d8", "\u05dc\u05d0\u05dc", "\u05de\u05e2\u05d9\u05df", "\u05e4\u05e2\u05dd", "de", "\u05d0\u05d5\u05d9\u05e3", "\u05de\u05e2\u05d5\u05dc\u05dd", "\u05d1\u05d9\u05d7\u05d5\u05d3", "\u05d0\u05dc_\u05e2\u05d1\u05e8", "\u05d1\u05db\u05dc\u05dc", "\u05d1_\u05e7\u05e9\u05e8", "\u05de\u05d9\u05d3", "\u05e4\u05d9", "\u05dc\u05d7\u05d3\u05e8", "\u05db\u05d5\u05dc\u05dc", "\u05d1_\u05de\u05d4\u05dc\u05da", "\u05db\u05d9\u05d5\u05dd", "\u05dc\u05e2\u05d5\u05dc\u05dd", "\u05de\u05e2\u05d8"]

pybart/graph_token.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass, field
+from constants.constants import EUD_LITERAL_ALLOWED_LIST
 
 
 @dataclass
@@ -30,7 +31,9 @@ def to_str(self, remove_enhanced_extra_info, remove_bart_extra_info):
         eud = ""
         if self.eud is not None:
             if not remove_enhanced_extra_info:
-                eud = ":" + self.eud
+                if self.eud.lower() not in EUD_LITERAL_ALLOWED_LIST:
+                    self.eud = "_other"
+                eud = ":" + self.eud.lower()
 
         bart = ""
         if self.src is not None: