Skip to content

Fix bug where Keyword Tagging removes newlines. #365

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 8, 2024
Merged

Fix bug where Keyword Tagging removes newlines. #365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
11aca00
Fix bug where Keyword Tagging was removing newlines.
jrobble May 2, 2024
b7e8342
Merge branch 'develop' into bugfix/keyword-tagging-offsets
jrobble May 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 3 additions & 16 deletions cpp/KeywordTagging/KeywordTagging.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,18 +344,6 @@ void KeywordTagging::load_tags_json(const MPFJob &job, map<wstring, vector<pair<
LOG4CXX_DEBUG(hw_logger_, "Read JSON")
}

wstring clean_whitespace(const wstring &input) {
boost::wregex re(L"\n(\n|[[:space:]])+");
boost::wregex re2(L"\\\\n(\\\\n|[[:space:]])+");

wstring result = boost::regex_replace(input, re, L"\n");
wstring result2 = boost::regex_replace(result, re2, L"\\\\n");

result2 = boost::trim_copy(result2);

return result2;
}

bool is_only_ascii_whitespace(const wstring &str) {
auto it = str.begin();
do {
Expand Down Expand Up @@ -529,14 +517,13 @@ void KeywordTagging::process_text_tagging(Properties &detection_properties, cons
for (auto const& it : prop_texts) {
prop = it.first;
prop_text = it.second;
wstring text = clean_whitespace(prop_text);

LOG4CXX_DEBUG(hw_logger_, "Processing tags on " +
boost::locale::conv::utf_to_utf<char>(prop))
LOG4CXX_DEBUG(hw_logger_, "Text is: " +
boost::locale::conv::utf_to_utf<char>(text))
boost::locale::conv::utf_to_utf<char>(prop_text))

if (is_only_ascii_whitespace(text)) {
if (is_only_ascii_whitespace(prop_text)) {
LOG4CXX_WARN(hw_logger_, "No text to process for " +
boost::locale::conv::utf_to_utf<char>(prop))
continue;
Expand All @@ -546,7 +533,7 @@ void KeywordTagging::process_text_tagging(Properties &detection_properties, cons
bool full_regex = DetectionComponentUtils::GetProperty(job.job_properties, "FULL_REGEX_SEARCH", true);

map<wstring, map<wstring, vector<string>>> trigger_tags_words_offset;
set<wstring> found_tags_regex = search_regex(job, text, json_kvs_regex, trigger_tags_words_offset, full_regex);
set<wstring> found_tags_regex = search_regex(job, prop_text, json_kvs_regex, trigger_tags_words_offset, full_regex);
all_found_tags.insert(found_tags_regex.begin(), found_tags_regex.end());

wstring tag_string = boost::algorithm::join(found_tags_regex, L"; ");
Expand Down
9 changes: 9 additions & 0 deletions cpp/KeywordTagging/test/data/test-newlines.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
Contact Us | Other Text | This is an address street, MA, 55555
someemail@place.org
777-777-7777

Use this special code when you contact us:
456-4564-456-46


This sentence finally ends in a period.
64 changes: 42 additions & 22 deletions cpp/KeywordTagging/test/test_keyword_tagging.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,11 @@ bool containsProp(const std::string &exp_text, const std::vector<MPFGenericTrack
const std::string &property, int index = -1) {
if (index != -1) {
if (tracks[index].detection_properties.count(property) == 0) {
return true;
return false;
}

std::string text = tracks[index].detection_properties.at(property);


return text.find(exp_text) != std::string::npos;
}

Expand All @@ -71,18 +70,17 @@ bool containsProp(const std::string &exp_text, const std::vector<MPFGenericTrack

std::string text = tracks[i].detection_properties.at(property);


if (text.find(exp_text) != std::string::npos)
return true;
}

return false;
}

void assertInText(const std::string &image_path, const std::string &expected_value,
void assertInText(const std::string &file_path, const std::string &expected_value,
const std::vector<MPFGenericTrack> &tracks, const std::string &prop, int index = -1) {
ASSERT_TRUE(containsProp(expected_value, tracks, prop, index))
<< "Expected tagger to detect " << prop << " \"" << expected_value << "\" in " << image_path;
<< "Expected tagger to detect " << prop << " \"" << expected_value << "\" in " << file_path;
}

void assertNotInText(const std::string &file_path, const std::string &expected_text,
Expand Down Expand Up @@ -111,7 +109,6 @@ bool logging_initialized = init_logging();
TEST(KEYWORDTAGGING, TaggingTest) {
KeywordTagging tagger;
std::vector<MPFGenericTrack> results;
std::map<std::string, std::string> custom_properties_disabled = {{"FULL_REGEX_SEARCH", "false"}};
std::map<std::string, std::string> custom_properties = {{}};

std::vector<MPFImageLocation> result;
Expand Down Expand Up @@ -140,7 +137,6 @@ TEST(KEYWORDTAGGING, TaggingTest) {
TEST(KEYWORDTAGGING, MulitpleTagsTest) {
KeywordTagging tagger;
std::vector<MPFGenericTrack> results;
std::map<std::string, std::string> custom_properties_disabled = {{"FULL_REGEX_SEARCH", "false"}};
std::map<std::string, std::string> custom_properties = {{}};

tagger.SetRunDirectory("../plugin");
Expand Down Expand Up @@ -228,18 +224,17 @@ TEST(KEYWORDTAGGING, FullSearch) {
TEST(KEYWORDTAGGING, LanguageTest) {
KeywordTagging tagger;
std::vector<MPFGenericTrack> results;
std::map<std::string, std::string> custom_properties_disabled = {{"FULL_REGEX_SEARCH", "false"}};
std::map<std::string, std::string> custom_properties = {{}};

tagger.SetRunDirectory("../plugin");

ASSERT_TRUE(tagger.Init());

ASSERT_NO_FATAL_FAILURE(runKeywordTagging("data/eng-bul.txt", tagger, results, custom_properties));
assertInText("data/eng-bul.txt", "foreign-text", results, "TAGS", 0);
assertInText("data/eng-bul.txt", "свободни", results, "TEXT TRIGGER WORDS", 0);
assertInText("data/eng-bul.txt", "106-113", results, "TEXT TRIGGER WORDS OFFSET", 0);
assertInText("data/eng-bul.txt", "Всички хора се раждат свободни", results, "TEXT", 0);
assertInText("data/eng-bul.txt", "foreign-text", results, "TAGS");
assertInText("data/eng-bul.txt", "свободни", results, "TEXT FOREIGN-TEXT TRIGGER WORDS");
assertInText("data/eng-bul.txt", "106-113", results, "TEXT FOREIGN-TEXT TRIGGER WORDS OFFSET");
assertInText("data/eng-bul.txt", "Всички хора се раждат свободни", results, "TEXT");

ASSERT_TRUE(tagger.Close());
}
Expand Down Expand Up @@ -572,16 +567,41 @@ TEST(KEYWORDTAGGING, FeedForwardTags) {
ASSERT_TRUE(tagger.Init());

MPFGenericTrack track(0.9,
{{"TAGS", "FeedForwardTag"},
{"BAR", "cash"}});
MPFGenericJob job("JOB NAME", "/some/path", track,
{ { "FEED_FORWARD_PROP_TO_PROCESS", "FOO,BAR" } }, {});
{{"TAGS", "FeedForwardTag"},
{"BAR", "cash"}});
MPFGenericJob job("JOB NAME", "/some/path", track,
{ { "FEED_FORWARD_PROP_TO_PROCESS", "FOO,BAR" } }, {});

std::vector<MPFGenericTrack> results = tagger.GetDetections(job);
ASSERT_EQ(1, results.size());
ASSERT_EQ(track.confidence, results.at(0).confidence);
std::vector<MPFGenericTrack> results = tagger.GetDetections(job);
ASSERT_EQ(1, results.size());
ASSERT_EQ(track.confidence, results.at(0).confidence);

Properties props = results.at(0).detection_properties;
ASSERT_EQ(4, props.size());
ASSERT_EQ("feedforwardtag; financial", props["TAGS"]);
Properties props = results.at(0).detection_properties;
ASSERT_EQ(4, props.size());
ASSERT_EQ("feedforwardtag; financial", props["TAGS"]);
}


TEST(KEYWORDTAGGING, NewLines) {
KeywordTagging tagger;
std::vector<MPFGenericTrack> results;
std::map<std::string, std::string> custom_properties = {{}};

tagger.SetRunDirectory("../plugin");

ASSERT_TRUE(tagger.Init());

ASSERT_NO_FATAL_FAILURE(runKeywordTagging("data/test-newlines.txt", tagger, results, custom_properties));
assertInText("data/test-newlines.txt", "identity document", results, "TAGS");
assertInText("data/test-newlines.txt", "address", results, "TEXT IDENTITY DOCUMENT TRIGGER WORDS");
assertInText("data/test-newlines.txt", "37-43", results, "TEXT IDENTITY DOCUMENT TRIGGER WORDS OFFSET");
assertInText("data/test-newlines.txt", "personal", results, "TAGS");
assertInText("data/test-newlines.txt", "777-777-7777", results, "TEXT PERSONAL TRIGGER WORDS");
assertInText("data/test-newlines.txt", "83-94", results, "TEXT PERSONAL TRIGGER WORDS OFFSET");
assertInText("data/test-newlines.txt", "564-456-46", results, "TEXT PERSONAL TRIGGER WORDS");
assertInText("data/test-newlines.txt", "145-154", results, "TEXT PERSONAL TRIGGER WORDS OFFSET");
assertInText("data/test-newlines.txt", "Text", results, "TEXT PERSONAL TRIGGER WORDS");
assertInText("data/test-newlines.txt", "19-22", results, "TEXT PERSONAL TRIGGER WORDS OFFSET");

ASSERT_TRUE(tagger.Close());
}