diff --git a/src/mongo/db/fts/SConscript b/src/mongo/db/fts/SConscript index 5c9c28dcd0572..25f3b467c5764 100644 --- a/src/mongo/db/fts/SConscript +++ b/src/mongo/db/fts/SConscript @@ -34,6 +34,7 @@ baseEnv.Library('base', [ 'fts_spec.cpp', 'fts_spec_legacy.cpp', 'fts_language.cpp', + 'fts_basic_phrase_matcher.cpp', 'fts_basic_tokenizer.cpp', 'fts_util.cpp', 'fts_element_iterator.cpp', @@ -93,3 +94,6 @@ env.CppUnitTest( "fts_matcher_test", "fts_matcher_test.cpp", env.CppUnitTest( "fts_element_iterator_test", "fts_element_iterator_test.cpp", LIBDEPS=["base"] ) + +env.CppUnitTest( "fts_basic_phrase_matcher_test", "fts_basic_phrase_matcher_test.cpp", + LIBDEPS=["base"] ) diff --git a/src/mongo/db/fts/fts_basic_phrase_matcher.cpp b/src/mongo/db/fts/fts_basic_phrase_matcher.cpp new file mode 100644 index 0000000000000..e32174d3b155d --- /dev/null +++ b/src/mongo/db/fts/fts_basic_phrase_matcher.cpp @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2015 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/fts/fts_basic_phrase_matcher.h" + +#include "mongo/platform/strcasestr.h" + +namespace mongo { +namespace fts { + +using std::string; + +bool BasicFTSPhraseMatcher::phraseMatches(const string& phrase, + const string& haystack, + PhraseMatcherOptions options) const { + if (options & kCaseSensitive) { + return haystack.find(phrase) != string::npos; + } + + return strcasestr(haystack.c_str(), phrase.c_str()) != NULL; +} + +} // namespace fts +} // namespace mongo diff --git a/src/mongo/db/fts/fts_basic_phrase_matcher.h b/src/mongo/db/fts/fts_basic_phrase_matcher.h new file mode 100644 index 0000000000000..c595b9c3bf910 --- /dev/null +++ b/src/mongo/db/fts/fts_basic_phrase_matcher.h @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2015 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/fts/fts_phrase_matcher.h" + +namespace mongo { +namespace fts { + +/** + * A phrase matcher that looks for exact substring matches with optional ASCII-aware case + * insensitivity. + */ +class BasicFTSPhraseMatcher final : public FTSPhraseMatcher { + MONGO_DISALLOW_COPYING(BasicFTSPhraseMatcher); + +public: + BasicFTSPhraseMatcher() = default; + + bool phraseMatches(const std::string& phrase, + const std::string& haystack, + PhraseMatcherOptions options) const final; +}; + +} // namespace fts +} // namespace mongo diff --git a/src/mongo/db/fts/fts_basic_phrase_matcher_test.cpp b/src/mongo/db/fts/fts_basic_phrase_matcher_test.cpp new file mode 100644 index 0000000000000..9f193e836ad44 --- /dev/null +++ b/src/mongo/db/fts/fts_basic_phrase_matcher_test.cpp @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2015 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/fts/fts_basic_phrase_matcher.h" + +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace fts { + +// Case insensitive match. +TEST(FtsBasicPhraseMatcher, CaseInsensitive) { + std::string str1 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + std::string find1 = "Consectetur adipiscing"; + std::string nofind1 = "dolor amet"; + + std::string str2 = "Duis aute irure dolor in reprehenderit in Voluptate velit esse cillum."; + std::string find2 = "In Voluptate"; + std::string nofind2 = "dolor velit"; + + BasicFTSPhraseMatcher phraseMatcher; + FTSPhraseMatcher::PhraseMatcherOptions options = FTSPhraseMatcher::kNone; + + ASSERT(phraseMatcher.phraseMatches(find1, str1, options)); + ASSERT(phraseMatcher.phraseMatches(find2, str2, options)); + + ASSERT_FALSE(phraseMatcher.phraseMatches(nofind1, str1, options)); + ASSERT_FALSE(phraseMatcher.phraseMatches(nofind2, str2, options)); +} + +// Case sensitive match. +TEST(FtsBasicPhraseMatcher, CaseSensitive) { + std::string str1 = "Lorem ipsum dolor sit amet, consectetur adipiscing elit."; + std::string find1 = "Lorem ipsum"; + std::string nofind1 = "Sit amet"; + + std::string str2 = "Duis aute irure dolor in reprehenderit in Voluptate velit esse cillum."; + std::string find2 = "in Voluptate"; + std::string nofind2 = "Irure dolor"; + + BasicFTSPhraseMatcher phraseMatcher; + FTSPhraseMatcher::PhraseMatcherOptions options = FTSPhraseMatcher::kCaseSensitive; + + ASSERT(phraseMatcher.phraseMatches(find1, str1, options)); + ASSERT(phraseMatcher.phraseMatches(find2, str2, options)); + + ASSERT_FALSE(phraseMatcher.phraseMatches(nofind1, str1, options)); + ASSERT_FALSE(phraseMatcher.phraseMatches(nofind2, str2, options)); +} + +} // namespace fts +} // namespace mongo diff --git a/src/mongo/db/fts/fts_language.cpp b/src/mongo/db/fts/fts_language.cpp index 7a0c64ab1cf74..1180cfa17b114 100644 --- a/src/mongo/db/fts/fts_language.cpp +++ b/src/mongo/db/fts/fts_language.cpp @@ -33,6 +33,7 @@ #include #include "mongo/base/init.h" +#include "mongo/db/fts/fts_basic_phrase_matcher.h" #include "mongo/db/fts/fts_basic_tokenizer.h" #include "mongo/stdx/memory.h" #include "mongo/util/assert_util.h" @@ -85,6 +86,10 @@ std::unique_ptr BasicFTSLanguage::createTokenizer() const { return stdx::make_unique(this); } +const FTSPhraseMatcher& BasicFTSLanguage::getPhraseMatcher() const { + return _basicPhraseMatcher; +} + MONGO_INITIALIZER_GROUP(FTSAllLanguagesRegistered, MONGO_NO_PREREQUISITES, MONGO_NO_DEPENDENTS); // diff --git a/src/mongo/db/fts/fts_language.h b/src/mongo/db/fts/fts_language.h index facdb8c9ce0cb..6c986f5de6e6b 100644 --- a/src/mongo/db/fts/fts_language.h +++ b/src/mongo/db/fts/fts_language.h @@ -30,6 +30,8 @@ #pragma once +#include "mongo/db/fts/fts_basic_phrase_matcher.h" +#include "mongo/db/fts/fts_phrase_matcher.h" #include "mongo/db/fts/fts_util.h" #include "mongo/base/status_with.h" @@ -86,6 +88,11 @@ class FTSLanguage { */ virtual std::unique_ptr createTokenizer() const = 0; + /** + * Returns a reference to the phrase matcher instance that this language owns. + */ + virtual const FTSPhraseMatcher& getPhraseMatcher() const = 0; + /** * Register std::string 'languageName' as a new language with text index version * 'textIndexVersion'. Saves the resulting language to out-argument 'languageOut'. @@ -133,7 +140,11 @@ typedef StatusWith StatusWithFTSLanguage; class BasicFTSLanguage : public FTSLanguage { public: - std::unique_ptr createTokenizer() const override; + std::unique_ptr createTokenizer() const final; + const FTSPhraseMatcher& getPhraseMatcher() const final; + +private: + BasicFTSPhraseMatcher _basicPhraseMatcher; }; extern BasicFTSLanguage languagePorterV1; diff --git a/src/mongo/db/fts/fts_matcher.cpp b/src/mongo/db/fts/fts_matcher.cpp index 544ef93cf367d..52c67e337e9c5 100644 --- a/src/mongo/db/fts/fts_matcher.cpp +++ b/src/mongo/db/fts/fts_matcher.cpp @@ -31,9 +31,9 @@ #include "mongo/platform/basic.h" #include "mongo/db/fts/fts_matcher.h" +#include "mongo/db/fts/fts_phrase_matcher.h" #include "mongo/db/fts/fts_tokenizer.h" #include "mongo/db/fts/fts_element_iterator.h" -#include "mongo/platform/strcasestr.h" namespace mongo { @@ -41,17 +41,6 @@ namespace fts { using std::string; -/** - * Does the string 'phrase' occur in the string 'haystack'? Match is case-insensitive if - * 'caseSensitive' is false; otherwise, an exact substring match is performed. - */ -static bool phraseMatches(const string& phrase, const string& haystack, bool caseSensitive) { - if (caseSensitive) { - return haystack.find(phrase) != string::npos; - } - return strcasestr(haystack.c_str(), phrase.c_str()) != NULL; -} - FTSMatcher::FTSMatcher(const FTSQuery& query, const FTSSpec& spec) : _query(query), _spec(spec) {} bool FTSMatcher::matches(const BSONObj& obj) const { @@ -163,7 +152,12 @@ bool FTSMatcher::_phraseMatch(const string& phrase, const BSONObj& obj) const { while (it.more()) { FTSIteratorValue val = it.next(); - if (phraseMatches(phrase, val._text, _query.getCaseSensitive())) { + + if (val._language->getPhraseMatcher().phraseMatches(phrase, + val._text, + _query.getCaseSensitive() + ? FTSPhraseMatcher::kCaseSensitive + : FTSPhraseMatcher::kNone)) { return true; } } diff --git a/src/mongo/db/fts/fts_phrase_matcher.h b/src/mongo/db/fts/fts_phrase_matcher.h new file mode 100644 index 0000000000000..ae7b8c8a9f912 --- /dev/null +++ b/src/mongo/db/fts/fts_phrase_matcher.h @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2015 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include + +namespace mongo { +namespace fts { + +/** + * An interface for substring matching routines. + */ +class FTSPhraseMatcher { +public: + virtual ~FTSPhraseMatcher() = default; + + using PhraseMatcherOptions = uint8_t; + + /** + * Use no options. + */ + static const int kNone = 0; + + /** + * Lowercase strings as part of phrase matching. + */ + static const int kCaseSensitive = 1 << 0; + + /** + * Does the string 'phrase' occur in the string 'haystack'? + */ + virtual bool phraseMatches(const std::string& phrase, + const std::string& haystack, + PhraseMatcherOptions options) const = 0; +}; + +} // namespace fts +} // namespace mongo diff --git a/src/mongo/db/fts/fts_query.cpp b/src/mongo/db/fts/fts_query.cpp index 8dec8e29204da..4fa332266a08b 100644 --- a/src/mongo/db/fts/fts_query.cpp +++ b/src/mongo/db/fts/fts_query.cpp @@ -106,10 +106,11 @@ Status FTSQuery::parse(const string& query, unsigned phraseStart = quoteOffset + 1; unsigned phraseLength = t.offset - phraseStart; StringData phrase = StringData(query).substr(phraseStart, phraseLength); - if (inNegation) - _negatedPhrases.push_back(normalizeString(phrase)); - else - _positivePhrases.push_back(normalizeString(phrase)); + if (inNegation) { + _negatedPhrases.push_back(phrase.toString()); + } else { + _positivePhrases.push_back(phrase.toString()); + } inNegation = false; inPhrase = false; } else { @@ -170,13 +171,6 @@ void FTSQuery::_addTerms(FTSTokenizer* tokenizer, const string& sentence, bool n } } -string FTSQuery::normalizeString(StringData str) const { - if (_caseSensitive) { - return str.toString(); - } - return tolowerString(str); -} - namespace { void _debugHelp(stringstream& ss, const set& s, const string& sep) { bool first = true; diff --git a/src/mongo/db/fts/fts_query.h b/src/mongo/db/fts/fts_query.h index 10e0cd2faaf44..cac73425ffbba 100644 --- a/src/mongo/db/fts/fts_query.h +++ b/src/mongo/db/fts/fts_query.h @@ -84,11 +84,6 @@ class FTSQuery { BSONObj toBSON() const; - /** - * Lowercases "str" if _caseSensitive is set, else returns a copy of "str" unchanged. - */ - std::string normalizeString(StringData str) const; - static const bool caseSensitiveDefault; private: diff --git a/src/mongo/db/fts/fts_tokenizer.h b/src/mongo/db/fts/fts_tokenizer.h index 40cdbde2cb851..4c99506eae277 100644 --- a/src/mongo/db/fts/fts_tokenizer.h +++ b/src/mongo/db/fts/fts_tokenizer.h @@ -40,9 +40,9 @@ class StopWords; /** * FTSTokenizer - * A iterator of "documents" where a document contains space delimited words. - * For each word returns a stem or lemma version of a word optimized for full text indexing. - * Supports various options to control how tokens are generated. + * A iterator of "documents" where a document contains space delimited words. For each word returns + * a stem or lemma version of a word optimized for full text indexing. Supports various options to + * control how tokens are generated. */ class FTSTokenizer { public: