diff --git a/include/library.h b/include/library.h
index 1763de7d5..ef43f2a7e 100644
--- a/include/library.h
+++ b/include/library.h
@@ -26,6 +26,7 @@
#include
#include
#include
+#include
#include "book.h"
#include "bookmark.h"
@@ -140,6 +141,22 @@ class Filter {
bool accept(const Book& book) const;
};
+
+class ZimSearcher : public zim::Searcher
+{
+ public:
+ explicit ZimSearcher(zim::Searcher&& searcher)
+ : zim::Searcher(searcher)
+ {}
+
+ std::unique_lock getLock() {
+ return std::unique_lock(m_mutex);
+ }
+ virtual ~ZimSearcher() = default;
+ private:
+ std::mutex m_mutex;
+};
+
/**
* A Library store several books.
*/
@@ -152,6 +169,7 @@ class Library
typedef uint64_t Revision;
typedef std::vector BookIdCollection;
typedef std::map AttributeCounts;
+ typedef std::set BookIdSet;
public:
Library();
@@ -207,6 +225,10 @@ class Library
DEPRECATED std::shared_ptr getReaderById(const std::string& id);
std::shared_ptr getArchiveById(const std::string& id);
+ std::shared_ptr getSearcherById(const std::string& id) {
+ return getSearcherByIds(BookIdSet{id});
+ }
+ std::shared_ptr getSearcherByIds(const BookIdSet& ids);
/**
* Remove a book from the library.
@@ -338,7 +360,7 @@ class Library
std::vector getBookPropValueSet(BookStrPropMemFn p) const;
BookIdCollection filterViaBookDB(const Filter& filter) const;
void updateBookDB(const Book& book);
- void dropReader(const std::string& bookId);
+ void dropCache(const std::string& bookId);
private: //data
std::unique_ptr mp_impl;
diff --git a/include/search_renderer.h b/include/search_renderer.h
index cd499d13d..22dccef1f 100644
--- a/include/search_renderer.h
+++ b/include/search_renderer.h
@@ -81,9 +81,9 @@ class SearchRenderer
void setSearchPattern(const std::string& pattern);
/**
- * Set the search content id.
+ * Set the querystring used to select books
*/
- void setSearchContent(const std::string& name);
+ void setSearchBookQuery(const std::string& bookQuery);
/**
* Set protocol prefix.
@@ -112,7 +112,7 @@ class SearchRenderer
zim::SearchResultSet m_srs;
NameMapper* mp_nameMapper;
Library* mp_library;
- std::string searchContent;
+ std::string searchBookQuery;
std::string searchPattern;
std::string protocolPrefix;
std::string searchProtocolPrefix;
diff --git a/include/server.h b/include/server.h
index b3cd240fd..0cd579c57 100644
--- a/include/server.h
+++ b/include/server.h
@@ -54,6 +54,7 @@ namespace kiwix
void setAddress(const std::string& addr) { m_addr = addr; }
void setPort(int port) { m_port = port; }
void setNbThreads(int threads) { m_nbThreads = threads; }
+ void setMultiZimSearchLimit(unsigned int limit) { m_multizimSearchLimit = limit; }
void setIpConnectionLimit(int limit) { m_ipConnectionLimit = limit; }
void setVerbose(bool verbose) { m_verbose = verbose; }
void setIndexTemplateString(const std::string& indexTemplateString) { m_indexTemplateString = indexTemplateString; }
@@ -63,7 +64,7 @@ namespace kiwix
{ m_blockExternalLinks = blockExternalLinks; }
int getPort();
std::string getAddress();
-
+
protected:
Library* mp_library;
NameMapper* mp_nameMapper;
@@ -72,6 +73,7 @@ namespace kiwix
std::string m_indexTemplateString = "";
int m_port = 80;
int m_nbThreads = 1;
+ unsigned int m_multizimSearchLimit = 0;
bool m_verbose = false;
bool m_withTaskbar = true;
bool m_withLibraryButton = true;
diff --git a/src/library.cpp b/src/library.cpp
index f77059404..ad97a0c38 100644
--- a/src/library.cpp
+++ b/src/library.cpp
@@ -27,10 +27,13 @@
#include "tools/regexTools.h"
#include "tools/pathTools.h"
#include "tools/stringTools.h"
+#include "tools/otherTools.h"
+#include "tools/concurrent_cache.h"
#include
#include
#include
+#include
#include
#include
@@ -56,6 +59,27 @@ bool booksReferToTheSameArchive(const Book& book1, const Book& book2)
&& book1.getPath() == book2.getPath();
}
+template
+class MultiKeyCache: public ConcurrentCache, Value>
+{
+ public:
+ explicit MultiKeyCache(size_t maxEntries)
+ : ConcurrentCache, Value>(maxEntries)
+ {}
+
+ bool drop(const Key& key)
+ {
+ std::unique_lock l(this->lock_);
+ bool removed = false;
+ for(auto& cache_key: this->impl_.keys()) {
+ if(cache_key.find(key)!=cache_key.end()) {
+ removed |= this->impl_.drop(cache_key);
+ }
+ }
+ return removed;
+ }
+};
+
} // unnamed namespace
struct Library::Impl
@@ -63,15 +87,14 @@ struct Library::Impl
struct Entry : Book
{
Library::Revision lastUpdatedRevision = 0;
-
- // May also keep the Archive and Reader pointers here and get
- // rid of the m_readers and m_archives data members in Library
};
Library::Revision m_revision;
std::map m_books;
- std::map> m_readers;
- std::map> m_archives;
+ using ArchiveCache = ConcurrentCache>;
+ std::unique_ptr mp_archiveCache;
+ using SearcherCache = MultiKeyCache>;
+ std::unique_ptr mp_searcherCache;
std::vector m_bookmarks;
Xapian::WritableDatabase m_bookDB;
@@ -85,7 +108,9 @@ struct Library::Impl
};
Library::Impl::Impl()
- : m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
+ : mp_archiveCache(new ArchiveCache(std::max(getEnvVar("KIWIX_ARCHIVE_CACHE_SIZE", 1), 1))),
+ mp_searcherCache(new SearcherCache(std::max(getEnvVar("KIWIX_SEARCHER_CACHE_SIZE", 1), 1))),
+ m_bookDB("", Xapian::DB_BACKEND_INMEMORY)
{
}
@@ -139,7 +164,7 @@ bool Library::addBook(const Book& book)
try {
auto& oldbook = mp_impl->m_books.at(book.getId());
if ( ! booksReferToTheSameArchive(oldbook, book) ) {
- dropReader(book.getId());
+ dropCache(book.getId());
}
oldbook.update(book); // XXX: This may have no effect if oldbook is readonly
// XXX: Then m_bookDB will become out-of-sync with
@@ -150,6 +175,13 @@ bool Library::addBook(const Book& book)
auto& newEntry = mp_impl->m_books[book.getId()];
static_cast(newEntry) = book;
newEntry.lastUpdatedRevision = mp_impl->m_revision;
+ size_t new_cache_size = std::ceil(mp_impl->getBookCount(true, true)*0.1);
+ if (getEnvVar("KIWIX_ARCHIVE_CACHE_SIZE", -1) <= 0) {
+ mp_impl->mp_archiveCache->setMaxSize(new_cache_size);
+ }
+ if (getEnvVar("KIWIX_SEARCHER_CACHE_SIZE", -1) <= 0) {
+ mp_impl->mp_searcherCache->setMaxSize(new_cache_size);
+ }
return true;
}
}
@@ -173,17 +205,23 @@ bool Library::removeBookmark(const std::string& zimId, const std::string& url)
}
-void Library::dropReader(const std::string& id)
+void Library::dropCache(const std::string& id)
{
- mp_impl->m_readers.erase(id);
- mp_impl->m_archives.erase(id);
+ mp_impl->mp_archiveCache->drop(id);
+ mp_impl->mp_searcherCache->drop(id);
}
bool Library::removeBookById(const std::string& id)
{
std::lock_guard lock(m_mutex);
mp_impl->m_bookDB.delete_document("Q" + id);
- dropReader(id);
+ dropCache(id);
+ // We do not change the cache size here
+ // Most of the time, the book is remove in case of library refresh, it is
+ // often associated with addBook calls (which will properly set the cache size)
+ // Having a too big cache is not a problem here (or it would have been before)
+ // (And setMaxSize doesn't actually reduce the cache size, extra cached items
+ // will be removed in put or getOrPut).
return mp_impl->m_books.erase(id) == 1;
}
@@ -242,35 +280,49 @@ const Book& Library::getBookByPath(const std::string& path) const
std::shared_ptr Library::getReaderById(const std::string& id)
{
- try {
- std::lock_guard lock(m_mutex);
- return mp_impl->m_readers.at(id);
- } catch (std::out_of_range& e) {}
-
- const auto archive = getArchiveById(id);
- if ( !archive )
+ auto archive = getArchiveById(id);
+ if(archive) {
+ return std::make_shared(archive);
+ } else {
return nullptr;
-
- const shared_ptr reader(new Reader(archive, true));
- std::lock_guard lock(m_mutex);
- mp_impl->m_readers[id] = reader;
- return reader;
+ }
}
std::shared_ptr Library::getArchiveById(const std::string& id)
{
- std::lock_guard lock(m_mutex);
try {
- return mp_impl->m_archives.at(id);
- } catch (std::out_of_range& e) {}
-
- auto book = getBookById(id);
- if (!book.isPathValid())
+ return mp_impl->mp_archiveCache->getOrPut(id,
+ [&](){
+ auto book = getBookById(id);
+ if (!book.isPathValid()) {
+ throw std::invalid_argument("");
+ }
+ return std::make_shared(book.getPath());
+ });
+ } catch (std::invalid_argument&) {
return nullptr;
+ }
+}
- auto sptr = make_shared(book.getPath());
- mp_impl->m_archives[id] = sptr;
- return sptr;
+std::shared_ptr Library::getSearcherByIds(const BookIdSet& ids)
+{
+ assert(!ids.empty());
+ try {
+ return mp_impl->mp_searcherCache->getOrPut(ids,
+ [&](){
+ std::vector archives;
+ for(auto& id:ids) {
+ auto archive = getArchiveById(id);
+ if(!archive) {
+ throw std::invalid_argument("");
+ }
+ archives.push_back(*archive);
+ }
+ return std::make_shared(zim::Searcher(archives));
+ });
+ } catch (std::invalid_argument&) {
+ return nullptr;
+ }
}
unsigned int Library::getBookCount(const bool localBooks,
diff --git a/src/search_renderer.cpp b/src/search_renderer.cpp
index 8f9685563..8878ead80 100644
--- a/src/search_renderer.cpp
+++ b/src/search_renderer.cpp
@@ -71,9 +71,9 @@ void SearchRenderer::setSearchPattern(const std::string& pattern)
searchPattern = pattern;
}
-void SearchRenderer::setSearchContent(const std::string& content)
+void SearchRenderer::setSearchBookQuery(const std::string& bookQuery)
{
- searchContent = content;
+ searchBookQuery = bookQuery;
}
void SearchRenderer::setProtocolPrefix(const std::string& prefix)
@@ -90,13 +90,13 @@ kainjow::mustache::data buildQueryData
(
const std::string& searchProtocolPrefix,
const std::string& pattern,
- const std::string& searchContent
+ const std::string& bookQuery
) {
kainjow::mustache::data query;
query.set("pattern", kiwix::encodeDiples(pattern));
std::ostringstream ss;
ss << searchProtocolPrefix << "?pattern=" << urlEncode(pattern, true);
- ss << "&content=" << urlEncode(searchContent, true);
+ ss << "&" << bookQuery;
query.set("unpaginatedQuery", ss.str());
return query;
}
@@ -197,7 +197,7 @@ std::string SearchRenderer::getHtml()
kainjow::mustache::data query = buildQueryData(
searchProtocolPrefix,
searchPattern,
- searchContent
+ searchBookQuery
);
std::string template_str = RESOURCE::templates::search_result_html;
diff --git a/src/server.cpp b/src/server.cpp
index 52355afe7..bd478c526 100644
--- a/src/server.cpp
+++ b/src/server.cpp
@@ -45,6 +45,7 @@ bool Server::start() {
m_port,
m_root,
m_nbThreads,
+ m_multizimSearchLimit,
m_verbose,
m_withTaskbar,
m_withLibraryButton,
diff --git a/src/server/internalServer.cpp b/src/server/internalServer.cpp
index 0290d79a7..772f6a4da 100644
--- a/src/server/internalServer.cpp
+++ b/src/server/internalServer.cpp
@@ -95,35 +95,197 @@ inline std::string normalizeRootUrl(std::string rootUrl)
return rootUrl.empty() ? rootUrl : "/" + rootUrl;
}
-// Returns the value of env var `name` if found, otherwise returns defaultVal
-unsigned int getCacheLength(const char* name, unsigned int defaultVal) {
- try {
- const char* envString = std::getenv(name);
- if (envString == nullptr) {
- throw std::runtime_error("Environment variable not set");
- }
- return extractFromString(envString);
- } catch (...) {}
+Filter get_search_filter(const RequestContext& request, const std::string& prefix="")
+{
+ auto filter = kiwix::Filter().valid(true).local(true);
+ try {
+ filter.query(request.get_argument(prefix+"q"));
+ } catch (const std::out_of_range&) {}
+ try {
+ filter.maxSize(request.get_argument(prefix+"maxsize"));
+ } catch (...) {}
+ try {
+ filter.name(request.get_argument(prefix+"name"));
+ } catch (const std::out_of_range&) {}
+ try {
+ filter.category(request.get_argument(prefix+"category"));
+ } catch (const std::out_of_range&) {}
+ try {
+ filter.lang(request.get_argument(prefix+"lang"));
+ } catch (const std::out_of_range&) {}
+ try {
+ filter.acceptTags(kiwix::split(request.get_argument(prefix+"tag"), ";"));
+ } catch (...) {}
+ try {
+ filter.rejectTags(kiwix::split(request.get_argument(prefix+"notag"), ";"));
+ } catch (...) {}
+ return filter;
+}
- return defaultVal;
+template
+std::vector subrange(const std::vector& v, size_t s, size_t n)
+{
+ const size_t e = std::min(v.size(), s+n);
+ return std::vector(v.begin()+std::min(v.size(), s), v.begin()+e);
}
+
+std::string renderUrl(const std::string& root, const std::string& urlTemplate)
+{
+ MustacheData data;
+ data.set("root", root);
+ auto url = kainjow::mustache::mustache(urlTemplate).render(data);
+ if ( url.back() == '\n' )
+ url.pop_back();
+ return url;
+}
+
+std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
+{
+ return i18n::expandParameterizedString(lang, "suggest-full-text-search",
+ {
+ {"SEARCH_TERMS", queryString}
+ }
+ );
+}
+
+ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
+{
+ return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
+}
+
+ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
+{
+ return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
+}
+
+ParameterizedMessage noValueForArgMsg(const std::string& argument)
+{
+ return ParameterizedMessage("no-value-for-arg", { {"ARGUMENT", argument} });
+}
+
+ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
+{
+ return ParameterizedMessage("raw-entry-not-found",
+ {
+ {"DATATYPE", dt},
+ {"ENTRY", entry},
+ }
+ );
+}
+
+ParameterizedMessage tooManyBooksMsg(size_t nbBooks, size_t limit)
+{
+ return ParameterizedMessage("too-many-books",
+ {
+ {"NB_BOOKS", beautifyInteger(nbBooks)},
+ {"LIMIT", beautifyInteger(limit)},
+ }
+ );
+}
+
+ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
+{
+ const ParameterizedMessage::Parameters noParams;
+ return ParameterizedMessage(msgId, noParams);
+}
+
+struct Error : public std::runtime_error {
+ explicit Error(const ParameterizedMessage& message)
+ : std::runtime_error("Error while handling request"),
+ _message(message)
+ {}
+
+ const ParameterizedMessage& message() const
+ {
+ return _message;
+ }
+
+ const ParameterizedMessage _message;
+};
+
+void checkBookNumber(const Library::BookIdSet& bookIds, size_t limit) {
+ if (bookIds.empty()) {
+ throw Error(nonParameterizedMessage("no-book-found"));
+ }
+ if (limit > 0 && bookIds.size() > limit) {
+ throw Error(tooManyBooksMsg(bookIds.size(), limit));
+ }
+}
+
} // unnamed namespace
-SearchInfo::SearchInfo(const std::string& pattern)
- : pattern(pattern),
- geoQuery()
-{}
+std::pair InternalServer::selectBooks(const RequestContext& request) const
+{
+ // Try old API
+ try {
+ auto bookName = request.get_argument("content");
+ try {
+ const auto bookIds = Library::BookIdSet{mp_nameMapper->getIdForName(bookName)};
+ const auto queryString = request.get_query([&](const std::string& key){return key == "content";}, true);
+ return {queryString, bookIds};
+ } catch (const std::out_of_range&) {
+ throw Error(noSuchBookErrorMsg(bookName));
+ }
+ } catch(const std::out_of_range&) {
+ // We've catch the out_of_range of get_argument
+ // continue
+ }
-SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery)
- : pattern(pattern),
- geoQuery(geoQuery)
-{}
+ // Does user directly gives us ids ?
+ try {
+ auto id_vec = request.get_arguments("books.id");
+ if (id_vec.empty()) {
+ throw Error(noValueForArgMsg("books.id"));
+ }
+ for(const auto& bookId: id_vec) {
+ try {
+ // This is a silly way to check that bookId exists
+ mp_nameMapper->getNameForId(bookId);
+ } catch (const std::out_of_range&) {
+ throw Error(noSuchBookErrorMsg(bookId));
+ }
+ }
+ const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
+ const auto queryString = request.get_query([&](const std::string& key){return key == "books.id";}, true);
+ return {queryString, bookIds};
+ } catch(const std::out_of_range&) {}
+
+ // Use the names
+ try {
+ auto name_vec = request.get_arguments("books.name");
+ if (name_vec.empty()) {
+ throw Error(noValueForArgMsg("books.name"));
+ }
+ Library::BookIdSet bookIds;
+ for(const auto& bookName: name_vec) {
+ try {
+ bookIds.insert(mp_nameMapper->getIdForName(bookName));
+ } catch(const std::out_of_range&) {
+ throw Error(noSuchBookErrorMsg(bookName));
+ }
+ }
+ const auto queryString = request.get_query([&](const std::string& key){return key == "books.name";}, true);
+ return {queryString, bookIds};
+ } catch(const std::out_of_range&) {}
+
+ // Check for filtering
+ Filter filter = get_search_filter(request, "books.filter.");
+ auto id_vec = mp_library->filter(filter);
+ if (id_vec.empty()) {
+ throw Error(nonParameterizedMessage("no-book-found"));
+ }
+ const auto bookIds = Library::BookIdSet(id_vec.begin(), id_vec.end());
+ const auto queryString = request.get_query([&](const std::string& key){return startsWith(key, "books.filter.");}, true);
+ return {queryString, bookIds};
+}
-SearchInfo::SearchInfo(const RequestContext& request)
- : pattern(request.get_optional_param("pattern", "")),
- geoQuery(),
- bookName(request.get_optional_param("content", ""))
+SearchInfo InternalServer::getSearchInfo(const RequestContext& request) const
{
+ auto bookIds = selectBooks(request);
+ checkBookNumber(bookIds.second, m_multizimSearchLimit);
+ auto pattern = request.get_optional_param("pattern", "");
+ GeoQuery geoQuery;
+
/* Retrive geo search */
try {
auto latitude = request.get_argument("latitude");
@@ -134,10 +296,19 @@ SearchInfo::SearchInfo(const RequestContext& request)
catch(const std::invalid_argument&) {}
if (!geoQuery && pattern.empty()) {
- throw std::invalid_argument("No query provided.");
+ throw Error(nonParameterizedMessage("no-query"));
}
+
+ return SearchInfo(pattern, geoQuery, bookIds.second, bookIds.first);
}
+SearchInfo::SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterQuery)
+ : pattern(pattern),
+ geoQuery(geoQuery),
+ bookIds(bookIds),
+ bookFilterQuery(bookFilterQuery)
+{}
+
zim::Query SearchInfo::getZimQuery(bool verbose) const {
zim::Query query;
if (verbose) {
@@ -175,6 +346,7 @@ InternalServer::InternalServer(Library* library,
int port,
std::string root,
int nbThreads,
+ unsigned int multizimSearchLimit,
bool verbose,
bool withTaskbar,
bool withLibraryButton,
@@ -185,6 +357,7 @@ InternalServer::InternalServer(Library* library,
m_port(port),
m_root(normalizeRootUrl(root)),
m_nbThreads(nbThreads),
+ m_multizimSearchLimit(multizimSearchLimit),
m_verbose(verbose),
m_withTaskbar(withTaskbar),
m_withLibraryButton(withLibraryButton),
@@ -194,9 +367,8 @@ InternalServer::InternalServer(Library* library,
mp_daemon(nullptr),
mp_library(library),
mp_nameMapper(nameMapper ? nameMapper : &defaultNameMapper),
- searcherCache(getCacheLength("SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U))),
- searchCache(getCacheLength("SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
- suggestionSearcherCache(getCacheLength("SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
+ searchCache(getEnvVar("KIWIX_SEARCH_CACHE_SIZE", DEFAULT_CACHE_SIZE)),
+ suggestionSearcherCache(getEnvVar("KIWIX_SUGGESTION_SEARCHER_CACHE_SIZE", std::max((unsigned int) (mp_library->getBookCount(true, true)*0.1), 1U)))
{}
bool InternalServer::start() {
@@ -439,56 +611,6 @@ SuggestionsList_t getSuggestions(SuggestionSearcherCache& cache, const zim::Arch
return suggestions;
}
-namespace
-{
-
-std::string renderUrl(const std::string& root, const std::string& urlTemplate)
-{
- MustacheData data;
- data.set("root", root);
- auto url = kainjow::mustache::mustache(urlTemplate).render(data);
- if ( url.back() == '\n' )
- url.pop_back();
- return url;
-}
-
-std::string makeFulltextSearchSuggestion(const std::string& lang, const std::string& queryString)
-{
- return i18n::expandParameterizedString(lang, "suggest-full-text-search",
- {
- {"SEARCH_TERMS", queryString}
- }
- );
-}
-
-ParameterizedMessage noSuchBookErrorMsg(const std::string& bookName)
-{
- return ParameterizedMessage("no-such-book", { {"BOOK_NAME", bookName} });
-}
-
-ParameterizedMessage invalidRawAccessMsg(const std::string& dt)
-{
- return ParameterizedMessage("invalid-raw-data-type", { {"DATATYPE", dt} });
-}
-
-ParameterizedMessage rawEntryNotFoundMsg(const std::string& dt, const std::string& entry)
-{
- return ParameterizedMessage("raw-entry-not-found",
- {
- {"DATATYPE", dt},
- {"ENTRY", entry},
- }
- );
-}
-
-ParameterizedMessage nonParameterizedMessage(const std::string& msgId)
-{
- const ParameterizedMessage::Parameters noParams;
- return ParameterizedMessage(msgId, noParams);
-}
-
-} // unnamed namespace
-
std::unique_ptr InternalServer::handle_suggest(const RequestContext& request)
{
if (m_verbose.load()) {
@@ -591,41 +713,18 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re
}
try {
- auto searchInfo = SearchInfo(request);
-
- std::string bookId;
- std::shared_ptr archive;
- if (!searchInfo.bookName.empty()) {
- try {
- bookId = mp_nameMapper->getIdForName(searchInfo.bookName);
- archive = mp_library->getArchiveById(bookId);
- } catch (const std::out_of_range&) {
- throw std::invalid_argument("The requested book doesn't exist.");
- }
- }
-
+ auto searchInfo = getSearchInfo(request);
+ auto bookIds = searchInfo.getBookIds();
/* Make the search */
// Try to get a search from the searchInfo, else build it
+ auto searcher = mp_library->getSearcherByIds(bookIds);
+ auto lock(searcher->getLock());
+
std::shared_ptr search;
try {
search = searchCache.getOrPut(searchInfo,
[=](){
- std::shared_ptr searcher;
- if (archive) {
- searcher = searcherCache.getOrPut(bookId, [=](){ return std::make_shared(*archive);});
- } else {
- for (auto& bookId: mp_library->filter(kiwix::Filter().local(true).valid(true))) {
- auto currentArchive = mp_library->getArchiveById(bookId);
- if (currentArchive) {
- if (! searcher) {
- searcher = std::make_shared(*currentArchive);
- } else {
- searcher->addArchive(*currentArchive);
- }
- }
- }
- }
return make_shared(searcher->search(searchInfo.getZimQuery(m_verbose.load())));
}
);
@@ -638,11 +737,14 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re
"404-page-heading",
cssUrl);
response += nonParameterizedMessage("no-search-results");
- response += TaskbarInfo(searchInfo.bookName, archive.get());
+ if(bookIds.size() == 1) {
+ auto bookId = *bookIds.begin();
+ auto bookName = mp_nameMapper->getNameForId(bookId);
+ response += TaskbarInfo(bookName, mp_library->getArchiveById(bookId).get());
+ }
return response;
}
-
auto start = 0;
try {
start = request.get_argument("start");
@@ -663,17 +765,21 @@ std::unique_ptr InternalServer::handle_search(const RequestContext& re
SearchRenderer renderer(search->getResults(start, pageLength), mp_nameMapper, mp_library, start,
search->getEstimatedMatches());
renderer.setSearchPattern(searchInfo.pattern);
- renderer.setSearchContent(searchInfo.bookName);
+ renderer.setSearchBookQuery(searchInfo.bookFilterQuery);
renderer.setProtocolPrefix(m_root + "/");
renderer.setSearchProtocolPrefix(m_root + "/search");
renderer.setPageLength(pageLength);
auto response = ContentResponse::build(*this, renderer.getHtml(), "text/html; charset=utf-8");
- response->set_taskbar(searchInfo.bookName, archive.get());
+ if(bookIds.size() == 1) {
+ auto bookId = *bookIds.begin();
+ auto bookName = mp_nameMapper->getNameForId(bookId);
+ response->set_taskbar(bookName, mp_library->getArchiveById(bookId).get());
+ }
return std::move(response);
- } catch (const std::invalid_argument& e) {
+ } catch (const Error& e) {
return HTTP400HtmlResponse(*this, request)
+ invalidUrlMsg
- + std::string(e.what());
+ + e.message();
}
}
@@ -776,45 +882,6 @@ std::unique_ptr InternalServer::handle_catalog(const RequestContext& r
return std::move(response);
}
-namespace
-{
-
-Filter get_search_filter(const RequestContext& request)
-{
- auto filter = kiwix::Filter().valid(true).local(true);
- try {
- filter.query(request.get_argument("q"));
- } catch (const std::out_of_range&) {}
- try {
- filter.maxSize(request.get_argument("maxsize"));
- } catch (...) {}
- try {
- filter.name(request.get_argument("name"));
- } catch (const std::out_of_range&) {}
- try {
- filter.category(request.get_argument("category"));
- } catch (const std::out_of_range&) {}
- try {
- filter.lang(request.get_argument("lang"));
- } catch (const std::out_of_range&) {}
- try {
- filter.acceptTags(kiwix::split(request.get_argument("tag"), ";"));
- } catch (...) {}
- try {
- filter.rejectTags(kiwix::split(request.get_argument("notag"), ";"));
- } catch (...) {}
- return filter;
-}
-
-template
-std::vector subrange(const std::vector& v, size_t s, size_t n)
-{
- const size_t e = std::min(v.size(), s+n);
- return std::vector(v.begin()+std::min(v.size(), s), v.begin()+e);
-}
-
-} // unnamed namespace
-
std::vector
InternalServer::search_catalog(const RequestContext& request,
kiwix::OPDSDumper& opdsDumper)
diff --git a/src/server/internalServer.h b/src/server/internalServer.h
index 69e8166c4..a0cffa95b 100644
--- a/src/server/internalServer.h
+++ b/src/server/internalServer.h
@@ -68,27 +68,26 @@ struct GeoQuery {
class SearchInfo {
public:
- SearchInfo(const std::string& pattern);
- SearchInfo(const std::string& pattern, GeoQuery geoQuery);
- SearchInfo(const RequestContext& request);
+ SearchInfo(const std::string& pattern, GeoQuery geoQuery, const Library::BookIdSet& bookIds, const std::string& bookFilterString);
zim::Query getZimQuery(bool verbose) const;
+ const Library::BookIdSet& getBookIds() const { return bookIds; }
friend bool operator<(const SearchInfo& l, const SearchInfo& r)
{
- return std::tie(l.bookName, l.pattern, l.geoQuery)
- < std::tie(r.bookName, r.pattern, r.geoQuery); // keep the same order
+ return std::tie(l.bookIds, l.pattern, l.geoQuery)
+ < std::tie(r.bookIds, r.pattern, r.geoQuery); // keep the same order
}
public: //data
std::string pattern;
GeoQuery geoQuery;
- std::string bookName;
+ Library::BookIdSet bookIds;
+ std::string bookFilterQuery;
};
typedef kainjow::mustache::data MustacheData;
-typedef ConcurrentCache> SearcherCache;
typedef ConcurrentCache> SearchCache;
typedef ConcurrentCache> SuggestionSearcherCache;
@@ -103,6 +102,7 @@ class InternalServer {
int port,
std::string root,
int nbThreads,
+ unsigned int multizimSearchLimit,
bool verbose,
bool withTaskbar,
bool withLibraryButton,
@@ -150,12 +150,15 @@ class InternalServer {
bool etag_not_needed(const RequestContext& r) const;
ETag get_matching_if_none_match_etag(const RequestContext& request) const;
+ std::pair selectBooks(const RequestContext& r) const;
+ SearchInfo getSearchInfo(const RequestContext& r) const;
private: // data
std::string m_addr;
int m_port;
std::string m_root;
int m_nbThreads;
+ unsigned int m_multizimSearchLimit;
std::atomic_bool m_verbose;
bool m_withTaskbar;
bool m_withLibraryButton;
@@ -167,7 +170,6 @@ class InternalServer {
Library* mp_library;
NameMapper* mp_nameMapper;
- SearcherCache searcherCache;
SearchCache searchCache;
SuggestionSearcherCache suggestionSearcherCache;
diff --git a/src/server/request_context.cpp b/src/server/request_context.cpp
index 6b435e5ad..53a5cde21 100644
--- a/src/server/request_context.cpp
+++ b/src/server/request_context.cpp
@@ -106,7 +106,7 @@ MHD_Result RequestContext::fill_argument(void *__this, enum MHD_ValueKind kind,
const char *key, const char* value)
{
RequestContext *_this = static_cast(__this);
- _this->arguments[key] = value == nullptr ? "" : value;
+ _this->arguments[key].push_back(value == nullptr ? "" : value);
return MHD_YES;
}
@@ -121,8 +121,14 @@ void RequestContext::print_debug_info() const {
printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
}
printf("arguments :\n");
- for (auto it=arguments.begin(); it!=arguments.end(); it++) {
- printf(" - %s : '%s'\n", it->first.c_str(), it->second.c_str());
+ for (auto& pair:arguments) {
+ printf(" - %s :", pair.first.c_str());
+ bool first = true;
+ for (auto& v: pair.second) {
+ printf("%s %s", first?"":",", v.c_str());
+ first = false;
+ }
+ printf("\n");
}
printf("Parsed : \n");
printf("full_url: %s\n", full_url.c_str());
@@ -176,23 +182,13 @@ ByteRange RequestContext::get_range() const {
template<>
std::string RequestContext::get_argument(const std::string& name) const {
- return arguments.at(name);
+ return arguments.at(name)[0];
}
std::string RequestContext::get_header(const std::string& name) const {
return headers.at(lcAll(name));
}
-std::string RequestContext::get_query() const {
- std::string q;
- const char* sep = "";
- for ( const auto& a : arguments ) {
- q += sep + a.first + '=' + a.second;
- sep = "&";
- }
- return q;
-}
-
std::string RequestContext::get_user_language() const
{
try {
diff --git a/src/server/request_context.h b/src/server/request_context.h
index 2c4c8902e..f0b79b58e 100644
--- a/src/server/request_context.h
+++ b/src/server/request_context.h
@@ -25,6 +25,7 @@
#include
#include
#include
- The requested book doesn't exist.
+ No such book: non-existing-book
)" },
{ /* url */ "/ROOT/search?content=non-existing-book&pattern=a\"