Skip to content

Commit

Permalink
Started with searching from ES
Browse files Browse the repository at this point in the history
  • Loading branch information
@andris9
andris9 committed Jun 22, 2023
1 parent 5a9f584 commit 4514046
Show file tree
Hide file tree
Showing 5 changed files with 134 additions and 72 deletions.
2 changes: 2 additions & 0 deletions indexer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,8 @@ function indexingJob(esclient) {
uid: messageData.uid,
answered: messageData.flags ? messageData.flags.includes('\\Answered') : null,

ha: (messageData.attachments && messageData.attachments.length > 0) || false,

attachments:
(messageData.attachments &&
messageData.attachments.map(attachment =>
Expand Down
21 changes: 20 additions & 1 deletion lib/api/messages.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ const { nextPageCursorSchema, previousPageCursorSchema, pageNrSchema, sessSchema
const { preprocessAttachments } = require('../data-url');
const TaskHandler = require('../task-handler');
const prepareSearchFilter = require('../prepare-search-filter');
const { getMongoDBQuery } = require('../search-query');
const { getMongoDBQuery, getElasticSearchQuery } = require('../search-query');
const { getClient } = require('./lib/elasticsearch');

const BimiHandler = require('../bimi-handler');

Expand Down Expand Up @@ -584,6 +585,24 @@ module.exports = (db, server, messageHandler, userHandler, storageHandler, setti
let query;

if (result.value.q) {
let hasESFeatureFlag = await db.redis.sismember(`feature:indexing`, user.toString());
if (hasESFeatureFlag) {
// search from ElasticSearch

let searchQuery = await getElasticSearchQuery(db, user, result.value.q);

const esclient = getClient();

let searchResult = await esclient.search({
index: config.elasticsearch.index,
query: searchQuery,
sort: { uid: 'desc' }
});

console.log('ES RESULTS');
console.log(util.inspect(searchResult, false, 22, true));
}

filter = await getMongoDBQuery(db, user, result.value.q);
query = result.value.q;
} else {
Expand Down
5 changes: 5 additions & 0 deletions lib/ensure-es-index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ const mappings = {
type: 'boolean'
},

// has attachments
ha: {
type: 'boolean'
},

attachments: {
type: 'nested',
properties: {
Expand Down
164 changes: 100 additions & 64 deletions lib/search-query.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ const getMongoDBQuery = async (db, user, queryStr) => {

return { user: false };
};
/*

const getElasticSearchQuery = async (db, user, queryStr) => {
const parsed = parseSearchQuery(queryStr);

Expand All @@ -330,9 +330,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
}
};

let curNode = searchQuery;
let walkTree = async (node, curNode) => {
let walkTree = async node => {
if (Array.isArray(node)) {
let branches = [];
for (let entry of node) {
Expand Down Expand Up @@ -390,7 +388,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
};

if (node.text.negated) {
// FIXME: negation support!
branch = { bool: { must_not: branch.bool.should } };
}

return branch;
Expand All @@ -401,48 +399,83 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
if (keyword) {
let { value, negated } = node.keywords[keyword];
switch (keyword) {
case 'from':
case 'subject':
{
let regex = escapeRegexStr(value);
let branch = {
headers: {
$elemMatch: {
key: keyword,
value: {
$regex: regex,
$options: 'i'
}
match: {
subject: {
query: value,
operator: 'and'
}
}
};
if (negated) {
branch = { $not: branch };
branch = { bool: { must_not: branch } };
}
branches.push(branch);
}
break;

case 'from':
{
let branch = {
bool: {
should: [
{
match: {
[`from.name`]: {
query: value,
operator: 'and'
}
}
},
{
term: {
[`from.address`]: value
}
}
],
minimum_should_match: 1
}
};
if (negated) {
branch = { bool: { must_not: branch } };
}
branches.push(branch);
}
break;

case 'to':
{
let regex = escapeRegexStr(value);
let branch = {
bool: {
should: [],
minimum_should_match: 1
}
};

for (let toKey of ['to', 'cc', 'bcc']) {
let branch = {
headers: {
$elemMatch: {
key: toKey,
value: {
$regex: regex,
$options: 'i'
branch.bool.should.push(
{
match: {
[`${toKey}.name`]: {
query: value,
operator: 'and'
}
}
},
{
term: {
[`${toKey}.address`]: value
}
}
};
if (negated) {
branch = { $not: branch };
}
branches.push(branch);
);
}

if (negated) {
branch = { bool: { must_not: branch } };
}
branches.push(branch);
}
break;

Expand All @@ -468,9 +501,9 @@ const getElasticSearchQuery = async (db, user, queryStr) => {

let mailboxEntry = await db.database.collection('mailboxes').findOne(resolveQuery, { project: { _id: -1 } });

let branch = { mailbox: mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24)) };
let branch = { term: { mailbox: (mailboxEntry ? mailboxEntry._id : new ObjectId('0'.repeat(24))).toString() } };
if (negated) {
branch = { $not: branch };
branch = { bool: { must_not: [branch] } };
}
branches.push(branch);

Expand All @@ -481,9 +514,12 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
{
value = (value || '').toString().trim();
if (/^[0-9a-f]{24}$/i.test(value)) {
let branch = { thread: new ObjectId(value) };
let branch = { term: { thread: value } };
if (negated) {
branch = { $not: branch };
branch = { bool: { must_not: [branch] } };
}
if (negated) {
branch = { bool: { must_not: [branch] } };
}
branches.push(branch);
}
Expand All @@ -493,7 +529,7 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
case 'has': {
switch (value) {
case 'attachment': {
branches.push({ ha: true });
branches.push({ term: { ha: true } });
break;
}
}
Expand All @@ -506,39 +542,39 @@ const getElasticSearchQuery = async (db, user, queryStr) => {
};

if (parsed && parsed.length) {
let filter = await walkTree(Array.isArray(parsed) ? { $and: parsed } : parsed);
let extras = { user };
if (hasTextFilter) {
extras.searchable = true;
}
return Object.assign({ user: null }, filter, extras);
let filter = await walkTree({ $and: parsed });
searchQuery.bool.must = searchQuery.bool.must.concat(filter);
}

return { user: false };
return searchQuery;
};
*/

module.exports = { parseSearchQuery, getMongoDBQuery /*, getElasticSearchQuery*/ };

/*
const util = require('util');
let main = () => {
let db = require('./db');
db.connect(() => {
let run = async () => {
let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)'];
for (let query of queries) {
console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true));
console.log(util.inspect({ query, parsed: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true));
}
};
module.exports = { parseSearchQuery, getMongoDBQuery, getElasticSearchQuery };

if (process.env.DEBUG_TEST_QUERY && process.env.NODE_ENV !== 'production') {
const util = require('util'); // eslint-disable-line
let main = () => {
let db = require('./db'); // eslint-disable-line
db.connect(() => {
let run = async () => {
let queries = ['from:"amy namy" kupi in:spam to:greg has:attachment -subject:"dinner and movie tonight" (jupi OR subject:tere)'];

for (let query of queries) {
console.log('PARSED QUERY');
console.log(util.inspect({ query, parsed: parseSearchQuery(query) }, false, 22, true));
console.log('MongoDB');
console.log(util.inspect({ query, filter: await getMongoDBQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true));
console.log('ElasticSearch');
console.log(
util.inspect({ query, filter: await getElasticSearchQuery(db, new ObjectId('64099fff101ca2ef6aad8be7'), query) }, false, 22, true)
);
}
};

run();
});
};
main();
*/
run()
.catch(err => console.error(err))
.finally(() => process.exit());
});
};
main();
}
14 changes: 7 additions & 7 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"ajv": "8.12.0",
"chai": "4.3.7",
"docsify-cli": "4.4.4",
"eslint": "8.41.0",
"eslint": "8.43.0",
"eslint-config-nodemailer": "1.2.0",
"eslint-config-prettier": "8.8.0",
"grunt": "1.6.1",
Expand All @@ -35,7 +35,7 @@
"grunt-mocha-test": "0.13.3",
"grunt-shell-spawn": "0.4.0",
"grunt-wait": "0.3.0",
"imapflow": "1.0.128",
"imapflow": "1.0.130",
"mailparser": "3.6.4",
"mocha": "10.2.0",
"request": "2.88.2",
Expand All @@ -53,8 +53,8 @@
"base32.js": "0.1.0",
"bcryptjs": "2.4.3",
"bson": "5.3.0",
"bullmq": "3.14.0",
"fido2-lib": "3.4.0",
"bullmq": "3.15.8",
"fido2-lib": "3.4.1",
"gelf": "2.0.1",
"generate-password": "1.7.0",
"hash-wasm": "4.9.0",
Expand All @@ -64,7 +64,7 @@
"iconv-lite": "0.6.3",
"ioredfour": "1.2.0-ioredis-07",
"ioredis": "5.3.2",
"ipaddr.js": "2.0.1",
"ipaddr.js": "2.1.0",
"isemail": "3.2.0",
"joi": "17.9.2",
"js-yaml": "4.1.0",
Expand All @@ -73,7 +73,7 @@
"libmime": "5.2.1",
"libqp": "2.0.1",
"logic-query-parser": "0.0.5",
"mailauth": "4.3.4",
"mailauth": "4.4.0",
"mailsplit": "5.4.0",
"mobileconfig": "2.4.0",
"mongo-cursor-pagination": "8.1.3",
Expand All @@ -82,7 +82,7 @@
"msgpack5": "6.0.2",
"node-forge": "1.3.1",
"node-html-parser": "6.1.5",
"nodemailer": "6.9.2",
"nodemailer": "6.9.3",
"npmlog": "7.0.1",
"openpgp": "5.9.0",
"pem-jwk": "2.0.0",
Expand Down

0 comments on commit 4514046

Please sign in to comment.