Skip to content

Commit

Permalink
[Rework] Drop url tags
Browse files Browse the repository at this point in the history
  • Loading branch information
@vstakhov
vstakhov committed Aug 21, 2019
1 parent e971c53 commit 6b67691
Show file tree
Hide file tree
Showing 6 changed files with 1 addition and 289 deletions.
3 changes: 0 additions & 3 deletions src/libserver/html.c
View file
Expand Up @@ -721,9 +721,6 @@ rspamd_html_url_is_phished (rspamd_mempool_t *pool,
href_url->phished_url = text_url;
phished_tld.begin = href_tok.begin;
phished_tld.len = href_tok.len;
rspamd_url_add_tag (text_url, "phishing",
rspamd_mempool_ftokdup (pool, &phished_tld),
pool);
text_url->flags |= RSPAMD_URL_FLAG_HTML_DISPLAYED;
}
}
Expand Down
28 changes: 0 additions & 28 deletions src/libserver/url.c
View file
Expand Up @@ -3176,34 +3176,6 @@ rspamd_url_task_subject_callback (struct rspamd_url *url, gsize start_offset,
}
}

void
rspamd_url_add_tag (struct rspamd_url *url, const gchar *tag,
const gchar *value,
rspamd_mempool_t *pool)
{
struct rspamd_url_tag *found, *ntag;

g_assert (url != NULL && tag != NULL && value != NULL);

if (url->tags == NULL) {
url->tags = g_hash_table_new (rspamd_strcase_hash, rspamd_strcase_equal);
rspamd_mempool_add_destructor (pool,
(rspamd_mempool_destruct_t)g_hash_table_unref, url->tags);
}

found = g_hash_table_lookup (url->tags, tag);

ntag = rspamd_mempool_alloc0 (pool, sizeof (*ntag));
ntag->data = rspamd_mempool_strdup (pool, value);

if (found == NULL) {
g_hash_table_insert (url->tags, rspamd_mempool_strdup (pool, tag),
ntag);
}

DL_APPEND (found, ntag);
}

guint
rspamd_url_hash (gconstpointer u)
{
Expand Down
11 changes: 0 additions & 11 deletions src/libserver/url.h
View file
Expand Up @@ -70,7 +70,6 @@ struct rspamd_url {

enum rspamd_url_flags flags;
guint count;
GHashTable *tags;
};

enum uri_errno {
Expand Down Expand Up @@ -214,16 +213,6 @@ void rspamd_url_task_subject_callback (struct rspamd_url *url,
gsize start_offset,
gsize end_offset, gpointer ud);

/**
* Adds a tag for url
* @param url
* @param tag
* @param pool
*/
void rspamd_url_add_tag (struct rspamd_url *url, const gchar *tag,
const gchar *value,
rspamd_mempool_t *pool);

guint rspamd_url_hash (gconstpointer u);

guint rspamd_email_hash (gconstpointer u);
Expand Down
130 changes: 1 addition & 129 deletions src/lua/lua_url.c
View file
Expand Up @@ -59,10 +59,7 @@ LUA_FUNCTION_DEF (url, is_obscured);
LUA_FUNCTION_DEF (url, is_html_displayed);
LUA_FUNCTION_DEF (url, is_subject);
LUA_FUNCTION_DEF (url, get_phished);
LUA_FUNCTION_DEF (url, get_tag);
LUA_FUNCTION_DEF (url, get_count);
LUA_FUNCTION_DEF (url, get_tags);
LUA_FUNCTION_DEF (url, add_tag);
LUA_FUNCTION_DEF (url, get_visible);
LUA_FUNCTION_DEF (url, create);
LUA_FUNCTION_DEF (url, init);
Expand All @@ -87,9 +84,7 @@ static const struct luaL_reg urllib_m[] = {
LUA_INTERFACE_DEF (url, is_html_displayed),
LUA_INTERFACE_DEF (url, is_subject),
LUA_INTERFACE_DEF (url, get_phished),
LUA_INTERFACE_DEF (url, get_tag),
LUA_INTERFACE_DEF (url, get_tags),
LUA_INTERFACE_DEF (url, add_tag),

LUA_INTERFACE_DEF (url, get_visible),
LUA_INTERFACE_DEF (url, get_count),
LUA_INTERFACE_DEF (url, get_flags),
Expand Down Expand Up @@ -438,129 +433,6 @@ lua_url_is_subject (lua_State *L)
return 1;
}

/***
* @method url:get_tag(tag)
* Returns list of string for a specific tagname for an url
* @return {table/strings} list of tags for an url
*/
static gint
lua_url_get_tag (lua_State *L)
{
LUA_TRACE_POINT;
struct rspamd_lua_url *url = lua_check_url (L, 1);
guint i;
const gchar *tag = luaL_checkstring (L, 2);
struct rspamd_url_tag *tval, *cur;

if (url != NULL && tag != NULL) {

if (url->url->tags == NULL) {
lua_createtable (L, 0, 0);
}
else {
tval = g_hash_table_lookup (url->url->tags, tag);

if (tval) {
lua_newtable (L);
i = 1;

DL_FOREACH (tval, cur) {
lua_pushstring (L, cur->data);
lua_rawseti (L, -2, i ++);
}

lua_settable (L, -3);
}
else {
lua_createtable (L, 0, 0);
}
}
}
else {
lua_pushnil (L);
}

return 1;
}


/***
* @method url:get_tags()
* Returns list of string tags for an url
* @return {table/strings} list of tags for an url
*/
static gint
lua_url_get_tags (lua_State *L)
{
LUA_TRACE_POINT;
struct rspamd_lua_url *url = lua_check_url (L, 1);
guint i;
GHashTableIter it;
struct rspamd_url_tag *tval, *cur;
gpointer k, v;

if (url != NULL) {
if (url->url->tags == NULL) {
lua_createtable (L, 0, 0);
}
else {
lua_createtable (L, 0, g_hash_table_size (url->url->tags));
g_hash_table_iter_init (&it, url->url->tags);

while (g_hash_table_iter_next (&it, &k, &v)) {
tval = v;
lua_pushstring (L, (const gchar *)k);
lua_newtable (L);
i = 1;

DL_FOREACH (tval, cur) {
lua_pushstring (L, cur->data);
lua_rawseti (L, -2, i ++);
}

lua_settable (L, -3);
}
}
}
else {
lua_pushnil (L);
}

return 1;
}

/***
* @method url:add_tag(tag, mempool)
* Adds a new tag for url
* @param {string} tag new tag to add
* @param {mempool} mempool memory pool (e.g. `task:get_pool()`)
*/
static gint
lua_url_add_tag (lua_State *L)
{
LUA_TRACE_POINT;
struct rspamd_lua_url *url = lua_check_url (L, 1);
rspamd_mempool_t *mempool = rspamd_lua_check_mempool (L, 4);
const gchar *tag = luaL_checkstring (L, 2);
const gchar *value;

if (lua_type (L, 3) == LUA_TSTRING) {
value = lua_tostring (L, 3);
}
else {
value = "1"; /* Some stupid placeholder */
}

if (url != NULL && mempool != NULL && tag != NULL) {
rspamd_url_add_tag (url->url, tag, value, mempool);
}
else {
return luaL_error (L, "invalid arguments");
}

return 0;
}

/***
* @method url:get_phished()
* Get another URL that pretends to be this URL (e.g. used in phishing)
Expand Down
62 changes: 0 additions & 62 deletions src/plugins/surbl.c
View file
Expand Up @@ -1584,9 +1584,6 @@ process_dns_results (struct rspamd_task *task,
bit->bit);
rspamd_task_insert_result (task, bit->symbol, 1, resolved_name);

if (surbl_module_ctx->use_tags) {
rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool);
}
got_result = TRUE;
}
}
Expand All @@ -1605,10 +1602,6 @@ process_dns_results (struct rspamd_task *task,
resolved_name, suffix->suffix,
bit->bit);
rspamd_task_insert_result (task, bit->symbol, 1, resolved_name);

if (surbl_module_ctx->use_tags) {
rspamd_url_add_tag (uri, "surbl", bit->symbol, task->task_pool);
}
}
}
}
Expand All @@ -1618,10 +1611,6 @@ process_dns_results (struct rspamd_task *task,
msg_info_surbl ("domain [%s] is in surbl %s",
resolved_name, suffix->suffix);
rspamd_task_insert_result (task, suffix->symbol, 1, resolved_name);

if (surbl_module_ctx->use_tags) {
rspamd_url_add_tag (uri, "surbl", suffix->symbol, task->task_pool);
}
}
else {
ina.s_addr = addr;
Expand Down Expand Up @@ -1795,11 +1784,6 @@ surbl_redirector_finish (struct rspamd_http_connection *conn,
else {
existing->count ++;
}

if (surbl_module_ctx->use_tags) {
rspamd_url_add_tag (param->url, "redirector", urlstr,
task->task_pool);
}
}
else {
msg_info_surbl ("cannot parse redirector reply: %s", urlstr);
Expand Down Expand Up @@ -1879,38 +1863,6 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
}
}

static gboolean
surbl_test_tags (struct rspamd_task *task, struct redirector_param *param,
struct rspamd_url *url)
{
struct rspamd_url_tag *tag = NULL, *cur;
gchar *ftld = NULL;
rspamd_ftok_t tld;
gboolean processed = FALSE;

if (url->tags) {
tag = g_hash_table_lookup (url->tags, "surbl");
}

if (tag) {
tld.begin = url->tld;
tld.len = url->tldlen;

ftld = rspamd_mempool_ftokdup (task->task_pool, &tld);
/* We know results for this URL */

DL_FOREACH (tag, cur) {
msg_info_surbl ("domain [%s] is in surbl %s (tags)",
ftld, cur->data);
rspamd_task_insert_result (task, cur->data, 1, ftld);
}

processed = TRUE;
}

return processed;
}

static void
surbl_tree_redirector_callback (gpointer key, gpointer value, void *data)
{
Expand Down Expand Up @@ -2026,15 +1978,6 @@ surbl_tree_url_callback (gpointer key, gpointer value, void *data)
msg_debug_surbl ("check url %*s in %s", url->urllen, url->string,
param->suffix->suffix);

if (surbl_module_ctx->use_tags && surbl_test_tags (param->task, param, url)) {
return;
}

if (url->tags && g_hash_table_lookup (url->tags, "redirector")) {
/* URL is redirected, skip from checks */
return;
}

make_surbl_requests (url, param->task, param->item, param->suffix, FALSE,
param->tree, surbl_module_ctx);
}
Expand Down Expand Up @@ -2324,11 +2267,6 @@ surbl_continue_process_handler (lua_State *L)
redirected_url->phished_url = param->url;
redirected_url->flags |= RSPAMD_URL_FLAG_REDIRECTED;
}

if (surbl_module_ctx->use_tags) {
rspamd_url_add_tag (param->url, "redirector", urlstr,
task->task_pool);
}
}
else {
msg_info_surbl ("could not resolve '%*s' on redirector",
Expand Down
56 changes: 0 additions & 56 deletions test/functional/lua/url_tags.lua
View file

This file was deleted.

0 comments on commit 6b67691

Please sign in to comment.