Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Implements handler_sphinx, a full-text search in the style of DBslaye…

…r, extensively used on openkvk.nl

TODO: configure.in needs some love to figure out where -lsphinxclient and sphinxclient.h is located, and the ability not to compile the handler.
TODO: we are currently not using a balancer scenario, but we might be able to do so.

http://code.google.com/p/cherokee/issues/detail?id=691
  • Loading branch information...
commit 72dbee13932a0237ac1f14e4faa6fef2fa48e2dc 1 parent 0ceae65
@skinkie authored
View
1  admin/plugins/Makefile.am
@@ -45,6 +45,7 @@ rrd.py \
scgi.py \
secdownload.py \
server_info.py \
+sphinx.py \
ssi.py \
streaming.py \
target_ip.py \
View
66 admin/plugins/sphinx.py
@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+#
+# Cherokee-admin
+#
+# Authors:
+# Alvaro Lopez Ortega <alvaro@alobbs.com>
+# Stefan de Konink <stefan@konink.de>
+#
+# Copyright (C) 2011 Alvaro Lopez Ortega
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 2 of the GNU General Public
+# License as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+#
+
+import CTK
+import Handler
+import Cherokee
+import Balancer
+
+from util import *
+from consts import *
+
+URL_APPLY = '/plugin/sphinx/apply'
+
+NOTE_LANG = N_("Language from which the information will be consumed.")
+NOTE_DB = N_("Index to connect to.")
+
+
+class Plugin_sphinx (Handler.PluginHandler):
+ def __init__ (self, key, **kwargs):
+ kwargs['show_document_root'] = False
+ Handler.PluginHandler.__init__ (self, key, **kwargs)
+ Handler.PluginHandler.AddCommon (self)
+
+ # DB-Slayer alike
+ table = CTK.PropsTable()
+ table.Add (_('Language'), CTK.ComboCfg('%s!lang'%(key), trans_options(DWRITER_LANGS)), _(NOTE_LANG))
+ table.Add (_('Index'), CTK.TextCfg('%s!index'%(key), True), _(NOTE_DB))
+
+ submit = CTK.Submitter (URL_APPLY)
+ submit += table
+
+ self += CTK.RawHTML ("<h2>%s</h2>" %(_('Serialization')))
+ self += CTK.Indenter (submit)
+
+ # Load Balancing (currently only one source is actually used!)
+ modul = CTK.PluginSelector('%s!balancer'%(key), trans_options(Cherokee.support.filter_available (BALANCERS)))
+ table = CTK.PropsTable()
+ table.Add (_("Balancer"), modul.selector_widget, _(Balancer.NOTE_BALANCER))
+
+ self += CTK.RawHTML ('<h2>%s</h2>' %(_('Data Base Balancing')))
+ self += CTK.Indenter (table)
+ self += modul
+
+CTK.publish ('^%s$'%(URL_APPLY), CTK.cfg_apply_post, method="POST")
View
21 cherokee/Makefile.am
@@ -568,6 +568,25 @@ dynamic_handler_file_lib = libplugin_file.la
endif
#
+# Handler sphinx
+#
+handler_sphinx = \
+handler_sphinx.h \
+handler_sphinx.c
+
+libplugin_sphinx_la_LDFLAGS = $(module_ldflags) $(SPHINXCLIENT_LDFLAGS)
+libplugin_sphinx_la_SOURCES = $(handler_sphinx)
+libplugin_sphinx_la_CFLAGS = $(SPHINXCLIENT_CFLAGS)
+
+if STATIC_HANDLER_SPHINX
+static_handler_sphinx_src = $(handler_sphinx)
+static_handler_sphinx_lib = $(SPHINXCLIENT_LDFLAGS)
+static_handler_sphinx_flags = $(SPHINXCLIENT_CFLAGS)
+else
+dynamic_handler_sphinx_lib = libplugin_sphinx.la
+endif
+
+#
# Handler tile
#
handler_tile = \
@@ -1563,6 +1582,7 @@ $(static_handler_post_report_src) \
$(static_collector_rrd_src) \
\
$(static_handler_file_src) \
+$(static_handler_sphinx_src) \
$(static_handler_tile_src) \
$(static_handler_dirlist_src) \
$(static_handler_cgi_src) \
@@ -1754,6 +1774,7 @@ $(dynamic_gen_evhost_lib) \
$(dynamic_post_track_lib) \
$(dynamic_handler_post_report_lib) \
$(dynamic_handler_file_lib) \
+$(dynamic_handler_sphinx_lib) \
$(dynamic_handler_tile_lib) \
$(dynamic_handler_server_info_lib) \
$(dynamic_handler_render_rrd_lib) \
View
261 cherokee/handler_sphinx.c
@@ -0,0 +1,261 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+
+/* Cherokee
+ *
+ * Authors:
+ * Alvaro Lopez Ortega <alvaro@alobbs.com>
+ * Stefan de Konink <stefan@konink.de>
+ *
+ * Copyright (C) 2001-2011 Alvaro Lopez Ortega
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+#include "common-internal.h"
+#include "handler_sphinx.h"
+#include "connection-protected.h"
+#include "thread.h"
+
+PLUGIN_INFO_HANDLER_EASIEST_INIT (sphinx, http_get);
+
+ret_t
+cherokee_handler_sphinx_init (cherokee_handler_sphinx_t *hdl)
+{
+ ret_t ret;
+ cherokee_connection_t *conn = HANDLER_CONN(hdl);
+ cherokee_handler_sphinx_props_t *props = HANDLER_SPHINX_PROPS(hdl);
+ cherokee_buffer_t *tmp = &HANDLER_THREAD(hdl)->tmp_buf1;
+ cuint_t len;
+ cuint_t limit = props->max_results;
+
+
+ hdl->client = sphinx_create ( SPH_TRUE );
+ if (unlikely(!hdl->client))
+ return ret_error;
+
+ cherokee_connection_parse_args (conn);
+
+ if (conn->arguments != NULL) {
+ cherokee_buffer_t *value;
+ cherokee_buffer_t limit_arg = CHEROKEE_BUF_INIT;
+ cherokee_buffer_fake_str(&limit_arg, "max");
+ ret = cherokee_avl_get (conn->arguments, &limit_arg, (void **)&value);
+ if (ret == ret_ok) {
+ limit = (int) strtol(value->buf, NULL, 10);
+ }
+ }
+
+ sphinx_set_limits ( hdl->client, 0, limit, limit, 0);
+
+ if ((cherokee_buffer_is_empty (&conn->web_directory)) ||
+ (cherokee_buffer_is_ending (&conn->web_directory, '/')))
+ {
+ len = conn->web_directory.len;
+ } else {
+ len = conn->web_directory.len + 1;
+ }
+
+ cherokee_buffer_clean (tmp);
+ cherokee_buffer_add (tmp,
+ conn->request.buf + len,
+ conn->request.len - len);
+
+
+ hdl->res = sphinx_query ( hdl->client, tmp->buf, props->index.buf, NULL );
+ if (unlikely(!hdl->res))
+ return ret_error;
+
+ return ret_ok;
+}
+
+
+static ret_t
+sphinx_add_headers (cherokee_handler_sphinx_t *hdl,
+ cherokee_buffer_t *buffer)
+{
+ switch (HANDLER_SPHINX_PROPS(hdl)->lang) {
+ case dwriter_json:
+ cherokee_buffer_add_str (buffer, "Content-Type: application/json" CRLF);
+ break;
+ case dwriter_python:
+ cherokee_buffer_add_str (buffer, "Content-Type: application/x-python" CRLF);
+ break;
+ case dwriter_php:
+ cherokee_buffer_add_str (buffer, "Content-Type: application/x-php" CRLF);
+ break;
+ case dwriter_ruby:
+ cherokee_buffer_add_str (buffer, "Content-Type: application/x-ruby" CRLF);
+ break;
+ default:
+ SHOULDNT_HAPPEN;
+ return ret_error;
+ }
+
+ return ret_ok;
+}
+
+
+ret_t
+sphinx_step (cherokee_handler_sphinx_t *hdl,
+ cherokee_buffer_t *buffer)
+{
+ int i;
+ cherokee_dwriter_set_buffer (&hdl->writer, buffer);
+
+ /* Open the result list */
+ cherokee_dwriter_list_open (&hdl->writer);
+
+ cherokee_dwriter_dict_open (&hdl->writer);
+ cherokee_dwriter_cstring (&hdl->writer, "RESULT");
+ cherokee_dwriter_list_open (&hdl->writer);
+
+ if (hdl->res) {
+ for ( i = 0; i < hdl->res->num_matches; i++ ) {
+ cherokee_dwriter_integer (&hdl->writer, sphinx_get_id ( hdl->res, i ));
+// cherokee_dwriter_integer (&hdl->writer, sphinx_get_weight ( hdl->res, i ));
+ }
+ }
+
+ cherokee_dwriter_list_close (&hdl->writer);
+ cherokee_dwriter_dict_close (&hdl->writer);
+
+ /* Close results list */
+ cherokee_dwriter_list_close (&hdl->writer);
+
+ return ret_eof_have_data;
+}
+
+
+static ret_t
+sphinx_free (cherokee_handler_sphinx_t *hdl)
+{
+ if (hdl->client)
+ sphinx_destroy(hdl->client);
+
+ cherokee_dwriter_mrproper (&hdl->writer);
+
+ return ret_ok;
+}
+
+ret_t
+cherokee_handler_sphinx_new (cherokee_handler_t **hdl,
+ void *cnt,
+ cherokee_module_props_t *props)
+{
+ CHEROKEE_NEW_STRUCT (n, handler_sphinx);
+
+ /* Init the base class object
+ */
+ cherokee_handler_init_base (HANDLER(n), cnt, HANDLER_PROPS(props), PLUGIN_INFO_HANDLER_PTR(sphinx));
+
+ MODULE(n)->init = (handler_func_init_t) cherokee_handler_sphinx_init;
+ MODULE(n)->free = (module_func_free_t) sphinx_free;
+ HANDLER(n)->step = (handler_func_step_t) sphinx_step;
+ HANDLER(n)->add_headers = (handler_func_add_headers_t) sphinx_add_headers;
+
+ /* Supported features
+ */
+ HANDLER(n)->support = hsupport_nothing;
+
+ /* Data writer */
+ cherokee_dwriter_init (&n->writer, &CONN_THREAD(cnt)->tmp_buf1);
+ n->writer.lang = PROP_SPHINX(props)->lang;
+
+ *hdl = HANDLER(n);
+ return ret_ok;
+}
+
+
+static ret_t
+props_free (cherokee_handler_sphinx_props_t *props)
+{
+/* if (props->balancer)
+ cherokee_balancer_free (props->balancer);*/
+
+ return ret_ok;
+}
+
+
+ret_t
+cherokee_handler_sphinx_configure (cherokee_config_node_t *conf,
+ cherokee_server_t *srv,
+ cherokee_module_props_t **_props)
+{
+ ret_t ret;
+ cherokee_list_t *i;
+ cherokee_handler_sphinx_props_t *props;
+
+ /* Instance a new property object
+ */
+ if (*_props == NULL) {
+ CHEROKEE_NEW_STRUCT (n, handler_sphinx_props);
+
+ cherokee_handler_props_init_base (HANDLER_PROPS(n),
+ MODULE_PROPS_FREE(props_free));
+/* n->balancer = NULL;*/
+
+ cherokee_buffer_init (&n->index);
+ cherokee_buffer_add_str (&n->index, "*"); /* default value for all indices */
+ n->max_results = 20;
+
+ *_props = MODULE_PROPS(n);
+ }
+
+ props = PROP_SPHINX(*_props);
+
+ /* Parse the configuration tree
+ */
+ cherokee_config_node_foreach (i, conf) {
+ cherokee_config_node_t *subconf = CONFIG_NODE(i);
+
+/* if (equal_buf_str (&subconf->key, "balancer")) {
+ ret = cherokee_balancer_instance (&subconf->val, subconf, srv, &props->balancer);
+ if (ret != ret_ok)
+ return ret;
+
+ } else */
+ if (equal_buf_str (&subconf->key, "lang")) {
+
+ if (equal_buf_str (&subconf->val, "json")) {
+ props->lang = dwriter_json;
+ } else if (equal_buf_str (&subconf->val, "python")) {
+ props->lang = dwriter_python;
+ } else if (equal_buf_str (&subconf->val, "php")) {
+ props->lang = dwriter_php;
+ } else if (equal_buf_str (&subconf->val, "ruby")) {
+ props->lang = dwriter_ruby;
+
+ } else {
+ return ret_error;
+ }
+ } else if (equal_buf_str (&subconf->key, "index")) {
+ cherokee_buffer_clean (&props->index);
+ cherokee_buffer_add_buffer (&props->index, &subconf->val);
+ } else if (equal_buf_str (&subconf->key, "max")) {
+ props->max_results = atoi(subconf->val.buf);
+ }
+
+ }
+
+ /* Final checks
+ */
+/* if (props->balancer == NULL) {
+ return ret_error;
+ }*/
+
+ return ret_ok;
+}
+
+
View
83 cherokee/handler_sphinx.h
@@ -0,0 +1,83 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+
+/* Cherokee
+ *
+ * Authors:
+ * Alvaro Lopez Ortega <alvaro@alobbs.com>
+ * Stefan de Konink <stefan@konink.de>
+ *
+ * Copyright (C) 2001-2011 Alvaro Lopez Ortega
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+#ifndef CHEROKEE_HANDLER_SPHINX_H
+#define CHEROKEE_HANDLER_SPHINX_H
+
+#include "handler.h"
+#include "dwriter.h"
+#include "balancer.h"
+#include "plugin_loader.h"
+#include "source.h"
+
+#include "sphinxclient.h"
+
+typedef ret_t (* dbs_func_t) (void *conn);
+typedef ret_t (* dbs_func_query_t) (void *conn, cherokee_buffer_t *buf);
+
+typedef struct {
+ cherokee_module_props_t base;
+// cherokee_balancer_t *balancer;
+ cherokee_dwriter_lang_t lang;
+ cherokee_buffer_t index;
+ cuint_t max_results;
+} cherokee_handler_sphinx_props_t;
+
+typedef struct {
+ cherokee_handler_t base;
+ cherokee_dwriter_t writer;
+// cherokee_source_t *src_ref;
+ sphinx_client *client;
+ sphinx_result *res;
+} cherokee_handler_sphinx_t;
+
+#define HDL_SPHINX(x) ((cherokee_handler_sphinx_t *)(x))
+#define PROP_SPHINX(x) ((cherokee_handler_sphinx_props_t *)(x))
+#define HANDLER_SPHINX_PROPS(x) (PROP_SPHINX(MODULE(x)->props))
+
+#define add_cstr_str(w,key,val,len) \
+ do { \
+ cherokee_dwriter_string (w, key, sizeof(key)-1); \
+ cherokee_dwriter_string (w, val, len); \
+ } while (false)
+
+#define add_cstr_int(w,key,val) \
+ do { \
+ cherokee_dwriter_string (w, key, sizeof(key)-1); \
+ cherokee_dwriter_integer (w, val); \
+ } while (false)
+
+
+/* Library init function
+ */
+void PLUGIN_INIT_NAME(sphinx) (cherokee_plugin_loader_t *loader);
+
+/* Methods
+ */
+ret_t cherokee_handler_sphinx_new (cherokee_handler_t **hdl, void *cnt, cherokee_module_props_t *props);
+ret_t cherokee_handler_sphinx_free (cherokee_handler_sphinx_t *hdl);
+ret_t cherokee_handler_sphinx_init (cherokee_handler_sphinx_t *hdl);
+
+#endif /* CHEROKEE_HANDLER_SPHINX_H */
View
8 configure.in
@@ -1441,12 +1441,13 @@ AC_ARG_ENABLE(static-module,
modules="error_redir error_nn server_info file dirlist cgi fcgi scgi uwsgi proxy redir common ssi secdownload empty_gif drop admin custom_error dbslayer streaming gzip deflate ncsa combined custom pam ldap mysql htpasswd plain htdigest authlist round_robin ip_hash failover directory extensions request header exists fullpath method from bind tls geoip url_arg v_or wildcard rehost target_ip evhost post_track post_report libssl render_rrd rrd not and or"
-beta_modules="filetime tile"
+beta_modules="filetime tile sphinx"
# Include beta modules only when --enable-beta is provided
#
if test "x$is_beta" = "xyes"; then
modules="$modules $beta_modules"
+ SPHINX_LDFLAGS="-lsphinxclient"
fi
# Remove modules that will not be compiled
@@ -1473,6 +1474,9 @@ fi
if test "x$have_ffmpeg" != "xyes"; then
modules=`echo $modules | sed s/streaming//`
fi
+#if test "x$have_sphinxclient" != "xyes"; then
+# modules=`echo $modules | sed s/sphinx//`
+#fi
add_calls=""
init_calls=""
@@ -1526,6 +1530,7 @@ AM_CONDITIONAL(STATIC_HANDLER_DROP, grep drop $conf_h >/dev/
AM_CONDITIONAL(STATIC_HANDLER_CUSTOM_ERROR, grep custom_error $conf_h >/dev/null)
AM_CONDITIONAL(STATIC_HANDLER_DBSLAYER, grep dbslayer $conf_h >/dev/null)
AM_CONDITIONAL(STATIC_HANDLER_STREAMING, grep streaming $conf_h >/dev/null)
+AM_CONDITIONAL(STATIC_HANDLER_SPHINX, grep sphinx $conf_h >/dev/null)
AM_CONDITIONAL(STATIC_HANDLER_RENDER_RRD, grep render_rrd $conf_h >/dev/null)
AM_CONDITIONAL(STATIC_HANDLER_TILE, grep tile $conf_h >/dev/null)
AM_CONDITIONAL(STATIC_ENCODER_GZIP, grep gzip $conf_h >/dev/null)
@@ -1656,6 +1661,7 @@ echo "LDAP $have_ldap"
echo "MySQL $have_mysql"
echo "GeoIP $have_geoip"
echo "FFMpeg $have_ffmpeg"
+#echo "Sphinx Client $have_sphinxclient"
echo "crypt support $crypt_type"
if test "x$is_beta" = "xyes"; then
echo "Beta release $PACKAGE_PATCH_VERSION"
Please sign in to comment.
Something went wrong with that request. Please try again.