Initial Commit of Sphinx for SMF

Signed-off-by: jdarwood007 <unmonitored+github@sleepycode.com>

Admin-Sphinx.english.php

@@ -0,0 +1,52 @@
+<?php
+// Version: 2.0; Admin-SphinxQL
+global $helptxt;
+
+$txt['search_index_sphinxql'] = 'SphinxQL';
+$txt['search_index_sphinxql_desc'] = '3rd party Sphinx Search Engine.';
+
+$txt['sphinx_download_config'] = 'Download Sphinx.conf';
+
+$txt['sphinx_admin_title'] = 'Sphinx';
+$txt['sphinx_server_config_tittle'] = 'Sphinx Server Config Settings';
+$txt['sphinx_server_config_note'] = 'After changing these settings you must update your Sphinx configuration file';
+$txt['sphinx_smf_sphinx_tittle'] = 'Sphinx SMF Settings';
+$txt['sphinx_smf_tittle'] = 'SMF Settings';
+$txt['sphinx_config_hints_title'] = 'Sphinx Server Configuration Hints';
+
+$txt['sphinx_data_path'] = 'Index data path';
+$txt['sphinx_data_path_subtext'] = 'This is the path that will be containing the search index files used by Sphinx.';
+
+$txt['sphinx_log_path'] = 'Log path';
+$txt['sphinx_log_path_subtext'] = 'Server path that will contain the log files created by Sphinx.';
+
+$txt['sphinx_stopword_path'] = 'Stopword path';
+$txt['sphinx_stopword_path_subtext'] = 'The server path to the stopword list (leave empty for no stopword list).';
+
+$txt['sphinx_indexer_mem'] = 'Memory limit indexer';
+$txt['sphinx_indexer_mem_subtext'] = 'The maximum amount of (RAM) memory the indexer is allowed to be using.';
+$txt['sphinx_indexer_mem_postinput'] = 'MB';
+
+$txt['sphinx_searchd_server'] = 'Sphinx server';
+$txt['sphinx_searchd_server_subtext'] = 'Server the Sphinx search deamon resides on.';
+
+$txt['sphinxql_searchd_port'] = 'Sphinx port';
+$txt['sphinxql_searchd_port_subtext'] = 'Port on which the search deamon will listen.';
+
+$txt['sphinx_max_results'] = 'Maximum # matches';
+$txt['sphinx_max_results_subtext'] = 'Maximum amount of matches the search deamon will return.';
+
+$txt['search_index'] = 'SMF Search API';
+$txt['sphinx_smf_search_standard'] = 'Standard (Default)';
+$txt['sphinx_smf_search_fulltext'] = 'Fulltext';
+$txt['sphinx_smf_search_custom'] = 'Custom index';
+$txt['sphinx_smf_search_sphinx'] = 'Sphinx (Legacy)';
+$txt['sphinx_smf_search_sphinxql'] = 'SphinxQL';
+
+$txt['sphinx_config_hints_save'] = 'Please save your configuration first.';
+$txt['sphinx_config_hints_desc'] = 'Create directories for storing the indexes: %1$s';
+$txt['sphinx_config_hints_pgsql_func'] = 'Execute the following the following statement on your PostgreSQL database';
+$txt['sphinx_config_hints_index_start'] = 'It\'s time to create the indexes. Do not proceed with the next command if it fails.';
+$txt['sphinx_config_hints_index_finish'] = 'If everything worked so far, congratulations, Sphinx has been installed and works! Next step is modifying SMF\'s search to work with Sphinx';
+$txt['sphinx_config_hints_cron_start'] = 'In order to keep the full-text index up to date, you need to add a cron job that will update the index from time to time. The configuration file defines two indexes: <tt>smf_delta_index</tt>, an index that only stores the recent changes and can be called frequently.  <tt>smf_base_index</tt>, an index that stores the full database and should be called less frequently.<br />
+Adding the following lines to /etc/crontab would let the index rebuild every day (at 3 am) and update the most recently changed messages each hour:';

Dev/Sphinxql-RTUpdates.php

@@ -0,0 +1,276 @@
+<?php
+
+/**
+ * Simple Machines Forum (SMF)
+ *
+ * @package SMF
+ * @author Simple Machines http://www.simplemachines.org
+ * @copyright 2015 Simple Machines and individual contributors
+ * @license http://www.simplemachines.org/about/smf/license.php BSD
+ *
+ * The Following fixes a bug in SMF to show this in the settings section.
+ * SearchAPI-Sphinxql.php
+ * @version 2.0.12
+ */
+
+if (!defined('SMF'))
+	die('Hacking attempt...');
+
+/**
+ * Our Search API class
+ * This Class should not load, it is used to store possible changes to the API
+ * These are not included for lack of testing or lack of Sphinx support
+ * At this time it includes functions which Sphinx only supports with RT indexes
+ * We do not make use of RT indexes at this time.
+ * @package SearchAPI
+ */
+class sphinxql_search_rtIndexes extends search_api
+{
+	/**
+	 * Callback when a post is created
+	 * @see createPost()
+	 *
+	 * @access public
+	 * @param array $msgOptions An array of post data
+	 * @param array $topicOptions An array of topic data
+	 * @param array $posterOptions An array of info about the person who made this post
+	 * @return void
+	 */
+	public function postCreated(array &$msgOptions, array &$topicOptions, array &$posterOptions)
+	{
+		return true;
+		//	!! SphinxQL Does support updating the search index from its QL interface.
+		//	!! Sphinx for SMF does not support this at this time. Code is provided
+		//	!! here as examples/testing purposes.
+
+		global $smcFunc, $modSettings;
+
+		// Create an instance of the sphinx client.
+		$mySphinx = $this->dbfunc_connect();
+
+		// Figure out our weights.
+		$weight_factors = array(
+			'age',
+			'length',
+			'first_message',
+			'sticky',
+		);
+		$weight = array();
+		$weight_total = 0;
+		foreach ($weight_factors as $weight_factor)
+		{
+			$weight[$weight_factor] = empty($modSettings['search_weight_' . $weight_factor]) ? 0 : (int) $modSettings['search_weight_' . $weight_factor];
+			$weight_total += $weight[$weight_factor];
+		}
+		if ($weight_total === 0)
+		{
+			$weight = array(
+				'age' => 25,
+				'length' => 25,
+				'first_message' => 25,
+				'sticky' => 25,
+			);
+			$weight_total = 100;
+		}
+
+		// The data was inserted at this point, lets get some data as the passed variables don't contain all we need.
+		$request = $smcFunc['db_query']('', '
+			SELECT 
+				m.id_msg, m.id_topic, m.id_board, IF(m.id_member = 0, 4294967295, m.id_member) AS id_member, m.poster_time, m.body, m.subject,
+				t.num_replies + 1 AS num_replies,
+				CEILING(1000000 * (
+					IF(m.id_msg < 0.7 * s.value, 0, (m.id_msg - 0.7 * s.value) / (0.3 * s.value)) * {int:weight_age} +
+					IF(t.num_replies < 200, t.num_replies / 200, 1) * {int:weight_length} +
+					IF(m.id_msg = t.id_first_msg, 1, 0) * {int:weight_first_msg} +
+					IF(t.is_sticky = 0, 0, 1) * {int:weight_sticky}
+				) / {int:weight_total) AS relevance
+			FROM {db_prefix}messages AS m
+				INNER JOIN {db_prefix}topics AS t ON (t.id_topic = m.id_topic)
+			WHERE m.id_msg = {int:newMessage}',
+			array(
+				'newMessage' => $msgOptions['id'],
+				'weight_age' => $weight['age'],
+				'weight_length' => $weight['length'],
+				'weight_first_msg' => $weight['first_message'],
+				'weight_sticky' => $weight['sticky'],
+				'weight_total' => $weight_total,
+			)
+		);
+		$tempMessage = $smcFunc['db_fetch_assoc']($request);
+		$smcFunc['db_free_result']($request);
+
+		$insertValues = array(
+			'id_msg' => $tempMessage['id_msg'],
+			'id_topic' => $tempMessage['id_topic'],
+			'id_board' => $tempMessage['id_board'],
+			'id_member' => $tempMessage['id_member'],
+			'poster_time' => $tempMessage['poster_time'],
+			'body' => '"' . $tempMessage['body'] . '"',
+			'subject' => '"' . $tempMessage['subject'] . '"',
+			'num_replies' => $tempMessage['num_replies'],
+			'relevance' => $tempMessage['relevance'],
+		);
+
+		// The insert query, use replace to make sure we don't get duplicates.
+		$query = '
+			REPLACE INTO smf_index (' . implode(', ', array_keys($insertValues)) . ')
+			VALUES (' . implode(', ', array_values($insertValues)) . ')';
+
+		// Execute the search query.
+		$request = $this->dbfunc_query($query, $mySphinx);
+
+		// Can a connection to the daemon be made?
+		if ($request === false)
+		{
+			// Just log the error.
+			if ($this->dbfunc_error($mySphinx))
+				log_error($this->dbfunc_error($mySphinx));
+
+			// Silently bail out, We can let the reindex cron take care of fixing this.
+			return true;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Callback when a post is modified
+	 * @see modifyPost()
+	 *
+	 * @access public
+	 * @param array $msgOptions An array of post data
+	 * @param array $topicOptions An array of topic data
+	 * @param array $posterOptions An array of info about the person who made this post
+	 * @return void
+	 */
+	public function postModified(array &$msgOptions, array &$topicOptions, array &$posterOptions)
+	{
+		return true;
+		//	!! SphinxQL Does support updating the search index from its QL interface.
+		//	!! Sphinx for SMF does not support this at this time. Code is provided
+		//	!! here as examples/testing purposes.
+
+		// Just call the postCreated as it does a replace.
+		$this->postCreated($msgOptions, $topicOptions, $posterOptions);
+	}
+
+	/**
+	 * Callback when a post is removed, not recycled.
+	 *
+	 * @access public
+	 * @param int $id_msg The ID of the post that was removed
+	 * @return void
+	 */
+	public function postRemoved($id_msg)
+	{
+		return true;
+		//	!! SphinxQL Does support updating the search index from its QL interface.
+		//	!! Sphinx for SMF does not support this at this time. Code is provided
+		//	!! here as examples/testing purposes.
+
+		global $smcFunc, $modSettings;
+
+		// Create an instance of the sphinx client.
+		$mySphinx = $this->dbfunc_connect();
+
+		// SMF only calls this search API when we delete, not recycle. So this will always be a remove.
+		$query = '
+			DELETE FROM smf_index
+			WHERE id_msg = ' . $id_msg;
+
+		// Execute the search query.
+		$request = $this->dbfunc_query($query, $mySphinx);
+
+		// Can a connection to the daemon be made?
+		if ($request === false)
+		{
+			// Just log the error.
+			if ($this->dbfunc_error($mySphinx))
+				log_error($this->dbfunc_error($mySphinx));
+
+			// Silently bail out, We can let the reindex cron take care of fixing this.
+			return true;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Callback when a topic is removed
+	 *
+	 * @access public
+	 * @param array $topics The ID(s) of the removed topic(s)
+	 * @return void
+	 */
+	public function topicsRemoved(array $topics)
+	{
+		return true;
+		//	!! SphinxQL Does support updating the search index from its QL interface.
+		//	!! Sphinx for SMF does not support this at this time. Code is provided
+		//	!! here as examples/testing purposes.
+
+		global $smcFunc, $modSettings;
+
+		// Create an instance of the sphinx client.
+		$mySphinx = $this->dbfunc_connect();
+
+		// Execute the search query.
+		$request = $this->dbfunc_query($query, $mySphinx);
+
+		// Can a connection to the daemon be made?
+		if ($request === false)
+		{
+			// Just log the error.
+			if ($this->dbfunc_error($mySphinx))
+				log_error($this->dbfunc_error($mySphinx));
+
+			// Silently bail out, We can let the reindex cron take care of fixing this.
+			return true;
+		}
+
+		return true;
+	}
+
+	/**
+	 * Callback when a topic is moved
+	 *
+	 * @access public
+	 * @param array $topics The ID(s) of the moved topic(s)
+	 * @param int $board_to The board that the topics were moved to
+	 * @return void
+	 */
+	public function topicsMoved(array $topics, $board_to)
+	{
+		return ture;
+		//	!! SphinxQL Does support updating the search index from its QL interface.
+		//	!! Sphinx for SMF does not support this at this time. Code is provided
+		//	!! here as examples/testing purposes.
+
+		global $smcFunc, $modSettings;
+
+		// Create an instance of the sphinx client.
+		$mySphinx = $this->dbfunc_connect();
+
+		// SMF only calls this search API when we delete, not recycle. So this will always be a remove.
+		$query = '
+			UPDATE smf_index
+			SET id_board = ' . $board_to . '
+			WHERE id_topic IN (' . implode(', ', $topics) . ')';
+
+		// Execute the search query.
+		$request = $this->dbfunc_query($query, $mySphinx);
+
+		// Can a connection to the daemon be made?
+		if ($request === false)
+		{
+			// Just log the error.
+			if ($this->dbfunc_error($mySphinx))
+				log_error($this->dbfunc_error($mySphinx));
+
+			// Silently bail out, We can let the reindex cron take care of fixing this.
+			return true;
+		}
+
+		return true;
+	}
+}