Much faster sorting with internal @id attribute + a lot of bug fixes

Compressed document id data can now be read and decompressed in both
directions - ascending and descending. This enables the PMBApi to choose
an appropriate direction when sorting by the internal @id attribute and
simply stop the decompression when a suitable amount of results has been
found. Huge performance improvements ( +1000% ) in selected situations.

Now, there is, however, a downside using the internal @id attribute for
sorting: the exact amount of results is unknown.  Based on the density (
actual matches / decompressed values ) of results PMBApi attempts to
calculate an estimated number, but more often than not it is inaccurate.
Only way to know for sure would be to decompress all the data, like
before. For implementing this feature correctly please see the updated
PMBApi instructions on our website.

This release contains also numerous bug fixes. Updating is certainly
recommended.

Unfortunately the index data format has been also changed, so
re-indexing is required. If you are using the internal @id attribute for
sorting results descending order in your application, please refrain
from replacing the PMBApi.php file with the updated version until you've
had a change to re-index.

README.txt

@@ -7,7 +7,7 @@ Published: 18.07.2017
 copyright 2015-2017 Henri Ruutinen 
 
 email: henri.ruutinen@gmail.com
-website: http://www.hollilla.com/pickmybrain
+website: http://www.pickmybra.in
 
 
 Overview
@@ -47,7 +47,7 @@ under the GPLv3, or purchase a commercial Pickmybrain source license.
 If you're interested in commercial licensing, please see the Pickmybrain 
 web site:
 
-    http://www.hollilla.com/pickmybrain
+    http://www.pickmybra.in
 
 
 Compatibility
@@ -75,7 +75,7 @@ Getting Pickmybrain
 -------------------
 
 The latest version is available from:
-http://www.hollilla.com/pickmybrain
+http://www.pickmybra.in
 
 
 Installation

autostop.php

@@ -13,13 +13,45 @@
 $total_size = 0;
 if ( $enable_exec && $enable_ext_sorting )
 {
+	$not_readable = array();
+
 	for ( $i = 0 ; $i < $dist_threads ; ++$i ) 
 	{
 		$filename = $directory . "/datatemp_".$index_id."_".$i.".txt";
 		if ( is_readable($filename) )
 		{
 			$total_size += filesize($filename);
 		}
+		else
+		{
+			# add into another array for later inspection
+			$not_readable[$i] = 1;
+		}
+	}
+
+	# some files couldn't be read - they were still open in another process
+	# try accessing them now
+	if ( !empty($not_readable) )
+	{
+		foreach ( $not_readable as $i => $error_count ) 
+		{
+			$filename = $directory . "/datatemp_".$index_id."_".$i.".txt";
+
+			while ( $not_readable[$i] < 10 ) 
+			{
+				if ( is_readable($filename) )
+				{
+					# everything is OK now ! 
+					$not_readable[$i] = 10;
+					$total_size += filesize($filename);
+				}
+				else
+				{
+					++$not_readable[$i];
+					usleep(300000); # wait for 300ms 
+				}
+			}
+		}
 	}
 }
 else

db_tokenizer.php

@@ -29,6 +29,9 @@
 	require_once("tokenizer_functions.php");
 }
 
+# set process state on
+SetProcessState($index_id, $process_number, 1);
+
 register_shutdown_function($shutdown_function);
 
 define("CHARSET_REGEXP", "/[^" . $charset . preg_quote(implode("", $blend_chars)) . "]/u");
@@ -591,9 +594,6 @@
 		}
 	}
 
-	# set process state on
-	SetProcessState($index_id, $process_number, 1);
-
 	# fetch data from ( external ) database
 	$mainpdo = $ext_connection->query($main_sql_query);	
 }
@@ -883,6 +883,17 @@
 		$fields[$f_id] = $field;
 	}
 
+	/*
+		INSERT CUSTOM FUNCTIONS HERE
+	*/
+	if ( !empty($approved_custom_functions) )
+	{
+		foreach ( $approved_custom_functions as $f_name ) 
+		{
+
+		}
+	}
+
 	foreach ( $fields as $f_id => $field ) 
 	{
 		$pos = 1;

db_tokenizer_ext.php

@@ -29,6 +29,9 @@
 	require_once("tokenizer_functions.php");
 }
 
+# set process state on
+SetProcessState($index_id, $process_number, 1);	
+
 register_shutdown_function($shutdown_function);
 
 define("CHARSET_REGEXP", "/[^" . $charset . preg_quote(implode("", $blend_chars)) . "]/u");
@@ -545,9 +548,6 @@
 		}
 	}
 
-	# set process state on
-	SetProcessState($index_id, $process_number, 1);
-
 	# fetch data from ( external ) database
 	$mainpdo = $ext_connection->query($main_sql_query);	
 }