This commit is deep refactoring of AQO machinery and machine learning core.

Many weaknesses was removed. Comments added.
At this moment we know about possible problems with clauses and selectivities collection algorithm.
We need to extract AQO knowledge base from database to file storage. Anyway we need to get rid of
disk access during prediction and learning procedures.

Author: danolivo

aqo.c

@@ -14,6 +14,7 @@ static const struct config_enum_entry format_options[] = {
 	{"forced", AQO_MODE_FORCED, false},
 	{"controlled", AQO_MODE_CONTROLLED, false},
 	{"learn", AQO_MODE_LEARN, false},
+	{"fixed", AQO_MODE_FIXED, false},
 	{"disabled", AQO_MODE_DISABLED, false},
 	{NULL, 0, false}
 };
@@ -28,11 +29,24 @@ int			auto_tuning_infinite_loop = 8;
 /* stat_size > infinite_loop + window_size + 3 is required for auto_tuning*/
 
 /* Machine learning parameters */
-double		object_selection_prediction_threshold = 0.3;
-double		object_selection_object_threshold = 0.1;
-double		learning_rate = 1e-1;
+
+/*
+ * Defines where we do not perform learning procedure
+ */
+const double	object_selection_prediction_threshold = 0.3;
+
+/*
+ * This parameter tell us that the new learning sample object has very small
+ * distance from one whose features stored in matrix already.
+ * In this case we will not to add new line in matrix, but will modify this
+ * nearest neighbor features and cardinality with linear smoothing by
+ * learning_rate coefficient.
+ */
+const double	object_selection_threshold = 0.1;
+const double	learning_rate = 1e-1;
+
+/* The number of nearest neighbors which will be chosen for ML-operations */
 int			aqo_k = 3;
-int			aqo_K = 30;
 double		log_selectivity_lower_bound = -30;
 
 /*
@@ -70,7 +84,7 @@ _PG_init(void)
 							 &aqo_mode,
 							 AQO_MODE_CONTROLLED,
 							 format_options,
-							 PGC_SUSET,
+							 PGC_USERSET,
 							 0,
 							 NULL,
 							 NULL,
@@ -83,7 +97,7 @@ _PG_init(void)
 	prev_ExecutorStart_hook						= ExecutorStart_hook;
 	ExecutorStart_hook							= aqo_ExecutorStart;
 	prev_ExecutorEnd_hook						= ExecutorEnd_hook;
-	ExecutorEnd_hook							= learn_query_stat;
+	ExecutorEnd_hook							= aqo_ExecutorEnd;
 	prev_set_baserel_rows_estimate_hook			= set_baserel_rows_estimate_hook;
 	set_baserel_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
 	prev_get_parameterized_baserel_size_hook	= get_parameterized_baserel_size_hook;
@@ -96,6 +110,7 @@ _PG_init(void)
 	copy_generic_path_info_hook					= aqo_copy_generic_path_info;
 	prev_ExplainOnePlan_hook					= ExplainOnePlan_hook;
 	ExplainOnePlan_hook							= print_into_explain;
+	parampathinfo_postinit_hook					= ppi_hook;
 
 	init_deactivated_queries_storage();
 	AQOMemoryContext = AllocSetContextCreate(TopMemoryContext, "AQOMemoryContext", ALLOCSET_DEFAULT_SIZES);

aqo.h

@@ -133,6 +133,7 @@
 #include "executor/execdesc.h"
 #include "nodes/makefuncs.h"
 #include "nodes/nodeFuncs.h"
+#include "optimizer/pathnode.h"
 #include "optimizer/planmain.h"
 #include "optimizer/planner.h"
 #include "optimizer/cost.h"
@@ -144,7 +145,6 @@
 #include "utils/hsearch.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
-#include "utils/tqual.h"
 #include "utils/fmgroids.h"
 #include "utils/snapmgr.h"
 
@@ -165,11 +165,19 @@ typedef enum
 	AQO_MODE_CONTROLLED,
 	/* Creates new feature space for each query type without auto-tuning */
 	AQO_MODE_LEARN,
+	/* Use only current AQO estimations, without learning or tuning */
+	AQO_MODE_FIXED,
 	/* Aqo is disabled for all queries */
 	AQO_MODE_DISABLED,
 }	AQO_MODE;
 extern int	aqo_mode;
 
+/*
+ * It is mostly needed for auto tuning of query. with auto tuning mode aqo
+ * checks stability of last executions of the query, bad influence of strong
+ * cardinality estimation on query execution (planner bug?) and so on.
+ * It can induce aqo to suppress machine learning for this query.
+ */
 typedef struct
 {
 	double	   *execution_time_with_aqo;
@@ -178,12 +186,14 @@ typedef struct
 	double	   *planning_time_without_aqo;
 	double	   *cardinality_error_with_aqo;
 	double	   *cardinality_error_without_aqo;
+
 	int			execution_time_with_aqo_size;
 	int			execution_time_without_aqo_size;
 	int			planning_time_with_aqo_size;
 	int			planning_time_without_aqo_size;
 	int			cardinality_error_with_aqo_size;
 	int			cardinality_error_without_aqo_size;
+
 	int64		executions_with_aqo;
 	int64		executions_without_aqo;
 }	QueryStat;
@@ -205,6 +215,9 @@ typedef struct QueryContextData
 	double		query_planning_time;
 } QueryContextData;
 
+extern double predicted_ppi_rows;
+extern double fss_ppi_hash;
+
 /* Parameters of autotuning */
 extern int	aqo_stat_size;
 extern int	auto_tuning_window_size;
@@ -213,11 +226,12 @@ extern int	auto_tuning_max_iterations;
 extern int	auto_tuning_infinite_loop;
 
 /* Machine learning parameters */
-extern double object_selection_prediction_threshold;
-extern double object_selection_object_threshold;
-extern double learning_rate;
+#define	aqo_K	(30)
+
+extern const double object_selection_prediction_threshold;
+extern const double object_selection_threshold;
+extern const double learning_rate;
 extern int	aqo_k;
-extern int	aqo_K;
 extern double log_selectivity_lower_bound;
 
 /* Parameters for current query */
@@ -244,11 +258,12 @@ extern		copy_generic_path_info_hook_type
 			prev_copy_generic_path_info_hook;
 extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
 
+extern void ppi_hook(ParamPathInfo *ppi);
 
 /* Hash functions */
 int			get_query_hash(Query *parse, const char *query_text);
-void get_fss_for_object(List *clauselist, List *selectivities, List *relidslist,
-				   int *nfeatures, int *fss_hash, double **features);
+extern int get_fss_for_object(List *clauselist, List *selectivities,
+						List *relidslist, int *nfeatures, double **features);
 void		get_eclasses(List *clauselist, int *nargs, int **args_hash, int **eclass_hash);
 int			get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash);
 
@@ -264,9 +279,8 @@ bool update_query(int query_hash, bool learn_aqo, bool use_aqo,
 bool		add_query_text(int query_hash, const char *query_text);
 bool load_fss(int fss_hash, int ncols,
 		 double **matrix, double *targets, int *rows);
-bool update_fss(int fss_hash, int nrows, int ncols,
-		   double **matrix, double *targets,
-		   int old_nrows, List *changed_rows);
+extern bool update_fss(int fss_hash, int nrows, int ncols,
+					   double **matrix, double *targets);
 QueryStat  *get_aqo_stat(int query_hash);
 void		update_aqo_stat(int query_hash, QueryStat * stat);
 void		init_deactivated_queries_storage(void);
@@ -288,7 +302,7 @@ void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into,
 void		disable_aqo_for_query(void);
 
 /* Cardinality estimation hooks */
-void		aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel);
+extern void aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel);
 double aqo_get_parameterized_baserel_size(PlannerInfo *root,
 								   RelOptInfo *rel,
 								   List *param_clauses);
@@ -314,22 +328,21 @@ List	   *get_list_of_relids(PlannerInfo *root, Relids relids);
 List	   *get_path_clauses(Path *path, PlannerInfo *root, List **selectivities);
 
 /* Cardinality estimation */
-double predict_for_relation(List *restrict_clauses,
-					 List *selectivities,
-					 List *relids);
+double predict_for_relation(List *restrict_clauses, List *selectivities,
+					 List *relids, int *fss_hash);
 
 /* Query execution statistics collecting hooks */
 void		aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);
 void		aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src);
-void		learn_query_stat(QueryDesc *queryDesc);
+void		aqo_ExecutorEnd(QueryDesc *queryDesc);
 
 /* Machine learning techniques */
-double OkNNr_predict(int matrix_rows, int matrix_cols,
-			  double **matrix, double *targets,
-			  double *nw_features);
-List *OkNNr_learn(int matrix_rows, int matrix_cols,
+extern double OkNNr_predict(int nrows, int ncols,
+							double **matrix, const double *targets,
+							double *features);
+extern int OkNNr_learn(int matrix_rows, int matrix_cols,
 			double **matrix, double *targets,
-			double *nw_features, double nw_target);
+			double *features, double target);
 
 /* Automatic query tuning */
 void		automatical_query_tuning(int query_hash, QueryStat * stat);

auto_tuning.c

@@ -168,7 +168,8 @@ automatical_query_tuning(int query_hash, QueryStat * stat)
 	}
 
 	if (num_iterations <= auto_tuning_max_iterations || p_use > 0.5)
-		update_query(query_hash, query_context.learn_aqo, query_context.use_aqo, query_context.fspace_hash, true);
+		update_query(query_hash, query_context.learn_aqo, query_context.use_aqo,
+											query_context.fspace_hash, true);
 	else
 		update_query(query_hash, false, false, query_context.fspace_hash, false);
 }

cardinality_estimation.c

@@ -13,38 +13,42 @@
  * General method for prediction the cardinality of given relation.
  */
 double
-predict_for_relation(List *restrict_clauses, List *selectivities, List *relids)
+predict_for_relation(List *restrict_clauses, List *selectivities, List *relids, int *fss_hash)
 {
-	int			nfeatures;
-	int			fss_hash;
-	double	  **matrix;
-	double	   *target;
-	double	   *features;
-	double		result;
-	int			rows;
-	int			i;
+	int		nfeatures;
+	double	*matrix[aqo_K];
+	double	targets[aqo_K];
+	double	*features;
+	double	result;
+	int		rows;
+	int		i;
 
-	get_fss_for_object(restrict_clauses, selectivities, relids,
-					   &nfeatures, &fss_hash, &features);
+	*fss_hash = get_fss_for_object(restrict_clauses, selectivities, relids,
+														&nfeatures, &features);
 
-	matrix = palloc(sizeof(*matrix) * aqo_K);
-	for (i = 0; i < aqo_K; ++i)
-		matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
-	target = palloc0(sizeof(*target) * aqo_K);
+	if (nfeatures > 0)
+		for (i = 0; i < aqo_K; ++i)
+			matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
 
-	if (load_fss(fss_hash, nfeatures, matrix, target, &rows))
-		result = OkNNr_predict(rows, nfeatures, matrix, target, features);
+	if (load_fss(*fss_hash, nfeatures, matrix, targets, &rows))
+		result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
 	else
+	{
+		/*
+		 * Due to planning optimizer tries to build many alternate paths. Many
+		 * of these not used in final query execution path. Consequently, only
+		 * small part of paths was used for AQO learning and fetch into the AQO
+		 * knowledge base.
+		 */
 		result = -1;
+	}
 
 	pfree(features);
-	for (i = 0; i < aqo_K; ++i)
-		pfree(matrix[i]);
-	pfree(matrix);
-	pfree(target);
-	list_free_deep(selectivities);
-	list_free(restrict_clauses);
-	list_free(relids);
+	if (nfeatures > 0)
+	{
+		for (i = 0; i < aqo_K; ++i)
+			pfree(matrix[i]);
+	}
 
 	if (result < 0)
 		return -1;