From 0ac196a4639d01c627aa6b2886a84f5ee9eeb4b4 Mon Sep 17 00:00:00 2001 From: Paul Richards Date: Sun, 5 Apr 2009 16:54:24 +0100 Subject: [PATCH] Sync my working checkout of misc performance changes/mssql fixes: 1) add some caching around version/categories/projects to speed up projects with a large number of categories/versions etc 2) database_api: improve db_fetch_array when running under pgsql 3) tag_api: fix for mssql+mysql by implementing different query logic :( 4) manage_proj_page: add improvement noted in bug #8850 --- changelog_page.php | 6 ++-- core/category_api.php | 62 ++++++++++++++++++++++++++++++++-- core/database_api.php | 30 +++++++++++++++- core/helper_api.php | 15 +------- core/news_api.php | 8 ++--- core/project_hierarchy_api.php | 20 +++++------ core/summary_api.php | 12 ++++--- core/tag_api.php | 26 +++++++++++--- core/user_api.php | 38 +++++++++++++++------ core/version_api.php | 52 +++++++++++++++++++++++++++- manage_proj_page.php | 2 ++ roadmap_page.php | 6 ++-- summary_page.php | 27 ++------------- 13 files changed, 221 insertions(+), 83 deletions(-) diff --git a/changelog_page.php b/changelog_page.php index becfa198e7..cfd53781de 100644 --- a/changelog_page.php +++ b/changelog_page.php @@ -131,8 +131,10 @@ function print_project_header_changelog ( $p_project_name ) { $t_project_index = 0; + version_cache_array_rows( $t_project_ids ); + category_cache_array_rows_by_project( $t_project_ids ); + foreach( $t_project_ids as $t_project_id ) { - $c_project_id = db_prepare_int( $t_project_id ); $t_project_name = project_get_field( $t_project_id, 'name' ); $t_can_view_private = access_has_project_level( config_get( 'private_bug_threshold' ), $t_project_id ); @@ -178,7 +180,7 @@ function print_project_header_changelog ( $p_project_name ) { $t_issue_parents = array(); $t_issue_handlers = array(); - $t_result = db_query_bound( $query, Array( $c_project_id, $t_version ) ); + $t_result = db_query_bound( $query, Array( $t_project_id, $t_version ) ); while ( $t_row = db_fetch_array( $t_result ) ) { # hide private bugs if user doesn't have access to view them. diff --git a/core/category_api.php b/core/category_api.php index 667a616e7d..169333ad55 100644 --- a/core/category_api.php +++ b/core/category_api.php @@ -335,6 +335,47 @@ function category_sort_rows_by_project( $p_category1, $p_category2 = null ) { return strcasecmp( $p_category1['name'], $p_category2['name'] ); } +$g_cache_category_project = null; + +function category_cache_array_rows_by_project( $p_project_id_array ) { + global $g_category_cache, $g_cache_category_project; + + $c_project_id_array = array(); + + foreach( $p_project_id_array as $t_project_id ) { + if( !isset( $g_cache_category_project[(int) $t_project_id] ) ) { + $c_project_id_array[] = (int) $t_project_id; + $g_cache_category_project[(int) $t_project_id] = array(); + } + } + + if( empty( $c_project_id_array ) ) { + return; + } + + $t_category_table = db_get_table( 'mantis_category_table' ); + $t_project_table = db_get_table( 'mantis_project_table' ); + + $query = "SELECT c.*, p.name AS project_name FROM $t_category_table AS c + LEFT JOIN $t_project_table AS p + ON c.project_id=p.id + WHERE project_id IN ( " . implode( ', ', $c_project_id_array ) . " ) + ORDER BY c.name "; + $result = db_query_bound( $query ); + + $rows = array(); + while( $row = db_fetch_array( $result ) ) { + $g_category_cache[(int) $row['id']] = $row; + + $rows[ (int)$row[ 'project_id' ] ][] = $row['id']; + } + + foreach( $rows as $t_project_id => $t_row ) { + $g_cache_category_project[ (int)$t_project_id ] = $t_row; + } + return; +} + /** * Return all categories for the specified project id. * Obeys project hierarchies and such. @@ -345,8 +386,25 @@ function category_sort_rows_by_project( $p_category1, $p_category2 = null ) { * @access public */ function category_get_all_rows( $p_project_id, $p_inherit = true, $p_sort_by_project = false ) { - global $g_category_cache; - + global $g_category_cache, $g_cache_category_project; + + if( isset( $g_cache_category_project[ (int)$p_project_id ] ) ) { + if( !empty( $g_cache_category_project[ (int)$p_project_id ]) ) { + foreach( $g_cache_category_project[ (int)$p_project_id ] as $t_id ) { + $t_categories[] = category_get_row( $t_id ); + } + + if( $p_sort_by_project ) { + category_sort_rows_by_project( $p_project_id ); + usort( $t_categories, 'category_sort_rows_by_project' ); + category_sort_rows_by_project( null ); + } + return $t_categories; + } else { + return array(); + } + } + project_hierarchy_cache(); $c_project_id = db_prepare_int( $p_project_id ); diff --git a/core/database_api.php b/core/database_api.php index 14530ab5e4..a71dce55c0 100644 --- a/core/database_api.php +++ b/core/database_api.php @@ -209,6 +209,22 @@ function db_is_pgsql() { return false; } +/** + * Checks if the database driver is MS SQL + * @return bool true if postgres + */ +function db_is_mssql() { + $t_db_type = config_get_global( 'db_type' ); + + switch( $t_db_type ) { + case 'mssql': + case 'odbc_mssql': + return true; + } + + return false; +} + /** * Checks if the database driver is DB2 * @return bool true if db2 @@ -435,10 +451,17 @@ function db_fetch_array( &$p_result ) { static $t_array_fields; if ($t_array_result != $p_result) { + // new query $t_array_result = $p_result; $t_array_fields = null; + } else { + if ( $t_array_fields === null ) { + $p_result->MoveNext(); + return $t_row; + } } - + + $t_convert = false; $t_fieldcount = $p_result->FieldCount(); for( $i = 0; $i < $t_fieldcount; $i++ ) { if (isset( $t_array_fields[$i] ) ) { @@ -457,11 +480,16 @@ function db_fetch_array( &$p_result ) { $t_row[$t_field->name] = true; break; } + $t_convert= true; break; default : break; } } + + if ( $t_convert == false ) { + $t_array_fields = null; + } $p_result->MoveNext(); return $t_row; } diff --git a/core/helper_api.php b/core/helper_api.php index 69d39cbafb..5a2b1ff9f7 100644 --- a/core/helper_api.php +++ b/core/helper_api.php @@ -294,20 +294,7 @@ function helper_project_specific_where( $p_project_id, $p_user_id = null ) { $p_user_id = auth_get_current_user_id(); } - if( ALL_PROJECTS == $p_project_id ) { - $t_topprojects = $t_project_ids = user_get_accessible_projects( $p_user_id ); - foreach( $t_topprojects as $t_project ) { - $t_project_ids = array_merge( $t_project_ids, user_get_all_accessible_subprojects( $p_user_id, $t_project ) ); - } - - $t_project_ids = array_unique( $t_project_ids ); - } else { - access_ensure_project_level( VIEWER, $p_project_id ); - $t_project_ids = user_get_all_accessible_subprojects( $p_user_id, $p_project_id ); - array_unshift( $t_project_ids, $p_project_id ); - } - - $t_project_ids = array_map( 'db_prepare_int', $t_project_ids ); + $t_project_ids = user_get_all_accessible_projects( $p_user_id, $p_project_id ); if( 0 == count( $t_project_ids ) ) { $t_project_filter = ' 1<>1'; diff --git a/core/news_api.php b/core/news_api.php index 0f8aad9c38..0306b2f210 100644 --- a/core/news_api.php +++ b/core/news_api.php @@ -196,14 +196,12 @@ function news_get_rows( $p_project_id, $p_sitewide = true ) { $t_news_table = db_get_table( 'mantis_news_table' ); $t_projects = current_user_get_all_accessible_subprojects( $p_project_id ); - $t_projects[] = $p_project_id; + $t_projects[] = (int)$p_project_id; if( $p_sitewide && ALL_PROJECTS != $p_project_id ) { $t_projects[] = ALL_PROJECTS; } - $t_projects = array_map( 'db_prepare_int', $t_projects ); - $query = "SELECT * FROM $t_news_table"; @@ -254,13 +252,11 @@ function news_get_limited_rows( $p_offset, $p_project_id = null ) { $c_offset = db_prepare_int( $p_offset ); $t_projects = current_user_get_all_accessible_subprojects( $p_project_id ); - $t_projects[] = $p_project_id; + $t_projects[] = (int)$p_project_id; if( ALL_PROJECTS != $p_project_id ) { $t_projects[] = ALL_PROJECTS; } - $t_projects = array_map( 'db_prepare_int', $t_projects ); - $t_news_table = db_get_table( 'mantis_news_table' ); $t_news_view_limit = config_get( 'news_view_limit' ); $t_news_view_limit_days = config_get( 'news_view_limit_days' ); diff --git a/core/project_hierarchy_api.php b/core/project_hierarchy_api.php index eb38f4e3e3..f5c40a79f1 100644 --- a/core/project_hierarchy_api.php +++ b/core/project_hierarchy_api.php @@ -134,23 +134,23 @@ function project_hierarchy_cache( $p_show_disabled = false ) { $row['parent_id'] = ALL_PROJECTS; } - if( isset( $g_cache_project_hierarchy[$row['parent_id']] ) ) { - $g_cache_project_hierarchy[$row['parent_id']][] = $row['id']; + if( isset( $g_cache_project_hierarchy[(int)$row['parent_id']] ) ) { + $g_cache_project_hierarchy[(int)$row['parent_id']][] = (int)$row['id']; } else { - $g_cache_project_hierarchy[$row['parent_id']] = array( - $row['id'], + $g_cache_project_hierarchy[(int)$row['parent_id']] = array( + (int)$row['id'], ); } - if( !isset( $g_cache_project_inheritance[$row['id']] ) ) { - $g_cache_project_inheritance[$row['id']] = array(); + if( !isset( $g_cache_project_inheritance[(int)$row['id']] ) ) { + $g_cache_project_inheritance[(int)$row['id']] = array(); } - if( $row['inherit_global'] && !isset( $g_cache_project_inheritance[$row['id']][ALL_PROJECTS] ) ) { - $g_cache_project_inheritance[$row['id']][] = ALL_PROJECTS; + if( $row['inherit_global'] && !isset( $g_cache_project_inheritance[(int)$row['id']][ALL_PROJECTS] ) ) { + $g_cache_project_inheritance[(int)$row['id']][] = ALL_PROJECTS; } - if( $row['inherit_parent'] && !isset( $g_cache_project_inheritance[$row['id']][$row['parent_id']] ) ) { - $g_cache_project_inheritance[$row['id']][] = (int) $row['parent_id']; + if( $row['inherit_parent'] && !isset( $g_cache_project_inheritance[(int)$row['id']][(int)$row['parent_id']] ) ) { + $g_cache_project_inheritance[(int)$row['id']][] = (int) $row['parent_id']; } } } diff --git a/core/summary_api.php b/core/summary_api.php index 0a164d670a..414b0b947e 100644 --- a/core/summary_api.php +++ b/core/summary_api.php @@ -29,7 +29,7 @@ function summary_helper_print_row( $p_label, $p_open, $p_resolved, $p_closed, $p_total ) { printf( '', helper_alternate_class() ); - printf( '%s', string_display( $p_label ) ); + printf( '%s', string_display_line( $p_label ) ); printf( '%s', $p_open ); printf( '%s', $p_resolved ); printf( '%s', $p_closed ); @@ -745,11 +745,13 @@ function summary_print_by_project( $p_projects = null, $p_level = 0, $p_cache = $t_bugs_total = $t_bugs_open + $t_bugs_resolved + $t_bugs_closed; summary_helper_print_row( $t_name, $t_bugs_open, $t_bugs_resolved, $t_bugs_closed, $t_bugs_total ); + + if ( count( project_hierarchy_get_subprojects ( $t_project ) ) > 0 ) { + $t_subprojects = current_user_get_accessible_subprojects( $t_project ); - $t_subprojects = current_user_get_accessible_subprojects( $t_project ); - - if( count( $t_subprojects ) > 0 ) { - summary_print_by_project( $t_subprojects, $p_level + 1, $p_cache ); + if( count( $t_subprojects ) > 0 ) { + summary_print_by_project( $t_subprojects, $p_level + 1, $p_cache ); + } } } } diff --git a/core/tag_api.php b/core/tag_api.php index d49ade890b..dcbf520b7a 100644 --- a/core/tag_api.php +++ b/core/tag_api.php @@ -411,11 +411,27 @@ function tag_get_candidates_for_bug( $p_bug_id ) { $t_params = array(); if ( 0 != $p_bug_id ) { $t_bug_tag_table = db_get_table( 'mantis_bug_tag_table' ); - $query = "SELECT id, name, description FROM $t_tag_table WHERE id IN ( - SELECT t.id FROM $t_tag_table t - LEFT JOIN $t_bug_tag_table b ON t.id=b.tag_id - WHERE b.bug_id IS NULL OR b.bug_id != " . db_param() . - ')'; + + if ( db_is_mssql() ) { + $t_params[] = $p_bug_id; + $query = "SELECT t.id FROM $t_tag_table t + LEFT JOIN $t_bug_tag_table b ON t.id=b.tag_id + WHERE b.bug_id IS NULL OR b.bug_id != " . db_param(); + $result = db_query_bound( $query, $t_params ); + + $t_subquery_results = array(); + + while( $row = db_fetch_array( $result ) ) { + $t_subquery_results[] = (int)$row; + } + $query = "SELECT id, name, description FROM $t_tag_table WHERE id IN ( " . implode( ', ', $t_subquery_results ) . ")"; + } else { + $query = "SELECT id, name, description FROM $t_tag_table WHERE id IN ( + SELECT t.id FROM $t_tag_table t + LEFT JOIN $t_bug_tag_table b ON t.id=b.tag_id + WHERE b.bug_id IS NULL OR b.bug_id != " . db_param() . + ')'; + } $t_params[] = $p_bug_id; } else { $query = 'SELECT id, name, description FROM ' . $t_tag_table; diff --git a/core/user_api.php b/core/user_api.php index b601b7a9fc..6e3129b98d 100644 --- a/core/user_api.php +++ b/core/user_api.php @@ -880,7 +880,7 @@ function user_get_accessible_projects( $p_user_id, $p_show_disabled = false ) { for( $i = 0;$i < $row_count;$i++ ) { $row = db_fetch_array( $result ); - $t_projects[$row['id']] = ( $row['parent_id'] === NULL ) ? 0 : $row['parent_id']; + $t_projects[(int)$row['id']] = ( $row['parent_id'] === NULL ) ? 0 : (int)$row['parent_id']; } # prune out children where the parents are already listed. Make the list @@ -962,34 +962,34 @@ function user_get_accessible_subprojects( $p_user_id, $p_project_id, $p_show_dis for( $i = 0;$i < $row_count;$i++ ) { $row = db_fetch_array( $result ); - if( !isset( $t_projects[$row['parent_id']] ) ) { - $t_projects[$row['parent_id']] = array(); + if( !isset( $t_projects[(int)$row['parent_id']] ) ) { + $t_projects[(int)$row['parent_id']] = array(); } - array_push( $t_projects[$row['parent_id']], $row['id'] ); + array_push( $t_projects[(int)$row['parent_id']], (int)$row['id'] ); } if( auth_get_current_user_id() == $p_user_id ) { $g_user_accessible_subprojects_cache = $t_projects; } - - if( !isset( $t_projects[$p_project_id] ) ) { - $t_projects[$p_project_id] = array(); + + if( !isset( $t_projects[(int)$p_project_id] ) ) { + $t_projects[(int)$p_project_id] = array(); } - return $t_projects[$p_project_id]; + return $t_projects[(int)$p_project_id]; } # -------------------- function user_get_all_accessible_subprojects( $p_user_id, $p_project_id ) { /** @todo (thraxisp) Should all top level projects be a sub-project of ALL_PROJECTS implicitly? * affects how news and some summaries are generated - */ + */ $t_todo = user_get_accessible_subprojects( $p_user_id, $p_project_id ); $t_subprojects = Array(); while( $t_todo ) { - $t_elem = array_shift( $t_todo ); + $t_elem = (int)array_shift( $t_todo ); if( !in_array( $t_elem, $t_subprojects ) ) { array_push( $t_subprojects, $t_elem ); $t_todo = array_merge( $t_todo, user_get_accessible_subprojects( $p_user_id, $t_elem ) ); @@ -999,6 +999,24 @@ function user_get_all_accessible_subprojects( $p_user_id, $p_project_id ) { return $t_subprojects; } +function user_get_all_accessible_projects( $p_user_id, $p_project_id ) { + if( ALL_PROJECTS == $p_project_id ) { + $t_topprojects = $t_project_ids = user_get_accessible_projects( $p_user_id ); + foreach( $t_topprojects as $t_project ) { + $t_project_ids = array_merge( $t_project_ids, user_get_all_accessible_subprojects( $p_user_id, $t_project ) ); + } + + $t_project_ids = array_unique( $t_project_ids ); + } else { + access_ensure_project_level( VIEWER, $p_project_id ); + $t_project_ids = user_get_all_accessible_subprojects( $p_user_id, $p_project_id ); + array_unshift( $t_project_ids, $p_project_id ); + } + + return $t_project_ids; +} + + # -------------------- # return the number of open assigned bugs to a user in a project function user_get_assigned_open_bug_count( $p_user_id, $p_project_id = ALL_PROJECTS ) { diff --git a/core/version_api.php b/core/version_api.php index b0fcc73007..72b9d9d5e3 100644 --- a/core/version_api.php +++ b/core/version_api.php @@ -267,9 +267,59 @@ function version_remove_all( $p_project_id ) { # Data Access # =================================== # -------------------- +$g_cache_versions_project = null; + +function version_cache_array_rows( $p_project_id_array ) { + global $g_cache_versions, $g_cache_versions_project; + + $c_project_id_array = array(); + + foreach( $p_project_id_array as $t_project_id ) { + if( !isset( $g_cache_versions_project[(int) $t_project_id] ) ) { + $c_project_id_array[] = (int) $t_project_id; + $g_cache_versions_project[(int) $t_project_id] = array(); + } + } + + if( empty( $c_project_id_array ) ) { + return; + } + + $t_project_version_table = db_get_table( 'mantis_project_version_table' ); + + $query = "SELECT * + FROM $t_project_version_table + WHERE project_id IN (" . implode( ',', $c_project_id_array ) . ')'; + $result = db_query_bound( $query ); + + $rows = array(); + while( $row = db_fetch_array( $result ) ) { + $row['date_order'] = db_unixtimestamp( $row['date_order'] ); + $g_cache_versions[(int) $row['id']] = $row; + + $rows[ (int)$row[ 'project_id' ] ][] = $row['id']; + } + + foreach( $rows as $t_project_id => $t_row ) { + $g_cache_versions_project[ (int)$t_project_id ] = $t_row; + } + return; +} + # Return all versions for the specified project function version_get_all_rows( $p_project_id, $p_released = null, $p_obsolete = false ) { - global $g_cache_versions; + global $g_cache_versions, $g_cache_versions_project; + + if( isset( $g_cache_versions_project[ (int)$p_project_id ] ) ) { + if( !empty( $g_cache_versions_project[ (int)$p_project_id ]) ) { + foreach( $g_cache_versions_project[ (int)$p_project_id ] as $t_id ) { + $t_versions[] = version_cache_row( $t_id ); + } + return $t_versions; + } else { + return array(); + } + } $c_project_id = db_prepare_int( $p_project_id ); $t_project_version_table = db_get_table( 'mantis_project_version_table' ); diff --git a/manage_proj_page.php b/manage_proj_page.php index 343c03fd25..07f7700a47 100644 --- a/manage_proj_page.php +++ b/manage_proj_page.php @@ -44,6 +44,8 @@ html_page_top2(); print_manage_menu( 'manage_proj_page.php' ); + + project_hierarchy_cache( true ); # Project Menu Form BEGIN ?>
diff --git a/roadmap_page.php b/roadmap_page.php index 0b7b2e1ca9..0d23e47ae4 100644 --- a/roadmap_page.php +++ b/roadmap_page.php @@ -128,8 +128,10 @@ function print_project_header_roadmap( $p_project_name ) { $t_project_index = 0; + version_cache_array_rows( $t_project_ids ); + category_cache_array_rows_by_project( $t_project_ids ); + foreach( $t_project_ids as $t_project_id ) { - $c_project_id = db_prepare_int( $t_project_id ); $t_project_name = project_get_field( $t_project_id, 'name' ); $t_can_view_private = access_has_project_level( config_get( 'private_bug_threshold' ), $t_project_id ); @@ -175,7 +177,7 @@ function print_project_header_roadmap( $p_project_name ) { $t_first_entry = true; - $t_result = db_query_bound( $query, Array( $c_project_id, $c_version ) ); + $t_result = db_query_bound( $query, Array( $t_project_id, $c_version ) ); $t_issue_ids = array(); $t_issue_parents = array(); diff --git a/summary_page.php b/summary_page.php index 0ac65217e9..7dbd490f14 100644 --- a/summary_page.php +++ b/summary_page.php @@ -38,31 +38,8 @@ $t_user_id = auth_get_current_user_id(); - # @@@ giallu: this block of code is duplicated from helper_project_specific_where - # the only diff is the commented line below: can we do better than this ? - if ( ALL_PROJECTS == $f_project_id ) { - $t_topprojects = $t_project_ids = user_get_accessible_projects( $t_user_id ); - foreach ( $t_topprojects as $t_project ) { - $t_project_ids = array_merge( $t_project_ids, user_get_all_accessible_subprojects( $t_user_id, $t_project ) ); - } - - $t_project_ids = array_unique( $t_project_ids ); - } else { - # access_ensure_project_level( VIEWER, $p_project_id ); - $t_project_ids = user_get_all_accessible_subprojects( $t_user_id, $f_project_id ); - array_unshift( $t_project_ids, $f_project_id ); - } - - $t_project_ids = array_map( 'db_prepare_int', $t_project_ids ); - - if ( 0 == count( $t_project_ids ) ) { - $specific_where = ' 1 <> 1'; - } elseif ( 1 == count( $t_project_ids ) ) { - $specific_where = ' project_id=' . $t_project_ids[0]; - } else { - $specific_where = ' project_id IN (' . join( ',', $t_project_ids ) . ')'; - } - # end @@@ block + $t_project_ids = user_get_all_accessible_projects( $t_user_id, $f_project_id); + $specific_where = helper_project_specific_where( $f_project_id, $t_user_id); $t_bug_table = db_get_table( 'mantis_bug_table' ); $t_history_table = db_get_table( 'mantis_bug_history_table' );