From 6da8b2349c6cb792e0b4d7291b05c25259d95b91 Mon Sep 17 00:00:00 2001 From: Damien Regad Date: Fri, 8 Mar 2013 12:37:23 +0100 Subject: [PATCH] Full rewrite the Move Attachments admin page The old move_db2disk.php program was completely outdated and did not work properly. This commit introduces completely new page, with dissociated page to select which attachments to move, and action page to actually move the files from DB to disk Fixes #15496 --- admin/move_attachments.php | 218 ++++++++++++++++++++++++++++++++ admin/move_attachments_page.php | 160 +++++++++++++++++++++++ admin/move_db2disk.php | 200 ----------------------------- admin/system_utils.php | 4 +- core/file_api.php | 2 + core/project_api.php | 24 ++++ 6 files changed, 406 insertions(+), 202 deletions(-) create mode 100644 admin/move_attachments.php create mode 100644 admin/move_attachments_page.php delete mode 100644 admin/move_db2disk.php diff --git a/admin/move_attachments.php b/admin/move_attachments.php new file mode 100644 index 0000000000..a79a6bc151 --- /dev/null +++ b/admin/move_attachments.php @@ -0,0 +1,218 @@ +. + +/** + * This upgrade moves attachments from the database to the disk + * @package MantisBT + * @copyright Copyright (C) 2000 - 2002 Kenzaburo Ito - kenito@300baud.org + * @copyright Copyright (C) 2002 - 2013 MantisBT Team - mantisbt-dev@lists.sourceforge.net + * @link http://www.mantisbt.org + */ + + +/** + * MantisBT Core API's + */ +require_once( dirname( dirname( __FILE__ ) ) . DIRECTORY_SEPARATOR . 'core.php' ); + +form_security_validate( 'move_attachments_project_select' ); + +access_ensure_global_level( config_get_global( 'admin_site_threshold' ) ); + + +$f_file_type = gpc_get( 'type' ); +$f_projects_to_disk = gpc_get( 'to_disk', null ); + + +/** + * Moves attachments from the specified list of projects from database to disk + * @param string $p_type Attachment type ('bug' or 'project') + * @param array $p_projects List of projects to process + * @return array summary of moves per project + */ +function move_attachments_to_disk( $p_type, $p_projects ) { + if( empty( $p_projects ) ) { + return array(); + } + + # Build the SQL query based on attachment type + $t_file_table = db_get_table( "mantis_${p_type}_file_table" ); + switch( $p_type ) { + case 'project': + + $t_query = "SELECT f.* + FROM $t_file_table f + WHERE content <> '' + AND f.project_id = " . db_param() . " + ORDER BY f.filename"; + break; + + case 'bug': + $t_bug_table = db_get_table( 'mantis_bug_table' ); + + $t_query = "SELECT f.* + FROM $t_file_table f + JOIN $t_bug_table b ON b.id = f.bug_id + WHERE content <> '' + AND b.project_id = " . db_param() . " + ORDER BY f.bug_id, f.filename"; + break; + } + + # Process projects list + foreach( $p_projects as $t_project ) { + # Retrieve attachments for the project + $t_result = db_query_bound( $t_query, array( $t_project ) ); + + # Project upload path + $t_upload_path = project_get_upload_path( $t_project ); + if( is_blank( $t_upload_path ) + || !file_exists( $t_upload_path ) + || !is_dir( $t_upload_path ) + || !is_writable( $t_upload_path ) + ) { + # Invalid path + $t_failures = db_num_rows( $t_result ); + $t_data = "ERROR: Upload path '$t_upload_path' does not exist or is not writable"; + } else { + # Process attachments + $t_failures = 0; + $t_data = array(); + + if( $p_type == 'project' ) { + $t_seed = config_get( 'document_files_prefix', null, ALL_USERS, $t_project ) . $t_project; + } + + while( $t_row = db_fetch_array( $t_result ) ) { + if( $p_type == 'bug' ) { + $t_seed = $t_row['bug_id'] . $t_row['filename']; + } + + $t_filename = $t_upload_path . file_generate_unique_name( $t_seed, $t_upload_path ); + + # write file to disk + if( file_put_contents( $t_filename, $t_row['content'] ) ) { + # successful, update database + # @todo do we want to check the size of data transfer matches here? + $t_update_query = "UPDATE $t_file_table + SET diskfile = " . db_param() . ", + folder = " . db_param() . ", + content = '' + WHERE id = " . db_param(); + $t_update_result = db_query_bound( + $t_update_query, + array( $t_filename, $t_upload_path, $t_row['id'] ) + ); + + if( !$t_update_result ) { + $t_status = 'Database update failed'; + $t_failures++; + } else { + $t_status = "Moved to '$t_filename'"; + } + } else { + $t_status = "Copy to '$t_file_name' failed"; + $t_failures++; + } + + # Add the file and status to the list of processed attachments + $t_file = array( + 'id' => $t_row['id'], + 'filename' => $t_row['filename'], + 'status' => $t_status, + ); + if( $p_type == 'bug' ) { + $t_file['bug_id'] = $t_row['bug_id']; + } + $t_data[] = $t_file; + } + } + + $t_moved[] = array( + 'name' => project_get_name( $t_project ), + 'path' => $t_upload_path, + 'rows' => db_num_rows( $t_result ), + 'failed' => $t_failures, + 'data' => $t_data, + ); + + } + return $t_moved; +} + + +$t_moved = move_attachments_to_disk( $f_file_type, $f_projects_to_disk ); + +form_security_purge( 'move_attachments_project_select' ); + +$t_redirect_url = 'system_utils.php'; + +# Page header, menu +html_page_top( + 'MantisBT Administration - Moving Attachments', + empty( $t_result ) ? $t_redirect_url : null +); + +?> + +
+ +Nothing to do.

\n"; +} else { + foreach( $t_moved as $t_row ) { + printf( + "

Project '%s' : %d attachments %s.

\n", + $t_row['name'], + $t_row['rows'], + ( 0 == $t_row['failed'] + ? 'moved successfully' + : 'to move, ' . $t_row['failed'] . ' failures') + ); + + if( is_array( $t_row['data'] ) ) { + # Display details of moved attachments + echo '
', "\n", + '', + $f_file_type == 'bug' ? '' : '', + '', + ''; + foreach( $t_row['data'] as $t_data ) { + echo ''; + if( $f_file_type == 'bug' ) { + printf( '', bug_format_id( $t_data['bug_id'] ) ); + } + printf( '' . "\n", + $t_data['id'], + $t_data['filename'], + $t_data['status'] + ); + } + echo '
Bug IDFileFilenameStatus
%s%s%s%s

'; + } else { + # No data rows - display error message + echo '

' . $t_row['data'] . '

'; + } + echo '
'; + } +} + +print_bracket_link( $t_redirect_url, 'Back to System Utilities' ); + +html_page_bottom(); diff --git a/admin/move_attachments_page.php b/admin/move_attachments_page.php new file mode 100644 index 0000000000..c199c87896 --- /dev/null +++ b/admin/move_attachments_page.php @@ -0,0 +1,160 @@ +. + +/** + * This upgrade moves attachments from the database to the disk + * + * @package MantisBT + * @copyright Copyright (C) 2000 - 2002 Kenzaburo Ito - kenito@300baud.org + * @copyright Copyright (C) 2002 - 2013 MantisBT Team - mantisbt-dev@lists.sourceforge.net + * @link http://www.mantisbt.org + */ + + +/** + * MantisBT Core API's + */ +require_once( dirname( dirname( __FILE__ ) ) . DIRECTORY_SEPARATOR . 'core.php' ); + +access_ensure_global_level( config_get_global( 'admin_site_threshold' ) ); + +# Page header, menu +html_page_top( 'MantisBT Administration - Moving Attachments' ); +echo '

'; +print_bracket_link( helper_mantis_url( 'admin/system_utils.php' ), 'Back to System Utilities' ); +echo '

'; + + +# File type should be 'bug' (default) or 'project' +$f_file_type = gpc_get( 'type', 'bug' ); + +$t_bug_table = db_get_table( 'mantis_bug_table' ); +$t_project_table = db_get_table( 'mantis_project_table' ); + +switch( $f_file_type ) { + case 'project': + $t_type = 'Project Files'; + $t_file_table = db_get_table( 'mantis_project_file_table' ); + $t_query = "SELECT p.id, p.name, COUNT(f.id) disk + FROM $t_file_table f + LEFT JOIN $t_project_table p ON p.id = f.project_id + WHERE content <> '' + GROUP BY p.id, p.name + ORDER BY p.name"; + break; + + case 'bug': + default: + $t_type = 'Attachments'; + $t_file_table = db_get_table( 'mantis_bug_file_table' ); + $t_query = "SELECT p.id, p.name, COUNT(f.id) disk + FROM $t_file_table f + JOIN $t_bug_table b ON b.id = f.bug_id + JOIN $t_project_table p ON p.id = b.project_id + WHERE content <> '' + GROUP BY p.id, p.name + ORDER BY p.name"; + break; +} + +# Move to disk: projects having non-empty attachments in the DB +$t_result = db_query_bound( $t_query ); + +# Build list, excluding projects having upload method other than DISK +$t_projects = array(); +while( $t_row = db_fetch_array( $t_result ) ) { + $t_project_id = (int) $t_row['id']; + $t_upload_method = config_get( 'file_upload_method', null, ALL_USERS, $t_project_id ); + if( $t_upload_method == DISK ) { + $t_projects[$t_project_id] = $t_row; + } +} + +if( count( $t_projects ) == 0 ) { + # Nothing to do + echo '

No attachments need to be moved.

'; +} else { + # Display name for All Projects + if( isset( $t_projects[ALL_PROJECTS] ) ) { + $t_projects[ALL_PROJECTS]['name'] = 'All Projects'; + } + + # Display table of projects for user selection + +?> + +
+ +
+ + + + + + + + + + + + $t_project ) { + echo ''; + printf( + '', + $t_project['name'], + $t_project['disk'], + $t_id + ); + echo "\n"; + } + + if( ON == config_get( 'use_javascript' ) ) { +?> + + + + + + + + +
+ +
Project nameAttachmentsTo Disk
%s%s' + . '
Select All + +
+ +
+ + + + +
+
+ +. - -/** - * This upgrade moves attachments from the database to the disk - * @package MantisBT - * @copyright Copyright (C) 2000 - 2002 Kenzaburo Ito - kenito@300baud.org - * @copyright Copyright (C) 2002 - 2013 MantisBT Team - mantisbt-dev@lists.sourceforge.net - * @link http://www.mantisbt.org - */ -/** - * MantisBT Core API's - */ -require_once( dirname( dirname( __FILE__ ) ) . DIRECTORY_SEPARATOR . 'core.php' ); - -access_ensure_global_level( config_get_global( 'admin_site_threshold' ) ); - -// Move type should be attachment or project. -$f_move_type = gpc_get( 'doc' ); - -function get_prefix( $file_path ) { - if( substr( $file_path, 0, 1 ) == '/' ) { - - # Unix absolute - return ''; - } - if( substr( $file_path, 0, 1 ) == '\\' ) { - - # Windows absolute - return ''; - } - if( substr( $file_path, 1, 2 ) == ':\\' ) { - - # Windows absolute - return ''; - } - return dirname( dirname( __FILE__ ) ) . DIRECTORY_SEPARATOR; -} - -# ------ move file attachments to issues from database to disk -# select non-empty data fields -# match with the project to get the file path -# store the file in the correct folder -# -# Assumptions: only supports storage in local file system (not FTP) -# file paths are set up and working -# -# Re-running this is safe because the data -# is not removed from the database until it is successfully copied. -# -function upgrade_move_att2disk( $p_source ) { - - # $p_source is the string "attachment" or "project" - if( $p_source == 'attachment' ) { - $t_file_table = db_get_table( 'mantis_bug_file_table' ); - $t_bug_label = "Bug"; - } - if( $p_source == 'project' ) { - $t_file_table = db_get_table( 'mantis_project_file_table' ); - $t_bug_label = "Project"; - } - - # check that the source was valid - if( !isset( $t_file_table ) ) { - echo 'Failure: Internal Error: File source not set'; - return; - } - - # check that the destination is set up properly - $t_upload_method = config_get_global( 'file_upload_method' ); - if( $t_upload_method <> DISK ) { - echo 'Failure: Upload Method is not DISK'; - return; - } - - $query = 'SELECT * FROM ' . $t_file_table . ' WHERE content <> \'\''; - - $result = @db_query_bound( $query ); - - if( false == $result ) { - echo '

No attachments need to be moved.

'; - return; - } - - $count = db_num_rows( $result ); - echo '

Found ' . $count . ' attachments to be moved.

'; - $t_failures = 0; - - if( $count > 0 ) { - echo ''; - - # Headings - echo ''; - } - - for( $i = 0;$i < $count;$i++ ) { - $t_row = db_fetch_array( $result ); - - // trace bug id back to project to determine the proper file path - if( $p_source == 'attachment' ) { - $t_project_id = bug_get_field( $t_row['bug_id'], 'project_id' ); - $t_bug_id = $t_row['bug_id']; - } else { - $t_project_id = (int) $t_row['project_id']; - $t_bug_id = $t_project_id; - } - - $t_file_path = project_get_field( $t_project_id, 'file_path' ); - $prefix = get_prefix( $t_file_path ); - $t_real_file_path = $prefix . $t_file_path; - $c_filename = file_clean_name( $t_row['filename'] ); - - printf( "\n'; - } - - echo '
' . $t_bug_label . 'AttachmentStatus
%8d%s", helper_alternate_class(), $t_bug_id, $t_row['filename'] ); - - if( is_blank( $t_real_file_path ) || !file_exists( $t_real_file_path ) || !is_dir( $t_real_file_path ) || !is_writable( $t_real_file_path ) ) { - echo 'Destination ' . $t_real_file_path . ' not writable'; - $t_failures++; - } else { - $t_file_name = $t_real_file_path . $c_filename; - - // write file to disk store after adjusting the path - if( file_put_contents( $t_file_name, $t_row['content'] ) ) { - // successful, update database - /** @todo do we want to check the size of data transfer matches here? */ - $c_new_file_name = $t_file_path . $c_filename; - $query2 = "UPDATE $t_file_table SET diskfile = " . db_param() . ", - folder = " . db_param() . ", content = '' WHERE id = " . db_param(); - $update = @db_query_bound( $query2, Array( $c_new_file_name, $t_file_path, $t_row['id'] ) ); - if( !$update ) { - echo 'database update failed'; - $t_failures++; - } else { - echo 'moved to ' . $t_file_name; - } - } else { - echo 'copy to ' . $t_file_name . ' failed'; - $t_failures++; - } - } - - echo '

' . $count . ' attachments processed, ' . $t_failures . ' failures'; -} - -# --------------------- -# main code -# -if( $f_move_type == 'attachment' ) { - $t_type = 'Attachments'; -} else { - if( $f_move_type == 'project' ) { - $t_type = 'Project Files'; - } else { - echo "

Invalid value '$f_move_type' for parameter 'doc'.

"; - exit; - } -} -?> - - - -MantisBT Administration - Move <?php echo $t_type?> to Disk - - - - - - - - - -
- [ Back to System Utilities ] - [ Refresh view ] - - Move to Disk -
-

- -Completed...

'; -?> - - diff --git a/admin/system_utils.php b/admin/system_utils.php index dbd6474260..559fee15f8 100644 --- a/admin/system_utils.php +++ b/admin/system_utils.php @@ -52,12 +52,12 @@ Move attachments stored in database schema to disk files.
- 'attachment' ) );?> + 'bug' ) );?>
Move project files stored in database schema to disk.
- 'project' ) );?> + 'project' ) );?>
diff --git a/core/file_api.php b/core/file_api.php index 17187edcdf..877ac61017 100644 --- a/core/file_api.php +++ b/core/file_api.php @@ -957,6 +957,8 @@ function file_get_content_type_override( $p_filename ) { * @param int $p_bug_id ID of bug containing attachments to be moved * @param int $p_project_id_to destination project ID for the bug * @return null + * + * @todo: this function can't cope with source or target storing attachments in DB */ function file_move_bug_attachments( $p_bug_id, $p_project_id_to ) { $t_project_id_from = bug_get_field( $p_bug_id, 'project_id' ); diff --git a/core/project_api.php b/core/project_api.php index e5972ae14e..1c80b85f9c 100644 --- a/core/project_api.php +++ b/core/project_api.php @@ -650,6 +650,30 @@ function project_get_all_user_rows( $p_project_id = ALL_PROJECTS, $p_access_leve return array_values( $t_users ); } +/** + * Returns the upload path for the specified project, empty string if + * file_upload_method is DATABASE + * @param int $p_project_id + * @return string upload path + */ +function project_get_upload_path( $p_project_id ) { + + if( DATABASE == config_get( 'file_upload_method', null, ALL_USERS, $p_project_id ) ) { + return ''; + } + + if( $p_project_id == ALL_PROJECTS ) { + $t_path = config_get( 'absolute_path_default_upload_folder', '', ALL_USERS, ALL_PROJECTS ); + } else { + $t_path = project_get_field( $p_project_id, 'file_path' ); + if( is_blank( $t_path ) ) { + $t_path = config_get( 'absolute_path_default_upload_folder', '', ALL_USERS, $p_project_id ); + } + } + + return $t_path; +} + # =================================== # Data Modification # ===================================