From da0b92c4e31454e209c70aefa4a2bec15dae0233 Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Tue, 7 Feb 2012 13:46:40 -0500 Subject: [PATCH 1/4] Linked files with a download link in DQG if IsFile is set to true in parameter_type table --- htdocs/mri/jiv/get_file.php | 28 ++++++++++++++--- htdocs/query_gui_data_download.php | 49 +++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/htdocs/mri/jiv/get_file.php b/htdocs/mri/jiv/get_file.php index f31846200c3..b4b072f86ca 100644 --- a/htdocs/mri/jiv/get_file.php +++ b/htdocs/mri/jiv/get_file.php @@ -29,10 +29,28 @@ if ($blessPass) { $imagePath = $config->getSetting('imagePath'); $imageFile = $imagePath."/".$_GET['file']; - if(!file_exists($imageFile)) exit; - $pf = fopen($imageFile, 'r'); - fpassthru($pf); - fclose($pf); + if(file_exists($imageFile)) { + $pf = fopen($imageFile, 'r'); + fpassthru($pf); + fclose($pf); + } +} else { + // If the file isn't blessed because it's the wrong type or starts with a /, + // it's still available if it's under DownloadPath. We just check to make sure it doesn't + // have ".." in it so that the user can't escape DownloadPath. + $downloadPath = $config->getSetting('DownloadPath'); + $downloadFile = $downloadPath.$_GET['file']; + if(file_exists($downloadFile) && strrpos($_GET['file'], '..') === FALSE) { + header("Content-type: application/x-minc"); + header("Content-Disposition: attachment; filename=\"".basename($_GET['file'])."\""); + + $pf = fopen($downloadFile, 'r'); + fpassthru($pf); + fclose($pf); + exit; + } else { + print "File not found\n"; + } } function blessPassThrough($toBless) { @@ -42,7 +60,7 @@ function blessPassThrough($toBless) { return false; } - if (strrpos($toBless, '.jpg') || strrpos($toBless, '.raw_byte.gz') || strrpos($toBless, '.header')) { + if (strrpos($toBless, '.jpg') || strrpos($toBless, '.raw_byte.gz') || strrpos($toBless, '.header') || strrpos($toBless, '.mnc')) { return true; } diff --git a/htdocs/query_gui_data_download.php b/htdocs/query_gui_data_download.php index e590777afa7..df3cdfb2347 100644 --- a/htdocs/query_gui_data_download.php +++ b/htdocs/query_gui_data_download.php @@ -24,13 +24,15 @@ $config =& NDB_Config::singleton(); $css = $config->getSetting('css'); $studyTitle = $config->getSetting('title'); +$downloadPath= $config->getSetting('DownloadPath'); +print "Download Path: $downloadPath"; $query = ""; if(!empty($_GET['queryID'])) { - // get the query $query = $DB->selectOne("SELECT query FROM query_gui_downloadable_queries WHERE queryID='$_GET[queryID]'"); + $format = $_GET['format']; // run the query $results = array(); @@ -40,11 +42,29 @@ exit; } // add the description as the second row - foreach(array_keys($results[0]) AS $key) { - $desc[$key] = $DB->selectOne("SELECT Description FROM parameter_type WHERE Name='$key'"); + $files_cols = array(); + foreach(array_keys($results[0]) AS $i => $key) { + $parameter_type = $DB->pselectRow("SELECT Description, IsFile FROM parameter_type WHERE Name=:key", array('key' => $key)); + $desc[$key] = $parameter_type['Description']; + if($parameter_type['IsFile']) { + $files_cols[] = $key; + } } $newResults[0] = $desc; + $files = array(); + for($row=0; $row < count($results); $row++) { + foreach($files_cols as $col) { + if(!empty($results[$row][$col]) && ($format=='html' || $format=='download_files')) { + $file = $results[$row][$col]; + if($format == 'html') { + $results[$row][$col] = "$file"; + } else if($format == 'download_files') { + $files[] = $file; + } + } + } + } for($i=0; $i < count($results); $i++) { $newResults[$i+1] = $results[$i]; } @@ -62,7 +82,13 @@ } // build the file buffer - list($buffer, $format) = buildFileBuffer($results, $_GET['format'], $css, $studyTitle); + + $buffer = "Format: $format"; + if($format == 'download_files') { + list($buffer, $format) = buildFileBuffer($files, $_GET['format'], $css, $studyTitle); + } else { + list($buffer, $format) = buildFileBuffer($results, $_GET['format'], $css, $studyTitle); + } $filename = "requested_data.".time().".$format"; // set the headers @@ -84,6 +110,21 @@ function buildFileBuffer($data, $format, $css, $studyTitle) { switch($format) { + case 'download_files': + $buffer = ''; + foreach($data as $file) { + global $downloadPath; + print "downloadPath: $downloadPath"; + chdir($downloadPath); + $file = "./$file"; + //print $file; + if(file_exists($file)) { + `tar rvf /var/www/neurodb/htdocs/foo.tar $file`; + } + $buffer .= "$file\n"; + } + $format = 'txt'; + break; case 'xls': $format = 'xls'; require_once 'Spreadsheet/Excel/Writer.php'; From a24538891a3b827af198c1f5e4d3cce34ce2575a Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Tue, 14 Feb 2012 12:21:29 -0500 Subject: [PATCH 2/4] First pass at download from DQG --- SQL/0000-00-00-schema.sql | 22 ++++ SQL/2012-02-14-Query_GUI_Download.sql | 8 ++ htdocs/downloads | 1 + htdocs/dqgui_step3.js | 11 +- htdocs/query_gui.php | 13 ++- htdocs/query_gui_data_loader.php | 49 ++++++++ .../NDB_Menu_Filter_download_files.class.inc | 109 ++++++++++++++++++ smarty/templates/menu_download_files.tpl | 77 +++++++++++++ userdownloads/.gitignore | 1 + 9 files changed, 288 insertions(+), 3 deletions(-) create mode 100644 SQL/2012-02-14-Query_GUI_Download.sql create mode 120000 htdocs/downloads create mode 100644 php/libraries/NDB_Menu_Filter_download_files.class.inc create mode 100644 smarty/templates/menu_download_files.tpl create mode 100644 userdownloads/.gitignore diff --git a/SQL/0000-00-00-schema.sql b/SQL/0000-00-00-schema.sql index 8d3e93ad7df..3f5cbc1e06f 100644 --- a/SQL/0000-00-00-schema.sql +++ b/SQL/0000-00-00-schema.sql @@ -1097,6 +1097,28 @@ LOCK TABLES `query_gui_stored_queries` WRITE; /*!40000 ALTER TABLE `query_gui_stored_queries` ENABLE KEYS */; UNLOCK TABLES; +-- +-- Table structure for table `query_gui_user_files` +-- +DROP TABLE IF EXISTS `query_gui_user_files`; +CREATE TABLE query_gui_user_files ( + UserFileID integer auto_increment primary key, + UserID integer REFERENCES users(ID), + filename varchar(255), + downloadDate timestamp DEFAULT CURRENT_TIMESTAMP, + md5sum varchar(32), + status enum('ready', 'packaging', 'expired') +); + +-- +-- Dumping data for table `query_gui_user_files` +-- + +LOCK TABLES `query_gui_user_files` WRITE; +/*!40000 ALTER TABLE `query_gui_user_files` DISABLE KEYS */; +/*!40000 ALTER TABLE `query_gui_user_files` ENABLE KEYS */; +UNLOCK TABLES; + -- -- Table structure for table `session` -- diff --git a/SQL/2012-02-14-Query_GUI_Download.sql b/SQL/2012-02-14-Query_GUI_Download.sql new file mode 100644 index 00000000000..68917f0feb3 --- /dev/null +++ b/SQL/2012-02-14-Query_GUI_Download.sql @@ -0,0 +1,8 @@ +CREATE TABLE query_gui_user_files ( + UserFileID integer auto_increment primary key, + UserID integer REFERENCES users(ID), + filename varchar(255), + downloadDate timestamp DEFAULT CURRENT_TIMESTAMP, + md5sum varchar(32), + status enum('ready', 'packaging', 'expired') +); diff --git a/htdocs/downloads b/htdocs/downloads new file mode 120000 index 00000000000..685ffaf65c7 --- /dev/null +++ b/htdocs/downloads @@ -0,0 +1 @@ +../userdownloads/ \ No newline at end of file diff --git a/htdocs/dqgui_step3.js b/htdocs/dqgui_step3.js index e579e7416cd..12056b95005 100644 --- a/htdocs/dqgui_step3.js +++ b/htdocs/dqgui_step3.js @@ -1,5 +1,12 @@ -function executeQuery(){ +function executeQuery(sendTo){ + if(sendTo == 'download') { + var oCbrainField=IFrameDoc.getElementById("download"); + oCbrainField.value="execute"; + var m = document.getElementById("message"); + m.innerHTML = "Packaging your files. They'll be ready to download here in a few minutes."; + } + var oQueryField=IFrameDoc.getElementById("queryData"); var oModeField=IFrameDoc.getElementById("mode"); oModeField.value="executeQuery"; @@ -65,4 +72,4 @@ function querySaved(){ function doNothing(){ return true; -} \ No newline at end of file +} diff --git a/htdocs/query_gui.php b/htdocs/query_gui.php index f13c41ccfbc..336023b2a6d 100644 --- a/htdocs/query_gui.php +++ b/htdocs/query_gui.php @@ -164,7 +164,18 @@ - + + +hasPermission('download_files')) { +?> +  + + +

diff --git a/htdocs/query_gui_data_loader.php b/htdocs/query_gui_data_loader.php index d31955a4b35..0972efd6ec8 100644 --- a/htdocs/query_gui_data_loader.php +++ b/htdocs/query_gui_data_loader.php @@ -285,6 +285,10 @@ // reference id, and send that to a popup $DB->insert("query_gui_downloadable_queries", array('query'=>$query)); $queryID = $DB->lastInsertID; + $download = $_POST['download']; + if($download === 'execute') { + SendFilesToPackage($querySegments['selected_fields'], $query); + } $script .= "window.open('query_gui_data_download.php?queryID=$queryID&format=$_REQUEST[outputFormat]', 'query_download_window');"; @@ -293,6 +297,50 @@ } +function getFiles($cols, $query, $timestamp, $prefix="cbrain") { + $DB = Database::singleton(); + $fileCols= array(); + $lines=explode("\n",trim($cols)); + foreach($lines as $field) { + $fileCol = $DB->pselectRow("SELECT IsFile, Name FROM parameter_type WHERE ParameterTypeID=:ptid", array("ptid" => $field)); + if($fileCol['IsFile'] == true) { + $fileCols[] = ParameterTypeIDToFieldName($field); + } + } + + // Strip off everything before from from the query that was generated above, then just re-add the file fields + // as determined above + $new_query = "SELECT " . join(',', $fileCols) . " " . substr($query, strpos($query, "FROM")); + $DB->select($new_query, $results); + $input = ""; + $fp = fopen("/tmp/$prefix.$timestamp.txt", 'w'); + foreach ($results as $row) { + foreach ($row as $key => $val) { + if(!empty($val)) { + fwrite($fp, $val . "\n"); + $input .= $val . "\n"; + } + } + } + fclose($fp); + return "/tmp/$prefix.$timestamp.txt"; +} + +function SendFilesToPackage($cols, $query) { + $DB = Database::singleton(); + $user = User::singleton(); + $timestamp = time(); + $filelist = getFiles($cols,$query,$timestamp, "download"); + $user = User::singleton(); + chdir("../tools"); + $cmd = sprintf("cat $filelist | ./%s %s.$timestamp > %s 2>&1 & echo $! >> %s", "package_files.sh", $user->getUsername(), "../logs/download.$timestamp", $filelist); + exec($cmd); + $cmd = "rm -f $filelist"; + exec($cmd); + $DB->insert("query_gui_user_files", array("UserID" => $user->getData("ID"), + "filename" => $user->getUsername() . ".$timestamp.tar.gz" + )); +} function printArray($array, $indent=""){ while(list($key,$val)=each($array)){ if(is_array($val)){ @@ -427,6 +475,7 @@ function treeToExpression(&$expression, &$tree, &$conditionals, $parentID=null)
+
diff --git a/php/libraries/NDB_Menu_Filter_download_files.class.inc b/php/libraries/NDB_Menu_Filter_download_files.class.inc new file mode 100644 index 00000000000..56965710893 --- /dev/null +++ b/php/libraries/NDB_Menu_Filter_download_files.class.inc @@ -0,0 +1,109 @@ +getMessage()); + } + return $user->hasPermission('download_files'); + } + + function _setupVariables() + { + + $this->query = " FROM query_gui_user_files f WHERE 1=1"; + $this->columns = array('f.filename as File', 'f.downloadDate as QueryTime', 'f.md5sum as MD5', 'f.status as Status'); + $this->order_by = 'downloadDate DESC'; + $this->headers = array('Filename', + 'QueryTime', + 'MD5', + 'Status' + ); + $this->validFilters = array('f.filename', 'f.status'); + $this->formToFilter = array ( + 'Filename' => 'f.filename' + ); + } + + function _setFilterForm() + { + $this->form->addElement('text', 'Filename', 'Filename:', array("size"=>9, "maxlength"=>11)); + } + + function _setDataTableRows($count) + { + $config = NDB_Config::singleton(); + $db = Database::singleton(); + $paths = $config->getSetting('paths'); + $base = $paths['base']; + // print out + $x = 0; + foreach ($this->list as $item) { + //count column + $this->tpl_data['items'][$x][0]['value'] = $x + $count; + + $file = false; + $md5 = false; + + //print out data rows + $i = 1; + foreach ($item as $key => $val) { + $this->tpl_data['items'][$x][$i]['name'] = $key; + switch($key) { + case 'File': $file = $val; break; + case 'MD5': $md5 = $val; break; + case 'Status': + if($file && file_exists("$base/userdownloads/$file")) { + if(!$md5) { + $md5 = md5_file("$base/userdownloads/$file"); + // We're in the Status, MD5 was the previous column + $this->tpl_data['items'][$x][$i-1]['value'] = $md5; + $db->update("query_gui_user_files", array("status" => 'ready', 'md5sum' => $md5), array("filename" => $file)); + + } + $val = 'ready'; + } else { + // If we're in this block, it means the file doesn't exist. + // If there's no MD5 sum calculated yet, the file was never there. So it's packaging + // If there is an MD5 sum but the file doesn't exist, it was deleted from the + // file system for space reasons, and it's "expired" for the user + + if(!$val || $val == 'ready') { + if(!$md5) { + $db->update("query_gui_user_files", array("status" => 'packaging'), array("filename" => $file)); + $val = 'packaging'; + } else { + $db->update("query_gui_user_files", array("status" => 'expired'), array("filename" => $file)); + $val = 'expired'; + + } + } + } + break; + } + + $this->tpl_data['items'][$x][$i]['value'] = $val; + $i++; + } + /* + $this->tpl_data['items'][$x][$i]['name'] = 'Status'; + if($file && file_exists("$base/userdownloads/$file")) { + $this->tpl_data['items'][$x][$i]['value'] = 'Ready for download'; + } + } else { + $this->tpl_data['items'][$x][$i]['value'] = 'Packaging files'; + $db->update("query_gui_user_files", array("status" => 'packaging'), array("filename" => $file)); + } + */ + + $x++; + } + + return true; + } + +} diff --git a/smarty/templates/menu_download_files.tpl b/smarty/templates/menu_download_files.tpl new file mode 100644 index 00000000000..47f72ab8c94 --- /dev/null +++ b/smarty/templates/menu_download_files.tpl @@ -0,0 +1,77 @@ + +
+ + + + + + + + + + + + + + +
Selection Filter
{$form.Filename.label}{$form.Filename.html}  
Actions:  +
+
+ + + + + + + +
+ + + + + {section name=header loop=$headers} + + {/section} + + +{section name=item loop=$items} + + + {section name=piece loop=$items[item]} + {if $items[item][piece].name == 'File'} + + {else} + + {/if} + {/section} + +{sectionelse} + +{/section} + + +
No.{$headers[header].displayName}
+ {* column 4 is the status *} + {if $items[item][4].value == 'ready'} + {$items[item][piece].value} + {else} + {$items[item][piece].value} + {/if} + + + + {$items[item][piece].value} +
Nothing found
+
+ diff --git a/userdownloads/.gitignore b/userdownloads/.gitignore new file mode 100644 index 00000000000..72e8ffc0db8 --- /dev/null +++ b/userdownloads/.gitignore @@ -0,0 +1 @@ +* From ecc5384fbd9680c206e37ff652da77f50ef0a865 Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Thu, 1 Mar 2012 15:38:30 -0500 Subject: [PATCH 3/4] Fixed bug where all users files were showing on download page --- php/libraries/NDB_Menu_Filter_download_files.class.inc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/php/libraries/NDB_Menu_Filter_download_files.class.inc b/php/libraries/NDB_Menu_Filter_download_files.class.inc index 56965710893..521c3e96b01 100644 --- a/php/libraries/NDB_Menu_Filter_download_files.class.inc +++ b/php/libraries/NDB_Menu_Filter_download_files.class.inc @@ -14,8 +14,10 @@ class NDB_Menu_Filter_download_files extends NDB_Menu_Filter function _setupVariables() { + $user =& User::singleton(); + + $this->query = " FROM query_gui_user_files f WHERE UserID=" . $user->getData("ID"); - $this->query = " FROM query_gui_user_files f WHERE 1=1"; $this->columns = array('f.filename as File', 'f.downloadDate as QueryTime', 'f.md5sum as MD5', 'f.status as Status'); $this->order_by = 'downloadDate DESC'; $this->headers = array('Filename', From f79e0055903ba5747e3160d015c1fe3ff881f8b5 Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Fri, 23 Mar 2012 15:07:59 -0400 Subject: [PATCH 4/4] Added missing script --- tools/package_files.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 tools/package_files.sh diff --git a/tools/package_files.sh b/tools/package_files.sh new file mode 100755 index 00000000000..e8b90dd11da --- /dev/null +++ b/tools/package_files.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Reads a list of files from stdin and packages them up into a tar file + +GetConfigOption() { + eval $1=`sed -n -e "/$2/ { s/^[ \t\n]*//; s/[ \t\n]*$//; s#<[/]*$2>##gp }" ../project/config.xml` +} +GetConfigOption SavePath 'UserDownloadFiles' +GetConfigOption DownloadPath 'DownloadPath' +if [ $# -lt 1 ]; then + echo "Usage: $0 destination\n"; + exit 2; +fi; +echo "$SavePath" +echo "$DownloadPath" + +cd $DownloadPath +while read filename; do + tar rfv /tmp/$1.tar ./$filename +done; +# gzip changes the md5sum based on the filename without the -n +# parameter.. even if it's the same data. +# The -n makes it possible to replace duplicate queries with +# sym/hard links +gzip -n /tmp/$1.tar; +mv /tmp/$1.tar.gz $SavePath