Permalink
Browse files

fix commit

  • Loading branch information...
lstein committed Sep 18, 2012
2 parents 6cf8f3f + 80dc1e7 commit 4c1c8e39016a6c9314024e4d59cd70803667da88
Showing with 2,578 additions and 40,832 deletions.
  1. +4 −2 Build.PL
  2. +32 −0 Changes
  3. +7 −0 MANIFEST
  4. +4 −1 MANIFEST.SKIP
  5. +1 −0 TODO
  6. +38 −8 bin/gbrowse_clean.pl
  7. +142 −0 bin/gbrowse_grow_cloud_vol.pl
  8. +758 −0 bin/gbrowse_import_ucsc_db.pl
  9. +9 −0 bin/gbrowse_metadb_config.pl
  10. +12 −12 cgi-bin/gbrowse_details
  11. +3 −3 conf/GBrowse.conf
  12. +2 −2 conf/languages/is.pm
  13. +12 −65 conf/plugins/BatchDumper.pm
  14. +2 −2 conf/plugins/FastaDumper.pm
  15. +7 −67 conf/plugins/SequenceDumper.pm
  16. +12 −20 conf/plugins/TrackFinder.pm
  17. +1 −1 conf/synteny/oryza.synconf.disabled
  18. +99 −0 htdocs/cloud_index.html
  19. +7 −7 htdocs/gbrowse_syn_help.html
  20. +9 −9 htdocs/index.html
  21. +2 −2 htdocs/js/ajax_upload.js
  22. +1 −1 htdocs/js/login.js
  23. +5 −5 htdocs/js/track_configure.js
  24. +102 −0 htdocs/vbox_index.html
  25. +61 −38 install_util/GBrowseInstall.pm
  26. +0 −1 install_util/conf_install.pl
  27. +621 −0 lib/Bio/DB/SeqFeature/Store/Alias.pm
  28. +4 −2 lib/Bio/Graphics/Browser2.pm
  29. +23 −19 lib/Bio/Graphics/Browser2/Action.pm
  30. +4 −1 lib/Bio/Graphics/Browser2/DataBase.pm
  31. +18 −3 lib/Bio/Graphics/Browser2/DataLoader.pm
  32. +4 −8 lib/Bio/Graphics/Browser2/DataLoader/bam.pm
  33. +40 −6 lib/Bio/Graphics/Browser2/DataLoader/bed.pm
  34. +55 −155 lib/Bio/Graphics/Browser2/DataLoader/bigwig.pm
  35. +10 −1 lib/Bio/Graphics/Browser2/DataLoader/sam.pm
  36. +217 −0 lib/Bio/Graphics/Browser2/DataLoader/wig2bigwig.pm
  37. +15 −5 lib/Bio/Graphics/Browser2/DataSource.pm
  38. +2 −5 lib/Bio/Graphics/Browser2/RegionSearch.pm
  39. +44 −26 lib/Bio/Graphics/Browser2/Render.pm
  40. +1 −2 lib/Bio/Graphics/Browser2/Render/HTML.pm
  41. +2 −2 lib/Bio/Graphics/Browser2/Render/TrackConfig.pm
  42. +15 −8 lib/Bio/Graphics/Browser2/RenderPanels.pm
  43. +1 −1 lib/Bio/Graphics/Browser2/Session.pm
  44. +4 −4 lib/Bio/Graphics/Browser2/SubtrackTable.pm
  45. +9 −68 lib/Bio/Graphics/Browser2/TrackDumper.pm
  46. +114 −0 lib/Bio/Graphics/Browser2/TrackDumper/RichSeqMaker.pm
  47. +16 −10 lib/Bio/Graphics/Browser2/UserTracks.pm
  48. +5 −4 lib/Bio/Graphics/Browser2/UserTracks/Database.pm
  49. +10 −2 lib/Bio/Graphics/Karyotype.pm
  50. +1 −1 lib/Legacy/Graphics/Browser/Synteny.pm
  51. +6 −11 lib/Legacy/Graphics/Browser/Util.pm
  52. BIN sample_data/gbrowse_syn/alignments/rice.aln.gz
  53. +2 −19,891 sample_data/gbrowse_syn/rice/rice.gff3
  54. +1 −20,351 sample_data/gbrowse_syn/wild_rice/wild_rice.gff3
  55. +2 −0 t/00.compile.t
View
@@ -18,7 +18,7 @@ my $build = GBrowseInstall->new(
requires => {
'perl' => '5.008',
'Bio::Root::Version' => '1.0069',
- 'Bio::Graphics' => '2.26',
+ 'Bio::Graphics' => '2.28',
'CGI::Session' => '4.02',
'Digest::MD5' => 0,
'ExtUtils::CBuilder' => 0,
@@ -38,7 +38,7 @@ my $build = GBrowseInstall->new(
build_requires => {'Capture::Tiny' => 0,},
recommends => {
'Bio::Das' => 0,
- 'Bio::DB::Sam' => 1.20,
+ 'Bio::DB::Sam' => 1.36,
'Bio::DB::BigFile' => 1.00,
'Crypt::SSLeay' => 0,
'DB_File::Lock' => 0,
@@ -61,9 +61,11 @@ my $build = GBrowseInstall->new(
script_files => [
'bin/gbrowse_clean.pl',
'bin/gbrowse_set_admin_passwd.pl',
+ 'bin/gbrowse_import_ucsc_db.pl',
'bin/gbrowse_create_account.pl',
'bin/gbrowse_change_passwd.pl',
'bin/gbrowse_metadb_config.pl',
+ 'bin/gbrowse_grow_cloud_vol.pl',
'bin/gbrowse_slave',
'bin/gbrowse_syn_load_alignment_database.pl',
'bin/gbrowse_syn_load_alignments_msa.pl',
View
32 Changes
@@ -1,3 +1,35 @@
+2.51
+ * Fix pod documentation for gbrowse_grow_cloud_vol.pl
+ * Volumes added to cloud instance have deleteOnTermination set to true.
+
+2.50
+ * Add gbrowse_import_ucsc_db.pl, which will create a fully functional starter database
+ from a UCSC data source (e.g. 'hg19').
+ * Add VirtualBox and EC2 virtual machine support (see http://gmod.org/wiki/GBrowse2_VMs).
+ * Prompt user before overwriting a customized config file.
+
+2.49
+ * Add new "metadata" option in track stanzas for making subtracks selectable via
+ structured metadata; see http://gmod.org/wiki/Creating_and_Managing_Subtracks_with_GBrowse2
+ for details.
+ * Specify which database to use for search result karyotype display using following
+ syntax:
+ [builtin:karyotype]
+ database = my_database
+ * Fixed error reporting when uploading badly-formatted WIG files.
+ * Do something sensible when user tries to upload a bigwig file.
+ * Some support for UCSC-style uploads with "track type=bigWig bigDataUrl=http://something...".
+
+2.48
+ * Fix track-level genbank and fasta dumping.
+ * Fix (mostly) the batch and sequence dumping plugins.
+
+2.47
+ * Fix bugs in demo mode.
+ * Fix session locking bug; FastCGI performance subjectively improved.
+ * Revert to using inline imagemaps when not running in an accelerated environment.
+ * Require Bio::Graphics version 2.26, so that transparency is supported.
+
2.46
* Support for transparent overlay tracks. These will appear as a user settable formatting option
for any track with subtracks. To make this default for a track set "bump = overlap" and consider
View
@@ -5,6 +5,8 @@ bin/gbrowse_attach_slaves.pl
bin/gbrowse_change_passwd.pl
bin/gbrowse_clean.pl
bin/gbrowse_create_account.pl
+bin/gbrowse_grow_cloud_vol.pl
+bin/gbrowse_import_ucsc_db.pl
bin/gbrowse_metadb_config.pl
bin/gbrowse_netinstall.pl
bin/gbrowse_netinstall2.pl
@@ -185,6 +187,7 @@ docs/README.tutorial
etc/default/gbrowse-slave
etc/init.d/gbrowse-slave
htdocs/annotation_help.html
+htdocs/cloud_index.html
htdocs/css/dropdown/default_theme.css
htdocs/css/dropdown/dropdown.css
htdocs/css/dropdown/helper.css
@@ -535,6 +538,7 @@ htdocs/tutorial/figures/wiggle_density.png
htdocs/tutorial/tutorial.css
htdocs/tutorial/tutorial.html
htdocs/tutorial/tutorial.phyTreeEdit.html
+htdocs/vbox_index.html
INSTALL
install_util/cgi_install.pl
install_util/conf_install.pl
@@ -551,6 +555,7 @@ lib/Bio/DB/GFF/Aggregator/match_gap.pm
lib/Bio/DB/GFF/Aggregator/reftranscript.pm
lib/Bio/DB/GFF/Aggregator/waba_alignment.pm
lib/Bio/DB/GFF/Aggregator/wormbase_gene.pm
+lib/Bio/DB/SeqFeature/Store/Alias.pm
lib/Bio/DB/SeqFeature/Store/BedLoader.pm
lib/Bio/DB/Tagger.pm
lib/Bio/DB/Tagger/mysql.pm
@@ -571,6 +576,7 @@ lib/Bio/Graphics/Browser2/DataLoader/generic.pm
lib/Bio/Graphics/Browser2/DataLoader/gff.pm
lib/Bio/Graphics/Browser2/DataLoader/gff3.pm
lib/Bio/Graphics/Browser2/DataLoader/sam.pm
+lib/Bio/Graphics/Browser2/DataLoader/wig2bigwig.pm
lib/Bio/Graphics/Browser2/DataLoader/wiggle.pm
lib/Bio/Graphics/Browser2/DataSource.pm
lib/Bio/Graphics/Browser2/ExternalData.pm
@@ -601,6 +607,7 @@ lib/Bio/Graphics/Browser2/Session.pm
lib/Bio/Graphics/Browser2/Shellwords.pm
lib/Bio/Graphics/Browser2/SubtrackTable.pm
lib/Bio/Graphics/Browser2/TrackDumper.pm
+lib/Bio/Graphics/Browser2/TrackDumper/RichSeqMaker.pm
lib/Bio/Graphics/Browser2/UserDB.pm
lib/Bio/Graphics/Browser2/UserTracks.pm
lib/Bio/Graphics/Browser2/UserTracks/Database.pm
View
@@ -80,4 +80,7 @@ slave\.pid
^gbrowse_users.*
.*\.new
^META\.
-^nytprof
+^nytprof
+\.out$
+^htdocs/i$
+
View
1 TODO
@@ -1,3 +1,4 @@
+* Specify intervals using unc-13..unc-30 (whatever..whatever)
* Document the REST API; in particular the action=list and action=scan commands.
* Reload globals when GBrowse.conf changes under FastCGI.
* Bring internationalization support up to date.
View
@@ -6,7 +6,7 @@
use Pod::Usage;
use File::Find ();
-use File::Basename 'basename';
+use File::Basename 'basename','dirname';
use Bio::Graphics::Browser2;
use Bio::Graphics::Browser2::Util 'shellwords';
use POSIX 'ENOTEMPTY';
@@ -41,6 +41,7 @@
my $locks_dir = $globals->session_locks;
my $images_dir = $globals->tmpimage_dir;
my $user_dir = $globals->user_dir();
+my $uploads_db = $globals->user_account_db;
my $cache_secs = $globals->time2sec($globals->cache_time);
my $uploads_secs = $globals->time2sec($globals->upload_time);
@@ -53,7 +54,7 @@
my $remember_settings_time = $globals->remember_settings_time;
-logit("Expiring sessions older than $remember_settings_time...");
+logit("Expiring sessions older than $remember_settings_time...\n");
my $dsn = $globals->code_setting(general=>'session driver');
my %dsn = shellwords($globals->code_setting(general=>'session args'));
my $rst = $globals->time2sec($remember_settings_time);
@@ -82,6 +83,10 @@
$rdev,$size,$atime,$mtime,$ctime) = stat($_);
next if $name =~ m!$tmpdir/[^/]+$!; # don't remove toplevel!
+ next if $name eq $tmpdir;
+ next if $name eq $user_dir;
+
+ my $is_userdata = $name =~ m/^$user_dir/;
if (-d _ ) { # attempt to remove directories - will have no effect unless empty
if (rmdir($name)) {
@@ -94,10 +99,10 @@
}
my $secs = $name =~ m/^($cache_dir|$locks_dir|$images_dir)/ ? $cache_secs
- :$name =~ m/^$user_dir/ ? $uploads_secs
+ :$is_userdata ? $uploads_secs
:0;
return unless $secs;
- my $time = $name =~ m/^$user_dir/ ? -A _ : -M _;
+ my $time = $is_userdata ? -A _ : -M _;
my $days = $secs/SECS_PER_DAY;
@@ -112,14 +117,39 @@
# Traverse desired filesystems
-logit("Deleting cache files and directories...");
-File::Find::finddepth( {wanted=>$wanted},
- $tmpdir);
-logit("$directories directories and $files files deleted.\n");
+logit("Deleting cache files and directories older than ".$globals->cache_time." (see GBrowse.conf \"expire cache\" option)...\n");
+File::Find::finddepth( {wanted=>$wanted}, $tmpdir);
+logit("Deleting unaccessed user uploads older than ".$globals->upload_time." (see GBrowse.conf \"expire uploads\" option)...\n");
+File::Find::finddepth( {wanted=>$wanted}, $user_dir);
+logit("Deleted $directories directories and $files files.\n");
+if ($uploads_db) {
+ logit("Cleaning uploads db...\n");
+ clean_uploads();
+}
logit("*** ",scalar localtime,"$0 done ***\n\n");
exit 0;
+sub clean_uploads {
+ eval {require DBI; 1} or return;
+
+ my $db = DBI->connect($globals->user_account_db) or return;
+ my $query = $db->prepare('select uploadsid,data_source,path,trackid from uploads,session where uploads.userid=session.userid') or return;
+ $query->execute or return;
+
+ my %flag_for_deletion;
+ while (my ($uploadsid,$dsn,$path,$trackid) = $query->fetchrow_array) {
+ my $full_path = "$user_dir/$dsn/$uploadsid/$path";
+ next if -e $full_path;
+ $flag_for_deletion{$trackid}++;
+ }
+
+ my @delete = keys %flag_for_deletion or return;
+ my $to_remove = join ',',map {"'$_'"}@delete;
+ $db->do("delete from uploads where trackid in ($to_remove)") or warn $db->errstr;
+ logit("Deleted ".scalar @delete." dangling uploads.\n");
+}
+
sub verbose {
my @mess = @_;
return unless $verbose;
@@ -0,0 +1,142 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+gbrowse_grow_cloud-vol.pl Grow the GBrowse volume by the requested amount
+
+=head1 SYNOPSYS
+
+Grow /opt/gbrowse by another 100 gigabytes
+
+ % gbrowse_grow_cloud_vol.pl 100
+
+=head1 DESCRIPTION
+
+This script grows /opt/gbrowse by the requested number of
+gigabytes. The single argument must be a number between 1 and 1000,
+which indicates the number of GB to grow the volume by (not the new
+size of the volume).
+
+It works by creating a new EBS volume and adding it to the logical
+volume manager (LVM2) for this machine. The filesystem is then
+extended to the desired size.
+
+=head1 COMMAND-LINE OPTIONS
+
+Options can be abbreviated. For example, you can use -a for
+--access_key:
+
+ --access_key EC2 access key
+ --secret_key EC2 secret key
+
+=head1 ENVIRONMENT VARIABLES
+
+The following environment variables are used if the corresponding
+options are not present:
+
+ EC2_ACCESS_KEY your access key
+ EC2_SECRET_KEY your secret key
+
+=head1 SEE ALSO
+
+L<VM::EC2>, L<VM::EC2::Staging::Manager>
+
+=head1 AUTHOR
+
+Lincoln Stein, lincoln.stein@gmail.com
+
+Copyright (c) 2012 Ontario Institute for Cancer Research
+
+This package and its accompanying libraries is free software; you can
+redistribute it and/or modify it under the terms of the GPL (either
+version 1, or at your option, any later version) or the Artistic
+License 2.0. Refer to LICENSE for the full license text. In addition,
+please see DISCLAIMER.txt for disclaimers of warranty.
+
+=cut
+
+use strict;
+use lib '../lib';
+
+use VM::EC2;
+use File::Basename 'basename';
+use Getopt::Long;
+
+my($Access_key,$Secret_key,$Endpoint);
+my $Program_name = basename($0);
+
+GetOptions(
+ 'access_key=s' => \$Access_key,
+ 'secret_key=s' => \$Secret_key,
+ ) or exec 'perldoc',$0;
+
+my $extra_size = shift or die "Please provide size to grow /opt/gbrowse by. Use --help for details.";
+
+#setup defaults
+$ENV{EC2_ACCESS_KEY} = $Access_key if defined $Access_key;
+$ENV{EC2_SECRET_KEY} = $Secret_key if defined $Secret_key;
+
+my $meta = VM::EC2->instance_metadata;
+my $zone = $meta->availabilityZone;
+(my $region = $zone) =~ s/[a-z]$//; # hack
+my $instance = $meta->instanceId;
+my ($ebs_device,$local_device) = unused_block_device() or die "Couldn't find a suitable device to attach to";
+
+my $ec2 = VM::EC2->new(-region=>$region) or die VM::EC2->error_str;
+
+print STDERR "Creating $extra_size EBS volume.\n";
+my $vol = $ec2->create_volume(-availability_zone => $zone,
+ -size => $extra_size) or die "Couldn't create EBS volume: ",$ec2->error_str;
+$vol->add_tag(Name=>"GBrowse lvm disk $local_device");
+
+print STDERR "Attaching volume.\n";
+my $a = $vol->attach($instance=>$ebs_device) or die "Couldn't attach EBS volume to $ebs_device: ",$ec2->error_str;
+$ec2->wait_for_attachments($a);
+$a->deleteOnTermination(1);
+-e $local_device or die "EBS volume did not appear at $local_device as expected";
+
+print STDERR "Creating lvm physical device.\n";
+system("sudo pvcreate $local_device") == 0 or die "pvcreate failed";
+
+print STDERR "Extending 'volumes' volume group.\n";
+system("sudo vgextend volumes $local_device") == 0 or die "vgextend failed";
+
+my $result = `sudo vgdisplay -c volumes` or die "vgdisplay filed";
+my @result = split /:/,$result;
+my $free = $result[15] or die "volume group has no free extents";
+
+print STDERR "Extending 'gbrowse' logical volume.\n";
+system("sudo lvextend -l +$free /dev/volumes/gbrowse") == 0 or die "lvextend failed";
+
+print STDERR "Extending /opt/gbrowse filesystem.\n";
+system("sudo resize2fs /dev/volumes/gbrowse") == 0 or die "resize2fs failed";
+
+print STDERR "Volume resized successfully\n";
+
+exit 0;
+
+sub unused_block_device {
+ my $major_start = shift || 'g';
+
+ my @devices = `ls -1 /dev/sd?* /dev/xvd?* 2>/dev/null`;
+ chomp(@devices);
+ return unless @devices;
+ my %used = map {$_ => 1} @devices;
+
+ my $base = $used{'/dev/sda1'} ? "/dev/sd"
+ : $used{'/dev/xvda1'} ? "/dev/xvd"
+ : '';
+ die "Device list contains neither /dev/sda1 nor /dev/xvda1; don't know how blocks are named on this system"
+ unless $base;
+
+ my $ebs = '/dev/sd';
+ for my $major ($major_start..'p') {
+ for my $minor (1..15) {
+ my $local_device = "${base}${major}${minor}";
+ next if $used{$local_device}++;
+ my $ebs_device = "/dev/sd${major}${minor}";
+ return ($ebs_device,$local_device);
+ }
+ }
+ return;
+}
Oops, something went wrong.

0 comments on commit 4c1c8e3

Please sign in to comment.