Skip to content
Browse files

documentation updates.

for all new commands, mogstats, and MogileFS::Utils notes.
  • Loading branch information...
1 parent 8cb5913 commit b2606f09c537f5e6dc82ab1d7d0ce86d782fa67a @dormando dormando committed Jan 5, 2011
Showing with 418 additions and 19 deletions.
  1. +19 −3 lib/MogileFS/Utils.pm
  2. +0 −14 mogadm
  3. +40 −0 mogdelete
  4. +48 −0 mogfetch
  5. +40 −0 mogfileinfo
  6. +45 −1 moglistfids
  7. +48 −0 moglistkeys
  8. +120 −0 mogstats
  9. +4 −1 mogtool
  10. +54 −0 mogupload
View
22 lib/MogileFS/Utils.pm
@@ -31,7 +31,7 @@ sub _readconf {
my $args = shift;
# Liftedish from mogadm, but we can refactor mogadm to use this instead.
- my @configs = ($args->{config}, $ENV{MOGUTILSCONF},
+ my @configs = ($args->{conf}, $ENV{MOGUTILSCONF},
"$ENV{HOME}/.mogilefs.conf",
"/etc/mogilefs/mogilefs.conf");
my %opts = ();
@@ -62,7 +62,7 @@ sub getopts {
my %opts = ();
$self->abort_usage($usage) unless @ARGV;
- GetOptions(\%opts, @want, qw/help trackers=s domain=s/)
+ GetOptions(\%opts, @want, qw/help trackers=s domain=s conf=s/)
or $self->abort_usage($usage);
my $config = $self->_readconf(\%opts);
@@ -120,7 +120,7 @@ MogileFS::Utils - Command line utilities for the MogileFS distributed file syste
L<mogadm>
-L<mogtool>
+L<mogstats>
L<mogupload>
@@ -134,14 +134,30 @@ L<moglistkeys>
L<moglistfids>
+L<mogtool> (DEPRECATED: Do not use!)
+
=head1 SUMMARY
Please refer to the documentation for the tools included in this distribution.
+=head1 CONFIGURATION FILE
+
+Most of the utilities in this package support a configuration file. Common
+options can be pushed into the config file, such as trackers, domain, or
+class. The file is in B</etc/mogilefs/mogilefs.conf> and B<~/.mogilefs.conf>
+by default. You may also specify a configuration via B<--conf=filename>
+
+Example:
+
+ trackers = 10.0.0.1:7001,10.0.0.3:7001
+ domain = foo
+
=head1 AUTHOR
Brad Fitzpatrick E<lt>L<brad@danga.com>E<gt>
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
=head1 BUGS
Please report any on the MogileFS mailing list: L<http://groups.google.com/group/mogile/>.
View
14 mogadm
@@ -1437,16 +1437,6 @@ Check to ensure that all of the MogileFS system components are functioning
and that we can contact everybody. The quickest way of ensuring that the
entire MogileFS system is functional from the current machine's point of view.
-=item B<stats>
-
-Attempt to get statistics for the MogileFS world. This should provide
-statistics on: the number of files present on each device; the number of files
-in each (domain,class) tuple; statistics about present replication devcounts
-and pending replications.
-
-Due to design constraints, the stats command may presently fail if some
-statistic takes too long to generate.
-
=item B<host add E<lt>hostE<gt> [host options]>
=item B<host modify E<lt>hostE<gt> [host options]>
@@ -1662,10 +1652,6 @@ Check the status of your entire MogileFS system:
$ mogadm check
-Check MogileFS system statistics:
-
- $ mogadm stats
-
Check every file in the entire MogileFS system:
$ mogadm fsck reset
View
40 mogdelete
@@ -1,5 +1,45 @@
#!/usr/bin/perl
+=head1 NAME
+
+mogdelete -- Delete keys from a MogileFS installation
+
+=head1 SYNOPSIS
+
+ $ mogdelete --trackers=host --domain=foo --key="/hello.jpg"
+
+=head1 OPTIONS
+
+=over
+
+=item --trackers=host1:7001,host2:7001
+
+Use these MogileFS trackers to negotiate with.
+
+=item --domain=<domain>
+
+Set the MogileFS domain to use.
+
+=item --key="<key>"
+
+A key to delete. Can be an arbitrary string.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+Produces a slightly weird error if key does not exist.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
+
use strict;
use warnings;
View
48 mogfetch
@@ -1,5 +1,53 @@
#!/usr/bin/perl
+=head1 NAME
+
+mogfetch -- Fetch data from a MogileFS installation
+
+=head1 SYNOPSIS
+
+ $ mogfetch [options]
+ $ mogfetch [options] --file="-" > filename
+
+ $ mogfetch --trackers=host --domain=foo \
+ --key="/hello.jpg" --file="output.jpg"
+
+=head1 OPTIONS
+
+=over
+
+=item --trackers=host1:7001,host2:7001
+
+Use these MogileFS trackers to negotiate with.
+
+=item --domain=<domain>
+
+Set the MogileFS domain to use.
+
+=item --key="<key>"
+
+The key to locate the data with. Can be an arbitrary string.
+
+=item --file="<filename|->"
+
+A local destination file. If '-', data is written to STDOUT instead.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+None known.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
+
use strict;
use warnings;
View
40 mogfileinfo
@@ -1,5 +1,45 @@
#!/usr/bin/perl
+=head1 NAME
+
+mogfileinfo -- Fetch key metadata from a MogileFS installation
+
+=head1 SYNOPSIS
+
+ $ mogfileinfo --trackers=host --domain=foo --key="/hello.jpg"
+
+=head1 OPTIONS
+
+=over
+
+=item --trackers=host1:7001,host2:7001
+
+Use these MogileFS trackers to negotiate with.
+
+=item --domain=<domain>
+
+Set the MogileFS domain to use.
+
+=item --key="<key>"
+
+The key to inspect. Can be an arbitrary string.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+None known, but output might change in the future.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
+
use strict;
use warnings;
View
46 moglistfids
@@ -1,5 +1,49 @@
#!/usr/bin/perl
-# Example for building into a backup program.
+
+=head1 NAME
+
+moglistfids -- Iterate fid/key data from a MogileFS installation
+
+=head1 DESCRIPTION
+
+Example utility for pulling all file data out of a MogileFS installation.
+Utilities like this can be built on for creating backup systems,
+Mogile<->Mogile syncronization systems, or Mogile->S3 syncronization.
+
+This method is only a way of pulling new files which have existed since the
+last time it was checked, as there's no logging of deleted files.
+
+=head1 OPTIONS
+
+=over
+
+=item --trackers=host1:7001,host2:7001
+
+Use these MogileFS trackers to negotiate with.
+
+=item --fromfid=<fid>
+
+The highest numbered fid fetched the last time this utility was run.
+
+=item --count=<count>
+
+Numer of fids to inspect and return.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+None known.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
use strict;
use warnings;
View
48 moglistkeys
@@ -1,4 +1,52 @@
#!/usr/bin/perl
+
+=head1 NAMe
+
+moglistkeys -- Lists keys out of a MogileFS domain
+
+=head1 SYNOPSIS
+
+ $ moglistkeys --trackers=host --domain=foo --key_prefix="/foo/bar/"
+
+=head1 DESCRIPTION
+
+If you store your MogileFS keys in a logical "structure", you may use this
+tool to view lists of subsets of keys. Note that this is not going to be
+equivalent to "cd" and "ls" tools, as listing "foo/" will list everything
+underneath, so it's more akin to "ls -R"
+
+=head1 OPTIONS
+
+=over
+
+=item --trackers=host1:7001,host2:7001
+
+Use these MogileFS trackers to negotiate with.
+
+=item --domain=<domain>
+
+Set the MogileFS domain to use.
+
+=item --key_prefix="/foo/bar/"
+
+Search for keys starting with this prefix. Can be an arbitrary string.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+None known.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
+
# TODO: Add ways to limit # of keys displayed
use strict;
View
120 mogstats
@@ -5,6 +5,125 @@
# timing output during fetch
# support different db backends better (something mildly generic).
+=head1 NAME
+
+mogstats -- Utility for calculating slow stats directly against a MogileFS DB
+
+=head1 SYNOPSIS
+
+ $ mogstats --db_dsn="DBI:mysql:mfs:host=mfshost" --db_user="mfs" \
+ --db_pass="mfs" --verbose --stats="devices,files"
+ $ mogstats --stats="all"
+ $ mogstats [all options in ~/.mogilefs.conf]
+
+=head1 DESCRIPTION
+
+Utility for inspecting queues and running general statistics against a
+MogileFS database. Some of these stats can take a very long time to run
+against a large instance, so the utility can be pointed at a read slave or
+special account.
+
+=head1 OPTIONS
+
+=over
+
+=item --db_dsn=<DBI_dsn>
+
+The DSN to use for connecting to the MogileFS database server.
+
+=item --db_user=<username>
+
+A database user for connecting to the database.
+
+=item --db_pass=<password>
+
+An optional password for the database user.
+
+=item --config=<file>
+
+An explicit config file to use. By default /etc/mogilefs/mogilefs.conf and
+~/.mogilefs.conf are checked.
+
+=item --verbose
+
+Print some extra text during processing. Mostly notes about what stats are
+starting or finishing.
+
+=item --stats=<stats>
+
+A list of which statistics to calculate. Notes on some of them are listed
+below, see --help for full list. A value of "all" fetches all possible stats.
+
+=item --help
+
+List usage info and supported statistics.
+
+=back
+
+=head1 AVAILABLE STATISTICS
+
+Contains notes on which stats may be fast or slow.
+
+=over
+
+=item devices
+
+Lists count of files and current database status per-device. Can be very slow.
+
+=item fids
+
+Lists the current highest file id. Should be fast.
+
+=item files
+
+Gives a breakdown of where files are by domain and class. Displays the size of
+all stored files pre-replication, as well as post-replication size. The latter
+being closer to the actual storage amount. Can be very slow.
+
+=item domains
+
+Shows a simple count of where files are by domain and class. Faster than using
+"files" but displays less information.
+
+=item replication
+
+Displays a breakdown of devcount per domain/class combo. Shows number of files
+in domain "foo" with class "bar" that have a current devcount of 0, 1, 2, 3,
+etc. Useful for spotting broken files (devcount 0), replication lag, or over
+replication bugs. Can be very slow.
+
+=item replication-queue
+
+Quick breakdown of how many fids are due for replication. Fids listed as
+"manual" need manual intervention before they can be replicated, and could be
+broken. Will be fast unless there are many files in queue.
+
+=item delete-queue
+
+Similar to replication-queue.
+
+=item general-queue
+
+Displays a breakdown of what's in the general queue. This includes FSCK,
+Rebalance, and other temporary queueing systems MogileFS has. Should be fast
+unless you have configured MogileFS to queue many fids at once.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+None known.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
+
use strict;
use warnings;
use DBI;
@@ -20,6 +139,7 @@ my $DB_TYPE = '';
my %opts;
+# FIXME: Use MogileFS::Utils for configuration junk.
Getopt::Long::Configure("require_order", "pass_through");
GetOptions(
"config=s" => \$opts{config},
View
5 mogtool
@@ -8,6 +8,8 @@ eval 'exec /usr/bin/perl -S $0 ${1+"$@"}'
mogtool -- Inject/extract data to/from a MogileFS installation
+B<WARNING>: this utility is deprecated! See L<MogileFS::Utils>
+
=head1 SYNOPSIS
$ mogtool [general-opts] <command> [command-opts] <command-args>
@@ -1417,7 +1419,8 @@ sub mdelete {
abortWithUsage() if $opts{help};
sub abortWithUsage {
- my $msg = join '', @_;
+ my $msg = "!!!mogtool is DEPRECATED and will be removed in the future!!!\n";
+ $msg .= join '', @_;
if ( $msg ) {
pod2usage( -verbose => 1, -exitval => 1, -message => "\n$msg\n" );
View
54 mogupload
@@ -1,5 +1,59 @@
#!/usr/bin/perl
+=head1 NAME
+
+mogupload -- Upload data to a MogileFS installation
+
+=head1 SYNOPSIS
+
+ $ mogupload [options]
+ $ mogupload [options] --file="-" < filename
+
+ $ mogupload --trackers=host --domain=foo --class=bar \
+ --key="/hello.jpg" --file="input.jpg"
+
+ $ echo "why hello" | mogupload [opts] --key="world" --file="-"
+
+=head1 OPTIONS
+
+=over
+
+=item --trackers=host1:7001,host2:7001
+
+Use these MogileFS trackers to negotiate with.
+
+=item --domain=<domain>
+
+Set the MogileFS domain to use.
+
+=item --class=<class>
+
+Set the class to use. Will use default class if not specified
+
+=item --key="<key>"
+
+A key to store the file under. Can be an arbitrary string.
+
+=item --file="<filename|->"
+
+A local file to upload. If '-', read file from STDIN instead.
+
+=back
+
+=head1 AUTHOR
+
+Dormando E<lt>L<dormando@rydia.net>E<gt>
+
+=head1 BUGS
+
+mogupload must buffer the upload in memory before transferring it. This makes it difficult to upload very large files. Future versions will lift this limitation.
+
+=head1 LICENSE
+
+Licensed for use and redistribution under the same terms as Perl itself.
+
+=cut
+
use strict;
use warnings;

0 comments on commit b2606f0

Please sign in to comment.
Something went wrong with that request. Please try again.