Permalink
Browse files

Initial commit of work in progress.

  • Loading branch information...
rcaputo committed May 20, 2011
0 parents commit 70306672654e2a0e5db689c4cc6ad1d85d276eb9
@@ -0,0 +1,25 @@
+CVS
+\.\#
+\.bak$
+\.cvsignore
+\.git
+\.gz$
+\.orig$
+\.patch$
+\.ppd$
+\.rej$
+\.rej$
+\.svn
+\.swo$
+\.swp$
+^Makefile$
+^Makefile\.old$
+^\.
+^_Inline
+^_build
+^blib/
+^comptest
+^cover_db
+^coverage\.report$
+^pm_to_blib$
+~$
@@ -0,0 +1,5 @@
+#!/usr/bin/perl
+
+use App::PipeFilter::JsonCut;
+
+App::PipeFilter::JsonCut->new_with_options()->run();
@@ -0,0 +1,5 @@
+#!/usr/bin/perl
+
+use App::PipeFilter::JsonMap;
+
+App::PipeFilter::JsonMap->new_with_options()->run();
@@ -0,0 +1,5 @@
+#!/usr/bin/perl
+
+use App::PipeFilter::JsonToYaml;
+
+exit App::PipeFilter::JsonToYaml->new_with_options()->run();
@@ -0,0 +1,5 @@
+#!/usr/bin/perl
+
+use App::PipeFilter::JsonSort;
+
+exit App::PipeFilter::JsonSort->new_with_options()->run();
@@ -0,0 +1,29 @@
+#!/usr/bin/perl
+
+use App::PipeFilter::MysqlToJson;
+
+exit App::PipeFilter::MysqlToJson->new_with_options()->run();
+
+__END__
+
+mysql -u user -password -h 10.0.0.5 database \
+ -e 'select crontab_id, task_id from crontab' -B \
+ | ./bin/mysql2json \
+ | ./bin/jsort -k task_id -rn
+
+{"crontab_id":"102","task_id":"701"}
+{"crontab_id":"101","task_id":"700"}
+{"crontab_id":"100","task_id":"650"}
+{"crontab_id":"8","task_id":"599"}
+{"crontab_id":"14","task_id":"38"}
+{"crontab_id":"12","task_id":"36"}
+{"crontab_id":"10","task_id":"35"}
+{"crontab_id":"11","task_id":"34"}
+{"crontab_id":"9","task_id":"33"}
+{"crontab_id":"7","task_id":"10"}
+{"crontab_id":"6","task_id":"9"}
+{"crontab_id":"5","task_id":"8"}
+{"crontab_id":"4","task_id":"7"}
+{"crontab_id":"3","task_id":"6"}
+{"crontab_id":"2","task_id":"5"}
+{"crontab_id":"1","task_id":"4"}
@@ -0,0 +1,19 @@
+name = App-PipeFilters
+version = 0.001
+author = Rocco Caputo <rcaputo@cpan.org>
+license = Perl_5
+copyright_holder = Rocco Caputo
+
+[Prereqs]
+JSON::XS = 0
+Moose = 0
+MooseX::Getopt = 0
+
+[Repository]
+git_remote = gh
+
+[ReadmeFromPod]
+[ReadmeMarkdownFromPod]
+[ReportVersions]
+[ChangelogFromGit]
+[@Classic]
@@ -0,0 +1,9 @@
+{"network":"freenode","channel":"#perl","nick":"dngor","karma":"120"}
+{"network":"efnet","channel":"#perl","nick":"dngor","karma":"300"}
+{"network":"magnet","channel":"#perl","nick":"dngor","karma":"250"}
+{"network":"freenode","channel":"#poe","nick":"dngor","karma":"9"}
+{"network":"efnet","channel":"#poe","nick":"dngor","karma":"66"}
+{"network":"magnet","channel":"#poe","nick":"dngor","karma":"9001"}
+{"network":"freenode","channel":"#reflex","nick":"dngor","karma":"3"}
+{"network":"efnet","channel":"#reflex","nick":"dngor","karma":"1"}
+{"network":"magnet","channel":"#reflex","nick":"dngor","karma":"5"}
@@ -0,0 +1,55 @@
+=head1 NAME
+
+App::PipeFilter
+
+=head1 DESCRIPTION
+
+App::PipeFilter is a distribution of shell pipeline filters that mimic
+some of the standard UNIX tools but process structured data.
+
+For example, jcut is a simple version of cut(1) that understands JSON
+objects rather than whitespace separated fields.
+
+ % head -1 sample.json
+ {"network":"freenode","channel":"#perl","nick":"dngor","karma":"120"}
+
+ % jcut -o network -o channel < eg/sample.json | sort | uniq
+ {"network":"efnet","channel":"#perl"}
+ {"network":"efnet","channel":"#poe"}
+ {"network":"efnet","channel":"#reflex"}
+ {"network":"freenode","channel":"#perl"}
+ {"network":"freenode","channel":"#poe"}
+ {"network":"freenode","channel":"#reflex"}
+ {"network":"magnet","channel":"#perl"}
+ {"network":"magnet","channel":"#poe"}
+ {"network":"magnet","channel":"#reflex"}
+
+=head1 DESIGN GOAL
+
+Follow the UNIX convention of one record per line of text. This
+ensures App::PipeFilter tools are compatible with many standard UNIX
+filters. In the example above, jcut output is piped through sort(1)
+and uniq(1) as one might expect.
+
+=head1 SEE ALSO
+
+jcut - Extract one or more named fields from JSON input.
+
+jmap - Rename one or more named fields from JSON input.
+
+json2yaml - Convert JSON input records to YAML output records. Some
+people may find YAML output to be more readable.
+
+jsort - Sort JSON input on one or more key fields.
+
+myswl2json - Convert mysql(1) batch output (-B) into JSON records.
+
+=head1 COPYRIGHT and LICENSE
+
+App::PipeFilter is Copyright 2001 by Rocco Caputo. All rights are
+reserved. App::PipeFilter is released under the same terms as Perl
+itself.
+
+=cut
+
+# vim: ts=2 sw=2 expandtab
@@ -0,0 +1,60 @@
+package App::PipeFilter::Generic;
+
+use Moose;
+with 'App::PipeFilter::Role::Flags::Standard';
+
+sub run {
+ my $self = shift();
+
+ my $ofh;
+ if ($self->output() eq '-') {
+ $ofh = \*STDOUT;
+ }
+ else {
+ open $ofh, ">", $self->output() or die(
+ "can't write ", $self->output(), ": $!"
+ );
+ }
+
+ # TODO - Pass input and output around in chunks to reduce method
+ # call overhead.
+
+ while (defined (my $input_filename = $self->next_input_file())) {
+ my $ifh;
+ if ($input_filename eq '-') {
+ warn "reading from standard input\n" if $self->verbose();
+ $ifh = \*STDIN;
+ }
+ else {
+ open $ifh, "<", $input_filename or die "can't open $input_filename: $!";
+ warn "reading from $input_filename\n" if $self->verbose();
+ }
+
+ $self->file_start($ifh, $ofh);
+
+ while (<$ifh>) {
+ next unless defined(my $input = $self->decode_input($_));
+ next unless my (@output) = $self->transform($input);
+ print $ofh $_ foreach $self->encode_output(@output);
+ }
+
+ $self->file_end($ofh);
+ }
+
+ # Exit value.
+ 0;
+}
+
+# Do nothing in particular before and after each file.
+#
+# TODO - Refactor the code to process a single file into its own
+# method. Replace file_start() with "before", and file_end() with
+# "after".
+
+sub file_start { undef }
+
+sub file_end { undef }
+
+1;
+
+# vim: ts=2 sw=2 expandtab
@@ -0,0 +1,14 @@
+package App::PipeFilter::Generic::Json;
+
+use Moose;
+
+extends 'App::PipeFilter::Generic';
+
+with qw(
+ App::PipeFilter::Role::Input::Json
+ App::PipeFilter::Role::Output::Json
+);
+
+1;
+
+# vim: ts=2 sw=2 expandtab
@@ -0,0 +1,29 @@
+package App::PipeFilter::JsonCut;
+
+use Moose;
+extends 'App::PipeFilter::Generic::Json';
+
+has o => (
+ is => 'rw',
+ isa => 'ArrayRef',
+ default => sub { die "requires one or more -o flag" },
+ lazy => 1,
+ documentation => 'output fields (a subset of the input)',
+);
+
+sub transform {
+ my $self = shift();
+
+ my @fields = @{$self->o()};
+
+ return map {
+ # TODO - Can this be done without a temporary variable?
+ my %x;
+ @x{@fields} = @{$_}{@fields};
+ \%x;
+ } @_;
+}
+
+1;
+
+# vim: ts=2 sw=2 expandtab
@@ -0,0 +1,45 @@
+package App::PipeFilter::JsonMap;
+
+use Moose;
+extends 'App::PipeFilter::Generic::Json';
+
+has i => (
+ is => 'rw',
+ isa => 'ArrayRef',
+ default => sub { die "requires one or more -i flag" },
+ lazy => 1,
+ documentation => 'input fields (a subset of the input)',
+);
+
+has o => (
+ is => 'rw',
+ isa => 'ArrayRef',
+ default => sub { die "requires one or more -o flag" },
+ lazy => 1,
+ documentation => 'output fields (a subset of the input)',
+);
+
+sub BUILD {
+ my $self = shift;
+ die "must have same number of -i fields as -o fields" if (
+ @{$self->i()} != @{$self->o()}
+ );
+}
+
+sub transform {
+ my $self = shift();
+
+ my %field_map;
+ @field_map{@{$self->i()}} = @{$self->o()};
+
+ return map {
+ # TODO - Can this be done without a temporary variable?
+ my %x = %$_;
+ $x{$field_map{$_}} = delete $x{$_} foreach keys %field_map;
+ \%x;
+ } @_;
+}
+
+1;
+
+# vim: ts=2 sw=2 expandtab
@@ -0,0 +1,61 @@
+package App::PipeFilter::JsonSort;
+
+use Moose;
+extends 'App::PipeFilter::Generic';
+with qw(
+ App::PipeFilter::Role::Input::Json
+ App::PipeFilter::Role::Transform::None
+);
+
+use JSON::XS;
+
+has k => (
+ is => 'rw',
+ isa => 'ArrayRef',
+ default => sub { die "requires one or more -k flag" },
+ lazy => 1,
+ documentation => 'key fields to sort on',
+);
+
+has n => (
+ is => 'rw',
+ isa => 'Bool',
+ default => 0,
+ documentation => 'sort fields numerically',
+);
+
+has r => (
+ is => 'rw',
+ isa => 'Bool',
+ default => 0,
+ documentation => 'reverse sort order',
+);
+
+sub BUILD {
+ my $self = shift;
+
+ my $sort = (
+ 'sort' .
+ ($self->n() ? ' -n' : '') .
+ ($self->r() ? ' -r' : '') .
+ ' | cut -f ' . (scalar(@{$self->k()}) + 1) . '-'
+ );
+
+ open STDOUT, '|-', $sort or die "Can't pipe into sort: $!";
+}
+
+sub encode_output {
+ my $self = shift();
+
+ my @fields = @{$self->k()};
+
+ return map {
+ my %sortable;
+ @sortable{@fields} = @{$_}{@fields};
+ join("\t", @sortable{@fields}, encode_json($_)) . "\n";
+ } @_;
+}
+
+1;
+
+# vim: ts=2 sw=2 expandtab
@@ -0,0 +1,15 @@
+package App::PipeFilter::JsonToYaml;
+
+use Moose;
+
+extends 'App::PipeFilter::Generic';
+
+with qw(
+ App::PipeFilter::Role::Input::Json
+ App::PipeFilter::Role::Transform::None
+ App::PipeFilter::Role::Output::Yaml
+);
+
+1;
+
+# vim: ts=2 sw=2 expandtab
Oops, something went wrong.

0 comments on commit 7030667

Please sign in to comment.