NetKAN-bot Rewrite

.gitignore

@@ -18,3 +18,5 @@ nytprof.out
 *.o
 *.bs
 /_eumm/
+App-KSP_CKAN-*
+*.swp

.travis.yml

@@ -0,0 +1,27 @@
+# thanks -> http://blogs.perl.org/users/alex_balhatchet/2013/04/travis-ci-perl.html
+language: perl
+perl:
+#  - "5.22" 
+  - "5.20"
+  - "5.18"
+  - "5.16"
+  - "5.14"
+  - "5.12"
+  - "5.10"
+before_install:
+  # Prevent "Please tell me who you are" errors for certain DZIL configs
+  - git config --global user.name "TravisCI"
+install:
+  # Deal with all of the DZIL dependancies, quickly and quietly
+  - pip install --user jsonschema
+  - export PYTHONPATH=~/.local/lib/python2.7/site-packages/
+  - cpanm --quiet --notest --skip-satisfied Dist::Zilla
+  - cpanm --quiet --notest --skip-satisfied Test::Perl::Critic
+  - dzil authordeps | grep -vP '[^\w:]' | xargs -n 5 -P 10 cpanm --quiet --notest --skip-satisfied
+  - dzil listdeps | grep -vP '[^\w:]' |  xargs -n 5 -P 10 cpanm --quiet --notest --skip-satisfied
+  - cpanm --quiet --notest Devel::Cover::Report::Coveralls
+  - cpanm --quiet --notest Dist::Zilla::App::Command::cover
+script:
+  - dzil test
+after_success:
+  - dzil cover -outputdir cover_db -report coveralls

README.md

@@ -1,9 +1,7 @@
-# NetKAN-bot
-NetKAN indexing service
+App::KSP-CKAN    [![Build Status](https://travis-ci.org/KSP-CKAN/NetKAN-bot.svg?branch=master)](https://travis-ci.org/KSP-CKAN/NetKAN-bot)  [![Coverage Status](https://coveralls.io/repos/KSP-CKAN/NetKAN-bot/badge.svg?branch=master)](https://coveralls.io/r/KSP-CKAN/NetKAN-bot?branch=master)
 
-TODO: Expand this!
-
-We'll need some deps. 
+Non Perl Dependencies
+=====================
 ```bash
 apt-get install liblocal-lib-perl cpanminus install build-essential mono-complete libcurl4-openssl-dev python-jsonschema  
 ```
@@ -13,19 +11,71 @@ NetKAN will need certs for mono
 mozroots --import --ask-remove
 ```
 
-We'll be using lib local for our Perl deps.
+Configure local::lib if you haven't already done so:
+```bash
+$ perl -Mlocal::lib >> ~/.bashrc
+$ eval $(perl -Mlocal::lib)
+```
+
+Installation
+============
+
+Install from git, you can then use:
+```bash
+$ dzil authordeps | cpanm
+$ dzil listdeps   | cpanm
+$ dzil install
+```
+
+or cpanm via the tar.gz on the GitHub Release page
+
+```bash
+cpanm App-KSP_CKAN-0.001.tar.gz
+```
+
+Configuration
+=============
+
+An ini file with the following contents will need to created at ~/.ksp-ckan
+```
+CKAN_meta=git@github.com:KSP-CKAN/CKAN-meta.git
+NetKAN=git@github.com:KSP-CKAN/NetKAN-bot.git
+netkan_exe=https://ckan-travis.s3.amazonaws.com/netkan.exe
+ckan_validate=https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py
+ckan_schema=https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/CKAN.schema
+working=/home/NetKAN/NetKAN
+```
+
+If you have a GitHub token, add the following line (helpful for prevent expending the GitHub public API limits):
+```
+GH_token=1234567890
+```
+
+Running
+=======
+
+Completing a full index is as straight forward as:
 ```bash
-perl -Mlocal::lib >> ~/.bashrc
+netkan-indexer
 ```
 
-Our Perl Deps
+Debugging will print debug messages to the logfile and to the screen. It is enabled with
 ```bash
-cpanm File::Basename File::chdir File::Path Try::Tiny HTTP::Tiny Log::Tiny IPC::System::Simple
+netkan-indexer --debug
+```
+
+Enable it in cron with (crontab -e as the netkan user):
+```
+# Run full index every 3 hours
+00 */3 * * * PERL5LIB=/home/netkan/perl5/lib/perl5/ netkan-indexer
 ```
 
-Currently everything is hardcoded, if you generate a github token, NetKAN will use it and 
-it will need to go here ~/.NetKAN/github.token
+There is a 'lite' cli option is not implemented. It's a future concept to allow 'lite' 
+skimming of metadata API endpoints without performing a full metadata inflation.
+
+License
+=======
 
-And _only_ contain the the token on the first line.
+Dist::Zilla handles the generation of the license file.
 
-It will generate a log file at ~/.NetKAN/NetKAN.log
+However this project is covered by The MIT License (MIT)

bin/netkan-indexer

@@ -1,145 +1,111 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
+
 use 5.010;
 use strict;
 use warnings;
 use autodie qw(:all);
-use File::Basename qw(basename);
-use FindBin qw($Bin);
-use File::chdir;
-use File::Path 'rmtree';
-use Try::Tiny;
-use HTTP::Tiny;
-use Log::Tiny;
 use Sys::RunAlone;
-use Time::Limit '3000';
+use Time::Limit '3000'; # Something wrong if we are taking longer than 50 mins
+use Getopt::Long;
+use File::Spec;
+use App::KSP_CKAN::NetKAN;
+use App::KSP_CKAN::Tools::Config;
 
-# Convert KerbalStuff and GitHub releases into CKAN metadata!
-# It's the Networked Kerbal Archive Network. (NetKAN) :)
+# PODNAME: netkan-indexer
 
-our $DEBUG = 0;
-if ($ARGV[0]) {
-    $DEBUG = 1 if $ARGV[0] eq '--debug';
-}
+# ABSTRACT: netkan-indexer - Extant NetKAN indexing bot
 
-# TODO: Make these configurable
-my $NETKAN_DATA = "$ENV{HOME}/.NetKAN";
-my $NETKAN_DIR = "$NETKAN_DATA/NetKAN";
-my $CKAN_META  = "$NETKAN_DATA/CKAN-meta";
+# VERSION
 
-if ( ! -d $NETKAN_DATA ) {
-    mkdir $NETKAN_DATA;
-}
-if ( ! -d "$NETKAN_DATA/cache" ) {
-    mkdir "$NETKAN_DATA/cache";
-}
+=head1 SYNOPSIS
 
-my $log = Log::Tiny->new( "$NETKAN_DATA/NetKAN.log" );
+Usage:
 
-my $token;
-if ( -e "$NETKAN_DATA/github.token" ) {
-    # This makes the brash assumption there is just a token
-    # string in the first line of the file.
-    open(my $fh, "<", "$NETKAN_DATA/github.token");
-    $token = <$fh>;
-}
+  Debugging commands:
+
+  netkan-indexer --debug        : Run with debugging enabled.
 
-# Update our External Dependencies
-# Using HTTP here due to S3 Certificate issue
-# https://forums.aws.amazon.com/thread.jspa?threadID=164095
-mirror_file( "http://ckan-travis.s3.amazonaws.com/netkan.exe", "$NETKAN_DATA/netkan.exe" );
-mirror_file( "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/bin/ckan-validate.py", "$NETKAN_DATA/ckan-validate.py" );
-mirror_file( "https://raw.githubusercontent.com/KSP-CKAN/CKAN/master/CKAN.schema", "$NETKAN_DATA/CKAN.schema" );
-
-# Make them executable
-chmod 0755, "$NETKAN_DATA/netkan.exe";
-chmod 0755, "$NETKAN_DATA/ckan-validate.py";
-
-# Get Fresh MetaData
-chdir($NETKAN_DATA);
-if (-d "CKAN-meta/") {
-  $log->DEBUG("Removing CKAN-meta") if $DEBUG;
-  rmtree("CKAN-meta");
-}
-system("git", "clone", "--recursive", 'git@github.com:KSP-CKAN/CKAN-meta');
+=head1 SETUP
 
-# Download NetKAN Meta Data
-chdir($NETKAN_DIR);
-system("git", "pull", "-X", "theirs");
+=head2 Installation
 
-foreach my $file (glob("NetKAN/*.netkan")) {
-    my $basename = basename($file,  ".netkan");
+If you have not already installed this software, the easiest way
+is to use L<cpanm> and L<local::lib>. If you don't have them installed,
+it's easy with:
 
-    $log->DEBUG("Downloading metadata for $basename...") if $DEBUG;
-
-    if (! -d "$CKAN_META/$basename" ) {
-        mkdir "$CKAN_META/$basename";
-    }
-
-    # TODO: It'd be nice to catch the errors and report them or at least log them.
-    try {
-        if ($token) {
-            system("$NETKAN_DATA/netkan.exe", "--outputdir=$CKAN_META/$basename",  "--cachedir=$NETKAN_DATA/cache", "--github-token=$token" , $file);
-        }
-        else {
-            system("$NETKAN_DATA/netkan.exe", "--outputdir=$CKAN_META/$basename",  "--cachedir=$NETKAN_DATA/cache", $file);
-        }
-    }
-    catch {
-        $log->WARN("Processing $file FAILED");
-    };
-}
+    curl -L http://cpanmin.us/ | perl - --self-upgrade
+    ~/perl5/bin/cpanm -L ~/perl5 App::local::lib::helper
+    source ~/perl5/bin/localenv-bashrc
 
-# Process Chagnes
-chdir($CKAN_META);
-system("git", "add", "-A");
-my @changes = `git diff --name-only --stat origin/master`;
-chomp(@changes);
-
-foreach my $changed (@changes) {
-    if ( ! validate("$CKAN_META/$changed") ) {
-        $log->WARN("Failed to Parse $changed");
-        system("git", "reset", $changed);
-    }
-    else {
-        $log->INFO("Commiting $changed");
-        system("git", "commit", $changed, "-m", "'NetKAN generated mods - $changed'");
-    }
-}
+You might want to put that last line in your F<~/.bashrc> file.
+
+You can then install C<netkan-indexer> and related utilities with:
+
+    cpanm App::KSP_CKAN
+
+=head1 DESCRIPTION
+
+This is the extant NetKAN Indexing Bot for KSP-CKAN
 
-unless  ($DEBUG) {
-    system("git", "pull", "-X", "ours");
-    system("git", "push");
+=head1 BUGS/Features Requests
+
+Please submit any bugs, feature requests to
+L<https://github.com/KSP-CKAN/NetKAN-bot/issues> .
+
+Contributions are more than welcome!
+
+=head1 SEE ALSO
+
+L<App::KSP-CKAN>
+
+=cut
+
+my $PROGNAME = (File::Spec->splitpath($0))[2];
+$PROGNAME ||= 'netkan-indexer';
+
+my $DEBUG  = 0;
+my $LITE = 0;
+
+my $getopts_rc = GetOptions(
+  "version"       => \&version,
+  "debug!"        => \$DEBUG,
+  "lite!"          => \$LITE,
+
+  "help|?"        => \&print_usage,
+);
+
+# TODO: Allow config to be specified
+my $config = App::KSP_CKAN::Tools::Config->new(
+  debugging => $DEBUG,
+);
+
+my $netkan = App::KSP_CKAN::NetKAN->new(
+  config => $config,
+);
+
+if (! $LITE ) {
+  $netkan->full_index;
+} else {
+  $netkan->lite_index;
 }
 
-# Shortcuts
-sub mirror_file {
-    my ($url, $output) = @_;
-    my $http = HTTP::Tiny->new( timeout => 15, verify_SSL => 1 );
-    my $response = $http->mirror( $url, $output );
-
-    if ( ! $response->{success} ) {
-        $log->WARN("Downloading '$url' failed: $response->{reason}");
-    }
+sub version {
+  $::VERSION ||= "Unreleased";
+  say "netkan-indexer version          : $::VERSION";
+  exit 1;
 }
 
-sub validate {
-    my ($file) = @_;
-    local $CWD = $NETKAN_DATA;
-
-    my $return; # Return in finally will not return out of the Sub, just itself.
-    try {
-        system("python", "ckan-validate.py", "$file");
-    } 
-    finally {
-        if (@_) {
-            $log->DEBUG(@_);
-            $return = 0;
-        }
-        else {
-            $return = 1;
-        }
-    };
-    return $return;
+sub print_usage {
+  say q{
+  Usage:
+
+  netkan-indexer --debug        : Run with debugging enabled.
+  netkan-indexer --version      : Show version information
+
+  For more documentation, use `perldoc netkan-indexer`.
+  };
+
+  exit 1;
 }
 
 __END__