Initial prototype of DbBuilder class and its supporting script

Build.PL

@@ -11,19 +11,6 @@ my $build = Module::Build->new(
 
     # and now the list of perl module dependencies
     requires => {
-        'CGI'             => 0,
-        'HTML::Template'  => 0,
-        'Mojolicious'     => 0,
-        'YAML'            => 0,
-        'File::Slurp'     => 0,
-        'File::Path'      => 0,
-        'JSON'            => 0,
-        'List::UtilsBy'   => '0.10',
-        'IO::Socket::SSL'                => '1.94',
-        'File::Spec::Functions'          => 0,
-        'Mojolicious::Plugin::AssetPack' => '0.68',
-        'Imager::File::PNG'              => '0.92',
-        'Time::Moment'                   => '0.36',
 
         ## Required by the Mojo app:
           'Carp' => 0,

web/build-project-list.pl

@@ -1,69 +1,30 @@
 #!/usr/bin/perl
-use strict;
-use warnings;
-use 5.010;
 
-use lib qw{lib  lib-db-builder};
-use File::Path qw(make_path  remove_tree);
-use File::Spec::Functions qw(catdir);
-use Getopt::Long qw(GetOptions);
-use P6Project;
+use strictures 2;
 
-GetOptions(
-    'limit=s'      => \my $limit,
-    'no-app-start' => \my $no_app_start,
-);
-
-my $output_dir = shift(@ARGV) || './';
-binmode STDOUT, ':encoding(UTF-8)';
-
-local $| = 1;
+use File::Spec::Functions qw/catdir  catfile/;
+use Getopt::Long;
 
-my $min_popular = 10;
+use lib qw/lib/;
+use DbBuilder;
 
-my $list_url = 'https://raw.githubusercontent.com/perl6/ecosystem/master/META.list';
+use constant DB_FILE        => 'modulesperl6.db';
+use constant APP            => catfile qw/bin  ModulesPerl6.pl/;
+use constant META_LIST_FILE => 'https://raw.githubusercontent.com'
+                                    . '/perl6/ecosystem/master/META.list';
 
-my $template = './index.tmpl';
-
-my $logos_dir = catdir $output_dir, qw/public content-pics dist-logos/;
-remove_tree $logos_dir;
-make_path $logos_dir, => { mode => 0755 };
-
-my $p6p = P6Project->new(
-    output_dir   => $output_dir,
-    min_popular  => $min_popular,
-    template     => $template,
-    limit        => $limit,
-    no_app_start => $no_app_start,
+my $meta_list = META_LIST_FILE;
+GetOptions(
+    'meta-list=s' => \$meta_list,
+    'limit=i'     => \my $limit,
+    'restart-app' => \my $restart_app,
 );
 
-$p6p->load_projects($list_url);
-
-my $success = $p6p->stats->success;
-my $failed  = $p6p->stats->failed;
-print "ok - $success\nnok - $failed\n";
-
-my @errors = $p6p->stats->errors;
-warn join "\n", @errors, '' if @errors;
-
-die "Too many errors no output generated"
-  if $failed > $success;
-
-$p6p->write_json('proto.json');
-$p6p->write_html('index.html'); # this doesn't actually write anything ATM
-# $p6p->write_sprite;
-$p6p->write_dist_db;
-
-unless ( $output_dir eq './' )  {
-    system qw/
-        cp -r
-            bin
-            lib
-            modules_perl6.conf
-            public
-            templates
-    /, $output_dir;
-}
-
-$p6p->restart_app;
-
+DbBuilder->new(
+    app         => APP,
+    db_file     => DB_FILE,
+    limit       => $limit,
+    logos_dir   => catdir(qw/public  content-pics  dist-logos/),
+    meta_list   => $meta_list,
+    restart_app => $restart_app,
+)->run;

web/lib/DbBuilder.pm

@@ -0,0 +1,131 @@
+package DbBuilder;
+
+use strictures 2;
+
+use Data::GUID;
+use File::Path             qw/make_path  remove_tree/;
+use Mojo::URL;
+use Mojo::UserAgent;
+use Mojo::Util             qw/slurp  trim/;
+use Types::Common::Numeric qw/PositiveNum/;
+use Types::Standard        qw/InstanceOf  Str  Bool  Maybe/;
+
+use DbBuilder::Log;
+use DbBuilder::Dist;
+use ModulesPerl6::Model::BuildStats;
+use ModulesPerl6::Model::Dists;
+
+use Moo;
+use namespace::clean;
+
+has _app => (
+    init_arg => 'app',
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+has _db_file => (
+    init_arg => 'db_file',
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+has _limit => (
+    init_arg => 'limit',
+    is  => 'ro',
+    isa => Maybe[ PositiveNum ],
+);
+
+has _logos_dir => (
+    init_arg => 'logos_dir',
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+has _restart_app => (
+    init_arg => 'restart_app',
+    is  => 'ro',
+    isa => Maybe[ Bool ],
+);
+
+has _meta_list => (
+    init_arg => 'meta_list',
+    is       => 'ro',
+    isa      => Str | InstanceOf[qw/Mojo::URL  URI/],
+    required => 1,
+);
+
+#########################
+
+sub run {
+    my $self = shift;
+
+    $self->_prep_dirs;
+
+    my $build_id = Data::GUID->new->as_base64;
+    log info => "Starting build $build_id";
+
+    my $m = ModulesPerl6::Model::Dists->new( db_file => $self->db_file );
+    $m->add(
+        DbBuilder::Dist->new( meta_url => $_, build_id => $build_id )->as_hash
+    ) for $self->_meta_list;
+
+    if ( $self->_restart_app ) {
+        system $^O eq 'MSWin32'
+            ? $self->_app => 'daemon' # hypnotoad is not supported on Win32
+            : hypnotoad   => $self->_app;
+    }
+
+    $self;
+}
+
+#########################
+
+sub _meta_list {
+    my $self = shift;
+    my $meta_list = $self->_meta_list;
+
+    log info => "Fetching META.list from $meta_list";
+    my $url = Mojo::URL->new( $meta_list );
+    my $raw_data;
+    if ( $url->scheme =~ /(ht|f)tps?/i ) {
+        log info => '... a URL detected; trying to fetch';
+        my $tx = Mojo::UserAgent->new( max_redirects => 10 )->get( $url );
+
+        if ( $tx->success ) { $raw_data = $tx->res->body }
+        else {
+            my $err = $tx->error;
+            log fatal => "$err->{code} response: $err->{message}"
+                if $err->{code};
+            log fatal => "Connection error: $err->{message}";
+        }
+    }
+    elsif ( -r $meta_list ) {
+        log info => '... a file detected; trying to read';
+        $raw_data = slurp $meta_list;
+    }
+    else {
+        log fatal => 'Could not figure out how to read';
+    }
+
+    my @metas = grep /\S/, map trim($_), split m{\Q$/\E}, $raw_data;
+    log info => 'Found ' . @metas . ' dists';
+
+    return @metas;
+}
+
+sub _prep_dirs {
+    my $self = shift;
+    my $logos_dir = $self->logos_dir;
+
+    log info => "Cleaning up dist logos dir [$logos_dir]";
+    remove_tree $logos_dir;
+    make_path   $logos_dir => { mode => 0755 };
+}
+
+1;
+
+__END__

web/lib/DbBuilder/Dist.pm

@@ -0,0 +1,24 @@
+package DbBuilder::Dist;
+
+use strictures 2;
+
+use Types::Standard qw/Str/;
+
+use Moo;
+use namespace::clean;
+
+has _build_id => (
+    init_arg => 'build_id',
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+has _meta_url => (
+    init_arg => 'meta_url',
+    is       => 'ro',
+    isa      => Str,
+    required => 1,
+);
+
+1;