Merge pull request #1357 from mudler/config_plugin

Allow plugins to load specific configurations from Ini file
os-autoinst · Jun 8, 2017 · da931c4 · da931c4
2 parents f7a233c + 8cefc1b
commit da931c4
Show file tree

Hide file tree

Showing 13 changed files with 247 additions and 40 deletions.
diff --git a/docs/Contributing.asciidoc b/docs/Contributing.asciidoc
@@ -266,8 +266,7 @@ to its value (or default OpenID) OpenQA tries to require OpenQA::WebAPI::Auth::$
 If successful, module for given method is imported or the OpenQA ends with error.
 
 
-Each authentication module is expected to export +auth_config+,
-+auth_login+ and +auth_logout+ functions. In case of request-response mechanism (as in
+Each authentication module is expected to export +auth_login+ and +auth_logout+ functions. In case of request-response mechanism (as in
 OpenID), +auth_response+ is imported on demand.
 
 Currently there is no login page because all implemented methods use either 3rd party

diff --git a/lib/OpenQA/ServerStartup.pm b/lib/OpenQA/ServerStartup.pm
@@ -102,6 +102,7 @@ sub read_config {
 
     if (-e $cfgfile) {
         $cfg = Config::IniFiles->new(-file => $cfgfile->to_string) || undef;
+        $app->config->{ini_config} = $cfg;
     }
     else {
         $app->log->warn("No configuration file supplied, will fallback to default configuration");
@@ -156,4 +157,35 @@ sub setup_logging {
     $OpenQA::Utils::app = $app;
 }
 
+# Update config definition from plugin requests
+sub update_config {
+    my ($config, @namespaces) = @_;
+    return unless exists $config->{ini_config};
+
+    # Filter out what plugins are loaded from the used namespaces
+    foreach my $plugin (loaded_plugins(@namespaces)) {
+
+        # We take config only if the plugin has the method declared
+        next unless ($plugin->can("configuration_fields"));
+
+        # If it is a Mojo::Base class, it requires to be instantiated
+        # because the attributes are not populated until creation.
+        my $fields
+          = UNIVERSAL::isa($plugin, "Mojo::Base") ?
+          do { $plugin->new->configuration_fields() }
+          : $plugin->configuration_fields();
+
+        # We expect just hashrefs
+        next unless (ref($fields) eq "HASH");
+
+        # Walk the hash with the plugin returns that needs to be fetched
+        # by our Ini file parser and fill config from it
+        hashwalker $fields => sub {
+            my ($key, undef, $keys) = @_;
+            my $v = $config->{ini_config}->val(@$keys[0], $key);
+            $config->{@$keys[0]}->{$key} = $v if defined $v;
+        };
+    }
+}
+
 1;
diff --git a/lib/OpenQA/Utils.pm b/lib/OpenQA/Utils.pm
@@ -66,9 +66,12 @@ $VERSION = sprintf "%d.%03d", q$Revision: 1.12 $ =~ /(\d+)/g;
   &detect_current_version
   wait_with_progress
   mark_job_linked
+  path_to_class
+  loaded_modules
+  loaded_plugins
+  hashwalker
 );
 
-
 if ($0 =~ /\.t$/) {
     # This should result in the 't' directory, even if $0 is in a subdirectory
     my ($tdirname) = $0 =~ qr/((.*\/t\/|^t\/)).+$/;
@@ -707,5 +710,38 @@ sub detect_current_version {
     return $current_version;
 }
 
+# Resolves a path to class
+# path is expected to be in the form of the keys of %INC. e.g. : foo/bar/baz.pm
+sub path_to_class { substr join('::', split(/\//, shift)), 0, -3 }
+
+# Returns all modules that are loaded into memory
+sub loaded_modules {
+    map { path_to_class($_); } sort keys %INC;
+}
+
+# Fallback to loaded_modules if no arguments are given.
+# Accepts namespaces as arguments. If supplied, it will filter by them
+sub loaded_plugins {
+    my $ns = join("|", map { quotemeta } @_);
+    return @_ ? grep { /^$ns/ } loaded_modules() : loaded_modules();
+}
+
+# Walks a hash keeping keys as a stack
+sub hashwalker {
+    my ($hash, $callback, $keys) = @_;
+    $keys = [] if !$keys;
+    while (my ($key, $value) = each %$hash) {
+        push @$keys, $key;
+        if (ref($value) eq 'HASH') {
+            hashwalker($value, $callback, $keys);
+        }
+        else {
+            $callback->($key, $value, $keys);
+        }
+        pop @$keys;
+    }
+}
+
+
 1;
 # vim: set sw=4 et:
diff --git a/lib/OpenQA/WebAPI.pm b/lib/OpenQA/WebAPI.pm
@@ -94,14 +94,16 @@ sub startup {
 
     unshift @{$self->renderer->paths}, '/etc/openqa/templates';
 
+    # Load plugins
+    push @{$self->plugins->namespaces}, 'OpenQA::WebAPI::Plugin';
     $self->plugin(AssetPack => {pipes => [qw(Sass Css JavaScript Fetch OpenQA::WebAPI::AssetPipe Combine)]});
-    $self->plugin('OpenQA::WebAPI::Plugin::Helpers');
-    $self->plugin('OpenQA::WebAPI::Plugin::CSRF');
-    $self->plugin('OpenQA::WebAPI::Plugin::REST');
-    $self->plugin('OpenQA::WebAPI::Plugin::HashedParams');
-    $self->plugin('OpenQA::WebAPI::Plugin::Gru');
+
+    foreach my $plugin (qw(Helpers CSRF REST HashedParams Gru)) {
+        $self->plugin($plugin);
+    }
+
     if ($self->config->{global}{audit_enabled}) {
-        $self->plugin('OpenQA::WebAPI::Plugin::AuditLog', Mojo::IOLoop->singleton);
+        $self->plugin('AuditLog', Mojo::IOLoop->singleton);
     }
     # Load arbitrary plugins defined in config: 'plugins' in section
     # '[global]' can be a space-separated list of plugins to load, by
@@ -110,12 +112,24 @@ sub startup {
         my @plugins = split(' ', $self->config->{global}->{plugins});
         for my $plugin (@plugins) {
             $self->log->info("Loading external plugin $plugin");
-            $self->plugin("OpenQA::WebAPI::Plugin::$plugin");
+            $self->plugin($plugin);
         }
     }
     if ($self->config->{global}{profiling_enabled}) {
         $self->plugin(NYTProf => {nytprof => {}});
     }
+    # load auth module
+    my $auth_method = $self->config->{auth}->{method};
+    my $auth_module = "OpenQA::WebAPI::Auth::$auth_method";
+    eval "require $auth_module";    ## no critic
+    if ($@) {
+        die sprintf('Unable to load auth module %s for method %s', $auth_module, $auth_method);
+    }
+
+    # Read configurations expected by plugins.
+    OpenQA::ServerStartup::update_config($self->config, @{$self->plugins->namespaces}, "OpenQA::WebAPI::Auth");
+
+    # End plugin loading/handling
 
     # read assets/assetpack.def
     $self->asset->process;
@@ -146,15 +160,6 @@ sub startup {
             $tx->req->headers->header('X-Build-Tx-Time' => time);
         });
 
-    # load auth module
-    my $auth_method = $self->config->{auth}->{method};
-    my $auth_module = "OpenQA::WebAPI::Auth::$auth_method";
-    eval "require $auth_module";    ## no critic
-    if ($@) {
-        die sprintf('Unable to load auth module %s for method %s', $auth_module, $auth_method);
-    }
-    $auth_module->auth_config($self->config);
-
     # Router
     my $r         = $self->routes;
     my $logged_in = $r->under('/')->to("session#ensure_user");

diff --git a/lib/OpenQA/WebAPI/Auth/Fake.pm b/lib/OpenQA/WebAPI/Auth/Fake.pm
@@ -20,13 +20,7 @@ use strict;
 require Exporter;
 our (@ISA, @EXPORT_OK);
 @ISA       = qw(Exporter);
-@EXPORT_OK = qw(auth_config auth_login auth_logout);
-
-sub auth_config {
-    my ($config) = @_;
-    # no config needed
-    return;
-}
+@EXPORT_OK = qw(auth_login auth_logout);
 
 sub auth_logout {
     return;

diff --git a/lib/OpenQA/WebAPI/Auth/OpenID.pm b/lib/OpenQA/WebAPI/Auth/OpenID.pm
@@ -23,13 +23,7 @@ use Net::OpenID::Consumer;
 require Exporter;
 our (@ISA, @EXPORT_OK);
 @ISA       = qw(Exporter);
-@EXPORT_OK = qw(auth_config auth_login auth_response);
-
-sub auth_config {
-    my ($config) = @_;
-    # no config needed
-    return;
-}
+@EXPORT_OK = qw(auth_login auth_response);
 
 sub auth_login {
     my ($self) = @_;

diff --git a/lib/OpenQA/WebAPI/Auth/iChain.pm b/lib/OpenQA/WebAPI/Auth/iChain.pm
@@ -20,13 +20,7 @@ use strict;
 require Exporter;
 our (@ISA, @EXPORT_OK);
 @ISA       = qw(Exporter);
-@EXPORT_OK = qw(auth_config auth_login auth_logout);
-
-sub auth_config {
-    my ($config) = @_;
-    # no config needed
-    return;
-}
+@EXPORT_OK = qw(auth_login auth_logout);
 
 sub auth_logout {
     return;

diff --git a/t/16-utils.t b/t/16-utils.t
@@ -128,4 +128,37 @@ EOT
     is detect_current_version($git_dir), undef, "Git ref file shows no tag, version is undef";
 };
 
+subtest 'Plugins handling' => sub {
+
+    is path_to_class('foo/bar.pm'),     "foo::bar";
+    is path_to_class('foo/bar/baz.pm'), "foo::bar::baz";
+
+    ok grep("OpenQA::Utils", loaded_modules), "Can detect loaded modules";
+    ok grep("Test::More",    loaded_modules), "Can detect loaded modules";
+
+    is_deeply [loaded_plugins("OpenQA::Utils", "Test::More")], ["OpenQA::Utils", "Test::More"],
+      "Can detect loaded plugins, filtering by namespace";
+    ok grep("Test::More", loaded_plugins),
+      "loaded_plugins() behave like loaded_modules() when no arguments are supplied";
+
+    my $test_hash = {
+        auth => {
+            method => "Fake",
+            foo    => "bar"
+        },
+        baz => {
+            bar => "test"
+        }};
+
+    my $reconstructed_hash;
+
+    hashwalker $test_hash => sub {
+        my ($key, $value, $keys) = @_;
+        $reconstructed_hash->{@$keys[0]}->{$key} = $value;
+    };
+
+    is_deeply $reconstructed_hash, $test_hash, "hashwalker() reconstructed original hash correctly";
+
+};
+
 done_testing;
diff --git a/t/25-serverstartup.t b/t/25-serverstartup.t
@@ -15,6 +15,16 @@
 
 BEGIN {
     unshift @INC, 'lib';
+
+    package OpenQA::FakePlugin::Fuzz;
+    use Mojo::Base -base;
+
+    has 'configuration_fields' => sub {
+        {
+            baz => {
+                test => 1
+            }};
+    };
 }
 
 use strict;
@@ -103,8 +113,79 @@ subtest 'Setup logging to file (ENV)' => sub {
     like $content, qr/\[.*\] \[test:info\] Works too/,    'right info message';
 };
 
+subtest 'Update configuration from Plugin requirements' => sub {
+    use Config::IniFiles;
+    use OpenQA::FakePlugin::Foo;
+    use OpenQA::FakePlugin::FooBar;
+    use OpenQA::FakePlugin::FooBaz;
+    use Mojolicious;
+
+    my $config;
+    $config->{ini_config} = Config::IniFiles->new();
+    $config->{ini_config}->AddSection("auth");
+    $config->{ini_config}->AddSection("bar");
+    $config->{ini_config}->AddSection("baz");
+    $config->{ini_config}->AddSection("bazzer");
+    $config->{ini_config}->AddSection("foofoo");
+
+    $config->{ini_config}->newval("auth",   "method",   "foobar");
+    $config->{ini_config}->newval("bar",    "foo",      "test");
+    $config->{ini_config}->newval("baz",    "foo",      "test2");
+    $config->{ini_config}->newval("baz",    "test",     "bartest");
+    $config->{ini_config}->newval("bazzer", "realfoo",  "win");
+    $config->{ini_config}->newval("foofoo", "is_there", "wohoo");
+
+    # Check if  Config::IniFiles object returns the right values
+    is $config->{ini_config}->val("auth", "method"), "foobar",
+      "Ini parser contains the right data for OpenQA::FakePlugin::Foo";
+    is $config->{ini_config}->val("bar", "foo"), "test",
+      "Ini parser contains the right data for OpenQA::FakePlugin::FooBar";
+    is $config->{ini_config}->val("baz", "foo"), "test2",
+      "Ini parser contains the right data for OpenQA::FakePlugin::FooBaz";
+    is $config->{ini_config}->val("baz", "test"), "bartest",
+      "Ini parser contains the right data for OpenQA::FakePlugin::Fuzz";
+    is $config->{ini_config}->val("bazzer", "realfoo"), "win",
+      "Ini parser contains the right data for OpenQA::FakePlugin::Fuzzer";
+    is $config->{ini_config}->val("foofoo", "is_there"), "wohoo",
+      "Ini parser contains the right data for OpenQA::FakePlugin::FooFoo";
+
+    # inline packages declaration needs to appear as "loaded"
+    $INC{"OpenQA/FakePlugin/Fuzz.pm"}   = undef;
+    $INC{"OpenQA/FakePlugin/Fuzzer.pm"} = undef;
+    OpenQA::ServerStartup::update_config($config, "OpenQA::FakePlugin");
+
+    ok exists($config->{auth}->{method}),      "Config option exists for OpenQA::FakePlugin::Foo";
+    ok exists($config->{bar}->{foo}),          "Config option exists for OpenQA::FakePlugin::FooBar";
+    ok exists($config->{baz}->{foo}),          "Config option exists for OpenQA::FakePlugin::FooBaz";
+    ok exists($config->{baz}->{test}),         "Config option exists for OpenQA::FakePlugin::Fuzz";
+    ok exists($config->{bazzer}->{realfoo}),   "Config option exists for OpenQA::FakePlugin::Fuzzer";
+    ok !exists($config->{foofoo}->{is_there}), "Config option doesn't exists(yet) for OpenQA::FakePlugin::Foofoo";
+
+    is $config->{auth}->{method},    "foobar",  "Right config option for OpenQA::FakePlugin::Foo";
+    is $config->{bar}->{foo},        "test",    "Right config option for OpenQA::FakePlugin::FooBar";
+    is $config->{baz}->{foo},        "test2",   "Right config option for OpenQA::FakePlugin::FooBaz";
+    is $config->{baz}->{test},       "bartest", "Right config option for OpenQA::FakePlugin::Fuzz";
+    is $config->{bazzer}->{realfoo}, "win",     "Right config option for OpenQA::FakePlugin::Fuzzer";
+
+    my $app = Mojolicious->new();
+    push @{$app->plugins->namespaces}, "OpenQA::FakePlugin";
+    $app->config->{ini_config} = $config->{ini_config};
+    $app->plugin("FooFoo");
+    OpenQA::ServerStartup::update_config($app->config, "OpenQA::FakePlugin");
+    is $app->config->{foofoo}->{is_there}, "wohoo", "Right config option for OpenQA::FakePlugin::Foofoo";
+};
 done_testing();
 
+package OpenQA::FakePlugin::Fuzzer;
+use Mojo::Base -base;
+
+sub configuration_fields {
+    {
+        bazzer => {
+            realfoo => 1
+        }};
+}
+
 package db_profiler;
 no warnings 'redefine';
 sub enable_sql_debugging {

diff --git a/t/lib/OpenQA/FakePlugin/Foo.pm b/t/lib/OpenQA/FakePlugin/Foo.pm
@@ -0,0 +1,9 @@
+package OpenQA::FakePlugin::Foo;
+use Mojo::Base -base;
+has 'configuration_fields' => sub {
+    {
+        auth => {
+            method => 1
+        }};
+};
+1;
diff --git a/t/lib/OpenQA/FakePlugin/FooBar.pm b/t/lib/OpenQA/FakePlugin/FooBar.pm
@@ -0,0 +1,8 @@
+package OpenQA::FakePlugin::FooBar;
+sub configuration_fields {
+    {
+        bar => {
+            foo => 1
+        }};
+}
+1;
diff --git a/t/lib/OpenQA/FakePlugin/FooBaz.pm b/t/lib/OpenQA/FakePlugin/FooBaz.pm
@@ -0,0 +1,10 @@
+package OpenQA::FakePlugin::FooBaz;
+use Mojo::Base -base;
+
+sub configuration_fields {
+    {
+        baz => {
+            foo => 1
+        }};
+}
+1;
diff --git a/t/lib/OpenQA/FakePlugin/FooFoo.pm b/t/lib/OpenQA/FakePlugin/FooFoo.pm
@@ -0,0 +1,12 @@
+package OpenQA::FakePlugin::FooFoo;
+use Mojo::Base 'Mojolicious::Plugin';
+has 'configuration_fields' => sub {
+    {
+        foofoo => {
+            is_there => 1
+        }};
+};
+sub register {
+    1;
+}
+1;