Replace zip packages with plugins via git

Gemfile

@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 
 gem 'activesupport'
 gem 'eventmachine'
+gem 'git'
 gem 'mechanize'
 gem 'nokogiri'
 gem 'open_uri_redirections'

Gemfile.lock

@@ -34,6 +34,7 @@ GEM
     ffi (1.9.18)
     ffi (1.9.18-x64-mingw32)
     gemoji (3.0.0)
+    git (1.5.0)
     gli (2.16.1)
     hashie (3.5.6)
     http-cookie (1.0.3)
@@ -162,6 +163,7 @@ DEPENDENCIES
   eventmachine
   faye-websocket
   gemoji
+  git
   mechanize
   nokogiri
   open_uri_redirections

PLUGINS.md

@@ -1,12 +1,6 @@
 # Plugin development
 
-## Scaffold
-
-There is a rake task to create a skeleton plugin. This uses the Ping plugin as a template.
-
-    rake make_plugin[name]
-
-The created plugin is located at `./lib/plugins/name/name.rb`.
+Please see [betabot-example-plugin](https://github.com/gyng/betabot-example-plugin) for an example of an external (ie. not bundled) plugin.
 
 ## Example plugin
 
@@ -148,9 +142,7 @@ The method `auth_r(level, message)` replies to the message sender if authenticat
 
 ## Packaging plugins
 
-Installable plugins can be packaged with `rake package_plugin[plugin_name]`.
-
-The zip package will be located in the `./packages` folder. This package can be installed by running `rake install_plugin[http://myurl.com/plugin_name.sha31fda.plugin.zip]`. Do not change the filename as it is used in the install process.
+Installable plugins should be an accessible git repository. Please see [betabot-example-plugin](https://github.com/gyng/betabot-example-plugin) for an example of an external (ie. not bundled) plugin.
 
 ## Web hook
 
@@ -183,3 +175,11 @@ end
 ```
 
 Check out the Image plugin for a concrete example.
+
+## Default plugin scaffold
+
+There is a rake task to create a skeleton bundled plugin, but this is not intended for external developers. This uses the Ping plugin as a template.
+
+    rake make_plugin[name]
+
+The created plugin is located at `./lib/plugins/name/name.rb`.

README.md

@@ -50,7 +50,13 @@ You can choose either to use or not to use Docker to run betabot.
 
 4. [Configure the bot](#configuration)
 
-5. Start the bot
+5. Install external plugins if wanted
+
+  ```
+  rake install_plugin[https://example.com/manifest.json]
+  ```
+
+6. Start the bot
 
        docker-compose up
 
@@ -142,7 +148,7 @@ For example: `!ping`, `MyBot: ping`
 * **unicode** – Search for Unicode characters or emoji by description, and identify Unicode characters
 * **wolfram** – Queries Wolfram|Alpha. *[requires setup of API keys](https://developer.wolframalpha.com/portal/apisignup.html)*
 
-For all plugins, see the plugin directory.
+For all plugins, see the plugins and external_plugins directory.
 
 #### A few core commands
 
@@ -166,8 +172,8 @@ For all plugins, see the plugin directory.
 
 To install a plugin from a URL:
 
-1. Run `rake install_plugin[http://www.example.com/myplugin.af84ad46.package.zip]`.
-2. Run `bundle install` to install plugin dependencies.
+1. Run `rake install_plugin[$MANIFEST_URL]`. An example can be installed using `rake install_plugin[https://raw.githubusercontent.com/gyng/betabot-example-plugin/master/manifest.json]`.
+2. Run `bundle install` to install plugin dependencies, if needed.
 3. If there is a running bot instance, `reload` to reload all plugins.
 
 ### Plugin development
@@ -176,7 +182,7 @@ See: [Plugin development](PLUGINS.md)
 
 ## Tests
 
-Tests can be run with `rspec`. Current test coverage is very limited.
+Tests can be run with `bundle exec rspec`. Current test coverage is very limited. Ruocop can be run with `bundle exec rubocop`.
 
 The EventMachine reactor has to be set up and torn down for most specs so there is a helper method `with_em(&block)` included in `spec_helper.rb`.
 

Rakefile

@@ -1,5 +1,3 @@
-# rubocop:disable Metrics/BlockLength
-
 task :make_user do
   require 'io/console'
   require 'json'
@@ -65,64 +63,49 @@ task :make_plugin, :name do |_t, args|
   File.write(File.join(dir, 'settings', '.gitignore'), 'settings.json')
 end
 
-task :package_plugin, :name do |_t, args|
-  require 'zip'
-  require 'fileutils'
-  require 'digest/sha1'
+task :install_plugin, :url do |_t, args|
+  require 'git'
+  require 'json'
+  require 'open-uri'
 
-  packages_dir = 'packages'
-  name = args[:name]
+  url = args[:url]
 
-  FileUtils.mkdir_p(packages_dir)
-  dir = File.join('lib', 'plugins', name)
-  sha = Digest::SHA1.hexdigest(File.read(File.join(dir, "#{name}.rb")))[0..7]
-  out = File.join(packages_dir, "#{name}.#{sha}.plugin.zip")
+  puts "ℹ Grabbing manifest from #{url}..."
+  manifest = open(url).read
+
+  puts 'ℹ Parsing manifest JSON...'
+  parsed = JSON.parse(manifest, symbolize_names: true)
+  puts "ℹ Parsed manifest: #{parsed}"
+
+  repo = parsed[:git]
+  plugin_name = parsed[:name]
+  puts "ℹ Git repo is at #{repo}"
 
-  Zip::File.open(out, Zip::File::CREATE) do |zipfile|
-    Dir[File.join(dir, '**', '**')].each do |file|
-      zipfile.add(file.sub(dir + File::SEPARATOR, ''), file)
-    end
+  plugins_dir = 'lib/plugins'
+  FileUtils.mkdir_p(plugins_dir)
+  plugin_path = File.join(plugins_dir, plugin_name)
+
+  if File.directory?(plugin_path)
+    puts "🔥 Directory #{plugin_path} already exists! Delete it first or run `rake update_plugin[#{plugin_name}]`"
+    next
   end
-  puts "Package created in #{out}."
-end
 
-task :install_plugin, :url do |_t, args|
-  require 'zip'
-  require 'fileutils'
-  require 'openssl'
-  require 'open-uri'
-  require 'uri'
+  puts 'ℹ Cloning plugin...'
+  Git.clone(repo, plugin_name, path: plugins_dir)
 
-  url = args[:url]
-  packages_dir = 'packages'
-  FileUtils.mkdir_p(packages_dir)
+  puts "ℹ Plugin #{plugin_name} installed. Run `bundle install` if needed."
+end
 
-  package_name = File.basename(URI.parse(url).path)
-  package_path = File.join(packages_dir, package_name)
+task :update_plugin, :name do |_t, args|
+  require 'git'
 
-  puts "Downloading package from #{url}"
-  open(url, ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE) do |f|
-    File.open(package_path, 'wb') do |file|
-      file.puts f.read
-    end
-  end
+  name = args[:name]
+  plugin_path = File.join('lib', 'external_plugins', name)
 
-  puts 'Package downloaded. Extracting...'
-  plugin_name = package_name.match(/(?<name>.+?)\..+/)[:name]
-  plugin_dir = File.join('lib', 'plugins', plugin_name)
-  FileUtils.mkdir_p(plugin_dir)
-
-  Zip::File.open(package_path) do |zip_file|
-    zip_file.each do |f|
-      f_path = File.join(plugin_dir, f.name)
-      FileUtils.mkdir_p(File.dirname(f_path))
-      zip_file.extract(f, f_path) unless File.exist?(f_path)
-    end
-  end
+  puts "ℹ Updating plugin #{name} at #{plugin_path}..."
 
-  puts 'Cleaning up downloaded files...'
-  File.delete(package_path)
+  g = Git.open(plugin_path)
+  g.pull
 
-  puts '\nPlugin #{plugin_name} installed to #{plugin_dir}!\n' \
-       'Run `bundle install` to install plugin dependencies.'
+  puts "ℹ Plugin #{name} updated to #{g.show('HEAD')}."
 end

lib/bot/core.rb

@@ -48,6 +48,7 @@ def initialize(bot_settings_path)
 
       initialize_objects(:adapter)
       initialize_objects(:plugin)
+      initialize_objects(:external_plugin)
       Bot.log.info "#{@adapters.length} adapter(s) and #{@plugins.length} plugin(s) loaded."
       @s[:adapters][:autostart].each { |regex| start_adapters(regex) }
     end
@@ -61,11 +62,12 @@ def initialize_objects(type)
       end
 
       # rubocop:disable Style/GuardClause
-      if type == :plugin || type.nil?
+      if type == :plugin || type == :external_plugin || type.nil?
         @plugins = {}
         @plugin_mapping = {}
         @subscribed_plugins = []
         load_objects(:plugin, mode)
+        load_objects(:external_plugin, mode)
       end
       # rubocop:enable Style/GuardClause
     end
@@ -93,6 +95,7 @@ def core_triggers(trigger, m)
           restart
         when 'reload'
           reload(:plugin)
+          reload(:external_plugin)
           m.reply 'Reloaded.' if m.respond_to? :reply
         when 'useradd'
           @authenticator.make_account(m.args[0], m.args[1], m.args[2])
@@ -109,6 +112,7 @@ def core_triggers(trigger, m)
         when 'unblacklist_plugin'
           unblacklist(:plugin, m.args[0])
           m.reply "Plugin #{m.args[0]} unblacklisted. Reload for this to take effect."
+        # TODO: Blacklist external plugins
         when 'blacklist'
           m.reply 'Adapters: ' + @s[:adapters][:blacklist].join(', ')
           m.reply 'Plugins: ' + @s[:plugins][:blacklist].join(', ')
@@ -199,6 +203,7 @@ def reload(type, name = nil)
         @plugins = nil
         initialize_objects(:adapter)
         initialize_objects(:plugin)
+        initialize_objects(:external_plugin)
       elsif name.nil?
         initialize_objects(type)
       else

lib/bot/core/object_loader.rb

@@ -5,6 +5,7 @@ def get_objects_dir(type)
   end
 
   def load_objects(type, mode = :all)
+    Bot.log.info "Loading #{type}s..."
     type = type.to_s
     objects_dir = get_objects_dir(type)
 
@@ -20,13 +21,24 @@ def load_objects(type, mode = :all)
   def load_curry(type)
     proc do |f|
       begin
-        Bot.log.info "Loading #{type} #{f}..."
         path = File.join(get_objects_dir(type), f.to_s)
-        load File.join(path, "#{f}.rb")
+        full_path = File.join(path, "#{f}.rb")
+        Bot.log.info "Loading #{type} #{f} from #{full_path}..."
+        load full_path
+
+        types = {
+          'plugin' => :plugin,
+          'external_plugin' => :plugin,
+          'adapter' => :adapter
+        }
+        actual_type = types[type]
+
         # Initialize the loaded object
-        object = Bot.module_eval(type.capitalize.to_s).const_get(f.capitalize).new(self)
+        object = Bot.module_eval(actual_type.capitalize.to_s).const_get(f.capitalize).new(self)
         # And store a reference to that object in @types (eg. @plugins)
-        instance_eval("@#{type}s")[f.downcase.to_sym] = object
+        # Store external plugins in the plugins list
+        # TODO: check for name collisions
+        instance_eval("@#{actual_type}s")[f.downcase.to_sym] = object
       rescue LoadError, StandardError, SyntaxError => e
         Bot.log.warn "Failed to load #{f} - #{e}\n\t#{e.backtrace.join("\n\t")}"
       end

lib/settings/bot_settings.json

@@ -16,6 +16,13 @@
     "blacklist": ["dummy", "script"]
   },
 
+  "external_plugins": {
+    "dir": "external_plugins",
+    "load_mode": "blacklist",
+    "whitelist": [],
+    "blacklist": []
+  },
+
   "databases": {
     "shared_db": true
   },

spec/fixtures/data/bot_settings_blacklist_fixture.json

@@ -1,29 +1,36 @@
 {
-    "short_trigger": "~",
+  "short_trigger": "~",
 
-    "adapters": {
-        "dir": ["adapters"],
-        "load_mode": "blacklist",
-        "autostart": [],
-        "whitelist": ["dummy"],
-        "blacklist": ["irc"]
-    },
+  "adapters": {
+    "dir": ["adapters"],
+    "load_mode": "blacklist",
+    "autostart": [],
+    "whitelist": ["dummy"],
+    "blacklist": ["irc"]
+  },
 
-    "plugins": {
-        "dir": "plugins",
-        "load_mode": "whitelist",
-        "whitelist": [],
-        "blacklist": ["dummy", "ping"]
-    },
+  "plugins": {
+    "dir": "plugins",
+    "load_mode": "whitelist",
+    "whitelist": [],
+    "blacklist": ["dummy", "ping"]
+  },
 
-    "databases": {
-        "shared_db": false
-    },
+  "external_plugins": {
+    "dir": "external_plugins",
+    "load_mode": "blacklist",
+    "whitelist": [],
+    "blacklist": []
+  },
 
-    "webserver": {
-        "enabled": false,
-        "link_url": "http://localhost:80",
-        "host": "0.0.0.0",
-        "port": "80"
-    }
+  "databases": {
+    "shared_db": false
+  },
+
+  "webserver": {
+    "enabled": false,
+    "link_url": "http://localhost:80",
+    "host": "0.0.0.0",
+    "port": "80"
+  }
 }