Fixes #13013 - Adds API to display pulp dir status

.gitignore

@@ -0,0 +1,3 @@
+.idea/
+Gemfile.lock
+logs/

Gemfile

@@ -2,5 +2,5 @@ source 'https://rubygems.org'
 gemspec
 
 group :development do
-  gem 'smart_proxy', :git => "https://github.com/theforeman/smart-proxy"
+  gem 'smart_proxy', :git => 'https://github.com/theforeman/smart-proxy', :branch => 'develop'
 end

lib/smart_proxy_pulp_plugin/disk_usage.rb

@@ -0,0 +1,71 @@
+module PulpProxy
+  class DiskUsage
+    include ::Proxy::Util
+    include ::Proxy::Log
+    SIZE = { :kilobyte => 1_024, :megabyte => 1_048_576, :gigabyte => 1_073_741_824, :terabyte => 1_099_511_627_776 }
+
+    attr_reader :path, :stat, :size
+
+    def initialize(opts ={})
+      raise(::Proxy::Error::ConfigurationError, 'Unable to continue - must provide a path.') if opts[:path].nil?
+      @paths_hash = path_hash(opts[:path])
+      @path = @paths_hash.values
+      @size = SIZE[opts[:size]] || SIZE[:kilobyte]
+      @stat = {}
+      find_df
+      get_stat
+    end
+
+    def to_json
+      stat.to_json
+    end
+
+    private
+
+    attr_reader :command_path
+
+    def find_df
+      @command_path = which('df') || raise(::Proxy::Error::ConfigurationError, 'df command was not found unable to retrieve usage information.')
+    end
+
+    def command
+      [command_path, "-B", "#{size}", *path]
+    end
+
+    def get_stat
+      raw = Open3::popen3({"LC_ALL" => "C"}, *command) do |stdin, stdout, stderr, thread|
+        unless stderr.read.empty?
+          error_line = stderr.read
+          logger.error "[#{command_path}] #{error_line}"
+          raise(::Proxy::Error::ConfigurationError, "#{command_path} raised an error: #{error_line}")
+        end
+        stdout.read.split("\n")
+      end
+      logger.debug "[#{command_path}] #{raw.to_s}"
+
+      titles = normalize_titles(raw)
+      raw.each_with_index do |line, index|
+        mount_path = path[index]
+        values = normalize_values(line.split)
+        @stat[@paths_hash.key(mount_path)] = Hash[titles.zip(values)].merge({:path => mount_path, :size => SIZE.key(size)})
+      end
+    end
+
+    def path_hash(path)
+      path.is_a?(Hash) ? path : Hash[path, path]
+    end
+
+    def normalize_titles(raw)
+      replacers = {"mounted on" => :mounted, "use%" => :percent}
+      raw.shift.downcase.gsub(/(use%|mounted on)/) { |m| replacers.fetch(m,m)}.split.map(&:to_sym)
+    end
+
+    def normalize_values(values)
+      values.each_with_index do |value, index|
+        is_int = Integer(value) rescue false
+        values[index] = is_int if is_int
+      end
+      values
+    end
+  end
+end

lib/smart_proxy_pulp_plugin/pulp_api.rb

@@ -1,5 +1,6 @@
 require 'sinatra'
 require 'smart_proxy_pulp_plugin/pulp_client'
+require 'smart_proxy_pulp_plugin/disk_usage'
 
 module PulpProxy
   class Api < Sinatra::Base
@@ -17,5 +18,17 @@ class Api < Sinatra::Base
         log_halt 503, "Pulp server '#{URI.parse(::PulpProxy::Plugin.settings.pulp_url.to_s).host}' is unknown"
       end
     end
+
+    get '/status/disk_usage' do
+      size = (params[:size] && DiskUsage::SIZE.keys.include?(params[:size].to_sym)) ? params[:size].to_sym : :kilobyte
+      monitor_dirs = ::PulpProxy::Plugin.settings.to_h.select { |key, _| key == :pulp_dir || key == :pulp_content_dir || key == :mongodb_dir }
+      begin
+        pulp_disk = DiskUsage.new({:path => monitor_dirs, :size => size})
+        pulp_disk.to_json
+      rescue ::Proxy::Error::ConfigurationError
+        log_halt 500, 'Could not find df command to evaluate disk space'
+      end
+
+    end
   end
 end

lib/smart_proxy_pulp_plugin/pulp_node_plugin.rb

@@ -1,7 +1,12 @@
 module PulpMasterProxy
   class Plugin < ::Proxy::Plugin
     plugin "pulpnode", ::PulpProxy::VERSION
-    default_settings :pulp_url => 'https://localhost/pulp'
+    default_settings :pulp_url => 'https://localhost/pulp',
+                     :pulp_dir => '/var/lib/pulp',
+                     :pulp_content_dir => '/var/lib/pulp/content',
+                     :mongodb_dir => '/var/lib/mongodb'
+
+    validate_readable :pulp_dir, :pulp_content_dir, :mongodb_dir
 
     http_rackup_path File.expand_path("pulp_node_http_config.ru", File.expand_path("../", __FILE__))
     https_rackup_path File.expand_path("pulp_node_http_config.ru", File.expand_path("../", __FILE__))

lib/smart_proxy_pulp_plugin/pulp_plugin.rb

@@ -1,9 +1,13 @@
 module PulpProxy
   class Plugin < ::Proxy::Plugin
     plugin "pulp", ::PulpProxy::VERSION
-    default_settings :pulp_url => 'https://localhost/pulp'
+    default_settings :pulp_url => 'https://localhost/pulp',
+                     :pulp_dir => '/var/lib/pulp',
+                     :pulp_content_dir => '/var/lib/pulp/content',
+                     :mongodb_dir => '/var/lib/mongodb'
 
+    validate_readable :pulp_dir, :pulp_content_dir, :mongodb_dir
     http_rackup_path File.expand_path("pulp_http_config.ru", File.expand_path("../", __FILE__))
     https_rackup_path File.expand_path("pulp_http_config.ru", File.expand_path("../", __FILE__))
   end
-end
+end

settings.d/pulp.yml.example

@@ -1,3 +1,7 @@
 ---
 :enabled: true
-#:pulp_url: https://localhost/pulp
+#:pulp_url: https://localhost/pulp
+# Path to pulp, pulp content and mongodb directories
+#:pulp_dir: /var/lib/pulp
+#:pulp_content_dir: /var/lib/pulp/content
+#:mongodb_dir: /var/lib/mongodb

settings.d/pulpnode.yml.example

@@ -1,3 +1,7 @@
 ---
 :enabled: true
-#:pulp_url: https://localhost/pulp
+#:pulp_url: https://localhost/pulp
+# Path to pulp, pulp content and mongodb directories
+#:pulp_dir: /var/lib/pulp
+#:pulp_content_dir: /var/lib/pulp/content
+#:mongodb_dir: /var/lib/mongodb

test/disk_usage_test.rb

@@ -0,0 +1,52 @@
+require 'test_helper'
+require 'rack/test'
+
+require 'smart_proxy_pulp_plugin/pulp_plugin'
+require 'smart_proxy_pulp_plugin/disk_usage'
+
+class DiskUsageTest < Test::Unit::TestCase
+  include ::Proxy::Util
+  def test_has_path_should_be_true
+    disk_test = ::PulpProxy::DiskUsage.new(:path => {:root => ::Sinatra::Application.settings.root})
+    assert_equal(::Sinatra::Application.settings.root, disk_test.path.first)
+  end
+
+  def test_hash_of_paths
+    paths_array = [::Sinatra::Application.settings.root, '/tmp']
+    disk_test = ::PulpProxy::DiskUsage.new(:path => {:root => paths_array.first, :tmp => paths_array.last})
+    assert_equal(paths_array, disk_test.path)
+  end
+
+  def test_path_can_be_string
+    disk_test = ::PulpProxy::DiskUsage.new(:path => ::Sinatra::Application.settings.root)
+    assert_equal(::Sinatra::Application.settings.root, disk_test.path.first)
+  end
+
+  def test_raise_error_if_path_nil
+    assert_raises Proxy::Error::ConfigurationError do
+      ::PulpProxy::DiskUsage.new
+    end
+  end
+
+  def test_command_df
+    disk_test = ::PulpProxy::DiskUsage.new(:path => ::Sinatra::Application.settings.root)
+    assert(disk_test.send(:command).include?(which('df')))
+  end
+
+  def test_has_stat_should_be_true
+    disk_test = ::PulpProxy::DiskUsage.new(:path => {:root => ::Sinatra::Application.settings.root})
+    assert_not_nil(disk_test.stat)
+    assert(disk_test.stat.is_a?(Hash))
+  end
+
+  def test_should_return_valid_json
+    paths_array = [::Sinatra::Application.settings.root, '/tmp']
+    disk_test = ::PulpProxy::DiskUsage.new(:path => {:root => paths_array.first, :tmp => paths_array.last})
+    data = disk_test.stat
+    assert_equal([:filesystem, :"1k-blocks", :used, :available, :percent, :mounted, :path, :size], data[:root].keys)
+    json = disk_test.to_json
+    assert_nothing_raised JSON::ParserError do
+      JSON.parse(json)
+    end
+  end
+end

test/pulp_api_test.rb

@@ -1,7 +1,7 @@
 require 'test_helper'
 require 'webmock/test_unit'
 require 'mocha/test_unit'
-require "rack/test"
+require 'rack/test'
 
 require 'smart_proxy_pulp_plugin/pulp_plugin'
 require 'smart_proxy_pulp_plugin/pulp_api'
@@ -36,4 +36,34 @@ def test_returns_50X_on_socket_error
   ensure
     Net::HTTP.any_instance.unstub(:request)
   end
-end
+
+  def test_returns_pulp_disk_on_200
+    PulpProxy::Plugin.load_test_settings(:pulp_dir => ::Sinatra::Application.settings.root,
+                                         :pulp_content_dir => ::Sinatra::Application.settings.root,
+                                         :mongodb_dir => ::Sinatra::Application.settings.root)
+    get '/status/disk_usage'
+    response = JSON.parse(last_response.body)
+    assert last_response.ok?, "Last response was not ok: #{last_response.body}"
+    assert_equal(%w(filesystem 1k-blocks used available percent mounted path size), response['pulp_dir'].keys)
+  end
+
+  def test_change_pulp_disk_size
+    PulpProxy::Plugin.load_test_settings(:pulp_dir => ::Sinatra::Application.settings.root,
+                                         :pulp_content_dir => ::Sinatra::Application.settings.root,
+                                         :mongodb_dir => ::Sinatra::Application.settings.root)
+    get '/status/disk_usage?size=megabyte'
+    response = JSON.parse(last_response.body)
+    assert last_response.ok?, "Last response was not ok: #{last_response.body}"
+    assert_equal('megabyte', response['pulp_dir']['size'])
+  end
+
+  def test_default_pulp_disk_size
+    PulpProxy::Plugin.load_test_settings(:pulp_dir => ::Sinatra::Application.settings.root,
+                                         :pulp_content_dir => ::Sinatra::Application.settings.root,
+                                         :mongodb_dir => ::Sinatra::Application.settings.root)
+    get '/status/disk_usage?size=pitabyte'
+    response = JSON.parse(last_response.body)
+    assert last_response.ok?, "Last response was not ok: #{last_response.body}"
+    assert_equal('kilobyte', response['pulp_dir']['size'])
+  end
+end