Skip to content

Commit

Permalink
finished off beta 1 of the monitoring system, working daemon, working…
Browse files Browse the repository at this point in the history
… notifications, etc
  • Loading branch information
Jonathan Hoyt authored and Jonathan Hoyt committed Dec 27, 2008
1 parent 64627c1 commit ece1647
Show file tree
Hide file tree
Showing 20 changed files with 104 additions and 32 deletions.
6 changes: 6 additions & 0 deletions app/controllers/devices_controller.rb
Expand Up @@ -7,6 +7,12 @@ def index
@devices = Device.find(:all, :conditions => {:client_id => params[:client_id]})
elsif params[:q]
@devices = Device.search(params[:q], :include => [:client])
elsif params[:status]
if params[:status] == "down"
@devices = Device.find_all_in_trouble
else
@devices = []
end
else
@devices = []
end
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/sentries_controller.rb
Expand Up @@ -18,7 +18,7 @@ def edit
end

def create
@sentry = Sentry.nfew(params[:sentry])
@sentry = Sentry.new(params[:sentry])
if @sentry.save
flash[:notice] = "Sentry created successfully."
redirect_to :back
Expand Down
19 changes: 19 additions & 0 deletions app/models/device.rb
Expand Up @@ -52,6 +52,25 @@ def to_json(options={})
super(options.merge(:methods => :client_name))
end

def healthy?
state = 0
self.sentries.each do |sentry|
if sentry.state == false
state += 1
end
end
state == 0 ? true : false
end

def self.find_all_in_trouble
all_devices = Device.find(:all)
devices_in_trouble = []
all_devices.each do |device|
!device.healthy? ? devices_in_trouble << device : false
end
return devices_in_trouble
end

def generate
sma_dir = RAILS_ROOT+"/lib/sma/"
devices_dir = sma_dir+"devices/"
Expand Down
1 change: 1 addition & 0 deletions app/models/email.rb
Expand Up @@ -2,5 +2,6 @@ class Email < ActiveRecord::Base
belongs_to :client

validates_uniqueness_of :address
validates_format_of :address, :with => /^([^@\s]+)@((?:[-a-z0-9]+\.)+[a-z]{2,})$/i, :message => 'is not a valid email address'

end
13 changes: 13 additions & 0 deletions app/models/notification_mailer.rb
@@ -0,0 +1,13 @@
class NotificationMailer < ActionMailer::Base

def notification(subject, message, schedule)
technician = Client.find(schedule.user.client_id)
email = Email.find(:first, :conditions => {:client_id => technician.id})
@recipients = "#{email.address}"
@from = APP_CONFIG[:admin_email]
@subject = "#{subject}"
@sent_on = Time.now
@body[:message] = message
end

end
1 change: 1 addition & 0 deletions app/models/notification_queue.rb
@@ -1,2 +1,3 @@
class NotificationQueue < ActiveRecord::Base
belongs_to :schedule
end
1 change: 1 addition & 0 deletions app/models/schedule.rb
@@ -1,6 +1,7 @@
class Schedule < ActiveRecord::Base
belongs_to :user
has_many :sentries
has_many :notification_queues

validates_presence_of :user_id

Expand Down
4 changes: 2 additions & 2 deletions app/models/sentry.rb
Expand Up @@ -8,8 +8,8 @@ def survey!
StatusLang.run(self.id, self.goggle.script) ? true : false
end

def notify!(message=nil)
NotificationQueue.create(:message => message, :schedule_id => self.schedule_id)
def notify!(subject, message=nil)
NotificationMailer.deliver_notification(subject, message, self.schedule)
end

def to_json(options={})
Expand Down
3 changes: 2 additions & 1 deletion app/views/devices/_sentries.html.erb
Expand Up @@ -28,14 +28,15 @@
<h3>Sentries for this Device</h3>
<table class="itu">
<tr>
<th>ID</th><th>Last Message</th><th>State</th><th>Last Checked</th>
<th>ID</th><th>Last Message</th><th>State</th><th>Last Checked</th><th>Last Notified</th>
</tr>
<% @device.sentries.each do |sentry| %>
<tr alt="<%= sentry.message %>" id="sentry_<%= sentry.id %>">
<td><a href="#"><%= sentry.id %></a></td>
<td><a href="#"><%= sentry.message %></a></td>
<td><a href="#"><%= sentry_state(sentry) %></a></td>
<td><a href="#"><%= sentry.last_surveyed_at ? sentry.last_surveyed_at.strftime("%H:%M, %x") : "" %></a></td>
<td><a href="#"><%= sentry.last_notified_at ? sentry.last_notified_at.strftime("%H:%M, %x") : "" %></a></td>
</tr>
<% end %>
</table>
Expand Down
18 changes: 15 additions & 3 deletions app/views/devices/index.html.erb
@@ -1,21 +1,33 @@
<div id="content_search">
<div id="applesearch"><form onsubmit="return false"><span class="sbox_l"></span><span class="sbox"><input type="search" id="srch_fld" placeholder="Search..." autosave="applestyle_srch" results="5" onkeyup="" /></span><span class="sbox_r" id="srch_clear"></span> <input id="srch_button" name="commit" type="submit" value="Search" onclick="" /></form></div>
</div>
<div id="tickets" class="hide">
<div id="tickets">
<table class="itu">
<thead>
<tr><th class="first">Service Tag</th><th>Name</th><th>Client</th><th>Created On</th><th>Updated On</th></tr>
</thead>
<tbody>

<% @devices.each do |device| %>
<tr>
<td><a href="/devices/<%= device.id %>"><%= device.service_tag %></a></td>
<td><a href="/devices/<%= device.id %>"><%= device.name %></a></td>
<td><a href="/devices/<%= device.id %>"><%= device.client.fullname %></a></td>
<td><a href="/devices/<%= device.id %>"><%= device.created_at.strftime('%A, %b %d %Y') %></a></td>
<td><a href="/devices/<%= device.id %>"><%= device.updated_at.strftime('%A, %b %d %Y') %></a></td>
</tr>
<% end %>
</tbody>
</table>
</div>

<script>
$(document).ready(function() {
// make sidebar item appear selected
$("#sidebar li a.search").parent().addClass("selected");
<% if @devices.length == 0 %>
$("#sidebar li a.search").parent().addClass("selected");
<% else %>
$("#sidebar li a.in_trouble").parent().addClass("selected");
<% end %>
// setup search functionality
$("#content_search input[type=submit]").bind("click", function(){
$("div#tickets").slideDown(1500);
Expand Down
6 changes: 5 additions & 1 deletion app/views/devices/show.html.erb
Expand Up @@ -15,7 +15,7 @@
<li class="ui-tabs-nav-item"><a href="#description" title="description">Description</a></li>
<li class="ui-tabs-nav-item"><a href="#tickets" title="tickets">Tickets</a></li>
<li class="ui-tabs-nav-item"><a href="#checklists" title="checklists">Checklists</a></li>
<li class="ui-tabs-nav-item"><a href="#sentries" title="sentries">Sentries</a></li>
<li class="ui-tabs-nav-item" id="sentries_tab"><a href="#sentries" title="sentries">Sentries</a></li>
<li class="ui-tabs-nav-item"><a href="#files" title="files">Files</a></li>
</ul>

Expand Down Expand Up @@ -99,5 +99,9 @@
})
// Change submit url
$("#new_ticket form").attr("action", "/devices/"+<%= @device.id %>+"/tickets/");
// change sentries tab color based on device health
<% if !@device.healthy? %>
$("table#device_details").css({'background-color':'#fbfbfb'});
<% end %>
});
</script>
1 change: 1 addition & 0 deletions app/views/layouts/devices.html.erb
Expand Up @@ -27,6 +27,7 @@
<h2>Devices</h2>
<ul id="devices" class="itu">
<li><a href="/devices" class="search">Search</a></li>
<li><a href="/devices/?status=down" class="in_trouble">Devices Down</a></li>
</ul>
</div>
<div id="center" class="container">
Expand Down
1 change: 1 addition & 0 deletions app/views/notification_mailer/notification.html.erb
@@ -0,0 +1 @@
<%= @message %>
4 changes: 4 additions & 0 deletions config/settings.yml.example
Expand Up @@ -5,6 +5,8 @@ development: &non_production_settings
# we use https://www.grc.com/passwords.htm to generate some unique strings
event_api_username: ln4A9DvHOs2qZXTLOg8hPx5jV5cClb9Emra129Fi0Q2cqcMvIysiL0XCSKcu5S6
event_api_password: MPQ1CBykdvLmcjRs9el0DKbKMEkdGJPT2OD4Dn1jtXNZMSoHy8f8ViMt6auJhu3
# this is used by the device sentries section, for creating a windows installer
nsis_path: "/opt/nsis/bin"

test:
<<: *non_production_settings
Expand All @@ -16,3 +18,5 @@ production:
# we use https://www.grc.com/passwords.htm to generate some unique strings
event_api_username: ln4A9DvHOs2qZXTLOg8hPx5jV5cClb9Emra129Fi0Q2cqcMvIysiL0XCSKcu5S6
event_api_password: MPQ1CBykdvLmcjRs9el0DKbKMEkdGJPT2OD4Dn1jtXNZMSoHy8f8ViMt6auJhu3
# this is used by the device sentries section, for creating a windows installer
nsis_path: "/opt/nsis/bin"
1 change: 1 addition & 0 deletions db/migrate/20081212173924_create_notification_queues.rb
Expand Up @@ -3,6 +3,7 @@ def self.up
create_table :notification_queues do |t|
t.text :message
t.integer :schedule_id
t.string :subject
end
end

Expand Down
1 change: 1 addition & 0 deletions db/migrate/20081212174343_create_goggles.rb
Expand Up @@ -6,6 +6,7 @@ def self.up
t.string :script
t.text :note
end
Goggle.create(:name => "I am alive!", :module => "i_am_alive", :script => "last(1.minutes).any_post.have('alive')")
end

def self.down
Expand Down
11 changes: 5 additions & 6 deletions db/migrate/20081212174923_create_sentries.rb
Expand Up @@ -6,15 +6,14 @@ def self.up
t.integer :device_id
t.string :parameters
t.datetime :last_surveyed_at
t.integer :survey_interval
t.integer :notifications_to_send
t.integer :maximum_notify_frequency
t.integer :notifications_sent
t.datetime :last_notified_at
t.integer :survey_interval, :default => 5, :null => false
t.integer :notifications_to_send, :default => 5, :null => false
t.integer :maximum_notify_frequency, :default => 15, :null => false
t.integer :notifications_sent, :default => 0, :null => false
t.integer :schedule_id
t.integer :goggle_id

t.timestamps
t.datetime :last_notified_at
end
end

Expand Down
13 changes: 8 additions & 5 deletions db/schema.rb
Expand Up @@ -129,13 +129,15 @@

create_table "goggles", :force => true do |t|
t.string "name"
t.string "module"
t.string "script"
t.text "note"
end

create_table "notification_queues", :force => true do |t|
t.text "message"
t.integer "schedule_id"
t.string "subject"
end

create_table "passwords", :force => true do |t|
Expand Down Expand Up @@ -190,16 +192,17 @@
t.boolean "state"
t.text "message"
t.integer "device_id"
t.string "goggle_parameters"
t.string "parameters"
t.datetime "last_surveyed_at"
t.integer "survey_interval"
t.integer "notifications_to_send"
t.integer "maximum_notify_frequency"
t.integer "notifications_sent"
t.integer "survey_interval", :default => 5, :null => false
t.integer "notifications_to_send", :default => 5, :null => false
t.integer "maximum_notify_frequency", :default => 15, :null => false
t.integer "notifications_sent", :default => 0, :null => false
t.integer "schedule_id"
t.integer "goggle_id"
t.datetime "created_at"
t.datetime "updated_at"
t.datetime "last_notified_at"
end

create_table "sessions", :force => true do |t|
Expand Down
28 changes: 16 additions & 12 deletions lib/daemon.rb → lib/sma_daemon.rb
Expand Up @@ -22,40 +22,44 @@ def delete_pidfile

def watch!
# This will naturally migrate different surveys to need run slightly apart from one another.
while(sleep(1))
while(sleep(10))
Sentry.find(:all).each do |sentry|
begin
if (sentry.last_surveyed_at.nil? || Time.now > sentry.last_surveyed_at.to_time + sentry.survey_interval)
if (sentry.last_surveyed_at.nil? || Time.now > sentry.last_surveyed_at.to_time + sentry.survey_interval*60)
begin
success = sentry.survey!
if success
loggit! "SUCCESS #{sentry.device.name} / #{sentry.goggle.name}"
loggit! "SUCCESS #{sentry.device.service_tag} / Sentry ##{sentry.id.to_s} #{sentry.goggle.name}"
sentry.last_notified_at = nil
sentry.notifications_sent = 0
sentry.state = true
else !success
loggit! "FAILURE #{sentry.device.name} / #{sentry.goggle.name}"
if sentry.last_notified_at.nil? || (sentry.notifications_to_send > 1 && Time.now > sentry.last_notified_at.to_time + sentry.maximum_notify_frequency)
if sentry.notify!
sentry.last_notified_at = Time.now
loggit! "FAILURE #{sentry.device.service_tag} / Sentry ##{sentry.id.to_s} #{sentry.goggle.name}"
sentry.state = false
if sentry.last_notified_at.nil? || (sentry.notifications_sent < sentry.notifications_to_send && Time.now > sentry.last_notified_at.to_time + sentry.maximum_notify_frequency*60)
if sentry.notify!("#{sentry.device.service_tag} / Sentry ##{sentry.id.to_s} #{sentry.goggle.name} failed", "I'm failing miserably :-(")
sentry.last_notified_at = Time.now
sentry.notifications_sent += 1
loggit! " -> NOTIFIED"
else
loggit! " -> Could Not Notify"
end
end
end
rescue => e
STDERR << loggit!("ERROR - (#{sentry.device.name} / #{sentry.goggle.name}): #{e}")
if sentry.last_notified_at.nil? || (sentry.notifications_to_send > 1 && Time.now > sentry.last_notified_at.to_time + sentry.maximum_notify_frequency)
STDERR << loggit!("ERROR - (#{sentry.device.service_tag} / #{sentry.goggle.name}): #{e}")
if sentry.last_notified_at.nil? || (sentry.notifications_sent < sentry.notifications_to_send && Time.now > sentry.last_notified_at.to_time + sentry.maximum_notify_frequency*60)
loggit! " -> NOTIFIED of ERROR"
sentry.last_notified_at = Time.now if sentry.notify!("Sentry #{sentry.device.name} / #{sentry.goggle.name} errored: #{e}")
sentry.last_notified_at = Time.now if sentry.notify!("#{sentry.device.service_tag} / Sentry ##{sentry.id.to_s} #{sentry.goggle.name} errored: #{e}", "I have failed...")
end
ensure
sentry.last_surveyed_at = Time.now
sentry.save
end
end
rescue => e
STDERR << loggit!("ERROR - Was going to cause exit on Sentry #{sentry.device.name}! Error: #{e}")
sentry.notify!("Sentry #{sentry.device.name} Exit-causing error: #{e}", 'dcparker@gmail.com')
STDERR << loggit!("ERROR - Was going to cause exit on Sentry #{sentry.device.service_tag}! Error: #{e}")
sentry.notify!("#{sentry.device.service_tag} / Sentry ##{sentry.id.to_s} #{sentry.goggle.name} Exit-causing error: #{e}")
end
end
end
Expand Down
2 changes: 1 addition & 1 deletion lib/statuslang/lang.rb
Expand Up @@ -20,7 +20,7 @@ def last(amount=nil)

def posts
if @options[:after]
Event.find(:all, :conditions => {:recordable_type => "Sentry", :recordable_id => @sentry_id, :created_at.gte => @options[:after]})
Event.find(:all, :conditions => {:recordable_type => "Sentry", :recordable_id => @sentry_id, :created_at.gte => @options[:after] + 5.hours})
elsif @options[:limit]
Event.find(:all, :conditions => {:recordable_type => "Sentry", :recordable_id => @sentry_id}, :limit => @options[:limit])
end
Expand Down

0 comments on commit ece1647

Please sign in to comment.