From 29308b0b20f02d2eb2ac46475187a4a98250b7a6 Mon Sep 17 00:00:00 2001
From: Joshua Sierles <jsierles@gmail.com>
Date: Mon, 19 Apr 2010 12:30:22 +0200
Subject: [PATCH] add utilities recipe

---
 utilities/files/default/logroll              |   1 +
 utilities/files/default/memory_stats         | 279 +++++++++++++++++++
 utilities/files/default/rotate-db-backups    |   4 +
 utilities/files/default/rotate-email-folders |  25 ++
 utilities/files/default/rotate-misc-log      |   7 +
 utilities/metadata.rb                        |   4 +
 utilities/recipes/default.rb                 |   8 +
 7 files changed, 328 insertions(+)
 create mode 100644 utilities/files/default/logroll
 create mode 100644 utilities/files/default/memory_stats
 create mode 100644 utilities/files/default/rotate-db-backups
 create mode 100644 utilities/files/default/rotate-email-folders
 create mode 100644 utilities/files/default/rotate-misc-log
 create mode 100644 utilities/metadata.rb
 create mode 100644 utilities/recipes/default.rb

diff --git a/utilities/files/default/logroll b/utilities/files/default/logroll
new file mode 100644
index 00000000..935d0b58
--- /dev/null
+++ b/utilities/files/default/logroll
@@ -0,0 +1 @@
+tail -150000 completed.log | grep -E 'Completed in [0-9][0-9][0-9][0-9]+.*destroy'
diff --git a/utilities/files/default/memory_stats b/utilities/files/default/memory_stats
new file mode 100644
index 00000000..0e8d9182
--- /dev/null
+++ b/utilities/files/default/memory_stats
@@ -0,0 +1,279 @@
+#!/usr/bin/env ruby
+#  Adopted from Phusion Passenger - http://www.modrails.com/
+#  Copyright (c) 2008, 2009 Phusion, 37signals
+#
+#  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in
+#  all copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+#  THE SOFTWARE.
+
+# ANSI color codes
+RESET   = "\e[0m"
+BOLD    = "\e[1m"
+WHITE   = "\e[37m"
+YELLOW  = "\e[33m"
+BLUE_BG = "\e[44m"
+
+# Container for tabular data.
+class Table
+  def initialize(column_names)
+    @column_names = column_names
+    @rows = []
+  end
+  
+  def add_row(values)
+    @rows << values.to_a
+  end
+  
+  def add_rows(list_of_rows)
+    list_of_rows.each do |row|
+      add_row(row)
+    end
+  end
+  
+  def remove_column(name)
+    i = @column_names.index(name)
+    @column_names.delete_at(i)
+    @rows.each do |row|
+      row.delete_at(i)
+    end
+  end
+  
+  def to_s(title = nil)
+    max_column_widths = [1] * @column_names.size
+    (@rows + [@column_names]).each do |row|
+      row.each_with_index do |value, i|
+        max_column_widths[i] = [value.to_s.size, max_column_widths[i]].max
+      end
+    end
+    
+    format_string = max_column_widths.map{ |i| "%#{-i}s" }.join("  ")
+    header = sprintf(format_string, *@column_names).rstrip << "\n"
+    if title
+      free_space = header.size - title.size - 2
+      if free_space <= 0
+        left_bar_size = 3
+        right_bar_size = 3
+      else
+        left_bar_size = free_space / 2
+        right_bar_size = free_space - left_bar_size
+      end
+      result = "#{BLUE_BG}#{BOLD}#{YELLOW}"
+      result << "#{"-" * left_bar_size} #{title} #{"-" * right_bar_size}\n"
+      if !@rows.empty?
+        result << WHITE
+        result << header
+      end
+    else
+      result = header.dup
+    end
+    if @rows.empty?
+      result << RESET
+    else
+      result << ("-" * header.size) << "#{RESET}\n"
+      @rows.each do |row|
+        result << sprintf(format_string, *row).rstrip << "\n"
+      end
+    end
+    result
+  end
+end
+
+class MemoryStats
+  class Process
+    attr_accessor :pid
+    attr_accessor :ppid
+    attr_accessor :threads
+    attr_accessor :vm_size              # in KB
+    attr_accessor :rss                  # in KB
+    attr_accessor :name
+    attr_accessor :private_dirty_rss    # in KB
+    
+    def vm_size_in_mb
+      return sprintf("%.1f MB", vm_size / 1024.0)
+    end
+
+    def rss_in_mb
+      return sprintf("%.1f MB", rss / 1024.0)
+    end
+    
+    def private_dirty_rss_in_mb
+      if private_dirty_rss.is_a?(Numeric)
+        return sprintf("%.1f MB", private_dirty_rss / 1024.0)
+      else
+        return "?"
+      end
+    end
+    
+    def to_a
+      return [pid, ppid, threads, vm_size_in_mb, private_dirty_rss_in_mb, rss_in_mb, name]
+    end
+  end
+
+  def start(match = 'rails')
+    procs = list_processes(:match => match).delete_if {|p| p.pid == ::Process.pid }
+    if !procs.empty?
+      if ::Process.uid != 0 && procs.any?{ |p| p.private_dirty_rss.nil? }
+        puts
+        puts "*** WARNING: Please run this tool as root. Otherwise the " <<
+          "private dirty RSS of processes cannot be determined."
+      end
+      print_process_list("#{match}", procs)
+    else
+      puts "No processes found matching '#{match}'"
+    end
+  end
+
+  # Returns a list of Process objects that match the given search criteria.
+  #
+  #  # Search by executable path.
+  #  list_processes(:exe => '/usr/sbin/apache2')
+  #  
+  #  # Search by executable name.
+  #  list_processes(:name => 'ruby1.8')
+  #  
+  #  # Search by process name.
+  #  list_processes(:match => 'Passenger FrameworkSpawner')
+  def list_processes(options)
+    if options[:exe]
+      name = options[:exe].sub(/.*\/(.*)/, '\1')
+      if RUBY_PLATFORM =~ /linux/
+        ps = "ps -C '#{name}'"
+      else
+        ps = "ps -A"
+        options[:match] = Regexp.new(Regexp.escape(name))
+      end
+    elsif options[:name]
+      if RUBY_PLATFORM =~ /linux/
+        ps = "ps -C '#{options[:name]}'"
+      else
+        ps = "ps -A"
+        options[:match] = Regexp.new(" #{Regexp.escape(options[:name])}")
+      end
+    elsif options[:match]
+      ps = "ps -A"
+    else
+      raise ArgumentError, "Invalid options."
+    end
+    
+    processes = []
+    case RUBY_PLATFORM
+    when /solaris/
+      list = `#{ps} -o pid,ppid,nlwp,vsz,rss,comm`.split("\n")
+      threads_known = true
+    when /darwin/
+      list = `#{ps} -w -o pid,ppid,vsz,rss,command`.split("\n")
+      threads_known = false
+    else
+      list = `#{ps} -w -o pid,ppid,nlwp,vsz,rss,command`.split("\n")
+      threads_known = true
+    end
+    list.shift
+    list.each do |line|
+      line.gsub!(/^ */, '')
+      line.gsub!(/ *$/, '')
+      
+      p = Process.new
+      if threads_known
+        p.pid, p.ppid, p.threads, p.vm_size, p.rss, p.name = line.split(/ +/, 6)
+      else
+        p.pid, p.ppid, p.vm_size, p.rss, p.name = line.split(/ +/, 5)
+        p.threads = "?"
+      end
+      p.name.sub!(/\Aruby: /, '')
+      p.name.sub!(/ \(ruby\)\Z/, '')
+      if p.name !~ /^ps/ && (!options[:match] || p.name.match(options[:match]))
+        # Convert some values to integer.
+        [:pid, :ppid, :vm_size, :rss].each do |attr|
+          p.send("#{attr}=", p.send(attr).to_i)
+        end
+        p.threads = p.threads.to_i if threads_known
+
+        if platform_provides_private_dirty_rss_information?
+          p.private_dirty_rss = determine_private_dirty_rss(p.pid)
+        end
+        processes << p
+      end
+    end
+    return processes
+  end
+
+private
+  def platform_provides_private_dirty_rss_information?
+    return RUBY_PLATFORM =~ /linux/
+  end
+
+  # Returns the private dirty RSS for the given process, in KB.
+  def determine_private_dirty_rss(pid)
+    total = 0
+    File.read("/proc/#{pid}/smaps").split("\n").each do |line|
+      line =~ /^(Private)_Dirty: +(\d+)/
+      if $2
+        total += $2.to_i
+      end
+    end
+    if total == 0
+      return nil
+    else
+      return total
+    end
+  rescue Errno::EACCES, Errno::ENOENT
+    return nil
+  end
+  
+  def print_process_list(title, processes, options = {})
+    table = Table.new(%w{PID PPID Threads VMSize Private Resident Name})
+    table.add_rows(processes)
+    if options.has_key?(:show_ppid) && !options[:show_ppid]
+      table.remove_column('PPID')
+    end
+    if platform_provides_private_dirty_rss_information?
+      table.remove_column('Resident')
+    else
+      table.remove_column('Private')
+    end
+    puts table.to_s(title)
+    
+    if platform_provides_private_dirty_rss_information?
+      total_private_dirty_rss = 0
+      some_private_dirty_rss_cannot_be_determined = false
+      processes.each do |p|
+        if p.private_dirty_rss.is_a?(Numeric)
+          total_private_dirty_rss += p.private_dirty_rss
+        else
+          some_private_dirty_rss_cannot_be_determined = true
+        end
+      end
+      puts   "### Processes: #{processes.size}"
+      printf "### Total private dirty RSS: %.2f MB", total_private_dirty_rss / 1024.0
+      if some_private_dirty_rss_cannot_be_determined
+        puts " (?)"
+      else
+        puts
+      end
+    end
+  end
+end
+
+if !ARGV[0]
+  puts "Prints memory usage stats for processes whose title matches <filter>"
+  puts "Usage: memory_stats <filter>"
+  exit 1
+end
+
+MemoryStats.new.start(ARGV[0])
\ No newline at end of file
diff --git a/utilities/files/default/rotate-db-backups b/utilities/files/default/rotate-db-backups
new file mode 100644
index 00000000..31ee9159
--- /dev/null
+++ b/utilities/files/default/rotate-db-backups
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+find /u/backup/db -type f -name \*mysql.tar.gz -mtime +5 -delete
+find /u/backup/db -mindepth 2 -maxdepth 2 -type d \! -name current -mtime +3 -exec rm -rf {} \;
diff --git a/utilities/files/default/rotate-email-folders b/utilities/files/default/rotate-email-folders
new file mode 100644
index 00000000..ed36003f
--- /dev/null
+++ b/utilities/files/default/rotate-email-folders
@@ -0,0 +1,25 @@
+#!/usr/local/bin/ruby
+
+require 'fileutils'
+
+date = Time.now.strftime("%m-%d-%Y")
+apps = ARGV[0].split(",")
+
+apps.each do |app|
+  email_dir = "/u/apps/#{app}/shared/email"
+  rotate_dir = File.join(email_dir, "/old/#{date}")
+
+  FileUtils.mkdir_p(rotate_dir)
+
+  %W(completed failed junk).each do |dir|
+    src = File.join(email_dir, dir)
+    next unless File.exist?(src)
+    dest = File.join(rotate_dir, dir)
+
+    FileUtils.mv(src, dest)
+
+    %W(cur new tmp).each do |folder|
+      FileUtils.mkdir_p(File.join(email_dir, dir, folder))
+    end
+  end
+end
diff --git a/utilities/files/default/rotate-misc-log b/utilities/files/default/rotate-misc-log
new file mode 100644
index 00000000..45b8bc22
--- /dev/null
+++ b/utilities/files/default/rotate-misc-log
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+APP=$1
+DATE=`date +%m%d`
+
+/bin/cp /u/logs/${APP}/${DATE}/misc.log /u/logs/${APP}/${DATE}/misc.log.1
+cat /dev/null > /u/logs/${APP}/${DATE}/misc.log
diff --git a/utilities/metadata.rb b/utilities/metadata.rb
new file mode 100644
index 00000000..7503b188
--- /dev/null
+++ b/utilities/metadata.rb
@@ -0,0 +1,4 @@
+maintainer        "37signals"
+maintainer_email  "sysadmins@37signals.com"
+description       "Configures custom scripts and utilities"
+version           "0.1"
diff --git a/utilities/recipes/default.rb b/utilities/recipes/default.rb
new file mode 100644
index 00000000..fdfbfdb1
--- /dev/null
+++ b/utilities/recipes/default.rb
@@ -0,0 +1,8 @@
+if node[:utilities]
+  node[:utilities].each do |name, config|
+    remote_file "/usr/local/bin/#{name}" do
+      config.each { |cmd, arg| send(cmd.to_sym, arg) } if config
+      mode 0755
+    end
+  end
+end
\ No newline at end of file