Elasticsearch puppet

apt/Modulefile

@@ -0,0 +1,8 @@
+name    'brightbox-apt'
+version '1.1.1'
+source ''
+author 'brightbox'
+license 'GPL3'
+summary 'Ubuntu apt package management configuration'
+description 'Ubuntu apt package management configuration. Most likely much like the other apt modules available, but battle-tested by Brightbox (and has a couple of neat features).'
+project_page 'https://github.com/brightbox/puppet'

apt/lib/puppet/parser/functions/ppa_filename.rb

@@ -0,0 +1,12 @@
+module Puppet::Parser::Functions
+  newfunction(:ppa_filename, :type => :rvalue) do |args|
+    # ppa:brightbox/passenger becomes brightbox-passenger-lucid.list
+    filename = args.first.to_s
+    raise Puppet::ParseError, "ppa_filename cannot be blank" if filename.empty?
+    filename.gsub!(/ppa\:/,'')
+    filename.gsub!('/', '-')
+    filename += "-#{lookupvar('lsbdistcodename')}"
+    filename += ".list"
+    "/etc/apt/sources.list.d/" + filename
+  end
+end

apt/manifests/init.pp

@@ -0,0 +1,107 @@
+class apt {
+  exec { "apt-update":
+    command => "/usr/bin/apt-get update",
+    schedule => daily,
+    refreshonly => true
+  }
+
+  # Ensure apt is setup before running apt-get update
+  Apt::Ppa <| |> -> Exec["apt-update"]
+  Apt::Key <| |> -> Exec["apt-update"]
+
+  # Ensure apt-get update has been run before installing any packages
+  Exec["apt-update"] -> Package <| |>
+}
+
+# Sets up a local apt repository, ready for using with apt::localpackage
+class apt::localrepo($repodir = "/var/cache/local-repo") {
+  file { "${repodir}":
+    ensure => directory,
+    mode => 755,
+    notify => Exec[apt-update-local-repo]
+  }
+  exec { "apt-update-local-repo":
+    cwd => $repodir,
+    command => "/usr/bin/apt-ftparchive packages . > Packages",
+    require => [File["${repodir}"]],
+    before => Exec["apt-update"],
+    notify => Exec["apt-update"],
+    refreshonly => true
+  }
+  apt::source { "apt-local-repo":
+    source => "deb [trusted=yes] file:${repodir} /",
+  }
+}
+
+# Defines a deb package to download and put into the local apt repository.
+# Requires that you set a url
+define apt::localpackage($url = "", $repodir = "/var/cache/local-repo") {
+  $url_tokens = split($url, '/')
+  $pkg_filename = $url_tokens[-1]
+  exec { "apt-localpackage-${name}":
+    command => "/usr/bin/curl -L -s -C - -O $url",
+    cwd => $repodir,
+    creates => "${repodir}/${pkg_filename}",
+    notify => Exec["apt-update-local-repo"],
+    require => File[$repodir]
+  }
+}
+
+# $name is arbitrary. attribute "ppa" should be the ppa name, such as
+# "brightbox/ruby-ng" (this is due to a bug in puppet with resources
+# with slashes in them)
+define apt::ppa($ppa = "") {
+  exec { "apt-add-repository-$ppa":
+    command => "/usr/bin/apt-add-repository ppa:$ppa",
+    creates => ppa_filename($ppa),
+    notify => Exec["apt-update"],
+  }
+}
+
+# id should be the 8 character hex string that represents the key (is
+# actually the last 8 characters of the fingeprint)
+define apt::key($id) {
+  exec { "apt-key-$name":
+    command => "/usr/bin/apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys $id",
+    unless => "/usr/bin/apt-key list | /bin/grep $id",
+    notify => Exec["apt-update"]
+  }
+}
+
+# name should be a nice key, like percona, or brightbox
+# source should be: "deb url $lsbdistcodename main"
+# gpg_key_id is a gpg key to import, can be left empty
+define apt::source($source = "", $gpg_key_id = "") {
+  file { "/etc/apt/sources.list.d/${name}.list":
+    content => "# This file is managed by Puppet\n$source",
+    notify => Exec["apt-update"]
+  }
+
+  if $gpg_key_id {
+    # Allow multiple definitions of same key - it's not a problem and
+    # could be a common ocurrence
+    apt::key { "${name}${gpg_key_id}": id => $gpg_key_id }
+    Apt::Key["${name}${gpg_key_id}"] -> File["/etc/apt/sources.list.d/${name}.list"]
+  }
+}
+
+# Adapted from https://github.com/camptocamp/puppet-apt primarily to
+# support their postgresql recpie.
+define apt::preferences($ensure="present", $package="", $pin, $priority) {
+  $pkg = $package ? {
+    "" => $name,
+    default => $package,
+  }
+
+  $fname = regsubst($name, '\.', '-', 'G')
+
+  file {"/etc/apt/preferences.d/$fname":
+    ensure  => $ensure,
+    owner   => root,
+    group   => root,
+    mode    => 644,
+    content => template("apt/preferences.erb"),
+    before  => Exec["apt-update"],
+    notify  => Exec["apt-update"],
+  }
+}

apt/metadata.json

@@ -0,0 +1,25 @@
+{
+  "author": "brightbox",
+  "dependencies": [
+
+  ],
+  "summary": "Ubuntu apt package management configuration",
+  "source": "",
+  "version": "1.1.1",
+  "description": "Ubuntu apt package management configuration. Most likely much like the other apt modules available, but battle-tested by Brightbox (and has a couple of neat features).",
+  "types": [
+
+  ],
+  "project_page": "https://github.com/brightbox/puppet",
+  "license": "GPL3",
+  "name": "brightbox-apt",
+  "checksums": {
+    "Modulefile": "6cbcb4185b61b75bbd3773b5fca2a959",
+    "spec/classes/apt_spec.rb": "e84cf4478715041533949ef7a501611c",
+    "templates/preferences.erb": "e2677dcf80de02ec699795e976a20daf",
+    "spec/defines/apt_spec.rb": "e35de8c653de5bf47220268976e6fcfa",
+    "manifests/init.pp": "eeee16c7c10bfde4b96f9703c02ec546",
+    "lib/puppet/parser/functions/ppa_filename.rb": "88d8c3f18a2f01c8d85df869e4c5b265",
+    "spec/spec_helper.rb": "67329d39c5c3635f0b8f79dab5811ea3"
+  }
+}

apt/spec/classes/apt_spec.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), '../spec_helper')
+
+describe 'apt', :type => :class do
+  it { should contain_exec('apt-update') }
+end
+
+describe 'apt::localrepo', :type => :class do
+end

apt/spec/defines/apt_spec.rb

@@ -0,0 +1,31 @@
+require File.join(File.dirname(__FILE__), '../spec_helper')
+
+describe 'apt::ppa', :type => :define do
+  let(:title) { 'brightbox-ruby-ng' }
+  let(:params) { { :ppa => 'brightbox/ruby-ng' } }
+  let(:facts) { { :lsbdistcodename => 'precise' } }
+  it { should contain_exec('apt-add-repository-brightbox-ruby-ng').
+   with( :creates => '/etc/apt/sources.list.d/brightbox-ruby-ng-precise.list' )  
+  }
+end
+
+describe 'apt::localpackage', :type => :define do
+  let(:title) { 'elasticsearch' }
+  let(:params) { { :repodir => '/some/path',
+                  :url => 'https://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.19.3.deb' } }
+  it { should contain_exec('apt-localpackage-elasticsearch').
+     with( :creates => '/some/path/elasticsearch-0.19.3.deb')
+  }
+end
+
+describe 'apt::source', :type => :define do
+  let(:title) { 'some-repo' }
+  let(:params) { { 
+                  :source => "deb http://some/url stable main",
+                  :gpg_key_id => "0090DAAD"
+                 } }
+  it { should contain_file('/etc/apt/sources.list.d/some-repo.list').
+     with_content(/deb http:\/\/some\/url/)
+  }
+  it { should contain_apt__key('some-repo0090DAAD').with_id('0090DAAD') }
+end

apt/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'rspec-puppet'
+
+RSpec.configure do |c|
+   c.module_path = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+   c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', '..','..','manifests'))
+end

apt/templates/preferences.erb

@@ -0,0 +1,4 @@
+# file managed by puppet
+Package: <%= pkg %>
+Pin: <%= pin %>
+Pin-Priority: <%= priority %>

elasticsearch/Modulefile

@@ -0,0 +1,9 @@
+name    'brightbox-elasticsearch'
+version '1.0.1'
+source ''
+author 'brightbox'
+license 'GPL3'
+summary 'Elasticsearch cluster management'
+description 'Base configuration of an elasticsearch cluster'
+project_page 'https://github.com/brightbox/puppet'
+dependency 'brightbox-apt', '>= 1.0.0'

elasticsearch/manifests/init.pp

@@ -0,0 +1,68 @@
+# discovery_hosts should be a an array of hostnames or IPs of other masters
+#
+# Will automatically download debs for versions >= 0.19.  For older
+# versions, you must provide a url to the appropriate deb file using
+# the deburl parameter (you still need to provide the right $version
+# parameter too).
+#
+# To run a client-only server, set $master and $data to false
+#
+# On Lucid it currently requires a manual step to install the package
+# as Lucid apt-get doesn't support trusted=yes parameter
+#
+class elasticsearch($version = "0.19.3", $heap_size = "1024m", $cluster_name = 'elasticsearch', $discovery_hosts = [], $minimum_master_nodes = 1, $deburl = false, $master = true, $data = true) {
+  package { "default-jre-headless":
+    ensure => installed
+  }
+  apt::localpackage { "elasticsearch":
+    url => $deburl ? {
+      false => "http://cloud.github.com/downloads/elasticsearch/elasticsearch/elasticsearch-${version}.deb",
+      default => $deburl
+    }
+  }
+  package { "elasticsearch":
+    ensure => $version
+  }
+  service { "elasticsearch":
+    ensure => true,
+    enable => true,
+    require => Package[elasticsearch],
+    hasstatus => true,
+    hasrestart => true
+  }
+  file { "/etc/default/elasticsearch":
+    content => template("elasticsearch/default-elasticsearch.erb"),
+    notify => Service[elasticsearch]
+  }
+  file { "/etc/elasticsearch":
+    ensure => directory,
+    require => Package[elasticsearch]
+  }
+  file { "/etc/elasticsearch/elasticsearch.yml":
+    content => template("elasticsearch/elasticsearch.yml.erb"),
+    notify => Service[elasticsearch],
+    require => [Package[elasticsearch], File["/etc/elasticsearch"]]
+  }
+  file { "/etc/elasticsearch/logging.yml":
+    content => template("elasticsearch/logging.yml.erb"),
+    notify => Service[elasticsearch],
+    require => [Package[elasticsearch], File["/etc/elasticsearch"]]
+  }
+  if tagged("nrpe") {
+    include elasticsearch::nrpe
+  }
+}
+
+class elasticsearch::apacheproxy($server_name = "elasticsearch", $htpasswd_filename = "/etc/apache2/elasticsearch.htpasswd") {
+  apache::site { "elasticsearch":
+    content => template("elasticsearch/elasticsearch-apache.conf.erb")
+  }
+  apache::module { 'proxy': conf => false }
+  apache::module { 'proxy_http': conf => false }
+}
+
+class elasticsearch::nrpe {
+  nagios::nrpe_config { "elasticsearch":
+    content => template("elasticsearch/es-nrpe.conf.erb")
+  }
+}

elasticsearch/metadata.json

@@ -0,0 +1,30 @@
+{
+  "author": "brightbox",
+  "dependencies": [
+    {
+      "version_requirement": ">= 1.0.0",
+      "name": "brightbox-apt"
+    }
+  ],
+  "summary": "Elasticsearch cluster management",
+  "source": "",
+  "version": "1.0.1",
+  "description": "Base configuration of an elasticsearch cluster",
+  "types": [
+
+  ],
+  "project_page": "https://github.com/brightbox/puppet",
+  "license": "GPL3",
+  "name": "brightbox-elasticsearch",
+  "checksums": {
+    "templates/es-nrpe.conf.erb": "61148bb9b28ce17fb6c6f92ef72499d4",
+    "Modulefile": "3ad88e62abe31223d617839454033e96",
+    "templates/elasticsearch.yml.erb": "ca3341e38414f66a9d82645d1ee07c19",
+    "spec/classes/elasticsearch_spec.rb": "87658d217cd0872396ee97bd5322a163",
+    "manifests/init.pp": "a5f8947c50c4d72bd01151d74191ab0b",
+    "templates/logging.yml.erb": "4c4e01504d644f67494d3a12ca1918f9",
+    "spec/spec_helper.rb": "67329d39c5c3635f0b8f79dab5811ea3",
+    "templates/elasticsearch-apache.conf.erb": "140fc111e07fd8211b6525c73da33560",
+    "templates/default-elasticsearch.erb": "d22abfafe5dfdbc9dc56746647138768"
+  }
+}

elasticsearch/spec/classes/elasticsearch_spec.rb

@@ -0,0 +1,35 @@
+require File.join(File.dirname(__FILE__), '../spec_helper')
+
+describe 'elasticsearch', :type => :class do
+  describe "with a version" do
+    let(:params) { { :version => "0.20.2" } }
+    it { should contain_package('elasticsearch').
+       with_ensure('0.20.2')
+    }
+    it { should contain_apt__localpackage("elasticsearch").
+       with_url(/elasticsearch-0.20.2.deb$/)
+    }
+  end
+
+  describe "with a version and a deburl" do
+    let(:params) { { :version => "0.30.3", :deburl => "https://somehost/elasticsearch.deb" } }
+    it { should contain_package('elasticsearch').
+       with_ensure('0.30.3')
+    }
+    it { should contain_apt__localpackage("elasticsearch").
+       with_url("https://somehost/elasticsearch.deb")
+    }
+  end
+
+  describe "config file" do
+    let(:params) { { :discovery_hosts => ["s_one", "s_two", "s_three"] } }
+    it { should contain_file('/etc/elasticsearch/elasticsearch.yml').
+       with_content(/["s_one", "s_two", "s_three"]/)
+    }
+  end
+
+end
+
+describe 'elasticsearch::nrpe', :type => :class do
+  it { should contain_nagios__nrpe_config('elasticsearch') }
+end

elasticsearch/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'rspec-puppet'
+
+RSpec.configure do |c|
+   c.module_path = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+   c.manifest_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', '..','..','manifests'))
+end

elasticsearch/templates/default-elasticsearch.erb

@@ -0,0 +1,22 @@
+# This file is managed by Puppet
+
+# Run ElasticSearch as this user ID and group ID
+ES_USER=elasticsearch
+ES_GROUP=elasticsearch
+
+ES_HEAP_SIZE=<%= heap_size %>
+
+# ElasticSearch log directory
+LOG_DIR=/var/log/elasticsearch
+
+# ElasticSearch data directory
+DATA_DIR=/var/lib/elasticsearch
+
+# ElasticSearch work directory
+WORK_DIR=/tmp/elasticsearch
+
+# ElasticSearch configuration directory
+CONF_DIR=/etc/elasticsearch
+
+# ElasticSearch configuration file (elasticsearch.yml)
+CONF_FILE=/etc/elasticsearch/elasticsearch.yml