Permalink
Browse files

Merge branch 'master' of github.com:infochimps-labs/ironfan-pantry

  • Loading branch information...
2 parents a9944cf + 395b63d commit 96c19b7590d8d2c15e26e368fd5d82eef1771fe1 @kornypoet kornypoet committed Sep 4, 2012
View
10 cookbooks/install_from/providers/release.rb
@@ -114,6 +114,16 @@
new_resource.updated_by_last_action(true)
end
+action :build_with_sbt do
+ action_build
+ bash "build #{new_resource.name} with sbt" do
+ user new_resource.user
+ cwd new_resource.install_dir
+ code "./sbt update; ./sbt package"
+ environment new_resource.environment
+ end
+end
+
action :install_binaries do
new_resource.has_binaries.each do |bin|
link ::File.join(new_resource.prefix_root, 'bin', ::File.basename(bin)) do
View
4 cookbooks/install_from/resources/release.rb
@@ -28,6 +28,7 @@
:configure_with_autoconf,
:build_with_make,
:build_with_ant,
+ :build_with_sbt,
:install_with_make,
:install_binaries,
:install_python
@@ -95,7 +96,7 @@ def assume_defaults!
# the release_url 'http://apache.org/pig/pig-0.8.0.tar.gz' has
# release_basename 'pig-0.8.0' and release_ext 'tar.gz'
release_basename = ::File.basename(release_url.gsub(/\?.*\z/, '')).gsub(/-bin\b/, '')
- release_basename =~ %r{^(.+?)\.(tar\.gz|tar\.bz2|zip)}
+ release_basename =~ %r{^(.+?)\.(tar\.gz|tar\.bz2|zip|tgz)}
@release_ext ||= $2
@home_dir ||= ::File.join(prefix_root, 'share', name)
@@ -105,6 +106,7 @@ def assume_defaults!
case release_ext
when 'tar' then untar_cmd('xf', release_file, install_dir)
when 'tar.gz' then untar_cmd('xzf', release_file, install_dir)
+ when 'tgz' then untar_cmd('xzf', release_file, install_dir)
when 'tar.bz2' then untar_cmd('xjf', release_file, install_dir)
when 'zip' then unzip_cmd(release_file, install_dir)
else raise "Don't know how to expand #{release_url} which has extension '#{release_ext}'"
View
36 cookbooks/kafka/CHANGELOG.md
@@ -0,0 +1,36 @@
+## 1.0.8
+* Added broker chroot prefix for zk.connect string
+
+## 1.0.6 - 1.0.7
+* Added changes to support kafka 0.7.1
+
+## 1.0.5
+* Added template to create a collectd plugin for kafka JMX objects.
+
+## 1.0.4
+* Fixed the creation the bin dir.
+
+## 1.0.3
+* Added logic to prevent kafka from being nuked each time Chef is run. A manual delete of the kafka install folder will trigger a re-deploy.
+
+## 1.0.2
+* Set default broker_id to nil and if not set will use the ip address without the '.'
+* Set the default broker_host_name to nil and if not set will use the server hostname
+* Fixed log4j.properties problems
+
+## 1.0.1
+
+* Use /opt/kafka as the default intall dir
+* Use /var/kafka as the default data dir
+* Remove the unnecessary platform case statement from the attributes file
+* Remove the attributes for user/group. Always run as kafka user/group
+* Remove tarball from the cookbook
+* Don't give kafka user a home directory or a valid shell
+* Fix runit script to work
+* Pull the source file down from a remote URL and not the cookbook
+* Use more restrictive permissions on config files
+* Use remote zookeeper nodes
+* Don't hardcode the broker ID
+
+## 1.0.0
+* Initial release with a changelog
View
46 cookbooks/kafka/README.rdoc
@@ -0,0 +1,46 @@
+= DESCRIPTION:
+Installs kafka 0.7.0
+
+= REQUIREMENTS:
+* Java cookbook version >= 1.5
+* Runit cookbook
+* Zookeeper cookbook - The Kafka cookbook will utilize the clientPort from the Zookeeper cookbook
+ as well as look for a role called "zookeeper" that is applied to nodes. All nodes with the role applied
+ to them will be used as the Zookeeper quorum that Kafka connects to.
+
+= ATTRIBUTES:
+
+* kafa.version - The Kafka version to pull and use
+* kafa.install_dir - Location for Kafka to be installed
+* kafa.data_dir - Location for Kafka logs
+* kafa.log_dir - Location for Kafka log4j logs
+* kafa.broker_id - The id of the broker. This must be set to a unique integer for each broker. If not set, it defaults to the machine's ip address without the '.'.
+* kafa.broker_host_name - Hostname the broker will advertise to consumers. If not set, kafka will use the host name for the server being deployed to..
+* kafa.port - The port the socket server listens on
+* kafa.threads - The number of processor threads the socket server uses for receiving and answering requests. If not set, defaults to the number of cores on the machine
+* kafa.log_flush_interval - The number of messages to accept before forcing a flush of data to disk
+* kafa.log_flush_time_interval - The maximum amount of time (ms) a message can sit in a log before we force a flush
+* kafa.log_flush_scheduler_time_interval - The interval (in ms) at which logs are checked to see if they need to be flushed to disk
+* kafa.log_retention_hours - The minimum age of a log file to be eligible for deletion
+
+= USAGE:
+
+* kafka - Install a Kafka broker.
+
+= LICENSE and AUTHOR:
+
+Author:: Ivan von Nagy ().
+
+Copyright:: 2012, Webtrends, Inc. Modifications by Infochimps
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
View
40 cookbooks/kafka/attributes/default.rb
@@ -0,0 +1,40 @@
+#
+# Cookbook Name:: kafka
+# Attributes:: default
+#
+# Copyright 2012, Webtrends, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install
+default[:kafka][:version] = "0.7.1-incubating"
+default[:kafka][:download_url] = "http://artifacts.chimpy.us.s3.amazonaws.com/tarballs"
+
+default[:kafka][:user] = "kafka"
+default[:kafka][:group] = "kafka"
+
+default[:kafka][:install_dir] = "/usr/local/share/kafka"
+default[:kafka][:data_dir] = "/var/kafka"
+default[:kafka][:log_dir] = "/var/log/kafka"
+default[:kafka][:chroot_suffix] = "brokers"
+
+default[:kafka][:broker_id] = nil
+default[:kafka][:broker_host_name] = nil
+default[:kafka][:port] = 9092
+default[:kafka][:threads] = nil
+default[:kafka][:log_flush_interval] = 10000
+default[:kafka][:log_flush_time_interval] = 1000
+default[:kafka][:log_flush_scheduler_time_interval] = 1000
+default[:kafka][:log_retention_hours] = 168
+
+
View
75 cookbooks/kafka/metadata.rb
@@ -0,0 +1,75 @@
+maintainer "Webtrends, Inc."
+maintainer_email "ivan.vonnagy@webtrends.com"
+license "Apache 2.0"
+description "Sets up Kafka"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.rdoc'))
+version "1.0.8"
+depends "java"
+depends "runit"
+depends "zookeeper"
+
+recipe "kafka::default", "Base configuration for kafka"
+
+%w{ debian ubuntu centos redhat fedora scientific amazon }.each do |os|
+ supports os
+end
+
+attribute "kafka/version",
+ :display_name => "Kafka Version",
+ :description => "The Kafka version to pull and use",
+ :default => "0.7.0"
+
+attribute "kafka/home_dir",
+ :display_name => "Kafka Home Directory",
+ :description => "Location for Kafka to be located.",
+ :default => "/usr/share/kafka"
+
+attribute "kafka/data_dir",
+ :display_name => "Kafka Log Directory",
+ :description => "Location for Kafka logs.",
+ :default => "/usr/share/kafka/kafka-logs"
+
+attribute "kafka/log_dir",
+ :display_name => "Kafka log4j Directory",
+ :description => ";.",
+ :default => "/var/log/kafka"
+
+attribute "kafka/broker_id",
+ :display_name => "Kafka Broker Id",
+ :description => "The id of the broker. This must be set to a unique integer for each broker. If not set, it defaults to the machine's ip address without the '.'.",
+ :default => ""
+
+attribute "kafka/broker_host_name",
+ :display_name => "Kafka Host Name",
+ :description => "Hostname the broker will advertise to consumers. If not set, kafka will use the host name for the server being deployed to.",
+ :default => ""
+
+attribute "kafka/port",
+ :display_name => "Kafka Port",
+ :description => "The port the socket server listens on.",
+ :default => "9092"
+
+attribute "kafka/threads",
+ :display_name => "Kafka Threads",
+ :description => "The number of processor threads the socket server uses for receiving and answering requests. If not set, defaults to the number of cores on the machine.",
+ :default => ""
+
+attribute "kafka/log_flush_interval",
+ :display_name => "Kafka Flush Interval",
+ :description => "The number of messages to accept before forcing a flush of data to disk.",
+ :default => "10000"
+
+attribute "kafka/log_flush_time_interval",
+ :display_name => "Kafka Flush Time Interval",
+ :description => "The maximum amount of time (ms) a message can sit in a log before we force a flush.",
+ :default => "1000"
+
+attribute "kafka/log_flush_scheduler_time_interval",
+ :display_name => "Kafka Flush Scheduler Time Interval",
+ :description => "The interval (in ms) at which logs are checked to see if they need to be flushed to disk.",
+ :default => "1000"
+
+attribute "kafka/log_retention_hours",
+ :display_name => "Kafka Log Retention Hours",
+ :description => "The minimum age of a log file to be eligible for deletion",
+ :default => "168"
View
34 cookbooks/kafka/recipes/default.rb
@@ -0,0 +1,34 @@
+#
+# Cookbook Name:: kafka
+# Description:: Base configuration for Kafka
+# Recipe:: default
+#
+# Copyright 2012, Webtrends, Inc. modifications by Infochimps
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# == Recipes
+include_recipe "java"
+include_recipe "runit"
+
+if node[:kafka][:broker_id].nil? || node[:kafka][:broker_id].empty?
+ node[:kafka][:broker_id] = node[:ipaddress].gsub(".","")
+end
+
+if node[:kafka][:broker_host_name].nil? || node[:kafka][:broker_host_name].empty?
+ node[:kafka][:broker_host_name] = node[:fqdn]
+end
+
+log "Broker id: #{node[:kafka][:broker_id]}"
+log "Broker name: #{node[:kafka][:broker_host_name]}"
+
View
18 cookbooks/kafka/recipes/directories.rb
@@ -0,0 +1,18 @@
+# create the log directory
+directory "#{node[:kafka][:log_dir]}" do
+ owner node[:kafka][:user]
+ group node[:kafka][:group]
+ mode 00755
+ recursive true
+ action :create
+end
+
+# create the data directory
+directory "#{node[:kafka][:data_dir]}" do
+ owner node[:kafka][:user]
+ group node[:kafka][:group]
+ mode 00755
+ recursive true
+ action :create
+end
+
View
72 cookbooks/kafka/recipes/install_from_release.rb
@@ -0,0 +1,72 @@
+# pull the remote file only if we create the directory
+tarball = "kafka-#{node[:kafka][:version]}-src.tgz"
+download_file = File.join("#{node[:kafka][:download_url]}", "#{tarball}")
+
+remote_file "#{Chef::Config[:file_cache_path]}/#{tarball}" do
+ source download_file
+ mode 00644
+ action :create_if_missing
+end
+
+install_from_release(:kafka) do
+ release_url download_file
+ home_dir node[:kafka][:install_dir]
+ version node[:kafka][:version]
+ checksum node[:kafka][:checksum]
+ action [ :build_with_sbt, :install ]
+end
+
+template "#{node[:kafka][:install_dir]}/bin/service-control" do
+ source "service-control.erb"
+ owner "root"
+ group "root"
+ mode 00755
+ variables({
+ :install_dir => node[:kafka][:install_dir],
+ :log_dir => node[:kafka][:log_dir],
+ :java_home => node['java']['java_home'],
+ :java_jmx_port => 9999,
+ :java_class => "kafka.Kafka",
+ :user => node[:kafka][:user]
+ })
+end
+
+# grab the zookeeper nodes that are currently available
+zookeeper_pairs = discover_all(:zookeeper, :server).map(&:private_ip).sort.map{|x| "#{x}:#{node[:zookeeper][:client_port]}"}
+kafka_chroot_suffix = node[:kafka][:chroot_suffix]
+
+%w[server.properties log4j.properties].each do |template_file|
+ template "#{node[:kafka][:install_dir]}/config/#{template_file}" do
+ source "#{template_file}.erb"
+ owner node[:kafka][:user]
+ group node[:kafka][:group]
+ mode 00755
+ variables({
+ :kafka => node[:kafka],
+ :zookeeper_pairs => zookeeper_pairs,
+ :client_port => node[:zookeeper][:client_port]
+ })
+ end
+end
+
+# fix perms and ownership
+execute "chmod" do
+ command "find #{node[:kafka][:install_dir]} -name bin -prune -o -type f -exec chmod 644 {} \\; && find #{node[:kafka][:install_dir]} -type d -exec chmod 755 {} \\;"
+ action :run
+end
+execute "chown" do
+ command "chown -R root:root #{node[:kafka][:install_dir]}"
+ action :run
+end
+execute "chmod" do
+ command "chmod -R 755 #{node[:kafka][:install_dir]}/bin"
+ action :run
+end
+
+# delete the application tarball
+execute "delete_install_source" do
+ user "root"
+ group "root"
+ command "rm -f #{Chef::Config[:file_cache_path]}/#{tarball}"
+ action :run
+end
View
22 cookbooks/kafka/recipes/server.rb
@@ -0,0 +1,22 @@
+# create the runit service
+runit_service "kafka" do
+ options({
+ :log_dir => node[:kafka][:log_dir],
+ :install_dir => node[:kafka][:install_dir],
+ :java_home => node['java']['java_home'],
+ :user => node[:kafka][:user]
+ })
+end
+
+announce(:kafka, :server)
+
+#Create collectd plugin for kafka JMX objects if collectd has been applied.
+if node.attribute?("collectd")
+ template "#{node[:collectd][:plugin_conf_dir]}/collectd_kafka-broker.conf" do
+ source "collectd_kafka-broker.conf.erb"
+ owner "root"
+ group "root"
+ mode 00644
+ notifies :restart, resources(:service => "collectd")
+ end
+end
View
15 cookbooks/kafka/recipes/users.rb
@@ -0,0 +1,15 @@
+# == Users
+
+# setup kafka group
+group node[:kafka][:group] do
+end
+
+# setup kafka user
+user node[:kafka][:user] do
+ comment "Kafka user"
+ gid "kafka"
+ home "/home/kafka"
+ shell "/bin/noshell"
+ supports :manage_home => false
+end
+
View
259 cookbooks/kafka/templates/default/collectd_kafka-broker.conf.erb
@@ -0,0 +1,259 @@
+##############################################################################
+# This file is autogenerated by chef #
+# #
+# Collectd Kafka Broker plugin configuration file #
+#----------------------------------------------------------------------------#
+# By: Josh Behrends - Webtrends 2012 #
+# #
+# Revisions: 06/12/2012 - Initial script creation #
+# #
+##############################################################################
+LoadPlugin java
+
+<Plugin "java">
+ JVMArg "-verbose:jni"
+ JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"
+ LoadPlugin "org.collectd.java.GenericJMX"
+ <Plugin "GenericJMX">
+ ################
+ # MBean blocks #
+ ################
+ <MBean "memory_pool">
+ ObjectName "java.lang:type=MemoryPool,*"
+ InstancePrefix "memory_pool-"
+ InstanceFrom "name"
+ <Value>
+ Type "memory"
+ #InstancePrefix ""
+ #InstanceFrom ""
+ Table true
+ Attribute "Usage"
+ </Value>
+ </MBean>
+ # Heap memory usage
+ <MBean "memory-heap">
+ ObjectName "java.lang:type=Memory"
+ #InstanceFrom ""
+ InstancePrefix "memory-heap"
+ # Creates four values: committed, init, max, used
+ <Value>
+ Type "memory"
+ #InstancePrefix ""
+ #InstanceFrom ""
+ Table true
+ Attribute "HeapMemoryUsage"
+ </Value>
+ </MBean>
+ # Non-heap memory usage
+ <MBean "memory_nonheap">
+ ObjectName "java.lang:type=Memory"
+ #InstanceFrom ""
+ InstancePrefix "memory-nonheap"
+ # Creates four values: committed, init, max, used
+ <Value>
+ Type "memory"
+ #InstancePrefix ""
+ #InstanceFrom ""
+ Table true
+ Attribute "NonHeapMemoryUsage"
+ </Value>
+ </MBean>
+ # Non-heap memory usage
+ <MBean "java_threads">
+ ObjectName "java.lang:type=Threading"
+ #InstanceFrom ""
+ InstancePrefix "java_threads"
+ <Value>
+ Type "gauge"
+ InstancePrefix "ThreadCount"
+ #InstanceFrom ""
+ Attribute "ThreadCount"
+ </Value>
+ </MBean>
+ # Garbage Collector stats
+ <MBean "garbage_collector">
+ ObjectName "java.lang:type=GarbageCollector,*"
+ InstancePrefix "gc-"
+ InstanceFrom "name"
+ <Value>
+ Type "invocations"
+ InstancePrefix "collection_count"
+ #InstanceFrom ""
+ Table false
+ Attribute "CollectionCount"
+ </Value>
+ <Value>
+ Type "total_time_in_ms"
+ InstancePrefix "collection_time"
+ #InstanceFrom ""
+ Table false
+ Attribute "CollectionTime"
+ </Value>
+ </MBean>
+ # Number of classes being loaded.
+ <MBean "classes">
+ ObjectName "java.lang:type=ClassLoading"
+ InstancePrefix "loaded_classes"
+ #InstanceFrom ""
+ <Value>
+ Type "gauge"
+ InstancePrefix "loaded_classes"
+ #InstanceFrom ""
+ Table false
+ Attribute "LoadedClassCount"
+ </Value>
+ </MBean>
+ #WT Metrics Below
+ <MBean "kafka_log-flush-stats">
+ ObjectName "kafka:type=kafka.LogFlushStats"
+ InstancePrefix "KafkaLogFlushStats"
+ #InstanceFrom ""
+ <Value>
+ Type "gauge"
+ InstancePrefix "AvgFlushMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "AvgFlushMs"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "FlushesPerSecond"
+ #InstanceFrom ""
+ Table false
+ Attribute "FlushesPerSecond"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "MaxFlushMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "MaxFlushMs"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "NumFlushes"
+ #InstanceFrom ""
+ Table false
+ Attribute "NumFlushes"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "TotalFlushMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "TotalFlushMs"
+ </Value>
+ </MBean>
+ <MBean "kafka_socket-server-stats">
+ ObjectName "kafka:type=kafka.SocketServerStats"
+ InstancePrefix "KafkaSocketServerStats"
+ #InstanceFrom ""
+ <Value>
+ Type "gauge"
+ InstancePrefix "AvgFetchRequestMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "AvgFetchRequestMs"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "AvgProduceRequestMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "AvgProduceRequestMs"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "BytesReadPerSecond"
+ #InstanceFrom ""
+ Table false
+ Attribute "BytesReadPerSecond"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "BytesWrittenPerSecond"
+ #InstanceFrom ""
+ Table false
+ Attribute "BytesWrittenPerSecond"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "FetchRequestsPerSecond"
+ #InstanceFrom ""
+ Table false
+ Attribute "FetchRequestsPerSecond"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "MaxFetchRequestMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "MaxFetchRequestMs"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "NumFetchRequests"
+ #InstanceFrom ""
+ Table false
+ Attribute "NumFetchRequests"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "NumProduceRequests"
+ #InstanceFrom ""
+ Table false
+ Attribute "NumProduceRequests"
+ </Value>
+ <Value>
+ Type "gauge"
+ InstancePrefix "ProduceRequestsPerSecond"
+ #InstanceFrom ""
+ Table false
+ Attribute "ProduceRequestsPerSecond"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "TotalBytesRead"
+ #InstanceFrom ""
+ Table false
+ Attribute "TotalBytesRead"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "TotalBytesWritten"
+ #InstanceFrom ""
+ Table false
+ Attribute "TotalBytesWritten"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "TotalFetchRequestMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "TotalFetchRequestMs"
+ </Value>
+ <Value>
+ Type "counter"
+ InstancePrefix "TotalProduceRequestMs"
+ #InstanceFrom ""
+ Table false
+ Attribute "TotalProduceRequestMs"
+ </Value>
+ </MBean>
+ #####################
+ # Connection blocks #
+ #####################
+ #Connection Block
+ <Connection>
+ Host "<%= @node[:hostname] %>"
+ ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:9999/jmxrmi"
+ Collect "classes"
+ Collect "garbage_collector"
+ Collect "java_threads"
+ Collect "memory-heap"
+ Collect "memory_nonheap"
+ Collect "kafka_socket-server-stats"
+ Collect "kafka_log-flush-stats"
+ </Connection>
+ </Plugin>
+</Plugin>
View
32 cookbooks/kafka/templates/default/consumer.properties.erb
@@ -0,0 +1,32 @@
+# This file created by Chef from template. Do not hand edit this file
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# see kafka.consumer.ConsumerConfig for more details
+
+# zk connection string
+# comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002"
+zk.connect=<%= @zookeeper_pairs.join(",") %>
+
+# timeout in ms for connecting to zookeeper
+zk.connectiontimeout.ms=1000000
+
+#consumer group id
+groupid=test-consumer-group
+
+#consumer timeout
+#consumer.timeout.ms=5000
View
21 cookbooks/kafka/templates/default/log4j.properties.erb
@@ -0,0 +1,21 @@
+# This file created by Chef from template. Do not hand edit this file
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO,R
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.File=<%= node["kafka"]["log_dir"] %>/kafka.log
+# Keep one backup file
+log4j.appender.R.MaxBackupIndex=1
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}] %-4r [%t] %-5p %c %x - %m%n
+
+log4j.logger.kafka=INFO
+log4j.logger.org.IOItec.zkclient.ZkClient=INFO
View
82 cookbooks/kafka/templates/default/producer.properties.erb
@@ -0,0 +1,82 @@
+# This file created by Chef from template. Do not hand edit this file
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# see kafka.producer.ProducerConfig for more details
+
+############################# Producer Basics #############################
+
+# need to set either broker.list or zk.connect
+
+# configure brokers statically
+# format: brokerid1:host1:port1,brokerid2:host2:port2 ...
+broker.list=0:localhost:9092
+
+# discover brokers from ZK
+#zk.connect=
+
+# zookeeper session timeout; default is 6000
+#zk.sessiontimeout.ms=
+
+# the max time that the client waits to establish a connection to zookeeper; default is 6000
+#zk.connectiontimeout.ms
+
+# name of the partitioner class for partitioning events; default partition spreads data randomly
+#partitioner.class=
+
+# specifies whether the messages are sent asynchronously (async) or synchronously (sync)
+producer.type=sync
+
+# specify the compression codec for all data generated: 0: no compression, 1: gzip
+compression.codec=0
+
+# message encoder
+serializer.class=kafka.serializer.StringEncoder
+
+# allow topic level compression
+#compressed.topics=
+
+# max message size; messages larger than that size are discarded; default is 1000000
+#max.message.size=
+
+
+############################# Async Producer #############################
+# maximum time, in milliseconds, for buffering data on the producer queue
+#queue.time=
+
+# the maximum size of the blocking queue for buffering on the producer
+#queue.size=
+
+# Timeout for event enqueue:
+# 0: events will be enqueued immediately or dropped if the queue is full
+# -ve: enqueue will block indefinitely if the queue is full
+# +ve: enqueue will block up to this many milliseconds if the queue is full
+#queue.enqueueTimeout.ms=
+
+# the number of messages batched at the producer
+#batch.size=
+
+# the callback handler for one or multiple events
+#callback.handler=
+
+# properties required to initialize the callback handler
+#callback.handler.props=
+
+# the handler for events
+#event.handler=
+
+# properties required to initialize the event handler
+#event.handler.props=
+
View
126 cookbooks/kafka/templates/default/server.properties.erb
@@ -0,0 +1,126 @@
+# This file created by Chef from template. Do not hand edit this file
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# see kafka.server.KafkaConfig for additional details and defaults
+
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+brokerid=<%=node["kafka"]["broker_id"]%>
+
+# Hostname the broker will advertise to consumers. If not set, kafka will use the value returned
+# from InetAddress.getLocalHost(). If there are multiple interfaces getLocalHost
+# may not be what you want.
+<% if node[:kafka][:broker_host_name].nil? || node[:kafka][:broker_host_name].empty? %>
+#hostname
+<% else %>
+hostname=<%=node[:kafka][:broker_host_name]%>
+<% end %>
+
+############################# Socket Server Settings #############################
+
+# The port the socket server listens on
+port=<%=node["kafka"]["port"]%>
+
+# The number of processor threads the socket server uses for receiving and answering requests.
+# Defaults to the number of cores on the machine
+<% if (node[:kafka][:threads].nil? || node[:kafka][:threads].empty?) %>
+#num.threads
+<% else %>
+num.threads=<%=node[:kafka][:threads]%>
+<% end %>
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer=1048576
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer=1048576
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+max.socket.request.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# The directory under which to store log files
+log.dir=<%=node["kafka"]["data_dir"]%>
+
+# The number of logical partitions per topic per server. More partitions allow greater parallelism
+# for consumption, but also mean more files.
+num.partitions=1
+
+# Overrides for for the default given by num.partitions on a per-topic basis
+#topic.partition.count.map=topic1:3, topic2:4
+
+############################# Log Flush Policy #############################
+
+# The following configurations control the flush of data to disk. This is the most
+# important performance knob in kafka.
+# There are a few important trade-offs here:
+# 1. Durability: Unflushed data is at greater risk of loss in the event of a crash.
+# 2. Latency: Data is not made available to consumers until it is flushed (which adds latency).
+# 3. Throughput: The flush is generally the most expensive operation.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+log.flush.interval=<%=node["kafka"]["log_flush_interval"]%>
+
+# The maximum amount of time a message can sit in a log before we force a flush
+log.default.flush.interval.ms=<%=node["kafka"]["log_flush_time_interval"]%>
+
+# Per-topic overrides for log.default.flush.interval.ms
+#topic.flush.intervals.ms=topic1:1000, topic2:3000
+
+# The interval (in ms) at which logs are checked to see if they need to be flushed to disk.
+log.default.flush.scheduler.interval.ms=<%=node["kafka"]["log_flush_scheduler_time_interval"]%>
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion
+log.retention.hours=<%=node["kafka"]["log_retention_hours"]%>
+
+# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
+# segments don't drop below log.retention.size.
+#log.retention.size=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.file.size=536870912
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.cleanup.interval.mins=1
+
+############################# Zookeeper #############################
+
+# Enable connecting to zookeeper
+enable.zookeeper=<%= @zookeeper_pairs ? true : false %>
+
+# Zk connection string (see zk docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zk.connect=<%= @zookeeper_pairs.join(",") %>
+
+# Timeout in ms for connecting to zookeeper
+zk.connectiontimeout.ms=1000000
View
98 cookbooks/kafka/templates/default/service-control.erb
@@ -0,0 +1,98 @@
+#!/bin/sh
+
+# Setup basic path information.
+MAIN="<%=@java_class%>"
+LOG_OUT="<%=File.join(@log_dir,"process.out")%>"
+LOG_ERR="<%=File.join(@log_dir,"process.err")%>"
+HOME_DIR="<%=@install_dir%>"
+JAVA_HOME="<%=@java_home%>"
+SERVER_CONFIG="<%=@install_dir%>/config/server.properties"
+
+# Setup classpath, initially empty.
+CP=""
+
+# Add all .jar files to the classpath.
+for jar in `ls $HOME_DIR/lib/*`; do
+ if [ "" = "$CP" ]; then
+ CP=$jar
+ else
+ CP=$CP:$jar
+ fi
+done
+
+# Add the JMX options.
+if [ -z "$JMX_OPTS"]; then
+ JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# Add the JMX port.
+if [ -z "$JMX_PORT" ]; then
+ JMX_PORT="<%=@java_jmx_port%>"
+fi
+
+# Add JMX options and port together.
+JMX_OPTS="$JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT "
+
+if [ -z "$KAFKA_OPTS" ]; then
+ KAFKA_OPTS="-Xmx512M -server -Djava.net.preferIPv4Stack=true -Dlog4j.configuration=file:$HOME_DIR/config/log4j.properties"
+fi
+
+if [ -z "$JAVA_HOME" ]; then
+ JAVA="java"
+else
+ JAVA="$JAVA_HOME/bin/java"
+fi
+
+# Create the JSVC command line.
+CMD="$JAVA $KAFKA_OPTS $JMX_OPTS -cp $CP"
+
+# Find current PID if any.
+FOUND_PID=`ps -ef |grep $MAIN |grep -v grep |awk '{print $2}'`
+
+# Check what the caller wants.
+case "$1" in
+ start)
+ if [ "" = "$FOUND_PID" ]; then
+ cd $HOME_DIR
+ echo $CMD $MAIN $SERVER_CONFIG
+ # For compatibility with runit we cannot backgroud
+ # ourselves with a '&' at the end, though we do
+ # need to exec.
+ exec $CMD $MAIN $SERVER_CONFIG >$LOG_OUT 2>$LOG_ERR
+ else
+ echo "Service already running"
+ exit 1
+ fi
+ ;;
+ stop)
+ if [ "" = "$FOUND_PID" ]; then
+ echo "Service not running, nothing to stop"
+ exit 1
+ else
+ kill $FOUND_PID
+ sleep 4
+ # Find PID again to see if we need to use kill -9 termination.
+ FOUND_PID=`ps -ef |grep $MAIN |grep -v grep |awk '{print $2}'`
+ if [ "" != "$FOUND_PID" ]; then
+ echo "Using kill did not work, sending kill -9 to process"
+ kill -9 $FOUND_PID
+ fi
+ exit 0
+ fi
+ ;;
+ status)
+ # Status is found via actual running processes, rather than
+ # trusting the PID log file.
+ if [ "" = "$FOUND_PID" ]; then
+ echo "Not running"
+ exit 1
+ else
+ echo "Running"
+ exit 0
+ fi
+ ;;
+ *)
+ echo "usage: daemon {start|stop|status}" >&2
+ exit 1
+ ;;
+esac
View
4 cookbooks/kafka/templates/default/sv-kafka-log-run.erb
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+exec 2>&1
+exec svlogd -tt ./main
View
3 cookbooks/kafka/templates/default/sv-kafka-run.erb
@@ -0,0 +1,3 @@
+#!/bin/sh
+exec 2>&1
+exec chpst -u <%=@options[:user]%> <%=@options[:install_dir]%>/bin/kafka-server-start.sh <%=@options[:install_dir]%>/config/server.properties
View
23 cookbooks/kafka/templates/default/zookeeper.properties.erb
@@ -0,0 +1,23 @@
+# This file created by Chef from template. Do not hand edit this file
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# the directory where the snapshot is stored.
+dataDir=/tmp/zookeeper
+# the port at which the clients will connect
+clientPort=<%=@client_port%>
+# disable the per-ip limit on the number of connections since this is a non-production config
+maxClientCnxns=0
View
4 cookbooks/pig/recipes/piggybank.rb
@@ -19,10 +19,6 @@
# limitations under the License.
#
-package "sun-java6-jdk"
-package "sun-java6-bin"
-package "sun-java6-jre"
-
package "ivy"
bash 'build piggybank' do
View
24 roles/kafka.rb
@@ -0,0 +1,24 @@
+name 'flume_agent'
+description 'flume agent'
+
+run_list(*%w[
+ flume
+ flume::jars
+ flume::plugin-hbase_sink
+ role[maven]
+ flume::agent
+ flume_integration::jruby_classpath
+ flume::config_files
+ flume_integration::jruby_home
+])
+
+override_attributes({
+ :zabbix => {
+ :host_groups => {
+ :flume => ['Flume nodes'],
+ },
+ :templates => {
+ :flume => ['Template_Flume_Node']
+ }
+ }
+})
View
9 roles/kafka_client.rb
@@ -0,0 +1,9 @@
+name 'kafka_client'
+description 'kafka client'
+
+run_list(*%w[
+ kafka
+ kafka::users
+ kafka::directories
+ kafka::install_from_release
+])
View
10 roles/kafka_server.rb
@@ -0,0 +1,10 @@
+name 'kafka_server'
+description 'kafka server'
+
+run_list(*%w[
+ kafka
+ kafka::users
+ kafka::directories
+ kafka::install_from_release
+ kafka::server
+])

0 comments on commit 96c19b7

Please sign in to comment.