Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

added a "bin_dir" parameter to configure where the Kafka scripts are #147

Closed
wants to merge 11 commits into from
Closed

added a "bin_dir" parameter to configure where the Kafka scripts are #147

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f43ac28
issue#145: added the "bin_dir" parameter to configure where the Kafka…
LionelCons May 17, 2017
20f3791
modulesync 0.21.3
bastelfreak Jun 11, 2017
af44357
fix open ended dependency version requirement
bastelfreak Jun 11, 2017
c6c44ae
delete Guardfile
bastelfreak Jun 11, 2017
fff27dc
bump to minimal recommended puppet version
bastelfreak Jun 11, 2017
2819c0d
bump to minimal recommended stdlib version
bastelfreak Jun 11, 2017
34b72a2
test against latest dependencies
bastelfreak Jun 11, 2017
8942fca
only test for working URLs
bastelfreak Jun 11, 2017
23e8192
rebased from upstream
LionelCons Jun 29, 2017
232f96b
add support to kafka app logs dir
jolivares May 22, 2017
65e92b9
reuse params log_dir
jolivares May 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions manifests/broker.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
# [*config_dir*]
# The directory to create the kafka config files to
#
# [*bin_dir*]
# The directory where the kafka scripts are
#
# === Examples
#
# Create a single broker instance which talks to a local zookeeper instance.
Expand Down Expand Up @@ -68,6 +71,7 @@
$group_id = $kafka::params::group_id,
$user_id = $kafka::params::user_id,
$config_dir = $kafka::params::config_dir,
$bin_dir = $kafka::params::bin_dir,
) inherits kafka::params {

validate_re($::osfamily, 'RedHat|Debian\b', "${::operatingsystem} not supported")
Expand Down
1 change: 1 addition & 0 deletions manifests/broker/service.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
$heap_opts = $kafka::broker::heap_opts,
$opts = $kafka::broker::opts,
$config_dir = $kafka::broker::config_dir,
$bin_dir = $kafka::broker::bin_dir,
) {

if $caller_module_name != $module_name {
Expand Down
4 changes: 4 additions & 0 deletions manifests/consumer.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
# [*config_dir*]
# The directory to create the kafka config files to
#
# [*bin_dir*]
# The directory where the kafka scripts are
#
# === Examples
#
# Create the consumer service connecting to a local zookeeper
Expand All @@ -63,6 +66,7 @@
$consumer_jmx_opts = $kafka::params::consumer_jmx_opts,
$consumer_log4j_opts = $kafka::params::consumer_log4j_opts,
$config_dir = $kafka::params::config_dir,
$bin_dir = $kafka::params::bin_dir,
) inherits kafka::params {

validate_re($::osfamily, 'RedHat|Debian\b', "${::operatingsystem} not supported")
Expand Down
1 change: 1 addition & 0 deletions manifests/consumer/service.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
$service_config = $kafka::consumer::service_config,
$service_defaults = $kafka::consumer::service_defaults,
$service_requires_zookeeper = $kafka::consumer::service_requires_zookeeper,
$bin_dir = $kafka::consumer::bin_dir,
) {

if $caller_module_name != $module_name {
Expand Down
4 changes: 4 additions & 0 deletions manifests/init.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@
# [*config_dir*]
# The directory to create the kafka config files to
#
# [*bin_dir*]
# The directory where the kafka scripts are
#
# [*log_dir*]
# The directory for kafka log files
#
Expand All @@ -72,6 +75,7 @@
$user = $kafka::params::user,
$group = $kafka::params::group,
$config_dir = $kafka::params::config_dir,
$bin_dir = $kafka::params::bin_dir,
$log_dir = $kafka::params::log_dir,
) inherits kafka::params {

Expand Down
4 changes: 4 additions & 0 deletions manifests/mirror.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@
# [*config_dir*]
# The directory to create the kafka config files to
#
# [*bin_dir*]
# The directory where the kafka scripts are
#
# === Examples
#
# Create the mirror service connecting to a local zookeeper
Expand Down Expand Up @@ -85,6 +88,7 @@
$mirror_jmx_opts = $kafka::params::mirror_jmx_opts,
$mirror_log4j_opts = $kafka::params::mirror_log4j_opts,
$config_dir = $kafka::params::config_dir,
$bin_dir = $kafka::params::bin_dir,
) inherits kafka::params {

validate_re($::osfamily, 'RedHat|Debian\b', "${::operatingsystem} not supported")
Expand Down
1 change: 1 addition & 0 deletions manifests/mirror/service.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
$blacklist = $kafka::mirror::blacklist,
$max_heap = $kafka::mirror::max_heap,
$config_dir = $kafka::params::config_dir,
$bin_dir = $kafka::params::bin_dir,
) inherits ::kafka::params {

if $caller_module_name != $module_name {
Expand Down
1 change: 1 addition & 0 deletions manifests/params.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
$scala_version = '2.11'
$install_dir = "/opt/kafka-${scala_version}-${version}"
$config_dir = '/opt/kafka/config'
$bin_dir = '/opt/kafka/bin'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since we're on puppet4 (or even 5 \o/) now, can you please use the datatype Stdlib::Absolutepath here (and in the other classes where you use the $bin_dir)?

$log_dir = '/var/log/kafka'
$mirror_url = 'http://mirrors.ukfast.co.uk/sites/ftp.apache.org'
$install_java = true
Expand Down
4 changes: 4 additions & 0 deletions manifests/producer.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
# [*config_dir*]
# The directory to create the kafka config files to
#
# [*bin_dir*]
# The directory where the kafka scripts are
#
# === Examples
#
# Create the producer service connecting to a local zookeeper
Expand All @@ -65,6 +68,7 @@
$producer_jmx_opts = $kafka::params::producer_jmx_opts,
$producer_log4j_opts = $kafka::params::producer_log4j_opts,
$config_dir = $kafka::params::config_dir,
$bin_dir = $kafka::params::bin_dir,
) inherits kafka::params {

validate_re($::osfamily, 'RedHat|Debian\b', "${::operatingsystem} not supported")
Expand Down
1 change: 1 addition & 0 deletions manifests/producer/service.pp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
$service_config = $kafka::producer::service_config,
$service_defaults = $kafka::producer::service_defaults,
$service_requires_zookeeper = $kafka::producer::service_requires_zookeeper,
$bin_dir = $kafka::producer::bin_dir,
) {

if $caller_module_name != $module_name {
Expand Down
8 changes: 4 additions & 4 deletions templates/init.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ KAFKA_USER=kafka
when 'kafka' -%>
PGREP_PATTERN=kafka.Kafka

DAEMON="/opt/kafka/bin/kafka-server-start.sh"
DAEMON="<%= @bin_dir %>/kafka-server-start.sh"
DAEMON_OPTS="<%= @config_dir %>/server.properties"

export KAFKA_JMX_OPTS="<%= @jmx_opts %>"
Expand All @@ -36,15 +36,15 @@ export KAFKA_OPTS="<%= @opts %>"
<%- when 'kafka-consumer' -%>
PGREP_PATTERN=kafka.tools.ConsoleConsumer

DAEMON="/opt/kafka/bin/kafka-console-consumer.sh"
DAEMON="<%= @bin_dir %>/kafka-console-consumer.sh"
DAEMON_OPTS="<% @consumer_service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%>=<%= v.is_a?(Array) ? v.join(',') : v %> <% end -%><% end -%>"

export KAFKA_JMX_OPTS="<%= @consumer_jmx_opts %>"
export KAFKA_LOG4J_OPTS="<%= @consumer_log4j_opts %>"
<%- when 'kafka-mirror' -%>
PGREP_PATTERN=kafka.tools.MirrorMaker

DAEMON="/opt/kafka/bin/kafka-run-class.sh"
DAEMON="<%= @bin_dir %>/kafka-run-class.sh"
DAEMON_OPTS="kafka.tools.MirrorMaker --consumer.config <%= @consumer_config -%> --num.streams <%= @num_streams -%> --producer.config <%= @producer_config -%><%- if (scope.function_versioncmp([scope.lookupvar('kafka::version'), '0.9.0.0']) < 0) -%> --num.producers <%= @num_producers -%><%- end -%><%- if !@whitelist.eql?('') -%> --whitelist='<%= @whitelist -%>'<%- end %><%- if !@blacklist.eql?('') -%> --blacklist='<%= @blacklist -%>'<%- end -%> <%= @abort_on_send_failure_opt %>"

export KAFKA_HEAP_OPTS="-Xmx<%= @max_heap -%> -Xms<%= @max_heap -%>"
Expand All @@ -53,7 +53,7 @@ export KAFKA_LOG4J_OPTS="<%= @mirror_log4j_opts %>"
<%- when 'kafka-producer' -%>
PGREP_PATTERN=kafka.tools.ConsoleProducer

DAEMON="/opt/kafka/bin/kafka-console-producer.sh"
DAEMON="<%= @bin_dir %>/kafka-console-producer.sh"
DAEMON_OPTS="<% @producer_service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%>=<%= v.is_a?(Array) ? v.join(',') : v %> <% end -%><% end -%>"
PRODUCER_INPUT="<%= @input %>"

Expand Down
8 changes: 4 additions & 4 deletions templates/unit.erb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,20 @@ Environment='KAFKA_HEAP_OPTS=<%= @heap_opts %>'
Environment='KAFKA_LOG4J_OPTS=<%= @log4j_opts %>'
Environment='KAFKA_JMX_OPTS=<%= @jmx_opts %>'
Environment='KAFKA_OPTS=<%= @opts %>'
ExecStart=/opt/kafka/bin/kafka-server-start.sh <%= @config_dir %>/server.properties
ExecStart=<%= @bin_dir %>/kafka-server-start.sh <%= @config_dir %>/server.properties
<%- when 'kafka-consumer' -%>
Environment='KAFKA_LOG4J_OPTS=<%= @consumer_log4j_opts %>'
Environment='KAFKA_JMX_OPTS=<%= @consumer_jmx_opts %>'
ExecStart=/opt/kafka/bin/kafka-console-consumer.sh <% @consumer_service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%>=<%= v.is_a?(Array) ? v.join(',') : v %> <% end -%><% end -%>
ExecStart=<%= @bin_dir %>/kafka-console-consumer.sh <% @consumer_service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%>=<%= v.is_a?(Array) ? v.join(',') : v %> <% end -%><% end -%>
<%- when 'kafka-mirror' -%>
Environment='KAFKA_LOG4J_OPTS=<%= @mirror_log4j_opts %>'
Environment='KAFKA_JMX_OPTS=<%= @mirror_jmx_opts %>'
Environment='KAFKA_HEAP_OPTS=-Xmx<%= @max_heap -%>'
ExecStart=/opt/kafka/bin/kafka-run-class.sh kafka.tools.MirrorMaker --consumer.config <%= @consumer_config -%> --num.streams <%= @num_streams -%> --producer.config <%= @producer_config -%><%- if (scope.function_versioncmp([scope.lookupvar('kafka::version'), '0.9.0.0']) < 0) -%> --num.producers <%= @num_producers -%><%- end -%><%- if !@whitelist.eql?('') -%> --whitelist='<%= @whitelist -%>'<%- end %><%- if !@blacklist.eql?('') -%> --blacklist='<%= @blacklist -%>'<%- end -%> <%= @abort_on_send_failure_opt %>
ExecStart=<%= @bin_dir %>/kafka-run-class.sh kafka.tools.MirrorMaker --consumer.config <%= @consumer_config -%> --num.streams <%= @num_streams -%> --producer.config <%= @producer_config -%><%- if (scope.function_versioncmp([scope.lookupvar('kafka::version'), '0.9.0.0']) < 0) -%> --num.producers <%= @num_producers -%><%- end -%><%- if !@whitelist.eql?('') -%> --whitelist='<%= @whitelist -%>'<%- end %><%- if !@blacklist.eql?('') -%> --blacklist='<%= @blacklist -%>'<%- end -%> <%= @abort_on_send_failure_opt %>
<%- when 'kafka-producer' -%>
Environment='KAFKA_LOG4J_OPTS=<%= @producer_log4j_opts %>'
Environment='KAFKA_JMX_OPTS=<%= @producer_jmx_opts %>'
ExecStart=/opt/kafka/bin/kafka-console-producer.sh <% @producer_service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%>=<%= v.is_a?(Array) ? v.join(',') : v %> <% end -%><% end -%> <%= @input %>
ExecStart=<%= @bin_dir %>/kafka-console-producer.sh <% @producer_service_config.sort.each do |k,v| -%><% unless v.to_s.strip.empty? -%>--<%= k -%>=<%= v.is_a?(Array) ? v.join(',') : v %> <% end -%><% end -%> <%= @input %>
<%- end -%>
LimitNOFILE=65536
LimitCORE=infinity
Expand Down