diff --git a/Berksfile b/Berksfile
index 8063122b3..99554172b 100644
--- a/Berksfile
+++ b/Berksfile
@@ -11,10 +11,12 @@ solver ENV.fetch('BERKS_SOLVER', :gecode)
#
# Local cookbooks, inside our repository.
#
+cookbook 'bach_backup', path: './cookbooks/bach_backup'
cookbook 'bach_common', path: './cookbooks/bach_common'
cookbook 'bach_krb5', path: './cookbooks/bach_krb5'
cookbook 'bach_repository', path: './cookbooks/bach_repository'
cookbook 'bach_spark', path: './cookbooks/bach_spark'
+cookbook 'backup', path: './cookbooks/backup'
cookbook 'bcpc', path: './cookbooks/bcpc'
cookbook 'bcpc-hadoop', path: './cookbooks/bcpc-hadoop'
cookbook 'bcpc_jmxtrans', path: './cookbooks/bcpc_jmxtrans'
diff --git a/cookbooks/bach_backup/README.md b/cookbooks/bach_backup/README.md
new file mode 100644
index 000000000..2320a85b4
--- /dev/null
+++ b/cookbooks/bach_backup/README.md
@@ -0,0 +1,9 @@
+# bach_backup
+
+`bach_backup` is a wrapper cookbook to configure HDFS backups for BACH clusters.
+It overrides some attributes in the `backup` library cookbook for use on BACH clusters.
+
+# Managing Backup Jobs
+The Backup jobs schedules can be managed through the jobs.yml YAML files.
+
+* [hdfs](files/default/hdfs/jobs.yml)
diff --git a/cookbooks/bach_backup/attributes/default.rb b/cookbooks/bach_backup/attributes/default.rb
new file mode 100644
index 000000000..26c813fbc
--- /dev/null
+++ b/cookbooks/bach_backup/attributes/default.rb
@@ -0,0 +1,62 @@
+# Cookbook Name:: bach_backup_wrapper
+# Override Attributes
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## override global backup properties
+force_default[:backup][:user] = "bach_backup"
+force_default[:backup][:root] = "/archive"
+force_default[:backup][:local][:root] = "/etc/archive"
+
+
+# storage cluster
+set_hosts # set bcpc hadoop hosts
+p force_default[:backup][:namenode] = node[:bcpc][:hadoop][:hdfs_url]
+p force_default[:backup][:jobtracker] = node[:bcpc][:hadoop][:rm_address]
+p force_default[:backup][:oozie] = node[:bcpc][:hadoop][:oozie_url]
+
+# Mapreduce Queue
+force_default[:backup][:queue] = "root.default.#{node[:backup][:user]}"
+
+# hdfs backup jobs list
+## NOTE: refer to file/default/hdfs_jobs.yml for proper data scheme.
+## force_default[:backup][:hdfs][:schedules] = YAML.load_file(File.join(
+## Chef::Config[:file_cache_path],
+## 'cookbooks',
+## 'bach_backup',
+## 'files/default/hdfs/jobs.yml'
+## ))
+### force_default[:backup][:hdfs][:groups] = node[:backup][:hdfs][:jobs].keys
+
+force_default[:backup][:hdfs][:schedules] = {
+ hdfs: {
+ hdfs: 'hdfs://Test-Laptop',
+ start: '2018-02-16T12:00Z',
+ end: '2018-06-16T06:00Z',
+ jobs: [
+ { path: '/tmp', period: 360, },
+ { path: '/user', period: 480, },
+ ]
+ },
+ ubuntu: {
+ hdfs: 'hdfs://Test-Laptop',
+ start: '2018-02-16T12:00Z',
+ end: '2018-06-16T06:00Z',
+ jobs: [
+ { path: '/tmp', period: 1440, },
+ { path: '/user', period: 720, },
+ ]
+ },
+}
diff --git a/cookbooks/bach_backup/files/default/hdfs/jobs.yml b/cookbooks/bach_backup/files/default/hdfs/jobs.yml
new file mode 100644
index 000000000..1c380e706
--- /dev/null
+++ b/cookbooks/bach_backup/files/default/hdfs/jobs.yml
@@ -0,0 +1,34 @@
+# hdfs_jobs.yaml
+# YAML model of HDFS backup jobs.
+# All keys should be preceded with a ':' s.t. they're read as ruby symbols.
+#
+# :group_name:
+# :hdfs: valid hdfs source uri (string)
+# :start: ISO datetime start (string)
+# :end: ISO datetime end (string)
+# :jobs:
+# - :path: hdfs path to backup target (string)
+# :hdfs: (OPTIONAL) override top-level hdfs uri (string)
+# :period: period in minutes between backup actions (integer)
+---
+:hdfs:
+ :hdfs: 'hdfs://Test-Laptop'
+ :start: '2018-02-16T12:00Z'
+ :end: '2018-06-16T06:00Z'
+ :jobs:
+ - :path: '/tmp'
+ :period: 360
+ - :path: '/user'
+ :hdfs: 'hdfs://Test-Laptop'
+ :period: 480
+
+:ubuntu:
+ :hdfs: 'hdfs://Test-Laptop'
+ :start: '2018-02-16T08:00Z'
+ :end: '2018-06-16T08:00Z'
+ :jobs:
+ - :path: '/user'
+ :period: 720
+ - :path: '/tmp'
+ :period: 1440
+
diff --git a/cookbooks/bach_backup/metadata.rb b/cookbooks/bach_backup/metadata.rb
new file mode 100644
index 000000000..92848aab4
--- /dev/null
+++ b/cookbooks/bach_backup/metadata.rb
@@ -0,0 +1,28 @@
+# Cookbook Name:: bach_backup_wrapper
+# metadata.rb
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# encoding: utf-8
+name 'bach_backup'
+maintainer 'Bloomberg Finance L.P.'
+maintainer_email 'hadoop@bloomberg.net'
+description 'Overrides the default attributes in the backup cookbook.'
+license 'Apache 2.0'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version '0.1.0'
+
+# dependencies
+depends 'bcpc-hadoop' # needed for kerberos authentication
diff --git a/cookbooks/bach_backup/recipes/default.rb b/cookbooks/bach_backup/recipes/default.rb
new file mode 100644
index 000000000..e1475edd3
--- /dev/null
+++ b/cookbooks/bach_backup/recipes/default.rb
@@ -0,0 +1,31 @@
+# Cookbook Name:: bach_backup_wrapper
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Resources here are run at compile time.
+# This is necessary to avoid errors in bcpc-hadoop's resource search.
+
+backup_user = node[:backup][:user]
+
+# create hdfs home
+execute 'hdfs home for backup service' do
+ command "hdfs dfs -mkdir -p /user/#{backup_user}"
+ user 'hdfs'
+end
+
+execute 'chown hdfs home for backup service' do
+ command "hdfs dfs -chown #{backup_user} /user/#{backup_user}"
+ user 'hdfs'
+end
diff --git a/cookbooks/backup/README.md b/cookbooks/backup/README.md
new file mode 100644
index 000000000..6d31d511d
--- /dev/null
+++ b/cookbooks/backup/README.md
@@ -0,0 +1,8 @@
+# backup
+
+`backup` is a chef cookbook to setup periodic HDFS inter-cluster backups.
+
+The backup service regularly schedules HDFS distcp actions from source to backup cluster.
+Distcps are run periodically using oozie coordinators and workflows.
+
+## FUTURE: HBase, Hive, and Phoenix backups
diff --git a/cookbooks/backup/attributes/default.rb b/cookbooks/backup/attributes/default.rb
new file mode 100644
index 000000000..12b3248e4
--- /dev/null
+++ b/cookbooks/backup/attributes/default.rb
@@ -0,0 +1,30 @@
+# Cookbook Name:: backup
+# Default Attributes
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## global backup properties
+default[:backup][:user] = "backup"
+default[:backup][:root] = "/backup"
+default[:backup][:local][:root] = "/etc/backup"
+
+# list of enabled backup services
+default[:backup][:services] = [:hdfs]
+
+# storage cluster
+default[:backup][:namenode] = "hdfs://localhost:9000"
+default[:backup][:jobtracker] = "localhost:8032"
+default[:backup][:oozie] = "http://localhost:11000/oozie"
+default[:backup][:queue] = "default"
diff --git a/cookbooks/backup/attributes/hdfs.rb b/cookbooks/backup/attributes/hdfs.rb
new file mode 100644
index 000000000..2a722f5bf
--- /dev/null
+++ b/cookbooks/backup/attributes/hdfs.rb
@@ -0,0 +1,43 @@
+# Cookbook Name:: backup
+# HDFS Backup Attributes
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### hdfs backups
+default[:backup][:hdfs][:user] = node[:backup][:user]
+default[:backup][:hdfs][:root] = "#{node[:backup][:root]}/hdfs"
+default[:backup][:hdfs][:local][:root] = "#{node[:backup][:local][:root]}/hdfs"
+
+# local oozie config dir
+default[:backup][:hdfs][:local][:oozie] =
+ "#{node[:backup][:hdfs][:local][:root]}/oozie"
+
+## hdfs backup tuning parameters
+# timeout in minutes before aborting distcp request
+default[:backup][:hdfs][:timeout] = -1
+
+# bandlimit in MB/s per mapper
+default[:backup][:hdfs][:mapper][:bandwidth] = 25
+
+### hdfs backup requests
+default[:backup][:hdfs][:schedules] = {}
+
+## NOTE: refer to files/default/hdfs/jobs.yml for the proper data scheme.
+# default[:backup][:hdfs][:schedules] = YAML.load_file(File.join(
+# Chef::Config[:file_cache_path],
+# 'cookbooks',
+# 'backup',
+# 'files/default/hdfs/jobs.yml'
+# ))
diff --git a/cookbooks/backup/files/default/hdfs/jobs.yml b/cookbooks/backup/files/default/hdfs/jobs.yml
new file mode 100644
index 000000000..03dd520c1
--- /dev/null
+++ b/cookbooks/backup/files/default/hdfs/jobs.yml
@@ -0,0 +1,24 @@
+# hdfs_jobs.yaml
+# YAML model of backup jobs.
+# Disclaimer: This is an example.
+# All keys should be preceded with a ':' s.t. they're read as ruby symbols.
+#
+# :group_name:
+# :hdfs: valid hdfs source uri (string)
+# :start: ISO datetime start (string)
+# :end: ISO datetime end (string)
+# :jobs:
+# - :path: hdfs path to backup target (string)
+# :hdfs: (OPTIONAL) override top-level hdfs uri (string)
+# :period: period in minutes between backup actions (integer)
+---
+:group:
+ :hdfs: 'hdfs://localhost:9000'
+ :start: '2018-01-01T12:00Z'
+ :end: '2019-12-25T06:00Z'
+ :jobs:
+ - :path: '/tmp'
+ :period: 360
+ - :path: '/user'
+ :hdfs: 'hdfs://localhost:9000' # optional override
+ :period: 480
diff --git a/cookbooks/backup/libraries/oozie_client.rb b/cookbooks/backup/libraries/oozie_client.rb
new file mode 100644
index 000000000..cdfa4bd43
--- /dev/null
+++ b/cookbooks/backup/libraries/oozie_client.rb
@@ -0,0 +1,101 @@
+# oozie_client.rb
+# ruby client for managing oozie jobs
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Oozie
+ class ClientV1
+ attr_accessor :oozie_url, :user
+
+ def initialize(oozie_url='http://localhost:11000/oozie', user='oozie')
+ @oozie = oozie_url
+ @user = user
+ end
+
+ # list the oozie jobs on the server.
+ def jobs(filter={}, jobtype='workflow', len=10)
+ execute('jobs', user, {
+ oozie: @oozie,
+ jobtype: jobtype,
+ filter: "\"#{filter_string(filter)}\"",
+ len: len
+ })
+ end
+
+ # kill the oozie jobs the match the filter.
+ def kill_jobs(filter={}, jobtype='workflow', len=1000)
+ execute('jobs', user, {
+ oozie: @oozie,
+ jobtype: jobtype,
+ filter: "\"#{filter_string(filter)}\"",
+ len: len,
+ kill: nil
+ })
+ end
+
+ # run an oozie job.
+ def run(config, user=@user)
+ execute('job', user, {
+ oozie: @oozie,
+ config: config,
+ run: nil
+ })
+ end
+
+ # rerun an oozie action.
+ def rerun(action_id, config, user=@user)
+ execute('job', user, {
+ oozie: @oozie,
+ config: config,
+ rerun: action_id
+ })
+ end
+
+ # kill an oozie job with the given id.
+ def kill(job_id, user=@user)
+ execute('job', user, {
+ oozie: @oozie,
+ kill: job_id
+ })
+ end
+
+ # get the id of an oozie job with the given name (if it exists).
+ def get_id(job_name, jobtype='workflow', status='RUNNING')
+ jobs_cmd = jobs({ name: job_name, status: status }, 'coordinator', 1)
+ match = jobs_cmd.stdout.match(/(\S+)\s+#{job_name}/)
+ return match.nil? ? nil : match[1]
+ end
+
+ private ## private methods
+
+ def execute(subcommand, user=@user, options={})
+ require 'mixlib/shellout'
+ command = "oozie #{subcommand} #{options_string(options)}"
+ # puts command ## print debug command
+ return Mixlib::ShellOut.new(command, user: user, timeout: 90).run_command
+ end
+
+ def options_string(options)
+ options.map { |key, value| "-#{key.to_s} #{value}" }.join(' ')
+ end
+
+ def filter_string(filter)
+ filter.map { |key, value| "#{key.to_s}=#{value}" }.join(';')
+ end
+ end
+
+ class Client < ClientV1
+ end
+end
diff --git a/cookbooks/backup/metadata.rb b/cookbooks/backup/metadata.rb
new file mode 100644
index 000000000..9c9995cc1
--- /dev/null
+++ b/cookbooks/backup/metadata.rb
@@ -0,0 +1,27 @@
+#
+# Cookbook Name:: backup
+# metadata.rb
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# encoding: utf-8
+name 'backup'
+maintainer 'Bloomberg Finance L.P.'
+maintainer_email 'hadoop@bloomberg.net'
+description 'Cookbook to setup HDFS backups.'
+license 'Apache 2.0'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version '0.1.0'
diff --git a/cookbooks/backup/recipes/bootstrap.rb b/cookbooks/backup/recipes/bootstrap.rb
new file mode 100644
index 000000000..ae1e21d35
--- /dev/null
+++ b/cookbooks/backup/recipes/bootstrap.rb
@@ -0,0 +1,70 @@
+# Cookbook Name:: backup
+# Recipe:: bootstrap
+# Creates the local backup bootstrap directory
+# Generates the starter oozie configurations
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# create the local configuration root
+# holds local copies of the oozie configurations
+directory node[:backup][:local][:root] do
+ owner node[:backup][:user]
+ group node[:backup][:user]
+ mode "0755"
+ action :create
+end
+
+node[:backup][:services].each do |service|
+ # create the service backup root (drwxr-xr-x)
+ directory node[:backup][service][:local][:root] do
+ owner node[:backup][:user]
+ group node[:backup][service][:user]
+ mode "0755"
+ action :create
+ end
+
+ # create the oozie config directory (drwxr-xr-x)
+ directory node[:backup][service][:local][:oozie] do
+ owner node[:backup][:user]
+ group node[:backup][service][:user]
+ mode "0755"
+ action :create
+ end
+
+ # oozie config files
+ oozie_config_dir = node[:backup][service][:local][:oozie]
+ oozie_configs = %w(
+ groups.properties
+ groups.xml
+ workflow.xml
+ coordinator.xml
+ )
+
+ # source configuration templates
+ oozie_configs.each do |config|
+ template "#{oozie_config_dir}/#{config}" do
+ source "#{service}/#{config}.erb"
+ owner node[:backup][:user]
+ group node[:backup][service][:user]
+ mode "0755"
+ action :create
+ variables(
+ service: service,
+ groups: node[:backup][service][:schedules].keys,
+ mode: '-rwxrwx---'
+ )
+ end
+ end
+end
diff --git a/cookbooks/backup/recipes/cleanup.rb b/cookbooks/backup/recipes/cleanup.rb
new file mode 100644
index 000000000..fc0451443
--- /dev/null
+++ b/cookbooks/backup/recipes/cleanup.rb
@@ -0,0 +1,77 @@
+# Cookbook Name::backup
+# Recipe:: cleanup
+# Cleans up the state of the storage cluster.
+# Removes old configurations. Kills stale coordinators.
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# creates the properties files for the service's oozie jobs.
+def get_job_names(service)
+ job_schedules = node[:backup][service][:schedules]
+ job_schedules.inject([]) do |job_names, job_schedule|
+ group, schedule = job_schedule
+ names = (schedule[:jobs] || []).map do |job|
+ "#{group}-#{job[:name] || File.basename(job[:path])}"
+ end
+ job_names.push(*names)
+ end
+end
+
+# removes all local properties in #{path} not included in the #{filter}
+# kills the stale oozie coordinators
+def cleanup_service(filter, service, path)
+ # Get a list of stale jobs
+ # Checks the existing local properties files against current job set.
+ stale_jobs = Dir.glob("#{path}/*.properties").select do |entry|
+ File.file? entry
+ end.map do |filename|
+ /#{path}\/(.+).properties/.match(filename)[1]
+ end.select do |name|
+ !filter.include? name
+ end
+
+ puts "stale jobs: #{stale_jobs}"
+ stale_jobs.each do |name|
+ # remove the local properties file
+ file "#{path}/#{name}.properties#delete" do
+ path "#{path}/#{name}.properties"
+ action :delete
+ end
+
+ # remove the hdfs properties file
+ hdfs_file "#{path}/#{name}.properties#delete" do
+ hdfs node[:backup][:namenode]
+ path "#{node[:backup][service][:root]}/#{name}.properties"
+ admin node[:backup][:user]
+ action :delete
+ end
+
+ # kill stale oozie coordinator
+ oozie_job "backup.#{service}.#{name}#kill" do
+ url node[:backup][:oozie]
+ name "backup.#{service}.#{name}"
+ user node[:backup][:user]
+ action :kill
+ ignore_failure true
+ end
+ end
+end
+
+node[:backup][:services].each do |service|
+ oozie_config_dir = node[:backup][service][:local][:oozie]
+ jobnames = get_job_names(service)
+ jobnames << "groups"
+ cleanup_service(jobnames, service, oozie_config_dir)
+end
diff --git a/cookbooks/backup/recipes/default.rb b/cookbooks/backup/recipes/default.rb
new file mode 100644
index 000000000..195c28f0a
--- /dev/null
+++ b/cookbooks/backup/recipes/default.rb
@@ -0,0 +1,19 @@
+# Cookbook Name::backup
+# Recipe:: default
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Print the jobs list hash
+p node[:backup][:hdfs][:schedules]
diff --git a/cookbooks/backup/recipes/hdfs.rb b/cookbooks/backup/recipes/hdfs.rb
new file mode 100644
index 000000000..0c0121536
--- /dev/null
+++ b/cookbooks/backup/recipes/hdfs.rb
@@ -0,0 +1,37 @@
+# Cookbook Name:: backup
+# Recipe:: hdfs
+# Uploads the bootstrap directory to HDFS
+# Launches the group directory creation workflow
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# upload the bootstrap directory to HDFS
+hdfs_directory node[:backup][:root] do
+ hdfs node[:backup][:namenode]
+ source node[:backup][:local][:root]
+ path File.dirname("#{node[:backup][:root]}")
+ action :put
+end
+
+# launch the group dir creation workflow
+node[:backup][:services].each do |service|
+ oozie_config_dir = node[:backup][service][:local][:oozie]
+ oozie_job "backup.groups.#{service}" do
+ url node[:backup][:oozie]
+ config "#{oozie_config_dir}/groups.properties"
+ user node[:backup][:user]
+ action :run
+ end
+end
diff --git a/cookbooks/backup/recipes/properties.rb b/cookbooks/backup/recipes/properties.rb
new file mode 100644
index 000000000..0942ced20
--- /dev/null
+++ b/cookbooks/backup/recipes/properties.rb
@@ -0,0 +1,73 @@
+# Cookbook Name::backup
+# Recipe:: properties
+# Creates the local oozie job.properties files
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# parses the job properties from an hdfs backup job
+def parse_hdfs_properties(group, schedule, job)
+ # override schedule parameters
+ name = job[:name] || File.basename(job[:path])
+ hdfs_src = job[:hdfs] || schedule[:hdfs]
+ period = job[:period] || schedule[:period]
+ bandwidth = node[:backup][:hdfs][:mapper][:bandwidth] ||
+ node[:backup][:mapper][:bandwidth]
+ queue = node[:backup][:hdfs][:queue] ||
+ node[:backup][:queue]
+
+ return {
+ group: group,
+ path: job[:path],
+ basename: File.basename(job[:path]),
+ jobname: "#{group}-#{name}",
+ hdfs: hdfs_src,
+ period: period,
+ startdate: schedule[:start],
+ enddate: schedule[:end],
+ timeout: node[:backup][:hdfs][:timeout],
+ bandwidth: bandwidth,
+ queue: queue
+ }
+end
+
+def parse_service_properties(service, group, schedule, job)
+ case service.to_sym
+ when :hdfs
+ parse_hdfs_properties(group, schedule, job)
+ else
+ nil # service not found
+ end
+end
+
+# parse job schedules and create properties files
+node[:backup][:services].map do |service|
+ node[:backup][service][:schedules].each do |group, schedule|
+ schedule[:jobs].each do |job|
+
+ # oozie job.properties
+ oozie_config_dir = node[:backup][service][:local][:oozie]
+ job_props = parse_service_properties(service, group, schedule, job)
+ template "#{oozie_config_dir}/#{job_props[:jobname]}.properties" do
+ source "#{service}/backup.properties.erb"
+ owner node[:backup][:user]
+ group node[:backup][service][:user]
+ mode "0755"
+ action :create
+ variables job_props
+ end
+
+ end
+ end
+end
diff --git a/cookbooks/backup/recipes/scheduler.rb b/cookbooks/backup/recipes/scheduler.rb
new file mode 100644
index 000000000..1b8e54345
--- /dev/null
+++ b/cookbooks/backup/recipes/scheduler.rb
@@ -0,0 +1,43 @@
+# Cookbook Name:: backup
+# Recipe:: scheduler
+# Launches the oozie coordinators to schedule periodic backups
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Run each oozie coordinator tracked by the backup service.
+# Only runs the coordinator if it is not already RUNNING
+node[:backup][:services].each do |service|
+ node[:backup][service][:schedules].each do |group, schedule|
+ schedule[:jobs].each do |job|
+
+ # scheduler parameters
+ name = job[:name] ? job[:name] : File.basename(job[:path])
+ jobname = "#{group}-#{name}"
+ oozie_config_dir = node[:backup][service][:local][:oozie]
+ properties_file = "#{oozie_config_dir}/#{jobname}.properties"
+ coordinator_file = "#{oozie_config_dir}/coordinator.xml"
+
+ # restart oozie coordinators
+ oozie_job "backup.#{service}.#{jobname}" do
+ url node[:backup][:oozie]
+ config properties_file
+ user node[:backup][service][:user]
+ action :run
+ ignore_failure true
+ end
+
+ end
+ end
+end
diff --git a/cookbooks/backup/resources/hdfs_directory.rb b/cookbooks/backup/resources/hdfs_directory.rb
new file mode 100644
index 000000000..bd2bdda31
--- /dev/null
+++ b/cookbooks/backup/resources/hdfs_directory.rb
@@ -0,0 +1,68 @@
+# Cookbook Name:: backup
+# Custom hdfs directory resource
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource_name :hdfs_directory
+
+property :hdfs, String, required: true
+property :path, String, name_property: true
+property :admin, String, default: 'hdfs'
+property :source, String
+property :owner, String
+property :group, String
+property :mode, String
+
+action_class do
+ def execute(command, user=admin, timeout=90)
+ require 'mixlib/shellout'
+ Chef::Log.info("Running command(#{user}): #{command}")
+ return Mixlib::ShellOut.new("sudo -u #{user} " + command, timeout: timeout).run_command
+ end
+end
+
+action :create do
+ # create the directory
+ Chef::Log.info("HDFS dir #{path} creation")
+ execute("hdfs dfs -mkdir -p #{hdfs}/#{path}", admin).error!
+
+ # set the owner and group
+ if !(owner.nil? || group.nil?)
+ execute("hdfs dfs -chown #{owner}:#{group} #{hdfs}/#{path}", admin).error!
+ elsif !owner.nil?
+ execute("hdfs dfs -chown #{owner} #{hdfs}/#{path}", admin).error!
+ elsif !group.nil?
+ execute("hdfs dfs -chgrp #{group} #{hdfs}/#{path}", admin).error!
+ end
+
+ # set permissions
+ if !mode.nil?
+ execute("hdfs dfs -chmod #{mode} #{hdfs}/#{path}", admin).error!
+ end
+end
+
+action :put do
+ Chef::Log.info("Copying #{path} to HDFS")
+ execute("hdfs dfs -put -f -p #{source} #{hdfs}/#{path}", admin).run_command.error!
+end
+
+action :delete do
+ # speculative, recursive delete. (ignores error)
+ Chef::Log.info("HDFS dir #{path} deletion")
+ execute("hdfs dfs -rm -r -f #{hdfs}/#{path}", admin)
+end
+
+action :nothing do
+end
diff --git a/cookbooks/backup/resources/hdfs_file.rb b/cookbooks/backup/resources/hdfs_file.rb
new file mode 100644
index 000000000..7a7911ce6
--- /dev/null
+++ b/cookbooks/backup/resources/hdfs_file.rb
@@ -0,0 +1,73 @@
+# Cookbook Name:: backup
+# Custom hdfs file resource
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource_name :hdfs_file
+
+property :hdfs, String, required: true
+property :path, String, required: true
+property :admin, String, default: 'hdfs'
+property :source, String
+property :owner, String
+property :group, String
+property :mode, String
+
+action_class do
+ def execute(command, user=admin, timeout=90)
+ require 'mixlib/shellout'
+ Chef::Log.info("Running command(#{user}): #{command}")
+ puts command
+ return Mixlib::ShellOut.new("sudo -u #{user} " + command, timeout: timeout).run_command
+ end
+end
+
+action :create do
+ # create the file
+ Chef::Log.info("HDFS file #{path} creation")
+ if source.nil?
+ execute("hdfs dfs -touchz #{hdfs}/#{path}", admin).error!
+ else
+ execute("hdfs dfs -put -p -f #{source} #{hdfs}/#{path}", admin).error!
+ end
+
+ # set the owner and group
+ if !(owner.nil? || group.nil?)
+ execute("hdfs dfs -chown #{owner}:#{group} #{hdfs}/#{path}", admin).error!
+ elsif !owner.nil?
+ execute("hdfs dfs -chown #{owner} #{hdfs}/#{path}", admin).error!
+ elsif !group.nil?
+ execute("hdfs dfs -chgrp #{group} #{hdfs}/#{path}", admin).error!
+ end
+
+ # set permissions
+ if !mode.nil?
+ execute("hdfs dfs -chmod #{mode} #{hdfs}/#{path}", admin).error!
+ end
+end
+
+action :put do
+ Chef::Log.info("Copying #{path} to HDFS")
+ execute("hdfs dfs -put -f -p #{source} #{hdfs}/#{path}", admin).run_command.error!
+end
+
+action :delete do
+ # speculative, recursive delete. (ignores error)
+ Chef::Log.info("HDFS file #{path} deletion")
+ execute("hdfs dfs -rm -r -f #{hdfs}/#{path}", admin)
+end
+
+action :nothing do
+end
diff --git a/cookbooks/backup/resources/oozie_job.rb b/cookbooks/backup/resources/oozie_job.rb
new file mode 100644
index 000000000..1f5d8afc1
--- /dev/null
+++ b/cookbooks/backup/resources/oozie_job.rb
@@ -0,0 +1,74 @@
+# Cookbook Name:: backup
+# Custom oozie job resource
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource_name :oozie_job
+provides :oozie_job
+
+property :name, String, name_property: true
+property :url, String, required: true
+property :config, String, required: true
+property :user, String, default: 'oozie'
+
+action_class do
+ include Oozie
+end
+
+action :run do
+ Chef::Log.info("Starting oozie job: #{name}")
+ client = Oozie::Client.new(url, user)
+
+ # check if the job is already running
+ job_id = client.get_id(name, 'coordinator', 'RUNNING')
+
+ # start the service
+ if job_id.nil?
+ run_cmd = client.run(config, user)
+ run_cmd.error!
+ end
+end
+
+action :restart do
+ Chef::Log.info("Rerunning oozie job: #{name}")
+ client = Oozie::Client.new(url, user)
+
+ # check if the job is already running
+ job_id = client.get_id(name, 'coordinator', 'RUNNING')
+
+ if job_id.nil?
+ # start the service
+ run_cmd = client.run(config, user)
+ run_cmd.error!
+ else
+ # kill and restart the service
+ kill_cmd = client.kill(job_id, user)
+ kill_cmd.error!
+ rerun_cmd = client.run(config, user)
+ rerun_cmd.error!
+ end
+end
+
+action :kill do
+ Chef::Log.info("Killing oozie job: #{name}")
+ client = Oozie::Client.new(url, user)
+
+ # kill the service (if it exists)
+ jobs_cmd = client.kill_jobs({ name: name }, "coordinator")
+ jobs_cmd.error!
+end
+
+action :nothing do
+end
diff --git a/cookbooks/backup/templates/default/hdfs/backup.properties.erb b/cookbooks/backup/templates/default/hdfs/backup.properties.erb
new file mode 100644
index 000000000..6a6fdf317
--- /dev/null
+++ b/cookbooks/backup/templates/default/hdfs/backup.properties.erb
@@ -0,0 +1,66 @@
+#
+# Cookbook Name:: backup
+# hdfs.properties
+# Properties file for an oozie backup job
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# group name [A-Za-z_]+
+group=<%= @group %>
+
+# job name [A-Za-z_]+
+jobname=<%= @jobname %>
+
+# backup file / dir
+target=<%= @path %>
+basename=<%= @basename %>
+
+# backup period in minutes
+period=<%= @period %>
+
+# backup timeout in minutes (-1 = no timeout)
+timeout=<%= @timeout %>
+
+# bandwidth per distcp mapper in MB/s
+bandwidth=<%= @bandwidth %>
+
+# yarn queue
+queue=<%= @queue %>
+
+# source cluster
+hdfs.src.namenode=<%= @hdfs %>
+
+# storage cluster
+hdfs.dest.namenode=<%= node[:backup][:namenode] %>
+hdfs.backup.root=${hdfs.dest.namenode}/<%= node[:backup][:hdfs][:root].sub("/", "") %>
+
+# group properties
+group.backup.coordinator=${hdfs.backup.root}/oozie/coordinator.xml
+group.backup.workflow=${hdfs.backup.root}/oozie/workflow.xml
+
+# backup schedule (ISO date)
+group.backup.start=<%= @startdate %>
+group.backup.end=<%= @enddate %>
+
+# oozie properties
+oozie.use.system.libpath=true
+# oozie.wf.application.path=${group.backup.workflow}
+oozie.coord.application.path=${group.backup.coordinator}
+oozie.launcher.mapreduce.job.hdfs-servers=${hdfs.src.namenode},${hdfs.dest.namenode}
+
+# hadoop properties
+namenode=<%= node[:backup][:namenode] %>
+jobtracker=<%= node[:backup][:jobtracker] %>
diff --git a/cookbooks/backup/templates/default/hdfs/coordinator.xml.erb b/cookbooks/backup/templates/default/hdfs/coordinator.xml.erb
new file mode 100644
index 000000000..f193491d8
--- /dev/null
+++ b/cookbooks/backup/templates/default/hdfs/coordinator.xml.erb
@@ -0,0 +1,39 @@
+
+
+
+
+
+ ${timeout}
+ 1
+ LAST_ONLY
+
+
+
+
+ ${coord:conf('group.backup.workflow')}
+
+
+
+
diff --git a/cookbooks/backup/templates/default/hdfs/groups.properties.erb b/cookbooks/backup/templates/default/hdfs/groups.properties.erb
new file mode 100644
index 000000000..705a09f38
--- /dev/null
+++ b/cookbooks/backup/templates/default/hdfs/groups.properties.erb
@@ -0,0 +1,31 @@
+#
+# Cookbook Name:: backup
+# groups.properties
+# Properties file for an oozie backup job
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# storage cluster
+hdfs.dest.namenode=<%= node[:backup][:namenode] %>
+hdfs.backup.root=${hdfs.dest.namenode}/<%= node[:backup][:hdfs][:root].sub("/", "") %>
+
+# oozie properties
+oozie.use.system.libpath=true
+oozie.wf.application.path=${hdfs.backup.root}/oozie/groups.xml
+
+# hadoop properties
+namenode=<%= node[:backup][:namenode] %>
+jobtracker=<%= node[:backup][:jobtracker] %>
diff --git a/cookbooks/backup/templates/default/hdfs/groups.xml.erb b/cookbooks/backup/templates/default/hdfs/groups.xml.erb
new file mode 100644
index 000000000..3286d05c3
--- /dev/null
+++ b/cookbooks/backup/templates/default/hdfs/groups.xml.erb
@@ -0,0 +1,46 @@
+
+
+
+
+
+
+
+
+
+
+ <% @groups.each do |group| %>
+
+
+
+ <%- end %>
+
+
+
+
+
+
+ [${wf:errorMessage(wf:lastErrorNode())}]
+
+
+
+
diff --git a/cookbooks/backup/templates/default/hdfs/workflow.xml.erb b/cookbooks/backup/templates/default/hdfs/workflow.xml.erb
new file mode 100644
index 000000000..f6e8bcc34
--- /dev/null
+++ b/cookbooks/backup/templates/default/hdfs/workflow.xml.erb
@@ -0,0 +1,53 @@
+
+
+
+
+
+
+
+
+
+ ${jobtracker}
+ ${namenode}
+
+
+ -Dmapred.job.queue.name=${queue}
+
+
+ -ppugbtxc
+ -update
+ -bandwidth
+ ${bandwidth}
+ -i
+
+
+ ${wf:conf("hdfs.src.namenode")}/${target}
+ ${wf:conf("hdfs.backup.root")}/${group}/${basename}
+
+
+
+
+
+
+ [${wf:errorMessage(wf:lastErrorNode())}]
+
+
+
+
diff --git a/cookbooks/bcpc-hadoop/attributes/backup.rb b/cookbooks/bcpc-hadoop/attributes/backup.rb
new file mode 100644
index 000000000..70cbbe4ea
--- /dev/null
+++ b/cookbooks/bcpc-hadoop/attributes/backup.rb
@@ -0,0 +1,18 @@
+# Cookbook Name:: bcpc-hadoop
+# BCPC Hadoop Attributes for backup jobs
+#
+# Copyright 2018, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default[:bcpc][:hadoop][:backup][:user] = "bach_backup"
diff --git a/cookbooks/bcpc-hadoop/attributes/kerberos.rb b/cookbooks/bcpc-hadoop/attributes/kerberos.rb
index 79fa2a737..3b4577da7 100644
--- a/cookbooks/bcpc-hadoop/attributes/kerberos.rb
+++ b/cookbooks/bcpc-hadoop/attributes/kerberos.rb
@@ -128,6 +128,15 @@
perms: '0440',
spnego_keytab: 'spnego.service.keytab'
},
+ backup: {
+ principal: node[:bcpc][:hadoop][:backup][:user],
+ keytab: 'backup.service.keytab',
+ owner: node[:bcpc][:hadoop][:backup][:user],
+ group: node[:bcpc][:hadoop][:backup][:user],
+ princhost: '_HOST',
+ perms: '0440',
+ spnego_keytab: 'spnego.service.keytab'
+ },
ambari: {
principal: "#{node['bcpc']['hadoop']['proxyuser']['ambari']}",
keytab: 'ambari.service.keytab',
diff --git a/cookbooks/bcpc-hadoop/recipes/bach_backup_user.rb b/cookbooks/bcpc-hadoop/recipes/bach_backup_user.rb
new file mode 100644
index 000000000..f5ba0e4f0
--- /dev/null
+++ b/cookbooks/bcpc-hadoop/recipes/bach_backup_user.rb
@@ -0,0 +1,34 @@
+# Cookbook Name:: bcpc-hadoop
+# Recipe:: bach_backup_user
+# Creates the user needed for hadoop cluster backup jobs
+#
+# Copyright 2016, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+backup_user = node[:bcpc][:hadoop][:backup][:user]
+
+user backup_user do
+ action :create
+ comment 'backup service user'
+end
+
+# make backup user an hdfs superuser
+group 'hdfs' do
+ members backup_user
+ append true
+end
+
+configure_kerberos 'backup_kerberos' do
+ service_name 'backup'
+end
diff --git a/cookbooks/bcpc-hadoop/recipes/bach_backup_wrapper.rb b/cookbooks/bcpc-hadoop/recipes/bach_backup_wrapper.rb
new file mode 100644
index 000000000..7fc33c0c9
--- /dev/null
+++ b/cookbooks/bcpc-hadoop/recipes/bach_backup_wrapper.rb
@@ -0,0 +1,19 @@
+# Cookbook Name:: bcpc-hadoop
+# Recipe:: bach_backup_wrapper
+# Wrapper recipe for bach_backup hadoop cluster support
+#
+# Copyright 2016, Bloomberg Finance L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_recipe 'bcpc-hadoop::bach_backup_user'
diff --git a/stub-environment/roles/BACH-Backup.json b/stub-environment/roles/BACH-Backup.json
new file mode 100644
index 000000000..7bff90289
--- /dev/null
+++ b/stub-environment/roles/BACH-Backup.json
@@ -0,0 +1,19 @@
+{
+ "name": "BACH-Backup",
+ "description": "Configures the backup service on a backup cluster leader.",
+ "chef_type": "role",
+ "json_class": "Chef::Role",
+ "run_list": [
+ "recipe[bach_backup::default]",
+ "recipe[backup::default]",
+ "recipe[backup::bootstrap]",
+ "recipe[backup::properties]",
+ "recipe[backup::hdfs]",
+ "recipe[backup::scheduler]",
+ "recipe[backup::cleanup]"
+ ],
+ "default_attributes": {
+ },
+ "override_attributes": {
+ }
+}
diff --git a/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json b/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json
index 795c32430..0f837baec 100644
--- a/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json
+++ b/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json
@@ -4,7 +4,8 @@
"run_list": [
"role[Basic]",
"recipe[bcpc-hadoop::resource_manager]",
- "recipe[bcpc-hadoop::smoke_test_user]"
+ "recipe[bcpc-hadoop::smoke_test_user]",
+ "recipe[bcpc-hadoop::bach_backup_user]"
],
"description": "A highly-available head node in a BCPC Hadoop cluster",
"chef_type": "role",
diff --git a/stub-environment/roles/BCPC-Hadoop-Head.json b/stub-environment/roles/BCPC-Hadoop-Head.json
index 0707e5cc1..a29607fe6 100644
--- a/stub-environment/roles/BCPC-Hadoop-Head.json
+++ b/stub-environment/roles/BCPC-Hadoop-Head.json
@@ -23,6 +23,7 @@
"recipe[hdfsdu::create_user]",
"recipe[bcpc-hadoop::configs]",
"recipe[pam::default]",
+ "recipe[bcpc-hadoop::bach_backup_wrapper]",
"recipe[bcpc-hadoop::zookeeper_server]",
"recipe[bcpc-hadoop::journalnode]",
"recipe[bcpc-hadoop::graphite_to_zabbix]"
diff --git a/stub-environment/roles/BCPC-Hadoop-Worker.json b/stub-environment/roles/BCPC-Hadoop-Worker.json
index df4c2e905..cc1200f8b 100644
--- a/stub-environment/roles/BCPC-Hadoop-Worker.json
+++ b/stub-environment/roles/BCPC-Hadoop-Worker.json
@@ -12,6 +12,7 @@
"recipe[bcpc-hadoop::hdp_repo]",
"recipe[bach_krb5::krb5_client]",
"recipe[hdfsdu::create_user]",
+ "recipe[bcpc-hadoop::bach_backup_wrapper]",
"recipe[bcpc-hadoop::configs]",
"recipe[pam::default]",
"recipe[bach_spark::default]",