diff --git a/Berksfile b/Berksfile index 8063122b3..99554172b 100644 --- a/Berksfile +++ b/Berksfile @@ -11,10 +11,12 @@ solver ENV.fetch('BERKS_SOLVER', :gecode) # # Local cookbooks, inside our repository. # +cookbook 'bach_backup', path: './cookbooks/bach_backup' cookbook 'bach_common', path: './cookbooks/bach_common' cookbook 'bach_krb5', path: './cookbooks/bach_krb5' cookbook 'bach_repository', path: './cookbooks/bach_repository' cookbook 'bach_spark', path: './cookbooks/bach_spark' +cookbook 'backup', path: './cookbooks/backup' cookbook 'bcpc', path: './cookbooks/bcpc' cookbook 'bcpc-hadoop', path: './cookbooks/bcpc-hadoop' cookbook 'bcpc_jmxtrans', path: './cookbooks/bcpc_jmxtrans' diff --git a/cookbooks/bach_backup/README.md b/cookbooks/bach_backup/README.md new file mode 100644 index 000000000..2320a85b4 --- /dev/null +++ b/cookbooks/bach_backup/README.md @@ -0,0 +1,9 @@ +# bach_backup + +`bach_backup` is a wrapper cookbook to configure HDFS backups for BACH clusters. +It overrides some attributes in the `backup` library cookbook for use on BACH clusters. + +# Managing Backup Jobs +The Backup jobs schedules can be managed through the jobs.yml YAML files. + +* [hdfs](files/default/hdfs/jobs.yml) diff --git a/cookbooks/bach_backup/attributes/default.rb b/cookbooks/bach_backup/attributes/default.rb new file mode 100644 index 000000000..26c813fbc --- /dev/null +++ b/cookbooks/bach_backup/attributes/default.rb @@ -0,0 +1,62 @@ +# Cookbook Name:: bach_backup_wrapper +# Override Attributes +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## override global backup properties +force_default[:backup][:user] = "bach_backup" +force_default[:backup][:root] = "/archive" +force_default[:backup][:local][:root] = "/etc/archive" + + +# storage cluster +set_hosts # set bcpc hadoop hosts +p force_default[:backup][:namenode] = node[:bcpc][:hadoop][:hdfs_url] +p force_default[:backup][:jobtracker] = node[:bcpc][:hadoop][:rm_address] +p force_default[:backup][:oozie] = node[:bcpc][:hadoop][:oozie_url] + +# Mapreduce Queue +force_default[:backup][:queue] = "root.default.#{node[:backup][:user]}" + +# hdfs backup jobs list +## NOTE: refer to file/default/hdfs_jobs.yml for proper data scheme. +## force_default[:backup][:hdfs][:schedules] = YAML.load_file(File.join( +## Chef::Config[:file_cache_path], +## 'cookbooks', +## 'bach_backup', +## 'files/default/hdfs/jobs.yml' +## )) +### force_default[:backup][:hdfs][:groups] = node[:backup][:hdfs][:jobs].keys + +force_default[:backup][:hdfs][:schedules] = { + hdfs: { + hdfs: 'hdfs://Test-Laptop', + start: '2018-02-16T12:00Z', + end: '2018-06-16T06:00Z', + jobs: [ + { path: '/tmp', period: 360, }, + { path: '/user', period: 480, }, + ] + }, + ubuntu: { + hdfs: 'hdfs://Test-Laptop', + start: '2018-02-16T12:00Z', + end: '2018-06-16T06:00Z', + jobs: [ + { path: '/tmp', period: 1440, }, + { path: '/user', period: 720, }, + ] + }, +} diff --git a/cookbooks/bach_backup/files/default/hdfs/jobs.yml b/cookbooks/bach_backup/files/default/hdfs/jobs.yml new file mode 100644 index 000000000..1c380e706 --- /dev/null +++ b/cookbooks/bach_backup/files/default/hdfs/jobs.yml @@ -0,0 +1,34 @@ +# hdfs_jobs.yaml +# YAML model of HDFS backup jobs. +# All keys should be preceded with a ':' s.t. they're read as ruby symbols. +# +# :group_name: +# :hdfs: valid hdfs source uri (string) +# :start: ISO datetime start (string) +# :end: ISO datetime end (string) +# :jobs: +# - :path: hdfs path to backup target (string) +# :hdfs: (OPTIONAL) override top-level hdfs uri (string) +# :period: period in minutes between backup actions (integer) +--- +:hdfs: + :hdfs: 'hdfs://Test-Laptop' + :start: '2018-02-16T12:00Z' + :end: '2018-06-16T06:00Z' + :jobs: + - :path: '/tmp' + :period: 360 + - :path: '/user' + :hdfs: 'hdfs://Test-Laptop' + :period: 480 + +:ubuntu: + :hdfs: 'hdfs://Test-Laptop' + :start: '2018-02-16T08:00Z' + :end: '2018-06-16T08:00Z' + :jobs: + - :path: '/user' + :period: 720 + - :path: '/tmp' + :period: 1440 + diff --git a/cookbooks/bach_backup/metadata.rb b/cookbooks/bach_backup/metadata.rb new file mode 100644 index 000000000..92848aab4 --- /dev/null +++ b/cookbooks/bach_backup/metadata.rb @@ -0,0 +1,28 @@ +# Cookbook Name:: bach_backup_wrapper +# metadata.rb +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# encoding: utf-8 +name 'bach_backup' +maintainer 'Bloomberg Finance L.P.' +maintainer_email 'hadoop@bloomberg.net' +description 'Overrides the default attributes in the backup cookbook.' +license 'Apache 2.0' +long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) +version '0.1.0' + +# dependencies +depends 'bcpc-hadoop' # needed for kerberos authentication diff --git a/cookbooks/bach_backup/recipes/default.rb b/cookbooks/bach_backup/recipes/default.rb new file mode 100644 index 000000000..e1475edd3 --- /dev/null +++ b/cookbooks/bach_backup/recipes/default.rb @@ -0,0 +1,31 @@ +# Cookbook Name:: bach_backup_wrapper +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Resources here are run at compile time. +# This is necessary to avoid errors in bcpc-hadoop's resource search. + +backup_user = node[:backup][:user] + +# create hdfs home +execute 'hdfs home for backup service' do + command "hdfs dfs -mkdir -p /user/#{backup_user}" + user 'hdfs' +end + +execute 'chown hdfs home for backup service' do + command "hdfs dfs -chown #{backup_user} /user/#{backup_user}" + user 'hdfs' +end diff --git a/cookbooks/backup/README.md b/cookbooks/backup/README.md new file mode 100644 index 000000000..6d31d511d --- /dev/null +++ b/cookbooks/backup/README.md @@ -0,0 +1,8 @@ +# backup + +`backup` is a chef cookbook to setup periodic HDFS inter-cluster backups. + +The backup service regularly schedules HDFS distcp actions from source to backup cluster. +Distcps are run periodically using oozie coordinators and workflows. + +## FUTURE: HBase, Hive, and Phoenix backups diff --git a/cookbooks/backup/attributes/default.rb b/cookbooks/backup/attributes/default.rb new file mode 100644 index 000000000..12b3248e4 --- /dev/null +++ b/cookbooks/backup/attributes/default.rb @@ -0,0 +1,30 @@ +# Cookbook Name:: backup +# Default Attributes +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## global backup properties +default[:backup][:user] = "backup" +default[:backup][:root] = "/backup" +default[:backup][:local][:root] = "/etc/backup" + +# list of enabled backup services +default[:backup][:services] = [:hdfs] + +# storage cluster +default[:backup][:namenode] = "hdfs://localhost:9000" +default[:backup][:jobtracker] = "localhost:8032" +default[:backup][:oozie] = "http://localhost:11000/oozie" +default[:backup][:queue] = "default" diff --git a/cookbooks/backup/attributes/hdfs.rb b/cookbooks/backup/attributes/hdfs.rb new file mode 100644 index 000000000..2a722f5bf --- /dev/null +++ b/cookbooks/backup/attributes/hdfs.rb @@ -0,0 +1,43 @@ +# Cookbook Name:: backup +# HDFS Backup Attributes +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### hdfs backups +default[:backup][:hdfs][:user] = node[:backup][:user] +default[:backup][:hdfs][:root] = "#{node[:backup][:root]}/hdfs" +default[:backup][:hdfs][:local][:root] = "#{node[:backup][:local][:root]}/hdfs" + +# local oozie config dir +default[:backup][:hdfs][:local][:oozie] = + "#{node[:backup][:hdfs][:local][:root]}/oozie" + +## hdfs backup tuning parameters +# timeout in minutes before aborting distcp request +default[:backup][:hdfs][:timeout] = -1 + +# bandlimit in MB/s per mapper +default[:backup][:hdfs][:mapper][:bandwidth] = 25 + +### hdfs backup requests +default[:backup][:hdfs][:schedules] = {} + +## NOTE: refer to files/default/hdfs/jobs.yml for the proper data scheme. +# default[:backup][:hdfs][:schedules] = YAML.load_file(File.join( +# Chef::Config[:file_cache_path], +# 'cookbooks', +# 'backup', +# 'files/default/hdfs/jobs.yml' +# )) diff --git a/cookbooks/backup/files/default/hdfs/jobs.yml b/cookbooks/backup/files/default/hdfs/jobs.yml new file mode 100644 index 000000000..03dd520c1 --- /dev/null +++ b/cookbooks/backup/files/default/hdfs/jobs.yml @@ -0,0 +1,24 @@ +# hdfs_jobs.yaml +# YAML model of backup jobs. +# Disclaimer: This is an example. +# All keys should be preceded with a ':' s.t. they're read as ruby symbols. +# +# :group_name: +# :hdfs: valid hdfs source uri (string) +# :start: ISO datetime start (string) +# :end: ISO datetime end (string) +# :jobs: +# - :path: hdfs path to backup target (string) +# :hdfs: (OPTIONAL) override top-level hdfs uri (string) +# :period: period in minutes between backup actions (integer) +--- +:group: + :hdfs: 'hdfs://localhost:9000' + :start: '2018-01-01T12:00Z' + :end: '2019-12-25T06:00Z' + :jobs: + - :path: '/tmp' + :period: 360 + - :path: '/user' + :hdfs: 'hdfs://localhost:9000' # optional override + :period: 480 diff --git a/cookbooks/backup/libraries/oozie_client.rb b/cookbooks/backup/libraries/oozie_client.rb new file mode 100644 index 000000000..cdfa4bd43 --- /dev/null +++ b/cookbooks/backup/libraries/oozie_client.rb @@ -0,0 +1,101 @@ +# oozie_client.rb +# ruby client for managing oozie jobs +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Oozie + class ClientV1 + attr_accessor :oozie_url, :user + + def initialize(oozie_url='http://localhost:11000/oozie', user='oozie') + @oozie = oozie_url + @user = user + end + + # list the oozie jobs on the server. + def jobs(filter={}, jobtype='workflow', len=10) + execute('jobs', user, { + oozie: @oozie, + jobtype: jobtype, + filter: "\"#{filter_string(filter)}\"", + len: len + }) + end + + # kill the oozie jobs the match the filter. + def kill_jobs(filter={}, jobtype='workflow', len=1000) + execute('jobs', user, { + oozie: @oozie, + jobtype: jobtype, + filter: "\"#{filter_string(filter)}\"", + len: len, + kill: nil + }) + end + + # run an oozie job. + def run(config, user=@user) + execute('job', user, { + oozie: @oozie, + config: config, + run: nil + }) + end + + # rerun an oozie action. + def rerun(action_id, config, user=@user) + execute('job', user, { + oozie: @oozie, + config: config, + rerun: action_id + }) + end + + # kill an oozie job with the given id. + def kill(job_id, user=@user) + execute('job', user, { + oozie: @oozie, + kill: job_id + }) + end + + # get the id of an oozie job with the given name (if it exists). + def get_id(job_name, jobtype='workflow', status='RUNNING') + jobs_cmd = jobs({ name: job_name, status: status }, 'coordinator', 1) + match = jobs_cmd.stdout.match(/(\S+)\s+#{job_name}/) + return match.nil? ? nil : match[1] + end + + private ## private methods + + def execute(subcommand, user=@user, options={}) + require 'mixlib/shellout' + command = "oozie #{subcommand} #{options_string(options)}" + # puts command ## print debug command + return Mixlib::ShellOut.new(command, user: user, timeout: 90).run_command + end + + def options_string(options) + options.map { |key, value| "-#{key.to_s} #{value}" }.join(' ') + end + + def filter_string(filter) + filter.map { |key, value| "#{key.to_s}=#{value}" }.join(';') + end + end + + class Client < ClientV1 + end +end diff --git a/cookbooks/backup/metadata.rb b/cookbooks/backup/metadata.rb new file mode 100644 index 000000000..9c9995cc1 --- /dev/null +++ b/cookbooks/backup/metadata.rb @@ -0,0 +1,27 @@ +# +# Cookbook Name:: backup +# metadata.rb +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# encoding: utf-8 +name 'backup' +maintainer 'Bloomberg Finance L.P.' +maintainer_email 'hadoop@bloomberg.net' +description 'Cookbook to setup HDFS backups.' +license 'Apache 2.0' +long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) +version '0.1.0' diff --git a/cookbooks/backup/recipes/bootstrap.rb b/cookbooks/backup/recipes/bootstrap.rb new file mode 100644 index 000000000..ae1e21d35 --- /dev/null +++ b/cookbooks/backup/recipes/bootstrap.rb @@ -0,0 +1,70 @@ +# Cookbook Name:: backup +# Recipe:: bootstrap +# Creates the local backup bootstrap directory +# Generates the starter oozie configurations +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# create the local configuration root +# holds local copies of the oozie configurations +directory node[:backup][:local][:root] do + owner node[:backup][:user] + group node[:backup][:user] + mode "0755" + action :create +end + +node[:backup][:services].each do |service| + # create the service backup root (drwxr-xr-x) + directory node[:backup][service][:local][:root] do + owner node[:backup][:user] + group node[:backup][service][:user] + mode "0755" + action :create + end + + # create the oozie config directory (drwxr-xr-x) + directory node[:backup][service][:local][:oozie] do + owner node[:backup][:user] + group node[:backup][service][:user] + mode "0755" + action :create + end + + # oozie config files + oozie_config_dir = node[:backup][service][:local][:oozie] + oozie_configs = %w( + groups.properties + groups.xml + workflow.xml + coordinator.xml + ) + + # source configuration templates + oozie_configs.each do |config| + template "#{oozie_config_dir}/#{config}" do + source "#{service}/#{config}.erb" + owner node[:backup][:user] + group node[:backup][service][:user] + mode "0755" + action :create + variables( + service: service, + groups: node[:backup][service][:schedules].keys, + mode: '-rwxrwx---' + ) + end + end +end diff --git a/cookbooks/backup/recipes/cleanup.rb b/cookbooks/backup/recipes/cleanup.rb new file mode 100644 index 000000000..fc0451443 --- /dev/null +++ b/cookbooks/backup/recipes/cleanup.rb @@ -0,0 +1,77 @@ +# Cookbook Name::backup +# Recipe:: cleanup +# Cleans up the state of the storage cluster. +# Removes old configurations. Kills stale coordinators. +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# creates the properties files for the service's oozie jobs. +def get_job_names(service) + job_schedules = node[:backup][service][:schedules] + job_schedules.inject([]) do |job_names, job_schedule| + group, schedule = job_schedule + names = (schedule[:jobs] || []).map do |job| + "#{group}-#{job[:name] || File.basename(job[:path])}" + end + job_names.push(*names) + end +end + +# removes all local properties in #{path} not included in the #{filter} +# kills the stale oozie coordinators +def cleanup_service(filter, service, path) + # Get a list of stale jobs + # Checks the existing local properties files against current job set. + stale_jobs = Dir.glob("#{path}/*.properties").select do |entry| + File.file? entry + end.map do |filename| + /#{path}\/(.+).properties/.match(filename)[1] + end.select do |name| + !filter.include? name + end + + puts "stale jobs: #{stale_jobs}" + stale_jobs.each do |name| + # remove the local properties file + file "#{path}/#{name}.properties#delete" do + path "#{path}/#{name}.properties" + action :delete + end + + # remove the hdfs properties file + hdfs_file "#{path}/#{name}.properties#delete" do + hdfs node[:backup][:namenode] + path "#{node[:backup][service][:root]}/#{name}.properties" + admin node[:backup][:user] + action :delete + end + + # kill stale oozie coordinator + oozie_job "backup.#{service}.#{name}#kill" do + url node[:backup][:oozie] + name "backup.#{service}.#{name}" + user node[:backup][:user] + action :kill + ignore_failure true + end + end +end + +node[:backup][:services].each do |service| + oozie_config_dir = node[:backup][service][:local][:oozie] + jobnames = get_job_names(service) + jobnames << "groups" + cleanup_service(jobnames, service, oozie_config_dir) +end diff --git a/cookbooks/backup/recipes/default.rb b/cookbooks/backup/recipes/default.rb new file mode 100644 index 000000000..195c28f0a --- /dev/null +++ b/cookbooks/backup/recipes/default.rb @@ -0,0 +1,19 @@ +# Cookbook Name::backup +# Recipe:: default +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print the jobs list hash +p node[:backup][:hdfs][:schedules] diff --git a/cookbooks/backup/recipes/hdfs.rb b/cookbooks/backup/recipes/hdfs.rb new file mode 100644 index 000000000..0c0121536 --- /dev/null +++ b/cookbooks/backup/recipes/hdfs.rb @@ -0,0 +1,37 @@ +# Cookbook Name:: backup +# Recipe:: hdfs +# Uploads the bootstrap directory to HDFS +# Launches the group directory creation workflow +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# upload the bootstrap directory to HDFS +hdfs_directory node[:backup][:root] do + hdfs node[:backup][:namenode] + source node[:backup][:local][:root] + path File.dirname("#{node[:backup][:root]}") + action :put +end + +# launch the group dir creation workflow +node[:backup][:services].each do |service| + oozie_config_dir = node[:backup][service][:local][:oozie] + oozie_job "backup.groups.#{service}" do + url node[:backup][:oozie] + config "#{oozie_config_dir}/groups.properties" + user node[:backup][:user] + action :run + end +end diff --git a/cookbooks/backup/recipes/properties.rb b/cookbooks/backup/recipes/properties.rb new file mode 100644 index 000000000..0942ced20 --- /dev/null +++ b/cookbooks/backup/recipes/properties.rb @@ -0,0 +1,73 @@ +# Cookbook Name::backup +# Recipe:: properties +# Creates the local oozie job.properties files +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# parses the job properties from an hdfs backup job +def parse_hdfs_properties(group, schedule, job) + # override schedule parameters + name = job[:name] || File.basename(job[:path]) + hdfs_src = job[:hdfs] || schedule[:hdfs] + period = job[:period] || schedule[:period] + bandwidth = node[:backup][:hdfs][:mapper][:bandwidth] || + node[:backup][:mapper][:bandwidth] + queue = node[:backup][:hdfs][:queue] || + node[:backup][:queue] + + return { + group: group, + path: job[:path], + basename: File.basename(job[:path]), + jobname: "#{group}-#{name}", + hdfs: hdfs_src, + period: period, + startdate: schedule[:start], + enddate: schedule[:end], + timeout: node[:backup][:hdfs][:timeout], + bandwidth: bandwidth, + queue: queue + } +end + +def parse_service_properties(service, group, schedule, job) + case service.to_sym + when :hdfs + parse_hdfs_properties(group, schedule, job) + else + nil # service not found + end +end + +# parse job schedules and create properties files +node[:backup][:services].map do |service| + node[:backup][service][:schedules].each do |group, schedule| + schedule[:jobs].each do |job| + + # oozie job.properties + oozie_config_dir = node[:backup][service][:local][:oozie] + job_props = parse_service_properties(service, group, schedule, job) + template "#{oozie_config_dir}/#{job_props[:jobname]}.properties" do + source "#{service}/backup.properties.erb" + owner node[:backup][:user] + group node[:backup][service][:user] + mode "0755" + action :create + variables job_props + end + + end + end +end diff --git a/cookbooks/backup/recipes/scheduler.rb b/cookbooks/backup/recipes/scheduler.rb new file mode 100644 index 000000000..1b8e54345 --- /dev/null +++ b/cookbooks/backup/recipes/scheduler.rb @@ -0,0 +1,43 @@ +# Cookbook Name:: backup +# Recipe:: scheduler +# Launches the oozie coordinators to schedule periodic backups +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run each oozie coordinator tracked by the backup service. +# Only runs the coordinator if it is not already RUNNING +node[:backup][:services].each do |service| + node[:backup][service][:schedules].each do |group, schedule| + schedule[:jobs].each do |job| + + # scheduler parameters + name = job[:name] ? job[:name] : File.basename(job[:path]) + jobname = "#{group}-#{name}" + oozie_config_dir = node[:backup][service][:local][:oozie] + properties_file = "#{oozie_config_dir}/#{jobname}.properties" + coordinator_file = "#{oozie_config_dir}/coordinator.xml" + + # restart oozie coordinators + oozie_job "backup.#{service}.#{jobname}" do + url node[:backup][:oozie] + config properties_file + user node[:backup][service][:user] + action :run + ignore_failure true + end + + end + end +end diff --git a/cookbooks/backup/resources/hdfs_directory.rb b/cookbooks/backup/resources/hdfs_directory.rb new file mode 100644 index 000000000..bd2bdda31 --- /dev/null +++ b/cookbooks/backup/resources/hdfs_directory.rb @@ -0,0 +1,68 @@ +# Cookbook Name:: backup +# Custom hdfs directory resource +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource_name :hdfs_directory + +property :hdfs, String, required: true +property :path, String, name_property: true +property :admin, String, default: 'hdfs' +property :source, String +property :owner, String +property :group, String +property :mode, String + +action_class do + def execute(command, user=admin, timeout=90) + require 'mixlib/shellout' + Chef::Log.info("Running command(#{user}): #{command}") + return Mixlib::ShellOut.new("sudo -u #{user} " + command, timeout: timeout).run_command + end +end + +action :create do + # create the directory + Chef::Log.info("HDFS dir #{path} creation") + execute("hdfs dfs -mkdir -p #{hdfs}/#{path}", admin).error! + + # set the owner and group + if !(owner.nil? || group.nil?) + execute("hdfs dfs -chown #{owner}:#{group} #{hdfs}/#{path}", admin).error! + elsif !owner.nil? + execute("hdfs dfs -chown #{owner} #{hdfs}/#{path}", admin).error! + elsif !group.nil? + execute("hdfs dfs -chgrp #{group} #{hdfs}/#{path}", admin).error! + end + + # set permissions + if !mode.nil? + execute("hdfs dfs -chmod #{mode} #{hdfs}/#{path}", admin).error! + end +end + +action :put do + Chef::Log.info("Copying #{path} to HDFS") + execute("hdfs dfs -put -f -p #{source} #{hdfs}/#{path}", admin).run_command.error! +end + +action :delete do + # speculative, recursive delete. (ignores error) + Chef::Log.info("HDFS dir #{path} deletion") + execute("hdfs dfs -rm -r -f #{hdfs}/#{path}", admin) +end + +action :nothing do +end diff --git a/cookbooks/backup/resources/hdfs_file.rb b/cookbooks/backup/resources/hdfs_file.rb new file mode 100644 index 000000000..7a7911ce6 --- /dev/null +++ b/cookbooks/backup/resources/hdfs_file.rb @@ -0,0 +1,73 @@ +# Cookbook Name:: backup +# Custom hdfs file resource +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource_name :hdfs_file + +property :hdfs, String, required: true +property :path, String, required: true +property :admin, String, default: 'hdfs' +property :source, String +property :owner, String +property :group, String +property :mode, String + +action_class do + def execute(command, user=admin, timeout=90) + require 'mixlib/shellout' + Chef::Log.info("Running command(#{user}): #{command}") + puts command + return Mixlib::ShellOut.new("sudo -u #{user} " + command, timeout: timeout).run_command + end +end + +action :create do + # create the file + Chef::Log.info("HDFS file #{path} creation") + if source.nil? + execute("hdfs dfs -touchz #{hdfs}/#{path}", admin).error! + else + execute("hdfs dfs -put -p -f #{source} #{hdfs}/#{path}", admin).error! + end + + # set the owner and group + if !(owner.nil? || group.nil?) + execute("hdfs dfs -chown #{owner}:#{group} #{hdfs}/#{path}", admin).error! + elsif !owner.nil? + execute("hdfs dfs -chown #{owner} #{hdfs}/#{path}", admin).error! + elsif !group.nil? + execute("hdfs dfs -chgrp #{group} #{hdfs}/#{path}", admin).error! + end + + # set permissions + if !mode.nil? + execute("hdfs dfs -chmod #{mode} #{hdfs}/#{path}", admin).error! + end +end + +action :put do + Chef::Log.info("Copying #{path} to HDFS") + execute("hdfs dfs -put -f -p #{source} #{hdfs}/#{path}", admin).run_command.error! +end + +action :delete do + # speculative, recursive delete. (ignores error) + Chef::Log.info("HDFS file #{path} deletion") + execute("hdfs dfs -rm -r -f #{hdfs}/#{path}", admin) +end + +action :nothing do +end diff --git a/cookbooks/backup/resources/oozie_job.rb b/cookbooks/backup/resources/oozie_job.rb new file mode 100644 index 000000000..1f5d8afc1 --- /dev/null +++ b/cookbooks/backup/resources/oozie_job.rb @@ -0,0 +1,74 @@ +# Cookbook Name:: backup +# Custom oozie job resource +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource_name :oozie_job +provides :oozie_job + +property :name, String, name_property: true +property :url, String, required: true +property :config, String, required: true +property :user, String, default: 'oozie' + +action_class do + include Oozie +end + +action :run do + Chef::Log.info("Starting oozie job: #{name}") + client = Oozie::Client.new(url, user) + + # check if the job is already running + job_id = client.get_id(name, 'coordinator', 'RUNNING') + + # start the service + if job_id.nil? + run_cmd = client.run(config, user) + run_cmd.error! + end +end + +action :restart do + Chef::Log.info("Rerunning oozie job: #{name}") + client = Oozie::Client.new(url, user) + + # check if the job is already running + job_id = client.get_id(name, 'coordinator', 'RUNNING') + + if job_id.nil? + # start the service + run_cmd = client.run(config, user) + run_cmd.error! + else + # kill and restart the service + kill_cmd = client.kill(job_id, user) + kill_cmd.error! + rerun_cmd = client.run(config, user) + rerun_cmd.error! + end +end + +action :kill do + Chef::Log.info("Killing oozie job: #{name}") + client = Oozie::Client.new(url, user) + + # kill the service (if it exists) + jobs_cmd = client.kill_jobs({ name: name }, "coordinator") + jobs_cmd.error! +end + +action :nothing do +end diff --git a/cookbooks/backup/templates/default/hdfs/backup.properties.erb b/cookbooks/backup/templates/default/hdfs/backup.properties.erb new file mode 100644 index 000000000..6a6fdf317 --- /dev/null +++ b/cookbooks/backup/templates/default/hdfs/backup.properties.erb @@ -0,0 +1,66 @@ +# +# Cookbook Name:: backup +# hdfs.properties +# Properties file for an oozie backup job +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# group name [A-Za-z_]+ +group=<%= @group %> + +# job name [A-Za-z_]+ +jobname=<%= @jobname %> + +# backup file / dir +target=<%= @path %> +basename=<%= @basename %> + +# backup period in minutes +period=<%= @period %> + +# backup timeout in minutes (-1 = no timeout) +timeout=<%= @timeout %> + +# bandwidth per distcp mapper in MB/s +bandwidth=<%= @bandwidth %> + +# yarn queue +queue=<%= @queue %> + +# source cluster +hdfs.src.namenode=<%= @hdfs %> + +# storage cluster +hdfs.dest.namenode=<%= node[:backup][:namenode] %> +hdfs.backup.root=${hdfs.dest.namenode}/<%= node[:backup][:hdfs][:root].sub("/", "") %> + +# group properties +group.backup.coordinator=${hdfs.backup.root}/oozie/coordinator.xml +group.backup.workflow=${hdfs.backup.root}/oozie/workflow.xml + +# backup schedule (ISO date) +group.backup.start=<%= @startdate %> +group.backup.end=<%= @enddate %> + +# oozie properties +oozie.use.system.libpath=true +# oozie.wf.application.path=${group.backup.workflow} +oozie.coord.application.path=${group.backup.coordinator} +oozie.launcher.mapreduce.job.hdfs-servers=${hdfs.src.namenode},${hdfs.dest.namenode} + +# hadoop properties +namenode=<%= node[:backup][:namenode] %> +jobtracker=<%= node[:backup][:jobtracker] %> diff --git a/cookbooks/backup/templates/default/hdfs/coordinator.xml.erb b/cookbooks/backup/templates/default/hdfs/coordinator.xml.erb new file mode 100644 index 000000000..f193491d8 --- /dev/null +++ b/cookbooks/backup/templates/default/hdfs/coordinator.xml.erb @@ -0,0 +1,39 @@ + + + + + + ${timeout} + 1 + LAST_ONLY + + + + + ${coord:conf('group.backup.workflow')} + + + + diff --git a/cookbooks/backup/templates/default/hdfs/groups.properties.erb b/cookbooks/backup/templates/default/hdfs/groups.properties.erb new file mode 100644 index 000000000..705a09f38 --- /dev/null +++ b/cookbooks/backup/templates/default/hdfs/groups.properties.erb @@ -0,0 +1,31 @@ +# +# Cookbook Name:: backup +# groups.properties +# Properties file for an oozie backup job +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# storage cluster +hdfs.dest.namenode=<%= node[:backup][:namenode] %> +hdfs.backup.root=${hdfs.dest.namenode}/<%= node[:backup][:hdfs][:root].sub("/", "") %> + +# oozie properties +oozie.use.system.libpath=true +oozie.wf.application.path=${hdfs.backup.root}/oozie/groups.xml + +# hadoop properties +namenode=<%= node[:backup][:namenode] %> +jobtracker=<%= node[:backup][:jobtracker] %> diff --git a/cookbooks/backup/templates/default/hdfs/groups.xml.erb b/cookbooks/backup/templates/default/hdfs/groups.xml.erb new file mode 100644 index 000000000..3286d05c3 --- /dev/null +++ b/cookbooks/backup/templates/default/hdfs/groups.xml.erb @@ -0,0 +1,46 @@ + + + + + + + + + + + <% @groups.each do |group| %> + + + + <%- end %> + + + + + + + [${wf:errorMessage(wf:lastErrorNode())}] + + + + diff --git a/cookbooks/backup/templates/default/hdfs/workflow.xml.erb b/cookbooks/backup/templates/default/hdfs/workflow.xml.erb new file mode 100644 index 000000000..f6e8bcc34 --- /dev/null +++ b/cookbooks/backup/templates/default/hdfs/workflow.xml.erb @@ -0,0 +1,53 @@ + + + + + + + + + + ${jobtracker} + ${namenode} + + + -Dmapred.job.queue.name=${queue} + + + -ppugbtxc + -update + -bandwidth + ${bandwidth} + -i + + + ${wf:conf("hdfs.src.namenode")}/${target} + ${wf:conf("hdfs.backup.root")}/${group}/${basename} + + + + + + + [${wf:errorMessage(wf:lastErrorNode())}] + + + + diff --git a/cookbooks/bcpc-hadoop/attributes/backup.rb b/cookbooks/bcpc-hadoop/attributes/backup.rb new file mode 100644 index 000000000..70cbbe4ea --- /dev/null +++ b/cookbooks/bcpc-hadoop/attributes/backup.rb @@ -0,0 +1,18 @@ +# Cookbook Name:: bcpc-hadoop +# BCPC Hadoop Attributes for backup jobs +# +# Copyright 2018, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +default[:bcpc][:hadoop][:backup][:user] = "bach_backup" diff --git a/cookbooks/bcpc-hadoop/attributes/kerberos.rb b/cookbooks/bcpc-hadoop/attributes/kerberos.rb index 79fa2a737..3b4577da7 100644 --- a/cookbooks/bcpc-hadoop/attributes/kerberos.rb +++ b/cookbooks/bcpc-hadoop/attributes/kerberos.rb @@ -128,6 +128,15 @@ perms: '0440', spnego_keytab: 'spnego.service.keytab' }, + backup: { + principal: node[:bcpc][:hadoop][:backup][:user], + keytab: 'backup.service.keytab', + owner: node[:bcpc][:hadoop][:backup][:user], + group: node[:bcpc][:hadoop][:backup][:user], + princhost: '_HOST', + perms: '0440', + spnego_keytab: 'spnego.service.keytab' + }, ambari: { principal: "#{node['bcpc']['hadoop']['proxyuser']['ambari']}", keytab: 'ambari.service.keytab', diff --git a/cookbooks/bcpc-hadoop/recipes/bach_backup_user.rb b/cookbooks/bcpc-hadoop/recipes/bach_backup_user.rb new file mode 100644 index 000000000..f5ba0e4f0 --- /dev/null +++ b/cookbooks/bcpc-hadoop/recipes/bach_backup_user.rb @@ -0,0 +1,34 @@ +# Cookbook Name:: bcpc-hadoop +# Recipe:: bach_backup_user +# Creates the user needed for hadoop cluster backup jobs +# +# Copyright 2016, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +backup_user = node[:bcpc][:hadoop][:backup][:user] + +user backup_user do + action :create + comment 'backup service user' +end + +# make backup user an hdfs superuser +group 'hdfs' do + members backup_user + append true +end + +configure_kerberos 'backup_kerberos' do + service_name 'backup' +end diff --git a/cookbooks/bcpc-hadoop/recipes/bach_backup_wrapper.rb b/cookbooks/bcpc-hadoop/recipes/bach_backup_wrapper.rb new file mode 100644 index 000000000..7fc33c0c9 --- /dev/null +++ b/cookbooks/bcpc-hadoop/recipes/bach_backup_wrapper.rb @@ -0,0 +1,19 @@ +# Cookbook Name:: bcpc-hadoop +# Recipe:: bach_backup_wrapper +# Wrapper recipe for bach_backup hadoop cluster support +# +# Copyright 2016, Bloomberg Finance L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_recipe 'bcpc-hadoop::bach_backup_user' diff --git a/stub-environment/roles/BACH-Backup.json b/stub-environment/roles/BACH-Backup.json new file mode 100644 index 000000000..7bff90289 --- /dev/null +++ b/stub-environment/roles/BACH-Backup.json @@ -0,0 +1,19 @@ +{ + "name": "BACH-Backup", + "description": "Configures the backup service on a backup cluster leader.", + "chef_type": "role", + "json_class": "Chef::Role", + "run_list": [ + "recipe[bach_backup::default]", + "recipe[backup::default]", + "recipe[backup::bootstrap]", + "recipe[backup::properties]", + "recipe[backup::hdfs]", + "recipe[backup::scheduler]", + "recipe[backup::cleanup]" + ], + "default_attributes": { + }, + "override_attributes": { + } +} diff --git a/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json b/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json index 795c32430..0f837baec 100644 --- a/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json +++ b/stub-environment/roles/BCPC-Hadoop-Head-ResourceManager.json @@ -4,7 +4,8 @@ "run_list": [ "role[Basic]", "recipe[bcpc-hadoop::resource_manager]", - "recipe[bcpc-hadoop::smoke_test_user]" + "recipe[bcpc-hadoop::smoke_test_user]", + "recipe[bcpc-hadoop::bach_backup_user]" ], "description": "A highly-available head node in a BCPC Hadoop cluster", "chef_type": "role", diff --git a/stub-environment/roles/BCPC-Hadoop-Head.json b/stub-environment/roles/BCPC-Hadoop-Head.json index 0707e5cc1..a29607fe6 100644 --- a/stub-environment/roles/BCPC-Hadoop-Head.json +++ b/stub-environment/roles/BCPC-Hadoop-Head.json @@ -23,6 +23,7 @@ "recipe[hdfsdu::create_user]", "recipe[bcpc-hadoop::configs]", "recipe[pam::default]", + "recipe[bcpc-hadoop::bach_backup_wrapper]", "recipe[bcpc-hadoop::zookeeper_server]", "recipe[bcpc-hadoop::journalnode]", "recipe[bcpc-hadoop::graphite_to_zabbix]" diff --git a/stub-environment/roles/BCPC-Hadoop-Worker.json b/stub-environment/roles/BCPC-Hadoop-Worker.json index df4c2e905..cc1200f8b 100644 --- a/stub-environment/roles/BCPC-Hadoop-Worker.json +++ b/stub-environment/roles/BCPC-Hadoop-Worker.json @@ -12,6 +12,7 @@ "recipe[bcpc-hadoop::hdp_repo]", "recipe[bach_krb5::krb5_client]", "recipe[hdfsdu::create_user]", + "recipe[bcpc-hadoop::bach_backup_wrapper]", "recipe[bcpc-hadoop::configs]", "recipe[pam::default]", "recipe[bach_spark::default]",