From d672287bd648ec6a972c753ff0c02f24ecaf4319 Mon Sep 17 00:00:00 2001 From: Oleksiy Molchanov Date: Tue, 27 Oct 2015 13:26:51 +0200 Subject: [PATCH] Ceph mon and osd - add possibility to use puppet-librarian-simple in library - add fuel main repo - add disk configuration for osd - add osds - add mons - change node names to solar-dev --- examples/library_ceph/README.md | 8 ++-- examples/library_ceph/ceph.py | 61 ++++++++++++++++++++++---- resources/ceph_disk/README.md | 5 +++ resources/ceph_disk/actions/run.yaml | 19 ++++++++ resources/ceph_disk/meta.yaml | 7 +++ resources/ceph_keys/README.md | 9 ++++ resources/ceph_keys/actions/run.sh | 3 ++ resources/ceph_mon/README.md | 42 ++++++++++++++++++ resources/ceph_mon/actions/run.pp | 8 ++-- resources/ceph_mon/meta.yaml | 3 ++ resources/ceph_osd/README.md | 42 ++++++++++++++++++ resources/ceph_osd/actions/run.pp | 59 +++++++++++++++++++++++++ resources/ceph_osd/meta.yaml | 41 +++++++++++++++++ resources/fuel_library/README.md | 5 +++ resources/fuel_library/actions/run.sh | 7 +++ resources/fuel_library/meta.yaml | 3 ++ resources/managed_apt/actions/run.yaml | 2 +- resources/managed_apt/meta.yaml | 4 ++ resources/remote_file/README.md | 3 ++ resources/remote_file/actions/run.sh | 2 +- templates/mos_repos.yaml | 15 ++++++- templates/nodes.yaml | 6 +-- 22 files changed, 331 insertions(+), 23 deletions(-) create mode 100644 resources/ceph_disk/README.md create mode 100644 resources/ceph_disk/actions/run.yaml create mode 100644 resources/ceph_disk/meta.yaml create mode 100644 resources/ceph_keys/README.md create mode 100644 resources/ceph_mon/README.md create mode 100644 resources/ceph_osd/README.md create mode 100644 resources/ceph_osd/actions/run.pp create mode 100644 resources/ceph_osd/meta.yaml create mode 100644 resources/fuel_library/README.md create mode 100644 resources/remote_file/README.md diff --git a/examples/library_ceph/README.md b/examples/library_ceph/README.md index 1e19af44..b0ba9d62 100644 --- a/examples/library_ceph/README.md +++ b/examples/library_ceph/README.md @@ -3,15 +3,15 @@ Current example will do following things: - fetch fuel-library from github - use ./update_modules.sh to fetch librarian dependencies - generate ceph keys on a solar-dev1 -- install ceph-mon on solar-dev1 (INPROGRESS) -- install ceph-osd on solar-dev2 (TODO) +- install ceph-mon on solar-dev1 +- install ceph-osd on solar-dev2 - imlement removal mechanism for ceph-mon/ceph-osd (TODO) - +- configure 10GB hdd for work with ceph-osd (disk should be added manually) To use it: ``` -python exaples/library_ceph/ceph.py +python examples/library_ceph/ceph.py solar ch stage && solar ch process solar or run-once last -w 120 ``` diff --git a/examples/library_ceph/ceph.py b/examples/library_ceph/ceph.py index 0e0f7361..78a99ea1 100644 --- a/examples/library_ceph/ceph.py +++ b/examples/library_ceph/ceph.py @@ -1,4 +1,4 @@ - +from solar import events as evapi from solar.core.resource import virtual_resource as vr from solar.interfaces.db import get_db @@ -36,21 +36,37 @@ def deploy(): db.clear() resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2}) - first_node, second_node = [x for x in resources if x.name.startswith('node')] + first_node, second_node = [x for x in resources if x.name.startswith('solar-dev')] first_transp = next(x for x in resources if x.name.startswith('transport')) + + host1, host2 = [x for x in resources if x.name.startswith('hosts_file')] - library = vr.create('library1', 'resources/fuel_library', {})[0] - first_node.connect(library) + library1 = vr.create('library1', 'resources/fuel_library', {})[0] + library2 = vr.create('library2', 'resources/fuel_library', + {'temp_directory': '/tmp/solar', + 'puppet_modules': '/etc/fuel/modules', + 'git':{'branch': 'master', 'repository': 'https://github.com/stackforge/fuel-library'}, + 'librarian_puppet_simple': 'true'})[0] keys = vr.create('ceph_key', 'resources/ceph_keys', {})[0] first_node.connect(keys) + remote_file = vr.create('ceph_key2', 'resources/remote_file', + {'dest': '/var/lib/astute/'})[0] + second_node.connect(remote_file) + keys.connect(remote_file, {'ip': 'remote_ip', 'path': 'remote_path'}) + first_transp.connect(remote_file, {'transports': 'remote'}) + remote_file = vr.create('ceph_key2', 'resources/remote_file', {'dest': '/var/lib/astute/'})[0] second_node.connect(remote_file) keys.connect(remote_file, {'ip': 'remote_ip', 'path': 'remote_path'}) first_transp.connect(remote_file, {'transports': 'remote'}) + ceph_disk = vr.create('ceph_disk1', 'resources/ceph_disk', + {'disk_name': '/dev/vdb'})[0] + + second_node.connect(ceph_disk, {}) ceph_mon = vr.create('ceph_mon1', 'resources/ceph_mon', {'storage': STORAGE, @@ -58,18 +74,45 @@ def deploy(): 'network_scheme': NETWORK_SCHEMA, 'ceph_monitor_nodes': NETWORK_METADATA, 'ceph_primary_monitor_node': NETWORK_METADATA, - 'role': 'controller', + 'role': 'primary-controller', + })[0] + + ceph_osd = vr.create('ceph_osd2', 'resources/ceph_osd', + {'storage': STORAGE, + 'keystone': KEYSTONE, + 'network_scheme': NETWORK_SCHEMA, + 'ceph_monitor_nodes': NETWORK_METADATA, + 'ceph_primary_monitor_node': NETWORK_METADATA, + 'role': 'ceph-osd', })[0] - managed_apt = vr.create( + managed_apt1 = vr.create( 'managed_apt1', 'templates/mos_repos.yaml', {'node': first_node.name, 'index': 0})[-1] - keys.connect(ceph_mon, {}) + managed_apt2 = vr.create( + 'managed_apt2', 'templates/mos_repos.yaml', + {'node': second_node.name, 'index': 1})[-1] + + first_node.connect(library1, {}) + second_node.connect(library2, {}) + first_node.connect(ceph_mon, {'ip': ['ip', 'public_vip', 'management_vip']}) - library.connect(ceph_mon, {'puppet_modules': 'puppet_modules'}) - managed_apt.connect(ceph_mon, {}) + second_node.connect(ceph_osd, + {'ip': ['ip', 'public_vip', 'management_vip']}) + library1.connect(ceph_mon, {'puppet_modules': 'puppet_modules'}) + library2.connect(ceph_osd, {'puppet_modules': 'puppet_modules'}) + + evapi.add_dep(second_node.name, ceph_osd.name, actions=('run',)) + evapi.add_dep(first_node.name, ceph_mon.name, actions=('run',)) + evapi.add_dep(keys.name, ceph_mon.name, actions=('run',)) + evapi.add_dep(remote_file.name, ceph_osd.name, actions=('run',)) + evapi.add_dep(managed_apt1.name, ceph_mon.name, actions=('run',)) + evapi.add_dep(managed_apt2.name, ceph_osd.name, actions=('run',)) + evapi.add_dep(ceph_mon.name, ceph_osd.name, actions=('run',)) + evapi.add_dep(ceph_disk.name, ceph_osd.name, actions=('run',)) + if __name__ == '__main__': deploy() diff --git a/resources/ceph_disk/README.md b/resources/ceph_disk/README.md new file mode 100644 index 00000000..91d2569a --- /dev/null +++ b/resources/ceph_disk/README.md @@ -0,0 +1,5 @@ +# Prepare disk for ceph osd deployment + +Takes only one parameter as disk name and configure it for osd deploy: +- make label +- set guid diff --git a/resources/ceph_disk/actions/run.yaml b/resources/ceph_disk/actions/run.yaml new file mode 100644 index 00000000..11c96f0c --- /dev/null +++ b/resources/ceph_disk/actions/run.yaml @@ -0,0 +1,19 @@ +- hosts: [{{ host }}] + sudo: yes + tasks: + - name: check if disk has gpt + shell: "parted {{disk_name}} print | grep 'Partition Table: gpt'" + register: gpt_created + ignore_errors: True + - name: set gpt label + shell: "echo yes | parted {{disk_name}} mklabel gpt" + ignore_errors: True + when: gpt_created|failed + - name: check if guid set + shell: "sgdisk -i1 {{disk_name}} | grep 'Partition GUID code: 4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D'" + register: guid_set + ignore_errors: True + - name: set guid + shell: "sgdisk -n 1:0:9G -t 1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -p {{disk_name}}" + ignore_errors: True + when: guid_set|failed diff --git a/resources/ceph_disk/meta.yaml b/resources/ceph_disk/meta.yaml new file mode 100644 index 00000000..92057f1b --- /dev/null +++ b/resources/ceph_disk/meta.yaml @@ -0,0 +1,7 @@ +id: ceph_disk +handler: ansible +version: 1.0.0 +input: + disk_name: + schema: str + value: '/dev/vdb' diff --git a/resources/ceph_keys/README.md b/resources/ceph_keys/README.md new file mode 100644 index 00000000..23ea406b --- /dev/null +++ b/resources/ceph_keys/README.md @@ -0,0 +1,9 @@ +# Prepare ceph keys + +This resource prepare ceph keys for future ceph cluster deployment. + +Takes 3 parameters: +- target_directory, default - '/var/lib/astute/' +- key_name, default - 'ceph' +- path, default - '/var/lib/astute/ceph/' + diff --git a/resources/ceph_keys/actions/run.sh b/resources/ceph_keys/actions/run.sh index 7fa849cd..b35f471a 100644 --- a/resources/ceph_keys/actions/run.sh +++ b/resources/ceph_keys/actions/run.sh @@ -12,6 +12,9 @@ function generate_ssh_keys { else echo 'Key $key_path already exists' fi + +### FIXME: Dirty hack to allow scp under vagrant user ### +chmod +r $key_path } generate_ssh_keys diff --git a/resources/ceph_mon/README.md b/resources/ceph_mon/README.md new file mode 100644 index 00000000..f92fc685 --- /dev/null +++ b/resources/ceph_mon/README.md @@ -0,0 +1,42 @@ +# Deploy ceph mon + +This resource deploy ceph mon. + +Example: + +``` +STORAGE = {'objects_ceph': True, + 'osd_pool_size': 2, + 'pg_num': 128} + +KEYSTONE = {'admin_token': 'abcde'} + + +NETWORK_SCHEMA = { + 'endpoints': {'eth1': {'IP': ['10.0.0.3/24']}}, + 'roles': {'ceph/replication': 'eth1', + 'ceph/public': 'eth1'} + } + +NETWORK_METADATA = yaml.load(""" + solar-dev1: + uid: '1' + fqdn: solar-dev1 + network_roles: + ceph/public: 10.0.0.3 + ceph/replication: 10.0.0.3 + node_roles: + - ceph-mon + name: solar-dev1 + + """) + + ceph_mon = vr.create('ceph_mon1', 'resources/ceph_mon', + {'storage': STORAGE, + 'keystone': KEYSTONE, + 'network_scheme': NETWORK_SCHEMA, + 'ceph_monitor_nodes': NETWORK_METADATA, + 'ceph_primary_monitor_node': NETWORK_METADATA, + 'role': 'primary-controller', + })[0] +``` diff --git a/resources/ceph_mon/actions/run.pp b/resources/ceph_mon/actions/run.pp index 6b172a81..155eb869 100644 --- a/resources/ceph_mon/actions/run.pp +++ b/resources/ceph_mon/actions/run.pp @@ -1,6 +1,5 @@ notice('MODULAR: ceph/mon.pp') - $storage_hash = hiera('storage', {}) $public_vip = hiera('public_vip') $management_vip = hiera('management_vip') @@ -27,6 +26,8 @@ $use_ceph = false } +class {'firewall': } + if $use_ceph { $ceph_primary_monitor_node = hiera('ceph_primary_monitor_node') $primary_mons = keys($ceph_primary_monitor_node) @@ -76,7 +77,7 @@ hasrestart => true, } - Class['ceph'] ~> Service['cinder-volume'] + Class['firewall'] -> Class['ceph'] ~> Service['cinder-volume'] Class['ceph'] ~> Service['cinder-backup'] } @@ -89,7 +90,6 @@ hasrestart => true, } - Class['ceph'] ~> Service['glance-api'] + Class['firewall'] -> Class['ceph'] ~> Service['glance-api'] } - } diff --git a/resources/ceph_mon/meta.yaml b/resources/ceph_mon/meta.yaml index ec92d1a8..f6260540 100644 --- a/resources/ceph_mon/meta.yaml +++ b/resources/ceph_mon/meta.yaml @@ -1,6 +1,9 @@ id: ceph_mon handler: puppetv2 version: 1.0.0 +actions: + run: run.pp + update: run.pp input: ip: schema: str! diff --git a/resources/ceph_osd/README.md b/resources/ceph_osd/README.md new file mode 100644 index 00000000..00540105 --- /dev/null +++ b/resources/ceph_osd/README.md @@ -0,0 +1,42 @@ +# Deploy ceph osd + +This resource deploy ceph osd with preprepaired disk drives. + +Example: + +``` +STORAGE = {'objects_ceph': True, + 'osd_pool_size': 2, + 'pg_num': 128} + +KEYSTONE = {'admin_token': 'abcde'} + + +NETWORK_SCHEMA = { + 'endpoints': {'eth1': {'IP': ['10.0.0.3/24']}}, + 'roles': {'ceph/replication': 'eth1', + 'ceph/public': 'eth1'} + } + +NETWORK_METADATA = yaml.load(""" + solar-dev1: + uid: '1' + fqdn: solar-dev1 + network_roles: + ceph/public: 10.0.0.3 + ceph/replication: 10.0.0.3 + node_roles: + - ceph-mon + name: solar-dev1 + + """) + + ceph_osd = vr.create('ceph_osd2', 'resources/ceph_osd', + {'storage': STORAGE, + 'keystone': KEYSTONE, + 'network_scheme': NETWORK_SCHEMA, + 'ceph_monitor_nodes': NETWORK_METADATA, + 'ceph_primary_monitor_node': NETWORK_METADATA, + 'role': 'ceph-osd', + })[0] +``` diff --git a/resources/ceph_osd/actions/run.pp b/resources/ceph_osd/actions/run.pp new file mode 100644 index 00000000..5cdac8ee --- /dev/null +++ b/resources/ceph_osd/actions/run.pp @@ -0,0 +1,59 @@ +notice('MODULAR: ceph-osd.pp') + +# Pulling hiera +$storage_hash = hiera('storage', {}) +$public_vip = hiera('public_vip') +$management_vip = hiera('management_vip') +$use_neutron = hiera('use_neutron', false) +#$mp_hash = hiera('mp') +$verbose = pick($storage_hash['verbose'], true) +$debug = pick($storage_hash['debug'], hiera('debug', true)) +$use_monit = false +$auto_assign_floating_ip = hiera('auto_assign_floating_ip', false) +$keystone_hash = hiera('keystone', {}) +$access_hash = hiera('access', {}) +$network_scheme = hiera_hash('network_scheme') +$neutron_mellanox = hiera('neutron_mellanox', false) +$syslog_hash = hiera('syslog', {}) +$use_syslog = hiera('use_syslog', true) +$mon_address_map = get_node_to_ipaddr_map_by_network_role(hiera_hash('ceph_monitor_nodes'), 'ceph/public') +$ceph_primary_monitor_node = hiera('ceph_primary_monitor_node') +$primary_mons = keys($ceph_primary_monitor_node) +$primary_mon = $ceph_primary_monitor_node[$primary_mons[0]]['name'] +prepare_network_config($network_scheme) +$ceph_cluster_network = get_network_role_property('ceph/replication', 'network') +$ceph_public_network = get_network_role_property('ceph/public', 'network') + +class {'firewall': } -> + +class {'ceph': + primary_mon => $primary_mon, + mon_hosts => keys($mon_address_map), + mon_ip_addresses => values($mon_address_map), + cluster_node_address => $public_vip, + osd_pool_default_size => $storage_hash['osd_pool_size'], + osd_pool_default_pg_num => $storage_hash['pg_num'], + osd_pool_default_pgp_num => $storage_hash['pg_num'], + use_rgw => $storage_hash['objects_ceph'], + glance_backend => $glance_backend, + rgw_pub_ip => $public_vip, + rgw_adm_ip => $management_vip, + rgw_int_ip => $management_vip, + cluster_network => $ceph_cluster_network, + public_network => $ceph_public_network, + use_syslog => $use_syslog, + syslog_log_level => hiera('syslog_log_level_ceph', 'info'), + syslog_log_facility => hiera('syslog_log_facility_ceph','LOG_LOCAL0'), + rgw_keystone_admin_token => $keystone_hash['admin_token'], + ephemeral_ceph => $storage_hash['ephemeral_ceph'], +} + +$osd_devices = split($::osd_devices_list, ' ') +#Class Ceph is already defined so it will do it's thing. +notify {"ceph_osd: ${osd_devices}": } +notify {"osd_devices: ${::osd_devices_list}": } +# TODO(bogdando) add monit ceph-osd services monitoring, if required + +################################################################# + +# vim: set ts=2 sw=2 et : diff --git a/resources/ceph_osd/meta.yaml b/resources/ceph_osd/meta.yaml new file mode 100644 index 00000000..7bfdd745 --- /dev/null +++ b/resources/ceph_osd/meta.yaml @@ -0,0 +1,41 @@ +id: ceph_osd +handler: puppetv2 +version: 1.0.0 +actions: + run: run.pp + update: run.pp +input: + ip: + schema: str! + value: + public_vip: + schema: str! + value: + management_vip: + schema: str! + value: + use_syslog: + schema: bool + value: true + keystone: + schema: {'admin_token': 'str'} + value: {} + ceph_monitor_nodes: + schema: [] + value: [] + ceph_primary_monitor_node: + schema: [] + value: [] + storage: + schema: {} + value: {} + network_scheme: + schema: {} + value: {} + role: + schema: str! + value: + puppet_modules: + schema: str! + value: +tags: [] diff --git a/resources/fuel_library/README.md b/resources/fuel_library/README.md new file mode 100644 index 00000000..e1de8fab --- /dev/null +++ b/resources/fuel_library/README.md @@ -0,0 +1,5 @@ +# Fetch fuel-library repo + +This resource can download custom fuel-library branch. + +It uses update_modules.sh with puppet_librarian(_simple). diff --git a/resources/fuel_library/actions/run.sh b/resources/fuel_library/actions/run.sh index e16245b4..3f42c805 100644 --- a/resources/fuel_library/actions/run.sh +++ b/resources/fuel_library/actions/run.sh @@ -1,6 +1,8 @@ #!/bin/bash mkdir -p {{temp_directory}} +use_librarin_simple={{librarian_puppet_simple}} + pushd {{temp_directory}} if [ ! -d fuel-library ] @@ -11,10 +13,15 @@ else git pull popd fi + +[ -n $use_librarian_puppet_simple ] && gem install librarian-puppet-simple --no-ri --no-rdoc + pushd ./fuel-library/deployment ./update_modules.sh popd +[ -n $use_librarian_puppet_simple ] && gem uninstall -x librarian-puppet-simple + mkdir -p {{puppet_modules}} cp -r ./fuel-library/deployment/puppet/* {{puppet_modules}} popd diff --git a/resources/fuel_library/meta.yaml b/resources/fuel_library/meta.yaml index 1d521038..485c5539 100644 --- a/resources/fuel_library/meta.yaml +++ b/resources/fuel_library/meta.yaml @@ -15,4 +15,7 @@ input: puppet_modules: schema: str! value: /etc/fuel/modules + librarian_puppet_simple: + schema: str! + value: tags: [] diff --git a/resources/managed_apt/actions/run.yaml b/resources/managed_apt/actions/run.yaml index 2776ef02..4923b229 100644 --- a/resources/managed_apt/actions/run.yaml +++ b/resources/managed_apt/actions/run.yaml @@ -3,5 +3,5 @@ tasks: - shell: echo 'Managed by solar' > /etc/apt/sources.list when: {{ensure_other_removed}} + - shell: wget -qO - {{gpg_key}} | sudo apt-key add - - shell: apt-get update - when: {{ensure_other_removed}} diff --git a/resources/managed_apt/meta.yaml b/resources/managed_apt/meta.yaml index 482bca4a..d5bac88e 100644 --- a/resources/managed_apt/meta.yaml +++ b/resources/managed_apt/meta.yaml @@ -15,3 +15,7 @@ input: ensure_other_removed: schema: bool value: true + gpg_key: + schema: [str!] + value: + diff --git a/resources/remote_file/README.md b/resources/remote_file/README.md new file mode 100644 index 00000000..651efc3a --- /dev/null +++ b/resources/remote_file/README.md @@ -0,0 +1,3 @@ +# Copies files to remote nodes + +This can be used for keys distribution. diff --git a/resources/remote_file/actions/run.sh b/resources/remote_file/actions/run.sh index 212bdfb4..d9698439 100644 --- a/resources/remote_file/actions/run.sh +++ b/resources/remote_file/actions/run.sh @@ -2,7 +2,7 @@ mkdir -p {{dest}} {% for transport in remote %} {% if transport.name == 'ssh' %} -scp -i {{transport.key}} -r {{transport.user}}@{{remote_ip}}:/{{remote_path}} {{dest}} +scp -o "StrictHostKeyChecking no" -i {{transport.key}} -r {{transport.user}}@{{remote_ip}}:/{{remote_path}} {{dest}} exit 0 {% endif %} {% endfor %} diff --git a/templates/mos_repos.yaml b/templates/mos_repos.yaml index dddf431e..6ee0eade 100644 --- a/templates/mos_repos.yaml +++ b/templates/mos_repos.yaml @@ -1,5 +1,14 @@ id: mos_repos resources: + - id: mos_main_{{index}} + from: resources/apt_repo + location: {{node}} + values: + name: mos + package: '*' + repo: deb http://mirror.fuel-infra.org/mos-repos/ubuntu/7.0/ mos7.0 main restricted + pin: release o=Mirantis,n=mos7.0,a=mos7.0-security,l=mos7.0 + pin_priority: 1100 - id: mos_holdback_{{index}} from: resources/apt_repo location: {{node}} @@ -13,7 +22,7 @@ resources: from: resources/apt_repo location: {{node}} values: - name: mos + name: mos_security package: '*' repo: deb http://mirror.fuel-infra.org/mos-repos/ubuntu/7.0/ mos7.0-security main restricted pin: release o=Mirantis,n=mos7.0,a=mos7.0-security,l=mos7.0 @@ -32,12 +41,16 @@ resources: location: {{node}} values: names: + - mos_main_{{index}}::name - mos_holdback_{{index}}::name - mos_security_{{index}}::name - mos_updates_{{index}}::name repos: + - mos_main_{{index}}::repo - mos_holdback_{{index}}::repo - mos_security_{{index}}::repo - mos_updates_{{index}}::repo ensure_other_removed: false + gpg_key: http://mirror.fuel-infra.org/mos-repos/ubuntu/7.0/archive-mos7.0.key + diff --git a/templates/nodes.yaml b/templates/nodes.yaml index 3c2e75a9..d122264d 100644 --- a/templates/nodes.yaml +++ b/templates/nodes.yaml @@ -14,14 +14,14 @@ resources: transports:user: ssh_transport{{j}}::ssh_user transports:port: ssh_transport{{j}}::ssh_port transports:name: ssh_transport{{j}}::name - - id: node{{j}} + - id: solar-dev{{j}} from: resources/ro_node values: - name: node{{j}} + name: solar-dev{{j}} ip: '10.0.0.{{i + 3}}' transports_id: transports{{j}}::transports_id - id: hosts_file{{j}} from: resources/hosts_file - location: node{{j}} + location: solar-dev{{j}} tags: ['location=node{{j}}'] {% endfor %}