From 1d3e30004319906da03efd92ea61783186d47879 Mon Sep 17 00:00:00 2001 From: David Lyle Date: Wed, 5 Apr 2017 17:02:38 -0400 Subject: [PATCH] METRON-826: Ambari MPack should utilize service specific repos --- .../ELASTICSEARCH/2.3.3/repos/repoinfo.xml | 32 +++++++++ .../addon-services/KIBANA/4.5.1/metainfo.xml | 4 -- .../KIBANA/4.5.1/repos/repoinfo.xml | 41 +++++++++++ .../METRON/CURRENT/repos/repoinfo.xml | 32 +++++++++ .../2.3.3/package/scripts/elastic_master.py | 12 +--- .../2.3.3/package/scripts/elastic_slave.py | 10 +-- .../4.5.1/package/scripts/kibana_master.py | 27 -------- .../CURRENT/configuration/metron-env.xml | 26 ------- .../package/scripts/enrichment_commands.py | 31 --------- .../package/scripts/enrichment_master.py | 2 - .../package/scripts/indexing_commands.py | 25 ------- .../package/scripts/indexing_master.py | 2 - .../CURRENT/package/scripts/metron_service.py | 1 - .../package/scripts/params/params_linux.py | 7 -- .../package/scripts/parser_commands.py | 25 ------- .../CURRENT/package/scripts/parser_master.py | 2 - .../METRON/CURRENT/themes/metron_theme.json | 68 ------------------- .../roles/metron-rpms/tasks/main.yml | 11 +++ 18 files changed, 120 insertions(+), 238 deletions(-) create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml create mode 100644 metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml new file mode 100644 index 0000000000..0a9caac3c1 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/2.3.3/repos/repoinfo.xml @@ -0,0 +1,32 @@ + + + + + + https://packages.elastic.co/elasticsearch/2.x/centos + elastic-2.x + ELASTICSEARCH + + + + + https://packages.elastic.co/elasticsearch/2.x/centos + elasticsearch-2.x + ELASTICSEARCH + + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml index 156a34ad1c..494f71b355 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/metainfo.xml @@ -20,10 +20,6 @@ 2.0 - - ELASTICSEARCH - common-services/ELASTICSEARCH/2.3.3 - KIBANA common-services/KIBANA/4.5.1 diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml new file mode 100644 index 0000000000..b05c2cfa09 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/4.5.1/repos/repoinfo.xml @@ -0,0 +1,41 @@ + + + + + + http://packages.elastic.co/kibana/4.5/centos + kibana-4.x + KIBANA + + + http://packages.elastic.co/curator/4/centos/6 + ES-Curator-4.x + CURATOR + + + + + http://packages.elastic.co/kibana/4.5/centos + kibana-4.x + KIBANA + + + http://packages.elastic.co/curator/4/centos/7 + ES-Curator-4.x + CURATOR + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml new file mode 100644 index 0000000000..fac4f772a5 --- /dev/null +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml @@ -0,0 +1,32 @@ + + + + + + file:///localrepo + METRON-${metron.version} + METRON + + + + + file:///localrepo + METRON-${metron.version} + METRON + + + + diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py index 3ad4e1f2b0..706610ee1f 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_master.py @@ -19,6 +19,7 @@ from resource_management.core.resources.system import Execute from resource_management.libraries.script import Script +from resource_management.core.logger import Logger from elastic import elastic @@ -27,16 +28,7 @@ class Elasticsearch(Script): def install(self, env): import params env.set_params(params) - - print 'Install the Master' - Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') - Execute("echo \"[elasticsearch-2.x]\n" - "name=Elasticsearch repository for 2.x packages\n" - "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n" - "gpgcheck=1\n" - "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") - + Logger.info('Install ES Master Node') self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py index 4b636e886a..7f7f21a129 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic_slave.py @@ -19,6 +19,7 @@ from resource_management.core.resources.system import Execute from resource_management.libraries.script import Script +from resource_management.core.logger import Logger from slave import slave @@ -27,14 +28,7 @@ class Elasticsearch(Script): def install(self, env): import params env.set_params(params) - print 'Install the Slave' - Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') - Execute("echo \"[elasticsearch-2.x]\n" - "name=Elasticsearch repository for 2.x packages\n" - "baseurl=https://packages.elastic.co/elasticsearch/2.x/centos\n" - "gpgcheck=1\n" - "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/elasticsearch.repo") + Logger.info('Install ES Data Node') self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py index 028ea9ea5c..faca7a83fc 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/kibana_master.py @@ -37,34 +37,7 @@ class Kibana(Script): def install(self, env): import params env.set_params(params) - Logger.info("Install Kibana Master") - - # TODO: Figure this out for all supported OSes - Execute('rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch') - Logger.info("Installing Kibana CentOS/RHEL repo") - Execute("echo \"[kibana-4.x]\n" - "name=Kibana repository for 4.5.x packages\n" - "baseurl=http://packages.elastic.co/kibana/4.5/centos\n" - "gpgcheck=1\n" - "gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/kibana.repo") - - majorVersion = OSCheck.get_os_major_version() - Logger.info("CentOS/RHEL major version reported by Ambari: " + majorVersion) - if majorVersion == "6" or majorVersion == "7": - repoName = "name=CentOS/RHEL {0} repository for Elasticsearch Curator 4.x packages\n".format(majorVersion) - baseUrl = "baseurl=http://packages.elastic.co/curator/4/centos/{0}\n".format(majorVersion) - Logger.info("Installing Elasticsearch Curator CentOS/RHEL {0} repo".format(majorVersion)) - Execute("echo \"[curator-4]\n" + - repoName + - baseUrl + - "gpgcheck=1\n" - "gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch\n" - "enabled=1\" > /etc/yum.repos.d/curator.repo") - else: - raise Exception("Unsupported CentOS/RHEL version: " + majorVersion) - self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml index 5598a9660b..90e0757cd6 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml @@ -124,32 +124,6 @@ URL of Zeppelin (zeppelin.server.addr:zeppelin.server.port) - - repo_type - Repository Type - Type of Repository: Local or Remote - local - - false - value-list - - - local - - - - remote - - - - 1 - - - - repo_url - Repository URL - - global-json global.json template diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py index a17145d089..3d7eb66939 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py @@ -74,37 +74,6 @@ def set_geo_configured(self): owner=self.__params.metron_user, mode=0775) - def setup_repo(self): - - def local_repo(): - Logger.info("Setting up local repo") - Execute("yum -y install createrepo") - Execute("createrepo /localrepo") - Execute("chmod -R o-w+r /localrepo") - - def remote_repo(): - Logger.info('Using remote repo') - - yum_repo_types = { - 'local': local_repo, - 'remote': remote_repo - } - - repo_type = self.__params.yum_repo_type - - if repo_type in yum_repo_types: - yum_repo_types[repo_type]() - Logger.info("Writing out repo file") - repo_template = ("echo \"[METRON-${metron.version}]\n" - "name=Metron ${metron.version} packages\n" - "baseurl={0}\n" - "gpgcheck=0\n" - "enabled=1\n\"" - " > /etc/yum.repos.d/metron.repo") - Execute(repo_template.format(self.__params.repo_url)) - else: - raise ValueError("Unsupported repo type '{0}'".format(repo_type)) - def init_geo(self): Logger.info("Creating HDFS location for GeoIP database") self.__params.HdfsResource(self.__params.geoip_hdfs_dir, diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py index 362b2e2b09..1e734b4dab 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py @@ -29,8 +29,6 @@ class Enrichment(Script): def install(self, env): from params import params env.set_params(params) - commands = EnrichmentCommands(params) - commands.setup_repo() self.install_packages(env) self.configure(env) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index a24a5107c5..819085f627 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -46,31 +46,6 @@ def set_configured(self): owner=self.__params.metron_user, mode=0775) - def setup_repo(self): - def local_repo(): - Logger.info("Setting up local repo") - Execute("yum -y install createrepo") - Execute("createrepo /localrepo") - Execute("chmod -R o-w+r /localrepo") - Execute("echo \"[METRON-${metron.version}]\n" - "name=Metron ${metron.version} packages\n" - "baseurl=file:///localrepo\n" - "gpgcheck=0\n" - "enabled=1\" > /etc/yum.repos.d/local.repo") - - def remote_repo(): - print('Using remote repo') - - yum_repo_types = { - 'local': local_repo, - 'remote': remote_repo - } - repo_type = self.__params.yum_repo_type - if repo_type in yum_repo_types: - yum_repo_types[repo_type]() - else: - raise ValueError("Unsupported repo type '{0}'".format(repo_type)) - def init_kafka_topics(self): Logger.info('Creating Kafka topics') command_template = """{0}/kafka-topics.sh \ diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 53fb17ba99..8189e3f348 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -33,8 +33,6 @@ class Indexing(Script): def install(self, env): from params import params env.set_params(params) - commands = IndexingCommands(params) - commands.setup_repo() self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py index 57da2c7c64..84154605a0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py @@ -23,7 +23,6 @@ from resource_management.core.source import InlineTemplate from resource_management.libraries.functions import format as ambari_format - def init_config(): Logger.info('Loading config into ZooKeeper') Execute(ambari_format( diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index a5e1e9d44c..a4bfcbdd7d 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -66,13 +66,6 @@ es_http_port = config['configurations']['metron-env']['es_http_port'] es_http_url = es_host_list[0] + ":" + es_http_port -# install repo -yum_repo_type = config['configurations']['metron-env']['repo_type'] -if yum_repo_type == 'local': - repo_url = 'file:///localrepo' -else: - repo_url = config['configurations']['metron-env']['repo_url'] - # hadoop params stack_root = Script.get_stack_root() # This is the cluster group named 'hadoop'. Its membership is the stack process user ids not individual users. diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py index 574d7e81a9..bd3ad2cd62 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_commands.py @@ -71,31 +71,6 @@ def init_parsers(self): def get_parser_list(self): return self.__parser_list - def setup_repo(self): - def local_repo(): - Logger.info("Setting up local repo") - Execute("yum -y install createrepo") - Execute("createrepo /localrepo") - Execute("chmod -R o-w+r /localrepo") - Execute("echo \"[METRON-${metron.version}]\n" - "name=Metron ${metron.version} packages\n" - "baseurl=file:///localrepo\n" - "gpgcheck=0\n" - "enabled=1\" > /etc/yum.repos.d/local.repo") - - def remote_repo(): - print('Using remote repo') - - yum_repo_types = { - 'local': local_repo, - 'remote': remote_repo - } - repo_type = self.__params.yum_repo_type - if repo_type in yum_repo_types: - yum_repo_types[repo_type]() - else: - raise ValueError("Unsupported repo type '{0}'".format(repo_type)) - def init_kafka_topics(self): Logger.info('Creating Kafka topics') command_template = """{0}/kafka-topics.sh \ diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py index e6d2ac609b..2721d137e2 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/parser_master.py @@ -34,8 +34,6 @@ def get_component_name(self): def install(self, env): from params import params env.set_params(params) - commands = ParserCommands(params) - commands.setup_repo() self.install_packages(env) def configure(self, env, upgrade_type=None, config_dir=None): diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json index 1f560708d9..9d0f56c933 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json @@ -31,37 +31,7 @@ } ] } - }, - { - "name": "metron_repo", - "display-name" : "Repository", - "layout": { - "tab-columns" : "1", - "tab-rows" : "1", - "sections" : [ - { - "name" : "section-repo", - "row-index": "0", - "column-index" : "0", - "row-span": "1", - "column-span": "1", - "section-columns": "1", - "section-rows": "1", - "subsections": [ - { - "name": "subsection-repo", - "row-index": "0", - "column-index": "0", - "row-span": "1", - "column-span": "1" - } - ] - } - ] - } } - ] - } ], "placement": { "configuration-layout": "default", @@ -85,32 +55,6 @@ { "config": "metron-env/es_cluster_name", "subsection-name": "subsection-general-indexing" - }, - { - "config": "metron-env/repo_type", - "subsection-name": "subsection-repo" - }, - { - "config": "metron-env/repo_url", - "subsection-name": "subsection-repo", - "depends-on": [ - { - "configs":[ - "metron-env/repo_type" - ], - "if": "${metron-env/repo_type} === remote", - "then": { - "property_value_attributes": { - "visible": true - } - }, - "else": { - "property_value_attributes": { - "visible": false - } - } - } - ] } ] }, @@ -138,18 +82,6 @@ "widget": { "type": "text-field" } - }, - { - "config": "metron-env/repo_type", - "widget": { - "type": "toggle" - } - }, - { - "config": "metron-env/repo_url", - "widget": { - "type": "text-field" - } } ] } diff --git a/metron-deployment/roles/metron-rpms/tasks/main.yml b/metron-deployment/roles/metron-rpms/tasks/main.yml index 7c7a2d7ab2..8ea30887ab 100644 --- a/metron-deployment/roles/metron-rpms/tasks/main.yml +++ b/metron-deployment/roles/metron-rpms/tasks/main.yml @@ -35,3 +35,14 @@ mode: 0755 with_fileglob: - "{{ metron_rpm_glob }}" + +- name: Install createrepo + package: + name: createrepo + state: installed + +- name: Create local repo with new packages + shell: "createrepo /localrepo" + args: + creates: /localrepo/repodata +