From 44a3968c3c6e07acc625de5b69e2ecef167022f2 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Wed, 19 Apr 2017 17:31:52 +0000 Subject: [PATCH 01/13] add version info; remove juju < 2 notes; add section for reporting issues; update reference links --- .../hadoop/layer-hadoop-namenode/README.md | 39 ++++---- .../layer-hadoop-namenode/metadata.yaml | 5 +- .../hadoop/layer-hadoop-plugin/README.md | 35 +++---- .../layer-hadoop-resourcemanager/README.md | 39 ++++---- .../metadata.yaml | 5 +- .../charm/hadoop/layer-hadoop-slave/README.md | 40 ++++---- .../hadoop/layer-hadoop-slave/metadata.yaml | 4 +- .../src/charm/hbase/layer-hbase/README.md | 80 +++++---------- .../src/charm/hbase/layer-hbase/metadata.yaml | 6 +- .../src/charm/kafka/layer-kafka/README.md | 50 ++++------ .../src/charm/kafka/layer-kafka/metadata.yaml | 25 +---- .../src/charm/mahout/layer-mahout/README.md | 54 +++++----- .../charm/mahout/layer-mahout/metadata.yaml | 7 +- .../src/charm/pig/layer-pig/README.md | 98 +++++++++---------- .../src/charm/pig/layer-pig/metadata.yaml | 5 +- .../src/charm/spark/layer-spark/README.md | 72 +++++--------- .../src/charm/spark/layer-spark/metadata.yaml | 6 +- .../charm/zeppelin/layer-zeppelin/README.md | 47 ++++----- .../zeppelin/layer-zeppelin/metadata.yaml | 7 +- .../charm/zookeeper/layer-zookeeper/README.md | 50 +++++----- .../zookeeper/layer-zookeeper/metadata.yaml | 7 +- 21 files changed, 291 insertions(+), 390 deletions(-) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md index e5cbf6352b..e22bbe7827 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md @@ -20,27 +20,22 @@ The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using a simple programming model. -This charm deploys the NameNode component of the [Apache Bigtop][] platform -to provide HDFS master resources. +This charm deploys version 2.7.3 of the NameNode component from +[Apache Bigtop][]. [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm is intended to be deployed via one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop cluster with this charm acting as the NameNode. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -52,7 +47,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -79,25 +73,16 @@ application is functioning as expected. Run the action as follows: juju run-action namenode/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do namenode/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities This charm includes Hadoop command line and web utilities that can be used to verify information about the cluster. @@ -117,6 +102,18 @@ The web interface will be available at the following URL: http://NAMENODE_PUBLIC_IP:50070 +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -127,6 +124,6 @@ The web interface will be available at the following URL: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/metadata.yaml b/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/metadata.yaml index a358a6d743..2a1806fcc3 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/metadata.yaml +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/metadata.yaml @@ -1,11 +1,12 @@ name: hadoop-namenode -summary: HDFS master (NameNode) from Apache Bigtop +summary: HDFS NameNode from Apache Bigtop maintainer: Juju Big Data description: > Hadoop is a software platform that lets one easily write and run applications that process vast amounts of data. - This charm provides the HDFS master node (NameNode). + This charm provides version 2.7.3 of the HDFS NameNode application from + Apache Bigtop. tags: [] provides: namenode: diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md index f9e44838fd..d32fbcf0a5 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md @@ -28,19 +28,14 @@ This charm facilitates communication between Hadoop components of an # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm is intended to be deployed via one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop cluster with a client unit preconfigured to work with the cluster. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -52,7 +47,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -79,25 +73,16 @@ application is functioning as expected. Run the action as follows: juju run-action plugin/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do plugin/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities This charm includes Hadoop command line utilities that can be used to verify information about the cluster. @@ -107,6 +92,18 @@ Show the dfsadmin report on the command line with the following: juju run --application plugin "su hdfs -c 'hdfs dfsadmin -report'" +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -117,6 +114,6 @@ Show the dfsadmin report on the command line with the following: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md index 829ea3a788..2d484366dd 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md @@ -20,27 +20,22 @@ The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using a simple programming model. -This charm deploys the ResourceManager component of the [Apache Bigtop][] -platform to provide YARN master resources. +This charm deploys version 2.7.3 of the ResourceManager component from +[Apache Bigtop][]. [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm is intended to be deployed via one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop cluster with this charm acting as the ResourceManager. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -52,7 +47,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -81,25 +75,16 @@ smoke tests provided by Apache Bigtop and may take up to juju run-action resourcemanager/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do resourcemanager/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities This charm includes Hadoop command line and web utilities that can be used to verify information about the cluster. @@ -168,6 +153,18 @@ cluster. Each benchmark is an action that can be run with `juju run-action`: started: 2016-02-04 14:55:27 +0000 UTC +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -178,6 +175,6 @@ cluster. Each benchmark is an action that can be run with `juju run-action`: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/metadata.yaml b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/metadata.yaml index 695d5bfab9..84e42555ac 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/metadata.yaml +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/metadata.yaml @@ -1,11 +1,12 @@ name: hadoop-resourcemanager -summary: YARN master (ResourceManager) from Apache Bigtop +summary: YARN ResourceManager from Apache Bigtop maintainer: Juju Big Data description: > Hadoop is a software platform that lets one easily write and run applications that process vast amounts of data. - This charm provides the YARN master node (ResourceManager). + This charm provides version 2.7.3 of the YARN ResourceManager application from + Apache Bigtop. tags: [] provides: resourcemanager: diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md index ee93239c5b..8c9a4f6ab6 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md @@ -20,28 +20,22 @@ The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using a simple programming model. -This charm deploys a combined slave node running the NodeManager -and DataNode components of the [Apache Bigtop][] platform -to provide YARN compute and HDFS storage resources. +This charm deploys a combined slave node running version 2.7.3 of both the +NodeManager and DataNode components from [Apache Bigtop][]. [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm is intended to be deployed via one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop cluster with 3 units of this charm acting as the combined DataNode/NodeManager application. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -53,7 +47,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -82,25 +75,16 @@ and 'mapreduce' smoke tests provided by Apache Bigtop and may take up to juju run-action slave/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do slave/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - # Scaling @@ -114,6 +98,18 @@ Multiple units may be added at once. For example, add four more slave units: juju add-unit -n4 slave +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -124,6 +120,6 @@ Multiple units may be added at once. For example, add four more slave units: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/metadata.yaml b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/metadata.yaml index 9e4601e1c7..0b56f32a7b 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/metadata.yaml +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/metadata.yaml @@ -5,5 +5,5 @@ description: > Hadoop is a software platform that lets one easily write and run applications that process vast amounts of data. - This charm provides both the storage node (DataNode) for HDFS and the - compute node (NodeManager) for Yarn. + This charm provides version 2.7.3 of both the HDFS DataNode and the + YARN NodeManager from Apache Bigtop. diff --git a/bigtop-packages/src/charm/hbase/layer-hbase/README.md b/bigtop-packages/src/charm/hbase/layer-hbase/README.md index d5b714a4d4..039fc6ba3e 100644 --- a/bigtop-packages/src/charm/hbase/layer-hbase/README.md +++ b/bigtop-packages/src/charm/hbase/layer-hbase/README.md @@ -21,58 +21,31 @@ store. Use HBase when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very large tables -- billions of rows X -millions of columns -- atop clusters of commodity hardware. - -HBase is an open-source, distributed, versioned, column-oriented store modeled -after Google's Bigtable: A Distributed Storage System for Structured Data by -Chang et al. Just as Bigtable leverages the distributed data storage provided -by the Google File System, HBase provides Bigtable-like capabilities on top of -Hadoop and HDFS. - -HBase provides: - -- Linear and modular scalability. -- Strictly consistent reads and writes. -- Automatic and configurable sharding of tables -- Automatic failover support between RegionServers. -- Convenient base classes for backing Hadoop MapReduce jobs with HBase tables. -- Easy to use Java API for client access. -- Block cache and Bloom Filters for real-time queries. -- Query predicate push down via server side Filters -- Thrift gateway and a REST-ful Web service that supports XML, Protobuf, - and binary data encoding options -- Extensible jruby-based (JIRB) shell -- Support for exporting metrics via the Hadoop metrics subsystem to files - or Ganglia; or via JMX. - -See [the homepage](http://hbase.apache.org) for more information. - -This charm deploys the hbase master and regionserver components of the -[Apache Bigtop][] platform. +millions of columns -- atop clusters of commodity hardware. Learn more at +[hbase.apache.org][]. +This charm deploys version 1.1.9 of the HBase master and regionserver +components from [Apache Bigtop][]. + +[hbase.apache.org]: http://hbase.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. An HBase deployment consists of HBase masters and HBase RegionServers. In a distributed HBase environment, one master and one regionserver are -deployed on each unit. HBase makes sure that only one master is active and -the rest are in standby mode in case the active one fails. +deployed on each unit. HBase ensures that only one master is active with +the rest in standby mode in case the active master fails. Because HBase requires HDFS, this charm is recommended to be deployed as part of the `hadoop-hbase` bundle: juju deploy hadoop-hbase -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop Hadoop cluster with 3 HBase units. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-hbase/). @@ -86,7 +59,6 @@ in this environment, configure a Juju model with appropriate proxy and/or mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -113,25 +85,16 @@ application is functioning as expected. Run the action as follows: juju run-action hbase/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do hbase/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## HBase web UI HBase provides a web console that can be used to verify information about the cluster. To access it, find the `PUBLIC-ADDRESS` of any hbase unit and @@ -149,9 +112,6 @@ The web interface will be available at the following URL: Once the deployment has been verified, there are a number of actions available in this charm. -> **Note**: Actions described below assume Juju 2.0 or greater. If using an -earlier version of Juju, the action syntax is: -`juju action do hbase/0 ; juju action fetch `. Run a performance test: @@ -189,6 +149,18 @@ following events will cause a restart: - Upgrading the hbase charm. +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -196,10 +168,10 @@ following events will cause a restart: # Resources -- [Apache HBase home page](https://hbase.apache.org/) -- [Apache HBase issue tracker](https://issues.apache.org/jira/browse/HBASE) +- [Apache HBase home page](http://hbase.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) +- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/hbase/layer-hbase/metadata.yaml b/bigtop-packages/src/charm/hbase/layer-hbase/metadata.yaml index 821f3fa8ea..49fef47019 100644 --- a/bigtop-packages/src/charm/hbase/layer-hbase/metadata.yaml +++ b/bigtop-packages/src/charm/hbase/layer-hbase/metadata.yaml @@ -1,9 +1,9 @@ name: hbase -summary: Apache Bitop HBase +summary: HBase from Apache Bigtop maintainer: Juju Big Data description: > - HBase is the Hadoop database. This charm provides HBase from the - Apache Bigtop project. + HBase is the Hadoop database. This charm provides version 1.1.9 of the + HBase application from Apache Bigtop. tags: [] requires: zookeeper: diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/README.md b/bigtop-packages/src/charm/kafka/layer-kafka/README.md index 93fca06cfe..2480a2e4dc 100644 --- a/bigtop-packages/src/charm/kafka/layer-kafka/README.md +++ b/bigtop-packages/src/charm/kafka/layer-kafka/README.md @@ -21,7 +21,8 @@ Software Foundation written in Scala. The project aims to provide a unified, high-throughput, low-latency platform for handling real-time data feeds. Learn more at [kafka.apache.org][]. -This charm deploys the Kafka component of the [Apache Bigtop][] platform. +This charm deploys version 0.10.1 of the Kafka component from +[Apache Bigtop][]. [kafka.apache.org]: http://kafka.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ @@ -29,9 +30,8 @@ This charm deploys the Kafka component of the [Apache Bigtop][] platform. # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. Kafka requires the Zookeeper distributed coordination service. Deploy and relate them as follows: @@ -52,9 +52,6 @@ mirror options. See [Configuring Models][] for more information. # Using Once deployed, there are a number of actions available in this charm. -> **Note**: Actions described below assume Juju 2.0 or greater. If using an -earlier version of Juju, the action syntax is: -`juju action do kafka/0 ; juju action fetch `. List the zookeeper servers that our kafka brokers are connected to. The following will list `:` information for each @@ -110,25 +107,16 @@ topics. Run the action as follows: juju run-action slave/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do kafka/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - # Scaling @@ -143,9 +131,6 @@ ready units, create a replicated topic as follows: juju run-action kafka/0 create-topic topic=my-replicated-topic \ partitions=1 replication=2 -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do kafka/0 create-topic `. - Query the description of the recently created topic: juju run --unit kafka/0 'kafka-topics.sh --describe \ @@ -191,9 +176,6 @@ network interface name or a CIDR range specifying a subnet. For example: juju config kafka network_interface=eth0 juju config kafka network_interface=10.0.2.0/24 -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju set-config kafka network_interface=eth0`. - Each kafka machine in the cluster will lookup the IP address of that network interface, or find the first network interface with an IP address in the specified subnet, and bind kafka to that address. @@ -205,16 +187,24 @@ run "juju resolved" on each unit: juju config kafka network_interface=eth0 juju resolved kafka/0 -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju set-config kafka network_interface=eth0; -juju resolved -r kafka/0`. - To go back to listening on any network interface on the machine, simply pass ``0.0.0.0`` to ``network_interface``. juju config kafka network_interface=0.0.0.0 +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -222,10 +212,10 @@ machine, simply pass ``0.0.0.0`` to ``network_interface``. # Resources +- [Apache Kafka home page](http://kafka.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) +- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Apache Kafka home page](http://kafka.apache.org/) -- [Apache Kafka issue tracker](https://issues.apache.org/jira/browse/KAFKA) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml b/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml index 5a11394874..ffd64f43ff 100644 --- a/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml +++ b/bigtop-packages/src/charm/kafka/layer-kafka/metadata.yaml @@ -1,26 +1,11 @@ name: kafka -summary: High-performance distributed messaging system from Apache Bigtop +summary: Kafka from Apache Bigtop maintainer: Juju Big Data -description: | - Fast - A single Kafka broker can handle hundreds of megabytes of reads and writes per - second from thousands of clients. +description: > + Kafka is a high-performance, scalable, distributed messaging system. - Scalable - Kafka is designed to allow a single cluster to serve as the central data - backbone for a large organization. It can be elastically and transparently - expanded without downtime. Data streams are partitioned and spread over a - cluster of machines to allow data streams larger than the capability of any - single machine and to allow clusters of co-ordinated consumers. - - Durable - Messages are persisted on disk and replicated within the cluster to prevent - data loss. Each broker can handle terabytes of messages without performance - impact. - - Distributed by Design - Kafka has a modern cluster-centric design that offers strong durability and - fault-tolerance guarantees. + This charm provides version 0.10.1 of the Kafka application from Apache + Bigtop. tags: [] provides: client: diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/README.md b/bigtop-packages/src/charm/mahout/layer-mahout/README.md index 2ecbf5ab78..30fb0aad8a 100644 --- a/bigtop-packages/src/charm/mahout/layer-mahout/README.md +++ b/bigtop-packages/src/charm/mahout/layer-mahout/README.md @@ -16,33 +16,27 @@ --> # Overview -The Apache Mahout project's goal is to build an environment for quickly creating -scalable performant machine learning applications. +The Apache Mahout project's goal is to build an environment for quickly +creating scalable, performant machine learning applications. Learn more at +[mahout.apache.org][]. -Apache Mahout software provides three major features: - * A simple and extensible programming environment and framework for building - scalable algorithms - * A wide variety of premade algorithms for Scala + Apache Spark, H2O, Apache - Flink - * Samsara, a vector math experimentation environment with R-like syntax which - works at scale +This charm deploys version 0.12.2 of the Mahout component from +[Apache Bigtop][]. + +[mahout.apache.org]: http://mahout.apache.org/ +[Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm is intended to be used with one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop Hadoop cluster. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -58,7 +52,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -85,24 +78,27 @@ application is functioning as expected. Run the action as follows: juju run-action mahout/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do mahout/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. + +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. # Contact Information @@ -112,10 +108,10 @@ of Juju, the syntax is `juju action fetch `. # Resources +- [Apache Mahout home page](http://mahout.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) +- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Apache Mahout home page](https://mahout.apache.org/) -- [Apache Mahout issue tracker](https://issues.apache.org/jira/browse/MAHOUT) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml index 7a71d70ff2..bf8ee0b2df 100644 --- a/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml +++ b/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml @@ -1,9 +1,12 @@ name: mahout -summary: Create scalable and performant machine learning applications +summary: Mahout from Apache Bigtop maintainer: Juju Big Data description: > The Apache Mahout project's goal is to build an environment for quickly - creating scalable and performant machine learning applications. + creating scalable, performant machine learning applications. + + This charm provides version 0.12.2 of the Mahout application from Apache + Bigtop. tags: [] subordinate: true requires: diff --git a/bigtop-packages/src/charm/pig/layer-pig/README.md b/bigtop-packages/src/charm/pig/layer-pig/README.md index 2334876653..affc26fe23 100644 --- a/bigtop-packages/src/charm/pig/layer-pig/README.md +++ b/bigtop-packages/src/charm/pig/layer-pig/README.md @@ -19,35 +19,24 @@ Apache Pig is a platform for creating MapReduce programs used with Hadoop. It consists of a high-level language (Pig Latin) for expressing data analysis programs, coupled with infrastructure for evaluating these programs. Learn more -at [pig.apache.org](http://pig.apache.org). +at [pig.apache.org][]. -This charm deploys the Pig component of the Apache Bigtop platform and -supports running Pig in two execution modes: +This charm deploys version 0.15.0 of the Pig component from [Apache Bigtop][]. - * Local Mode: Pig runs using your local host and file system. Specify local - mode using the -x flag: `pig -x local` - * Mapreduce Mode: Pig runs using a Hadoop cluster and HDFS. This is the default - mode; you can, optionally, specify it using the -x flag: - `pig` or `pig -x mapreduce` +[pig.apache.org]: http://pig.apache.org/ +[Apache Bigtop]: http://bigtop.apache.org/ -# Deploying / Using +# Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the -[getting-started](https://jujucharms.com/docs/2.0/getting-started) -instructions prior to deploying this charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. -This charm is intended to be used with one of the -[apache bigtop bundles](https://jujucharms.com/u/bigdata-charmers/#bundles). +This charm is intended to be deployed via one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart](https://launchpad.net/juju-quickstart) with the -following syntax: `juju quickstart hadoop-processing`. - This will deploy an Apache Bigtop Hadoop cluster. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -56,21 +45,14 @@ Now add Pig and relate it to the cluster via the hadoop-plugin: juju deploy pig juju add-relation pig plugin -Once deployment is complete, Apache Pig will be available to execute Pig Latin -jobs on your data. You can run Pig in a variety of modes: - -## Local Mode -Run Pig in local mode on the Pig unit with the following: - - juju ssh pig/0 - pig -x local - -## MapReduce Mode -MapReduce mode is the default for Pig. To run in this mode, ssh to the Pig unit -and run pig as follows: +## Network-Restricted Environments +Charms can be deployed in environments with limited network access. To deploy +in this environment, configure a Juju model with appropriate proxy and/or +mirror options. See [Configuring Models][] for more information. - juju ssh pig/0 - pig +[getting-started]: https://jujucharms.com/docs/stable/getting-started +[apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles +[Configuring Models]: https://jujucharms.com/docs/stable/models-config # Verifying @@ -84,7 +66,7 @@ are ready: This is particularly useful when combined with `watch` to track the on-going progress of the deployment: - watch -n 0.5 juju status + watch -n 2 juju status The message column will provide information about a given unit's state. This charm is ready for use once the status message indicates that it is @@ -96,15 +78,9 @@ application is functioning as expected. Run the action as follows: juju run-action pig/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do pig/0 smoke-test`. - Watch the progress of the smoke test actions with: - watch -n 0.5 juju show-action-status - -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. + watch -n 2 juju show-action-status Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get @@ -112,17 +88,36 @@ more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. +# Using -# Network-Restricted Environments +Once the deployment has been verified, Apache Pig will be available to execute +Pig Latin jobs on your data. You can run Pig in a variety of modes: -Charms can be deployed in environments with limited network access. To deploy -in this environment, configure a Juju model with appropriate -proxy and/or mirror options. See -[Configuring Models](https://jujucharms.com/docs/2.0/models-config) for more -information. +## Local Mode +Run Pig in local mode on the Pig unit with the following: + + juju ssh pig/0 + pig -x local + +## MapReduce Mode +MapReduce mode is the default for Pig. To run in this mode, ssh to the Pig unit +and run pig as follows: + + juju ssh pig/0 + pig + + +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. # Contact Information @@ -132,9 +127,10 @@ information. # Resources +- [Apache Pig home page](http://pig.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/pig/layer-pig/metadata.yaml b/bigtop-packages/src/charm/pig/layer-pig/metadata.yaml index 2898c6a002..4aa1f66120 100644 --- a/bigtop-packages/src/charm/pig/layer-pig/metadata.yaml +++ b/bigtop-packages/src/charm/pig/layer-pig/metadata.yaml @@ -1,9 +1,10 @@ name: pig -summary: Create Pig Latin programs for data warehouse analysis +summary: Pig from Apache Bigtop maintainer: Juju Big Data description: | Apache Pig is a platform for creating MapReduce programs used with Hadoop. It consists of a high-level language (Pig Latin) for expressing data analysis programs, coupled with infrastructure for evaluating these programs. - Learn more at http://pig.apache.org. + + This charm provides version 0.15.0 of the Pig application from Apache Bigtop. tags: [] diff --git a/bigtop-packages/src/charm/spark/layer-spark/README.md b/bigtop-packages/src/charm/spark/layer-spark/README.md index 6a55c8200b..84bba2809e 100644 --- a/bigtop-packages/src/charm/spark/layer-spark/README.md +++ b/bigtop-packages/src/charm/spark/layer-spark/README.md @@ -17,36 +17,18 @@ # Overview Apache Spark is a fast and general purpose engine for large-scale data -processing. This charm deploys the Spark component of the [Apache Bigtop][] -platform. Key features: +processing. Learn more at [spark.apache.org][]. - * **Speed** - - Run programs up to 100x faster than Hadoop MapReduce in memory, or 10x faster - on disk. Spark has an advanced DAG execution engine that supports cyclic data - flow and in-memory computing. - - * **Ease of Use** - - Write applications quickly in Java, Scala or Python. Spark offers over 80 - high-level operators that make it easy to build parallel apps for use - interactively from the Scala and Python shells. - - * **General Purpose Engine** - - Combine SQL, streaming, and complex analytics. Spark powers a stack of - high-level tools including Shark for SQL, MLlib for machine learning, GraphX, - and Spark Streaming. Combine these frameworks seamlessly in the same - application. +This charm deploys version 2.1.0 of the Spark component from [Apache Bigtop][]. +[spark.apache.org]: http://spark.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm supports running Spark in a variety of modes: @@ -82,26 +64,17 @@ the unit acting as master, query Zookeeper as follows: ### YARN-client and YARN-cluster This charm leverages our pluggable Hadoop model with the `hadoop-plugin` -interface. This means that this charm can be related to a base Apache Hadoop +interface. This means that this charm can be related to an Apache Hadoop cluster to run Spark jobs there. The suggested deployment method is to use the -[hadoop-processing][] bundle and add a relation between spark and the plugin. - +[hadoop-spark][] bundle: - juju deploy hadoop-processing - juju add-relation plugin spark - -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. + juju deploy hadoop-spark To switch among the above execution modes, set the `spark_execution_mode` config variable: juju config spark spark_execution_mode= -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju set spark spark_execution_mode=`. - See the **Configuring** section below for supported mode options. ## Network-Restricted Environments @@ -110,8 +83,7 @@ in this environment, configure a Juju model with appropriate proxy and/or mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started -[hadoop-processing]: https://jujucharms.com/hadoop-processing/ -[juju-quickstart]: https://launchpad.net/juju-quickstart +[hadoop-spark]: https://jujucharms.com/hadoop-spark/ [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -138,25 +110,16 @@ application is functioning as expected. Run the action as follows: juju run-action spark/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do spark/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Spark Master web UI Spark provides a web console that can be used to verify information about the cluster. To access it, find the `PUBLIC-ADDRESS` of the spark application @@ -322,6 +285,18 @@ Each benchmark is an action that can be run with `juju run-action`: [Spark Bench]: https://github.com/SparkTC/spark-bench +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -329,9 +304,10 @@ Each benchmark is an action that can be run with `juju run-action`: # Resources +- [Apache Spark home page](http://spark.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/spark/layer-spark/metadata.yaml b/bigtop-packages/src/charm/spark/layer-spark/metadata.yaml index efec6f0eb6..066642904c 100644 --- a/bigtop-packages/src/charm/spark/layer-spark/metadata.yaml +++ b/bigtop-packages/src/charm/spark/layer-spark/metadata.yaml @@ -1,11 +1,11 @@ name: spark -summary: Apache Spark from Apache Bigtop platform +summary: Spark from Apache Bigtop maintainer: Juju Big Data description: > Apache Spark is a fast and general engine for large-scale data processing. - Learn more at http://spark.apache.org. -tags: ["applications"] + This charm provides version 2.1.0 of the Spark application from Apache Bigtop. +tags: ["analytics"] provides: benchmark: interface: benchmark diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md index c76ad1e5a0..9f19b09c68 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md @@ -18,35 +18,25 @@ Apache Zeppelin is a web-based notebook that enables interactive data analytics. It allows for beautiful data-driven, interactive, and collaborative documents -with SQL, Scala and more. +with SQL, Scala and more. Learn more at [zeppelin.apache.org][]. -As a Multi-purpose Notebook, Apache Zeppelin is the place for interactive: - - * Data Ingestion - * Data Discovery - * Data Analytics - * Data Visualization & Collaboration - -This charm deploys the Zeppelin component of the [Apache Bigtop][] platform. +This charm deploys version 0.7.0 of the Zeppelin component from +[Apache Bigtop][]. +[zeppelin.apache.org]: http://zeppelin.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. This charm is intended to be deployed via one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - This will deploy an Apache Bigtop Hadoop cluster. More information about this deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). @@ -72,7 +62,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles -[juju-quickstart]: https://launchpad.net/juju-quickstart [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -99,24 +88,27 @@ application is functioning as expected. Run the action as follows: juju run-action zeppelin/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do zeppelin/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. + +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. # Contact Information @@ -126,9 +118,10 @@ of Juju, the syntax is `juju action fetch `. # Resources +- [Apache Zeppelin home page](http://zeppelin.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/metadata.yaml b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/metadata.yaml index f6f4be6b75..f07614d5f8 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/metadata.yaml +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/metadata.yaml @@ -1,10 +1,13 @@ name: zeppelin -summary: A web-based notebook that enables interactive data analytics. +summary: Zeppelin from Apache Bigtop maintainer: Juju Big Data -description: | +description: > Apache Zeppelin is a web-based notebook that enables interactive data analytics. You can make beautiful data-driven, interactive, and collaborative documents with SQL, Scala and more. + + This charm provides version 0.7.0 of the Zeppelin application from Apache + Bigtop. tags: ["analytics"] provides: client: diff --git a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md index 9c92d7366d..4619ebc9d1 100644 --- a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md +++ b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md @@ -20,18 +20,19 @@ Apache ZooKeeper is a high-performance coordination service for distributed applications. It exposes common services such as naming, configuration management, synchronization, and group services in a simple interface. Use it off-the-shelf to implement consensus, group management, leader election, and -presence protocols. +presence protocols. Learn more at [zookeeper.apache.org][]. -This charm provides the Zookeeper component of the [Apache Bigtop][] platform. +This charm provides version 3.4.6 of the Zookeeper component from +[Apache Bigtop][]. +[zookeeper.apache.org]: http://zookeeper.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -charm. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this charm. Deploy a Zookeeper unit. With only one unit, the application will be running in `standalone` mode: @@ -55,9 +56,6 @@ network interface name or a CIDR range specifying a subnet. For example: juju config zookeeper network_interface=eth0 juju config zookeeper network_interface=10.0.2.0/24 -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju set-config zookeeper network_interface=eth0`. - Each zookeeper unit in the cluster will lookup the IP address of that network interface, or find the first network interface with an IP address in the specified subnet, and bind Zookeeper to that address. @@ -69,18 +67,11 @@ run "juju resolved" on any failed units: juju config zookeeper network_interface=eth0 juju resolved zookeeper/0 -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju set-config zookeeper network_interface=eth0; -juju resolved -r zookeeper/0`. - To go back to listening on all interfaces, configure zookeeper with `network_interface=0.0.0.0`: juju config zookeeper network_interface=0.0.0.0 -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju set-config zookeeper network_interface=0.0.0.0`. - # Verifying @@ -105,25 +96,16 @@ application is functioning as expected. Run the action as follows: juju run-action zookeeper/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do zookeeper/0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, the action should settle to `status: completed`. If it reports `status: failed`, the application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities This charm includes Zookeeper command line utilities that can also be used to verify that the application is running as expected. Check the status of the @@ -176,6 +158,18 @@ that require Zookeeper as follows: ZK_port = relation_get('port') +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this charm at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Leave any uncertain fields blank. Typically, charm issues are filed in the +`deployment` component with the latest stable release selected as the affected +version. + + # Contact Information - @@ -183,10 +177,10 @@ that require Zookeeper as follows: # Resources +- [Apache Zookeeper home page](http://zookeeper.apache.org/) - [Apache Bigtop home page](http://bigtop.apache.org/) +- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Apache Zookeeper home page](https://zookeeper.apache.org/) -- [Apache Zookeeper issue tracker](https://issues.apache.org/jira/browse/ZOOKEEPER) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/metadata.yaml b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/metadata.yaml index d9818d792b..a56377592b 100644 --- a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/metadata.yaml +++ b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/metadata.yaml @@ -1,12 +1,15 @@ name: zookeeper +summary: Zookeeper from Apache Bigtop maintainer: Juju Big Data -summary: High-performance coordination service for distributed applications -description: | +description: > Apache ZooKeeper is a centralized, reliable application for maintaining configuration information, naming, synchronization, and group services. All of these kinds of services are used in some form or another by distributed applications. In order to install and configure Apache HBase and other Hadoop ecosystem components, you need ZooKeeper. + + This charm provides version 3.4.6 of the Zookeeper application from Apache + Bigtop. tags: [] provides: zookeeper: From e221bb70bbc76b558895086a760a256b2baa10ee Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Wed, 19 Apr 2017 18:53:25 +0000 Subject: [PATCH 02/13] more guidance in issue section --- .../src/charm/hadoop/layer-hadoop-namenode/README.md | 6 +++--- .../src/charm/hadoop/layer-hadoop-plugin/README.md | 6 +++--- .../src/charm/hadoop/layer-hadoop-resourcemanager/README.md | 6 +++--- .../src/charm/hadoop/layer-hadoop-slave/README.md | 6 +++--- bigtop-packages/src/charm/hbase/layer-hbase/README.md | 6 +++--- bigtop-packages/src/charm/kafka/layer-kafka/README.md | 6 +++--- bigtop-packages/src/charm/mahout/layer-mahout/README.md | 6 +++--- bigtop-packages/src/charm/pig/layer-pig/README.md | 6 +++--- bigtop-packages/src/charm/spark/layer-spark/README.md | 6 +++--- bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md | 6 +++--- .../src/charm/zookeeper/layer-zookeeper/README.md | 6 +++--- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md index e22bbe7827..b92ab41cb8 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-namenode/README.md @@ -109,9 +109,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md index d32fbcf0a5..8c9c032284 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-plugin/README.md @@ -99,9 +99,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md index 2d484366dd..bd82d008b8 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/README.md @@ -160,9 +160,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md index 8c9a4f6ab6..41635dc514 100644 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/README.md @@ -105,9 +105,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/hbase/layer-hbase/README.md b/bigtop-packages/src/charm/hbase/layer-hbase/README.md index 039fc6ba3e..a9be1acd18 100644 --- a/bigtop-packages/src/charm/hbase/layer-hbase/README.md +++ b/bigtop-packages/src/charm/hbase/layer-hbase/README.md @@ -156,9 +156,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/README.md b/bigtop-packages/src/charm/kafka/layer-kafka/README.md index 2480a2e4dc..9d7f6059d0 100644 --- a/bigtop-packages/src/charm/kafka/layer-kafka/README.md +++ b/bigtop-packages/src/charm/kafka/layer-kafka/README.md @@ -200,9 +200,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/README.md b/bigtop-packages/src/charm/mahout/layer-mahout/README.md index 30fb0aad8a..2b64b207b9 100644 --- a/bigtop-packages/src/charm/mahout/layer-mahout/README.md +++ b/bigtop-packages/src/charm/mahout/layer-mahout/README.md @@ -96,9 +96,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/pig/layer-pig/README.md b/bigtop-packages/src/charm/pig/layer-pig/README.md index affc26fe23..f38763dceb 100644 --- a/bigtop-packages/src/charm/pig/layer-pig/README.md +++ b/bigtop-packages/src/charm/pig/layer-pig/README.md @@ -115,9 +115,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/spark/layer-spark/README.md b/bigtop-packages/src/charm/spark/layer-spark/README.md index 84bba2809e..0fc596789c 100644 --- a/bigtop-packages/src/charm/spark/layer-spark/README.md +++ b/bigtop-packages/src/charm/spark/layer-spark/README.md @@ -292,9 +292,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md index 9f19b09c68..a303bc2d74 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/README.md @@ -106,9 +106,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information diff --git a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md index 4619ebc9d1..a3d12f18e9 100644 --- a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md +++ b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/README.md @@ -165,9 +165,9 @@ issue for this charm at: https://issues.apache.org/jira/secure/CreateIssue!default.jspa -Leave any uncertain fields blank. Typically, charm issues are filed in the -`deployment` component with the latest stable release selected as the affected -version. +Ensure `Bigtop` is selected as the project. Typically, charm issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. # Contact Information From 6a3ea3a2bee3bf06948948b4e5572626a069d31d Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Wed, 19 Apr 2017 19:13:46 +0000 Subject: [PATCH 03/13] sparkbench does not yet work with spark 2.1, so remove refs from the benchmarking section of the readme --- .../src/charm/spark/layer-spark/README.md | 53 ++++++------------- 1 file changed, 15 insertions(+), 38 deletions(-) diff --git a/bigtop-packages/src/charm/spark/layer-spark/README.md b/bigtop-packages/src/charm/spark/layer-spark/README.md index 0fc596789c..c482ea93e9 100644 --- a/bigtop-packages/src/charm/spark/layer-spark/README.md +++ b/bigtop-packages/src/charm/spark/layer-spark/README.md @@ -233,27 +233,16 @@ the `spark_execution_mode` config variable. # Benchmarking -This charm provides several benchmarks, including the [Spark Bench][] -benchmarking suite (if enabled), to gauge the performance of the environment. +This charm provides benchmarks to gauge the performance of the Spark cluster. Each benchmark is an action that can be run with `juju run-action`: - $ juju actions spark | grep Bench - connectedcomponent Run the Spark Bench ConnectedComponent benchmark. - decisiontree Run the Spark Bench DecisionTree benchmark. - kmeans Run the Spark Bench KMeans benchmark. - linearregression Run the Spark Bench LinearRegression benchmark. - logisticregression Run the Spark Bench LogisticRegression benchmark. - matrixfactorization Run the Spark Bench MatrixFactorization benchmark. - pagerank Run the Spark Bench PageRank benchmark. - pca Run the Spark Bench PCA benchmark. - pregeloperation Run the Spark Bench PregelOperation benchmark. - shortestpaths Run the Spark Bench ShortestPaths benchmark. - sql Run the Spark Bench SQL benchmark. - stronglyconnectedcomponent Run the Spark Bench StronglyConnectedComponent benchmark. - svdplusplus Run the Spark Bench SVDPlusPlus benchmark. - svm Run the Spark Bench SVM benchmark. - - $ juju run-action spark/0 svdplusplus + $ juju actions spark + ... + pagerank Calculate PageRank for a sample data set + sparkpi Calculate Pi + ... + + $ juju run-action spark/0 pagerank Action queued with id: 339cec1f-e903-4ee7-85ca-876fb0c3d28e $ juju show-action-output 339cec1f-e903-4ee7-85ca-876fb0c3d28e @@ -262,27 +251,15 @@ Each benchmark is an action that can be run with `juju run-action`: composite: direction: asc units: secs - value: "200.754000" - raw: | - SVDPlusPlus,2016-11-02-03:08:26,200.754000,85.974071,.428255,0,SVDPlusPlus-MLlibConfig,,,,,10,,,50000,4.0,1.3, - start: 2016-11-02T03:08:26Z - stop: 2016-11-02T03:11:47Z - results: - duration: - direction: asc - units: secs - value: "200.754000" - throughput: - direction: desc - units: x/sec - value: ".428255" + value: "83" + start: 2017-04-12T23:22:38Z + stop: 2017-04-12T23:24:01Z + output: '{''status'': ''completed''}' status: completed timing: - completed: 2016-11-02 03:11:48 +0000 UTC - enqueued: 2016-11-02 03:08:21 +0000 UTC - started: 2016-11-02 03:08:26 +0000 UTC - -[Spark Bench]: https://github.com/SparkTC/spark-bench + completed: 2017-04-12 23:24:02 +0000 UTC + enqueued: 2017-04-12 23:22:36 +0000 UTC + started: 2017-04-12 23:22:37 +0000 UTC # Issues From eaa628982b08fd32eaf22c28c9ae0ed51e291bd6 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Thu, 20 Apr 2017 23:25:45 +0000 Subject: [PATCH 04/13] consistent UX for actions --- .../actions/mrbench | 1 + .../actions/nnbench | 1 + .../actions/parseBenchmark.py | 3 +- .../actions/smoke-test | 10 +++---- .../actions/teragen | 1 + .../actions/terasort | 1 + .../actions/testdfsio | 1 + .../layer-hadoop-slave/actions/smoke-test | 10 +++---- .../charm/hbase/layer-hbase/actions/perf-test | 3 +- .../hbase/layer-hbase/actions/smoke-test | 4 +-- .../kafka/layer-kafka/actions/create-topic | 7 +++-- .../kafka/layer-kafka/actions/kafkautils.py | 4 +-- .../kafka/layer-kafka/actions/list-topics | 7 +++-- .../charm/kafka/layer-kafka/actions/list-zks | 9 +++--- .../kafka/layer-kafka/actions/read-topic | 7 +++-- .../kafka/layer-kafka/actions/smoke-test | 6 ++-- .../kafka/layer-kafka/actions/write-topic | 7 +++-- .../mahout/layer-mahout/actions/smoke-test | 1 + .../charm/pig/layer-pig/actions/smoke-test | 15 +++++----- .../charm/spark/layer-spark/actions/pagerank | 30 ++++++++++--------- .../actions/restart-spark-job-history-server | 26 ++++++++-------- .../spark/layer-spark/actions/sparkbench | 10 +++++-- .../charm/spark/layer-spark/actions/sparkpi | 28 ++++++++++------- .../actions/start-spark-job-history-server | 26 ++++++++-------- .../actions/stop-spark-job-history-server | 26 ++++++++-------- .../charm/spark/layer-spark/actions/submit | 3 +- .../zeppelin/layer-zeppelin/actions/restart | 7 +++-- .../layer-zeppelin/actions/smoke-test | 3 ++ .../zookeeper/layer-zookeeper/actions/restart | 11 +++---- .../layer-zookeeper/actions/smoke-test | 10 +++---- 30 files changed, 153 insertions(+), 125 deletions(-) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/mrbench b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/mrbench index d177a2d84c..2b97413e76 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/mrbench +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/mrbench @@ -71,6 +71,7 @@ benchmark-finish cat ${RESULT_LOG} | $CHARM_DIR/actions/parseBenchmark.py DURATION=`expr $STOP - $START` benchmark-composite "${DURATION}" 'secs' 'asc' +action-set outcome="success" # clean out benchmark dir benchmark_cleanup diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/nnbench b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/nnbench index 13d7618928..64bb2301f7 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/nnbench +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/nnbench @@ -75,6 +75,7 @@ benchmark-finish cat ${RESULT_LOG} | $CHARM_DIR/actions/parseBenchmark.py DURATION=`expr $STOP - $START` benchmark-composite "${DURATION}" 'secs' 'asc' +action-set outcome="success" # clean out benchmark dir benchmark_cleanup diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/parseBenchmark.py b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/parseBenchmark.py index 4abdea001b..2232aa562f 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/parseBenchmark.py +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/parseBenchmark.py @@ -37,7 +37,8 @@ def parse_benchmark_output(): m = regex.match(line) if m: results[m.group(1)] = m.group(2) - hookenv.action_set({"results.raw": json.dumps(results)}) + hookenv.action_set({"meta.raw": json.dumps(results)}) + if __name__ == "__main__": parse_benchmark_output() diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/smoke-test b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/smoke-test index 4218f7e3bc..34f160bac9 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/smoke-test +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/smoke-test @@ -23,12 +23,12 @@ from charms.layer.apache_bigtop_base import Bigtop # noqa: E402 from charms.reactive import is_state # noqa: E402 -def fail(msg, output=None): - if output: - hookenv.action_set({'output': output}) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() + if not is_state('apache-bigtop-resourcemanager.ready'): fail('Charm is not yet ready to run the Bigtop smoke test(s)') @@ -43,6 +43,6 @@ smoke_env = { bigtop = Bigtop() result = bigtop.run_smoke_tests(smoke_components, smoke_env) if result == 'success': - hookenv.action_set({'outcome': result}) + hookenv.action_set({'outcome': 'success'}) else: - fail('{} smoke tests failed'.format(smoke_components), result) + fail('{} smoke tests failed with: {}'.format(smoke_components, result)) diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/teragen b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/teragen index d08468c319..95a65e2986 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/teragen +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/teragen @@ -57,6 +57,7 @@ benchmark-finish cat ${RESULT_LOG} | $CHARM_DIR/actions/parseBenchmark.py DURATION=`expr $STOP - $START` benchmark-composite "${DURATION}" 'secs' 'asc' +action-set outcome="success" # clean out benchmark dir benchmark_cleanup diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/terasort b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/terasort index 5e3e592137..2611318a8d 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/terasort +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/terasort @@ -78,6 +78,7 @@ benchmark-finish cat ${RESULT_LOG} | $CHARM_DIR/actions/parseBenchmark.py DURATION=`expr $STOP - $START` benchmark-composite "${DURATION}" 'secs' 'asc' +action-set outcome="success" # clean out benchmark dir benchmark_cleanup diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/testdfsio b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/testdfsio index 558c9f435c..a53ccac73c 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/testdfsio +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-resourcemanager/actions/testdfsio @@ -76,6 +76,7 @@ benchmark-finish cat ${RESULT_LOG} | $CHARM_DIR/actions/parseBenchmark.py DURATION=`expr $STOP - $START` benchmark-composite "${DURATION}" 'secs' 'asc' +action-set outcome="success" # clean out benchmark dir benchmark_cleanup diff --git a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/actions/smoke-test b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/actions/smoke-test index 68e646d443..f1bb34d44c 100755 --- a/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/actions/smoke-test +++ b/bigtop-packages/src/charm/hadoop/layer-hadoop-slave/actions/smoke-test @@ -23,12 +23,12 @@ from charms.layer.apache_bigtop_base import Bigtop # noqa: E402 from charms.reactive import is_state # noqa: E402 -def fail(msg, output=None): - if output: - hookenv.action_set({'output': output}) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() + if not is_state('apache-bigtop-datanode.started'): fail('Charm is not yet ready to run the Bigtop smoke test(s)') @@ -44,6 +44,6 @@ smoke_env = { bigtop = Bigtop() result = bigtop.run_smoke_tests(smoke_components, smoke_env) if result == 'success': - hookenv.action_set({'outcome': result}) + hookenv.action_set({'outcome': 'success'}) else: - fail('{} smoke tests failed'.format(smoke_components), result) + fail('{} smoke tests failed with: {}'.format(smoke_components, result)) diff --git a/bigtop-packages/src/charm/hbase/layer-hbase/actions/perf-test b/bigtop-packages/src/charm/hbase/layer-hbase/actions/perf-test index 2209d37e46..215e0dcc24 100755 --- a/bigtop-packages/src/charm/hbase/layer-hbase/actions/perf-test +++ b/bigtop-packages/src/charm/hbase/layer-hbase/actions/perf-test @@ -42,4 +42,5 @@ benchmark-finish DURATION=`expr $STOP - $START` benchmark-composite "${DURATION}" 'secs' 'asc' -benchmark-raw ${RESULT_LOG} +action-set meta.raw=${RESULT_LOG} +action-set outcome="success" diff --git a/bigtop-packages/src/charm/hbase/layer-hbase/actions/smoke-test b/bigtop-packages/src/charm/hbase/layer-hbase/actions/smoke-test index bc6f45b661..652b70634c 100755 --- a/bigtop-packages/src/charm/hbase/layer-hbase/actions/smoke-test +++ b/bigtop-packages/src/charm/hbase/layer-hbase/actions/smoke-test @@ -44,6 +44,6 @@ echo "HBase shell return code was: " $status if [ $status -eq 0 ]; then action-set outcome="success" else - action-set outcome="failed with return code ${status}" - action-fail "smoke-test failed" + action-set outcome="failure" + action-fail "smoke-test failed with return code ${status}" fi diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic index b402fe8939..159eac71ff 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/create-topic @@ -43,8 +43,9 @@ if host.service_available('kafka-server') and host.service_running('kafka-server '--replication-factor', topic_replication, capture_output=True) except subprocess.CalledProcessError as e: - kafkautils.fail('Kafka command failed', e.output) + kafkautils.fail('Kafka command failed: {}'.format(e.output)) else: - hookenv.action_set({'output': output}) + hookenv.action_set({'raw': output}) + hookenv.action_set({'outcome': 'success'}) else: - kafkautils.fail('Kafka service is not running', 'Please start kafka-server') + kafkautils.fail('kafka-server service is not running') diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py b/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py index 52e529901f..892502723d 100644 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/kafkautils.py @@ -19,8 +19,8 @@ from charmhelpers.core import hookenv -def fail(msg, output=''): - hookenv.action_set({'output': output}) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics index 0ef3a3f027..0241586f6e 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-topics @@ -35,8 +35,9 @@ if host.service_available('kafka-server') and host.service_running('kafka-server '--zookeeper', zookeepers, '--list', capture_output=True) except subprocess.CalledProcessError as e: - kafkautils.fail('Kafka command failed', e.output) + kafkautils.fail('Kafka command failed: {}'.format(e.output)) else: - hookenv.action_set({'output': output}) + hookenv.action_set({'raw': output}) + hookenv.action_set({'outcome': 'success'}) else: - kafkautils.fail('Kafka service is not running') + kafkautils.fail('kafka-server service is not running') diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks index 517d60c0b9..be9541a9f0 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/list-zks @@ -22,15 +22,16 @@ from charms.reactive import is_state if not is_state('kafka.started'): - kafkautils.fail('Kafka service not yet ready', 'Please deploy kafka and required relations') + kafkautils.fail('Kafka service not yet ready') # List zookeepers if kafka is running if host.service_available('kafka-server') and host.service_running('kafka-server'): zookeepers = kafkautils.get_zookeepers() if zookeepers: - hookenv.action_set({'output': zookeepers}) + hookenv.action_set({'raw': zookeepers}) + hookenv.action_set({'outcome': 'success'}) else: - kafkautils.fail('No zookeeper.connect string found', 'Please relate kafka to zookeeper') + kafkautils.fail('No zookeeper.connect string found') else: - kafkautils.fail('Kafka service is not running', 'Please start kafka-server') + kafkautils.fail('kafka-server service is not running') diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic index 8b7a27d6ff..a1053e7c95 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/read-topic @@ -47,8 +47,9 @@ if host.service_available('kafka-server') and host.service_running('kafka-server '--no-wait-at-logend', capture_output=True) except subprocess.CalledProcessError as e: - kafkautils.fail('Kafka command failed', e.output) + kafkautils.fail('Kafka command failed: {}'.format(e.output)) else: - hookenv.action_set({'output': output}) + hookenv.action_set({'raw': output}) + hookenv.action_set({'outcome': 'success'}) else: - kafkautils.fail('Kafka service is not running') + kafkautils.fail('kafka-server service is not running') diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test b/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test index 5c1435a194..e9ac52de5a 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test @@ -48,7 +48,7 @@ if host.service_available('kafka-server') and host.service_running('kafka-server '--replication-factor', topic_replication, capture_output=True) except subprocess.CalledProcessError as e: - kafkautils.fail('Kafka command failed', e.output) + kafkautils.fail('Kafka command failed: {}'.format(e.output)) # List topics try: @@ -56,9 +56,9 @@ if host.service_available('kafka-server') and host.service_running('kafka-server '--zookeeper', zookeepers, '--list', capture_output=True) except subprocess.CalledProcessError as e: - kafkautils.fail('Kafka command failed', e.output) + kafkautils.fail('Kafka command failed: {}'.format(e.output)) # If we haven't failed yet, we passed hookenv.action_set({'outcome': 'success'}) else: - kafkautils.fail('Kafka service is not running') + kafkautils.fail('kafka-server service is not running') diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic index 6a56265df6..813c541d7f 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/write-topic @@ -46,8 +46,9 @@ if host.service_available('kafka-server') and host.service_running('kafka-server capture_output=True, input=bytes(data, 'UTF-8')) except subprocess.CalledProcessError as e: - kafkautils.fail('Kafka command failed', e.output) + kafkautils.fail('Kafka command failed: {}'.format(e.output)) else: - hookenv.action_set({'output': output}) + hookenv.action_set({'raw': output}) + hookenv.action_set({'outcome': 'success'}) else: - kafkautils.fail('Kafka service is not running') + kafkautils.fail('kafka-server service is not running') diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test b/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test index cc98b4615f..f903da4a01 100755 --- a/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test +++ b/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test @@ -62,3 +62,4 @@ hadoop jar /usr/lib/mahout/mahout-mr-*-job.jar \ -s SIMILARITY_LOGLIKELIHOOD 2>&1 | tee -a ${RESULT_LOG} EOF echo 'mahout smoke-test complete' +action-set outcome="success" diff --git a/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test b/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test index 378d3da478..87f128d50d 100755 --- a/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test +++ b/bigtop-packages/src/charm/pig/layer-pig/actions/smoke-test @@ -18,16 +18,15 @@ import sys sys.path.append('lib') -from charmhelpers.core import hookenv -from charms.layer.apache_bigtop_base import Bigtop -from charms.reactive import is_state - -from jujubigdata.utils import run_as - -from path import Path +from charmhelpers.core import hookenv # noqa: E402 +from charms.layer.apache_bigtop_base import Bigtop # noqa: E402 +from charms.reactive import is_state # noqa: E402 +from jujubigdata.utils import run_as # noqa: E402 +from path import Path # noqa: E402 def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() @@ -63,8 +62,8 @@ else: # The smoke tests analyze /etc/passwd. Successful runs will have an 'ubuntu' # passwd entry in the output. +hookenv.action_set({'raw': output}) if 'ubuntu' not in output: - hookenv.action_set({'output': output}) fail('Unable to analyze passwd file with Pig') else: hookenv.action_set({'outcome': 'success'}) diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank b/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank index b2d85e6ace..a54269cc4f 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank @@ -21,17 +21,16 @@ import sys from path import Path from time import time -from charmhelpers.contrib.benchmark import Benchmark from charmhelpers.core import hookenv +from charms.benchmark import Benchmark from charms.reactive import is_state from jujubigdata import utils -def fail(msg, output): - print(msg) - hookenv.action_set({'output': output}) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) - sys.exit(1) + sys.exit() def main(): @@ -39,7 +38,7 @@ def main(): if not is_state('spark.started'): msg = 'Spark is not started yet' - fail(msg, 'error') + fail(msg) # gather params and create dir to store results num_iter = hookenv.action_get('iterations') @@ -54,7 +53,7 @@ def main(): sample = "/home/ubuntu/SparkBench/PageRank/web-Google.txt" if not os.path.isfile(sample): msg = 'Could not find pagerank sample data' - fail('{}: {}'.format(msg, sample), 'error') + fail('{}: {}'.format(msg, sample)) # Benchmark input data is packed into our sparkbench.tgz, which makes # it available on all spark units. In yarn mode, however, the nodemanagers @@ -69,12 +68,12 @@ def main(): capture_output=True) except subprocess.CalledProcessError as e: msg = 'Unable to copy pagerank sample data to hdfs' - fail('{}: {}'.format(msg, e), 'error') + fail('{}: {}'.format(msg, e)) else: sample = "/user/ubuntu/web-Google.txt" else: msg = 'Spark is configured for yarn mode, but HDFS is not ready yet' - fail(msg, 'error') + fail(msg) # find jar location spark_home = "/usr/lib/spark" @@ -86,7 +85,7 @@ def main(): if not example_jar_path: msg = 'Could not find {}'.format(example_jar_name) - fail(msg, 'error') + fail(msg) print('Calculating PageRank') bench.start() @@ -107,14 +106,17 @@ def main(): stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: msg = 'SparkPageRank command failed: {}'.format(' '.join(arg_list)) - fail('{}: {}'.format(msg, e), 'error') + fail('{}: {}'.format(msg, e)) stop = int(time()) bench.finish() duration = stop - start bench.set_composite_score(duration, 'secs') - subprocess.check_call(['benchmark-raw', result_log]) + + # Tell the user how they can get the full result log + scp_log_msg = "juju scp {}:{} .".format(hookenv.local_unit(), result_log) + hookenv.action_set({'meta.fetchcmd': scp_log_msg}) with open(result_log) as log: success = False @@ -125,9 +127,9 @@ def main(): if not success: msg = 'Spark-submit failed to calculate pagerank' - fail(msg, 'error') + fail(msg) - hookenv.action_set({'output': {'status': 'completed'}}) + hookenv.action_set({'outcome': 'success'}) if __name__ == '__main__': diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/restart-spark-job-history-server b/bigtop-packages/src/charm/spark/layer-spark/actions/restart-spark-job-history-server index 411c335b44..331239afe6 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/restart-spark-job-history-server +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/restart-spark-job-history-server @@ -15,22 +15,22 @@ # limitations under the License. import sys +from charmhelpers.core import hookenv, host +from charms.reactive import is_state -try: - from charmhelpers.core import host, hookenv, unitdata - from jujubigdata import utils - charm_ready = True -except ImportError: - charm_ready = False -if not charm_ready: - from subprocess import call - call(['action-fail', 'Spark service not yet ready']) - sys.exit(1) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) + hookenv.action_fail(msg) + sys.exit() + + +if not is_state('spark.started'): + msg = 'Spark is not started yet' + fail(msg) if not host.service_available('spark-history'): - from subprocess import call - call(['action-fail', 'Spark history service not available']) - sys.exit(1) + msg = 'Spark history service not available' + fail(msg) host.service_restart('spark-history-server') diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/sparkbench b/bigtop-packages/src/charm/spark/layer-spark/actions/sparkbench index 906a30e961..3f626e3e6c 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/sparkbench +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/sparkbench @@ -16,8 +16,9 @@ set -ex if ! charms.reactive is_state 'spark.started'; then + action-set outcome="failure" action-fail 'Spark not yet ready' - exit + exit 1 fi # Do not call this script directly. Call it via one of the symlinks. The @@ -138,11 +139,14 @@ if [ -f "${SB_APPS}" ]; then # send raw data (benchmark-raw takes a file) echo ${DATA} > ${RESULT_LOG} benchmark-raw ${RESULT_LOG} + action-set outcome="success" else - echo "ERROR: Invalid benchmark (${BENCHMARK})" + action-set outcome="failure" + action-fail "ERROR: Invalid benchmark (${BENCHMARK})" exit 1 fi else - echo "ERROR: Could not find SparkBench application list" + action-set outcome="failure" + action-fail "ERROR: Could not find SparkBench application list" exit 1 fi diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi b/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi index 99fded6e6b..f94befc474 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi @@ -21,22 +21,23 @@ from path import Path from time import time import subprocess -from charmhelpers.contrib.benchmark import Benchmark from charmhelpers.core import hookenv +from charms.benchmark import Benchmark from charms.reactive import is_state -def fail(msg, output): - hookenv.action_set({'output': output}) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) - sys.exit(1) + sys.exit() def main(): bench = Benchmark() if not is_state('spark.started'): - fail('Spark not yet ready', 'error') + msg = 'Spark is not started yet' + fail(msg) num_partitions = hookenv.action_get('partitions') or '' @@ -64,7 +65,8 @@ def main(): example_jar_path = os.path.join(root, example_jar_name) if not example_jar_path: - fail('could not find {}'.format(example_jar_name), 'error') + msg = 'Could not find {}'.format(example_jar_name) + fail(msg) with open(result_log, 'w') as log_file: arg_list = [ @@ -84,26 +86,32 @@ def main(): except subprocess.CalledProcessError as e: print('smoke test command failed: ') print('{}'.format(' '.join(arg_list))) - fail('spark-submit failed: {}'.format(e), 'error') + fail('spark-submit failed: {}'.format(e)) stop = int(time()) bench.finish() duration = stop - start bench.set_composite_score(duration, 'secs') - subprocess.check_call(['benchmark-raw', result_log]) + + # Tell the user how they can get the full result log + scp_log_msg = "juju scp {}:{} .".format(hookenv.local_unit(), result_log) + hookenv.action_set({'meta.fetchcmd': scp_log_msg}) with open(result_log) as log: success = False for line in log.readlines(): if 'Pi is roughly 3.1' in line: success = True + pi_calc = line break if not success: - fail('spark-submit did not calculate pi', 'error') + msg = 'Spark-submit failed to calculate pi' + fail(msg) - hookenv.action_set({'output': {'status': 'completed'}}) + hookenv.action_set({'meta.raw': pi_calc}) + hookenv.action_set({'outcome': 'success'}) if __name__ == '__main__': diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/start-spark-job-history-server b/bigtop-packages/src/charm/spark/layer-spark/actions/start-spark-job-history-server index 9677a383e8..cc218dbbbc 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/start-spark-job-history-server +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/start-spark-job-history-server @@ -15,22 +15,22 @@ # limitations under the License. import sys +from charmhelpers.core import hookenv, host +from charms.reactive import is_state -try: - from charmhelpers.core import host, hookenv, unitdata - from jujubigdata import utils - charm_ready = True -except ImportError: - charm_ready = False -if not charm_ready: - from subprocess import call - call(['action-fail', 'Spark service not yet ready']) - sys.exit(1) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) + hookenv.action_fail(msg) + sys.exit() + + +if not is_state('spark.started'): + msg = 'Spark is not started yet' + fail(msg) if not host.service_available('spark-history-server'): - from subprocess import call - call(['action-fail', 'Spark history service not available']) - sys.exit(1) + msg = 'Spark history service not available' + fail(msg) host.service_start('spark-history-server') diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/stop-spark-job-history-server b/bigtop-packages/src/charm/spark/layer-spark/actions/stop-spark-job-history-server index fbe41cf9fa..368b1e9177 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/stop-spark-job-history-server +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/stop-spark-job-history-server @@ -15,22 +15,22 @@ # limitations under the License. import sys +from charmhelpers.core import hookenv, host +from charms.reactive import is_state -try: - from charmhelpers.core import host, hookenv, unitdata - from jujubigdata import utils - charm_ready = True -except ImportError: - charm_ready = False -if not charm_ready: - from subprocess import call - call(['action-fail', 'Spark service not yet ready']) - sys.exit(1) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) + hookenv.action_fail(msg) + sys.exit() + + +if not is_state('spark.started'): + msg = 'Spark is not started yet' + fail(msg) if not host.service_available('spark-history-server'): - from subprocess import call - call(['action-fail', 'Spark history service not available']) - sys.exit(1) + msg = 'Spark history service not available' + fail(msg) host.service_stop('spark-history-server') diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/submit b/bigtop-packages/src/charm/spark/layer-spark/actions/submit index a25a7af1a5..ece0b20609 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/submit +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/submit @@ -16,8 +16,9 @@ set -e if ! charms.reactive is_state 'spark.started'; then + action-set outcome="failure" action-fail 'Spark not yet ready' - exit + exit 1 fi py_files="$(action-get py-files)" diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/restart b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/restart index bb37376f95..6e822b440f 100755 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/restart +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/restart @@ -18,12 +18,13 @@ import sys sys.path.append('lib') -from charmhelpers.core import hookenv -from charms.reactive import is_state -from charms.layer.bigtop_zeppelin import Zeppelin +from charmhelpers.core import hookenv # noqa: E402 +from charms.reactive import is_state # noqa: E402 +from charms.layer.bigtop_zeppelin import Zeppelin # noqa: E402 def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test index 0339d46509..823580d9f8 100755 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test @@ -28,6 +28,7 @@ from charms.reactive import is_state def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() @@ -91,3 +92,5 @@ try: fail('Notebook failed: {}'.format(errmsg)) except requests.exceptions.RequestException as e: fail('Request failed: {}: {}'.format(e.request.url, e)) + +hookenv.action_set({'outcome': 'success'}) diff --git a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/restart b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/restart index 7bb5136441..acbd448bed 100755 --- a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/restart +++ b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/restart @@ -22,18 +22,15 @@ from charmhelpers.core import hookenv # noqa: E402 from charms.layer.bigtop_zookeeper import Zookeeper # noqa: E402 from charms.reactive import is_state # noqa: E402 -LOG = hookenv.log - -LOG("starting restart handler.") - def main(): if not is_state('zookeeper.started'): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail('Cannot restart: Zookeeper has not yet started!') - return + sys.exit() - LOG("starting restart handler main.") - hookenv.status_set('maintenance', 'restarting ...') + hookenv.log("starting restart handler main.") + hookenv.status_set('maintenance', 'restarting zookeeper') zookeeper = Zookeeper() zookeeper.install() hookenv.status_set('active', 'ready {}'.format(zookeeper.quorum_check())) diff --git a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/smoke-test b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/smoke-test index 64814629a1..0f72ca0abb 100755 --- a/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/smoke-test +++ b/bigtop-packages/src/charm/zookeeper/layer-zookeeper/actions/smoke-test @@ -23,12 +23,12 @@ from charms.layer.apache_bigtop_base import Bigtop # noqa: E402 from charms.reactive import is_state # noqa: E402 -def fail(msg, output=None): - if output: - hookenv.action_set({'output': output}) +def fail(msg): + hookenv.action_set({'outcome': 'failure'}) hookenv.action_fail(msg) sys.exit() + if not is_state('zookeeper.started'): fail('Charm is not yet ready to run the Bigtop smoke test(s)') @@ -38,6 +38,6 @@ smoke_components = ['zookeeper'] bigtop = Bigtop() result = bigtop.run_smoke_tests(smoke_components) if result == 'success': - hookenv.action_set({'outcome': result}) + hookenv.action_set({'outcome': 'success'}) else: - fail('{} smoke tests failed'.format(smoke_components), result) + fail('{} smoke tests failed with: {}'.format(smoke_components, result)) From a7192cd506b124c711028bfee2aaaa8bfc1518ee Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Fri, 21 Apr 2017 15:35:42 +0000 Subject: [PATCH 05/13] update charm instructions for 2.0 --- bigtop-packages/src/charm/README.md | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/bigtop-packages/src/charm/README.md b/bigtop-packages/src/charm/README.md index 04213ad8ba..cdc50e44b8 100644 --- a/bigtop-packages/src/charm/README.md +++ b/bigtop-packages/src/charm/README.md @@ -28,9 +28,10 @@ Bigtop components. Built charms are published to the [Juju charm store][] and can be deployed directly from there, either individually or with [bundles][]. They can also be built from these layers and deployed locally. -For the remainder of this guide, a working Juju installation is assumed to be -present. If Juju is not yet set up, please follow the [getting-started][] -instructions prior to deploying locally built charms and bundles. +For the remainder of this guide, a working Juju 2.0 or greater installation is +assumed to be present. If Juju is not yet set up, please follow the +[getting-started][] instructions prior to deploying locally built charms and +bundles. [Juju charm store]: https://jujucharms.com/ [bundles]: https://jujucharms.com/hadoop-processing @@ -48,11 +49,12 @@ For example: mkdir $JUJU_REPOSITORY cd bigtop-packages/src/charms/hadoop/layer-hadoop-namenode - charm build + charm build --series xenial --report -This will build the NameNode charm, pulling in the appropriate base and -interface layers from [interfaces.juju.solutions][]. You can get local copies -of those layers as well by using `charm pull-source`: +This will build the NameNode charm in the +$JUJU_REPOSITORY/xenial/hadoop-namenode directory, pulling in the appropriate +base and interface layers from [interfaces.juju.solutions][]. You can get +local copies of those layers as well by using `charm pull-source`: export LAYER_PATH=$HOME/layers export INTERFACE_PATH=$HOME/interfaces @@ -65,17 +67,10 @@ You can deploy the locally built charms individually, for example: juju deploy $JUJU_REPOSITORY/xenial/hadoop-namenode -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is: `juju deploy local:xenial/hadoop-namenode`. - You can also deploy the local version of a bundle: juju deploy ./bigtop-deploy/juju/hadoop-processing/bundle-local.yaml -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -./bigtop-deploy/juju/hadoop-processing/bundle-local.yaml`. - [charm-tools]: https://jujucharms.com/docs/stable/tools-charm-tools [Getting Started]: https://jujucharms.com/docs/stable/developer-getting-started [interfaces.juju.solutions]: http://interfaces.juju.solutions/ From 0775fbe3d1495e35c6b098e9e0d93b67bc48e5f5 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Fri, 21 Apr 2017 16:14:46 -0500 Subject: [PATCH 06/13] update charm revs; update readmes to remove juju-1 syntax and include app versions and issue tracking section --- bigtop-deploy/juju/hadoop-hbase/README.md | 67 +++++---- bigtop-deploy/juju/hadoop-hbase/bundle.yaml | 12 +- bigtop-deploy/juju/hadoop-kafka/README.md | 49 +++---- bigtop-deploy/juju/hadoop-kafka/bundle.yaml | 12 +- .../juju/hadoop-processing/README.md | 45 +++--- .../juju/hadoop-processing/bundle.yaml | 8 +- bigtop-deploy/juju/hadoop-spark/README.md | 134 +++++++----------- bigtop-deploy/juju/hadoop-spark/bundle.yaml | 42 ++---- bigtop-deploy/juju/spark-processing/README.md | 109 ++++++-------- .../juju/spark-processing/bundle.yaml | 4 +- 10 files changed, 192 insertions(+), 290 deletions(-) diff --git a/bigtop-deploy/juju/hadoop-hbase/README.md b/bigtop-deploy/juju/hadoop-hbase/README.md index b45bf7bc2b..2825327799 100644 --- a/bigtop-deploy/juju/hadoop-hbase/README.md +++ b/bigtop-deploy/juju/hadoop-hbase/README.md @@ -26,14 +26,20 @@ to deliver high-availability, Hadoop can detect and handle failures at the application layer. This provides a highly-available service on top of a cluster of machines, each of which may be prone to failure. -HBase is the Hadoop database. Think of it as a distributed, scalable Big Data -store. +Apache HBase is the Hadoop database. Think of it as a distributed, scalable +Big Data store. -This bundle provides a complete deployment of Hadoop and HBase components from -[Apache Bigtop][] that performs distributed data processing at scale. Ganglia -and rsyslog applications are also provided to monitor cluster health and syslog -activity. +Use HBase when you need random, realtime read/write access to your Big Data. +This project's goal is the hosting of very large tables -- billions of rows X +millions of columns -- atop clusters of commodity hardware. Learn more at +[hbase.apache.org][]. +This bundle provides a complete deployment of Hadoop and HBase components +from [Apache Bigtop][] that performs distributed data processing at scale. +Ganglia and rsyslog applications are also provided to monitor cluster health +and syslog activity. + +[hbase.apache.org]: http://hbase.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ ## Bundle Composition @@ -41,14 +47,14 @@ activity. The applications that comprise this bundle are spread across 8 units as follows: - * NameNode (HDFS) - * ResourceManager (YARN) + * NameNode v2.7.3 + * ResourceManager v2.7.3 * Colocated on the NameNode unit - * Zookeeper + * Zookeeper v3.4.6 * 3 separate units - * Slave (DataNode and NodeManager) + * Slave (DataNode and NodeManager) v2.7.3 * 3 separate units - * HBase + * HBase v1.1.9 * 3 units colocated with the Hadoop Slaves * Client (Hadoop endpoint) * Plugin (Facilitates communication with the Hadoop cluster) @@ -65,9 +71,8 @@ demands. # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -bundle. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this bundle. > **Note**: This bundle requires hardware resources that may exceed limits of Free-tier or Trial accounts on some clouds. To deploy to these @@ -79,18 +84,10 @@ Deploy this bundle from the Juju charm store with the `juju deploy` command: juju deploy hadoop-hbase -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-hbase`. - Alternatively, deploy a locally modified `bundle.yaml` with: juju deploy /path/to/bundle.yaml -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -/path/to/bundle.yaml`. - The charms in this bundle can also be built from their source layers in the [Bigtop charm repository][]. See the [Bigtop charm README][] for instructions on building and deploying these charms locally. @@ -102,7 +99,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [bundle.yaml]: https://github.com/apache/bigtop/blob/master/bigtop-deploy/juju/hadoop-hbase/bundle.yaml -[juju-quickstart]: https://launchpad.net/juju-quickstart [Bigtop charm repository]: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm [Bigtop charm README]: https://github.com/apache/bigtop/blob/master/bigtop-packages/src/charm/README.md [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -138,25 +134,16 @@ complete. Run the smoke-test actions as follows: juju run-action hbase/0 smoke-test juju run-action zookeeper/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do /0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, all of the actions should settle to `status: completed`. If any report `status: failed`, that application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities Applications in this bundle include command line and web utilities that can be used to verify information about the cluster. @@ -314,6 +301,18 @@ Multiple units may be added at once. For example, add four more slave units: juju add-unit -n4 slave +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this bundle at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Ensure `Bigtop` is selected as the project. Typically, bundle issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. + + # Contact Information - @@ -324,6 +323,6 @@ Multiple units may be added at once. For example, add four more slave units: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-deploy/juju/hadoop-hbase/bundle.yaml b/bigtop-deploy/juju/hadoop-hbase/bundle.yaml index 9e3c823a76..618b196a40 100644 --- a/bigtop-deploy/juju/hadoop-hbase/bundle.yaml +++ b/bigtop-deploy/juju/hadoop-hbase/bundle.yaml @@ -1,6 +1,6 @@ services: namenode: - charm: "cs:xenial/hadoop-namenode-13" + charm: "cs:xenial/hadoop-namenode-14" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -9,7 +9,7 @@ services: to: - "0" resourcemanager: - charm: "cs:xenial/hadoop-resourcemanager-14" + charm: "cs:xenial/hadoop-resourcemanager-16" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -18,7 +18,7 @@ services: to: - "0" slave: - charm: "cs:xenial/hadoop-slave-13" + charm: "cs:xenial/hadoop-slave-15" constraints: "mem=7G root-disk=32G" num_units: 3 annotations: @@ -29,7 +29,7 @@ services: - "2" - "3" plugin: - charm: "cs:xenial/hadoop-plugin-13" + charm: "cs:xenial/hadoop-plugin-14" annotations: gui-x: "1000" gui-y: "400" @@ -43,7 +43,7 @@ services: to: - "4" hbase: - charm: "cs:xenial/hbase-11" + charm: "cs:xenial/hbase-14" constraints: "mem=7G root-disk=32G" num_units: 3 annotations: @@ -54,7 +54,7 @@ services: - "2" - "3" zookeeper: - charm: "cs:xenial/zookeeper-17" + charm: "cs:xenial/zookeeper-19" constraints: "mem=3G root-disk=32G" num_units: 3 annotations: diff --git a/bigtop-deploy/juju/hadoop-kafka/README.md b/bigtop-deploy/juju/hadoop-kafka/README.md index 4c7a581808..4caf77441c 100644 --- a/bigtop-deploy/juju/hadoop-kafka/README.md +++ b/bigtop-deploy/juju/hadoop-kafka/README.md @@ -44,15 +44,15 @@ activity. The applications that comprise this bundle are spread across 9 units as follows: - * NameNode (HDFS) - * ResourceManager (YARN) + * NameNode v2.7.3 + * ResourceManager v2.7.3 * Colocated on the NameNode unit - * Slave (DataNode and NodeManager) + * Slave (DataNode and NodeManager) v2.7.3 * 3 separate units - * Kafka + * Kafka v0.10.1 * Flume-Kafka * Colocated on the Kafka unit - * Zookeeper + * Zookeeper v3.4.6 * 3 separate units * Client (Hadoop endpoint) * Plugin (Facilitates communication with the Hadoop cluster) @@ -76,9 +76,8 @@ demands. # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -bundle. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this bundle. > **Note**: This bundle requires hardware resources that may exceed limits of Free-tier or Trial accounts on some clouds. To deploy to these @@ -90,18 +89,10 @@ Deploy this bundle from the Juju charm store with the `juju deploy` command: juju deploy hadoop-kafka -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-kafka`. - Alternatively, deploy a locally modified `bundle.yaml` with: juju deploy /path/to/bundle.yaml -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -/path/to/bundle.yaml`. - The charms in this bundle can also be built from their source layers in the [Bigtop charm repository][]. See the [Bigtop charm README][] for instructions on building and deploying these charms locally. @@ -113,7 +104,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [bundle.yaml]: https://github.com/apache/bigtop/blob/master/bigtop-deploy/juju/hadoop-kafka/bundle.yaml -[juju-quickstart]: https://launchpad.net/juju-quickstart [Bigtop charm repository]: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm [Bigtop charm README]: https://github.com/apache/bigtop/blob/master/bigtop-packages/src/charm/README.md [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -166,25 +156,16 @@ complete. Run the smoke-test actions as follows: juju run-action kafka/0 smoke-test juju run-action zookeeper/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do /0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, all of the actions should settle to `status: completed`. If any report `status: failed`, that application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities Applications in this bundle include command line and web utilities that can be used to verify information about the cluster. @@ -314,6 +295,18 @@ Multiple units may be added at once. For example, add four more slave units: juju add-unit -n4 slave +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this bundle at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Ensure `Bigtop` is selected as the project. Typically, bundle issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. + + # Contact Information - @@ -324,6 +317,6 @@ Multiple units may be added at once. For example, add four more slave units: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-deploy/juju/hadoop-kafka/bundle.yaml b/bigtop-deploy/juju/hadoop-kafka/bundle.yaml index 948c8896cd..6694d20d7c 100644 --- a/bigtop-deploy/juju/hadoop-kafka/bundle.yaml +++ b/bigtop-deploy/juju/hadoop-kafka/bundle.yaml @@ -1,6 +1,6 @@ services: namenode: - charm: "cs:xenial/hadoop-namenode-13" + charm: "cs:xenial/hadoop-namenode-14" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -9,7 +9,7 @@ services: to: - "0" resourcemanager: - charm: "cs:xenial/hadoop-resourcemanager-14" + charm: "cs:xenial/hadoop-resourcemanager-16" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -18,7 +18,7 @@ services: to: - "0" slave: - charm: "cs:xenial/hadoop-slave-13" + charm: "cs:xenial/hadoop-slave-15" constraints: "mem=7G root-disk=32G" num_units: 3 annotations: @@ -29,7 +29,7 @@ services: - "2" - "3" plugin: - charm: "cs:xenial/hadoop-plugin-13" + charm: "cs:xenial/hadoop-plugin-14" annotations: gui-x: "1000" gui-y: "400" @@ -52,7 +52,7 @@ services: to: - "4" zookeeper: - charm: "cs:xenial/zookeeper-17" + charm: "cs:xenial/zookeeper-19" constraints: "mem=3G root-disk=32G" num_units: 3 annotations: @@ -63,7 +63,7 @@ services: - "6" - "7" kafka: - charm: "cs:xenial/kafka-12" + charm: "cs:xenial/kafka-14" constraints: "mem=3G" num_units: 1 annotations: diff --git a/bigtop-deploy/juju/hadoop-processing/README.md b/bigtop-deploy/juju/hadoop-processing/README.md index 896a7936e2..9ad44b6ae9 100644 --- a/bigtop-deploy/juju/hadoop-processing/README.md +++ b/bigtop-deploy/juju/hadoop-processing/README.md @@ -38,10 +38,10 @@ and syslog activity. The applications that comprise this bundle are spread across 5 machines as follows: - * NameNode (HDFS) - * ResourceManager (YARN) + * NameNode v2.7.3 + * ResourceManager v2.7.3 * Colocated on the NameNode unit - * Slave (DataNode and NodeManager) + * Slave (DataNode and NodeManager) v2.7.3 * 3 separate units * Client (Hadoop endpoint) * Plugin (Facilitates communication with the Hadoop cluster) @@ -57,9 +57,8 @@ demands. # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -bundle. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this bundle. > **Note**: This bundle requires hardware resources that may exceed limits of Free-tier or Trial accounts on some clouds. To deploy to these @@ -70,18 +69,10 @@ Deploy this bundle from the Juju charm store with the `juju deploy` command: juju deploy hadoop-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-processing`. - Alternatively, deploy a locally modified `bundle.yaml` with: juju deploy /path/to/bundle.yaml -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -/path/to/bundle.yaml`. - The charms in this bundle can also be built from their source layers in the [Bigtop charm repository][]. See the [Bigtop charm README][] for instructions on building and deploying these charms locally. @@ -93,7 +84,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [bundle.yaml]: https://github.com/apache/bigtop/blob/master/bigtop-deploy/juju/hadoop-processing/bundle.yaml -[juju-quickstart]: https://launchpad.net/juju-quickstart [Bigtop charm repository]: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm [Bigtop charm README]: https://github.com/apache/bigtop/blob/master/bigtop-packages/src/charm/README.md [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -127,25 +117,16 @@ Run the smoke-test actions as follows: juju run-action resourcemanager/0 smoke-test juju run-action slave/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do /0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, all of the actions should settle to `status: completed`. If any report `status: failed`, that application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities Applications in this bundle include Hadoop command line and web utilities that can be used to verify information about the cluster. @@ -270,6 +251,18 @@ Multiple units may be added at once. For example, add four more slave units: juju add-unit -n4 slave +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this bundle at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Ensure `Bigtop` is selected as the project. Typically, bundle issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. + + # Contact Information - @@ -280,6 +273,6 @@ Multiple units may be added at once. For example, add four more slave units: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-deploy/juju/hadoop-processing/bundle.yaml b/bigtop-deploy/juju/hadoop-processing/bundle.yaml index c4d8005eec..ef5195aeeb 100644 --- a/bigtop-deploy/juju/hadoop-processing/bundle.yaml +++ b/bigtop-deploy/juju/hadoop-processing/bundle.yaml @@ -1,6 +1,6 @@ services: namenode: - charm: "cs:xenial/hadoop-namenode-13" + charm: "cs:xenial/hadoop-namenode-14" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -9,7 +9,7 @@ services: to: - "0" resourcemanager: - charm: "cs:xenial/hadoop-resourcemanager-14" + charm: "cs:xenial/hadoop-resourcemanager-16" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -18,7 +18,7 @@ services: to: - "0" slave: - charm: "cs:xenial/hadoop-slave-13" + charm: "cs:xenial/hadoop-slave-15" constraints: "mem=7G root-disk=32G" num_units: 3 annotations: @@ -29,7 +29,7 @@ services: - "2" - "3" plugin: - charm: "cs:xenial/hadoop-plugin-13" + charm: "cs:xenial/hadoop-plugin-14" annotations: gui-x: "1000" gui-y: "400" diff --git a/bigtop-deploy/juju/hadoop-spark/README.md b/bigtop-deploy/juju/hadoop-spark/README.md index cc956e9d6a..13905b3d1e 100644 --- a/bigtop-deploy/juju/hadoop-spark/README.md +++ b/bigtop-deploy/juju/hadoop-spark/README.md @@ -26,35 +26,36 @@ to deliver high-availability, Hadoop can detect and handle failures at the application layer. This provides a highly-available service on top of a cluster of machines, each of which may be prone to failure. -Spark is a fast and general engine for large-scale data processing. +Apache Spark is a fast and general engine for large-scale data processing. +Learn more at [spark.apache.org][]. This bundle provides a complete deployment of Hadoop and Spark components from [Apache Bigtop][] that performs distributed data processing at scale. Ganglia and rsyslog applications are also provided to monitor cluster health and syslog activity. +[spark.apache.org]: http://spark.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ ## Bundle Composition -The applications that comprise this bundle are spread across 9 units as +The applications that comprise this bundle are spread across 5 units as follows: - * NameNode (HDFS) - * ResourceManager (YARN) + * NameNode v2.7.3 + * ResourceManager v2.7.3 * Colocated on the NameNode unit - * Slave (DataNode and NodeManager) - * 3 separate units - * Spark (Master in yarn-client mode) - * Zookeeper + * Slave (DataNode and NodeManager) v2.7.3 * 3 separate units + * Spark (Driver in yarn-client mode) v2.1.0 * Client (Hadoop endpoint) + * Colocated on the Spark unit * Plugin (Facilitates communication with the Hadoop cluster) - * Colocated on the Spark and Client units + * Colocated on the Spark/Client unit * Ganglia (Web interface for monitoring cluster metrics) - * Colocated on the Client unit + * Colocated on the Spark/Client unit * Rsyslog (Aggregate cluster syslog events in a single location) - * Colocated on the Client unit + * Colocated on the Spark/Client unit Deploying this bundle results in a fully configured Apache Bigtop cluster on any supported cloud, which can be scaled to meet workload @@ -63,9 +64,8 @@ demands. # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -bundle. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this bundle. > **Note**: This bundle requires hardware resources that may exceed limits of Free-tier or Trial accounts on some clouds. To deploy to these @@ -77,18 +77,10 @@ Deploy this bundle from the Juju charm store with the `juju deploy` command: juju deploy hadoop-spark -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -hadoop-spark`. - Alternatively, deploy a locally modified `bundle.yaml` with: juju deploy /path/to/bundle.yaml -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -/path/to/bundle.yaml`. - The charms in this bundle can also be built from their source layers in the [Bigtop charm repository][]. See the [Bigtop charm README][] for instructions on building and deploying these charms locally. @@ -100,7 +92,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [bundle.yaml]: https://github.com/apache/bigtop/blob/master/bigtop-deploy/juju/hadoop-spark/bundle.yaml -[juju-quickstart]: https://launchpad.net/juju-quickstart [Bigtop charm repository]: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm [Bigtop charm README]: https://github.com/apache/bigtop/blob/master/bigtop-packages/src/charm/README.md [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -124,8 +115,8 @@ Once they all indicate that they are ready, perform application smoke tests to verify that the bundle is working as expected. ## Smoke Test -The charms for each core component (namenode, resourcemanager, slave, spark, -and zookeeper) provide a `smoke-test` action that can be used to verify the +The charms for each core component (namenode, resourcemanager, slave, and +spark) provide a `smoke-test` action that can be used to verify the application is functioning as expected. Note that the 'slave' component runs extensive tests provided by Apache Bigtop and may take up to 30 minutes to complete. Run the smoke-test actions as follows: @@ -134,27 +125,17 @@ complete. Run the smoke-test actions as follows: juju run-action resourcemanager/0 smoke-test juju run-action slave/0 smoke-test juju run-action spark/0 smoke-test - juju run-action zookeeper/0 smoke-test - -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do /0 smoke-test`. Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, all of the actions should settle to `status: completed`. If any report `status: failed`, that application is not working as expected. Get more information about a specific smoke test with: juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities Applications in this bundle include command line and web utilities that can be used to verify information about the cluster. @@ -165,10 +146,6 @@ of YARN NodeManager units with the following: juju run --application namenode "su hdfs -c 'hdfs dfsadmin -report'" juju run --application resourcemanager "su yarn -c 'yarn node -list'" -Show the list of Zookeeper nodes with the following: - - juju run --unit zookeeper/0 'echo "ls /" | /usr/lib/zookeeper/bin/zkCli.sh' - To access the HDFS web console, find the `PUBLIC-ADDRESS` of the namenode application and expose it: @@ -204,7 +181,7 @@ The web interface will be available at the following URL: # Monitoring This bundle includes Ganglia for system-level monitoring of the namenode, -resourcemanager, slave, spark, and zookeeper units. Metrics are sent to a +resourcemanager, slave, and spark units. Metrics are sent to a centralized ganglia unit for easy viewing in a browser. To view the ganglia web interface, find the `PUBLIC-ADDRESS` of the Ganglia application and expose it: @@ -219,7 +196,7 @@ The web interface will be available at: # Logging This bundle includes rsyslog to collect syslog data from the namenode, -resourcemanager, slave, spark, and zookeeper units. These logs are sent to a +resourcemanager, slave, and spark units. These logs are sent to a centralized rsyslog unit for easy syslog analysis. One method of viewing this log data is to simply cat syslog from the rsyslog unit: @@ -278,27 +255,17 @@ run with `juju run-action`: enqueued: 2016-02-04 14:55:14 +0000 UTC started: 2016-02-04 14:55:27 +0000 UTC -The `spark` charm in this bundle also provides several benchmarks to gauge -the performance of the Spark cluster. Each benchmark is an action that can be -run with `juju run-action`: +The `spark` charm in this bundle provides benchmarks to gauge the performance +of the Spark/YARN cluster. Each benchmark is an action that can be run with +`juju run-action`: - $ juju actions spark | grep Bench - connectedcomponent Run the Spark Bench ConnectedComponent benchmark. - decisiontree Run the Spark Bench DecisionTree benchmark. - kmeans Run the Spark Bench KMeans benchmark. - linearregression Run the Spark Bench LinearRegression benchmark. - logisticregression Run the Spark Bench LogisticRegression benchmark. - matrixfactorization Run the Spark Bench MatrixFactorization benchmark. - pagerank Run the Spark Bench PageRank benchmark. - pca Run the Spark Bench PCA benchmark. - pregeloperation Run the Spark Bench PregelOperation benchmark. - shortestpaths Run the Spark Bench ShortestPaths benchmark. - sql Run the Spark Bench SQL benchmark. - stronglyconnectedcomponent Run the Spark Bench StronglyConnectedComponent benchmark. - svdplusplus Run the Spark Bench SVDPlusPlus benchmark. - svm Run the Spark Bench SVM benchmark. - - $ juju run-action spark/0 svdplusplus + $ juju actions spark + ... + pagerank Calculate PageRank for a sample data set + sparkpi Calculate Pi + ... + + $ juju run-action spark/0 pagerank Action queued with id: 339cec1f-e903-4ee7-85ca-876fb0c3d28e $ juju show-action-output 339cec1f-e903-4ee7-85ca-876fb0c3d28e @@ -307,40 +274,41 @@ run with `juju run-action`: composite: direction: asc units: secs - value: "200.754000" - raw: | - SVDPlusPlus,2016-11-02-03:08:26,200.754000,85.974071,.428255,0,SVDPlusPlus-MLlibConfig,,,,,10,,,50000,4.0,1.3, - start: 2016-11-02T03:08:26Z - stop: 2016-11-02T03:11:47Z - results: - duration: - direction: asc - units: secs - value: "200.754000" - throughput: - direction: desc - units: MB/sec - value: ".428255" + value: "83" + start: 2017-04-12T23:22:38Z + stop: 2017-04-12T23:24:01Z + output: '{''status'': ''completed''}' status: completed timing: - completed: 2016-11-02 03:11:48 +0000 UTC - enqueued: 2016-11-02 03:08:21 +0000 UTC - started: 2016-11-02 03:08:26 +0000 UTC + completed: 2017-04-12 23:24:02 +0000 UTC + enqueued: 2017-04-12 23:22:36 +0000 UTC + started: 2017-04-12 23:22:37 +0000 UTC # Scaling -By default, three Hadoop slave and three zookeeper units are deployed. Scaling -these applications is as simple as adding more units. To add one unit: +By default, three Hadoop slave units are deployed. Scaling these is as simple +as adding more units. To add one unit: juju add-unit slave - juju add-unit zookeeper Multiple units may be added at once. For example, add four more slave units: juju add-unit -n4 slave +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this bundle at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Ensure `Bigtop` is selected as the project. Typically, bundle issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. + + # Contact Information - @@ -351,6 +319,6 @@ Multiple units may be added at once. For example, add four more slave units: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-deploy/juju/hadoop-spark/bundle.yaml b/bigtop-deploy/juju/hadoop-spark/bundle.yaml index 500ae78a16..0701f27975 100644 --- a/bigtop-deploy/juju/hadoop-spark/bundle.yaml +++ b/bigtop-deploy/juju/hadoop-spark/bundle.yaml @@ -1,6 +1,6 @@ services: namenode: - charm: "cs:xenial/hadoop-namenode-13" + charm: "cs:xenial/hadoop-namenode-14" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -9,7 +9,7 @@ services: to: - "0" resourcemanager: - charm: "cs:xenial/hadoop-resourcemanager-14" + charm: "cs:xenial/hadoop-resourcemanager-16" constraints: "mem=7G root-disk=32G" num_units: 1 annotations: @@ -18,7 +18,7 @@ services: to: - "0" slave: - charm: "cs:xenial/hadoop-slave-13" + charm: "cs:xenial/hadoop-slave-15" constraints: "mem=7G root-disk=32G" num_units: 3 annotations: @@ -29,13 +29,13 @@ services: - "2" - "3" plugin: - charm: "cs:xenial/hadoop-plugin-13" + charm: "cs:xenial/hadoop-plugin-14" annotations: gui-x: "1000" gui-y: "400" client: charm: "cs:xenial/hadoop-client-3" - constraints: "mem=3G" + constraints: "mem=7G root-disk=32G" num_units: 1 annotations: gui-x: "1250" @@ -43,7 +43,7 @@ services: to: - "4" spark: - charm: "cs:xenial/spark-31" + charm: "cs:xenial/spark-33" constraints: "mem=7G root-disk=32G" num_units: 1 options: @@ -52,18 +52,7 @@ services: gui-x: "1000" gui-y: "0" to: - - "5" - zookeeper: - charm: "cs:xenial/zookeeper-17" - constraints: "mem=3G root-disk=32G" - num_units: 3 - annotations: - gui-x: "500" - gui-y: "400" - to: - - "6" - - "7" - - "8" + - "4" ganglia: charm: "cs:xenial/ganglia-12" num_units: 1 @@ -97,20 +86,17 @@ relations: - [resourcemanager, slave] - [plugin, namenode] - [plugin, resourcemanager] - - [client, plugin] - [spark, plugin] - - [spark, zookeeper] + - [client, plugin] - ["ganglia-node:juju-info", "namenode:juju-info"] - ["ganglia-node:juju-info", "resourcemanager:juju-info"] - ["ganglia-node:juju-info", "slave:juju-info"] - ["ganglia-node:juju-info", "spark:juju-info"] - - ["ganglia-node:juju-info", "zookeeper:juju-info"] - ["ganglia:node", "ganglia-node:node"] - ["rsyslog-forwarder-ha:juju-info", "namenode:juju-info"] - ["rsyslog-forwarder-ha:juju-info", "resourcemanager:juju-info"] - ["rsyslog-forwarder-ha:juju-info", "slave:juju-info"] - ["rsyslog-forwarder-ha:juju-info", "spark:juju-info"] - - ["rsyslog-forwarder-ha:juju-info", "zookeeper:juju-info"] - ["rsyslog:aggregator", "rsyslog-forwarder-ha:syslog"] machines: "0": @@ -126,17 +112,5 @@ machines: series: "xenial" constraints: "mem=7G root-disk=32G" "4": - series: "xenial" - constraints: "mem=3G" - "5": series: "xenial" constraints: "mem=7G root-disk=32G" - "6": - series: "xenial" - constraints: "mem=3G root-disk=32G" - "7": - series: "xenial" - constraints: "mem=3G root-disk=32G" - "8": - series: "xenial" - constraints: "mem=3G root-disk=32G" diff --git a/bigtop-deploy/juju/spark-processing/README.md b/bigtop-deploy/juju/spark-processing/README.md index c39fa2c872..a499a3842a 100644 --- a/bigtop-deploy/juju/spark-processing/README.md +++ b/bigtop-deploy/juju/spark-processing/README.md @@ -16,12 +16,14 @@ --> # Overview -This bundle provides a complete deployment of -[Apache Spark][] in standalone HA mode as provided -by [Apache Bigtop][]. Ganglia and rsyslog +Apache Spark is a fast and general engine for large-scale data processing. +Learn more at [spark.apache.org][]. + +This bundle provides a complete deployment of Spark (in standalone HA mode) +and Apache Zookeeper components from [Apache Bigtop][]. Ganglia and rsyslog applications are included to monitor cluster health and syslog activity. -[Apache Spark]: http://spark/apache.org/ +[spark.apache.org]: http://spark.apache.org/ [Apache Bigtop]: http://bigtop.apache.org/ ## Bundle Composition @@ -29,9 +31,9 @@ applications are included to monitor cluster health and syslog activity. The applications that comprise this bundle are spread across 6 units as follows: - * Spark (Master and Worker) + * Spark (Master and Worker) v2.1.0 * 2 separate units - * Zookeeper + * Zookeeper v3.4.6 * 3 separate units * Ganglia (Web interface for monitoring cluster metrics) * Rsyslog (Aggregate cluster syslog events in a single location) @@ -44,9 +46,8 @@ demands. # Deploying -A working Juju installation is assumed to be present. If Juju is not yet set -up, please follow the [getting-started][] instructions prior to deploying this -bundle. +This charm requires Juju 2.0 or greater. If Juju is not yet set up, please +follow the [getting-started][] instructions prior to deploying this bundle. > **Note**: This bundle requires hardware resources that may exceed limits of Free-tier or Trial accounts on some clouds. To deploy to these @@ -58,18 +59,10 @@ Deploy this bundle from the Juju charm store with the `juju deploy` command: juju deploy spark-processing -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -spark-processing`. - Alternatively, deploy a locally modified `bundle.yaml` with: juju deploy /path/to/bundle.yaml -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart -/path/to/bundle.yaml`. - The charms in this bundle can also be built from their source layers in the [Bigtop charm repository][]. See the [Bigtop charm README][] for instructions on building and deploying these charms locally. @@ -81,7 +74,6 @@ mirror options. See [Configuring Models][] for more information. [getting-started]: https://jujucharms.com/docs/stable/getting-started [bundle.yaml]: https://github.com/apache/bigtop/blob/master/bigtop-deploy/juju/spark-processing/bundle.yaml -[juju-quickstart]: https://launchpad.net/juju-quickstart [Bigtop charm repository]: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm [Bigtop charm README]: https://github.com/apache/bigtop/blob/master/bigtop-packages/src/charm/README.md [Configuring Models]: https://jujucharms.com/docs/stable/models-config @@ -112,25 +104,16 @@ actions as follows: juju run-action spark/0 smoke-test juju run-action zookeeper/0 smoke-test -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do /0 smoke-test`. - Watch the progress of the smoke test actions with: watch -n 2 juju show-action-status -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`. - Eventually, all of the actions should settle to `status: completed`. If any report `status: failed`, that application is not working as expected. Get more information about the smoke-test action juju show-action-output -> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `. - ## Utilities Applications in this bundle include Zookeeper command line and Spark web utilities that can be used to verify information about the cluster. @@ -181,27 +164,17 @@ the [rsyslog README](https://jujucharms.com/rsyslog/) for more information. # Benchmarking -The `spark` charm in this bundle provides several benchmarks to gauge -the performance of the Spark cluster. Each benchmark is an action that can be -run with `juju run-action`: - - $ juju actions spark | grep Bench - connectedcomponent Run the Spark Bench ConnectedComponent benchmark. - decisiontree Run the Spark Bench DecisionTree benchmark. - kmeans Run the Spark Bench KMeans benchmark. - linearregression Run the Spark Bench LinearRegression benchmark. - logisticregression Run the Spark Bench LogisticRegression benchmark. - matrixfactorization Run the Spark Bench MatrixFactorization benchmark. - pagerank Run the Spark Bench PageRank benchmark. - pca Run the Spark Bench PCA benchmark. - pregeloperation Run the Spark Bench PregelOperation benchmark. - shortestpaths Run the Spark Bench ShortestPaths benchmark. - sql Run the Spark Bench SQL benchmark. - stronglyconnectedcomponent Run the Spark Bench StronglyConnectedComponent benchmark. - svdplusplus Run the Spark Bench SVDPlusPlus benchmark. - svm Run the Spark Bench SVM benchmark. - - $ juju run-action spark/0 svdplusplus +The `spark` charm in this bundle provides benchmarks to gauge the performance +of the Spark cluster. Each benchmark is an action that can be run with +`juju run-action`: + + $ juju actions spark + ... + pagerank Calculate PageRank for a sample data set + sparkpi Calculate Pi + ... + + $ juju run-action spark/0 pagerank Action queued with id: 339cec1f-e903-4ee7-85ca-876fb0c3d28e $ juju show-action-output 339cec1f-e903-4ee7-85ca-876fb0c3d28e @@ -210,25 +183,15 @@ run with `juju run-action`: composite: direction: asc units: secs - value: "200.754000" - raw: | - SVDPlusPlus,2016-11-02-03:08:26,200.754000,85.974071,.428255,0,SVDPlusPlus-MLlibConfig,,,,,10,,,50000,4.0,1.3, - start: 2016-11-02T03:08:26Z - stop: 2016-11-02T03:11:47Z - results: - duration: - direction: asc - units: secs - value: "200.754000" - throughput: - direction: desc - units: x/sec - value: ".428255" + value: "83" + start: 2017-04-12T23:22:38Z + stop: 2017-04-12T23:24:01Z + output: '{''status'': ''completed''}' status: completed timing: - completed: 2016-11-02 03:11:48 +0000 UTC - enqueued: 2016-11-02 03:08:21 +0000 UTC - started: 2016-11-02 03:08:26 +0000 UTC + completed: 2017-04-12 23:24:02 +0000 UTC + enqueued: 2017-04-12 23:22:36 +0000 UTC + started: 2017-04-12 23:22:37 +0000 UTC # Scaling @@ -244,6 +207,18 @@ Multiple units may be added at once. For example, add four more spark units: juju add-unit -n4 spark +# Issues + +Apache Bigtop tracks issues using JIRA (Apache account required). File an +issue for this bundle at: + +https://issues.apache.org/jira/secure/CreateIssue!default.jspa + +Ensure `Bigtop` is selected as the project. Typically, bundle issues are filed +in the `deployment` component with the latest stable release selected as the +affected version. Any uncertain fields may be left blank. + + # Contact Information - @@ -254,6 +229,6 @@ Multiple units may be added at once. For example, add four more spark units: - [Apache Bigtop home page](http://bigtop.apache.org/) - [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju Big Data](https://jujucharms.com/big-data) +- [Juju Bigtop charms](https://jujucharms.com/q/bigtop) - [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) -- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-deploy/juju/spark-processing/bundle.yaml b/bigtop-deploy/juju/spark-processing/bundle.yaml index bb79d9459e..5a488850bd 100644 --- a/bigtop-deploy/juju/spark-processing/bundle.yaml +++ b/bigtop-deploy/juju/spark-processing/bundle.yaml @@ -1,6 +1,6 @@ services: spark: - charm: "cs:xenial/spark-31" + charm: "cs:xenial/spark-33" constraints: "mem=7G root-disk=32G" num_units: 2 options: @@ -13,7 +13,7 @@ services: - "0" - "1" zookeeper: - charm: "cs:xenial/zookeeper-17" + charm: "cs:xenial/zookeeper-19" constraints: "mem=3G root-disk=32G" num_units: 3 annotations: From cb43a52b1f9cdaa9f4e1e636524f9294c3a4bbed Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Sat, 22 Apr 2017 08:40:57 -0500 Subject: [PATCH 07/13] add direction to spark benchmarks --- bigtop-packages/src/charm/spark/layer-spark/actions/pagerank | 2 +- bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank b/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank index a54269cc4f..2650e74ad2 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/pagerank @@ -112,7 +112,7 @@ def main(): bench.finish() duration = stop - start - bench.set_composite_score(duration, 'secs') + bench.set_composite_score(duration, units='secs', direction='asc') # Tell the user how they can get the full result log scp_log_msg = "juju scp {}:{} .".format(hookenv.local_unit(), result_log) diff --git a/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi b/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi index f94befc474..93a4f10721 100755 --- a/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi +++ b/bigtop-packages/src/charm/spark/layer-spark/actions/sparkpi @@ -92,7 +92,7 @@ def main(): bench.finish() duration = stop - start - bench.set_composite_score(duration, 'secs') + bench.set_composite_score(duration, units='secs', direction='asc') # Tell the user how they can get the full result log scp_log_msg = "juju scp {}:{} .".format(hookenv.local_unit(), result_log) From 279eca7a083a6473336969dfd46185bf37d66439 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Sat, 22 Apr 2017 13:46:35 +0000 Subject: [PATCH 08/13] bump spark rev to pick up benchmark fix --- bigtop-deploy/juju/hadoop-spark/bundle.yaml | 2 +- bigtop-deploy/juju/spark-processing/bundle.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bigtop-deploy/juju/hadoop-spark/bundle.yaml b/bigtop-deploy/juju/hadoop-spark/bundle.yaml index 0701f27975..06114957e7 100644 --- a/bigtop-deploy/juju/hadoop-spark/bundle.yaml +++ b/bigtop-deploy/juju/hadoop-spark/bundle.yaml @@ -43,7 +43,7 @@ services: to: - "4" spark: - charm: "cs:xenial/spark-33" + charm: "cs:xenial/spark-34" constraints: "mem=7G root-disk=32G" num_units: 1 options: diff --git a/bigtop-deploy/juju/spark-processing/bundle.yaml b/bigtop-deploy/juju/spark-processing/bundle.yaml index 5a488850bd..f114ebfea8 100644 --- a/bigtop-deploy/juju/spark-processing/bundle.yaml +++ b/bigtop-deploy/juju/spark-processing/bundle.yaml @@ -1,6 +1,6 @@ services: spark: - charm: "cs:xenial/spark-33" + charm: "cs:xenial/spark-34" constraints: "mem=7G root-disk=32G" num_units: 2 options: From 63c997b3b74d3dcf27db03e51a67d1da547187d4 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Sat, 22 Apr 2017 13:12:09 -0500 Subject: [PATCH 09/13] enable heavy spark tests for spark-proc; light test for hadoop-spark --- bigtop-deploy/juju/hadoop-spark/tests/tests.yaml | 8 +++++++- bigtop-deploy/juju/spark-processing/tests/tests.yaml | 8 +------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bigtop-deploy/juju/hadoop-spark/tests/tests.yaml b/bigtop-deploy/juju/hadoop-spark/tests/tests.yaml index b951742128..cb745df7d7 100644 --- a/bigtop-deploy/juju/hadoop-spark/tests/tests.yaml +++ b/bigtop-deploy/juju/hadoop-spark/tests/tests.yaml @@ -5,9 +5,15 @@ sources: packages: - amulet - python3-yaml -# exclude tests that are unrelated to bigtop. +# exclude tests that are unrelated to bigtop. the exclusion of spark might +# look weird here, but for this bundle, we only care that spark is good in +# yarn mode (covered by this bundle when we invoke the spark smoke-test). the +# typical spark tests will test spark once in standalone and twice more in +# various HA modes. that takes forever, so leave those heavy tests for the +# spark-processing bundle. let's go fast on this one. excludes: - ganglia - ganglia-node - rsyslog - rsyslog-forwarder-ha + - spark diff --git a/bigtop-deploy/juju/spark-processing/tests/tests.yaml b/bigtop-deploy/juju/spark-processing/tests/tests.yaml index e4b472ea4a..b951742128 100644 --- a/bigtop-deploy/juju/spark-processing/tests/tests.yaml +++ b/bigtop-deploy/juju/spark-processing/tests/tests.yaml @@ -5,15 +5,9 @@ sources: packages: - amulet - python3-yaml -# exclude tests that are unrelated to bigtop. the exclusion of spark might -# look weird here, but for this bundle, we only care that spark is good in -# HA mode (covered by this bundle when we invoke the spark smoke-test). the -# typical spark tests will test spark once in standalone and twice more in -# various HA modes. that takes forever, so leave those heavy tests for the -# hadoop-spark bundle. let's go fast on this one. +# exclude tests that are unrelated to bigtop. excludes: - ganglia - ganglia-node - rsyslog - rsyslog-forwarder-ha - - spark From 2b45c7b097fac7e7909906aaefa751292e98ad3b Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Mon, 24 Apr 2017 17:27:18 +0000 Subject: [PATCH 10/13] use unique topic name for smoke test --- bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test b/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test index e9ac52de5a..a6cd3b6fa9 100755 --- a/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test +++ b/bigtop-packages/src/charm/kafka/layer-kafka/actions/smoke-test @@ -21,6 +21,7 @@ import subprocess from charmhelpers.core import hookenv, host from charms.reactive import is_state from jujubigdata.utils import run_as +from time import time if not is_state('kafka.started'): @@ -28,7 +29,7 @@ if not is_state('kafka.started'): # Define smoke test params -topic_name = "smoketest" +topic_name = "smoketest_{}".format(int(time())) topic_partitions = 1 topic_replication = 1 From b6fe2a68f4e499c48db5b03a1a9f0b9b4074f634 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Mon, 24 Apr 2017 17:32:32 +0000 Subject: [PATCH 11/13] bump kafka for smoke-test fix --- bigtop-deploy/juju/hadoop-kafka/bundle.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigtop-deploy/juju/hadoop-kafka/bundle.yaml b/bigtop-deploy/juju/hadoop-kafka/bundle.yaml index 6694d20d7c..abf74bba80 100644 --- a/bigtop-deploy/juju/hadoop-kafka/bundle.yaml +++ b/bigtop-deploy/juju/hadoop-kafka/bundle.yaml @@ -63,7 +63,7 @@ services: - "6" - "7" kafka: - charm: "cs:xenial/kafka-14" + charm: "cs:xenial/kafka-15" constraints: "mem=3G" num_units: 1 annotations: From 4a09b1d7395f8542b39afc44cf5c367d67d800bd Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Mon, 24 Apr 2017 19:38:36 +0000 Subject: [PATCH 12/13] BIGTOP-2742: workaround literal string bug in zeppelin init service --- .../lib/charms/layer/bigtop_zeppelin.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py index b600f4b7b4..5f3004b572 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py @@ -65,7 +65,33 @@ def trigger_bigtop(self): ], overrides=overrides, ) + + ########## + # BUG: BIGTOP-2742 + # Default zeppelin init script looks for the literal '$(hostname)' + # string. Symlink it so it's there when apt install (called during + # puppet apply) initially starts the service. + import subprocess + host = subprocess.check_output(['hostname']).decode('utf8').strip() + utils.run_as('root', 'mkdir', '/var/run/zeppelin') + utils.run_as('root', 'ln', '-sf', + '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host), + '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid') + ########## + bigtop.trigger_puppet() + + ########## + # BUG: BIGTOP-2742 + # Puppet apply will call systemctl daemon-reload, which removes the + # symlink we just created. Put it back so users will continue to have + # a working systemctl service. + utils.run_as('root', 'ln', '-sf', + '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host), + '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid') + utils.run_as('root', 'systemctl', 'restart', 'zeppelin') + ########## + self.wait_for_api(30) def setup_etc_env(self): From ee20fe7dfc2f8dbf9fd02fd2570f56520f91cd9a Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Tue, 25 Apr 2017 14:08:26 +0000 Subject: [PATCH 13/13] workaround zeppelin init script issue and set appropriate spark mode depending on zepp relations --- .../layer-zeppelin/actions/smoke-test | 11 --- .../lib/charms/layer/bigtop_zeppelin.py | 82 +++++++++++++------ .../layer-zeppelin/reactive/zeppelin.py | 3 +- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test index 823580d9f8..7305822b53 100755 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/actions/smoke-test @@ -16,7 +16,6 @@ # limitations under the License. import sys -import json from time import sleep from urllib.parse import urljoin from operator import itemgetter @@ -36,23 +35,13 @@ def fail(msg): if not is_state('zeppelin.installed'): fail('Zeppelin not yet ready') - notebook_id = '2A94M5J1Z' zep_addr = 'localhost' base_url = 'http://{}:9080/api/notebook/'.format(zep_addr) -interp_url = urljoin(base_url, 'interpreter/bind/%s' % notebook_id) job_url = urljoin(base_url, 'job/%s' % notebook_id) para_url = urljoin(base_url, '%s/paragraph/' % notebook_id) try: - # bind interpreters - resp = requests.get(interp_url, timeout=60) - resp.raise_for_status() - interpreters = resp.json() - interp_ids = list(map(itemgetter('id'), interpreters['body'])) - resp = requests.put(interp_url, data=json.dumps(interp_ids), timeout=60) - resp.raise_for_status() - # run notebook resp = requests.post(job_url, timeout=60) resp.raise_for_status() diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py index 5f3004b572..bb7cf08596 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py @@ -16,16 +16,17 @@ import os import json import time +import requests import socket -from urllib.parse import urljoin -import requests from path import Path +from urllib.parse import urljoin -from jujubigdata import utils from charmhelpers.core import hookenv, host, unitdata from charms import layer from charms.layer.apache_bigtop_base import Bigtop +from charms.reactive import is_state +from jujubigdata import utils class Zeppelin(object): @@ -43,55 +44,78 @@ def _add_override(self, name, value): def install(self): ''' - Trigger the Bigtop puppet recipe that handles the Zepplin service. + Perform initial one-time setup, workaround upstream bugs, and + trigger puppet. ''' # Dirs are handled by the bigtop deb, so no need to call out to # dist_config to do that work. However, we want to adjust the # groups for the `ubuntu` user for better interaction with Juju. self.dist_config.add_users() + + # Set ports based on layer.yaml options self._add_override('zeppelin::server::server_port', self.dist_config.port('zeppelin')) self._add_override('zeppelin::server::web_socket_port', self.dist_config.port('zeppelin_web')) - self.trigger_bigtop() - def trigger_bigtop(self): - bigtop = Bigtop() - overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.', - strip=True) - bigtop.render_site_yaml( - roles=[ - 'zeppelin-server', - ], - overrides=overrides, - ) + # Default spark to local mode on initial install. This will be + # reconfigured if/when hadoop or spark relations are made. + self._add_override('zeppelin::server::spark_master_url', 'local[*]') ########## # BUG: BIGTOP-2742 # Default zeppelin init script looks for the literal '$(hostname)' - # string. Symlink it so it's there when apt install (called during - # puppet apply) initially starts the service. + # string. Symlink it so it exists before the apt install from puppet + # tries to start the service. import subprocess host = subprocess.check_output(['hostname']).decode('utf8').strip() - utils.run_as('root', 'mkdir', '/var/run/zeppelin') + zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host) + utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin') utils.run_as('root', 'ln', '-sf', - '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host), + zepp_pid, '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid') ########## - bigtop.trigger_puppet() + self.trigger_bigtop() ########## # BUG: BIGTOP-2742 # Puppet apply will call systemctl daemon-reload, which removes the - # symlink we just created. Put it back so users will continue to have - # a working systemctl service. - utils.run_as('root', 'ln', '-sf', - '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host), - '/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid') - utils.run_as('root', 'systemctl', 'restart', 'zeppelin') + # symlink we just created. Now that the bits are on disk, update the + # init script $(hostname) that caused this mess to begin with. + zepp_init_script = '/etc/init.d/zeppelin' + utils.re_edit_in_place(zepp_init_script, { + r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid), + }) + utils.run_as('root', 'systemctl', 'daemon-reload') + self.restart() + self.wait_for_api(30) + ########## + + ########## + # BUG: BIGTOP-2154 + # The zep deb depends on spark-core and spark-python. However, because + # of the unholy requirement to have hive tightly coupled to spark, + # we need to ensure spark-datanucleus is installed. Do this after the + # initial install so the bigtop repo is available to us. + utils.run_as('root', 'apt-get', 'install', '-qy', 'spark-datanucleus') ########## + def trigger_bigtop(self): + ''' + Trigger the Bigtop puppet recipe that handles the Zeppelin service. + ''' + bigtop = Bigtop() + overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.', + strip=True) + bigtop.render_site_yaml( + roles=[ + 'zeppelin-server', + ], + overrides=overrides, + ) + + bigtop.trigger_puppet() self.wait_for_api(30) def setup_etc_env(self): @@ -135,6 +159,12 @@ def configure_hadoop(self): utils.run_as('hdfs', 'hdfs', 'dfs', '-mkdir', '-p', '/user/zeppelin') utils.run_as('hdfs', 'hdfs', 'dfs', '-chown', 'zeppelin', '/user/zeppelin') + # If spark is ready, let it handle the spark_master_url. Otherwise, + # zepp is in local mode; set it to yarn-client since hadoop is here. + if not is_state('spark.ready'): + self._add_override('zeppelin::server::spark_master_url', 'yarn-client') + self.trigger_bigtop() + def configure_spark(self, master_url): ''' Configure the zeppelin spark interpreter diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py index 6d37fdccc6..a04a59be3d 100644 --- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py +++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/reactive/zeppelin.py @@ -88,9 +88,8 @@ def configure_hadoop(hadoop): set_state('zeppelin.hadoop.configured') -@when('zeppelin.installed') +@when('zeppelin.installed', 'zeppelin.hadoop.configured') @when_not('hadoop.ready') -@when('zeppelin.hadoop.configured') def unconfigure_hadoop(): zeppelin = Zeppelin() zeppelin.remove_hadoop_notebooks()