From 211d3c1374d5d70a52e5ca4840e48893e362aadb Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Tue, 7 Jun 2016 17:01:49 +0300 Subject: [PATCH 1/3] BIGTOP-2483: Add Mahout Charm --- .../src/charm/mahout/layer-mahout/README.md | 103 +++++ .../charm/mahout/layer-mahout/actions.yaml | 2 + .../mahout/layer-mahout/actions/smoke-test | 35 ++ .../src/charm/mahout/layer-mahout/copyright | 16 + .../src/charm/mahout/layer-mahout/icon.svg | 411 +++++++++++++++++ .../src/charm/mahout/layer-mahout/layer.yaml | 5 + .../charm/mahout/layer-mahout/metadata.yaml | 12 + .../mahout/layer-mahout/reactive/mahout.py | 37 ++ .../resources/links-converted.txt | 426 ++++++++++++++++++ .../mahout/layer-mahout/resources/users.txt | 247 ++++++++++ .../layer-mahout/tests/01-mahout-test.py | 65 +++ .../mahout/layer-mahout/tests/tests.yaml | 3 + 12 files changed, 1362 insertions(+) create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/README.md create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/actions.yaml create mode 100755 bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/copyright create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/icon.svg create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/reactive/mahout.py create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/resources/links-converted.txt create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/resources/users.txt create mode 100755 bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py create mode 100644 bigtop-packages/src/charm/mahout/layer-mahout/tests/tests.yaml diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/README.md b/bigtop-packages/src/charm/mahout/layer-mahout/README.md new file mode 100644 index 000000000..6a27a6394 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/README.md @@ -0,0 +1,103 @@ + +## Overview + + +The Apache Mahout™ project's goal is to build an environment for quickly creating +scalable performant machine learning applications. + +Apache Mahout software provides three major features: + * A simple and extensible programming environment and framework for building scalable algorithms + * A wide variety of premade algorithms for Scala + Apache Spark, H2O, Apache Flink + * Samsara, a vector math experimentation environment with R-like syntax which works at scale + + +## Usage + +This charm is intended to be added to an existing cluster, +for example, Hadoop. A Hadoop cluster can be deployed via +one of the [apache bigtop bundles](https://jujucharms.com/u/bigdata-charmers/#bundles). +For example: + + juju deploy hadoop-processing + +> Note: With Juju versions < 2.0, you will need to use [juju-deployer][] to +deploy the bundle. + +This will deploy the Apache Bigtop Hadoop platform with a workload node +preconfigured to work with the cluster. + +You can then add Mahout by relating it to the client: + + juju deploy cs:trusty/mahout + juju add-relation client mahout + juju add-relation mahout openjdk + + +[juju-deployer]: https://pypi.python.org/pypi/juju-deployer/ + + +## Status and Smoke Test + +Apache Bigtop charms provide extended status reporting to indicate when they +are ready: + + juju status + +This is particularly useful when combined with `watch` to track the on-going +progress of the deployment: + + watch -n 0.5 juju status + +The message for each unit will provide information about that unit's state. +Once they all indicate that they are ready, you can perform a "smoke test" +to verify that Mahout is working as expected using the built-in `smoke-test` +action: + + juju run-action mahout/0 smoke-test + +_**Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action do mahout/0 smoke-test`._ + +After a few minutes, you can check the results of the smoke test: + + juju show-action-status + +_**Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action status`._ + +You will see `status: completed` if the smoke test was successful, or +`status: failed` if it was not. You can get more information on why it failed +via: + + juju show-action-output + +_**Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action fetch `._ + + +## Contact Information + +- + + +## Hadoop + +- [Apache Bigtop](http://bigtop.apache.org/) home page +- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) +- [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) +- [Apache Bigtop charms](https://jujucharms.com/q/apache/bigtop) diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/actions.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/actions.yaml new file mode 100644 index 000000000..670c09991 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/actions.yaml @@ -0,0 +1,2 @@ +"smoke-test": + "description": "Mahout mapreduce recommender job." diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test b/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test new file mode 100755 index 000000000..22cf7c713 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/actions/smoke-test @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +if hdfs dfs -stat /tmp/input/ &> /dev/null; then + hdfs dfs -rm -r -skipTrash /tmp/input/ || true +fi + +hdfs dfs -mkdir /tmp/input/ +hdfs dfs -put resources/links-converted.txt /tmp/input/ +hdfs dfs -put resources/users.txt /tmp/input/ + +if hdfs dfs -stat temp &> /dev/null; then + hdfs dfs -rm -r -skipTrash temp || true +fi +if hdfs dfs -stat output &> /dev/null; then + hdfs dfs -rm -r -skipTrash output || true +fi + +hadoop jar /usr/lib/mahout/mahout-mr-*-job.jar org.apache.mahout.cf.taste.hadoop.item.RecommenderJob -Dmapred.input.dir=/tmp/input/links-converted.txt -Dmapred.output.dir=output --usersFile /tmp/input/users.txt --booleanData -s SIMILARITY_LOGLIKELIHOOD diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/copyright b/bigtop-packages/src/charm/mahout/layer-mahout/copyright new file mode 100644 index 000000000..52de50a91 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/copyright @@ -0,0 +1,16 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2015, Canonical Ltd., All Rights Reserved, The Apache Software Foundation +License: Apache License 2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/icon.svg b/bigtop-packages/src/charm/mahout/layer-mahout/icon.svg new file mode 100644 index 000000000..977103ba4 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/icon.svg @@ -0,0 +1,411 @@ + +image/svg+xml \ No newline at end of file diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml new file mode 100644 index 000000000..c80751e91 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml @@ -0,0 +1,5 @@ +includes: + - 'layer:apache-bigtop-base' +options: + basic: + use_venv: true diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml new file mode 100644 index 000000000..3c7c97df5 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml @@ -0,0 +1,12 @@ +name: mahout +summary: Create scalable and performant machine learning applications +maintainer: Juju Big Data +description: > + The Apache Mahout project's goal is to build an environment for quickly + creating scalable and performant machine learning applications. +tags: ["apache", "bigdata", "bigtop"] +subordinate: true +requires: + mahout: + interface: mahout + scope: container diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/reactive/mahout.py b/bigtop-packages/src/charm/mahout/layer-mahout/reactive/mahout.py new file mode 100644 index 000000000..e8e68cac3 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/reactive/mahout.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jujubigdata import utils +from charms.reactive import when, when_not, set_state +from charms.layer.apache_bigtop_base import Bigtop +from charmhelpers.core import hookenv + + +@when('bigtop.available') +@when_not('mahout.installed') +def install_mahout(): + hookenv.status_set('maintenance', 'installing mahout') + bigtop = Bigtop() + bigtop.render_site_yaml( + roles=[ + 'mahout-client', + ], + ) + bigtop.trigger_puppet() + with utils.environment_edit_in_place('/etc/environment') as env: + env['MAHOUT_HOME'] = '/usr/lib/mahout' + + hookenv.status_set('active', 'ready') + set_state('mahout.installed') diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/resources/links-converted.txt b/bigtop-packages/src/charm/mahout/layer-mahout/resources/links-converted.txt new file mode 100644 index 000000000..5e7a7f91a --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/resources/links-converted.txt @@ -0,0 +1,426 @@ +1,1664968 +2,3 +2,747213 +2,1664968 +2,1691047 +2,4095634 +2,5535664 +3,9 +3,77935 +3,79583 +3,84707 +3,56457 +3,594898 +3,681805 +3,681886 +3,835470 +3,880698 +3,1109091 +3,1125108 +3,1279972 +3,1463445 +3,1497566 +3,1783284 +3,1997564 +3,2006526 +3,2070954 +3,2250217 +3,2268713 +3,2276203 +3,2374802 +3,2571397 +3,2640902 +3,2647217 +3,2732378 +3,2821237 +3,3088028 +3,3092827 +3,3211549 +3,3283735 +3,3491412 +3,3492254 +3,3498305 +3,3505664 +3,3547201 +3,3603437 +3,3617913 +3,3793767 +3,3907547 +3,4021634 +3,4025897 +3,4086017 +3,4183126 +3,4184025 +3,4189168 +3,4192731 +3,4395141 +3,4899940 +3,4987592 +3,4999120 +3,5017477 +3,5149173 +3,5149311 +3,5158741 +3,5223097 +3,5302153 +3,5474252 +3,5535280 +4,145 +5,8 +5,57544 +5,58089 +5,60048 +5,65880 +5,284186 +5,313376 +5,564578 +5,717529 +5,729993 +5,1097284 +5,1204280 +5,1204407 +5,1255317 +5,1670218 +5,1720928 +5,1850305 +5,2269887 +5,2333350 +5,2359764 +5,2640693 +5,2743982 +5,3303009 +5,3322952 +5,3492254 +5,3573013 +5,3721693 +5,3797343 +5,3797349 +5,3797359 +5,3849461 +5,4033556 +5,4173124 +5,4189215 +5,4207986 +5,4669945 +5,4817900 +5,4901416 +5,5010479 +5,5062062 +5,5072938 +5,5098953 +5,5292042 +5,5429924 +5,5599862 +5,5599863 +5,5689049 +6,8 +7,8 +8,5 +8,57544 +8,58089 +8,59375 +8,64985 +8,313376 +8,704624 +8,717529 +8,729993 +8,1204280 +8,1204407 +8,1254637 +8,1255317 +8,1497566 +8,1720928 +8,1850305 +8,2269887 +8,2333350 +8,2359764 +8,2496900 +8,2640848 +8,2743982 +8,3303009 +8,3322952 +8,3492254 +8,3573013 +8,3797343 +8,3797349 +8,3797359 +8,4033556 +8,4173124 +8,4189168 +8,4206743 +8,4207986 +8,4393611 +8,4813259 +8,4901416 +8,5010479 +8,5062062 +8,5072938 +8,5098953 +8,5292042 +8,5429924 +8,5599862 +8,5599863 +9,3 +9,74106 +9,75221 +9,275656 +9,313376 +9,1279972 +9,1565872 +9,1613838 +9,1997564 +9,2640650 +9,3092827 +9,3491412 +9,3492254 +9,3956845 +9,3973207 +9,4025897 +9,4189168 +9,4189215 +9,4813259 +10,3 +11,60956 +11,313376 +11,322893 +11,497519 +11,499246 +11,594399 +11,801968 +11,806840 +11,1123171 +11,1228259 +11,1463265 +11,1892998 +11,2022036 +11,2070954 +11,2639079 +11,3492254 +11,3594794 +11,3967074 +11,4096317 +11,4189168 +11,4189215 +11,4273212 +11,4611415 +11,4708418 +11,4813259 +11,5300058 +11,5575496 +12,5 +13,5534647 +14,4116750 +15,4095634 +16,5534647 +17,5703728 +18,4207272 +19,2402613 +20,2402613 +22,4095634 +23,5688890 +24,205444 +24,530901 +24,1601519 +24,2583882 +24,3072654 +24,3492254 +24,3498305 +24,4096317 +24,4189168 +24,4638601 +24,4751151 +24,5242252 +25,5688891 +26,5688892 +27,5688893 +28,5688894 +29,5688893 +30,5688896 +31,267480 +32,5688896 +33,5688899 +34,5688900 +35,5688901 +36,5688902 +37,5688903 +38,5688904 +39,5688904 +40,5688906 +41,5108264 +42,5688910 +43,5688909 +44,45 +45,77935 +45,349575 +45,730133 +45,1050004 +45,1296303 +45,1438941 +45,1861146 +45,2238687 +45,2415295 +45,2496868 +45,2504674 +45,2518439 +45,2640161 +45,2806797 +45,3270463 +45,3523345 +45,3709479 +45,3877647 +45,4068200 +45,4068202 +45,4267445 +45,4745746 +45,4786041 +45,4884810 +45,4884863 +45,4950172 +45,5061591 +45,5300058 +45,5444196 +45,5445898 +45,5596573 +46,5688911 +47,5688912 +48,5688912 +49,5688914 +50,5496703 +51,5688916 +52,5688916 +53,1965054 +54,5688918 +55,5688921 +56,5688923 +57,5688925 +58,5688925 +59,4536608 +60,4536608 +61,4536608 +62,4536608 +63,65 +64,5688927 +65,66 +65,373621 +65,497364 +65,660675 +65,668841 +65,726925 +65,801925 +65,875721 +65,1054785 +65,1137604 +65,1264295 +65,1485934 +65,1506476 +65,1525691 +65,1707455 +65,1875197 +65,1891648 +65,2012305 +65,2016813 +65,2495798 +65,2531695 +65,2531937 +65,2533922 +65,2566623 +65,2630316 +65,2630470 +65,3234828 +65,3253028 +65,3345822 +65,3603437 +65,3681859 +65,3880666 +65,3918951 +65,3999143 +65,3999230 +65,4040762 +65,4140473 +65,4186673 +65,4208272 +65,4222476 +65,4288342 +65,4444399 +65,4629446 +65,4732884 +65,4867692 +65,4877498 +66,65 +66,76573 +66,275656 +66,373621 +66,462484 +66,497364 +66,660675 +66,726925 +66,1051918 +66,1054809 +66,1081160 +66,1786747 +66,1872865 +66,1875197 +66,2013257 +66,2016813 +66,2495798 +66,2531695 +66,2531937 +66,2566623 +66,3234828 +66,3253028 +66,3345822 +66,3492254 +66,3880666 +66,3918951 +66,3999230 +66,4040762 +66,4140473 +66,4186673 +66,4189168 +66,4189215 +66,4288238 +66,4288342 +66,4444399 +66,4530182 +66,4629446 +66,4732884 +66,4813259 +66,4877491 +66,4877498 +66,5029787 +66,5361565 +67,5688928 +68,2965227 +69,5703728 +70,2810730 +71,4813077 +72,4813077 +73,5688931 +74,5688931 +75,5703728 +76,2783730 +77,5703728 +78,5703727 +79,5703729 +80,5703729 +81,5703728 +82,5703729 +83,5703729 +84,5665378 +85,5688933 +86,5688937 +87,5688937 +88,5079330 +89,5079330 +90,5703728 +91,5703728 +92,3721245 +93,3549534 +94,436730 +95,5688942 +96,2783730 +97,5688943 +98,5688946 +99,5688946 +100,5689134 +101,5688957 diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/resources/users.txt b/bigtop-packages/src/charm/mahout/layer-mahout/resources/users.txt new file mode 100644 index 000000000..cb68caa59 --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/resources/users.txt @@ -0,0 +1,247 @@ +1,1664968 +2,3 +2,747213 +2,1664968 +2,1691047 +2,4095634 +2,5535664 +3,9 +3,77935 +3,79583 +3,84707 +3,56457 +3,594898 +3,681805 +3,681886 +3,835470 +3,880698 +3,1109091 +3,1125108 +3,1279972 +3,1463445 +3,1497566 +3,1783284 +3,1997564 +3,2006526 +3,2070954 +3,2250217 +3,2268713 +3,2276203 +3,2374802 +3,2571397 +3,2640902 +3,2647217 +3,2732378 +3,2821237 +3,3088028 +3,3092827 +3,3211549 +3,3283735 +3,3491412 +3,3492254 +3,3498305 +3,3505664 +3,3547201 +3,3603437 +3,3617913 +3,3793767 +3,3907547 +3,4021634 +3,4025897 +3,4086017 +3,4183126 +3,4184025 +3,4189168 +3,4192731 +3,4395141 +3,4899940 +3,4987592 +3,4999120 +3,5017477 +3,5149173 +3,5149311 +3,5158741 +3,5223097 +3,5302153 +3,5474252 +3,5535280 +4,145 +5,8 +5,57544 +5,58089 +5,60048 +5,65880 +5,284186 +5,313376 +5,564578 +5,717529 +5,729993 +5,1097284 +5,1204280 +5,1204407 +5,1255317 +5,1670218 +5,1720928 +5,1850305 +5,2269887 +5,2333350 +5,2359764 +5,2640693 +5,2743982 +5,3303009 +5,3322952 +5,3492254 +5,3573013 +5,3721693 +5,3797343 +5,3797349 +5,3797359 +5,3849461 +5,4033556 +5,4173124 +5,4189215 +5,4207986 +5,4669945 +5,4817900 +5,4901416 +5,5010479 +5,5062062 +5,5072938 +5,5098953 +5,5292042 +5,5429924 +5,5599862 +5,5599863 +5,5689049 +6,8 +7,8 +8,5 +8,57544 +8,58089 +8,59375 +8,64985 +8,313376 +8,704624 +8,717529 +8,729993 +8,1204280 +8,1204407 +8,1254637 +8,1255317 +8,1497566 +8,1720928 +8,1850305 +8,2269887 +8,2333350 +8,2359764 +8,2496900 +8,2640848 +8,2743982 +8,3303009 +8,3322952 +8,3492254 +8,3573013 +8,3797343 +8,3797349 +8,3797359 +8,4033556 +8,4173124 +8,4189168 +8,4206743 +8,4207986 +8,4393611 +8,4813259 +8,4901416 +8,5010479 +8,5062062 +8,5072938 +8,5098953 +8,5292042 +8,5429924 +8,5599862 +8,5599863 +9,3 +9,74106 +9,75221 +9,275656 +9,313376 +9,1279972 +9,1565872 +9,1613838 +9,1997564 +9,2640650 +9,3092827 +9,3491412 +9,3492254 +9,3956845 +9,3973207 +9,4025897 +9,4189168 +9,4189215 +9,4813259 +10,3 +11,60956 +11,313376 +11,322893 +11,497519 +11,499246 +11,594399 +11,801968 +11,806840 +11,1123171 +11,1228259 +11,1463265 +11,1892998 +11,2022036 +11,2070954 +11,2639079 +11,3492254 +11,3594794 +11,3967074 +11,4096317 +11,4189168 +11,4189215 +11,4273212 +11,4611415 +11,4708418 +11,4813259 +11,5300058 +11,5575496 +12,5 +13,5534647 +14,4116750 +15,4095634 +16,5534647 +17,5703728 +18,4207272 +19,2402613 +20,2402613 +22,4095634 +23,5688890 +24,205444 +24,530901 +24,1601519 +24,2583882 +24,3072654 +24,3492254 +24,3498305 +24,4096317 +24,4189168 +24,4638601 +24,4751151 +24,5242252 +25,5688891 +26,5688892 +27,5688893 +28,5688894 +29,5688893 +30,5688896 +31,267480 +32,5688896 +33,5688899 +34,5688900 +35,5688901 +36,5688902 +37,5688903 +38,5688904 +39,5688904 diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py b/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py new file mode 100755 index 000000000..88c9410fa --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import amulet + + +class TestDeploy(unittest.TestCase): + """ + Deployment and smoke test for Apache Mahout. + """ + def setUp(self): + self.d = amulet.Deployment(series='trusty') + self.d.add('mahout', 'mahout') + self.d.add('client', 'hadoop-client') + self.d.add('resourcemgr', 'hadoop-resourcemanager') + self.d.add('namenode', 'hadoop-namenode') + self.d.add('slave', 'hadoop-slave') + self.d.add('plugin', 'hadoop-plugin') + self.d.add('openjdk', 'openjdk') + + self.d.configure('openjdk', {'java-type': 'jdk', + 'java-major': '8'}) + + self.d.relate('plugin:hadoop-plugin', 'client:hadoop') + self.d.relate('plugin:namenode', 'namenode:namenode') + self.d.relate('plugin:resourcemanager', 'resourcemgr:resourcemanager') + self.d.relate('slave:namenode', 'namenode:datanode') + self.d.relate('slave:resourcemanager', 'resourcemgr:nodemanager') + self.d.relate('namenode:namenode', 'resourcemgr:namenode') + self.d.relate('mahout:mahout', 'client:mahout') + + self.d.relate('plugin:java', 'openjdk:java') + self.d.relate('namenode:java', 'openjdk:java') + self.d.relate('slave:java', 'openjdk:java') + self.d.relate('resourcemgr:java', 'openjdk:java') + self.d.relate('mahout:java', 'openjdk:java') + self.d.relate('client:java', 'openjdk:java') + + self.d.setup(timeout=1800) + self.d.sentry.wait_for_messages({"mahout": "ready"}) + self.mahout = self.d.sentry['mahout'][0] + + def test_mahout(self): + smk_uuid = self.mahout.action_do("smoke-test") + output = self.d.action_fetch(smk_uuid, full_output=True) + assert "completed" in output['status'] + + +if __name__ == '__main__': + unittest.main() diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/tests/tests.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/tests/tests.yaml new file mode 100644 index 000000000..3b6ce3e5f --- /dev/null +++ b/bigtop-packages/src/charm/mahout/layer-mahout/tests/tests.yaml @@ -0,0 +1,3 @@ +reset: false +packages: + - amulet From 047e06144e9fd3b0057d71dd20d3dec8f2accf0b Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Tue, 11 Oct 2016 14:40:29 +0000 Subject: [PATCH 2/3] refresh mahout for 2.0 and xenial - update README with juju2 and consistent bigtop charm messaging - add layer repo key pointing to upstream; inherit tags from bigtop base layer - update amulet test for xenial and remove openjdk since it's optional --- .../src/charm/mahout/layer-mahout/README.md | 98 +++++++++++-------- .../src/charm/mahout/layer-mahout/layer.yaml | 1 + .../charm/mahout/layer-mahout/metadata.yaml | 2 +- .../layer-mahout/tests/01-mahout-test.py | 21 +--- 4 files changed, 65 insertions(+), 57 deletions(-) diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/README.md b/bigtop-packages/src/charm/mahout/layer-mahout/README.md index 6a27a6394..065fe00bd 100644 --- a/bigtop-packages/src/charm/mahout/layer-mahout/README.md +++ b/bigtop-packages/src/charm/mahout/layer-mahout/README.md @@ -14,45 +14,57 @@ See the License for the specific language governing permissions and limitations under the License. --> -## Overview +# Overview - -The Apache Mahout™ project's goal is to build an environment for quickly creating +The Apache Mahout project's goal is to build an environment for quickly creating scalable performant machine learning applications. Apache Mahout software provides three major features: - * A simple and extensible programming environment and framework for building scalable algorithms - * A wide variety of premade algorithms for Scala + Apache Spark, H2O, Apache Flink - * Samsara, a vector math experimentation environment with R-like syntax which works at scale + * A simple and extensible programming environment and framework for building + scalable algorithms + * A wide variety of premade algorithms for Scala + Apache Spark, H2O, Apache + Flink + * Samsara, a vector math experimentation environment with R-like syntax which + works at scale + +# Deploying -## Usage +A working Juju installation is assumed to be present. If Juju is not yet set +up, please follow the [getting-started][] instructions prior to deploying this +charm. -This charm is intended to be added to an existing cluster, -for example, Hadoop. A Hadoop cluster can be deployed via -one of the [apache bigtop bundles](https://jujucharms.com/u/bigdata-charmers/#bundles). +This charm is intended to be used with one of the [apache bigtop bundles][]. For example: juju deploy hadoop-processing -> Note: With Juju versions < 2.0, you will need to use [juju-deployer][] to -deploy the bundle. +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, use [juju-quickstart][] with the following syntax: `juju quickstart +hadoop-processing`. -This will deploy the Apache Bigtop Hadoop platform with a workload node -preconfigured to work with the cluster. +This will deploy an Apache Bigtop Hadoop cluster. More information about this +deployment can be found in the [bundle readme](https://jujucharms.com/hadoop-processing/). -You can then add Mahout by relating it to the client: +Now add Mahout and relate it to the cluster endpoint: - juju deploy cs:trusty/mahout - juju add-relation client mahout - juju add-relation mahout openjdk + juju deploy mahout + juju add-relation mahout client +## Network-Restricted Environments +Charms can be deployed in environments with limited network access. To deploy +in this environment, configure a Juju model with appropriate proxy and/or +mirror options. See [Configuring Models][] for more information. -[juju-deployer]: https://pypi.python.org/pypi/juju-deployer/ +[getting-started]: https://jujucharms.com/docs/stable/getting-started +[apache bigtop bundles]: https://jujucharms.com/u/bigdata-charmers/#bundles +[juju-quickstart]: https://launchpad.net/juju-quickstart +[Configuring Models]: https://jujucharms.com/docs/stable/models-config -## Status and Smoke Test +# Verifying +## Status Apache Bigtop charms provide extended status reporting to indicate when they are ready: @@ -61,43 +73,49 @@ are ready: This is particularly useful when combined with `watch` to track the on-going progress of the deployment: - watch -n 0.5 juju status + watch -n 2 juju status + +The message column will provide information about a given unit's state. +This charm is ready for use once the status message indicates that it is +ready. -The message for each unit will provide information about that unit's state. -Once they all indicate that they are ready, you can perform a "smoke test" -to verify that Mahout is working as expected using the built-in `smoke-test` -action: +## Smoke Test +This charm provides a `smoke-test` action that can be used to verify the +application is functioning as expected. Run the action as follows: juju run-action mahout/0 smoke-test -_**Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action do mahout/0 smoke-test`._ +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action do mahout/0 smoke-test`. -After a few minutes, you can check the results of the smoke test: +Watch the progress of the smoke test actions with: - juju show-action-status + watch -n 2 juju show-action-status -_**Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action status`._ +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action status`. -You will see `status: completed` if the smoke test was successful, or -`status: failed` if it was not. You can get more information on why it failed -via: +Eventually, the action should settle to `status: completed`. If it +reports `status: failed`, the application is not working as expected. Get +more information about a specific smoke test with: juju show-action-output -_**Note**: The above assumes Juju 2.0 or greater. If using an earlier version -of Juju, the syntax is `juju action fetch `._ +> **Note**: The above assumes Juju 2.0 or greater. If using an earlier version +of Juju, the syntax is `juju action fetch `. -## Contact Information +# Contact Information - -## Hadoop +# Resources - [Apache Bigtop](http://bigtop.apache.org/) home page -- [Apache Bigtop issue tracking](http://bigtop.apache.org/issue-tracking.html) - [Apache Bigtop mailing lists](http://bigtop.apache.org/mail-lists.html) -- [Apache Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Apache Mahout home page](https://mahout.apache.org/) +- [Apache Mahout issue tracker](https://issues.apache.org/jira/browse/MAHOUT) +- [Juju Bigtop charms](https://jujucharms.com/q/apache/bigtop) +- [Juju mailing list](https://lists.ubuntu.com/mailman/listinfo/juju) +- [Juju community](https://jujucharms.com/community) diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml index c80751e91..9bb67328e 100644 --- a/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml +++ b/bigtop-packages/src/charm/mahout/layer-mahout/layer.yaml @@ -1,3 +1,4 @@ +repo: https://github.com/apache/bigtop/tree/master/bigtop-packages/src/charm/mahout/layer-mahout includes: - 'layer:apache-bigtop-base' options: diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml b/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml index 3c7c97df5..7a71d70ff 100644 --- a/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml +++ b/bigtop-packages/src/charm/mahout/layer-mahout/metadata.yaml @@ -4,7 +4,7 @@ maintainer: Juju Big Data description: > The Apache Mahout project's goal is to build an environment for quickly creating scalable and performant machine learning applications. -tags: ["apache", "bigdata", "bigtop"] +tags: [] subordinate: true requires: mahout: diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py b/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py index 88c9410fa..f5d1e0c60 100755 --- a/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py +++ b/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py @@ -24,33 +24,22 @@ class TestDeploy(unittest.TestCase): Deployment and smoke test for Apache Mahout. """ def setUp(self): - self.d = amulet.Deployment(series='trusty') + self.d = amulet.Deployment(series='xenial') self.d.add('mahout', 'mahout') self.d.add('client', 'hadoop-client') - self.d.add('resourcemgr', 'hadoop-resourcemanager') + self.d.add('resourcemanager', 'hadoop-resourcemanager') self.d.add('namenode', 'hadoop-namenode') self.d.add('slave', 'hadoop-slave') self.d.add('plugin', 'hadoop-plugin') - self.d.add('openjdk', 'openjdk') - - self.d.configure('openjdk', {'java-type': 'jdk', - 'java-major': '8'}) self.d.relate('plugin:hadoop-plugin', 'client:hadoop') self.d.relate('plugin:namenode', 'namenode:namenode') - self.d.relate('plugin:resourcemanager', 'resourcemgr:resourcemanager') + self.d.relate('plugin:resourcemanager', 'resourcemanager:resourcemanager') self.d.relate('slave:namenode', 'namenode:datanode') - self.d.relate('slave:resourcemanager', 'resourcemgr:nodemanager') - self.d.relate('namenode:namenode', 'resourcemgr:namenode') + self.d.relate('slave:resourcemanager', 'resourcemanager:nodemanager') + self.d.relate('namenode:namenode', 'resourcemanager:namenode') self.d.relate('mahout:mahout', 'client:mahout') - self.d.relate('plugin:java', 'openjdk:java') - self.d.relate('namenode:java', 'openjdk:java') - self.d.relate('slave:java', 'openjdk:java') - self.d.relate('resourcemgr:java', 'openjdk:java') - self.d.relate('mahout:java', 'openjdk:java') - self.d.relate('client:java', 'openjdk:java') - self.d.setup(timeout=1800) self.d.sentry.wait_for_messages({"mahout": "ready"}) self.mahout = self.d.sentry['mahout'][0] From be477f97cce3b35c9fc2eed5c9877ba58380be54 Mon Sep 17 00:00:00 2001 From: Kevin W Monroe Date: Sun, 30 Oct 2016 22:25:13 +0000 Subject: [PATCH 3/3] be series specific in smoke test; use new action syntax --- .../layer-mahout/tests/01-mahout-test.py | 54 ++++++++++--------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py b/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py index f5d1e0c60..4e315d308 100755 --- a/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py +++ b/bigtop-packages/src/charm/mahout/layer-mahout/tests/01-mahout-test.py @@ -21,33 +21,39 @@ class TestDeploy(unittest.TestCase): """ - Deployment and smoke test for Apache Mahout. + Deployment and smoke test for Apache Bigtop Mahout. """ - def setUp(self): - self.d = amulet.Deployment(series='xenial') - self.d.add('mahout', 'mahout') - self.d.add('client', 'hadoop-client') - self.d.add('resourcemanager', 'hadoop-resourcemanager') - self.d.add('namenode', 'hadoop-namenode') - self.d.add('slave', 'hadoop-slave') - self.d.add('plugin', 'hadoop-plugin') - - self.d.relate('plugin:hadoop-plugin', 'client:hadoop') - self.d.relate('plugin:namenode', 'namenode:namenode') - self.d.relate('plugin:resourcemanager', 'resourcemanager:resourcemanager') - self.d.relate('slave:namenode', 'namenode:datanode') - self.d.relate('slave:resourcemanager', 'resourcemanager:nodemanager') - self.d.relate('namenode:namenode', 'resourcemanager:namenode') - self.d.relate('mahout:mahout', 'client:mahout') - - self.d.setup(timeout=1800) - self.d.sentry.wait_for_messages({"mahout": "ready"}) - self.mahout = self.d.sentry['mahout'][0] + @classmethod + def setUpClass(cls): + cls.d = amulet.Deployment(series='xenial') + cls.d.add('mahout', 'cs:xenial/mahout') + cls.d.add('client', 'cs:xenial/hadoop-client') + cls.d.add('resourcemanager', 'cs:xenial/hadoop-resourcemanager') + cls.d.add('namenode', 'cs:xenial/hadoop-namenode') + cls.d.add('slave', 'cs:xenial/hadoop-slave') + cls.d.add('plugin', 'cs:xenial/hadoop-plugin') + + cls.d.relate('plugin:hadoop-plugin', 'client:hadoop') + cls.d.relate('plugin:namenode', 'namenode:namenode') + cls.d.relate('plugin:resourcemanager', 'resourcemanager:resourcemanager') + cls.d.relate('slave:namenode', 'namenode:datanode') + cls.d.relate('slave:resourcemanager', 'resourcemanager:nodemanager') + cls.d.relate('namenode:namenode', 'resourcemanager:namenode') + cls.d.relate('mahout:mahout', 'client:mahout') + + cls.d.setup(timeout=3600) + cls.d.sentry.wait_for_messages({"mahout": "ready"}, timeout=3600) + cls.mahout = cls.d.sentry['mahout'][0] def test_mahout(self): - smk_uuid = self.mahout.action_do("smoke-test") - output = self.d.action_fetch(smk_uuid, full_output=True) - assert "completed" in output['status'] + """ + Validate Mahout by running the smoke-test action. + """ + uuid = self.mahout.run_action('smoke-test') + result = self.d.action_fetch(uuid, full_output=True) + # action status=completed on success + if (result['status'] != "completed"): + self.fail('Mahout smoke-test failed: %s' % result) if __name__ == '__main__':