Permalink
Cannot retrieve contributors at this time
Fetching contributors…
Cannot retrieve contributors at this time
| name: hadoop | |
| version: '1.2.0' | |
| summary: Apache Hadoop | |
| description: | | |
| Hadoop is a software platform that lets one easily write and run applications | |
| that process vast amounts of data. | |
| This snap installs Hadoop 2.7.3 and is compatible with Apache Bigtop 1.2.0. | |
| grade: stable | |
| confinement: strict | |
| apps: | |
| hadoop: | |
| command: wrappers/hadoop | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hdfs: | |
| command: wrappers/hdfs | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| mapred: | |
| command: wrappers/mapred | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| yarn: | |
| command: wrappers/yarn | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| daemon-wrapper: | |
| command: wrappers/hadoop-daemon-wrapper.sh | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hadoop-hdfs-namenode: | |
| command: wrappers/hadoop-daemon-wrapper.sh start namenode | |
| stop-command: wrappers/hadoop-daemon-wrapper.sh stop namenode | |
| daemon: forking | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hadoop-hdfs-datanode: | |
| command: wrappers/hadoop-daemon-wrapper.sh start datanode | |
| stop-command: wrappers/hadoop-daemon-wrapper.sh stop datanode | |
| daemon: forking | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hadoop-yarn-resourcemanager: | |
| command: wrappers/hadoop-daemon-wrapper.sh start resourcemanager | |
| stop-command: wrappers/hadoop-daemon-wrapper.sh stop resourcemanager | |
| daemon: forking | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hadoop-yarn-nodemanager: | |
| command: wrappers/hadoop-daemon-wrapper.sh start nodemanager | |
| stop-command: wrappers/hadoop-daemon-wrapper.sh stop nodemanager | |
| daemon: forking | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hadoop-mapreduce-historyserver: | |
| command: wrappers/hadoop-daemon-wrapper.sh start historyserver | |
| stop-command: wrappers/hadoop-daemon-wrapper.sh stop historyserver | |
| daemon: forking | |
| plugs: | |
| - 'network' | |
| - 'network-bind' | |
| hooks: | |
| install: | |
| plugs: | |
| # needs network to format the namenode | |
| - 'network' | |
| plugs: | |
| home: | |
| interface: home | |
| zookeeper: | |
| interface: content | |
| content: zookeeper | |
| target: $SNAP/zookeeper | |
| slots: | |
| hadoop: | |
| interface: content | |
| content: hadoop | |
| read: | |
| - $SNAP | |
| hadoop-conf: | |
| interface: content | |
| content: hadoop-conf | |
| write: | |
| - $SNAP_DATA/etc/hadoop | |
| parts: | |
| bigtop: | |
| source: https://github.com/apache/bigtop.git | |
| source-type: git | |
| source-branch: branch-1.2 | |
| plugin: gradle | |
| gradle-options: ['-b', 'packages.gradle', 'bom-snap'] | |
| build-packages: | |
| - curl | |
| - puppet-common | |
| stage-packages: | |
| - default-jdk-headless | |
| prepare: | | |
| # get bigtop buildreqs on the system | |
| if [ `id -u` -ne 0 ]; then | |
| PUPPET_CMD='sudo -E puppet apply' | |
| else | |
| PUPPET_CMD='puppet apply' | |
| fi | |
| ${PUPPET_CMD} -d --parser future --modulepath=".:/etc/puppet/modules" \ | |
| -e "include bigtop_toolchain::protobuf" | |
| ${PUPPET_CMD} -d --parser future --modulepath=".:/etc/puppet/modules" \ | |
| -e "include bigtop_toolchain::packages" | |
| # add a gradle task to generate a sourceable bom for use by other parts | |
| echo 'task "bom-snap" () << {' >> packages.gradle | |
| echo ' def bomWriter = new File("bigtop-snap.bom").newWriter()' >> packages.gradle | |
| echo ' bomVersions.each { bomWriter << "$it\\n"}' >> packages.gradle | |
| echo ' bomWriter.close()' >> packages.gradle | |
| echo '}' >> packages.gradle | |
| # there is no 'build' for bigtop-utils source, just set perms | |
| chmod 755 bigtop-packages/src/common/bigtop-utils/bigtop-detect-* | |
| # make sure utils point to the right default directory | |
| for i in `ls bigtop-packages/src/common/bigtop-utils/bigtop-*`; do | |
| sed -e 's|/etc/default|${SNAP}/etc/default|' -i $i; done | |
| # point bigtop-utils to SNAP_DATA for additional classpath entries | |
| echo 'export BIGTOP_CLASSPATH=${SNAP_DATA}/var/lib/bigtop' >> \ | |
| bigtop-packages/src/common/bigtop-utils/bigtop-utils.default | |
| # set bigtop-utils java home to the jdk packed into the snap | |
| echo 'export JAVA_HOME=${SNAP}/usr/lib/jvm/default-java' >> \ | |
| bigtop-packages/src/common/bigtop-utils/bigtop-utils.default | |
| install: | | |
| # adapted from bigtop/bigtop-packages/src/deb/bigtop-utils/rules | |
| install -d -p -m 755 $SNAPCRAFT_PART_INSTALL/usr/lib/bigtop-utils/ | |
| install -p -m 755 bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome \ | |
| $SNAPCRAFT_PART_INSTALL/usr/lib/bigtop-utils/ | |
| install -p -m 755 bigtop-packages/src/common/bigtop-utils/bigtop-detect-javalibs \ | |
| $SNAPCRAFT_PART_INSTALL/usr/lib/bigtop-utils/ | |
| install -p -m 755 bigtop-packages/src/common/bigtop-utils/bigtop-detect-classpath \ | |
| $SNAPCRAFT_PART_INSTALL/usr/lib/bigtop-utils/ | |
| install -p -m 755 bigtop-packages/src/common/bigtop-utils/bigtop-monitor-service \ | |
| $SNAPCRAFT_PART_INSTALL/usr/lib/bigtop-utils/ | |
| install -d -p -m 755 $SNAPCRAFT_PART_INSTALL/bigtop-utils/etc/default | |
| install -p -m 644 bigtop-packages/src/common/bigtop-utils/bigtop-utils.default \ | |
| $SNAPCRAFT_PART_INSTALL/etc/default/bigtop-utils | |
| stage: | |
| - etc/* | |
| - usr/lib/* | |
| bigtop-groovy: | |
| after: | |
| - bigtop | |
| source: https://dl.bintray.com/groovy/maven/apache-groovy-binary-2.4.10.zip | |
| plugin: dump | |
| prepare: | | |
| cp ../../bigtop/build/bigtop-snap.bom \ | |
| ../../bigtop/build/bigtop-packages/src/common/bigtop-groovy/bigtop.bom | |
| chmod 755 ../../bigtop/build/bigtop-packages/src/common/bigtop-groovy/do-component-build | |
| chmod 755 ../../bigtop/build/bigtop-packages/src/common/bigtop-groovy/install_groovy.sh | |
| build: | | |
| ../../bigtop/build/bigtop-packages/src/common/bigtop-groovy/do-component-build | |
| install: | | |
| # adapted from bigtop/bigtop-packages/src/deb/bigtop-groovy/rules | |
| ../../bigtop/build/bigtop-packages/src/common/bigtop-groovy/install_groovy.sh \ | |
| --build-dir=groovy-2.4.10 \ | |
| --prefix=$SNAPCRAFT_PART_INSTALL \ | |
| --bin-dir=usr/lib/bigtop-groovy/bin \ | |
| --version=2.4.10 \ | |
| --lib-dir=usr/lib/bigtop-groovy/lib \ | |
| --conf-dir=usr/lib/bigtop-groovy/conf | |
| stage: | |
| - usr/lib/* | |
| bigtop-jsvc: | |
| after: | |
| - bigtop | |
| source: https://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-native-src.tar.gz | |
| plugin: autotools | |
| prepare: | | |
| cp ../../bigtop/build/bigtop-snap.bom \ | |
| ../../bigtop/build/bigtop-packages/src/common/bigtop-jsvc/bigtop.bom | |
| chmod 755 ../../bigtop/build/bigtop-packages/src/common/bigtop-jsvc/do-component-build | |
| chmod 755 ../../bigtop/build/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh | |
| # patch jsvc sources with bigtop-specific changes | |
| for i in `ls ../../bigtop/build/bigtop-packages/src/common/bigtop-jsvc/patch*.diff`; do \ | |
| patch -p1 -i $i; done | |
| build: | | |
| # the autotools plugin ensures we get the right build tools, but we \ | |
| # still want to use the bigtop build recipe. | |
| bash -c '. ../../bigtop/build/bigtop-packages/src/common/bigtop-utils/bigtop-detect-javahome && \ | |
| ../../bigtop/build/bigtop-packages/src/common/bigtop-jsvc/do-component-build' | |
| install: | | |
| # adapted from bigtop/bigtop-packages/src/deb/bigtop-jsvc/rules | |
| ../../bigtop/build/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh \ | |
| --build-dir=. \ | |
| --prefix=$SNAPCRAFT_PART_INSTALL | |
| stage: | |
| - usr/lib/* | |
| - usr/share/* | |
| hadoop: | |
| after: | |
| - bigtop-groovy | |
| - bigtop-jsvc | |
| source: https://s3.amazonaws.com/jujubigdata/apache/noarch/hadoop-2.7.3-src.tar.gz | |
| plugin: maven | |
| maven-options: | |
| # We need the maven plugin to setup our env, but dont actually care about | |
| # the packaged build. Build as fast as we can, leaving the real build for | |
| # the install phase. | |
| - -Psrc | |
| - -DskipTests | |
| - -DskipITs | |
| maven-targets: | |
| - hadoop-dist | |
| stage-packages: | |
| - coreutils | |
| - util-linux | |
| prepare: | | |
| cp ../../bigtop/build/bigtop-snap.bom \ | |
| ../../bigtop/build/bigtop-packages/src/common/hadoop/bigtop.bom | |
| chmod 755 ../../bigtop/build/bigtop-packages/src/common/hadoop/do-component-build | |
| chmod 755 ../../bigtop/build/bigtop-packages/src/common/hadoop/install_hadoop.sh | |
| # patch bigtop sources with snap-specific changes for hadoop | |
| for i in `ls ../../../snap/sources/patch*.diff`; do \ | |
| patch -d ../../bigtop/build -p1 -i $i; done | |
| # patch hadoop sources with bigtop-specific changes | |
| for i in `ls ../../bigtop/build/bigtop-packages/src/common/hadoop/patch*.diff`; do \ | |
| patch -p1 -i $i; done | |
| # copy sources to expected locations; otherwise, when maven calls ant, | |
| # we lose the builder proxy config and cant download required sources | |
| mkdir -p hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads | |
| mkdir -p hadoop-common-project/hadoop-kms/downloads | |
| cp ../../../snap/sources/apache-tomcat* hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads | |
| cp ../../../snap/sources/apache-tomcat* hadoop-common-project/hadoop-kms/downloads | |
| install: | | |
| # Snap builders create a proxy-aware m2 settings file; use it if present | |
| if [ -e $SNAPCRAFT_PART_INSTALL/../m2/settings.xml ] ; then | |
| if [ ! -e $HOME/.m2/settings.xml ]; then | |
| mkdir -p $HOME/.m2 && \ | |
| ln -s $SNAPCRAFT_PART_INSTALL/../m2/settings.xml $HOME/.m2/settings.xml | |
| fi | |
| fi | |
| # Do not define build tasks because we want the maven plugin to setup any | |
| # proxy config (otherwise, we'd have to grok the env and do it ourselves). | |
| # We still want to use the bigtop build recipe for the actual build, so | |
| # do it asap during install. | |
| # NB: exit 1 on failure since install step failures do not stop the build | |
| ../../bigtop/build/bigtop-packages/src/common/hadoop/do-component-build || exit 1 | |
| # adapted from bigtop/bigtop-packages/src/deb/hadoop/rules | |
| ../../bigtop/build/bigtop-packages/src/common/hadoop/install_hadoop.sh \ | |
| --prefix=$SNAPCRAFT_PART_INSTALL \ | |
| --distro-dir=../../bigtop/build/bigtop-packages/src/common/hadoop \ | |
| --build-dir=build \ | |
| --httpfs-dir=$SNAPCRAFT_PART_INSTALL/usr/lib/hadoop-httpfs \ | |
| --httpfs-etc-dir=$SNAPCRAFT_PART_INSTALL/etc/hadoop-httpfs \ | |
| --system-lib-dir=$SNAPCRAFT_PART_INSTALL/usr/lib/ \ | |
| --system-libexec-dir=$SNAPCRAFT_PART_INSTALL/usr/lib/hadoop/libexec/ \ | |
| --system-include-dir=$SNAPCRAFT_PART_INSTALL/usr/include \ | |
| --doc-dir=$SNAPCRAFT_PART_INSTALL/usr/share/doc/hadoop-doc \ | |
| --man-dir=$SNAPCRAFT_PART_INSTALL/usr/share/man \ | |
| --example-dir=$SNAPCRAFT_PART_INSTALL/usr/share/doc/hadoop/examples | |
| install -d -m 0755 $SNAPCRAFT_PART_INSTALL/etc/security/limits.d | |
| cp ../../bigtop/build/bigtop-packages/src/common/hadoop/hdfs.conf \ | |
| $SNAPCRAFT_PART_INSTALL/etc/security/limits.d | |
| cp ../../bigtop/build/bigtop-packages/src/common/hadoop/mapreduce.conf \ | |
| $SNAPCRAFT_PART_INSTALL/etc/security/limits.d | |
| cp ../../bigtop/build/bigtop-packages/src/common/hadoop/yarn.conf \ | |
| $SNAPCRAFT_PART_INSTALL/etc/security/limits.d | |
| # create dirs for potential slot contents | |
| install -d -m 0755 $SNAPCRAFT_PART_INSTALL/zookeeper | |
| stage: | |
| - bin/* | |
| - etc/hadoop* | |
| - etc/security* | |
| - usr/* | |
| - zookeeper | |
| wrappers: | |
| after: | |
| - hadoop | |
| source: snap/ | |
| plugin: dump | |
| stage: | |
| - wrappers/* |