diff --git a/docker/inference/build-hadoop.sh b/docker/inference/build-hadoop.sh deleted file mode 100755 index b5bde0afc..000000000 --- a/docker/inference/build-hadoop.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -if [[ "$#" != 1 ]]; then - echo "ERROR: Must provide Hadoop version number!" - echo " Example: ${BASH_SOURCE[0]} \"3.3.2\"" - exit 1 -fi -HADOOP_VER="$1" - -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) -cd ${SCRIPT_DIR} - -# Download desired revision. -git clone --branch rel/release-${HADOOP_VER} --depth 1 https://github.com/apache/hadoop.git hadoop -cd hadoop - -# Temporarily disable name resolution for jboss repository. NVIDIA IT ticket number: "INC0866408" -if [[ ! "$(curl --connect-timeout 5 https://repository.jboss.org)" ]] 2>/dev/null; then - echo 'Unable to connect to repository.jboss.org. Disabling...' - echo '127.0.0.1 repository.jboss.org' >> /etc/hosts -fi - -# Build Hadoop. -mvn clean package \ - -Pdist,native \ - -DskipTests \ - -Dtar -Dmaven.javadoc.skip=true \ - -Drequire.snappy \ - -Drequire.zstd \ - -Drequire.openssl \ - -Drequire.pmdk - -# Move compiled distribution to $SCRIPT_DIR and delete temporary files. -mv hadoop-dist/target/hadoop-${HADOOP_VER}.tar.gz .. -cd .. -rm -rf hadoop /root/.m2 - -# Self-delete. -rm -rf ${BASH_SOURCE[0]} \ No newline at end of file diff --git a/docker/inference/install-hadoop.sh b/docker/inference/install-hadoop.sh deleted file mode 100755 index 1c3a73598..000000000 --- a/docker/inference/install-hadoop.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -if [[ "$#" != 1 ]]; then - echo "ERROR: Must provide Hadoop version number!" - echo " Example: ${BASH_SOURCE[0]} \"3.3.2\"" - exit 1 -fi -HADOOP_VER="$1" - -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) -cd ${SCRIPT_DIR} - -# Extract files and delete archive. -mkdir -p ${HADOOP_HOME}/logs -tar xf hadoop-${HADOOP_VER}.tar.gz --strip-components 1 --directory ${HADOOP_HOME} -rm -rf hadoop-${HADOOP_VER}.tar.gz - -# Cleanup reundant files. -for f in $(find ${HADOOP_HOME} -name *.cmd); do - rm -rf $f -done - -# Pretend that the package has been installed like any other. -ln -s ${HADOOP_HOME}/include/hdfs.h /usr/local/include/hdfs.h -ln -s ${HADOOP_HOME}/lib/native/libhdfs.so /usr/local/lib/libhdfs.so -ln -s ${HADOOP_HOME}/lib/native/libhdfs.so.0.0.0 /usr/local/lib/libhdfs.so.0.0.0 -ln -s ${HADOOP_HOME}/lib/native/libhadoop.so /usr/local/lib/libhadoop.so -ln -s ${HADOOP_HOME}/lib/native/libhadoop.so.1.0.0 /usr/local/lib/libhadoop.so.1.0.0 - -# Create minimalist single-node "default" configuration. -sed -i "s/^# export JAVA_HOME=$/export JAVA_HOME=${JAVA_HOME//\//\\\/}/g" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh - -echo ' - - - - - - fs.defaultFS - hdfs://localhost:9000 - -' > ${HADOOP_HOME}/etc/hadoop/core-site.xml - -echo ' - - - - - - dfs.replication - 1 - -' > ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - -ssh-keygen -q -t ecdsa -b 521 -N "" <<< "" -cat $HOME/.ssh/id_ecdsa.pub >> $HOME/.ssh/authorized_keys - -ldconfig -echo " -Hadoop version: $(hadoop version) - -To run a single-node hadoop instance (for development only): - - hadoop namenode -format - service ssh start - start-dfs.sh - -" - -# Self-delete. -rm -rf ${BASH_SOURCE[0]} \ No newline at end of file diff --git a/docker/training/build-hadoop.sh b/docker/training/build-hadoop.sh deleted file mode 100755 index b5bde0afc..000000000 --- a/docker/training/build-hadoop.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -if [[ "$#" != 1 ]]; then - echo "ERROR: Must provide Hadoop version number!" - echo " Example: ${BASH_SOURCE[0]} \"3.3.2\"" - exit 1 -fi -HADOOP_VER="$1" - -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) -cd ${SCRIPT_DIR} - -# Download desired revision. -git clone --branch rel/release-${HADOOP_VER} --depth 1 https://github.com/apache/hadoop.git hadoop -cd hadoop - -# Temporarily disable name resolution for jboss repository. NVIDIA IT ticket number: "INC0866408" -if [[ ! "$(curl --connect-timeout 5 https://repository.jboss.org)" ]] 2>/dev/null; then - echo 'Unable to connect to repository.jboss.org. Disabling...' - echo '127.0.0.1 repository.jboss.org' >> /etc/hosts -fi - -# Build Hadoop. -mvn clean package \ - -Pdist,native \ - -DskipTests \ - -Dtar -Dmaven.javadoc.skip=true \ - -Drequire.snappy \ - -Drequire.zstd \ - -Drequire.openssl \ - -Drequire.pmdk - -# Move compiled distribution to $SCRIPT_DIR and delete temporary files. -mv hadoop-dist/target/hadoop-${HADOOP_VER}.tar.gz .. -cd .. -rm -rf hadoop /root/.m2 - -# Self-delete. -rm -rf ${BASH_SOURCE[0]} \ No newline at end of file diff --git a/docker/training/install-hadoop.sh b/docker/training/install-hadoop.sh deleted file mode 100755 index 1c3a73598..000000000 --- a/docker/training/install-hadoop.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -if [[ "$#" != 1 ]]; then - echo "ERROR: Must provide Hadoop version number!" - echo " Example: ${BASH_SOURCE[0]} \"3.3.2\"" - exit 1 -fi -HADOOP_VER="$1" - -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) -cd ${SCRIPT_DIR} - -# Extract files and delete archive. -mkdir -p ${HADOOP_HOME}/logs -tar xf hadoop-${HADOOP_VER}.tar.gz --strip-components 1 --directory ${HADOOP_HOME} -rm -rf hadoop-${HADOOP_VER}.tar.gz - -# Cleanup reundant files. -for f in $(find ${HADOOP_HOME} -name *.cmd); do - rm -rf $f -done - -# Pretend that the package has been installed like any other. -ln -s ${HADOOP_HOME}/include/hdfs.h /usr/local/include/hdfs.h -ln -s ${HADOOP_HOME}/lib/native/libhdfs.so /usr/local/lib/libhdfs.so -ln -s ${HADOOP_HOME}/lib/native/libhdfs.so.0.0.0 /usr/local/lib/libhdfs.so.0.0.0 -ln -s ${HADOOP_HOME}/lib/native/libhadoop.so /usr/local/lib/libhadoop.so -ln -s ${HADOOP_HOME}/lib/native/libhadoop.so.1.0.0 /usr/local/lib/libhadoop.so.1.0.0 - -# Create minimalist single-node "default" configuration. -sed -i "s/^# export JAVA_HOME=$/export JAVA_HOME=${JAVA_HOME//\//\\\/}/g" ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh - -echo ' - - - - - - fs.defaultFS - hdfs://localhost:9000 - -' > ${HADOOP_HOME}/etc/hadoop/core-site.xml - -echo ' - - - - - - dfs.replication - 1 - -' > ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - -ssh-keygen -q -t ecdsa -b 521 -N "" <<< "" -cat $HOME/.ssh/id_ecdsa.pub >> $HOME/.ssh/authorized_keys - -ldconfig -echo " -Hadoop version: $(hadoop version) - -To run a single-node hadoop instance (for development only): - - hadoop namenode -format - service ssh start - start-dfs.sh - -" - -# Self-delete. -rm -rf ${BASH_SOURCE[0]} \ No newline at end of file