Skip to content
This repository has been archived by the owner on Feb 3, 2021. It is now read-only.

Commit

Permalink
Fix: optimize start task (#582)
Browse files Browse the repository at this point in the history 
* optimize start task

* functions for setup_host
  • Loading branch information
@jafreck
jafreck authored May 31, 2018
1 parent 1eeff23 commit e5e529a
Show file tree
Hide file tree
Showing 2 changed files with 154 additions and 111 deletions.
250 changes: 147 additions & 103 deletions aztk/node_scripts/setup_host.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,106 +11,150 @@ export PYTHONUNBUFFERED=TRUE
container_name=$1
docker_repo_name=$2

echo "Installing pre-reqs"
apt-get -y update
apt-get install -y --no-install-recommends linux-image-extra-$(uname -r) linux-image-extra-virtual
apt-get install -y --no-install-recommends apt-transport-https
apt-get install -y --no-install-recommends curl
apt-get install -y --no-install-recommends ca-certificates
apt-get install -y --no-install-recommends software-properties-common
apt-get install -y --no-install-recommends python3-pip python3-venv python-dev build-essential libssl-dev
echo "Done installing pre-reqs"

# Install docker
echo "Installing Docker"
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
apt-get -y update
apt-get -y install docker-ce
echo "Done installing Docker"

# set hostname in /etc/hosts if dns cannot resolve
if ! host $HOSTNAME ; then
echo $(hostname -I | awk '{print $1}') $HOSTNAME >> /etc/hosts
fi

# Install docker-compose
echo "Installing Docker-Componse"
sudo curl -L https://github.com/docker/compose/releases/download/1.19.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
echo "Done installing Docker-Compose"

if [ $AZTK_GPU_ENABLED == "true" ]; then
echo "running nvidia install"
sudo apt-get -y install nvidia-384
sudo apt-get -y install nvidia-modprobe

wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb
sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb
echo "nvidia install complete"
fi

if [ -z "$DOCKER_USERNAME" ]; then
echo "No Credentials provided. No need to login to dockerhub"
else
echo "Docker credentials provided. Login in."
docker login $DOCKER_ENDPOINT --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
fi

echo "Pulling $docker_repo_name"
(time docker pull $docker_repo_name) 2>&1

# Unzip resource files and set permissions
apt-get -y install unzip
chmod 777 $AZTK_WORKING_DIR/aztk/node_scripts/docker_main.sh

# Check docker is running
docker info > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "UNKNOWN - Unable to talk to the docker daemon"
exit 3
fi

echo "Node python version:"
python3 --version

# set up aztk python environment
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
python3 -m pip install pipenv
mkdir -p $AZTK_WORKING_DIR/.aztk-env
cp $AZTK_WORKING_DIR/aztk/node_scripts/Pipfile $AZTK_WORKING_DIR/.aztk-env
cp $AZTK_WORKING_DIR/aztk/node_scripts/Pipfile.lock $AZTK_WORKING_DIR/.aztk-env
cd $AZTK_WORKING_DIR/.aztk-env
export PIPENV_VENV_IN_PROJECT=true
pipenv install --python /usr/bin/python3.5m
pipenv run pip install --upgrade setuptools wheel #TODO: add pip when pipenv is compatible with pip10

# Install python dependencies
$AZTK_WORKING_DIR/.aztk-env/.venv/bin/pip install -r $(dirname $0)/requirements.txt
export PYTHONPATH=$PYTHONPATH:$AZTK_WORKING_DIR

# If the container already exists just restart. Otherwise create it
if [ "$(docker ps -a -q -f name=$container_name)" ]; then
echo "Docker container is already setup. Restarting it."
docker restart $container_name
else
echo "Creating docker container."

echo "Running setup python script"
$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python $(dirname $0)/main.py setup-node $docker_repo_name

# wait until container is running
until [ "`/usr/bin/docker inspect -f {{.State.Running}} $container_name`"=="true" ]; do
sleep 0.1;
done;

# wait until container setup is complete
echo "Waiting for spark docker container to setup."
docker exec spark /bin/bash -c '$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python $AZTK_WORKING_DIR/aztk/node_scripts/wait_until_setup_complete.py'

# Setup symbolic link for the docker logs
docker_log=$(docker inspect --format='{{.LogPath}}' $container_name)
mkdir -p $AZ_BATCH_TASK_WORKING_DIR/logs
ln -s $docker_log $AZ_BATCH_TASK_WORKING_DIR/logs/docker.log
fi
install_prerequisites () {
echo "Installing pre-reqs"

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"

packages=(
apt-transport-https
curl
ca-certificates
software-properties-common
python3-pip
python3-venv
docker-ce
)

echo "running apt-get install -y --no-install-recommends \"${packages[@]}\""
apt-get -y update &&
apt-get install -y --no-install-recommends "${packages[@]}"

if [ $AZTK_GPU_ENABLED == "true" ]; then
apt-get install -y nvidia-384 nvidia-modprobe
wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb
sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb
fi
echo "Finished installing pre-reqs"
}

install_docker_compose () {
echo "Installing Docker-Componse"
sudo curl -L https://github.com/docker/compose/releases/download/1.19.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
echo "Finished installing Docker-Compose"
}

pull_docker_container () {
echo "Pulling $docker_repo_name"

if [ -z "$DOCKER_USERNAME" ]; then
echo "No Credentials provided. No need to login to dockerhub"
else
echo "Docker credentials provided. Login in."
docker login $DOCKER_ENDPOINT --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
fi

docker pull $docker_repo_name
echo "Finished pulling $docker_repo_name"
}

install_python_dependencies () {
echo "Installing python dependencies"
pipenv install --python /usr/bin/python3.5m
pipenv run pip install --upgrade setuptools wheel #TODO: add pip when pipenv is compatible with pip10
echo "Finished installing python dependencies"

}

run_docker_container () {
echo "Running docker container"

# If the container already exists just restart. Otherwise create it
if [ "$(docker ps -a -q -f name=$container_name)" ]; then
echo "Docker container is already setup. Restarting it."
docker restart $container_name
else
echo "Creating docker container."

echo "Running setup python script"
$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python $(dirname $0)/main.py setup-node $docker_repo_name

# wait until container is running
until [ "`/usr/bin/docker inspect -f {{.State.Running}} $container_name`"=="true" ]; do
sleep 0.1;
done;

# wait until container setup is complete
echo "Waiting for spark docker container to setup."
docker exec spark /bin/bash -c '$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python $AZTK_WORKING_DIR/aztk/node_scripts/wait_until_setup_complete.py'

# Setup symbolic link for the docker logs
docker_log=$(docker inspect --format='{{.LogPath}}' $container_name)
mkdir -p $AZ_BATCH_TASK_WORKING_DIR/logs
ln -s $docker_log $AZ_BATCH_TASK_WORKING_DIR/logs/docker.log
fi
echo "Finished running docker container"

}





main () {

time(
install_prerequisites
) 2>&1


# set hostname in /etc/hosts if dns cannot resolve
if ! host $HOSTNAME ; then
echo $(hostname -I | awk '{print $1}') $HOSTNAME >> /etc/hosts
fi

time(
install_docker_compose
) 2>&1

time(
pull_docker_container
) 2>&1

# Unzip resource files and set permissions
chmod 777 $AZTK_WORKING_DIR/aztk/node_scripts/docker_main.sh

# Check docker is running
docker info > /dev/null 2>&1
if [ $? -ne 0 ]; then
echo "UNKNOWN - Unable to talk to the docker daemon"
exit 3
fi

echo "Node python version:"
python3 --version

# set up aztk python environment
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
python3 -m pip install pipenv
mkdir -p $AZTK_WORKING_DIR/.aztk-env
cp $AZTK_WORKING_DIR/aztk/node_scripts/Pipfile $AZTK_WORKING_DIR/.aztk-env
cp $AZTK_WORKING_DIR/aztk/node_scripts/Pipfile.lock $AZTK_WORKING_DIR/.aztk-env
cd $AZTK_WORKING_DIR/.aztk-env
export PIPENV_VENV_IN_PROJECT=true

time(
install_python_dependencies
) 2>&1

export PYTHONPATH=$PYTHONPATH:$AZTK_WORKING_DIR

time(
run_docker_container
) 2>&1
}

main
15 changes: 7 additions & 8 deletions aztk/spark/helpers/create_cluster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,17 +97,16 @@ def __cluster_install_cmd(zip_resource_file: batch_models.ResourceFile,
))

setup = [
'apt-get -y clean',
'apt-get -y update',
'apt-get install --fix-missing',
'apt-get -y install unzip',
'unzip -o $AZ_BATCH_TASK_WORKING_DIR/{0}'.format(
zip_resource_file.file_path),
'chmod 777 $AZ_BATCH_TASK_WORKING_DIR/aztk/node_scripts/setup_host.sh',
'time('\
'apt-get -y update;'\
'apt-get -y --no-install-recommends install unzip;'\
'unzip -o $AZ_BATCH_TASK_WORKING_DIR/{0};'\
'chmod 777 $AZ_BATCH_TASK_WORKING_DIR/aztk/node_scripts/setup_host.sh;'\
') 2>&1'.format(zip_resource_file.file_path),
'/bin/bash $AZ_BATCH_TASK_WORKING_DIR/aztk/node_scripts/setup_host.sh {0} {1}'.format(
constants.DOCKER_SPARK_CONTAINER_NAME,
docker_repo,
),
)
]

commands = shares + setup
Expand Down

0 comments on commit e5e529a

Please sign in to comment.