Skip to content

Commit

Permalink
Merge pull request #101 from AICONSlab/dev
Browse files Browse the repository at this point in the history 
Update README and readthedocs for new ACE flow.
  • Loading branch information
@anthonyprinaldi
anthonyprinaldi committed Jan 29, 2024
2 parents 1e70410 + ff138b7 commit cbd0c8a
Show file tree
Hide file tree
Showing 10 changed files with 412 additions and 123 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:
# Change permissions for code directory for any Singularity user
sed -i '/#STOPUNCOMMENT#/a # Change permissions for code dir\nchmod -R 777 /code' miracl.def && \
# Log in to cloud.sylabs.io
echo $SYLABS_TOKEN_DEC_22_2023 > tokenfile && \
echo $SYLABS_TOKEN_JAN_26_2024 > tokenfile && \
singularity remote login -u aiconslab --tokenfile tokenfile && \
# Setting trap in case image cannot be deleted
set -e
Expand Down
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ ___

\*\***NEW WORKFLOW/FEATURE RELEASE**\*\*

We will be releasing our AI-based Cartography of Ensembles (ACE) workflow, an end-to-end, automated pipeline that integrates cutting-edge deep learning
We have released our AI-based Cartography of Ensembles (ACE) workflow, an end-to-end, automated pipeline that integrates cutting-edge deep learning
segmentation models and advanced statistical methods to enable unbiased and generalizable brain-wide mapping of 3D alterations in neuronal activity,
morphology, or connectivity at the sub-regional and laminar levels beyond atlas-defined regions.

ACE will be available as part of MIRACL in **January 2024**.
ACE is now available [here](https://github.com/AICONSlab/MIRACL/tree/ace). Tutorials and usage examples for ACE can be found in our [docs](https://miracl.readthedocs.io/en/latest/tutorials/workflows/ace_flow/ace_flow.html).

___

Expand Down
238 changes: 186 additions & 52 deletions build.sh
View file
Original file line number Diff line number Diff line change
@@ -1,105 +1,213 @@
#!/bin/bash

# Build MIRACL Docker image with user information from host system
#

# - Checks if Docker and Docker Compose are installed
# - Dynamically adds and removes user information to
# prevent errors in Singularity container creation
# - Container user is the same as host user to prevent
# X11 issues with miraclGUI
#

# Current maintainer of script: Jonas Osmann @ github.com/jono3030
# MIRACL (C) Maged Groubran @ maged.goubran@utoronto.ca

# Set variable base state
#############
# VARIABLES #
#############

# Set variables base state
# Script version
version="1.1.4-beta"
version="2.0.0-beta"

# Set default log state
write_log=false
# Service name
service_name="miracl"

# Set default image name
img_name="mgoubran/miracl"
image_name="aicons/miracl"

# Set default container name
cont_name="miracl"
container_name="miracl"

# Set default image version
miracl_version="latest"

# Version file version
miracl_version_file=$(cat ./miracl/version.txt)

# Set default shared memory size
shm="64mb"

# Set array to capture volumes
volumes=()

# Base usage
base_usage="Usage: ./$(basename "$0") [-n service_name] [-i image_name] [-c container_name] [-t {auto, x.x.x}] [-g] [-v vol:vol] [-l] [-s] [-m] [-h]"

##########
# PARSER #
##########

# Usage function
function usage()
{

cat <<usage
Usage: ./build.sh [OPTIONS] [ARG] ...
$base_usage
Automatically build MIRACL Docker image with pseudo host user
Options:
-i, Specify image name (default: mgroubran/miracl)
-c, Specify container name (default: miracl)
-t, Set when using specific MIRACL tag/version (parses from miracl/version.txt)
-l, Write logfile of build process to build.log (default: false )
-v, Print version of build script and exit
-h, Print this help menu and exit
Version: $version
-n, name of the Docker service (default: 'miracl')
-i, specify image name (default: 'aicons/miracl')
-c, specify container name (default: 'miracl')
-t, set when using specific MIRACL tag/version. Use 'auto' to parse from 'miracl/version.txt' or specify version as floating point value in format 'x.x.x' (default: 'latest')
-g, enable Nvidia GPU passthrough mode for Docker container which is required for some of MIRACL's scripts e.g. ACE segmentation (default: false)
-d, set shared memory (shm) size which is important for some of MIRACL's scripts e.g. ACE segmentation (default: '64mb')
-v, mount volumes for MIRACL in docker-compose.yml, using a separate flag for each additional volume (format: '/path/on/host:/path/in/container'; default: none)
-l, write logfile of build process to 'build.log' in MIRACL root directory (default: false)
-s, print version of build script and exit
-m, print version of MIRACL on current Git branch and exit
-h, print this help menu and exit
Script version: $version
MIRACL version: $miracl_version_file
usage

}

# Parser for args and options
while getopts ':i:c:ltvh' opt; do
case "$opt" in
# Parse command line arguments
while getopts ":n:i:c:t:gd:v:lsmh" opt; do
case ${opt} in

n)
if [ "${OPTARG}" != "$service_name" ]; then
service_name=${OPTARG}
fi
;;

i)
if [ "${OPTARG}" != "$img_name" ]; then
img_name=${OPTARG}
if [ "${OPTARG}" != "$image_name" ]; then
image_name=${OPTARG}
fi
;;

c)
if [ "${OPTARG}" != "$cont_name" ]; then
cont_name=${OPTARG}
if [ "${OPTARG}" != "$container_name" ]; then
container_name=${OPTARG}
fi
;;

l)
write_log=true
t)
if [[ "${OPTARG}" == "auto" ]]; then
miracl_version=$miracl_version_file
elif [[ "${OPTARG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
miracl_version=${OPTARG}
else
echo 'Error: Version input is not a floating point number (format: x.x.x), "auto" or "latest"'
exit 1
fi
;;

t)
miracl_version=$(cat ./miracl/version.txt)
g)
gpu=true
;;

d)
if [ "${OPTARG}" != "$shm" ]; then
shm=${OPTARG}
fi
;;

v)
volumes+=("$OPTARG")
;;

l)
write_log=true
;;

s)
echo -e "v$version"
exit 0
;;

m)
echo -e "v$miracl_version_file"
exit 0
;;

h)
usage
exit 0
;;

:)
echo -e "Option requires an argument.\nUsage: $(basename $0) [-i arg] [-c arg]"
\?)
echo "Invalid option: -$OPTARG" 1>&2
printf "\n%s\n" "$base_usage"
exit 1
;;

?)
echo -e "Invalid command option.\nUsage: $(basename $0) [-i arg] [-c arg] [-l] [-v] [-h]"
:)
echo "Option -$OPTARG requires an argument." 1>&2
printf "\n%s\n" "$base_usage"
exit 1
;;

esac
done
shift "$(($OPTIND -1))"

shift "$((OPTIND -1))"

##################
# DOCKER COMPOSE #
##################

# Generate docker-compose.yml file
cat > docker-compose.yml <<EOF
version: "3.3"
services:
$service_name:
image: $image_name:$miracl_version
tty: true
environment:
- DISPLAY
stdin_open: true
network_mode: host
container_name: $container_name
shm_size: $shm
EOF

if [[ $gpu ]]; then

cat >> docker-compose.yml <<EOF
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
EOF

fi

# Append required volumes
cat >> docker-compose.yml <<EOF
volumes:
- /home/josmann/.Xauthority:/home/josmann/.Xauthority
EOF

# Add additional volumes
for v in "${volumes[@]}"; do
echo " - $v" >> docker-compose.yml
done

##############
# DOCKERFILE #
##############

# Information that needs to be added to Dockerfile
# to create pseudo host user. This is required to
# make X11 work correclty with miraclGUI
Expand All @@ -118,12 +226,7 @@ WORKDIR /home/\$USER
END
)

# Replace Docker image name and version
sed -i "s|\ \ \ \ image:.*|\ \ \ \ image: $img_name:$miracl_version|" docker-compose.yml

# Replace Docker container name
sed -i "s|\ \ \ \ container_name:.*|\ \ \ \ container_name: $cont_name|" docker-compose.yml

# Function to remove user text in Dockerfile if it is already present
function rm_USER_TEXT () {
sed -i '/STARTUNCOMMENT/,/STOPUNCOMMENT/{//!d}' Dockerfile
}
Expand All @@ -133,26 +236,57 @@ if [ "$(sed -n '/#STARTUNCOMMENT#/{n;p;}' Dockerfile)" != "#STOPUNCOMMENT#" ]; t
rm_USER_TEXT
fi

#########
# BUILD #
#########

# Check if Docker is installed
if [ -x "$(command -v docker)" ]; then
printf "\nDocker installation found. Building image.\n"
printf "\nDocker installation found.\n"
# Adding host user requirements to Dockerfile
sed -i "/#STARTUNCOMMENT#/a $USER_TEXT" Dockerfile
# Change user in docker-compose.yml to host user
export HOST_USER=$(whoami)
HOST_USER=$(whoami)
export HOST_USER
sed -i "s/\(\/home\/\).*\(\/.Xauthority:\/home\/\).*\(\/.Xauthority\)/\1$HOST_USER\2$HOST_USER\3/g" docker-compose.yml
# Build MIRACL image from Dockerfile

######################################
# Build MIRACL image from Dockerfile #
######################################

# Printing information about build process and docker-compose.yml to stdout
printf "\n[+] Building MIRACL and creating docker-compose.yml with the following parameters:\n"
printf " User: %s\n" "$HOST_USER"
printf " pid: %s\n" "$(id -u)"
printf " gid: %s\n" "$(id -g)"
printf " Service name: %s\n" "$service_name"
printf " Image name: %s\n" "$image_name:$miracl_version"
printf " Container name: %s\n" "$container_name"
if [[ $gpu ]]; then
printf " GPU passthrough: enabled\n"
else
printf " GPU passthrough: disabled\n"
fi
if [[ $write_log ]]; then
printf " Log file: enabled\n"
else
printf " Log file: disabled\n"
fi
for v in "${!volumes[@]}"; do
printf " Volume %s: %s\n" "$v" "${volumes[$v]}"
done

# Pass user name, UID and GID to Dockerfile
function docker_build () {
docker build \
--build-arg USER_ID=$(id -u) \
--build-arg GROUP_ID=$(id -g) \
--build-arg USER=$HOST_USER \
-t $img_name:$miracl_version .
--build-arg USER_ID="$(id -u)" \
--build-arg GROUP_ID="$(id -g)" \
--build-arg USER="$HOST_USER" \
-t "$image_name":"$miracl_version" .
}

# Check for log flag
if [ $write_log = true ];
if [ $write_log ];
then
docker_build | tee build.log
else
Expand All @@ -169,15 +303,15 @@ if [ -x "$(command -v docker)" ]; then
# Test if docker-compose is installed
# Should come by default with Docker-Desktop
if [ -x "$(command -v docker-compose)" ]; then
printf "Docker Compose installation found ($(docker-compose --version)). Run 'docker-compose up -d' to start the MIRACL container in background.\n"
printf "Docker Compose installation found (%s). Run 'docker-compose up -d' to start the MIRACL container in background.\n" "$(docker-compose --version)"
elif dcex=$(docker compose version); then
printf "Docker Compose installation found (%s). Run 'docker compose up -d' or use Docker Desktop (if installed) to start the MIRACL container in background.\n" "$dcex"
else
printf "Docker Compose installation not found. Please install Docker Compose plugin or standalone version to run the MIRACL container. Instructions on how to install Docker Compose can be found here: https://docs.docker.com/compose/install/\n"
fi
else
# Return error code if build was not successful
printf "\nBuild not successful! An error occured with exit code: $build_status_code\n"
printf "\nBuild not successful! An error occured with exit code: %s\n" $build_status_code
# Remove $USER_TEXT from Dockerfile
rm_USER_TEXT
# Exit with custom status code
Expand All @@ -187,7 +321,7 @@ if [ -x "$(command -v docker)" ]; then
else
di_status_code=$?
printf "\nDocker installation not found. Please install Docker first.\n"
printf "Exiting with status code $di_status_code\n"
printf "Exiting with status code %s\n" $di_status_code
# Exit with custom status code
exit $di_status_code
fi
fi
2 changes: 1 addition & 1 deletion docs/conf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@
'display_github': True,
'github_user': 'AICONSlab',
'github_repo': 'MIRACL',
'github_version': 'master/docs/',
'github_version': 'master/',
}

# Output file base name for HTML help builder.
Expand Down

0 comments on commit cbd0c8a

Please sign in to comment.