Tests running, still very slow

feup-infolab · Apr 22, 2018 · fd2f22d · fd2f22d
1 parent 6e8c57c
commit fd2f22d
Show file tree

Hide file tree

Showing 8 changed files with 961 additions and 1,095 deletions.
diff --git a/conf/deployment_configs.json b/conf/deployment_configs.json
@@ -1571,8 +1571,8 @@
         },
         "numCPUs": 1,
         "docker": {
-          "active": false,
-          "reuse_checkpoints": true
+          "active": true,
+          "reuse_checkpoints": false
         },
         "virtualbox": {
           "active": false,

diff --git a/conf/scripts/docker/start_containers.sh b/conf/scripts/docker/start_containers.sh
@@ -1,47 +1,105 @@
 #!/usr/bin/env bash
 
-# Start containers
+# DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG
+# stop and destroy all containers
+# docker stop $(docker ps -aq)
+# docker rm $(docker ps -aq)
+# docker system prune
+# DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG
 
-RUNNING_FOLDER=$(pwd)/data/current
-rm -rf RUNNING_FOLDER
+# Get network for the containers
+NETWORK_NAME="$1"
+NETWORK_SUBNET="$2"
 
-# create data directories to mount data folders in containers
-mkdir -p $RUNNING_FOLDER/virtuoso
-mkdir -p $RUNNING_FOLDER/elasticsearch
-mkdir -p $RUNNING_FOLDER/mysql
-mkdir -p $RUNNING_FOLDER/mongo
-#mkdir -p $RUNNING_FOLDER/redis-default
-#mkdir -p $RUNNING_FOLDER/redis-social
-#mkdir -p $RUNNING_FOLDER/redis-notifications
+echo "Network Name: $NETWORK_NAME"
+
+# adjust parameters for containers if a custom subnet is being used
+if [[ "$NETWORK_NAME" != "" && "$NETWORK_SUBNET" != "" ]]
+then
+    # calculate IP addresses of the containers
+    ELASTICSEARCH_IP="${NETWORK_SUBNET}.1"
+    echo "ELASTICSEARCH_IP: $ELASTICSEARCH_IP"
+
+    VIRTUOSO_IP="${NETWORK_SUBNET}.2"
+    echo "VIRTUOSO_IP: $VIRTUOSO_IP"
+
+    MYSQL_IP="${NETWORK_SUBNET}.3"
+    echo "MYSQL_IP: $MYSQL_IP"
+
+    MONGODB_IP="${NETWORK_SUBNET}.4"
+    echo "MONGODB_IP: $MONGODB_IP"
+
+    NETWORK_SUBNET="${NETWORK_SUBNET}.0/25" # max 255 containers on this subnet
+    echo "Network Subnet: $NETWORK_SUBNET"
+    echo "Creating docker subnet $NETWORK_NAME with value $NETWORK_SUBNET..."
+    docker network rm "$NETWORK_NAME"
+    docker network create \
+      --driver=bridge \
+      --subnet="$NETWORK_SUBNET" \
+      "$NETWORK_NAME"
+
+    # set running folder with network name suffix
+    RUNNING_FOLDER=$(pwd)/data/$NETWORK_NAME/current
+
+    #set container names (WITH SUBNET)
+    ELASTICSEARCH_CONTAINER_NAME="elasticsearch-dendro-${NETWORK_NAME}"
+    VIRTUOSO_CONTAINER_NAME="virtuoso-dendro-${NETWORK_NAME}"
+    MYSQL_CONTAINER_NAME="mysql-dendro-${NETWORK_NAME}"
+    MONGODB_CONTAINER_NAME="mongodb-dendro-${NETWORK_NAME}"
+else
+    # set running folder with network name suffix
+    RUNNING_FOLDER=$(pwd)/data/current
+    #set container names
+    ELASTICSEARCH_CONTAINER_NAME="elasticsearch-dendro"
+    VIRTUOSO_CONTAINER_NAME="virtuoso-dendro"
+    MYSQL_CONTAINER_NAME="mysql-dendro"
+    MONGODB_CONTAINER_NAME="mongodb-dendro"
+fi
 
-# start containers with the volumes mounted
+# create data directories to mount data folders in containers
+# rm -rf "$RUNNING_FOLDER"
+# mkdir -p "$RUNNING_FOLDER/virtuoso"
+# mkdir -p "$RUNNING_FOLDER/elasticsearch"
+# mkdir -p "$RUNNING_FOLDER/mysql"
+# mkdir -p "$RUNNING_FOLDER/mongo"
 
+# starts containers with the volumes mounted
 function wait_for_server_to_boot_on_port()
 {
-    local port=$1
+    local ip=$1
+
+    if [[ $ip == "" ]]; then
+      ip="127.0.0.1"
+    fi
+    local port=$2
     local attempts=0
     local max_attempts=60
 
-    echo "Waiting for server on port $port to boot up..."
+    echo "Waiting for server on $ip:$port to boot up..."
 
-    while nc 127.0.0.1 "$port" < /dev/null > /dev/null && [[ $attempts < $max_attempts ]]  ; do
+    # curl -s $URL 2>&1 > /dev/null
+    # nc "$ip" "$port" < /dev/null > /dev/null
+    while curl -s "$ip:$port" > /dev/null && [[ $attempts < $max_attempts ]]  ; do
         attempts=$((attempts+1))
         sleep 1;
         echo "waiting... (${attempts}/${max_attempts})"
     done
 
     if (( $attempts == $max_attempts ));
     then
-        echo "Server on port $port failed to start after $max_attempts"
+        echo "Server on $ip:$port failed to start after $max_attempts"
     elif (( $attempts < $max_attempts ));
     then
-        echo "Server on port $port started successfully at attempt (${attempts}/${max_attempts})"
+        echo "Server on $ip:$port started successfully at attempt (${attempts}/${max_attempts})"
     fi
 }
 
+
+# checks if a container is running or not
 function container_running
 {
-    container_name=$1
+    local container_name=$1
+    local running
     running=$(docker inspect -f "{{.State.Running}}" "$container_name")
     if [[ "$running" == "true" ]]
     then
@@ -53,93 +111,115 @@ function container_running
     fi
 }
 
-if container_running "elasticsearch-dendro" == 0
+## ELASTICSEARCH
+
+echo $RUNNING_FOLDER
+
+printf "\n\n"
+if container_running "$ELASTICSEARCH_CONTAINER_NAME" == 0
 then
-    ( docker start elasticsearch-dendro 1> /dev/null || \
-    docker pull docker.elastic.co/elasticsearch/elasticsearch:6.2.2 && \
-    docker run --name elasticsearch-dendro \
-        -p 9200:9200 \
-        -p 9300:9300 \
-        -e "discovery.type=single-node" \
-        -e "http.host=0.0.0.0" \
-        -e "transport.host=127.0.0.1" \
-        -v "$RUNNING_FOLDER/elasticsearch:/usr/share/elasticsearch/data" \
-        -d docker.elastic.co/elasticsearch/elasticsearch:6.2.2 ) && \
-        echo "Container elasticsearch-dendro started."
+    if ! docker start "$ELASTICSEARCH_CONTAINER_NAME"
+    then
+      docker pull docker.elastic.co/elasticsearch/elasticsearch:6.2.2
+      docker run \
+          -p 9200:9200 \
+          -p 9300:9300 \
+          -e "discovery.type=single-node" \
+          -e "http.host=0.0.0.0" \
+          -e "transport.host=${ELASTICSEARCH_IP}" \
+          -v "$RUNNING_FOLDER/elasticsearch:/usr/share/elasticsearch/data" \
+          --name "$ELASTICSEARCH_CONTAINER_NAME" \
+          -d docker.elastic.co/elasticsearch/elasticsearch:6.2.2
+
+          # --net "$NETWORK_NAME" \
+          # --ip "$ELASTICSEARCH_IP" \
+
+      docker start "$ELASTICSEARCH_CONTAINER_NAME" && echo "Container $ELASTICSEARCH_CONTAINER_NAME started." || echo "Container $ELASTICSEARCH_CONTAINER_NAME failed to start."
+    fi
 fi
 
-wait_for_server_to_boot_on_port 9200
-wait_for_server_to_boot_on_port 9300
+wait_for_server_to_boot_on_port "$ELASTICSEARCH_IP" 9200
+wait_for_server_to_boot_on_port "$ELASTICSEARCH_IP" 9300
+# docker ps -a
 
-if container_running "virtuoso-dendro" == 0
+## VIRTUOSO
+
+printf "\n\n"
+if container_running "$VIRTUOSO_CONTAINER_NAME" == 0
 then
-    ( docker start virtuoso-dendro 1> /dev/null || \
-    docker run --name virtuoso-dendro \
-        -p 8890:8890 \
-        -p 1111:1111 \
-        -e SPARQL_UPDATE=true \
-        -e "VIRT_Parameters_CheckpointSyncMode=2" \
-        -e "VIRT_Parameters_PageMapCheck=1" \
-        -e "VIRT_Parameters_CheckpointInterval=0" \
-        -v "$RUNNING_FOLDER/virtuoso:/data" \
-        -d tenforce/virtuoso:1.3.1-virtuoso7.2.4 ) && \
-        echo "Container virtuoso-dendro started."
+    if ! docker start "$VIRTUOSO_CONTAINER_NAME"
+    then
+      docker pull tenforce/virtuoso:1.3.1-virtuoso7.2.4
+      docker run \
+          -p 8890:8890 \
+          -p 1111:1111 \
+          -e SPARQL_UPDATE=true \
+          -e "VIRT_Parameters_CheckpointSyncMode=2" \
+          -e "VIRT_Parameters_PageMapCheck=1" \
+          -e "VIRT_Parameters_CheckpointInterval=0" \
+          -v "$RUNNING_FOLDER/virtuoso:/data" \
+          --name "$VIRTUOSO_CONTAINER_NAME" \
+          -d tenforce/virtuoso:1.3.1-virtuoso7.2.4
+
+          # --net "$NETWORK_NAME" \
+          # --ip "$VIRTUOSO_IP" \
+
+      docker start $VIRTUOSO_CONTAINER_NAME && echo "Container $VIRTUOSO_CONTAINER_NAME started." || echo "Container $VIRTUOSO_CONTAINER_NAME failed to start."
+    fi
 fi
 
 # -e "VIRT_Parameters_NumberOfBuffers=$((32*85000))" \
 
-wait_for_server_to_boot_on_port 8890
-wait_for_server_to_boot_on_port 1111
+wait_for_server_to_boot_on_port "$VIRTUOSO_IP" 8890
+wait_for_server_to_boot_on_port "$VIRTUOSO_IP" 1111
+# docker ps -a
 
-if container_running "mysql-dendro" == 0
+## MYSQL
+
+printf "\n\n"
+if container_running "$MYSQL_CONTAINER_NAME" == 0
 then
-    ( docker start mysql-dendro 1> /dev/null || \
-    docker run --name mysql-dendro \
-      -p 3306:3306 \
-      -e MYSQL_ROOT_PASSWORD=r00t \
-      -v "$RUNNING_FOLDER/mysql:/var/lib/mysql" \
-      -d mysql:8.0.3 || docker start mysql-dendro 1> /dev/null ) && \
-      echo "Container mysql-dendro started."
+    if ! docker start "$MYSQL_CONTAINER_NAME"
+    then
+      docker pull mysql:8.0.3
+      docker run \
+        -p 3306:3306 \
+        -e MYSQL_ROOT_PASSWORD=r00t \
+        -v "$RUNNING_FOLDER/mysql:/var/lib/mysql" \
+        --name "$MYSQL_CONTAINER_NAME" \
+        -d mysql:8.0.3
+
+        # --net "$NETWORK_NAME" \
+        # --ip "$MYSQL_IP" \
+
+        docker start $MYSQL_CONTAINER_NAME && echo "Container $MYSQL_CONTAINER_NAME started." || echo "Container $MYSQL_CONTAINER_NAME failed to start."
+        # "$(get_network_arguments $NETWORK_NAME $MYSQL_IP)" \
+    fi
 fi
 
-wait_for_server_to_boot_on_port 3306
+wait_for_server_to_boot_on_port "$MYSQL_IP" 3306
+# docker ps -a
 
-if container_running "mongo-dendro" == 0
+## MONGODB
+
+printf "\n\n"
+if container_running "$MONGODB_CONTAINER_NAME" == 0
 then
-    ( docker start mongo-dendro 1> /dev/null || \
-    docker run --name mongo-dendro \
-        -p 27017:27017 \
-        -v "$RUNNING_FOLDER/mongo:/data/db" \
-        -d mongo:3.4.10 ) && \
-    echo "Container mongo-dendro started."
+    if ! docker start "$MONGODB_CONTAINER_NAME"
+    then
+      docker pull mongo:3.4.10
+      docker run \
+          -p 27017:27017 \
+          -v "$RUNNING_FOLDER/mongo:/data/db" \
+          --name "$MONGODB_CONTAINER_NAME" \
+          -d mongo:3.4.10
+
+          # --net "$NETWORK_NAME" \
+          # --ip "$MONGODB_IP" \
+
+          docker start $MONGODB_CONTAINER_NAME &&  echo "Container $MONGODB_CONTAINER_NAME started." || echo "Container $MONGODB_CONTAINER_NAME failed to start."
+    fi
 fi
 
-wait_for_server_to_boot_on_port 27017
-
-#docker run --name redis-dendro-default \
-#    -p 6781:6780 \
-#    -v "$RUNNING_FOLDER/redis-default:/data" \
-#    -d redis:3.2.11 || docker start redis-dendro-default || docker restart redis-dendro-default
-#
-#docker run --name redis-dendro-social \
-#    -p 6782:6780 \
-#    -v "$RUNNING_FOLDER/redis-default:/data" \
-#    -d redis:3.2.11 || docker start redis-dendro-social || docker restart redis-dendro-social
-#
-#docker run --name redis-dendro-notifications \
-#    -p 6783:6780 \
-#    -v "$RUNNING_FOLDER/redis-default:/data" \
-#    -d redis:3.2.11 || docker start redis-dendro-notifications || docker restartredis-dendro-notifications
-
-#docker ps
-
-
-#docker run --name virtuoso-dendro \
-#    -p 8890:8890 \
-#    -p 1111:1111 \
-#    -e SPARQL_UPDATE=true \
-#    -v $RUNNING_FOLDER/virtuoso:/data \
-#    -d joaorosilva/virtuoso-7.2.2-dendro || docker start virtuoso-dendro-remote
-
-#for elasticsearch
-#-Des.network.host=0.0.0.0
+wait_for_server_to_boot_on_port "$MONGODB_IP" 27017
+# docker ps -a