Optimizing build to use Docker caching effectively

Dockerfile

@@ -3,18 +3,24 @@ FROM ubuntu:18.04 as os_dependencies_ready
 RUN apt-get update -qq
 RUN apt-get install -y -qq git curl wget build-essential
 
-FROM os_dependencies_ready as libraries_and_modules_installed
+FROM os_dependencies_ready as backend_dependencies
+
+WORKDIR /archgraph_setup
 COPY ./conf /archgraph_setup/conf
 COPY ./requirements.txt /archgraph_setup/requirements.txt
 COPY ./frontend/package.json /archgraph_setup/frontend/package.json
-WORKDIR /archgraph_setup
-RUN ./conf/install.sh
+RUN ./conf/install_backend.sh
 
-FROM libraries_installed as archgraph_installed
+FROM backend_dependencies as frontend_dependencies
+RUN ./conf/install_frontend.sh
+
+FROM frontend_dependencies as archgraph_installed
 COPY . /archgraph
-COPY /archgraph_setup/frontend/node_modules /archgraph/frontend
+RUN mv /archgraph_setup/frontend/* /archgraph/frontend
 
 FROM archgraph_installed as cleanup_complete
 RUN rm -rf /archgraph_setup
 
+WORKDIR /archgraph
+
 ENTRYPOINT [ "./conf/run.sh" ]

conf/install.sh

@@ -1,81 +1,11 @@
 #!/bin/bash
 
-ROOT_DIR=$(pwd)
-echo "Running at $ROOT_DIR"
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-ARCHGRAPH_ENV="archgraph"
-PYTHON27_ENV="nodegyp-python27"
+source "$DIR/install_backend.sh"
+source "$DIR/install_frontend.sh"
 
-if  [ "$(uname)" == "Darwin" ]; then
-    # Do something under Mac OS X platform
-    if [ $(conda > /dev/null) > /dev/null ]; then
-        brew cask install miniconda || brew cask upgrade miniconda
-    else
-        echo "Miniconda already installed, continuing..."
-    fi
-elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ] ; then
-    if ! type conda &> /dev/null ; then
-        # Do something under GNU/Linux platform
-        rm -rf Miniconda3-latest-Linux-x86_64.sh
-        wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
-        chmod +x Miniconda3-latest-Linux-x86_64.sh
-        ./Miniconda3-latest-Linux-x86_64.sh -b -p "$HOME/miniconda" || ./Miniconda3-latest-Linux-x86_64.sh -u -b -p "$HOME/miniconda"
-        rm -rf Miniconda3-latest-Linux-x86_64.sh
-    else
-        echo "Miniconda already installed, continuing..."
-    fi
-fi
 
-# run conda
-if [ "$(uname)" == "Darwin" ]; then
-      source "$HOME/.bash_profile"
-elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
-    source "$HOME/.profile"
-fi
 
-if  [ "$(uname)" == "Darwin" ]; then
-    # make conda binary available in path
-    export PATH="$HOME/miniconda/bin":$PATH
-elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
-    # init conda cli
-    source "$HOME/miniconda/etc/profile.d/conda.sh"
-fi
 
-conda init bash
-conda create --quiet -y -n "$PYTHON27_ENV" python=2.7 anaconda
-conda activate "$PYTHON27_ENV"
-PYTHON27_PATH=$(which python)
-echo "Node-Gyp interpreter Python (2.7) is at: ---> ${PYTHON27_PATH} <---"
-
-# create archgraph env
-conda create --quiet -y -n "$ARCHGRAPH_ENV" python=3.7 anaconda
-conda activate "$ARCHGRAPH_ENV"
-
-# Get location of python interpreter
-PYTHON_PATH=$(which python)
-echo "Python interpreter is at: ---> ${PYTHON_PATH} <---"
-
-# install pip
-conda install pip -y --all --quiet
-echo "Pip at: ---> $(which pip) <---"
-# (optional) install any requirements of your current app in this venv
-pip install -r "$ROOT_DIR/requirements.txt"
-
-
-# install nodejs and yarn
-curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.2/install.sh | bash
-
-# activate nvm
-export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
-
-#install nodejs
-nvm install v10
-nvm use v10
-
-# install frontend stuff
-cd frontend
-npm install -g npm@latest
-npm config set python "$PYTHON27_PATH"
-npm install
 

conf/install_backend.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+ROOT_DIR=$(pwd)
+echo "Running at $ROOT_DIR"
+
+ARCHGRAPH_ENV="archgraph"
+PYTHON27_ENV="nodegyp-python27"
+
+if  [ "$(uname)" == "Darwin" ]; then
+    # Do something under Mac OS X platform
+    if [ $(conda > /dev/null) > /dev/null ]; then
+        brew cask install miniconda || brew cask upgrade miniconda
+    else
+        echo "Miniconda already installed, continuing..."
+    fi
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ] ; then
+    if ! type conda &> /dev/null ; then
+        # Do something under GNU/Linux platform
+        rm -rf Miniconda3-latest-Linux-x86_64.sh
+        wget --quiet https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+        chmod +x Miniconda3-latest-Linux-x86_64.sh
+        ./Miniconda3-latest-Linux-x86_64.sh -b -p "$HOME/miniconda" || ./Miniconda3-latest-Linux-x86_64.sh -u -b -p "$HOME/miniconda"
+        rm -rf Miniconda3-latest-Linux-x86_64.sh
+    else
+        echo "Miniconda already installed, continuing..."
+    fi
+fi
+
+# run conda
+if [ "$(uname)" == "Darwin" ]; then
+      source "$HOME/.bash_profile"
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+    source "$HOME/.profile"
+fi
+
+if  [ "$(uname)" == "Darwin" ]; then
+    # make conda binary available in path
+    export PATH="$HOME/miniconda/bin":$PATH
+elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
+    # init conda cli
+    source "$HOME/miniconda/etc/profile.d/conda.sh"
+fi
+
+conda init bash
+conda create --quiet -y -n "$PYTHON27_ENV" python=2.7 anaconda
+conda activate "$PYTHON27_ENV"
+PYTHON27_PATH=$(which python)
+echo "Node-Gyp interpreter Python (2.7) is at: ---> ${PYTHON27_PATH} <---"
+
+# create archgraph env
+conda create --quiet -y -n "$ARCHGRAPH_ENV" python=3.7 anaconda
+conda activate "$ARCHGRAPH_ENV"
+
+# Get location of python interpreter
+PYTHON_PATH=$(which python)
+echo "Python interpreter is at: ---> ${PYTHON_PATH} <---"
+
+# install pip
+conda install pip -y --all --quiet
+echo "Pip at: ---> $(which pip) <---"
+# (optional) install any requirements of your current app in this venv
+pip install -r "$ROOT_DIR/requirements.txt"

conf/install_frontend.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+ROOT_DIR=$(pwd)
+echo "Installing frontend dependencies at $ROOT_DIR"
+
+ARCHGRAPH_ENV="archgraph"
+PYTHON27_ENV="nodegyp-python27"
+
+# install nodejs and yarn
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.2/install.sh | bash
+
+# activate nvm
+export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
+
+#install nodejs
+nvm install v10
+nvm use v10
+
+# install frontend stuff
+cd "$ROOT_DIR/frontend"
+npm install -g npm@latest
+npm config set python "$PYTHON27_PATH"
+npm install
+echo "Current directory: $(pwd)"
+ls -la
+cd -

conf/run.sh

@@ -37,20 +37,6 @@ export NVM_DIR="$([[ -z "${XDG_CONFIG_HOME-}" ]] && printf %s "${HOME}/.nvm" ||
 [[ -s "$NVM_DIR/nvm.sh" ]] && \. "$NVM_DIR/nvm.sh" # This loads nvm
 nvm use v10
 
-# preload graph
-if [[ -z "$INIT_GRAPH" ]] ; then
-    echo "Preload graph flag is not active, skipping tests"
-elif [[ ! -f "$ROOT_DIR/.preloaded.txt" ]] || [[ "$FORCE_RELOAD_GRAPH" == "1" ]] ; then
-    rm -f "$ROOT_DIR/.preloaded.txt"
-    echo "Preload graph flag is active, loading graph through tests"
-    coverage run -m unittest discover test
-    echo "true" > "$ROOT_DIR/.preloaded.txt"
-else
-    echo "Preload graph flag is active but the database has already been initialized once. \
-    To re-initialize, delete the $ROOT_DIR/.preloaded.txt file and run this script again, or \
-    set the FORCE_RELOAD_GRAPH environment variable before re-running this script."
-fi
-
 echo "Starting archgraph server at $ROOT_DIR"
 cd "$ROOT_DIR"
 
@@ -84,10 +70,21 @@ fi
 
 ## wait for servers to be active before running the application
 
-./conf/wait-for-it.sh "$MONGODB_HOST:$MONGODB_PORT" --timeout=60 &
-./conf/wait-for-it.sh "$NEO4J_HOST:$NEO4J_PORT" --timeout=60
+./conf/wait-for-it.sh "$NEO4J_HOST:$NEO4J_PORT" --timeout=120
 
-wait
+# preload graph
+if [[ -z "$INIT_GRAPH" ]] ; then
+    echo "Preload graph flag is not active, skipping tests"
+elif [[ ! -f "$ROOT_DIR/.preloaded.txt" ]] || [[ "$FORCE_RELOAD_GRAPH" == "1" ]] ; then
+    rm -f "$ROOT_DIR/.preloaded.txt"
+    echo "Preload graph flag is active, loading graph through tests"
+    coverage run -m unittest discover test
+    echo "true" > "$ROOT_DIR/.preloaded.txt"
+else
+    echo "Preload graph flag is active but the database has already been initialized once. \
+    To re-initialize, delete the $ROOT_DIR/.preloaded.txt file and run this script again, or \
+    set the FORCE_RELOAD_GRAPH environment variable before re-running this script."
+fi
 
 python "$ROOT_DIR/src/Routes/routes.py" &
 SERVER_PID=$!

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.3'
 services:
   mongodb:
     container_name: archgraph-mongodb
-    image: mongo:latest
+    image: library/mongo:4.2.1
     command: ["--bind_ip_all"]
     environment:
       MONGO_INITDB_ROOT_USERNAME: root

src/Routes/mongo.py

@@ -11,6 +11,7 @@
 client = MongoClient(
     host=EnvVarManager.get_from_env_or_return_default("MONGODB_HOST", "localhost"),
     port=int(EnvVarManager.get_from_env_or_return_default("MONGODB_PORT", "27017")),
+    connectTimeoutMS=120000
 )
 db = client.mydatabase
 date_now = datetime.now().strftime("%Y-%m-%d, %H:%M:%S")

src/Utils/EnvVarManager.py

@@ -5,14 +5,17 @@ def parse():
     import os
 
     args = dict()
-    if "NEO4J_CONNECTION_STRING" in os.environ:
-        args["NEO4J_CONNECTION_STRING"] = os.environ["NEO4J_CONNECTION_STRING"]
+    if "NEO4J_HOST" in os.environ:
+        args["NEO4J_HOST"] = os.environ["NEO4J_HOST"]
 
-    if "MONGODB_CONNECTION_STRING" in os.environ:
-        args["MONGODB_CONNECTION_STRING"] = os.environ["MONGODB_CONNECTION_STRING"]
+    if "NEO4J_PORT" in os.environ:
+        args["NEO4J_PORT"] = os.environ["NEO4J_PORT"]
 
-    if "CUSTOM_HOST_FOR_SERVER_BIND" in os.environ:
-        args["CUSTOM_HOST_FOR_SERVER_BIND"] = os.environ["CUSTOM_HOST_FOR_SERVER_BIND"]
+    if "MONGODB_HOST" in os.environ:
+        args["MONGODB_HOST"] = os.environ["MONGODB_HOST"]
+
+    if "MONGODB_PORT" in os.environ:
+        args["MONGODB_PORT"] = os.environ["MONGODB_PORT"]
 
     if "CUSTOM_HOST_FOR_SERVER_BIND" in os.environ:
         args["CUSTOM_HOST_FOR_SERVER_BIND"] = os.environ["CUSTOM_HOST_FOR_SERVER_BIND"]

src/Utils/Utils.py

@@ -293,9 +293,16 @@
 
 import src.Utils.Utils as self
 
-uri = "bolt://localhost:7687"
-driver = None
+import src.Utils.EnvVarManager as EnvVarManager
+
+uri = (
+    "bolt://neo4j:password@"
+    + EnvVarManager.get_from_env_or_return_default("NEO4J_HOST", "127.0.0.1")
+    + ":"
+    + EnvVarManager.get_from_env_or_return_default("NEO4J_PORT", "27017")
+)
 
+driver = None
 
 def get_driver():
     if self.driver is None: