Bug/MDD179: MISP Modules is not Starting Correctly (#6)

## Bug/MDD179: MISP Modules is not Starting Correctly ### Update Information We added a new Version 1.3-debian. The version 1.2-debian does not start correctly. ### General Changes In general, we made no changes. ### Corrections & Improvements - We added a new container version 1.3-debian - Improved CI scripts ### Detailed changes - We added a new Version 1.3-debian. The version 1.2-debian does not start correctly. The root cause was that MISP modules Git repository adds a pip package which is not available for Python 3.5.x. In Debian is Python 3.5.x the default. The new Container 1.3-debian has now Installed Python 3.7.x from Debian unstable branch. This fixes the problem with pip package url-normalizer. - We improved the CI files to a more easier way. Travis CI only test the Image, but our internal Gitlab CI build, test, and push the Images to internal Docker registry and hub.docker.com registry.
DCSO · Feb 14, 2019 · 136958b · 136958b
1 parent ca460a0
commit 136958b
Show file tree

Hide file tree

Showing 13 changed files with 546 additions and 47 deletions.
diff --git a/.ci/01_before_install.sh b/.ci/01_before_install.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+STARTMSG="[before_install]"
+
+# Install Requirements
+echo
+echo "$STARTMSG Install requirements..."
+    [ ! -z "$(which apk)" ] && apk add --no-cache make bash sudo git curl coreutils grep python3
+    [ ! -z "$(which apt-get)" ] && apt-get update; 
+    [ ! -z "$(which apt-get)" ] && apt-get install make bash sudo git curl coreutils grep python3
+    # Upgrade Docke
+    [ ! -z "$(which apt-get)" ] && apt-get install --only-upgrade docker-ce -y
+# Install docker-compose
+    # https://stackoverflow.com/questions/42295457/using-docker-compose-in-a-gitlab-ci-pipeline
+    [ -z "$(which docker-compose)" ] && pip3 install --no-cache-dir docker-compose
+# Show version of docker-compose:
+    docker-compose -v
+
+# Set Git Options
+    echo
+    echo "$STARTMSG Set Git options..."
+    git config --global user.name "MISP-dockerized-bot"
+
+echo "$STARTMSG $0 is finished."
diff --git a/.ci/02_build.sh b/.ci/02_build.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+STARTMSG="[build]"
+
+[ -z "$1" ] && echo "$STARTMSG No parameter with the image version. Exit now." && exit 1
+[ "$1" == "dev" ] && echo "$STARTMSG False first argument. Abort." && exit 1
+
+VERSION="$1"
+if [[ "$2" == "true" ]]; then ENVIRONMENT="prod"; fi;
+
+
+#################   MANUAL VARIABLES #################
+# path of the script
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+# dockerfile name:
+DOCKERFILE_NAME=Dockerfile
+# Which Folder the script should use
+
+echo "$STARTMSG Index all versions..."
+if [ -z $1 ] ;then
+    	# build all you find
+        FOLDER=( */)
+        FOLDER=( "${FOLDER[@]%/}" )
+else
+    # build only the argumented one
+    FOLDER="$VERSION"
+fi
+#########################################################
+
+#################   AUTOMATIC VARIABLES #################
+# Find Out Git Hub Repository
+echo "$STARTMSG Set GIT_REPO..."
+if [ ! -z "$(git remote get-url origin|grep git@)" ]
+then
+    GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
+elif [ ! -z "$(git remote get-url origin|grep http)" ] 
+then    
+    GIT_REPO="$(git remote get-url origin|sed 's,http.*//.*/,,'|sed 's,....$,,')"
+elif [ ! -z "$(echo $GIT_REPO|grep $GITLAB_HOST)" ] 
+then
+    GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
+else
+    echo "Can not found the Git URL. Exit now."
+    exit 1
+fi
+
+GIT_REPO_URL="https://github.com/$GIT_REPO"
+# Dockerifle Settings
+CONTAINER_NAME="$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
+DOCKER_REPO="not2push/$CONTAINER_NAME"
+#########################################################
+
+echo "$STARTMSG Start image building..."
+for FOLD in ${FOLDER[@]}
+do  
+    # Find Out Version from folder
+    VERSION=$(echo $FOLD|cut -d- -f 1)
+    DOCKERFILE_PATH="$SCRIPTPATH/../$FOLD"
+    # Load Variables from configuration file
+    source $DOCKERFILE_PATH/configuration.sh
+    # Default mode add "-dev" tag.
+    if [ "$ENVIRONMENT" == "prod" ]
+    then
+        # PROD Version
+        TAGS="-t $DOCKER_REPO:$FOLD"
+    else
+        # DEV Version
+        TAGS="-t $DOCKER_REPO:$FOLD-dev"
+    fi
+
+    # Default build args
+    BUILD_ARGS+="
+        --build-arg BUILD_DATE="$(date -u +"%Y-%m-%d")" \
+        --build-arg NAME="$CONTAINER_NAME" \
+        --build-arg GIT_REPO="$GIT_REPO_URL" \
+        --build-arg VCS_REF=$(git rev-parse --short HEAD) \
+        --build-arg VERSION="$VERSION" \
+    "
+    # build image
+    docker build \
+            $BUILD_ARGS \
+        -f $DOCKERFILE_PATH/$DOCKERFILE_NAME $TAGS $DOCKERFILE_PATH/
+done
+
+echo "$STARTMSG $0 is finished."
diff --git a/.ci/03_tagging.sh b/.ci/03_tagging.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+STARTMSG="[tagging]"
+
+[ -z "$1" ] && echo "$STARTMSG No parameter with the image version. Exit now." && exit 1
+[ "$1" == "true" ] && echo "$STARTMSG False first argument. Abort." && exit 1
+
+REGISTRY_URL="$1"
+if [[ "$2" == "true" ]]; then ENVIRONMENT="prod"; fi;
+
+# change directory to the top level:
+pushd ..
+
+# Docker Repo e.g. dcso/misp-dockerized-proxy
+[ -z "$(git remote get-url origin|grep git@)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
+[ -z "$(git remote get-url origin|grep http)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*github.com/,,'|sed 's,....$,,')"
+[ -z "$GITLAB_HOST" ] || [ -z "$(echo $GIT_REPO|grep $GITLAB_HOST)" ] ||  GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
+
+# Set Container Name
+CONTAINER_NAME="$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
+
+# Show Images before tagging
+echo  "$STARTMSG ### Show images before tagging:"
+docker images | grep $CONTAINER_NAME
+
+# Set Docker Repository
+DOCKER_REPO="$REGISTRY_URL/$CONTAINER_NAME"
+SOURCE_REPO="not2push"
+
+# Search the latest image
+    # Create the Array
+    FOLDER_ARRAY=( */)
+    FOLDER_ARRAY=( "${FOLDER_ARRAY[@]%/}" )
+    # How many items in your Array:
+    index=${#FOLDER_ARRAY[@]}       
+
+    # SORT ARRAY
+    IFS=$'\n' 
+        sorted=($(sort <<<"${FOLDER_ARRAY[*]}"))
+    unset IFS
+
+    # Latest Version
+    LATEST=$(echo ${sorted[$index-1]}|cut -d- -f 1)
+
+# Search the current major version
+    # All Latest Major versions
+    MAJOR_LATEST=""
+    # Run over all FOLDER versions and add all first digit numbers
+    for i in ${sorted[@]}
+    do
+        # change from 1.0-ubuntu -> 1
+        CURRENT_MAJOR_VERSION="$(echo $i|cut -d . -f 1)"
+        CURRENT_MINOR_VERSION="$(echo $i|cut -d . -f 2|cut -d - -f 1)"
+
+        # Check if there is any Version available for the current MAJOR version:
+        [ -z ${MAJOR_LATEST[$CURRENT_MAJOR_LATEST]} ] && MAJOR_LATEST[$CURRENT_MAJOR_VERSION]=$i && continue
+
+        # change the Folder Name which are written into the Array on position of the current_major_version from 1.0-ubuntu to 1
+        LIST_MINOR_VERSION=$(echo ${MAJOR_LATEST[$CURRENT_MAJOR_VERSION]}|cut -d . -f 2|cut -d - -f 1)
+        # Check if the current minor digit from Elelement i is higher than the one which are saved in the array
+        [[ $LIST_MINOR_VERSION < $CURRENT_MINOR_VERSION ]] && MAJOR_LATEST[$CURRENT_MAJOR_VERSION]=$i && continue
+    done
+
+
+# Lookup to all build versions of the current docker container
+ALL_BUILD_DOCKER_VERSIONS=$(docker images --format '{{.Repository}}={{.Tag}}'|grep $CONTAINER_NAME |cut -d = -f 2)
+
+# Tag Latest + Version Number
+for i in $ALL_BUILD_DOCKER_VERSIONS
+do
+    VERSION=$(echo $i|cut -d- -f 1)                 # for example 1.0
+    BASE=$(echo $i|cut -d- -f 2)                    # for example ubuntu
+    MAJOR_VERSION="$(echo $i|cut -d . -f 1)"        # for example 1
+
+    # Remove '-dev' tag
+    if [ "$ENVIRONMENT" == "prod" ]; then
+        #
+        #   If prod=true, ~ prodcutin ready image
+        #
+
+        # Add custom Docker registry tag
+        docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$VERSION-$BASE
+
+        # Add latest tag
+        if [ "$VERSION" == "$LATEST" ]; then
+            docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:latest
+        fi
+
+        # Add latest Major Version Tag
+        for k in ${MAJOR_LATEST[@]}
+        do
+            CURRENT_MAJOR_VERSION="$(echo $k|cut -d . -f 1)"
+            [ "$i" == $k"-dev" ] && docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$CURRENT_MAJOR_VERSION
+        done
+    else
+        #
+        #   Add '-dev' tag
+        #   
+
+        # Add custom Docker registry tag
+        docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$VERSION-$BASE-dev
+
+        # Add latest tag
+        if [ "$VERSION" == "$LATEST" ]; then
+            docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:latest-dev
+        fi
+
+        # Add latest Major Version Tag
+        for k in ${MAJOR_LATEST[@]}
+        do
+            CURRENT_MAJOR_VERSION="$(echo $k|cut -d . -f 1)"
+            [ "$i" == $k"-dev" ] && docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$CURRENT_MAJOR_VERSION-dev
+        done
+    fi
+done
+
+echo  "$STARTMSG ### Show images after tagging:"
+docker images | grep $DOCKER_REPO
+
+echo "$STARTMSG $0 is finished."
+
diff --git a/.ci/04_push.sh b/.ci/04_push.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+STARTMSG="[push]"
+
+[ -z "$1" ] && echo "$STARTMSG No parameter with the Docker registry URL. Exit now." && exit 1
+[ "$1" == "NOT2PUSH" ] && echo "$STARTMSG The NOT2PUSH slug is only for local build and retag not for pushin to docker registries. Exit now." && exit 1
+[ -z "$2" ] && echo "$STARTMSG No parameter with the Docker registry username. Exit now." && exit 1
+[ -z "$3" ] && echo "$STARTMSG No parameter with the Docker registry password. Exit now." && exit 1
+
+REGISTRY_URL="$1"
+REGISTRY_USER="$2"
+REGISTRY_PW="$3"
+
+##################################
+
+# Find the right Docker Repo name e.g. dcso/misp-dockerized-proxy
+[ -z "$(git remote get-url origin|grep git@)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
+[ -z "$(git remote get-url origin|grep http)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*github.com/,,'|sed 's,....$,,')"
+if [ ! -z $GITLAB_HOST ]; then
+    [ -z "$(echo $GIT_REPO | grep $GITLAB_HOST)" ] ||  GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
+fi
+
+# Set Container Name in lower case
+CONTAINER_NAME="$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
+
+# Set the right Docker Repository with the Docker registry URL
+DOCKER_REPO="$REGISTRY_URL/$CONTAINER_NAME"
+
+# Find all builded versions of the current Docker image
+ALL_BUILD_DOCKER_VERSIONS=$(docker images --format '{{.Repository}}={{.Tag}}'|grep $DOCKER_REPO|cut -d = -f 2)
+
+# Login to Docker registry
+[ "$REGISTRY_URL" != "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" "$REGISTRY_URL" --password-stdin)"
+[ "$REGISTRY_URL" == "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" --password-stdin)"
+echo $DOCKER_LOGIN_OUTPUT
+DOCKER_LOGIN_STATE="$(echo $DOCKER_LOGIN_OUTPUT | grep 'Login Succeeded')"
+
+if [ ! -z "$DOCKER_LOGIN_STATE" ]; then
+    # Push all Docker images
+    for i in $ALL_BUILD_DOCKER_VERSIONS
+    do
+        echo "$STARTMSG docker push $DOCKER_REPO:$i" && docker push $DOCKER_REPO:$i
+
+    done
+else
+    echo $DOCKER_LOGIN_OUTPUT
+    exit
+fi
+
+echo "$STARTMSG $0 is finished."
diff --git a/.ci/05_notify_hub.docker.com.sh b/.ci/05_notify_hub.docker.com.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+STARTMSG="[notify_hob.docker.com]"
+
+DOCKER_SLUG="$1"
+TOKEN="$2"
+
+echo "$STARTMSG Notify hub.docker.com"
+
+# Find Out Git Hub Repository
+echo "$STARTMSG Set GIT_REPO..."
+if [ ! -z "$(git remote get-url origin|grep git@)" ]
+then
+    GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
+elif [ ! -z "$(git remote get-url origin|grep http)" ] 
+then    
+    GIT_REPO="$(git remote get-url origin|sed 's,http.*//.*/,,'|sed 's,....$,,')"
+elif [ ! -z "$(echo $GIT_REPO|grep $GITLAB_HOST)" ] 
+then
+    GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
+else
+    echo "Can not found the Git URL. Exit now."
+    exit 1
+fi
+
+
+DOCKER_REPO="$DOCKER_SLUG/$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
+
+curl -X POST -H "Content-Type: application/json"  --data '{"docker_tag_name": "hub_automatic_untested"}'  https://registry.hub.docker.com/u/$DOCKER_REPO/trigger/$TOKEN/
+
+echo "$STARTMSG $0 is finished."
+
diff --git a/.ci/Makefile b/.ci/Makefile
@@ -0,0 +1,26 @@
+#
+#	Makefile
+#
+.PHONY: help build tags push notify-hub-docker-com
+
+help:
+	@echo -e "Please use a command: \n \
+		make build v=<2.3-debian> \n \
+		make build v=<2.3-debian> prod=true | If you want to build directly production ready images\n \
+		make tags $(REPOURL) \n \
+		make tags $(REPOURL) prod=true | If you want to add production ready images \n \
+		make push $(REPOURL) $(USER) $(PW) \n \
+		make notify-hub.docker.com TOKEN=<TOKEN> \n \
+	"
+
+build:
+	@bash 02_build.sh $(v) $(prod)
+
+tags:
+	@bash 03_tagging.sh $(REPOURL) $(prod)
+
+push:
+	@bash 04_push.sh $(REPOURL) $(USER) $(PW)
+
+notify-hub-docker-com:
+	@bash 05_notify_hub.docker.com.sh $(TOKEN)