Feat/MDD-172: Added MISP Version 2.4.102 (#27)

## Feat/MDD-172: Added MISP Version 2.4.102 ### Update Information Features of the release: - Added MISP server container for MISP release 2.4.102. For more information, please have a look at the official changelog at https://github.com/MISP/MISP/releases/tag/v2.4.102. - Added improved CI scripts ### General Changes Besides the new MISP version, we improved the CI scripts for Gitlab and Travis CI. ### Fixes and Improvements - Added MISP server container with MISP Version 2.4.102 - Added improved CI scripts ### Detailed Changes - Added new MISP server 2.4.102 on base of 2.4.97.
DCSO · Mar 15, 2019 · 29e1ef6 · 29e1ef6
1 parent 74f8aed
commit 29e1ef6
Show file tree

Hide file tree

Showing 40 changed files with 3,296 additions and 160 deletions.
diff --git a/.ci/01_before_install.sh b/.ci/01_before_install.sh
@@ -4,14 +4,14 @@ STARTMSG="[before_install]"
 # Install Requirements
 echo
 echo "$STARTMSG Install requirements..."
-    [ ! -z $(which apk) ] && apk add --no-cache make bash sudo git curl coreutils grep python3
-    [ ! -z $(which apt-get) ] && apt-get update; 
-    [ ! -z $(which apt-get) ] && apt-get install make bash sudo git curl coreutils grep python3
+    [ ! -z "$(which apk)" ] && apk add --no-cache make bash sudo git curl coreutils grep python3
+    [ ! -z "$(which apt-get)" ] && apt-get update; 
+    [ ! -z "$(which apt-get)" ] && apt-get install make bash sudo git curl coreutils grep python3
     # Upgrade Docke
-    [ ! -z $(which apt-get) ] && apt-get install --only-upgrade docker-ce -y
+    [ ! -z "$(which apt-get)" ] && apt-get install --only-upgrade docker-ce -y
 # Install docker-compose
     # https://stackoverflow.com/questions/42295457/using-docker-compose-in-a-gitlab-ci-pipeline
-    [ -z $(which docker-compose) ] && pip3 install --no-cache-dir docker-compose
+    [ -z "$(which docker-compose)" ] && pip3 install --no-cache-dir docker-compose
 # Show version of docker-compose:
     docker-compose -v
 

diff --git a/.ci/02_build.sh b/.ci/02_build.sh
@@ -5,7 +5,8 @@ STARTMSG="[build]"
 [ "$1" == "dev" ] && echo "$STARTMSG False first argument. Abort." && exit 1
 
 VERSION="$1"
-ENVIRONMENT="$2"
+if [[ "$2" == "true" ]]; then ENVIRONMENT="prod"; fi;
+
 
 #################   MANUAL VARIABLES #################
 # path of the script
@@ -34,7 +35,7 @@ then
 elif [ ! -z "$(git remote get-url origin|grep http)" ] 
 then    
     GIT_REPO="$(git remote get-url origin|sed 's,http.*//.*/,,'|sed 's,....$,,')"
-elif [ ! -z "$(echo $GIT_REPO|grep $GITLAB_HOST)" ] 
+elif [ ! -z "$(echo "$GIT_REPO"|grep "$GITLAB_HOST")" ] 
 then
     GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
 else
@@ -44,7 +45,7 @@ fi
 
 GIT_REPO_URL="https://github.com/$GIT_REPO"
 # Dockerifle Settings
-CONTAINER_NAME="$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
+CONTAINER_NAME="$(echo "$GIT_REPO"|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
 DOCKER_REPO="not2push/$CONTAINER_NAME"
 #########################################################
 
@@ -55,9 +56,9 @@ do
     VERSION=$(echo $FOLD|cut -d- -f 1)
     DOCKERFILE_PATH="$SCRIPTPATH/../$FOLD"
     # Load Variables from configuration file
-    source $DOCKERFILE_PATH/configuration.sh
+    source "$DOCKERFILE_PATH/configuration.sh"
     # Default mode add "-dev" tag.
-    if [ "$ENVIRONMENT" == "true" ]
+    if [ "$ENVIRONMENT" == "prod" ]
     then
         # PROD Version
         TAGS="-t $DOCKER_REPO:$FOLD"
@@ -77,7 +78,7 @@ do
     # build image
     docker build \
             $BUILD_ARGS \
-        -f $DOCKERFILE_PATH/$DOCKERFILE_NAME $TAGS $DOCKERFILE_PATH/
+        -f "$DOCKERFILE_PATH/$DOCKERFILE_NAME" $TAGS "$DOCKERFILE_PATH"/
 done
 
 echo "$STARTMSG $0 is finished."
diff --git a/.ci/03_tagging.sh b/.ci/03_tagging.sh
@@ -1,32 +1,34 @@
 #!/bin/bash
 STARTMSG="[tagging]"
 
-
 [ -z "$1" ] && echo "$STARTMSG No parameter with the image version. Exit now." && exit 1
 [ "$1" == "true" ] && echo "$STARTMSG False first argument. Abort." && exit 1
 
 REGISTRY_URL="$1"
-ENVIRONMENT="$2"
+if [[ "$2" == "true" ]]; then ENVIRONMENT="prod"; fi;
+
+# change directory to the top level:
+pushd ..
 
 # Docker Repo e.g. dcso/misp-dockerized-proxy
 [ -z "$(git remote get-url origin|grep git@)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
 [ -z "$(git remote get-url origin|grep http)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*github.com/,,'|sed 's,....$,,')"
-[ -z "$GITLAB_HOST" ] || [ -z "$(echo $GIT_REPO|grep $GITLAB_HOST)" ] ||  GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
+[ -z "$GITLAB_HOST" ] || [ -z "$(echo "$GIT_REPO"|grep "$GITLAB_HOST")" ] ||  GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
 
 # Set Container Name
 CONTAINER_NAME="$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
 
 # Show Images before tagging
 echo  "$STARTMSG ### Show images before tagging:"
-docker images | grep $CONTAINER_NAME
+docker images | grep "$CONTAINER_NAME"
 
 # Set Docker Repository
 DOCKER_REPO="$REGISTRY_URL/$CONTAINER_NAME"
 SOURCE_REPO="not2push"
 
 # Search the latest image
     # Create the Array
-    FOLDER_ARRAY=( ../*/)
+    FOLDER_ARRAY=( */)
     FOLDER_ARRAY=( "${FOLDER_ARRAY[@]%/}" )
     # How many items in your Array:
     index=${#FOLDER_ARRAY[@]}       
@@ -46,8 +48,8 @@ SOURCE_REPO="not2push"
     for i in ${sorted[@]}
     do
         # change from 1.0-ubuntu -> 1
-        CURRENT_MAJOR_VERSION="$(echo $i|cut -d . -f 1)"
-        CURRENT_MINOR_VERSION="$(echo $i|cut -d . -f 2|cut -d - -f 1)"
+        CURRENT_MAJOR_VERSION="$(echo "$i"|cut -d . -f 1)"
+        CURRENT_MINOR_VERSION="$(echo "$i"|cut -d . -f 2|cut -d - -f 1)"
 
         # Check if there is any Version available for the current MAJOR version:
         [ -z ${MAJOR_LATEST[$CURRENT_MAJOR_LATEST]} ] && MAJOR_LATEST[$CURRENT_MAJOR_VERSION]=$i && continue
@@ -65,53 +67,54 @@ ALL_BUILD_DOCKER_VERSIONS=$(docker images --format '{{.Repository}}={{.Tag}}'|gr
 # Tag Latest + Version Number
 for i in $ALL_BUILD_DOCKER_VERSIONS
 do
-    VERSION=$(echo $i|cut -d- -f 1)                 # for example 1.0
-    BASE=$(echo $i|cut -d- -f 2)                    # for example ubuntu
-    MAJOR_VERSION="$(echo $i|cut -d . -f 1)"        # for example 1
+    VERSION=$(echo "$i"|cut -d- -f 1)                 # for example 1.0
+    BASE=$(echo "$i"|cut -d- -f 2)                    # for example ubuntu
+    CURRENT_MAJOR_VERSION="$(echo "$i"|cut -d . -f 1)"        # for example 1
 
     # Remove '-dev' tag
-    if [ "$ENVIRONMENT" == "true" ]; then
+    if [ "$ENVIRONMENT" == "prod" ]; then
         #
         #   If prod=true, ~ prodcutin ready image
         #
 
         # Add custom Docker registry tag
-        docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$VERSION-$BASE
+        docker tag "$SOURCE_REPO/$CONTAINER_NAME:$i" "$DOCKER_REPO:$VERSION-$BASE"
 
         # Add latest tag
         if [ "$VERSION" == "$LATEST" ]; then
-            docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:latest
+            docker tag "$SOURCE_REPO/$CONTAINER_NAME:$i" "$DOCKER_REPO":latest
         fi
 
         # Add latest Major Version Tag
         for k in ${MAJOR_LATEST[@]}
         do
-            CURRENT_MAJOR_VERSION="$(echo $k|cut -d . -f 1)"
-            [ "$i" == $k"-dev" ] && docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$CURRENT_MAJOR_VERSION
+            #CURRENT_MAJOR_VERSION="$(echo $k|cut -d . -f 1)"
+            [ "$i" == "$k-dev" ] && docker tag "$SOURCE_REPO/$CONTAINER_NAME:$i" "$DOCKER_REPO:$CURRENT_MAJOR_VERSION"
         done
     else
         #
         #   Add '-dev' tag
         #   
 
         # Add custom Docker registry tag
-        docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$VERSION-$BASE-dev
+        docker tag "$SOURCE_REPO/$CONTAINER_NAME:$i" "$DOCKER_REPO:$VERSION-$BASE-dev"
 
         # Add latest tag
         if [ "$VERSION" == "$LATEST" ]; then
-            docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:latest-dev
+            docker tag "$SOURCE_REPO/$CONTAINER_NAME:$i" "$DOCKER_REPO:latest-dev"
         fi
 
         # Add latest Major Version Tag
         for k in ${MAJOR_LATEST[@]}
         do
-            CURRENT_MAJOR_VERSION="$(echo $k|cut -d . -f 1)"
-            [ "$i" == $k"-dev" ] && docker tag $SOURCE_REPO/$CONTAINER_NAME:$i $DOCKER_REPO:$CURRENT_MAJOR_VERSION-dev
+            CURRENT_MAJOR_VERSION="$(echo "$k"|cut -d . -f 1)"
+            [ "$i" == "$k-dev" ] && docker tag "$SOURCE_REPO/$CONTAINER_NAME:$i" "$DOCKER_REPO:$CURRENT_MAJOR_VERSION-dev"
         done
     fi
 done
 
 echo  "$STARTMSG ### Show images after tagging:"
-docker images | grep $DOCKER_REPO
+docker images | grep "$DOCKER_REPO"
 
 echo "$STARTMSG $0 is finished."
+
diff --git a/.ci/04_push.sh b/.ci/04_push.sh
@@ -15,33 +15,34 @@ REGISTRY_PW="$3"
 # Find the right Docker Repo name e.g. dcso/misp-dockerized-proxy
 [ -z "$(git remote get-url origin|grep git@)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
 [ -z "$(git remote get-url origin|grep http)" ] || GIT_REPO="$(git remote get-url origin|sed 's,.*github.com/,,'|sed 's,....$,,')"
-if [ ! -z $GITLAB_HOST ]; then
-    [ -z "$(echo $GIT_REPO | grep $GITLAB_HOST)" ] ||  GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
+if [ ! -z "$GITLAB_HOST" ]; then
+    [ -z "$(echo "$GIT_REPO" | grep "$GITLAB_HOST")" ] ||  GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
 fi
 
 # Set Container Name in lower case
-CONTAINER_NAME="$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
+CONTAINER_NAME="$(echo "$GIT_REPO"|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
 
 # Set the right Docker Repository with the Docker registry URL
 DOCKER_REPO="$REGISTRY_URL/$CONTAINER_NAME"
 
 # Find all builded versions of the current Docker image
-ALL_BUILD_DOCKER_VERSIONS=$(docker images --format '{{.Repository}}={{.Tag}}'|grep $DOCKER_REPO|cut -d = -f 2)
+ALL_BUILD_DOCKER_VERSIONS=$(docker images --format '{{.Repository}}={{.Tag}}'|grep "$DOCKER_REPO"|cut -d = -f 2)
 
 # Login to Docker registry
 [ "$REGISTRY_URL" != "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" "$REGISTRY_URL" --password-stdin)"
-[ "$REGISTRY_URL" == "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" --password-stdin)"
-echo $DOCKER_LOGIN_OUTPUT
-DOCKER_LOGIN_STATE="$(echo $DOCKER_LOGIN_OUTPUT | grep 'Login Succeeded')"
+[ "$REGISTRY_URL" = "dcso" ] && DOCKER_LOGIN_OUTPUT="$(echo "$REGISTRY_PW" | docker login -u "$REGISTRY_USER" --password-stdin)"
+echo "$DOCKER_LOGIN_OUTPUT"
+DOCKER_LOGIN_STATE="$(echo "$DOCKER_LOGIN_OUTPUT" | grep 'Login Succeeded')"
 
 if [ ! -z "$DOCKER_LOGIN_STATE" ]; then
     # Push all Docker images
     for i in $ALL_BUILD_DOCKER_VERSIONS
     do
-        echo "$STARTMSG docker push $DOCKER_REPO:$i" && docker push $DOCKER_REPO:$i
+        echo "$STARTMSG docker push $DOCKER_REPO:$i" && docker push "$DOCKER_REPO:$i"
+        [ "$?" -ne 0 ] && exit 1
     done
 else
-    echo $DOCKER_LOGIN_OUTPUT
+    echo "$DOCKER_LOGIN_OUTPUT"
     exit
 fi
 

diff --git a/.ci/05_notify_hub.docker.com.sh b/.ci/05_notify_hub.docker.com.sh
@@ -1,30 +1,10 @@
 #!/bin/bash
 STARTMSG="[notify_hob.docker.com]"
 
-DOCKER_SLUG="$1"
-TOKEN="$2"
+NOTIFY_URL="$1"
 
 echo "$STARTMSG Notify hub.docker.com"
 
-# Find Out Git Hub Repository
-echo "$STARTMSG Set GIT_REPO..."
-if [ ! -z "$(git remote get-url origin|grep git@)" ]
-then
-    GIT_REPO="$(git remote get-url origin|sed 's,.*:,,'|sed 's,....$,,')"
-elif [ ! -z "$(git remote get-url origin|grep http)" ] 
-then    
-    GIT_REPO="$(git remote get-url origin|sed 's,http.*//.*/,,'|sed 's,....$,,')"
-elif [ ! -z "$(echo $GIT_REPO|grep $GITLAB_HOST)" ] 
-then
-    GIT_REPO="$(git remote get-url origin|sed 's,.*'${GITLAB_HOST}'/'${GITLAB_GROUP}'/,,'|sed 's,....$,,')"
-else
-    echo "Can not found the Git URL. Exit now."
-    exit 1
-fi
-
-
-DOCKER_REPO="$DOCKER_SLUG/$(echo $GIT_REPO|cut -d / -f 2|tr '[:upper:]' '[:lower:]')"
-
-curl -X POST -H "Content-Type: application/json"  --data '{"docker_tag_name": "hub_automatic_untested"}'  https://registry.hub.docker.com/u/$DOCKER_REPO/trigger/$TOKEN/
+curl -X POST -H "Content-Type: application/json"  --data '{"docker_tag_name": "hub_automatic_untested"}'  "$NOTIFY_URL"
 
 echo "$STARTMSG $0 is finished."
diff --git a/.ci/Makefile b/.ci/Makefile
@@ -7,20 +7,20 @@ help:
 	@echo -e "Please use a command: \n \
 		make build v=<2.3-debian> \n \
 		make build v=<2.3-debian> prod=true | If you want to build directly production ready images\n \
-		make tags $(REPO) \n \
-		make tags $(REPO) prod=true | If you want to add production ready images \n \
-		make push $(REPO) $(USER) $(PW) \n \
-		make notify-hub.docker.com TOKEN=<TOKEN> \n \
+		make tags REPOURL= \n \
+		make tags REPOURL= prod=true | If you want to add production ready images \n \
+		make push REPOURL= USER= PW= \n \
+		make notify-hub.docker.com URL= \n \
 	"
 
 build:
 	@bash 02_build.sh $(v) $(prod)
 
 tags:
-	@bash 03_tagging.sh $(REPO) $(prod)
+	@bash 03_tagging.sh $(REPOURL) $(prod)
 
 push:
-	@bash 04_push.sh $(REPO) $(USER) $(PW)
+	@bash 04_push.sh $(REPOURL) $(USER) $(PW)
 
 notify-hub-docker-com:
-	@bash 05_notify_hub.docker.com.sh $(REPO) $(TOKEN)
+	@bash 05_notify_hub.docker.com.sh $(URL)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -9,62 +9,54 @@ stages:
 
 variables:
   # The following variables are setup via gitlab project group:
-  # DOCKER_HUB_TOKEN
+  # DOCKER_HUB_NOTIFY_URL
   # DOCKER_SLUG
   # DOCKER_USERNAME
   # DOCKER_PASSWORD
   # CUSTOM_REGISTRY_URL
   # CUSTOM_REGISTRY_USER
   # CUSTOM_REGISTRY_PW
 
-#
-# Install dependencies
-#
 before_script:
 - .ci/01_before_install.sh
 
-#
-# Build image for a short test to see if image can be build
-#
 .test:
   stage: test
   except: 
   - master  
   script:
-  - make -C .ci build v=$VERSION # build an 8ear/CONTAINERNAME image, if you want to change the slug please set DOCKER_SLUG=<Your slug>
+  - make -C .ci build v=$VERSION
 
-#
-# Build, tag, and push our image + trigger hub.docker.com automatic_build queue to update Readme
-#
 .build:
   stage: build
-  only: 
-  - master
+  # only: 
+  # - master
   script:
   - make -C .ci build v=$VERSION 
-  - make -C .ci tags REPO=$CUSTOM_REGISTRY_URL
-  - make -C .ci tags REPO=$DOCKER_SLUG
-  - make -C .ci push REPO=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW
-  - make -C .ci push REPO=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD
-  - make -C .ci notify-hub-docker-com REPO=$DOCKER_SLUG TOKEN=$DOCKER_HUB_TOKEN
+  - make -C .ci tags REPOURL=$CUSTOM_REGISTRY_URL
+  - make -C .ci tags REPOURL=$DOCKER_SLUG
+  - make -C .ci push REPOURL=$CUSTOM_REGISTRY_URL USER=$CUSTOM_REGISTRY_USER PW=$CUSTOM_REGISTRY_PW
+  - make -C .ci push REPOURL=$DOCKER_SLUG USER=$DOCKER_USERNAME PW=$DOCKER_PASSWORD
+  - make -C .ci notify-hub-docker-com URL=$DOCKER_HUB_NOTIFY_URL
 
 
 #
 # For a cleaner Gitlab CI file include all subfolder which should be build:
 #
 include:
-  - '2.4.nightly-ubuntu/.gitlab-ci.yml'
-  - '2.4.88-ubuntu/.gitlab-ci.yml'
-  - '2.4.89-ubuntu/.gitlab-ci.yml'
-  - '2.4.90-ubuntu/.gitlab-ci.yml'
-  - '2.4.91-ubuntu/.gitlab-ci.yml'
-  - '2.4.92-ubuntu/.gitlab-ci.yml'
-  - '2.4.93-ubuntu/.gitlab-ci.yml'
-  - '2.4.94-ubuntu/.gitlab-ci.yml'
-  - '2.4.95-ubuntu/.gitlab-ci.yml'
-  - '2.4.96-ubuntu/.gitlab-ci.yml'
-  - '2.4.97-debian/.gitlab-ci.yml'
-  - '2.4.98-debian/.gitlab-ci.yml'
-  - '2.4.99-debian/.gitlab-ci.yml'
-  - '2.4.100-debian/.gitlab-ci.yml'
-  - '2.4.101-debian/.gitlab-ci.yml'
+  # - '2.4.nightly-ubuntu/.gitlab-ci.yml'
+  # - '2.4.88-ubuntu/.gitlab-ci.yml'
+  # - '2.4.89-ubuntu/.gitlab-ci.yml'
+  # - '2.4.90-ubuntu/.gitlab-ci.yml'
+  # - '2.4.91-ubuntu/.gitlab-ci.yml'
+  # - '2.4.92-ubuntu/.gitlab-ci.yml'
+  # - '2.4.93-ubuntu/.gitlab-ci.yml'
+  # - '2.4.94-ubuntu/.gitlab-ci.yml'
+  # - '2.4.95-ubuntu/.gitlab-ci.yml'
+  # - '2.4.96-ubuntu/.gitlab-ci.yml'
+  # - '2.4.97-debian/.gitlab-ci.yml'
+  # - '2.4.98-debian/.gitlab-ci.yml'
+  # - '2.4.99-debian/.gitlab-ci.yml'
+  # - '2.4.100-debian/.gitlab-ci.yml'
+  # - '2.4.101-debian/.gitlab-ci.yml'
+  - '2.4.102-debian/.gitlab-ci.yml'