Feature/zenko 485 eve on s3

Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Giorgio Regni <gr@scality.com>
 WORKDIR /usr/src/app
 
 # Keep the .git directory in order to properly report version
-COPY . /usr/src/app
+COPY ./package.json .
 
 RUN apt-get update \
     && apt-get install -y jq python git build-essential --no-install-recommends \
@@ -15,6 +15,8 @@ RUN apt-get update \
     && rm -rf ~/.node-gyp \
     && rm -rf /tmp/npm-*
 
+COPY ./ ./
+
 VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
 
 ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]

eve/main.yml

@@ -0,0 +1,102 @@
+---
+version: 0.2
+
+branches:
+  feature/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*:
+    stage: pre-merge
+  development/*:
+    stage: post-merge
+
+models:
+  - Git: &clone
+      name: Pull repo
+      repourl: '%(prop:git_reference)s'
+      shallow: True
+      retryFetch: True
+      haltOnFailure: True
+      # Machine Setup
+  - ShellCommandWthSecrets: &credentials
+      name: Setup Credentials
+      command: bash eve/workers/build/credentials.bash
+      haltOnFailure: True
+  - ShellCommandWithSecrets: &npm-install
+      name: install modules
+      command: npm install
+      haltOnFailure: True
+stages:
+  pre-merge:
+    worker:
+      type: local
+    steps:
+      - SetProperty:
+          property: artifacts_name
+          value: "scality-s3-%(prop:buildnumber)s"
+          haltOnFailure: True
+
+      - TriggerStages:
+          name: Launch all workers
+          stage_names:
+            - linting-coverage
+            - mongo-ft-tests
+          waitForFinish: True
+          haltOnFailure: True
+
+  linting-coverage:
+    worker:
+      type: docker
+      path: eve/workers/build
+      volumes: &default_volumes
+        - '/home/eve/workspace'
+    steps:
+      - Git: *clone
+      - ShellCommand: *npm-install
+      - ShellCommand:
+          name: Linting
+          command: |
+            npm run --silent lint -- --max-warnings 0
+            npm run --silent lint_md
+            flake8 $(git ls-files "*.py")
+            yamllint $(git ls-files "*.yml")
+      - ShellCommand:
+          name: Unit Coverage
+          command: |
+            mkdir -p $CIRCLE_TEST_REPORTS/unit
+            npm run unit_coverage
+            npm run unit_coverage_legacy_location
+          haltOnFailure: True
+          env:
+            CIRCLE_TEST_REPORTS: /tmp
+
+  mongo-ft-tests:
+    worker:
+      type: kube_pod
+      path: eve/workers/pod.yaml
+      images:
+        aggressor: eve/workers/build
+        s3: "."
+      vars: &mongo-vars
+        S3BACKEND: "mem"
+        S3DATA: ""
+        MPU_TESTING: "yes"
+        S3METADATA: mongodb
+        CI_PROXY: "false"
+    steps:
+      - Git: *clone
+      - ShellCommandWithSecrets: *credentials
+      - ShellCommand: *npm-install
+      - ShellCommandWithSecrets:
+          command: |
+            . /root/.aws/exports &> /dev/null
+            bash wait_for_local_port.bash 8000 40
+            npm run ft_test
+          env:
+            <<: *mongo-vars
+
+  post-merge:
+    worker:
+      type: local
+    steps:
+      - Git: *clone
+      - ShellCommand:
+          name: Nightly Build
+          command: docker build -t zenko/cloudserver:nightly .

eve/workers/build/Dockerfile

@@ -0,0 +1,57 @@
+FROM buildpack-deps:stretch-curl
+
+#
+# Install packages needed by the buildchain
+#
+ENV LANG C.UTF-8
+COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
+RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+    && echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+    && apt-get update \
+    && cat /tmp/*packages.list | xargs apt-get install -y \
+    && git clone https://github.com/tj/n.git \
+    && make -C ./n \
+    && n 6 latest \
+    && pip install pip==9.0.1 \
+    && rm -rf ./n \
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -f /tmp/packages.list
+
+#
+# Add user eve
+#
+
+RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
+    && adduser eve sudo \
+    && sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
+#
+# Install Dependencies
+#
+
+# Install RVM and gems
+ENV RUBY_VERSION="2.4.1"
+COPY ./gems.list /tmp/
+RUN cat /tmp/gems.list | xargs gem install
+#RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
+#    && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
+#    && usermod -a -G rvm eve
+#RUN /bin/bash -l -c "\
+#    source /usr/local/rvm/scripts/rvm \
+#    && cat /tmp/gems.list | xargs gem install \
+#    && rm /tmp/gems.list"
+
+# Install Pip packages
+COPY ./pip_packages.list /tmp/
+RUN cat /tmp/pip_packages.list | xargs pip install \
+    && rm -f /tmp/pip_packages.list \
+    && mkdir /home/eve/.aws \
+    && chown eve /home/eve/.aws 
+
+#
+# Run buildbot-worker on startup
+#
+
+ARG BUILDBOT_VERSION
+RUN pip install buildbot-worker==$BUILDBOT_VERSION
+
+CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS   && buildbot-worker start --nodaemon"]

eve/workers/build/buildbot_worker_packages.list

@@ -0,0 +1,13 @@
+ca-certificates
+git
+gnupg
+libffi-dev
+libssl-dev
+python-pip
+python2.7
+python2.7-dev
+software-properties-common
+sudo
+tcl
+wget
+procps

eve/workers/build/credentials.bash

@@ -0,0 +1,41 @@
+#!/bin/bash -x
+set -x #echo on
+set -e #exit at the first error
+
+mkdir -p ~/.aws
+cat >>/root/.aws/exports <<EOF
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+export GOOGLE_SERVICE_EMAIL="$GCP_BACKEND_SERVICE_EMAIL"
+export GOOGLE_SERVICE_KEY="$GCP_BACKEND_SERVICE_KEY"
+export azurebackend2_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY_2"
+export azurebackend2_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME_2"
+export azurebackend2_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT_2"
+export azurebackend_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurebackend_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurebackend_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurebackendmismatch_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY="$AZURE_BACKEND_ACCESS_KEY"
+export azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME="$AZURE_BACKEND_ACCOUNT_NAME"
+export azurenonexistcontainer_AZURE_STORAGE_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+export azuretest_AZURE_BLOB_ENDPOINT="$AZURE_BACKEND_ENDPOINT"
+EOF
+
+source /root/.aws/exports &> /dev/null
+cat >>/root/.aws/credentials <<EOF
+[default]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY
+[default_2]
+aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY_2
+[google]
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY
+[google_2]
+aws_access_key_id = $AWS_GCP_BACKEND_ACCESS_KEY_2
+aws_secret_access_key = $AWS_GCP_BACKEND_SECRET_KEY_2
+EOF
+

eve/workers/build/gems.list

@@ -0,0 +1,4 @@
+fog-aws:1.3.0
+json
+mime-types:3.1
+rspec:3.5

eve/workers/build/pip_packages.list

@@ -0,0 +1,3 @@
+flake8
+s3cmd==1.6.1
+yamllint

eve/workers/build/s3_packages.list

@@ -0,0 +1,11 @@
+build-essential
+curl
+default-jdk
+libdigest-hmac-perl
+lsof
+maven
+netcat
+redis-server
+ruby-full
+yarn=1.7.0-1
+zlib1g-dev