Merge branch 'main' into person/aivanov/vdk-control-cli-fix-parsing-c…

…ontacts

projects/control-service/cicd/.gitlab-ci.yml

@@ -58,7 +58,6 @@ control_service_build_image:
       junit: ./projects/control-service/projects/**/build/test-results/test/TEST-*.xml
   only:
     refs:
-      - external_pull_requests
       - main
     changes: *control_service_change_locations
 
@@ -101,6 +100,34 @@ control_service_integration_test:
     changes:
       - projects/control-service/projects/helm_charts/pipelines-control-service/version.txt
 
+
+control_service_publish_job_base_image:
+  image: docker:19.03.8
+  services:
+    - docker:19.03.8-dind
+  variables:
+    DOCKER_HOST: tcp://localhost:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  stage: publish_artifacts
+  script:
+    - apk add --no-cache bash
+    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
+    - cd projects/control-service/projects/job-base-image
+    - export VERSION_TAG="1.$CI_PIPELINE_ID"
+    - bash -ex ./publish-job-base.sh
+  retry: !reference [.control_service_retry, retry_options]
+  only:
+    refs:
+      - main
+      - external_pull_requests
+    changes:
+      - projects/control-service/projects/job-base-image/**/*
+  except:
+    changes:
+      - projects/control-service/projects/helm_charts/pipelines-control-service/version.txt
+
+
 control_service_publish_job_builder_image:
   image: docker:19.03.8
   services:

projects/control-service/projects/job-base-image/Dockerfile-data-job-base

@@ -0,0 +1,33 @@
+# https://docs.docker.com/develop/develop-images/dockerfile_best-practices\
+# Creating data job base image with support for Oracle.
+
+ARG base_image=python:3.7-slim
+FROM $base_image
+
+# Set the working directory
+WORKDIR /job
+
+
+# Install native dependencies so that requirements in requirements.txt can be installed
+# some (like openssl) should be pre-installed in the base image but let's be explicit
+RUN set -ex \
+       && apt-get update \
+       && apt-get -y install --no-install-recommends \
+              build-essential openssl g++
+
+# Install the native dependencies necessary for cx_Oracle python library
+# See https://oracle.github.io/odpi/doc/installation.html#linux
+RUN set -ex \
+      && echo "Installing native dependencies related to support for cx_Oracle python library ..." \
+      && mkdir -p /opt/lib/native  \
+      && apt-get -y install  --no-install-recommends libaio1 curl unzip \
+      && curl --insecure --output oracle-instantclient.zip https://download.oracle.com/otn_software/linux/instantclient/1911000/instantclient-basic-linux.x64-19.11.0.0.0dbru.zip \
+      && unzip oracle-instantclient.zip -d /opt/lib/native/oracle && rm -f oracle-instantclient.zip \
+      && sh -c "echo /opt/lib/native/oracle/instantclient_19_11 > /etc/ld.so.conf.d/oracle-instantclient.conf" \
+      && ldconfig \
+      && apt-get purge -y --auto-remove curl unzip
+
+# libaio1 - LGPL-2.1+ https://developer.puri.sm/licenses/Librem5/Birch/libaio1/copyright
+# curl - MIT/X* modified https://curl.se/docs/copyright.html
+# unzip - MIT https://github.com/vipsoft/Unzip/blob/master/LICENSE
+# oracle instant client license - Oracle Technology Network Development and Distribution License Agreement https://www.oracle.com/downloads/licenses/instant-client-lic.html

projects/control-service/projects/job-base-image/README.md

@@ -0,0 +1,17 @@
+# Job base image
+
+Job base image is the container "base" image used when building per data job custom image during deployment.
+
+This directory provides the source of some base images for standard python versions.
+It's used by default installation of VDK and should also serve as an example on how operators of VDK can build their own base image.
+
+The current base image installs supporting libraries for Oracle database
+and some native bindings necessary for installing from source some python packages which user may specify for their data job.
+
+## Build
+
+To build the job_base images run `./publish-job-base` which will publish new base image to versatiledatakit container registry.
+
+## Use
+
+It's then set in values.yaml of the helm chart as `deploymentDataJobBaseImage` option

projects/control-service/projects/job-base-image/publish-job-base.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright 2021 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+VERSION_TAG="${VERSION_TAG:-"0.1dev"}"
+VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"}
+
+function build_and_push_image() {
+    name="$1"
+    docker_file="$2"
+    arguments="$3"
+
+    image_repo="$VDK_DOCKER_REGISTRY_URL/$name"
+    image_tag="$image_repo:$VERSION_TAG"
+
+    docker build -t "$image_tag" -t "$image_repo:latest" -f "$SCRIPT_DIR/$docker_file" $arguments "$SCRIPT_DIR"
+    docker push "$image_tag"
+    docker push "$image_repo:latest"
+}
+
+build_and_push_image \
+    "data-job-base-python-3.7" \
+    Dockerfile-data-job-base \
+    "--build-arg base_image=python:3.7-slim"
+
+build_and_push_image \
+    "data-job-base-python-3.8" \
+    Dockerfile-data-job-base \
+    "--build-arg base_image=python:3.8-slim"
+
+build_and_push_image \
+    "data-job-base-python-3.9" \
+    Dockerfile-data-job-base \
+    "--build-arg base_image=python:3.9-slim"
+
+build_and_push_image \
+    "data-job-base-python-3.10" \
+    Dockerfile-data-job-base \
+    "--build-arg base_image=python:3.10-slim"