From a6c583cf5862701b7987619c5bdce7b5932acd87 Mon Sep 17 00:00:00 2001 From: Ian Wong Date: Fri, 30 Mar 2018 10:01:13 +0800 Subject: [PATCH] remove jenkins related docker files which is only used internal (#2478) (#2479) --- docker/jenkins/README.md | 77 ------ docker/jenkins/jenkins-master/Dockerfile | 41 --- docker/jenkins/jenkins-master/init.groovy | 25 -- .../jenkins/jenkins-master/install-plugins.sh | 257 ------------------ docker/jenkins/jenkins-master/jenkins-support | 128 --------- docker/jenkins/jenkins-master/jenkins.sh | 27 -- docker/jenkins/jenkins-master/plugins.sh | 122 --------- docker/jenkins/jenkins-slave/Dockerfile | 163 ----------- docker/jenkins/jenkins-slave/pip.conf | 8 - docker/jenkins/jenkins-slave/run-slave.sh | 13 - docker/jenkins/jenkins-slave/settings.xml | 18 -- docker/jenkins/jenkins-slave/slave.groovy | 27 -- 12 files changed, 906 deletions(-) delete mode 100644 docker/jenkins/README.md delete mode 100644 docker/jenkins/jenkins-master/Dockerfile delete mode 100644 docker/jenkins/jenkins-master/init.groovy delete mode 100644 docker/jenkins/jenkins-master/install-plugins.sh delete mode 100644 docker/jenkins/jenkins-master/jenkins-support delete mode 100644 docker/jenkins/jenkins-master/jenkins.sh delete mode 100644 docker/jenkins/jenkins-master/plugins.sh delete mode 100644 docker/jenkins/jenkins-slave/Dockerfile delete mode 100644 docker/jenkins/jenkins-slave/pip.conf delete mode 100644 docker/jenkins/jenkins-slave/run-slave.sh delete mode 100644 docker/jenkins/jenkins-slave/settings.xml delete mode 100644 docker/jenkins/jenkins-slave/slave.groovy diff --git a/docker/jenkins/README.md b/docker/jenkins/README.md deleted file mode 100644 index 5a5b64e002d..00000000000 --- a/docker/jenkins/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Jenkins Docker -Jenkins is a self-contained, open source automation server which can be used to automate all sorts of tasks related to building, testing, and delivering or deploying software. Here we run Jenkins in Docker, based on Ubuntu 14.04. - -## 1. Build images -Jenkins has master nodes and slave agents. To make Jenkins well functioning, masters and slaves should be built separatedly. -### 1.1 Build master -First, you should step into the directory jenkins-master and run -```bash -sudo docker build --rm -t bigdl/jenkins-master . -``` -If proxy is needed for building the image, building arguments related to proxy should be added in the command above. For example, -```bash -sudo docker build --build-arg http_proxy=http://yourproxyhost:port \ - --build-arg https_proxy=https://yourproxyhost:port \ - --rm -t bigdl/jenkins-master . -``` -Note that you can set any tag of the image by changine the parameter after the option -t. - -There are some extra files other than Dockerfile in the directory jenkins-master. -+ jenkins.sh: launch Jenkins -+ jenkins-support: provide support of upgrading Jenkins and its plugins -+ install-plugins: resolve dependencies and download Jenkins plugins -+ init.groovy: provide initial configuration of launching Jenkins master - -### 1.2 Build slaves -Building Jenkins slave is similar to building Jenkins master. First, step into the directory jenkins-slave and run -```bash -sudo docker build --rm -t bigdl/jenkins-slave . -``` -If proxy is needed for building the image, the following command should be run. -```bash -sudo docker build --build-arg http_proxy=http://yourproxyhost:port \ - --build-arg https_proxy=https://yourproxyhost:port \ - --rm -t bigdl/jenkins-slave . -``` -There are some extra files other than Dockerfile in the directory jenkins-slave. -+ pip.conf: provide configurations of pip -+ run-slave.sh: launch Jenkins slave agents -+ settings-xml: provide settings of maven -+ slave.groovy: provide initial configurations of launching Jenkins slave agents -Please modify these files to add your own pypi index, maven settings like proxies, servers and mirrors before you build the images. - -## 2. Run images -### 2.1 Run master -After building images of Jenkins master and slave agents, you should run the image of Jenkins master first using -```bash -sudo docker run -itd --net=host -p 8080:8080 -p 50000:50000 bigdl/jenkins-master -``` -+ -p 8080:8080: This mapping of port is used for accessing Jenkins Web UI. -+ -p 50000:50000: This mapping of port aims at attaching build slave servers through JNLP (Java Web Start), which will be used when you connect a slave agent. -+ bigdl/jenkins-master: You can change it to your tag of Jenkins master images set in build step. -If you want to use proxy when running images, two more parameters should be added in the command. -```bash -sudo docker run -itd --net=host -p 8080:8080 -p 50000:50000 \ -                     --env PROXY_HOST=yourproxyhost --env PROXY_PORT=yourproxyport bigdl/jenkins-master -``` - -### 2.2 Run slave -Then, you can run the image of Jenkins slave using -```bash -sudo docker run -itd --net=host \ - -e jenkins_master_host=yourJenkinsmasterhost \ - -e jenkins_master_port=yourJenkinsmasterport \ - -e jenkins_slave_label=yourslavelabel \ - -e jenkins_slave_name=yourslavename \ - -e jenkins_slave_executors=numberofyourslaveexecutors \ - --memory=50g \ - bigdl/jenkins-slave -``` -Note that 8080 is the default Jenkins master port. After that, you can browse to http://yourJenkinsmasterhost:8080 to check information and status of both Jenkins master and slave agents. -Note that memory should be larger than 50G. - -## 3. Configure Github and Git plugins -If you want to link Jenkins with Github, you should conduct some configurations when you first start Jenkins and log in. -* In **Manage Jenkins -> Configure System**, you should add a Github server with API URL: https://api.github.com/ and set global configurations in Git plugin part as you set Git at the first time. -* In **Credentials -> Systems -> Add domains**, you should add a domain with the name api.github.com. Then you need to add credentials in the domain **api.github.com** with the kind Username and password. The username and password should correspond to those of github accounts of users. -* When creating a new job, you should modify the setting of git in **Source Code Management** part of job configuration. diff --git a/docker/jenkins/jenkins-master/Dockerfile b/docker/jenkins/jenkins-master/Dockerfile deleted file mode 100644 index 1c2c8582e72..00000000000 --- a/docker/jenkins/jenkins-master/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -FROM ubuntu:14.04 - -MAINTAINER The BigDL Authors https://github.com/intel-analytics/BigDL - -WORKDIR /opt/work - -ENV JAVA_HOME /opt/work/jdk -ENV JAVA_OPTS -Djenkins.install.runSetupWizard=false -ENV JENKINS_HOME /opt/work/jenkins -ENV JENKINS_SLAVE_AGENT_PORT 50000 -ENV JENKINS_VERSION 2.60.3 -ENV JENKINS_SHA 2d71b8f87c8417f9303a73d52901a59678ee6c0eefcf7325efed6035ff39372a -ENV JENKINS_URL https://repo.jenkins-ci.org/public/org/jenkins-ci/main/jenkins-war/${JENKINS_VERSION}/jenkins-war-${JENKINS_VERSION}.war -ENV JENKINS_UC https://updates.jenkins.io -ENV JENKINS_UC_EXPERIMENTAL https://updates.jenkins.io/experimental -ENV COPY_REFERENCE_FILE_LOG ${JENKINS_HOME}/copy_reference_file.log -ENV PATH ${JAVA_HOME}/bin:${PATH} - -COPY init.groovy ${JENKINS_HOME}/init.groovy.d/basic-security.groovy -COPY jenkins-support ${JENKINS_HOME}/jenkins-support -COPY jenkins.sh ${JENKINS_HOME}/jenkins.sh -COPY install-plugins.sh ${JENKINS_HOME}/install-plugins.sh - -RUN apt-get update && apt-get install -y git wget curl unzip && rm -rf /var/lib/apt/lists/* && \ -# Install JDK 8 - wget http://ftp.osuosl.org/pub/funtoo/distfiles/oracle-java/jdk-8u152-linux-x64.tar.gz && \ - gunzip jdk-8u152-linux-x64.tar.gz && \ - tar -xf jdk-8u152-linux-x64.tar -C /opt && \ - rm jdk-8u152-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_152 ${JAVA_HOME} && \ -# Install Jenkins Master - curl -fsSL ${JENKINS_URL} -o ${JENKINS_HOME}/jenkins.war && \ - echo "${JENKINS_SHA} ${JENKINS_HOME}/jenkins.war" | sha256sum -c - && \ - echo "${JENKINS_VERSION}" >> ${JENKINS_HOME}/jenkins.install.InstallUtil.lastExecVersion && \ - chmod a+x ${JENKINS_HOME}/jenkins.sh && \ - chmod a+x ${JENKINS_HOME}/install-plugins.sh && \ - chmod a+x ${JENKINS_HOME}/jenkins-support && \ - ${JENKINS_HOME}/install-plugins.sh nodelabelparameter parameterized-trigger github email-ext jenkins-multijob-plugin leastload - -ENTRYPOINT ["/opt/work/jenkins/jenkins.sh"] - diff --git a/docker/jenkins/jenkins-master/init.groovy b/docker/jenkins/jenkins-master/init.groovy deleted file mode 100644 index 8c1908a380e..00000000000 --- a/docker/jenkins/jenkins-master/init.groovy +++ /dev/null @@ -1,25 +0,0 @@ -#!groovy - -import hudson.model.*; -import jenkins.model.*; -import hudson.slaves.*; -import hudson.security.*; - -Thread.start { - println "--> setting agent port for jnlp" - def env = System.getenv() - int agentPort = env['JENKINS_SLAVE_AGENT_PORT'].toInteger() - Jenkins.instance.setSlaveAgentPort(agentPort) - println "--> setting agent port for jnlp... done" - def proxyHost = env['PROXY_HOST'].toString() - int proxyPort = env['PROXY_PORT'].toInteger() - Jenkins.instance.proxy = new hudson.ProxyConfiguration(proxyHost, proxyPort) - println "--> set proxy... done" - def hudsonRealm = new HudsonPrivateSecurityRealm(false) - hudsonRealm.createAccount('admin','admin') - Jenkins.instance.setSecurityRealm(hudsonRealm) - def strategy = new FullControlOnceLoggedInAuthorizationStrategy() - Jenkins.instance.setAuthorizationStrategy(strategy) - println "--> create admin... done" - Jenkins.instance.save() -} diff --git a/docker/jenkins/jenkins-master/install-plugins.sh b/docker/jenkins/jenkins-master/install-plugins.sh deleted file mode 100644 index ed4d19ed975..00000000000 --- a/docker/jenkins/jenkins-master/install-plugins.sh +++ /dev/null @@ -1,257 +0,0 @@ -#!/bin/bash -eu - -# Resolve dependencies and download plugins given on the command line -# -# FROM jenkins -# RUN install-plugins.sh docker-slaves github-branch-source - -set -o pipefail - -REF_DIR=${REF:-${JENKINS_HOME}/plugins} -FAILED="$REF_DIR/failed-plugins.txt" - -. ${JENKINS_HOME}/jenkins-support - -getLockFile() { - printf '%s' "$REF_DIR/${1}.lock" -} - -getArchiveFilename() { - printf '%s' "$REF_DIR/${1}.jpi" -} - -download() { - local plugin originalPlugin version lock ignoreLockFile - plugin="$1" - version="${2:-latest}" - ignoreLockFile="${3:-}" - lock="$(getLockFile "$plugin")" - - if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then - if ! doDownload "$plugin" "$version"; then - # some plugin don't follow the rules about artifact ID - # typically: docker-plugin - originalPlugin="$plugin" - plugin="${plugin}-plugin" - if ! doDownload "$plugin" "$version"; then - echo "Failed to download plugin: $originalPlugin or $plugin" >&2 - echo "Not downloaded: ${originalPlugin}" >> "$FAILED" - return 1 - fi - fi - - if ! checkIntegrity "$plugin"; then - echo "Downloaded file is not a valid ZIP: $(getArchiveFilename "$plugin")" >&2 - echo "Download integrity: ${plugin}" >> "$FAILED" - return 1 - fi - - resolveDependencies "$plugin" - fi -} - -doDownload() { - local plugin version url jpi - plugin="$1" - version="$2" - jpi="$(getArchiveFilename "$plugin")" - - # If plugin already exists and is the same version do not download - if test -f "$jpi" && unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | grep "^Plugin-Version: ${version}$" > /dev/null; then - echo "Using provided plugin: $plugin" - return 0 - fi - - if [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then - # If version-specific Update Center is available, which is the case for LTS versions, - # use it to resolve latest versions. - url="$JENKINS_UC_LATEST/latest/${plugin}.hpi" - elif [[ "$version" == "experimental" && -n "$JENKINS_UC_EXPERIMENTAL" ]]; then - # Download from the experimental update center - url="$JENKINS_UC_EXPERIMENTAL/latest/${plugin}.hpi" - else - JENKINS_UC_DOWNLOAD=${JENKINS_UC_DOWNLOAD:-"$JENKINS_UC/download"} - url="$JENKINS_UC_DOWNLOAD/plugins/$plugin/$version/${plugin}.hpi" - fi - - echo "Downloading plugin: $plugin from $url" - curl --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-5}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" -s -f -L "$url" -o "$jpi" - return $? -} - -checkIntegrity() { - local plugin jpi - plugin="$1" - jpi="$(getArchiveFilename "$plugin")" - - unzip -t -qq "$jpi" >/dev/null - return $? -} - -resolveDependencies() { - local plugin jpi dependencies - plugin="$1" - jpi="$(getArchiveFilename "$plugin")" - - dependencies="$(unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | tr '\n' '|' | sed -e 's#| ##g' | tr '|' '\n' | grep "^Plugin-Dependencies: " | sed -e 's#^Plugin-Dependencies: ##')" - - if [[ ! $dependencies ]]; then - echo " > $plugin has no dependencies" - return - fi - - echo " > $plugin depends on $dependencies" - - IFS=',' read -r -a array <<< "$dependencies" - - for d in "${array[@]}" - do - plugin="$(cut -d':' -f1 - <<< "$d")" - if [[ $d == *"resolution:=optional"* ]]; then - echo "Skipping optional dependency $plugin" - else - local pluginInstalled - if pluginInstalled="$(echo -e "${bundledPlugins}\n${installedPlugins}" | grep "^${plugin}:")"; then - pluginInstalled="${pluginInstalled//[$'\r']}" - local versionInstalled; versionInstalled=$(versionFromPlugin "${pluginInstalled}") - local minVersion; minVersion=$(versionFromPlugin "${d}") - if versionLT "${versionInstalled}" "${minVersion}"; then - echo "Upgrading bundled dependency $d ($minVersion > $versionInstalled)" - download "$plugin" & - else - echo "Skipping already installed dependency $d ($minVersion <= $versionInstalled)" - fi - else - download "$plugin" & - fi - fi - done - wait -} - -bundledPlugins() { - local JENKINS_WAR=${JENKINS_HOME}/jenkins.war - if [ -f $JENKINS_WAR ] - then - TEMP_PLUGIN_DIR=/tmp/plugintemp.$$ - for i in $(jar tf $JENKINS_WAR | grep -E '[^detached-]plugins.*\..pi' | sort) - do - rm -fr $TEMP_PLUGIN_DIR - mkdir -p $TEMP_PLUGIN_DIR - PLUGIN=$(basename "$i"|cut -f1 -d'.') - (cd $TEMP_PLUGIN_DIR;jar xf "$JENKINS_WAR" "$i";jar xvf "$TEMP_PLUGIN_DIR/$i" META-INF/MANIFEST.MF >/dev/null 2>&1) - VER=$(grep -E -i Plugin-Version "$TEMP_PLUGIN_DIR/META-INF/MANIFEST.MF"|cut -d: -f2|sed 's/ //') - echo "$PLUGIN:$VER" - done - rm -fr $TEMP_PLUGIN_DIR - else - rm -f "$TEMP_ALREADY_INSTALLED" - echo "ERROR file not found: $JENKINS_WAR" - exit 1 - fi -} - -versionFromPlugin() { - local plugin=$1 - if [[ $plugin =~ .*:.* ]]; then - echo "${plugin##*:}" - else - echo "latest" - fi - -} - -installedPlugins() { - for f in "$REF_DIR"/*.jpi; do - echo "$(basename "$f" | sed -e 's/\.jpi//'):$(get_plugin_version "$f")" - done -} - -jenkinsMajorMinorVersion() { - local JENKINS_WAR - JENKINS_WAR=${JENKINS_HOME}/jenkins.war - if [[ -f "$JENKINS_WAR" ]]; then - local version major minor - version="$(java -jar ${JENKINS_HOME}/jenkins.war --version)" - major="$(echo "$version" | cut -d '.' -f 1)" - minor="$(echo "$version" | cut -d '.' -f 2)" - echo "$major.$minor" - else - echo "ERROR file not found: $JENKINS_WAR" - return 1 - fi -} - -main() { - local plugin pluginVersion jenkinsVersion - local plugins=() - - mkdir -p "$REF_DIR" || exit 1 - - # Read plugins from stdin or from the command line arguments - if [[ ($# -eq 0) ]]; then - while read -r line || [ "$line" != "" ]; do - # Remove leading/trailing spaces, comments, and empty lines - plugin=$(echo "${line}" | sed -e 's/^[ \t]*//g' -e 's/[ \t]*$//g' -e 's/[ \t]*#.*$//g' -e '/^[ \t]*$/d') - - # Avoid adding empty plugin into array - if [ ${#plugin} -ne 0 ]; then - plugins+=("${plugin}") - fi - done - else - plugins=("$@") - fi - - # Create lockfile manually before first run to make sure any explicit version set is used. - echo "Creating initial locks..." - for plugin in "${plugins[@]}"; do - mkdir "$(getLockFile "${plugin%%:*}")" - done - - echo "Analyzing war..." - bundledPlugins="$(bundledPlugins)" - - echo "Registering preinstalled plugins..." - installedPlugins="$(installedPlugins)" - - # Check if there's a version-specific update center, which is the case for LTS versions - jenkinsVersion="$(jenkinsMajorMinorVersion)" - if curl -fsL -o /dev/null "$JENKINS_UC/$jenkinsVersion"; then - JENKINS_UC_LATEST="$JENKINS_UC/$jenkinsVersion" - echo "Using version-specific update center: $JENKINS_UC_LATEST..." - else - JENKINS_UC_LATEST= - fi - - echo "Downloading plugins..." - for plugin in "${plugins[@]}"; do - pluginVersion="" - - if [[ $plugin =~ .*:.* ]]; then - pluginVersion=$(versionFromPlugin "${plugin}") - plugin="${plugin%%:*}" - fi - - download "$plugin" "$pluginVersion" "true" & - done - wait - - echo - echo "WAR bundled plugins:" - echo "${bundledPlugins}" - echo - echo "Installed plugins:" - installedPlugins - - if [[ -f $FAILED ]]; then - echo "Some plugins failed to download!" "$(<"$FAILED")" >&2 - exit 1 - fi - - echo "Cleaning up locks" - rm -r "$REF_DIR"/*.lock -} - -main "$@" - diff --git a/docker/jenkins/jenkins-master/jenkins-support b/docker/jenkins/jenkins-master/jenkins-support deleted file mode 100644 index 50cbf1d5eb5..00000000000 --- a/docker/jenkins/jenkins-master/jenkins-support +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash -eu - -# compare if version1 < version2 -versionLT() { - local v1; v1=$(echo "$1" | cut -d '-' -f 1 ) - local q1; q1=$(echo "$1" | cut -s -d '-' -f 2- ) - local v2; v2=$(echo "$2" | cut -d '-' -f 1 ) - local q2; q2=$(echo "$2" | cut -s -d '-' -f 2- ) - if [ "$v1" = "$v2" ]; then - if [ "$q1" = "$q2" ]; then - return 1 - else - if [ -z "$q1" ]; then - return 1 - else - if [ -z "$q2" ]; then - return 0 - else - [ "$q1" = "$(echo -e "$q1\n$q2" | sort -V | head -n1)" ] - fi - fi - fi - else - [ "$v1" = "$(echo -e "$v1\n$v2" | sort -V | head -n1)" ] - fi -} - -# returns a plugin version from a plugin archive -get_plugin_version() { - local archive; archive=$1 - local version; version=$(unzip -p "$archive" META-INF/MANIFEST.MF | grep "^Plugin-Version: " | sed -e 's#^Plugin-Version: ##') - version=${version%%[[:space:]]} - echo "$version" -} - -# Copy files from /usr/share/jenkins/ref into $JENKINS_HOME -# So the initial JENKINS-HOME is set with expected content. -# Don't override, as this is just a reference setup, and use from UI -# can then change this, upgrade plugins, etc. -copy_reference_file() { - f="${1%/}" - b="${f%.override}" - rel="${b:23}" - version_marker="${rel}.version_from_image" - dir=$(dirname "${b}") - local action; - local reason; - local container_version; - local image_version; - local marker_version; - local log; log=false - if [[ ${rel} == plugins/*.jpi ]]; then - container_version=$(get_plugin_version "$JENKINS_HOME/${rel}") - image_version=$(get_plugin_version "${f}") - if [[ -e $JENKINS_HOME/${version_marker} ]]; then - marker_version=$(cat "$JENKINS_HOME/${version_marker}") - if versionLT "$marker_version" "$container_version"; then - action="SKIPPED" - reason="Installed version ($container_version) has been manually upgraded from initial version ($marker_version)" - log=true - else - if [[ "$image_version" == "$container_version" ]]; then - action="SKIPPED" - reason="Version from image is the same as the installed version $image_version" - else - if versionLT "$image_version" "$container_version"; then - action="SKIPPED" - log=true - reason="Image version ($image_version) is older than installed version ($container_version)" - else - action="UPGRADED" - log=true - reason="Image version ($image_version) is newer than installed version ($container_version)" - fi - fi - fi - else - if [[ -n "$TRY_UPGRADE_IF_NO_MARKER" ]]; then - if [[ "$image_version" == "$container_version" ]]; then - action="SKIPPED" - reason="Version from image is the same as the installed version $image_version (no marker found)" - # Add marker for next time - echo "$image_version" > "$JENKINS_HOME/${version_marker}" - else - if versionLT "$image_version" "$container_version"; then - action="SKIPPED" - log=true - reason="Image version ($image_version) is older than installed version ($container_version) (no marker found)" - else - action="UPGRADED" - log=true - reason="Image version ($image_version) is newer than installed version ($container_version) (no marker found)" - fi - fi - fi - fi - if [[ ! -e $JENKINS_HOME/${rel} || "$action" == "UPGRADED" || $f = *.override ]]; then - action=${action:-"INSTALLED"} - log=true - mkdir -p "$JENKINS_HOME/${dir:23}" - cp -r "${f}" "$JENKINS_HOME/${rel}"; - # pin plugins on initial copy - touch "$JENKINS_HOME/${rel}.pinned" - echo "$image_version" > "$JENKINS_HOME/${version_marker}" - reason=${reason:-$image_version} - else - action=${action:-"SKIPPED"} - fi - else - if [[ ! -e $JENKINS_HOME/${rel} || $f = *.override ]] - then - action="INSTALLED" - log=true - mkdir -p "$JENKINS_HOME/${dir:23}" - cp -r "${f}" "$JENKINS_HOME/${rel}"; - else - action="SKIPPED" - fi - fi - if [[ -n "$VERBOSE" || "$log" == "true" ]]; then - if [ -z "$reason" ]; then - echo "$action $rel" >> "$COPY_REFERENCE_FILE_LOG" - else - echo "$action $rel : $reason" >> "$COPY_REFERENCE_FILE_LOG" - fi - fi -} - diff --git a/docker/jenkins/jenkins-master/jenkins.sh b/docker/jenkins/jenkins-master/jenkins.sh deleted file mode 100644 index 6a3c9b250d1..00000000000 --- a/docker/jenkins/jenkins-master/jenkins.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -e - -: "${JENKINS_HOME:="/opt/work/jenkins"}" -touch "${COPY_REFERENCE_FILE_LOG}" || { echo "Can not write to ${COPY_REFERENCE_FILE_LOG}. Wrong volume permissions?"; exit 1; } -echo "--- Copying files at $(date)" >> "$COPY_REFERENCE_FILE_LOG" -find ${JENKINS_HOME}/ \( -type f -o -type l \) -exec bash -c '. ${JENKINS_HOME}/jenkins-support; for arg; do copy_reference_file "$arg"; done' _ {} + - -# if `docker run` first argument start with `--` the user is passing jenkins launcher arguments -if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then - - # read JAVA_OPTS and JENKINS_OPTS into arrays to avoid need for eval (and associated vulnerabilities) - java_opts_array=() - while IFS= read -r -d '' item; do - java_opts_array+=( "$item" ) - done < <([[ $JAVA_OPTS ]] && xargs printf '%s\0' <<<"$JAVA_OPTS") - - jenkins_opts_array=( ) - while IFS= read -r -d '' item; do - jenkins_opts_array+=( "$item" ) - done < <([[ $JENKINS_OPTS ]] && xargs printf '%s\0' <<<"$JENKINS_OPTS") - - exec java -Duser.home="$JENKINS_HOME" "${java_opts_array[@]}" -jar ${JENKINS_HOME}/jenkins.war "${jenkins_opts_array[@]}" "$@" -fi - -# As argument is not jenkins, assume user want to run his own process, for example a `bash` shell to explore this image -exec "$@" - diff --git a/docker/jenkins/jenkins-master/plugins.sh b/docker/jenkins/jenkins-master/plugins.sh deleted file mode 100644 index e6b26670f01..00000000000 --- a/docker/jenkins/jenkins-master/plugins.sh +++ /dev/null @@ -1,122 +0,0 @@ -#! /bin/bash - -# Parse a support-core plugin -style txt file as specification for jenkins plugins to be installed -# in the reference directory, so user can define a derived Docker image with just : -# -# FROM jenkins -# COPY plugins.txt /plugins.txt -# RUN /usr/local/bin/plugins.sh /plugins.txt -# -# Note: Plugins already installed are skipped -# - -set -e - -echo "WARN: plugins.sh is deprecated, please switch to install-plugins.sh" - -if [ -z "$1" ] -then - echo " -USAGE: - Parse a support-core plugin -style txt file as specification for jenkins plugins to be installed - in the reference directory, so user can define a derived Docker image with just : - FROM jenkins - COPY plugins.txt /plugins.txt - RUN /usr/local/bin/plugins.sh /plugins.txt - Note: Plugins already installed are skipped -" - exit 1 -else - JENKINS_INPUT_JOB_LIST=$1 - if [ ! -f "$JENKINS_INPUT_JOB_LIST" ] - then - echo "ERROR File not found: $JENKINS_INPUT_JOB_LIST" - exit 1 - fi -fi - -# the war includes a # of plugins, to make the build efficient filter out -# the plugins so we dont install 2x - there about 17! -if [ -d "$JENKINS_HOME" ] -then - TEMP_ALREADY_INSTALLED=$JENKINS_HOME/preinstalled.plugins.$$.txt -else - echo "ERROR $JENKINS_HOME not found" - exit 1 -fi - -JENKINS_PLUGINS_DIR=/var/jenkins_home/plugins -if [ -d "$JENKINS_PLUGINS_DIR" ] -then - echo "Analyzing: $JENKINS_PLUGINS_DIR" - for i in "$JENKINS_PLUGINS_DIR"/*/; do - JENKINS_PLUGIN=$(basename "$i") - JENKINS_PLUGIN_VER=$(grep -E -i Plugin-Version "$i/META-INF/MANIFEST.MF"|cut -d: -f2|sed 's/ //') - echo "$JENKINS_PLUGIN:$JENKINS_PLUGIN_VER" - done >"$TEMP_ALREADY_INSTALLED" -else - JENKINS_WAR=/usr/share/jenkins/jenkins.war - if [ -f "$JENKINS_WAR" ] - then - echo "Analyzing war: $JENKINS_WAR" - TEMP_PLUGIN_DIR=/tmp/plugintemp.$$ - while read -r i <&3; do - rm -fr "$TEMP_PLUGIN_DIR" - mkdir -p "$TEMP_PLUGIN_DIR" - PLUGIN=$(basename "$i"|cut -f1 -d'.') - (cd "$TEMP_PLUGIN_DIR" || exit; jar xf "$JENKINS_WAR" "$i"; jar xvf "$TEMP_PLUGIN_DIR/$i" META-INF/MANIFEST.MF >/dev/null 2>&1) - VER=$(grep -E -i Plugin-Version "$TEMP_PLUGIN_DIR/META-INF/MANIFEST.MF"|cut -d: -f2|sed 's/ //') - echo "$PLUGIN:$VER" - done 3< <(jar tf "$JENKINS_WAR" | grep -E '[^detached-]plugins.*\..pi' | sort) > "$TEMP_ALREADY_INSTALLED" - rm -fr "$TEMP_PLUGIN_DIR" - else - rm -f "$TEMP_ALREADY_INSTALLED" - echo "ERROR file not found: $JENKINS_WAR" - exit 1 - fi -fi - -REF=/usr/share/jenkins/ref/plugins -mkdir -p $REF -COUNT_PLUGINS_INSTALLED=0 -while read -r spec || [ -n "$spec" ]; do - - plugin=() - IFS=' ' read -r -a plugin <<< "${spec//:/ }" - [[ ${plugin[0]} =~ ^# ]] && continue - [[ ${plugin[0]} =~ ^[[:space:]]*$ ]] && continue - [[ -z ${plugin[1]} ]] && plugin[1]="latest" - - if [ -z "$JENKINS_UC_DOWNLOAD" ]; then - JENKINS_UC_DOWNLOAD=$JENKINS_UC/download - fi - - if ! grep -q "${plugin[0]}:${plugin[1]}" "$TEMP_ALREADY_INSTALLED" - then - echo "Downloading ${plugin[0]}:${plugin[1]}" - curl --retry 3 --retry-delay 5 -sSL -f "${JENKINS_UC_DOWNLOAD}/plugins/${plugin[0]}/${plugin[1]}/${plugin[0]}.hpi" -o "$REF/${plugin[0]}.jpi" - unzip -qqt "$REF/${plugin[0]}.jpi" - (( COUNT_PLUGINS_INSTALLED += 1 )) - else - echo " ... skipping already installed: ${plugin[0]}:${plugin[1]}" - fi -done < "$JENKINS_INPUT_JOB_LIST" - -echo "---------------------------------------------------" -if (( "$COUNT_PLUGINS_INSTALLED" > 0 )) -then - echo "INFO: Successfully installed $COUNT_PLUGINS_INSTALLED plugins." - - if [ -d $JENKINS_PLUGINS_DIR ] - then - echo "INFO: Please restart the container for changes to take effect!" - fi -else - echo "INFO: No changes, all plugins previously installed." - -fi -echo "---------------------------------------------------" - -#cleanup -rm "$TEMP_ALREADY_INSTALLED" -exit 0 diff --git a/docker/jenkins/jenkins-slave/Dockerfile b/docker/jenkins/jenkins-slave/Dockerfile deleted file mode 100644 index 61f9c8e151a..00000000000 --- a/docker/jenkins/jenkins-slave/Dockerfile +++ /dev/null @@ -1,163 +0,0 @@ -FROM ubuntu:14.04 - -MAINTAINER The BigDL Authors https://github.com/intel-analytics/BigDL - -WORKDIR /opt/work - -ENV JAVA_7_HOME /opt/work/jdk7 -ENV JAVA_8_HOME /opt/work/jdk8 -ENV JAVA_HOME ${JAVA_7_HOME} -ENV CONDA_HOME /opt/work/conda -ENV JENKINS_HOME /opt/work/jenkins -ENV SPARK_1_5_HOME /opt/work/spark-1.5.0 -ENV SPARK_1_6_HOME /opt/work/spark-1.6.0 -ENV SPARK_2_1_HOME /opt/work/spark-2.1.1 -ENV SPARK_2_2_HOME /opt/work/spark-2.2.0 -ENV TORCH_VERSION d03a42834bb1b674495b0c42de1716b66cc388f1 -ENV TORCH_HOME /opt/work/torch -ENV CAFFE_HOME /opt/work/caffe -ENV PATH ${TORCH_HOME}/install/bin:${JAVA_HOME}/bin:${CONDA_HOME}/bin:${PATH} -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -ADD ./run-slave.sh /opt/work/jenkins/run-slave.sh -ADD ./slave.groovy /opt/work/jenkins/slave.groovy -ADD ./pip.conf /root/.pip/pip.conf -ADD ./settings.xml /root/.m2/settings.xml - -RUN apt-get update --fix-missing && \ - apt-get install -y vim curl nano wget unzip maven git && \ - apt-get install -y locales && locale-gen en_US.UTF-8 - -#jdk8 -RUN wget http://ftp.osuosl.org/pub/funtoo/distfiles/oracle-java/jdk-8u152-linux-x64.tar.gz && \ - gunzip jdk-8u152-linux-x64.tar.gz && \ - tar -xf jdk-8u152-linux-x64.tar -C /opt && \ - rm jdk-8u152-linux-x64.tar && \ - ln -s /opt/jdk1.8.0_152 ${JAVA_8_HOME} && \ -#jdk7 - wget http://ftp.osuosl.org/pub/funtoo/distfiles/oracle-java/jdk-7u80-linux-x64.tar.gz && \ - gunzip jdk-7u80-linux-x64.tar.gz && \ - tar -xf jdk-7u80-linux-x64.tar -C /opt && \ - rm jdk-7u80-linux-x64.tar && \ - ln -s /opt/jdk1.7.0_80 ${JAVA_7_HOME} && \ -#Jenkins - chmod a+x /opt/work/jenkins/run-slave.sh && \ - chmod a+x /opt/work/jenkins/slave.groovy && \ - wget http://repo.jenkins-ci.org/releases/org/jenkins-ci/main/remoting/3.14/remoting-3.14.jar && \ - mv remoting-3.14.jar ${JENKINS_HOME} && \ -#spark 1.5.0 1.6.0 2.1.1 2.2.0 - wget http://archive.apache.org/dist/spark/spark-1.5.0/spark-1.5.0-bin-hadoop2.6.tgz && \ - tar -xf spark-1.5.0-bin-hadoop2.6.tgz -C /opt/work && \ - rm spark-1.5.0-bin-hadoop2.6.tgz && \ - ln -s /opt/work/spark-1.5.0-bin-hadoop2.6 ${SPARK_1_5_HOME} && \ - wget http://archive.apache.org/dist/spark/spark-1.6.0/spark-1.6.0-bin-hadoop2.6.tgz && \ - tar -xf spark-1.6.0-bin-hadoop2.6.tgz -C /opt/work && \ - rm spark-1.6.0-bin-hadoop2.6.tgz && \ - ln -s /opt/work/spark-1.6.0-bin-hadoop2.6 ${SPARK_1_6_HOME} && \ - wget http://archive.apache.org/dist/spark/spark-2.1.1/spark-2.1.1-bin-hadoop2.7.tgz && \ - tar -xf spark-2.1.1-bin-hadoop2.7.tgz -C /opt/work && \ - rm spark-2.1.1-bin-hadoop2.7.tgz && \ - ln -s /opt/work/spark-2.1.1-bin-hadoop2.7 ${SPARK_2_1_HOME} && \ - wget http://archive.apache.org/dist/spark/spark-2.2.0/spark-2.2.0-bin-hadoop2.7.tgz && \ - tar -xf spark-2.2.0-bin-hadoop2.7.tgz -C /opt/work && \ - rm spark-2.2.0-bin-hadoop2.7.tgz && \ - ln -s /opt/work/spark-2.2.0-bin-hadoop2.7 ${SPARK_2_2_HOME} - -#MKL -RUN apt-get install -y libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler && \ - apt-get install -y --no-install-recommends libboost-all-dev && \ - apt-get install -y gcc g++ gfortran wget cpio && \ - wget http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/12414/l_mkl_2018.1.163.tgz && \ - tar -xzvf l_mkl_2018.1.163.tgz && \ - sed -i 's/ACCEPT_EULA=decline/ACCEPT_EULA=accept/g' /opt/work/l_mkl_2018.1.163/silent.cfg && \ - sed -i 's/ARCH_SELECTED=ALL/ARCH_SELECTED=INTEL64/g' /opt/work/l_mkl_2018.1.163/silent.cfg && \ - sed -i 's/COMPONENTS=DEFAULTS/COMPONENTS=;intel-comp-l-all-vars__noarch;intel-comp-nomcu-vars__noarch;intel-openmp__x86_64;intel-tbb-libs__x86_64;intel-mkl-common__noarch;intel-mkl-installer-license__noarch;intel-mkl-core__x86_64;intel-mkl-core-rt__x86_64;intel-mkl-doc__noarch;intel-mkl-doc-ps__noarch;intel-mkl-gnu__x86_64;intel-mkl-gnu-rt__x86_64;intel-mkl-common-ps__noarch;intel-mkl-core-ps__x86_64;intel-mkl-common-c__noarch;intel-mkl-core-c__x86_64;intel-mkl-common-c-ps__noarch;intel-mkl-tbb__x86_64;intel-mkl-tbb-rt__x86_64;intel-mkl-gnu-c__x86_64;intel-mkl-common-f__noarch;intel-mkl-core-f__x86_64;intel-mkl-gnu-f-rt__x86_64;intel-mkl-gnu-f__x86_64;intel-mkl-f95-common__noarch;intel-mkl-f__x86_64;intel-mkl-psxe__noarch;intel-psxe-common__noarch;intel-psxe-common-doc__noarch;intel-compxe-pset/g' /opt/work/l_mkl_2018.1.163/silent.cfg && \ - /opt/work/l_mkl_2018.1.163/install.sh -s /opt/work/l_mkl_2018.1.163/silent.cfg && \ - ln -sf /opt/intel/mkl/lib/intel64/libmkl_rt.so /usr/lib/libblas.so && \ - ln -sf /opt/intel/mkl/lib/intel64/libmkl_rt.so /usr/lib/libblas.so.3 && \ - ln -sf /opt/intel/mkl/lib/intel64/libmkl_rt.so /usr/lib/liblapack.so && \ - ln -sf /opt/intel/mkl/lib/intel64/libmkl_rt.so /usr/lib/liblapack.so.3 && \ - echo "/opt/intel/mkl/lib/intel64" > /etc/ld.so.conf.d/libblas.conf && \ - echo "/opt/intel/lib/intel64" >> /etc/ld.so.conf.d/libblas.conf && \ - ldconfig && \ - rm -rf /opt/work/l_mkl_2018.1.163.tgz /opt/work/l_mkl_2018.1.163 - -ENV TENSOR_FLOW_VERION 1.2.0 -ENV KERAS_VERSION 1.2.2 - -#python-native -RUN apt-get install -y --no-install-recommends build-essential tcl tk gettext autoconf zlib1g-dev gcc python-dev python3-dev libcurl4-openssl-dev && \ - wget https://bootstrap.pypa.io/get-pip.py && \ - python2 get-pip.py && \ - pip2 install --no-cache-dir robotframework robotframework-requests requests && \ - pip2 install numpy scipy pandas scikit-learn matplotlib seaborn jupyter plotly nltk twine pytest pytest-xdist h5py ipython==5.0.0 tensorflow==$TENSOR_FLOW_VERION keras==$KERAS_VERSION && \ - python3 get-pip.py && \ - pip install --no-cache-dir robotframework robotframework-requests requests && \ - pip install numpy scipy pandas scikit-learn matplotlib seaborn jupyter plotly nltk twine pytest pytest-xdist h5py ipython==5.0.0 tensorflow==$TENSOR_FLOW_VERION keras==$KERAS_VERSION - -#python-conda -RUN wget https://repo.continuum.io/miniconda/Miniconda3-4.3.31-Linux-x86_64.sh && \ - /bin/bash Miniconda3-4.3.31-Linux-x86_64.sh -f -b -p ${CONDA_HOME} && \ - rm Miniconda3-4.3.31-Linux-x86_64.sh && \ - conda config --system --prepend channels conda-forge && \ - conda config --system --set auto_update_conda false && \ - conda config --system --set show_channel_urls true && \ - conda update --all --quiet --yes && \ - conda clean -tipsy && \ - conda create -y -n py27 python=2.7 mkdocs numpy scipy pandas scikit-learn matplotlib seaborn jupyter plotly nltk twine pytest pytest-xdist h5py tensorflow==$TENSOR_FLOW_VERION keras==$KERAS_VERSION && \ - conda create -y -n py35 python=3.5 mkdocs numpy scipy pandas scikit-learn matplotlib seaborn jupyter plotly nltk twine pytest pytest-xdist h5py tensorflow==$TENSOR_FLOW_VERION keras==$KERAS_VERSION && \ - conda create -y -n py36 python=3.6 mkdocs numpy scipy pandas scikit-learn matplotlib seaborn jupyter plotly nltk twine pytest pytest-xdist h5py tensorflow==$TENSOR_FLOW_VERION keras==$KERAS_VERSION && \ - conda info --env - -#TENSORFLOW MODELS -ENV PYTHONPATH /opt/work/models/research/slim:${PYTHONPATH} -RUN git clone https://github.com/tensorflow/models.git - -#Caffe -ENV PYTHONPATH=${CAFFE_HOME}/python:${CAFFE_HOME}/distribute/python:${PYTHONPATH} -RUN apt-get install -y libgflags-dev libgoogle-glog-dev liblmdb-dev && \ - git clone https://github.com/BVLC/caffe.git && \ - cd /opt/work/caffe && \ - cp /opt/work/caffe/Makefile.config.example Makefile.config && \ - echo "CPU_ONLY := 1" >> Makefile.config && \ - sed -i "s/BLAS := atlas/BLAS := mkl/g" Makefile.config && \ - pip2 install -r /opt/work/caffe/python/requirements.txt && \ - ln -s /usr/include/python2.7/ /usr/local/include/python2.7 && \ - ln -s /usr/local/lib/python2.7/dist-packages/numpy/core/include/numpy/ /usr/local/include/python2.7/numpy && \ - make all -j4 && make pycaffe && make distribute && \ - make test && make runtest && \ - cd /opt/work && \ - ln /dev/null /dev/raw1394 - -#torch -RUN git clone https://github.com/torch/distro.git /opt/work/torch --recursive && \ - cd /opt/work/torch && git checkout ${TORCH_VERSION} && \ - echo "Acquire::http::Proxy \"${http_proxy}\";\nAcquire::https::Proxy \"${https_proxy}\";" > /etc/apt/apt.conf && \ - export CMAKE_INCLUDE_PATH=/opt/intel/mkl/include && \ - export CMAKE_LIBRARY_PATH=/opt/intel/mkl/lib/intel64:/opt/intel/mkl/compiler/lib/intel64 && \ - export LD_LIBRARY_PATH=$CMAKE_LIBRARY_PATH:$LD_LIBRARY_PATH && \ - ./install-deps && \ - yes | ./install.sh && \ - cd /opt/work/ - -#torch-rnn -RUN git config --global url.https://github.com/.insteadOf git://github.com/ && \ - cat ~/.gitconfig && \ - #luarocks install torch && \ - #luarocks install nn && \ - #luarocks install dpnn && \ - luarocks install torchx && \ - git clone https://github.com/Element-Research/dpnn.git && \ - cd dpnn && \ - git reset --hard ca0e99fa6d0dfd971f0f284520562765ae0865fd && \ - luarocks make rocks/dpnn-scm-1.rockspec && \ - #luarocks install torchx && \ - cd /opt/work && \ - git clone https://github.com/Element-Research/rnn.git && \ - cd rnn && \ - luarocks make rocks/rnn-scm-1.rockspec && \ - cd /opt/work - -CMD ["/opt/work/jenkins/run-slave.sh"] - diff --git a/docker/jenkins/jenkins-slave/pip.conf b/docker/jenkins/jenkins-slave/pip.conf deleted file mode 100644 index fa77e374324..00000000000 --- a/docker/jenkins/jenkins-slave/pip.conf +++ /dev/null @@ -1,8 +0,0 @@ -[global] -timeout = 60 -#index = your-private-pypi-index -#index-url = your-private-pypi-index-url -#extra-index-url = https://pypi.tuna.tsinghua.edu.cn/simple - -[install] -#trusted-host = your-private-pypi-host diff --git a/docker/jenkins/jenkins-slave/run-slave.sh b/docker/jenkins/jenkins-slave/run-slave.sh deleted file mode 100644 index 6a31b21ecb1..00000000000 --- a/docker/jenkins/jenkins-slave/run-slave.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -sed -i -e "s/jenkins_slave_name/$jenkins_slave_name/g" /opt/work/jenkins/slave.groovy -sed -i -e "s/jenkins_slave_label/$jenkins_slave_label/g" /opt/work/jenkins/slave.groovy -sed -i -e "s/jenkins_slave_executors/$jenkins_slave_executors/g" /opt/work/jenkins/slave.groovy - -resp=$(curl -s --user "admin:admin" -d "script=$(<${JENKINS_HOME}/slave.groovy)" "http://${jenkins_master_host}:${jenkins_master_port}/scriptText") -token=$(echo $resp|cut -f1 -d" ") -slaveName=$(echo $resp|cut -f2 -d" ") -echo "Successfully retrived info from http://${jenkins_master_host}:${jenkins_master_port}" -echo "SlaveName is $slaveName" -echo "CSRF token is $token" -$JAVA_8_HOME/bin/java -Dorg.jenkinsci.remoting.engine.Jnlpotocol3.disabled=false -cp /opt/work/jenkins/remoting-3.14.jar hudson.remoting.jnlp.Main -headless -url "http://${jenkins_master_host}:${jenkins_master_port}" $token $slaveName diff --git a/docker/jenkins/jenkins-slave/settings.xml b/docker/jenkins/jenkins-slave/settings.xml deleted file mode 100644 index d3975361d95..00000000000 --- a/docker/jenkins/jenkins-slave/settings.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/docker/jenkins/jenkins-slave/slave.groovy b/docker/jenkins/jenkins-slave/slave.groovy deleted file mode 100644 index 754cb7100af..00000000000 --- a/docker/jenkins/jenkins-slave/slave.groovy +++ /dev/null @@ -1,27 +0,0 @@ -import hudson.model.*; -import jenkins.model.*; -import hudson.slaves.*; -import jenkins.*; - -def date = new Date() -def no = date.format('yyyyMMddHHmm') -def name = "jenkins_slave_name-$no" -Jenkins.instance.addNode( - new DumbSlave( - name, - "this is a slave node", - "/opt/work/jenkins", - "jenkins_slave_executors", - Node.Mode.NORMAL, - "jenkins_slave_label", - new JNLPLauncher(), - new RetentionStrategy.Always(), - new LinkedList() - ) -) - -def slaves = hudson.model.Hudson.instance.slaves -def slave = slaves.find { it.name == name } -def token = slave.getComputer().getJnlpMac() -println "$token $name" -