Merge branch 'dev' into feature/itf_statuses

Signed-off-by: Kitsu <mail@kitsu.me>
hyperledger-archives · Sep 28, 2018 · 537b522 · 537b522
2 parents a27006d + 0094df6
commit 537b522
Show file tree

Hide file tree

Showing 230 changed files with 6,046 additions and 3,335 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "deploy/ansible/kubespray"]
+	path = deploy/ansible/kubespray
+	url = https://github.com/soramitsu/kubespray.git
diff --git a/.jenkinsci/artifacts.groovy b/.jenkinsci/artifacts.groovy
@@ -1,10 +1,8 @@
 #!/usr/bin/env groovy
 
-def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.co.jp']) {
-  def baseUploadPath = 'files'
+def uploadArtifacts(filePaths, uploadPath, artifactServers=['nexus.iroha.tech']) {
   def filePathsConverted = []
   agentType = sh(script: 'uname', returnStdout: true).trim()
-  uploadPath = baseUploadPath + uploadPath
   filePaths.each {
     fp = sh(script: "ls -d ${it} | tr '\n' ','", returnStdout: true).trim()
     filePathsConverted.addAll(fp.split(','))
@@ -24,32 +22,22 @@ def uploadArtifacts(filePaths, uploadPath, artifactServers=['artifact.soramitsu.
       sh "gpg --yes --batch --no-tty --import ${CI_GPG_PRIVKEY} || true"
     }
     filePathsConverted.each {
-      sh "echo put ${it} $uploadPath >> \$(pwd)/batch.txt;"
+      sh "echo ${it} >> \$(pwd)/batch.txt;"
       sh "$shaSumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).sha256"
       sh "$md5SumBinary ${it} | cut -d' ' -f1 > \$(pwd)/\$(basename ${it}).md5"
       // TODO @bakhtin 30.05.18 IR-1384. Make gpg command options and paths compatible with Windows OS.
       if (!agentType.contains('MSYS_NT')) {
-        sh "echo \"${CI_GPG_MASTERKEY}\" | $gpgKeyBinary -o \$(pwd)/\$(basename ${it}).asc ${it}"
-        sh "echo put \$(pwd)/\$(basename ${it}).asc $uploadPath >> \$(pwd)/batch.txt;"
+        sh "echo \"${CI_GPG_MASTERKEY}\" | $gpgKeyBinary -o \$(pwd)/\$(basename ${it}).ascfile ${it}"
+        sh "echo \$(pwd)/\$(basename ${it}).ascfile >> \$(pwd)/batch.txt;"
       }
-      sh "echo put \$(pwd)/\$(basename ${it}).sha256 $uploadPath >> \$(pwd)/batch.txt;"
-      sh "echo put \$(pwd)/\$(basename ${it}).md5 $uploadPath >> \$(pwd)/batch.txt;"
+      sh "echo \$(pwd)/\$(basename ${it}).sha256 >> \$(pwd)/batch.txt;"
+      sh "echo \$(pwd)/\$(basename ${it}).md5 >> \$(pwd)/batch.txt;"
     }
   }
-  // mkdirs recursively
-  uploadPath = uploadPath.split('/')
-  def p = ''
-  sh "> \$(pwd)/mkdirs.txt"
-  uploadPath.each {
-    p += "/${it}"
-    sh("echo -mkdir $p >> \$(pwd)/mkdirs.txt")
-  }
 
-  sshagent(['jenkins-artifact']) {
-    sh "ssh-agent"
+  withCredentials([usernamePassword(credentialsId: 'ci_nexus', passwordVariable: 'NEXUS_PASS', usernameVariable: 'NEXUS_USER')]) {
     artifactServers.each {
-      sh "sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -b \$(pwd)/mkdirs.txt jenkins@${it} || true"
-      sh "sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -b \$(pwd)/batch.txt jenkins@${it}"
+      sh(script: "while read line; do curl -u ${NEXUS_USER}:${NEXUS_PASS} --upload-file \$line https://${it}/repository/artifacts/${uploadPath}/ ; done < \$(pwd)/batch.txt")
     }
   }
 }

diff --git a/.jenkinsci/release-build.groovy b/.jenkinsci/release-build.groovy
@@ -60,7 +60,9 @@ def doReleaseBuild() {
   // push Docker image in case the current branch is develop,
   // or it is a commit into PR which base branch is develop (usually develop -> master)
   if (GIT_LOCAL_BRANCH == 'develop' || CHANGE_BRANCH_LOCAL == 'develop' || GIT_LOCAL_BRANCH == 'dev' || CHANGE_BRANCH_LOCAL == 'dev') {
-    iCRelease.push("${platform}-develop")
+    docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
+      iCRelease.push("${platform}-develop")
+    }
     if (manifest.manifestSupportEnabled()) {
       manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:develop",
         ["${DOCKER_REGISTRY_BASENAME}:x86_64-develop",
@@ -81,7 +83,9 @@ def doReleaseBuild() {
     }
   }
   else if (GIT_LOCAL_BRANCH == 'master') {
-    iCRelease.push("${platform}-latest")
+    docker.withRegistry('https://registry.hub.docker.com', 'docker-hub-credentials') {
+      iCRelease.push("${platform}-latest")
+    }
     if (manifest.manifestSupportEnabled()) {
       manifest.manifestCreate("${DOCKER_REGISTRY_BASENAME}:latest",
         ["${DOCKER_REGISTRY_BASENAME}:x86_64-latest",

diff --git a/.jenkinsci/selected-branches-coverage.groovy b/.jenkinsci/selected-branches-coverage.groovy
@@ -1,13 +1,7 @@
 #!/usr/bin/env groovy
 
-def selectedBranchesCoverage(branches, PRCoverage=true) {
-	// trigger coverage if branch is either develop or master, or it is a PR
-	if (PRCoverage) {
-		return env.GIT_LOCAL_BRANCH in branches || env.CHANGE_ID != null
-	}
-	else {
-		return env.GIT_LOCAL_BRANCH in branches
-	}
+def selectedBranchesCoverage(branches) {
+	return env.GIT_LOCAL_BRANCH in branches
 }
 
-return this
+return this
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -18,7 +18,7 @@ properties([parameters([
   choice(choices: 'Release\nDebug', description: 'Android bindings build type', name: 'ABBuildType'),
   choice(choices: 'arm64-v8a\narmeabi-v7a\narmeabi\nx86_64\nx86', description: 'Android bindings platform', name: 'ABPlatform'),
   booleanParam(defaultValue: true, description: 'Build docs', name: 'Doxygen'),
-  string(defaultValue: '4', description: 'How much parallelism should we exploit. "4" is optimal for machines with modest amount of memory and at least 4 cores', name: 'PARALLELISM')])])
+  string(defaultValue: '8', description: 'Expect ~3GB memory consumtion per CPU core', name: 'PARALLELISM')])])
 
 
 pipeline {
@@ -75,12 +75,12 @@ pipeline {
             beforeAgent true
             expression { return params.x86_64_linux }
           }
-          agent { label 'x86_64' }
+          agent { label 'docker-build-agent' }
           steps {
             script {
               debugBuild = load ".jenkinsci/debug-build.groovy"
               coverage = load ".jenkinsci/selected-branches-coverage.groovy"
-              if (coverage.selectedBranchesCoverage(['develop', 'master', 'dev'])) {
+              if (coverage.selectedBranchesCoverage(['master'])) {
                 debugBuild.doDebugBuild(true)
               }
               else {
@@ -111,7 +111,7 @@ pipeline {
             script {
               debugBuild = load ".jenkinsci/debug-build.groovy"
               coverage = load ".jenkinsci/selected-branches-coverage.groovy"
-              if (!params.x86_64_linux && !params.armv8_linux && !params.x86_64_macos && (coverage.selectedBranchesCoverage(['develop', 'master', 'dev']))) {
+              if (!params.x86_64_linux && !params.armv8_linux && !params.x86_64_macos && (coverage.selectedBranchesCoverage(['master']))) {
                 debugBuild.doDebugBuild(true)
               }
               else {
@@ -142,7 +142,7 @@ pipeline {
             script {
               debugBuild = load ".jenkinsci/debug-build.groovy"
               coverage = load ".jenkinsci/selected-branches-coverage.groovy"
-              if (!params.x86_64_linux && !params.x86_64_macos && (coverage.selectedBranchesCoverage(['develop', 'master', 'dev']))) {
+              if (!params.x86_64_linux && !params.x86_64_macos && (coverage.selectedBranchesCoverage(['master']))) {
                 debugBuild.doDebugBuild(true)
               }
               else {
@@ -174,7 +174,7 @@ pipeline {
               def coverageEnabled = false
               def cmakeOptions = ""
               coverage = load ".jenkinsci/selected-branches-coverage.groovy"
-              if (!params.x86_64_linux && (coverage.selectedBranchesCoverage(['develop', 'master', 'dev']))) {
+              if (!params.x86_64_linux && (coverage.selectedBranchesCoverage(['master']))) {
                 coverageEnabled = true
                 cmakeOptions = " -DCOVERAGE=ON "
               }
@@ -248,7 +248,7 @@ pipeline {
                       def artifacts = load ".jenkinsci/artifacts.groovy"
                       def commit = env.GIT_COMMIT
                       filePaths = [ '\$(pwd)/build/*.tar.gz' ]
-                      artifacts.uploadArtifacts(filePaths, sprintf('/iroha/macos/%1$s-%2$s-%3$s', [GIT_LOCAL_BRANCH, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)]))
+                      artifacts.uploadArtifacts(filePaths, sprintf('iroha/macos/%1$s-%2$s-%3$s', [GIT_LOCAL_BRANCH, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)]))
                     }
                   }
                   finally {
@@ -276,7 +276,7 @@ pipeline {
             beforeAgent true
             expression { return params.x86_64_linux }
           }
-          agent { label 'x86_64' }
+          agent { label 'docker-build-agent' }
           steps {
             script {
               def releaseBuild = load ".jenkinsci/release-build.groovy"
@@ -355,7 +355,7 @@ pipeline {
                       def artifacts = load ".jenkinsci/artifacts.groovy"
                       def commit = env.GIT_COMMIT
                       filePaths = [ '\$(pwd)/build/*.tar.gz' ]
-                      artifacts.uploadArtifacts(filePaths, sprintf('/iroha/macos/%1$s-%2$s-%3$s', [GIT_LOCAL_BRANCH, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)]))
+                      artifacts.uploadArtifacts(filePaths, sprintf('iroha/macos/%1$s-%2$s-%3$s', [GIT_LOCAL_BRANCH, sh(script: 'date "+%Y%m%d"', returnStdout: true).trim(), commit.substring(0,6)]))
                     }
                   }
                   finally {
@@ -373,9 +373,7 @@ pipeline {
         beforeAgent true
         expression { return params.Doxygen }
       }
-      // build docs on any vacant node. Prefer `x86_64` over
-      // others as nodes are more powerful
-      agent { label 'x86_64' }
+      agent { label 'docker-build-agent' }
       steps {
         script {
           def doxygen = load ".jenkinsci/doxygen.groovy"
@@ -385,7 +383,7 @@ pipeline {
             "$platform-develop-build",
             "${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/develop/Dockerfile",
             "${env.GIT_RAW_BASE_URL}/${env.GIT_PREVIOUS_COMMIT}/docker/develop/Dockerfile",
-            "${env.GIT_RAW_BASE_URL}/develop/docker/develop/Dockerfile",
+            "${env.GIT_RAW_BASE_URL}/dev/docker/develop/Dockerfile",
             ['PARALLELISM': params.PARALLELISM])
           iC.inside() {
             doxygen.doDoxygen()
@@ -409,7 +407,7 @@ pipeline {
             beforeAgent true
             expression { return params.x86_64_linux }
           }
-          agent { label 'x86_64' }
+          agent { label 'docker-build-agent' }
           environment {
             JAVA_HOME = "/usr/lib/jvm/java-8-oracle"
           }
@@ -423,7 +421,7 @@ pipeline {
                   "$platform-develop-build",
                   "${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/develop/Dockerfile",
                   "${env.GIT_RAW_BASE_URL}/${env.GIT_PREVIOUS_COMMIT}/docker/develop/Dockerfile",
-                  "${env.GIT_RAW_BASE_URL}/develop/docker/develop/Dockerfile",
+                  "${env.GIT_RAW_BASE_URL}/dev/docker/develop/Dockerfile",
                   ['PARALLELISM': params.PARALLELISM])
                 if (params.JavaBindings) {
                   iC.inside("-v /tmp/${env.GIT_COMMIT}/bindings-artifact:/tmp/bindings-artifact") {
@@ -441,7 +439,7 @@ pipeline {
                   "android-${params.ABPlatform}-${params.ABBuildType}",
                   "${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/android/Dockerfile",
                   "${env.GIT_RAW_BASE_URL}/${env.GIT_PREVIOUS_COMMIT}/docker/android/Dockerfile",
-                  "${env.GIT_RAW_BASE_URL}/develop/docker/android/Dockerfile",
+                  "${env.GIT_RAW_BASE_URL}/dev/docker/android/Dockerfile",
                   ['PARALLELISM': params.PARALLELISM, 'PLATFORM': params.ABPlatform, 'BUILD_TYPE': params.ABBuildType])
                 sh "curl -L -o /tmp/${env.GIT_COMMIT}/entrypoint.sh ${env.GIT_RAW_BASE_URL}/${env.GIT_COMMIT}/docker/android/entrypoint.sh"
                 sh "chmod +x /tmp/${env.GIT_COMMIT}/entrypoint.sh"
@@ -458,15 +456,15 @@ pipeline {
                 def commit = env.GIT_COMMIT
                 if (params.JavaBindings) {
                   javaBindingsFilePaths = [ '/tmp/${GIT_COMMIT}/bindings-artifact/java-bindings-*.zip' ]
-                  artifacts.uploadArtifacts(javaBindingsFilePaths, '/iroha/bindings/java')
+                  artifacts.uploadArtifacts(javaBindingsFilePaths, 'iroha/bindings/java')
                 }
                 if (params.PythonBindings) {
                   pythonBindingsFilePaths = [ '/tmp/${GIT_COMMIT}/bindings-artifact/python-bindings-*.zip' ]
-                  artifacts.uploadArtifacts(pythonBindingsFilePaths, '/iroha/bindings/python')
+                  artifacts.uploadArtifacts(pythonBindingsFilePaths, 'iroha/bindings/python')
                 }
                 if (params.AndroidBindings) {
                   androidBindingsFilePaths = [ '/tmp/${GIT_COMMIT}/bindings-artifact/android-bindings-*.zip' ]
-                  artifacts.uploadArtifacts(androidBindingsFilePaths, '/iroha/bindings/android')
+                  artifacts.uploadArtifacts(androidBindingsFilePaths, 'iroha/bindings/android')
                 }
               }
             }
@@ -500,11 +498,11 @@ pipeline {
                 def commit = env.GIT_COMMIT
                 if (params.JavaBindings) {
                   javaBindingsFilePaths = [ '/tmp/${GIT_COMMIT}/bindings-artifact/java-bindings-*.zip' ]
-                  artifacts.uploadArtifacts(javaBindingsFilePaths, '/iroha/bindings/java')
+                  artifacts.uploadArtifacts(javaBindingsFilePaths, 'iroha/bindings/java')
                 }
                 if (params.PythonBindings) {
                   pythonBindingsFilePaths = [ '/tmp/${GIT_COMMIT}/bindings-artifact/python-bindings-*.zip' ]
-                  artifacts.uploadArtifacts(pythonBindingsFilePaths, '/iroha/bindings/python')
+                  artifacts.uploadArtifacts(pythonBindingsFilePaths, 'iroha/bindings/python')
                 }
               }
             }

diff --git a/clean.sh b/clean.sh
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
 
-rm -rf external
 rm -rf build
 rm -rf cmake-build-debug
diff --git a/deploy/ansible/inventory/group_vars/all.yml b/deploy/ansible/inventory/group_vars/all.yml
@@ -1,3 +1,13 @@
 ---
 torii_port: 50051
 internal_port: 10001
+
+# kubespray
+# Valid bootstrap options (required): ubuntu, coreos, centos, none
+bootstrap_os: ubuntu
+
+#Directory where etcd data stored
+etcd_data_dir: /var/lib/etcd
+
+# Directory where the binaries will be installed
+bin_dir: /usr/local/bin