Client with apiv2 (transfers in one step)

.ci/jenkins/Jenkinsfile

@@ -1,110 +1,136 @@
-def nightly = (env.JOB_NAME == "ConanNightly")
+// ########## CONFIGURATION ##################
+def full_py_vers = false
+def test_v2 = true
+// ############################
 
-def slaves = ['Linux', 'Windows', 'Macos']
 
-def pyvers 
 if (env.BRANCH_NAME =~ /(^release.*)|(^master)/) {
-    pyvers = ['py37', 'py36', 'py34', 'py27']
+    full_py_vers = true
 }
-else{
-	pyvers = ['py37', 'py36', 'py27']
+
+def slaves = ['Linux', 'Windows', 'Macos']
+def flavors = ["no_revisions"]
+
+
+def pyvers  = ['py36', 'py27']
+def api_confs = ["v1"]
+
+if (full_py_vers){
+    pyvers = ['py36', 'py34', 'py27']
+    api_confs = ["v1", "v2"]
+}
+else if(test_v2){
+    api_confs = ["v1", "v2"]
 }
 
+
 def module = "\"conans.test\""
 def numcores = "--num_cores=3"
 
-
 def win_tmp_base = "D:/J/t/"
 def rest_tmp_base = "/tmp/"
 def runner = ".ci/jenkins/runner.py"
 
 def commit
 def branch
 
-
-def builders = [:]
 try{
-    for (x in slaves) {
-        def slave = x
-        for (y in pyvers) {
-            def pyver = y
-            if(slave != "Linux" && pyver=="py37"){
-                continue;
-            }
+    for (flavor in flavors){
+        def builders = [:]
+        for (x in slaves) {
+            def slave = x
+            for (y in pyvers) {
+                for (z in api_confs){
+                    def pyver = y
+                    def api_conf = z
+                    def name = "${slave} - ${flavor} - ${pyver} - API${api_conf}"
+
+                    if (api_conf == "v2" && slave != "Linux"){
+                        continue;
+                    }
 
-            builders["${slave} - ${pyver}"] = {
-                node(slave) {
-                    stage("${slave} - ${pyver}"){
-                        def workdir
-                        def sourcedir
-                        def base_source
-                        lock('source_code') { // Prepare a clean new directory with the sources
-                            try{
-                                step ([$class: 'WsCleanup'])
-                            }
-                            catch(e){
-                                echo "Cannot clean WS"
-                            }
+                    if (flavor == "revisions" && api_conf == "v1"){
+                        continue;
+                    }
 
-                            def vars = checkout scm
-                            commit = vars["GIT_COMMIT"].substring(0, 4)
-                            branch = vars["GIT_BRANCH"]
-                            def base_dir = (slave == "Windows") ? win_tmp_base : rest_tmp_base
-                            workdir = "${base_dir}${commit}/${pyver}"
-                            base_source = "${base_dir}source/${commit}"
-                            sourcedir = "${base_source}/${pyver}"
-                            while(fileExists(sourcedir)){
-                                sourcedir = sourcedir + "_"
-                            }
+                    if (flavor == "revisions" && pyver == "py34"){
+                        continue;
+                    }
 
-                            dir(base_source){ // Trick to create the parent
-                                def escaped_ws = "${WORKSPACE}".replace("\\", "/")
-                                def cmd = "python -c \"import shutil; shutil.copytree('${escaped_ws}', '${sourcedir}')\""
-                                if (slave == "Windows"){
-                                    bat(script: cmd)
+                    builders[name] = {
+                        node(slave) {
+                            stage(name){
+                                def workdir
+                                def sourcedir
+                                def base_source
+                                lock('source_code') { // Prepare a clean new directory with the sources
+                                    try{
+                                        step ([$class: 'WsCleanup'])
+                                    }
+                                    catch(e){
+                                        echo "Cannot clean WS"
+                                    }
+
+                                    def vars = checkout scm
+                                    commit = vars["GIT_COMMIT"].substring(0, 4)
+                                    branch = vars["GIT_BRANCH"]
+                                    def base_dir = (slave == "Windows") ? win_tmp_base : rest_tmp_base
+                                    workdir = "${base_dir}${commit}/${pyver}/${api_conf}"
+                                    base_source = "${base_dir}source/${commit}"
+                                    sourcedir = "${base_source}/${pyver}/${api_conf}"
+                                    while(fileExists(sourcedir)){
+                                        sourcedir = sourcedir + "_"
+                                    }
+
+                                    dir(base_source){ // Trick to create the parent
+                                        def escaped_ws = "${WORKSPACE}".replace("\\", "/")
+                                        def cmd = "python -c \"import shutil; shutil.copytree('${escaped_ws}', '${sourcedir}')\""
+                                        if (slave == "Windows"){
+                                            bat(script: cmd)
+                                        }
+                                        else{
+                                            sh(script: cmd)
+                                        }
+                                    }
                                 }
-                                else{
-                                    sh(script: cmd)
+                                if(slave == "Linux"){
+                                    sh "docker pull lasote/conantests"
+                                    docker.image('lasote/conantests').inside("-e CONAN_USER_HOME=${sourcedir} -v${sourcedir}:${sourcedir}") {
+                                        sh(script: "python ${runner} ${module} ${pyver} ${sourcedir} ${workdir} -e rest_api ${numcores}  --server_api=${api_conf} -f ${flavor}")
+                                    }
                                 }
+                                else if(slave == "Windows"){
+                                    try{
+
+                                      withEnv(["CONAN_TEST_FOLDER=${workdir}"]){
+                                        bat(script: "python ${runner} ${module} ${pyver} ${sourcedir} \"${workdir}\" -e rest_api -e windows_ci_excluded ${numcores} --server_api=${api_conf} -f ${flavor}")
+                                      }
+                                    }
+                                    finally{
+                                      bat(script: "rd /s /q \"${workdir}\"")
+                                      bat(script: "rd /s /q \"${sourcedir}\"")
+                                    }
+                                }
+                                else if(slave == "Macos"){
+                                    try{
+                                      withEnv(['PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin']) {
+                                          sh(script: "python ${runner} ${module} ${pyver} ${sourcedir} ${workdir} -e rest_api ${numcores} --server_api=${api_conf} -f ${flavor}")
+                                      }
+                                    }
+                                    finally{
+                                      sh(script: "rm -rf ${workdir}")
+                                      sh(script: "rm -rf ${sourcedir}")
+                                    }
+                                }
+                                //step([$class: 'JUnitResultArchiver', testResults: '**/nosetests.xml'])
                             }
                         }
-
-                        if(slave == "Linux"){
-                            sh "docker pull lasote/conantests"
-                            docker.image('lasote/conantests').inside("-e CONAN_USER_HOME=${sourcedir} -v${sourcedir}:${sourcedir}") {
-                                sh(script: "python ${runner} ${module} ${pyver} ${sourcedir} ${workdir} -e rest_api ${numcores}")
-                            }
-                        }
-                        else if(slave == "Windows"){
-                            try{
-                              withEnv(["CONAN_TEST_FOLDER=${workdir}"]){
-                                bat(script: "python ${runner} ${module} ${pyver} ${sourcedir} \"${workdir}\" -e rest_api -e windows_ci_excluded ${numcores}")
-                              }
-                            }
-                            finally{
-                              bat(script: "rd /s /q \"${workdir}\"")
-                              bat(script: "rd /s /q \"${sourcedir}\"")
-                            }
-                        }
-                        else if(slave == "Macos"){
-                            try{
-                              withEnv(['PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin']) {
-                                  sh(script: "python ${runner} ${module} ${pyver} ${sourcedir} ${workdir} -e rest_api ${numcores}")
-                              }
-                            }
-                            finally{
-                              sh(script: "rm -rf ${workdir}")
-                              sh(script: "rm -rf ${sourcedir}")
-                            }
-                        }
-                        //step([$class: 'JUnitResultArchiver', testResults: '**/nosetests.xml'])
                     }
                 }
             }
         }
+        parallel builders
     }
-    parallel builders
-
 
 
 
@@ -148,16 +174,24 @@ try{
     }
 }
 catch(e){
-    if(nightly){
+    if(env.JOB_NAME == "ConanNightly"){
         def subject = "FAILED: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'"
         def summary = "${subject} (${env.BUILD_URL}), Branch: ${branch}"
         slackSend (color: '#FF0000', message: summary)
     }
     throw e
 }
 
-if(nightly){
+if(env.JOB_NAME == "ConanNightly"){
     def subject = "SUCCESS ${env.JOB_NAME}! Another day with a green ${branch}!"
     def summary = "${subject} (${env.BUILD_URL})"
     slackSend (color: '#00FF00', message: summary)
 }
+
+
+def build_linux(){
+    sh "docker pull lasote/conantests"
+    docker.image('lasote/conantests').inside("-e CONAN_USER_HOME=${sourcedir} -v${sourcedir}:${sourcedir}") {
+        sh(script: "python ${runner} ${module} ${pyver} ${sourcedir} ${workdir} -e rest_api ${numcores}  --server_api=${api_conf}")
+    }
+}

.ci/jenkins/runner.py

@@ -9,27 +9,31 @@
                "Darwin": macpylocation}[platform.system()]
 
 
-def run_tests(module_path, pyver, source_folder, tmp_folder,
-              exluded_tags, num_cores=3, verbosity=None):
+def run_tests(module_path, pyver, source_folder, tmp_folder, flavor,
+              excluded_tags, num_cores=3, verbosity=None, server_api=None):
 
     verbosity = verbosity or (2 if platform.system() == "Windows" else 1)
-    exluded_tags = exluded_tags or ""
     venv_dest = os.path.join(tmp_folder, "venv")
     if not os.path.exists(venv_dest):
         os.makedirs(venv_dest)
     venv_exe = os.path.join(venv_dest,
                             "bin" if platform.system() != "Windows" else "Scripts",
                             "activate")
-    if exluded_tags:
-        exluded_tags = '-A "%s"' % " and ".join(["not %s" % tag for tag in exluded_tags])
+
+    if flavor == "revisions":
+        excluded_tags.append("only_without_revisions")
+    elif flavor == "no_revisions":
+        excluded_tags.append("only_revisions")
+
+    exluded_tags_str = '-A "%s"' % " and ".join(["not %s" % tag for tag in excluded_tags]) if excluded_tags else ""
 
     pyenv = pylocations[pyver]
     source_cmd = "." if platform.system() != "Windows" else ""
     # Prevent OSX to lock when no output is received
     debug_traces = ""  # "--debug=nose,nose.result" if platform.system() == "Darwin" and pyver != "py27" else ""
     # pyenv = "/usr/local/bin/python2"
     multiprocess = ("--processes=%s --process-timeout=1000 "
-                    "--process-restartworker --with-coverage" % num_cores)
+                    "--process-restartworker --with-coverage" % num_cores) if platform.system() != "Darwin" or pyver == "py27" else ""
 
     if num_cores <= 1:
         multiprocess = ""
@@ -54,7 +58,7 @@ def run_tests(module_path, pyver, source_folder, tmp_folder,
               "&& codecov -t f1a9c517-3d81-4213-9f51-61513111fc28".format(
                                     **{"module_path": module_path,
                                        "pyenv": pyenv,
-                                       "excluded_tags": exluded_tags,
+                                       "excluded_tags": exluded_tags_str,
                                        "venv_dest": venv_dest,
                                        "verbosity": verbosity,
                                        "venv_exe": venv_exe,
@@ -65,9 +69,13 @@ def run_tests(module_path, pyver, source_folder, tmp_folder,
 
     env = get_environ(tmp_folder)
     env["PYTHONPATH"] = source_folder
-    # env["CONAN_RECIPE_LINTER"] = "False"
     env["CONAN_LOGGING_LEVEL"] = "50" if platform.system() == "Darwin" else "50"
     env["CHANGE_AUTHOR_DISPLAY_NAME"] = ""
+    if server_api == "v2":
+        env["CONAN_TESTING_SERVER_V2_ENABLED"] = "1"
+    if flavor == "revisions":
+        env["CONAN_TESTING_SERVER_REVISIONS_ENABLED"] = "1"
+
     with chdir(source_folder):
         with environment_append(env):
             run(command)
@@ -95,9 +103,12 @@ def run(command):
     parser.add_argument('source_folder', help='Folder containing the conan source code')
     parser.add_argument('tmp_folder', help='Folder to create the venv inside')
     parser.add_argument('--num_cores', type=int, help='Number of cores to use', default=3)
+    parser.add_argument('--server_api', help='Test all with v1 or v2', default="v1")
     parser.add_argument('--exclude_tag', '-e', nargs=1, action=Extender,
                         help='Tags to exclude from testing, e.g.: rest_api')
+    parser.add_argument('--flavor', '-f', help='Flavor (revisions, no_revisions)')
 
     args = parser.parse_args()
-    run_tests(args.module, args.pyver, args.source_folder, args.tmp_folder, args.exclude_tag, num_cores=args.num_cores)
 
+    run_tests(args.module, args.pyver, args.source_folder, args.tmp_folder, args.flavor,
+              args.exclude_tag, num_cores=args.num_cores, server_api=args.server_api)

conans/client/rest/rest_client.py

@@ -1,5 +1,8 @@
 from collections import defaultdict
+
+from conans import API_V2, CHECKSUM_DEPLOY
 from conans.client.rest.rest_client_v1 import RestV1Methods
+from conans.client.rest.rest_client_v2 import RestV2Methods
 
 
 class RestApiClient(object):
@@ -28,8 +31,14 @@ def _get_api(self):
             _, _, cap = tmp.server_info()
             self._capabilities[self.remote_url] = cap
 
-        return RestV1Methods(self.remote_url, self.token, self.custom_headers, self._output,
-                             self.requester, self.verify_ssl, self._put_headers)
+        if API_V2 in self._capabilities[self.remote_url]:
+            checksum_deploy = CHECKSUM_DEPLOY in self._capabilities[self.remote_url]
+            return RestV2Methods(self.remote_url, self.token, self.custom_headers, self._output,
+                                 self.requester, self.verify_ssl, self._put_headers,
+                                 checksum_deploy)
+        else:
+            return RestV1Methods(self.remote_url, self.token, self.custom_headers, self._output,
+                                 self.requester, self.verify_ssl, self._put_headers)
 
     def get_conan_manifest(self, conan_reference):
         return self._get_api().get_conan_manifest(conan_reference)