Release 0.20.0

.ci/travis/install.sh

@@ -40,6 +40,9 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
     esac
     pyenv rehash
     pyenv activate conan
+else
+    sudo apt-get update
+    sudo apt-get install gcc-multilib g++-multilib
 fi
 
 pip install -r conans/requirements_dev.txt

.travis.yml

@@ -15,26 +15,26 @@ matrix:
     include:
         - language: generic
           os: osx
-          env: PYVER=py27 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0
+          env: PYVER=py27 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0 CONAN_LOGGING_LEVEL=20
 
         - language: generic
           os: osx
-          env: PYVER=py34 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0
+          env: PYVER=py34 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0 CONAN_LOGGING_LEVEL=20
 
         - language: generic
           os: osx
-          env: PYVER=py35 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0
+          env: PYVER=py35 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0 CONAN_LOGGING_LEVEL=20
 
         - language: generic
           os: osx
-          env: PYVER=py36 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0
+          env: PYVER=py36 CONAN_COMPILER=apple-clang CONAN_COMPILER_VERSION=6.0 CONAN_LOGGING_LEVEL=20
 
 # command to install dependencies
 install:
   - ./.ci/travis/install.sh
 before_script:
   - export PYTHONPATH=$PYTHONPATH:$(pwd)
-  - export CONAN_LOGGING_LEVEL=10
+  - export CONAN_LOGGING_LEVEL=20
 # command to run tests
 script: 
   - ulimit -n 2048 # Error with py3 and OSX, max file descriptors

appveyor.yml

@@ -1,4 +1,6 @@
 build: false
+cache:
+    - C:\mingw64-> appveyor.yml
 
 environment:
   matrix:
@@ -10,7 +12,9 @@ init:
   - "ECHO %PYTHON%"
 
 install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - If Not Exist "C:\mingw64" appveyor DownloadFile "https://s3-eu-west-1.amazonaws.com/downloads.conan.io/x86_64-6.3.0-release-posix-sjlj-rt_v5-rev1.7z"
+  - If Not Exist "C:\mingw64" 7z x x86_64-6.3.0-release-posix-sjlj-rt_v5-rev1.7z -oc:\
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\mingw64\\bin;%PATH%"
   - "set PYTHONPATH=%PYTHONPATH%;%CD%"
   - "set CONAN_LOGGING_LEVEL=10"
   - "set CONAN_COMPILER=Visual Studio"
@@ -19,6 +23,7 @@ install:
   - "%PYTHON%/Scripts/pip.exe install -r conans/requirements_dev.txt"
   - "%PYTHON%/Scripts/pip.exe install -r conans/requirements_server.txt"
 
+
 test_script:
   - "nosetests --with-coverage conans.test"
 

conans/__init__.py

@@ -6,11 +6,13 @@
 from conans.client.cmake import CMake
 from conans.client.gcc import GCC
 from conans.client.configure_environment import ConfigureEnvironment
+from conans.client.configure_build_environment import (AutoToolsBuildEnvironment, VisualStudioBuildEnvironment)
 from conans.util.files import load
 import os
 
 # complex_search: With ORs and not filtering by not restricted settings
 COMPLEX_SEARCH_CAPABILITY = "complex_search"
 SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ]
 
-__version__ = '0.19.3'
+__version__ = '0.20.0'
+

conans/build_info/__init__.py

@@ -0,0 +1,2 @@
+"""Module for extract the JFrog Artifactory Build Info JSON from a conan tracer log:
+   https://github.com/JFrogDev/build-info"""

conans/build_info/command.py

@@ -0,0 +1,40 @@
+import argparse
+import json
+import os
+
+from conans.build_info.conan_build_info import get_build_info
+from conans.util.files import save
+
+
+def run():
+
+    parser = argparse.ArgumentParser(description='Extracts build-info from a specified '
+                                                 'conan trace log and return a valid JSON')
+    parser.add_argument('trace_path', help='Path to the conan trace log file e.j: '
+                                           '/tmp/conan_trace.log')
+    parser.add_argument("--output", default=False,
+                        help='Optional file to output the JSON contents, if not specified the JSON'
+                             ' will be printed to stdout')
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.trace_path):
+        print("Error, conan trace log not found! '%s'" % args.trace_path)
+        exit(1)
+    if args.output and not os.path.exists(os.path.dirname(args.output)):
+        print("Error, output file directory not found! '%s'" % args.trace_path)
+        exit(1)
+
+    try:
+        info = get_build_info(args.trace_path)
+        the_json = json.dumps(info.serialize())
+        if args.output:
+            save(args.output, the_json)
+        else:
+            print(the_json)
+    except Exception as exc:
+        print(exc)
+        exit(1)
+
+if __name__ == "__main__":
+    run()

conans/build_info/conan_build_info.py

@@ -0,0 +1,125 @@
+import json
+import os
+from collections import defaultdict
+
+from conans.build_info.model import BuildInfo, BuildInfoModule, BuildInfoModuleArtifact, BuildInfoModuleDependency
+from conans.model.info import ConanInfo, PackageReference
+from conans.model.ref import ConanFileReference
+from conans.util.files import load
+
+
+def _extract_uploads_from_conan_trace(path):
+    modules = defaultdict(dict)  # dict of {conan_ref: [abs_path1, abs_path2]}
+    try:
+        with open(path, "r") as traces:
+            for line in traces.readlines():
+                doc = json.loads(line)
+                if doc["_action"] in ("UPLOADED_RECIPE", "UPLOADED_PACKAGE"):
+                    module_type = "recipe" if doc["_action"] == "UPLOADED_RECIPE" else "package"
+                    modules[doc["_id"]] = {"remote": doc["remote"], "files": [], "type": module_type}
+                    modules[doc["_id"]]["files"].extend(doc["files"])
+    except ValueError as exc:
+        raise Exception("INVALID TRACE FILE! %s" % exc)
+
+    return modules
+
+
+def _extract_downloads_from_conan_trace(path):
+    downloaded_modules = defaultdict(dict)  # dict of {conan_ref: {"files": [doc_file, doc_file], "remote": remote }}
+
+    with open(path, "r") as traces:
+        for line in traces.readlines():
+            doc = json.loads(line)
+            if doc["_action"] in ["DOWNLOADED_PACKAGE", "DOWNLOADED_RECIPE"]:
+                downloaded_modules[doc["_id"]] = {"files": doc["files"], "remote": doc["remote"]}
+    return downloaded_modules
+
+
+def _get_type(file_path):
+    return os.path.splitext(file_path)[1].upper()[1:]
+
+
+def _get_build_info_artifact(file_doc):
+    the_type = _get_type(file_doc["path"])
+    ret = BuildInfoModuleArtifact(the_type, file_doc["sha1"], file_doc["md5"], file_doc["name"])
+    return ret
+
+
+def _get_dependency(file_doc, dep_ref):
+    the_type = _get_type(file_doc["path"])
+    the_id = "%s:%s" % (dep_ref, file_doc["name"])
+    ret = BuildInfoModuleDependency(the_id, the_type, file_doc["sha1"], file_doc["md5"])
+    return ret
+
+
+def _get_upload_modules_with_deps(uploaded_files, downloaded_files):
+    modules = []
+    deps = defaultdict(set)  # Reference: [Reference, Reference]
+    # Extract needed information
+    for module_id, mod_doc in uploaded_files.items():
+        module_id = ConanFileReference.loads(module_id) if mod_doc["type"] == "recipe" \
+            else PackageReference.loads(module_id)
+        # Store recipe and package dependencies
+        if mod_doc["type"] == "package":
+            conan_infos = [file_doc for file_doc in mod_doc["files"] if file_doc["name"] == "conaninfo.txt"]
+            if conan_infos:
+                conan_info = conan_infos[0]["path"]
+                info = ConanInfo.loads(load(conan_info))
+                for package_reference in info.full_requires:
+                    deps[str(module_id.conan)].add(str(package_reference.conan))
+                    deps[str(module_id)].add(str(package_reference))
+
+    # Add the modules
+    for module_id, mod_doc in uploaded_files.items():
+        module = BuildInfoModule()
+        module.id = str(module_id)
+        # Add artifacts
+        for file_doc in mod_doc["files"]:
+            artifact = _get_build_info_artifact(file_doc)
+            module.artifacts.append(artifact)
+
+        # Add dependencies, for each module dep modules
+        for mod_dep_id in deps[module_id]:
+            if mod_dep_id in downloaded_files:
+                down_module = downloaded_files[mod_dep_id]
+                # Check if the remote from the uploaded package matches the remote from the downloaded dependency
+                if down_module.get("remote", None) == mod_doc["remote"]:
+                    for file_doc in down_module["files"]:
+                        module.dependencies.append(_get_dependency(file_doc, mod_dep_id))
+
+        modules.append(module)
+    return modules
+
+
+def _get_only_downloads_module(downloaded_files):
+    """
+    Gets a BuildInfoModule for the downloaded_files
+    :param downloaded_files: {conan_ref: {"files": [doc_file, doc_file], "remote": remote }}
+    :return: BuildInfoModule object
+    """
+    ret = BuildInfoModule()
+    ret.id = "DownloadOnly"
+    for ref, file_docs in downloaded_files.items():
+        files = file_docs["files"]
+        for file_doc in files:
+            the_type = _get_type(file_doc["path"])
+            dep = BuildInfoModuleDependency(file_doc["name"], the_type, file_doc["sha1"], file_doc["md5"])
+            ret.dependencies.append(dep)
+    return ret
+
+
+def _build_modules(trace_path):
+
+    uploaded_files = _extract_uploads_from_conan_trace(trace_path)
+    downloaded_files = _extract_downloads_from_conan_trace(trace_path)
+    if uploaded_files:
+        return _get_upload_modules_with_deps(uploaded_files, downloaded_files)
+    else:
+        return [_get_only_downloads_module(downloaded_files)]
+
+
+def get_build_info(trace_path):
+    bi = BuildInfo()
+    modules = _build_modules(trace_path)
+    bi.modules.extend(modules)
+    return bi

conans/build_info/model.py

@@ -0,0 +1,33 @@
+from collections import namedtuple
+
+import conans
+
+
+class BuildInfo(object):
+
+    def __init__(self):
+        self.modules = []
+
+    def serialize(self):
+        return {"modules": [module.serialize() for module in self.modules],
+                "buildAgent": {"name": "Conan", "version": conans.__version__}}
+
+
+class BuildInfoModule(object):
+
+    def __init__(self):
+        # Conan package or recipe
+        self.id = ""
+        self.artifacts = []
+        self.dependencies = []
+
+    def serialize(self):
+        return {"id": self.id,
+                "artifacts": [ar._asdict() for ar in self.artifacts],
+                "dependencies": [dep._asdict() for dep in self.dependencies]}
+
+
+BuildInfoModuleArtifact = namedtuple("BuildInfoModuleArtifact", ['type', 'sha1', 'md5', 'name'])
+BuildInfoModuleDependency = namedtuple('BuildInfoModuleDependency', ['id', 'type', 'sha1', 'md5'])
+
+

conans/client/client_cache.py

@@ -1,12 +1,14 @@
 import os
+
+from conans.errors import ConanException
 from conans.util.files import save, load, mkdir, normalize
 from conans.model.settings import Settings
 from conans.client.conf import ConanClientConfigParser, default_client_conf, default_settings_yml
 from conans.model.values import Values
 from conans.client.detect import detect_defaults_settings
 from conans.model.ref import ConanFileReference
 from conans.model.manifest import FileTreeManifest
-from conans.paths import SimplePaths, CONANINFO
+from conans.paths import SimplePaths, CONANINFO, PUT_HEADERS
 from genericpath import isdir
 from conans.model.info import ConanInfo
 
@@ -31,6 +33,30 @@ def __init__(self, base_folder, store_folder, output):
         self._store_folder = store_folder or self.conan_config.storage_path or self.conan_folder
         super(ClientCache, self).__init__(self._store_folder)
 
+    @property
+    def put_headers_path(self):
+        return os.path.join(self.conan_folder, PUT_HEADERS)
+
+    def read_put_headers(self):
+        ret = {}
+        if not os.path.exists(self.put_headers_path):
+            save(self.put_headers_path, "")
+            return ret
+        try:
+            contents = load(self.put_headers_path)
+            for line in contents.splitlines():
+                if line:
+                    tmp = line.split("=", 1)
+                    if len(tmp) != 2:
+                        raise Exception()
+                    name = tmp[0].strip()
+                    value = tmp[1].strip()
+                    ret[str(name)] = str(value)
+            return ret
+        except Exception:
+            raise ConanException("Invalid %s file!" % self.put_headers_path)
+
+
     @property
     def registry(self):
         return os.path.join(self.conan_folder, REGISTRY)