From cba710d57293067229632ec6612a2e657186b458 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 29 Apr 2016 13:01:44 -0500 Subject: [PATCH 01/11] start splitting off recipe render from build --- bin/conda-render | 5 + conda_build/completers.py | 49 +++++++++ conda_build/main_build.py | 103 +----------------- conda_build/main_render.py | 218 +++++++++++++++++++++++++++++++++++++ conda_build/utils.py | 22 ++++ 5 files changed, 300 insertions(+), 97 deletions(-) create mode 100644 bin/conda-render create mode 100644 conda_build/completers.py create mode 100644 conda_build/main_render.py diff --git a/bin/conda-render b/bin/conda-render new file mode 100644 index 0000000000..c813e1c321 --- /dev/null +++ b/bin/conda-render @@ -0,0 +1,5 @@ +#!/usr/bin/env python +import sys +from conda_build.main_render import main + +sys.exit(main()) diff --git a/conda_build/completers.py b/conda_build/completers.py new file mode 100644 index 0000000000..81ff0b286e --- /dev/null +++ b/conda_build/completers.py @@ -0,0 +1,49 @@ +from os.path import isdir, isfile +from conda.cli.common import Completer + + +all_versions = { + 'python': [26, 27, 33, 34, 35], + 'numpy': [16, 17, 18, 19, 110], + 'perl': None, + 'R': None, + 'lua': ["2.0", "5.1", "5.2", "5.3"] +} + +conda_version = { + 'python': 'CONDA_PY', + 'numpy': 'CONDA_NPY', + 'perl': 'CONDA_PERL', + 'R': 'CONDA_R', + 'lua': 'CONDA_LUA', +} + +class RecipeCompleter(Completer): + def _get_items(self): + completions = [] + for path in listdir('.'): + if isdir(path) and isfile(join(path, 'meta.yaml')): + completions.append(path) + if isfile('meta.yaml'): + completions.append('.') + return completions + +# These don't represent all supported versions. It's just for tab completion. + +class PythonVersionCompleter(Completer): + def _get_items(self): + return ['all'] + [str(i/10) for i in all_versions['python']] + +class NumPyVersionCompleter(Completer): + def _get_items(self): + versions = [str(i) for i in all_versions['numpy']] + return ['all'] + ['%s.%s' % (ver[0], ver[1:]) for ver in versions] + +class RVersionsCompleter(Completer): + def _get_items(self): + return ['3.1.2', '3.1.3', '3.2.0', '3.2.1', '3.2.2'] + +class LuaVersionsCompleter(Completer): + def _get_items(self): + return ['all'] + [i for i in all_versions['lua']] + diff --git a/conda_build/main_build.py b/conda_build/main_build.py index c68aa1e14b..866aa077ce 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -20,65 +20,25 @@ from conda.compat import PY3 from conda.cli.common import add_parser_channels, Completer from conda.cli.conda_argparse import ArgumentParser +from conda.install import delete_trash from conda.resolve import NoPackagesFound, Unsatisfiable from conda_build import __version__, exceptions from conda_build.index import update_index -from conda.install import delete_trash +from conda_build.main_render import get_render_parser +from conda_build.completers import (all_versions, RecipeCompleter, PythonVersionCompleter, + RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) on_win = (sys.platform == 'win32') -all_versions = { - 'python': [26, 27, 33, 34, 35], - 'numpy': [16, 17, 18, 19, 110], - 'perl': None, - 'R': None, - 'lua': ["2.0", "5.1", "5.2", "5.3"] -} - -class RecipeCompleter(Completer): - def _get_items(self): - completions = [] - for path in listdir('.'): - if isdir(path) and isfile(join(path, 'meta.yaml')): - completions.append(path) - if isfile('meta.yaml'): - completions.append('.') - return completions - -# These don't represent all supported versions. It's just for tab completion. - -class PythonVersionCompleter(Completer): - def _get_items(self): - return ['all'] + [str(i/10) for i in all_versions['python']] - -class NumPyVersionCompleter(Completer): - def _get_items(self): - versions = [str(i) for i in all_versions['numpy']] - return ['all'] + ['%s.%s' % (ver[0], ver[1:]) for ver in versions] - -class RVersionsCompleter(Completer): - def _get_items(self): - return ['3.1.2', '3.1.3', '3.2.0', '3.2.1', '3.2.2'] - -class LuaVersionsCompleter(Completer): - def _get_items(self): - return ['all'] + [i for i in all_versions['lua']] def main(): - p = ArgumentParser( - description=""" + p=get_render_parser() + p.description=""" Tool for building conda packages. A conda package is a binary tarball containing system-level libraries, Python modules, executable programs, or other components. conda keeps track of dependencies between packages and platform specifics, making it simple to create working environments from different sets of packages.""" - ) - p.add_argument( - '-V', '--version', - action='version', - help='Show the conda-build version number and exit.', - version = 'conda-build %s' % __version__, - ) p.add_argument( "--check", action="store_true", @@ -121,14 +81,6 @@ def main(): action="store_true", help="Test package (assumes package is already build).", ) - p.add_argument( - 'recipe', - action="store", - metavar='RECIPE_PATH', - nargs='+', - choices=RecipeCompleter(), - help="Path to recipe directory.", - ) p.add_argument( '--no-test', action='store_true', @@ -157,49 +109,6 @@ def main(): action="store_true", help="do not display progress bar", ) - p.add_argument( - '--python', - action="append", - help="""Set the Python version used by conda build. Can be passed - multiple times to build against multiple versions. Can be 'all' to - build against all known versions (%r)""" % [i for i in - PythonVersionCompleter() if '.' in i], - metavar="PYTHON_VER", - choices=PythonVersionCompleter(), - ) - p.add_argument( - '--perl', - action="append", - help="""Set the Perl version used by conda build. Can be passed - multiple times to build against multiple versions.""", - metavar="PERL_VER", - ) - p.add_argument( - '--numpy', - action="append", - help="""Set the NumPy version used by conda build. Can be passed - multiple times to build against multiple versions. Can be 'all' to - build against all known versions (%r)""" % [i for i in - NumPyVersionCompleter() if '.' in i], - metavar="NUMPY_VER", - choices=NumPyVersionCompleter(), - ) - p.add_argument( - '--R', - action="append", - help="""Set the R version used by conda build. Can be passed - multiple times to build against multiple versions.""", - metavar="R_VER", - choices=RVersionsCompleter(), - ) - p.add_argument( - '--lua', - action="append", - help="""Set the Lua version used by conda build. Can be passed - multiple times to build against multiple versions (%r).""" % [i for i in LuaVersionsCompleter()], - metavar="LUA_VER", - choices=LuaVersionsCompleter(), - ) add_parser_channels(p) p.set_defaults(func=execute) diff --git a/conda_build/main_render.py b/conda_build/main_render.py new file mode 100644 index 0000000000..191677d0df --- /dev/null +++ b/conda_build/main_render.py @@ -0,0 +1,218 @@ +# (c) Continuum Analytics, Inc. / http://continuum.io +# All Rights Reserved +# +# conda is distributed under the terms of the BSD 3-clause license. +# Consult LICENSE.txt or http://opensource.org/licenses/BSD-3-Clause. + +from __future__ import absolute_import, division, print_function + +import sys +from collections import deque +from glob import glob +from locale import getpreferredencoding +import os +from os.path import exists, isdir, isfile, join, abspath + +# import conda.config as config +from conda_build.config import config +from conda.compat import PY3 +from conda.cli.common import add_parser_channels +from conda.cli.conda_argparse import ArgumentParser + +from conda_build import __version__, exceptions +from conda_build.metadata import MetaData +import conda_build.source as source +from conda_build.completers import (all_versions, conda_version, RecipeCompleter, PythonVersionCompleter, + RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) +from conda_build.utils import find_recipe + +on_win = (sys.platform == 'win32') + + +def get_render_parser(): + p = ArgumentParser( + description=""" +Tool for building conda packages. A conda package is a binary tarball +containing system-level libraries, Python modules, executable programs, or +other components. conda keeps track of dependencies between packages and +platform specifics, making it simple to create working environments from + different sets of packages.""", + conflict_handler='resolve' + ) + p.add_argument( + '-V', '--version', + action='version', + help='Show the conda-build version number and exit.', + version = 'conda-build %s' % __version__, + ) + p.add_argument( + '-s', "--source", + action="store_true", + help="Obtain the source and fill in related template variables.", + ) + p.add_argument( + 'recipe', + action="store", + metavar='RECIPE_PATH', + choices=RecipeCompleter(), + help="Path to recipe directory.", + ) + p.add_argument( + '--python', + action="append", + help="""Set the Python version used by conda build. Can be passed + multiple times to build against multiple versions. Can be 'all' to + build against all known versions (%r)""" % [i for i in + PythonVersionCompleter() if '.' in i], + metavar="PYTHON_VER", + choices=PythonVersionCompleter(), + ) + p.add_argument( + '--perl', + action="append", + help="""Set the Perl version used by conda build. Can be passed + multiple times to build against multiple versions.""", + metavar="PERL_VER", + ) + p.add_argument( + '--numpy', + action="append", + help="""Set the NumPy version used by conda build. Can be passed + multiple times to build against multiple versions. Can be 'all' to + build against all known versions (%r)""" % [i for i in + NumPyVersionCompleter() if '.' in i], + metavar="NUMPY_VER", + choices=NumPyVersionCompleter(), + ) + p.add_argument( + '--R', + action="append", + help="""Set the R version used by conda build. Can be passed + multiple times to build against multiple versions.""", + metavar="R_VER", + choices=RVersionsCompleter(), + ) + p.add_argument( + '--lua', + action="append", + help="""Set the Lua version used by conda build. Can be passed + multiple times to build against multiple versions (%r).""" % [i for i in LuaVersionsCompleter()], + metavar="LUA_VER", + choices=LuaVersionsCompleter(), + ) + add_parser_channels(p) + return p + + +def set_language_env_vars(args): + """Given args passed into conda command, set language env vars""" + for lang in all_versions: + versions = getattr(args, lang) + if not versions: + continue + if versions == ['all']: + if all_versions[lang]: + versions = all_versions[lang] + else: + parser.error("'all' is not supported for --%s" % lang) + if len(versions) > 1: + for ver in versions[:]: + setattr(args, lang, [str(ver)]) + execute(args, parser) + # This is necessary to make all combinations build. + setattr(args, lang, versions) + return + else: + version = versions[0] + if lang in ('python', 'numpy'): + version = int(version.replace('.', '')) + setattr(config, conda_version[lang], version) + if not len(str(version)) in (2, 3) and lang in ['python', 'numpy']: + if all_versions[lang]: + raise RuntimeError("%s must be major.minor, like %s, not %s" % + (conda_version[lang], all_versions[lang][-1]/10, version)) + else: + raise RuntimeError("%s must be major.minor, not %s" % + (conda_version[lang], version)) + + # Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc. + # Auto-set those env variables + for var in conda_version.values(): + if hasattr(config, var): + # Set the env variable. + os.environ[var] = str(getattr(config, var)) + + +def render_recipe(recipe_path, download_source=False): + import shutil + import tarfile + import tempfile + + from conda.lock import Locked + + with Locked(config.croot): + arg = recipe_path + try_again = False + # Don't use byte literals for paths in Python 2 + if not PY3: + arg = arg.decode(getpreferredencoding() or 'utf-8') + if isfile(arg): + if arg.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')): + recipe_dir = tempfile.mkdtemp() + t = tarfile.open(arg, 'r:*') + t.extractall(path=recipe_dir) + t.close() + need_cleanup = True + else: + print("Ignoring non-recipe: %s" % arg) + return + else: + recipe_dir = abspath(arg) + need_cleanup = False + + if not isdir(recipe_dir): + sys.exit("Error: no such directory: %s" % recipe_dir) + + try: + m = MetaData(recipe_dir) + except exceptions.YamlParsingError as e: + sys.stderr.write(e.error_msg()) + sys.exit(1) + + if download_source: + source.provide(m.path, m.get_section('source'), patch=False) + print('Source tree in:', source.get_dir()) + + try: + m.parse_again(permit_undefined_jinja=False) + except SystemExit: + # Something went wrong; possibly due to undefined GIT_ jinja variables. + # Maybe we need to actually download the source in order to resolve the build_id. + source.provide(m.path, m.get_section('source')) + + # Parse our metadata again because we did not initialize the source + # information before. + m.parse_again(permit_undefined_jinja=False) + + print(build.bldpkg_path(m)) + raise + + if need_cleanup: + shutil.rmtree(recipe_dir) + + return m + + +def main(): + import pprint + p = get_render_parser() + + args = p.parse_args() + set_language_env_vars(args) + + metadata = render_recipe(find_recipe(args.recipe), download_source=args.source) + pprint.pprint(metadata.meta) + + +if __name__ == '__main__': + main() diff --git a/conda_build/utils.py b/conda_build/utils.py index 9e18ae4d51..aacb0d25ab 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function import os +import fnmatch import sys import shutil import tarfile @@ -20,6 +21,27 @@ from conda.install import rm_rf rm_rf + +def _recursive_glob(treeroot, pattern): + results = [] + for base, dirs, files in os.walk(treeroot): + goodfiles = fnmatch.filter(files, pattern) + results.extend(os.path.join(base, f) for f in goodfiles) + return results + + +def find_recipe(path): + """recurse through a folder, locating meta.yaml. Raises error if more than one is found. + + Returns folder containing meta.yaml, to be built.""" + results = _recursive_glob(path, "meta.yaml") + if len(results) > 1: + raise IOError("More than one meta.yaml files found in %s" % path) + elif not results: + raise IOError("No meta.yaml files found in %s" % path) + return os.path.dirname(results[0]) + + def copy_into(src, dst): "Copy all the files and directories in src to the directory dst" From 5600f5181b0829c7788b943e707255d159ad545f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 29 Apr 2016 18:57:30 -0500 Subject: [PATCH 02/11] add option to output rendered recipe as yaml --- conda_build/main_render.py | 42 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 191677d0df..0274b253a0 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -13,7 +13,8 @@ import os from os.path import exists, isdir, isfile, join, abspath -# import conda.config as config +import yaml + from conda_build.config import config from conda.compat import PY3 from conda.cli.common import add_parser_channels @@ -203,15 +204,52 @@ def render_recipe(recipe_path, download_source=False): return m +# Next bit of stuff is to support YAML output in the order we expect. +# http://stackoverflow.com/a/17310199/1170370 +class MetaYaml(dict): + fields = ["package", "source", "build", "requirements", "test", "extra"] + def to_omap(self): + return [(field, self[field]) for field in MetaYaml.fields] + + +def represent_omap(dumper, data): + return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.to_omap()) + +def unicode_representer(dumper, uni): + node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni) + return node + + +class IndentDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super(IndentDumper, self).increase_indent(flow, False) + + +yaml.add_representer(MetaYaml, represent_omap) +if PY3: + yaml.add_representer(str, unicode_representer) +else: + yaml.add_representer(unicode, unicode_representer) + + def main(): import pprint p = get_render_parser() + p.add_argument( + '-y', '--yaml', + action="store_true", + help="print YAML, as opposed to printing the metadata as a dictionary" + ) args = p.parse_args() set_language_env_vars(args) metadata = render_recipe(find_recipe(args.recipe), download_source=args.source) - pprint.pprint(metadata.meta) + if args.yaml: + print(yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper, + default_flow_style=False, indent=4)) + else: + pprint.pprint(metadata.meta) if __name__ == '__main__': From 72ac7ffc00cab2c36b3d9527b30fb428fdf451de Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 29 Apr 2016 21:52:36 -0500 Subject: [PATCH 03/11] fix build tests; add nested recipe build test --- conda_build/completers.py | 5 ++-- conda_build/main_build.py | 23 +++++++++++-------- conda_build/main_render.py | 15 ++++++------ .../nested_recipe/build_number/meta.yaml | 6 +++++ .../nested_recipe/build_number/run_test.bat | 10 ++++++++ .../nested_recipe/build_number/run_test.sh | 6 +++++ 6 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 tests/test-recipes/metadata/nested_recipe/build_number/meta.yaml create mode 100644 tests/test-recipes/metadata/nested_recipe/build_number/run_test.bat create mode 100644 tests/test-recipes/metadata/nested_recipe/build_number/run_test.sh diff --git a/conda_build/completers.py b/conda_build/completers.py index 81ff0b286e..16441961c9 100644 --- a/conda_build/completers.py +++ b/conda_build/completers.py @@ -1,4 +1,5 @@ -from os.path import isdir, isfile +import os +from os.path import isdir, isfile, join from conda.cli.common import Completer @@ -21,7 +22,7 @@ class RecipeCompleter(Completer): def _get_items(self): completions = [] - for path in listdir('.'): + for path in os.listdir('.'): if isdir(path) and isfile(join(path, 'meta.yaml')): completions.append(path) if isfile('meta.yaml'): diff --git a/conda_build/main_build.py b/conda_build/main_build.py index 866aa077ce..3d976b1a8c 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -26,8 +26,9 @@ from conda_build import __version__, exceptions from conda_build.index import update_index from conda_build.main_render import get_render_parser -from conda_build.completers import (all_versions, RecipeCompleter, PythonVersionCompleter, +from conda_build.completers import (all_versions, conda_version, RecipeCompleter, PythonVersionCompleter, RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) +from conda_build.utils import find_recipe on_win = (sys.platform == 'win32') @@ -98,6 +99,14 @@ def main(): action="store_true", help="Run the post-build logic. Implies --no-test and --no-anaconda-upload.", ) + p.add_argument( + 'recipe', + action="store", + metavar='RECIPE_PATH', + nargs='+', + choices=RecipeCompleter(), + help="Path to recipe directory.", + ) p.add_argument( '--skip-existing', action='store_true', @@ -212,14 +221,6 @@ def execute(args, parser): "imported that is hard-linked by files in the trash. " "Will try again on next run.") - conda_version = { - 'python': 'CONDA_PY', - 'numpy': 'CONDA_NPY', - 'perl': 'CONDA_PERL', - 'R': 'CONDA_R', - 'lua': 'CONDA_LUA', - } - for lang in ['python', 'numpy', 'perl', 'R', 'lua']: versions = getattr(args, lang) if not versions: @@ -252,7 +253,7 @@ def execute(args, parser): # Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc. # Auto-set those env variables for var in conda_version.values(): - if getattr(config, var): + if hasattr(config, var): # Set the env variable. os_environ[var] = str(getattr(config, var)) @@ -287,6 +288,8 @@ def execute(args, parser): recipe_dir = abspath(arg) need_cleanup = False + # recurse looking for meta.yaml that is potentially not in immediate folder + recipe_dir = find_recipe(recipe_dir) if not isdir(recipe_dir): sys.exit("Error: no such directory: %s" % recipe_dir) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 0274b253a0..948b19443c 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -51,13 +51,6 @@ def get_render_parser(): action="store_true", help="Obtain the source and fill in related template variables.", ) - p.add_argument( - 'recipe', - action="store", - metavar='RECIPE_PATH', - choices=RecipeCompleter(), - help="Path to recipe directory.", - ) p.add_argument( '--python', action="append", @@ -240,6 +233,14 @@ def main(): action="store_true", help="print YAML, as opposed to printing the metadata as a dictionary" ) + # we do this one separately because we only allow one entry to conda render + p.add_argument( + 'recipe', + action="store", + metavar='RECIPE_PATH', + choices=RecipeCompleter(), + help="Path to recipe directory.", + ) args = p.parse_args() set_language_env_vars(args) diff --git a/tests/test-recipes/metadata/nested_recipe/build_number/meta.yaml b/tests/test-recipes/metadata/nested_recipe/build_number/meta.yaml new file mode 100644 index 0000000000..6211c1fc37 --- /dev/null +++ b/tests/test-recipes/metadata/nested_recipe/build_number/meta.yaml @@ -0,0 +1,6 @@ +package: + name: conda-build-test-build-number + version: 1.0 + +build: + number: 1 diff --git a/tests/test-recipes/metadata/nested_recipe/build_number/run_test.bat b/tests/test-recipes/metadata/nested_recipe/build_number/run_test.bat new file mode 100644 index 0000000000..03efa65f8f --- /dev/null +++ b/tests/test-recipes/metadata/nested_recipe/build_number/run_test.bat @@ -0,0 +1,10 @@ +conda list -p "%PREFIX%" --canonical +if errorlevel 1 exit 1 +for /f "delims=" %%i in ('conda list -p "%PREFIX%" --canonical') do set condalist=%%i +if errorlevel 1 exit 1 +echo "%condalist%" +if not "%condalist%"=="conda-build-test-build-number-1.0-1" exit 1 +cat "%PREFIX%\conda-meta\conda-build-test-build-number-1.0-1.json" +if errorlevel 1 exit 1 +cat "%PREFIX%\conda-meta\conda-build-test-build-number-1.0-1.json" | grep '"build_number": 1' +if errorlevel 1 exit 1 diff --git a/tests/test-recipes/metadata/nested_recipe/build_number/run_test.sh b/tests/test-recipes/metadata/nested_recipe/build_number/run_test.sh new file mode 100644 index 0000000000..e0c3296e85 --- /dev/null +++ b/tests/test-recipes/metadata/nested_recipe/build_number/run_test.sh @@ -0,0 +1,6 @@ +conda list -p $PREFIX --canonical +# This is actually the build string. We test the build number below +[ "$(conda list -p $PREFIX --canonical)" = "conda-build-test-build-number-1.0-1" ] + +cat $PREFIX/conda-meta/conda-build-test-build-number-1.0-1.json +cat $PREFIX/conda-meta/conda-build-test-build-number-1.0-1.json | grep '"build_number": 1' From 24259367d40e8df687c8e2ec4ad0b4966754b539 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 29 Apr 2016 22:20:16 -0500 Subject: [PATCH 04/11] separate more of conda-build into conda-render (filename output) --- conda_build/main_build.py | 56 ++------------------------------------ conda_build/main_render.py | 32 +++++++++++++++++++--- 2 files changed, 31 insertions(+), 57 deletions(-) diff --git a/conda_build/main_build.py b/conda_build/main_build.py index 3d976b1a8c..a0a7b38f02 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -29,6 +29,7 @@ from conda_build.completers import (all_versions, conda_version, RecipeCompleter, PythonVersionCompleter, RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) from conda_build.utils import find_recipe +from conda_build.main_render import render_recipe, get_package_build_string, set_language_env_vars on_win = (sys.platform == 'win32') @@ -66,12 +67,6 @@ def main(): dest='include_recipe', default=True, ) - p.add_argument( - "--output", - action="store_true", - help="Output the conda package filename which would have been " - "created and exit.", - ) p.add_argument( '-s', "--source", action="store_true", @@ -221,41 +216,7 @@ def execute(args, parser): "imported that is hard-linked by files in the trash. " "Will try again on next run.") - for lang in ['python', 'numpy', 'perl', 'R', 'lua']: - versions = getattr(args, lang) - if not versions: - continue - if versions == ['all']: - if all_versions[lang]: - versions = all_versions[lang] - else: - parser.error("'all' is not supported for --%s" % lang) - if len(versions) > 1: - for ver in versions[:]: - setattr(args, lang, [str(ver)]) - execute(args, parser) - # This is necessary to make all combinations build. - setattr(args, lang, versions) - return - else: - version = versions[0] - if lang in ('python', 'numpy'): - version = int(version.replace('.', '')) - setattr(config, conda_version[lang], version) - if not len(str(version)) in (2, 3) and lang in ['python', 'numpy']: - if all_versions[lang]: - raise RuntimeError("%s must be major.minor, like %s, not %s" % - (conda_version[lang], all_versions[lang][-1]/10, version)) - else: - raise RuntimeError("%s must be major.minor, not %s" % - (conda_version[lang], version)) - - # Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc. - # Auto-set those env variables - for var in conda_version.values(): - if hasattr(config, var): - # Set the env variable. - os_environ[var] = str(getattr(config, var)) + set_language_env_vars(args) if args.skip_existing: for d in config.bldpkgs_dirs: @@ -315,18 +276,7 @@ def execute(args, parser): "configuration." % m.dist()) continue if args.output: - try: - m.parse_again(permit_undefined_jinja=False) - except SystemExit: - # Something went wrong; possibly due to undefined GIT_ jinja variables. - # Maybe we need to actually download the source in order to resolve the build_id. - source.provide(m.path, m.get_section('source')) - - # Parse our metadata again because we did not initialize the source - # information before. - m.parse_again(permit_undefined_jinja=False) - - print(build.bldpkg_path(m)) + print(get_package_build_string(m)) continue elif args.test: build.test(m, move_broken=False) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 948b19443c..8ded83f376 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -51,6 +51,12 @@ def get_render_parser(): action="store_true", help="Obtain the source and fill in related template variables.", ) + p.add_argument( + "--output", + action="store_true", + help="Output the conda package filename which would have been " + "created", + ) p.add_argument( '--python', action="append", @@ -197,6 +203,21 @@ def render_recipe(recipe_path, download_source=False): return m +def get_package_build_string(metadata): + import conda_build.build as build + try: + metadata.parse_again(permit_undefined_jinja=False) + except SystemExit: + # Something went wrong; possibly due to undefined GIT_ jinja variables. + # Maybe we need to actually download the source in order to resolve the build_id. + source.provide(metadata.path, metadata.get_section('source')) + + # Parse our metadata again because we did not initialize the source + # information before. + metadata.parse_again(permit_undefined_jinja=False) + return build.bldpkg_path(metadata) + + # Next bit of stuff is to support YAML output in the order we expect. # http://stackoverflow.com/a/17310199/1170370 class MetaYaml(dict): @@ -246,11 +267,14 @@ def main(): set_language_env_vars(args) metadata = render_recipe(find_recipe(args.recipe), download_source=args.source) - if args.yaml: - print(yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper, - default_flow_style=False, indent=4)) + if args.output: + print(get_package_build_string(metadata)) else: - pprint.pprint(metadata.meta) + if args.yaml: + print(yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper, + default_flow_style=False, indent=4)) + else: + pprint.pprint(metadata.meta) if __name__ == '__main__': From 72336733da08a8ea4931afaae349baa5174059a7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 29 Apr 2016 22:41:25 -0500 Subject: [PATCH 05/11] fix pyflakes bugs, remove duplicated recursive glob function --- conda_build/main_build.py | 17 ++++++----------- conda_build/main_render.py | 26 ++++++-------------------- conda_build/utils.py | 14 ++------------ 3 files changed, 14 insertions(+), 43 deletions(-) diff --git a/conda_build/main_build.py b/conda_build/main_build.py index a0a7b38f02..ee6c9c4029 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -7,29 +7,24 @@ from __future__ import absolute_import, division, print_function import argparse +import os import sys from collections import deque from glob import glob from locale import getpreferredencoding -from os import listdir -from os import environ as os_environ -from os.path import exists, isdir, isfile, join import warnings import conda.config as config from conda.compat import PY3 -from conda.cli.common import add_parser_channels, Completer -from conda.cli.conda_argparse import ArgumentParser +from conda.cli.common import add_parser_channels from conda.install import delete_trash from conda.resolve import NoPackagesFound, Unsatisfiable -from conda_build import __version__, exceptions +from conda_build import exceptions from conda_build.index import update_index from conda_build.main_render import get_render_parser -from conda_build.completers import (all_versions, conda_version, RecipeCompleter, PythonVersionCompleter, - RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) from conda_build.utils import find_recipe -from conda_build.main_render import render_recipe, get_package_build_string, set_language_env_vars +from conda_build.main_render import get_package_build_string, set_language_env_vars, RecipeCompleter on_win = (sys.platform == 'win32') @@ -216,7 +211,7 @@ def execute(args, parser): "imported that is hard-linked by files in the trash. " "Will try again on next run.") - set_language_env_vars(args) + set_language_env_vars(args, parser, execute=execute) if args.skip_existing: for d in config.bldpkgs_dirs: @@ -315,7 +310,7 @@ def execute(args, parser): if pkg in skip_names: continue recipe_glob = glob(pkg + '-[v0-9][0-9.]*') - if exists(pkg): + if os.path.exists(pkg): recipe_glob.append(pkg) if recipe_glob: try_again = True diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 8ded83f376..d29a53f4d0 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -7,11 +7,9 @@ from __future__ import absolute_import, division, print_function import sys -from collections import deque -from glob import glob from locale import getpreferredencoding import os -from os.path import exists, isdir, isfile, join, abspath +from os.path import isdir, isfile, abspath import yaml @@ -104,7 +102,7 @@ def get_render_parser(): return p -def set_language_env_vars(args): +def set_language_env_vars(args, parser, execute=None): """Given args passed into conda command, set language env vars""" for lang in all_versions: versions = getattr(args, lang) @@ -118,7 +116,8 @@ def set_language_env_vars(args): if len(versions) > 1: for ver in versions[:]: setattr(args, lang, [str(ver)]) - execute(args, parser) + if execute: + execute(args, parser) # This is necessary to make all combinations build. setattr(args, lang, versions) return @@ -152,7 +151,6 @@ def render_recipe(recipe_path, download_source=False): with Locked(config.croot): arg = recipe_path - try_again = False # Don't use byte literals for paths in Python 2 if not PY3: arg = arg.decode(getpreferredencoding() or 'utf-8') @@ -183,19 +181,7 @@ def render_recipe(recipe_path, download_source=False): source.provide(m.path, m.get_section('source'), patch=False) print('Source tree in:', source.get_dir()) - try: - m.parse_again(permit_undefined_jinja=False) - except SystemExit: - # Something went wrong; possibly due to undefined GIT_ jinja variables. - # Maybe we need to actually download the source in order to resolve the build_id. - source.provide(m.path, m.get_section('source')) - - # Parse our metadata again because we did not initialize the source - # information before. - m.parse_again(permit_undefined_jinja=False) - - print(build.bldpkg_path(m)) - raise + m.parse_again(permit_undefined_jinja=False) if need_cleanup: shutil.rmtree(recipe_dir) @@ -264,7 +250,7 @@ def main(): ) args = p.parse_args() - set_language_env_vars(args) + set_language_env_vars(args, p) metadata = render_recipe(find_recipe(args.recipe), download_source=args.source) if args.output: diff --git a/conda_build/utils.py b/conda_build/utils.py index aacb0d25ab..96e152dae0 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -1,14 +1,13 @@ from __future__ import absolute_import, division, print_function -import os import fnmatch +import os import sys import shutil import tarfile import zipfile import subprocess import operator -import fnmatch from os.path import dirname, getmtime, getsize, isdir, join from collections import defaultdict @@ -21,20 +20,11 @@ from conda.install import rm_rf rm_rf - -def _recursive_glob(treeroot, pattern): - results = [] - for base, dirs, files in os.walk(treeroot): - goodfiles = fnmatch.filter(files, pattern) - results.extend(os.path.join(base, f) for f in goodfiles) - return results - - def find_recipe(path): """recurse through a folder, locating meta.yaml. Raises error if more than one is found. Returns folder containing meta.yaml, to be built.""" - results = _recursive_glob(path, "meta.yaml") + results = rec_glob(path, ["meta.yaml"]) if len(results) > 1: raise IOError("More than one meta.yaml files found in %s" % path) elif not results: From ca69c7d53f7516cfb230e1cc0a16eca1daf8ddd1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 30 Apr 2016 11:34:40 -0500 Subject: [PATCH 06/11] squelch pyflakes missing unicode define --- conda_build/main_render.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index d29a53f4d0..556a9f619d 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -228,6 +228,7 @@ def increase_indent(self, flow=False, indentless=False): yaml.add_representer(MetaYaml, represent_omap) if PY3: yaml.add_representer(str, unicode_representer) + unicode = None # silence pyflakes about unicode not existing in py3 else: yaml.add_representer(unicode, unicode_representer) From 0448b2e09c3d4b7e2daa438615bd224ab4b2dce7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 2 May 2016 13:59:36 -0500 Subject: [PATCH 07/11] output yaml by default. add -f (file) option. Download by default. --- conda_build/main_render.py | 62 +++++++++++++++++++++----------------- conda_build/metadata.py | 4 +-- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 556a9f619d..bd9fa4380e 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -45,9 +45,10 @@ def get_render_parser(): version = 'conda-build %s' % __version__, ) p.add_argument( - '-s', "--source", + '-n', "--no_source", action="store_true", - help="Obtain the source and fill in related template variables.", + help="When templating can't be completed, do not obtain the \ +source to try fill in related template variables.", ) p.add_argument( "--output", @@ -142,7 +143,24 @@ def set_language_env_vars(args, parser, execute=None): os.environ[var] = str(getattr(config, var)) -def render_recipe(recipe_path, download_source=False): +def parse_or_try_download(metadata, no_download_source): + try: + metadata.parse_again(permit_undefined_jinja=False) + except SystemExit: + if not no_download_source: + # Something went wrong; possibly due to undefined GIT_ jinja variables. + # Maybe we need to actually download the source in order to resolve the build_id. + source.provide(metadata.path, metadata.get_section('source')) + + # Parse our metadata again because we did not initialize the source + # information before. + metadata.parse_again(permit_undefined_jinja=False) + else: + raise + return metadata + + +def render_recipe(recipe_path, no_download_source=True): import shutil import tarfile import tempfile @@ -172,16 +190,12 @@ def render_recipe(recipe_path, download_source=False): sys.exit("Error: no such directory: %s" % recipe_dir) try: - m = MetaData(recipe_dir) + m = MetaData(recipe_dir, permit_undefined_jinja=False) except exceptions.YamlParsingError as e: sys.stderr.write(e.error_msg()) sys.exit(1) - if download_source: - source.provide(m.path, m.get_section('source'), patch=False) - print('Source tree in:', source.get_dir()) - - m.parse_again(permit_undefined_jinja=False) + m = parse_or_try_download(m, no_download_source=no_download_source) if need_cleanup: shutil.rmtree(recipe_dir) @@ -191,16 +205,7 @@ def render_recipe(recipe_path, download_source=False): def get_package_build_string(metadata): import conda_build.build as build - try: - metadata.parse_again(permit_undefined_jinja=False) - except SystemExit: - # Something went wrong; possibly due to undefined GIT_ jinja variables. - # Maybe we need to actually download the source in order to resolve the build_id. - source.provide(metadata.path, metadata.get_section('source')) - - # Parse our metadata again because we did not initialize the source - # information before. - metadata.parse_again(permit_undefined_jinja=False) + metadata = parse_or_try_download(metadata) return build.bldpkg_path(metadata) @@ -237,9 +242,10 @@ def main(): import pprint p = get_render_parser() p.add_argument( - '-y', '--yaml', - action="store_true", - help="print YAML, as opposed to printing the metadata as a dictionary" + '-f', '--file', + action="store", + help="write YAML to file, given as argument here.\ + Overwrites existing files." ) # we do this one separately because we only allow one entry to conda render p.add_argument( @@ -253,15 +259,17 @@ def main(): args = p.parse_args() set_language_env_vars(args, p) - metadata = render_recipe(find_recipe(args.recipe), download_source=args.source) + metadata = render_recipe(find_recipe(args.recipe), no_download_source=args.no_source) if args.output: print(get_package_build_string(metadata)) else: - if args.yaml: - print(yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper, - default_flow_style=False, indent=4)) + output = yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper, + default_flow_style=False, indent=4) + if args.file: + with open(args.file, "w") as f: + f.write(output) else: - pprint.pprint(metadata.meta) + print(output) if __name__ == '__main__': diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 1004c71709..9f5980ff66 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -312,7 +312,7 @@ def handle_config_version(ms, ver): class MetaData(object): - def __init__(self, path): + def __init__(self, path, permit_undefined_jinja=True): assert isdir(path) self.path = path self.meta_path = join(path, 'meta.yaml') @@ -331,7 +331,7 @@ def __init__(self, path): # (e.g. GIT_FULL_HASH, etc. are undefined) # Therefore, undefined jinja variables are permitted here # In the second pass, we'll be more strict. See build.build() - self.parse_again(permit_undefined_jinja=True) + self.parse_again(permit_undefined_jinja=permit_undefined_jinja) def parse_again(self, permit_undefined_jinja=False): """Redo parsing for key-value pairs that are not initialized in the From ddb852a1320f31a07a02574a5570ac2d0ae1c362 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 2 May 2016 14:08:20 -0500 Subject: [PATCH 08/11] pyflakes, no pprint used --- conda_build/main_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index bd9fa4380e..1d7048a208 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -239,7 +239,6 @@ def increase_indent(self, flow=False, indentless=False): def main(): - import pprint p = get_render_parser() p.add_argument( '-f', '--file', From 1c16171f7cf48d15b49abf8a053efa0896dbce28 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 3 May 2016 15:49:46 -0500 Subject: [PATCH 09/11] look for conda.yaml files and meta.yaml files --- conda_build/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index 96e152dae0..2ad4a7cf52 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -24,7 +24,7 @@ def find_recipe(path): """recurse through a folder, locating meta.yaml. Raises error if more than one is found. Returns folder containing meta.yaml, to be built.""" - results = rec_glob(path, ["meta.yaml"]) + results = rec_glob(path, ["meta.yaml", "conda.yaml"]) if len(results) > 1: raise IOError("More than one meta.yaml files found in %s" % path) elif not results: From 9f5185a6bb02b1b06ea5f27cda7b322abe69eedf Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 3 May 2016 15:57:23 -0500 Subject: [PATCH 10/11] revert strict jinja in MetaData constructor; omit non-existent fields from yaml output --- conda_build/main_render.py | 4 ++-- conda_build/metadata.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 1d7048a208..0ad55af97f 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -190,7 +190,7 @@ def render_recipe(recipe_path, no_download_source=True): sys.exit("Error: no such directory: %s" % recipe_dir) try: - m = MetaData(recipe_dir, permit_undefined_jinja=False) + m = MetaData(recipe_dir) except exceptions.YamlParsingError as e: sys.stderr.write(e.error_msg()) sys.exit(1) @@ -214,7 +214,7 @@ def get_package_build_string(metadata): class MetaYaml(dict): fields = ["package", "source", "build", "requirements", "test", "extra"] def to_omap(self): - return [(field, self[field]) for field in MetaYaml.fields] + return [(field, self[field]) for field in MetaYaml.fields if field in self] def represent_omap(dumper, data): diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 9f5980ff66..1004c71709 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -312,7 +312,7 @@ def handle_config_version(ms, ver): class MetaData(object): - def __init__(self, path, permit_undefined_jinja=True): + def __init__(self, path): assert isdir(path) self.path = path self.meta_path = join(path, 'meta.yaml') @@ -331,7 +331,7 @@ def __init__(self, path, permit_undefined_jinja=True): # (e.g. GIT_FULL_HASH, etc. are undefined) # Therefore, undefined jinja variables are permitted here # In the second pass, we'll be more strict. See build.build() - self.parse_again(permit_undefined_jinja=permit_undefined_jinja) + self.parse_again(permit_undefined_jinja=True) def parse_again(self, permit_undefined_jinja=False): """Redo parsing for key-value pairs that are not initialized in the From 1f3ef2bfed2d3368f0e564c4872d5217ba5e24cd Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 3 May 2016 18:15:32 -0500 Subject: [PATCH 11/11] add about section in rendering --- conda_build/main_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 0ad55af97f..7e265c960d 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -212,7 +212,7 @@ def get_package_build_string(metadata): # Next bit of stuff is to support YAML output in the order we expect. # http://stackoverflow.com/a/17310199/1170370 class MetaYaml(dict): - fields = ["package", "source", "build", "requirements", "test", "extra"] + fields = ["package", "source", "build", "requirements", "test", "about", "extra"] def to_omap(self): return [(field, self[field]) for field in MetaYaml.fields if field in self]