Permalink
Browse files

Implement ``conda_lint``.

With test cases for every failure mode for linting.
  • Loading branch information...
jmchilton committed Sep 21, 2016
1 parent ca88b0c commit 6a6f16491acfde04a474dd3435cd7042f113e5ee
Showing with 1,166 additions and 20 deletions.
  1. +25 −0 planemo/commands/cmd_conda_lint.py
  2. +210 −0 planemo/conda_lint.py
  3. +24 −0 planemo/conda_verify/LICENSE.txt
  4. +1 −0 planemo/conda_verify/__init__.py
  5. +49 −0 planemo/conda_verify/const.py
  6. +321 −0 planemo/conda_verify/recipe.py
  7. +68 −0 planemo/conda_verify/utils.py
  8. +26 −0 planemo/io.py
  9. +30 −0 planemo/lint_util.py
  10. +17 −1 planemo/options.py
  11. +4 −8 planemo/shed/__init__.py
  12. +3 −10 planemo/shed_lint.py
  13. +10 −0 scripts/update_extern.sh
  14. +1 −1 setup.cfg
  15. +1 −0 setup.py
  16. +6 −0 tests/data/recipes/fail_samtools_about_urls/build.sh
  17. +23 −0 tests/data/recipes/fail_samtools_about_urls/meta.yaml
  18. +6 −0 tests/data/recipes/fail_samtools_build_number/build.sh
  19. +23 −0 tests/data/recipes/fail_samtools_build_number/meta.yaml
  20. 0 tests/data/recipes/fail_samtools_directory_contents/bad.so
  21. +6 −0 tests/data/recipes/fail_samtools_directory_contents/build.sh
  22. +23 −0 tests/data/recipes/fail_samtools_directory_contents/meta.yaml
  23. +6 −0 tests/data/recipes/fail_samtools_fields/build.sh
  24. +24 −0 tests/data/recipes/fail_samtools_fields/meta.yaml
  25. +6 −0 tests/data/recipes/fail_samtools_fields_2/build.sh
  26. +24 −0 tests/data/recipes/fail_samtools_fields_2/meta.yaml
  27. +6 −0 tests/data/recipes/fail_samtools_license/build.sh
  28. +23 −0 tests/data/recipes/fail_samtools_license/meta.yaml
  29. +6 −0 tests/data/recipes/fail_samtools_name/build.sh
  30. +23 −0 tests/data/recipes/fail_samtools_name/meta.yaml
  31. +6 −0 tests/data/recipes/fail_samtools_requirements/build.sh
  32. +24 −0 tests/data/recipes/fail_samtools_requirements/meta.yaml
  33. +6 −0 tests/data/recipes/fail_samtools_source/build.sh
  34. +23 −0 tests/data/recipes/fail_samtools_source/meta.yaml
  35. +6 −0 tests/data/recipes/fail_samtools_summary/build.sh
  36. +22 −0 tests/data/recipes/fail_samtools_summary/meta.yaml
  37. +6 −0 tests/data/recipes/fail_samtools_version/build.sh
  38. +23 −0 tests/data/recipes/fail_samtools_version/meta.yaml
  39. +6 −0 tests/data/recipes/ok_samtools/build.sh
  40. +23 −0 tests/data/recipes/ok_samtools/meta.yaml
  41. +25 −0 tests/test_conda_lint.py
  42. +1 −0 tests/test_utils.py
@@ -0,0 +1,25 @@
"""Module describing the planemo ``conda_lint`` command."""
import click

from planemo import options
from planemo import conda_lint
from planemo.cli import command_function


@click.command('conda_lint')
@options.report_level_option()
@options.fail_level_option()
@options.recursive_option(
"Recursively perform command for nested conda directories.",
)
@options.recipe_arg(multiple=True)
@command_function
def cli(ctx, paths, **kwds):
"""Check conda recipe for common issues.
Built in large part on the work from the BSD licensed anaconda-verify
project. For more information on anacoda-verify see:
https://github.com/ContinuumIO/anaconda-verify.
"""
exit_code = conda_lint.lint_recipes_on_paths(ctx, paths, **kwds)
ctx.exit(exit_code)
@@ -0,0 +1,210 @@
"""Logic for linting conda recipes."""

import os

from functools import wraps

from planemo.conda_verify.recipe import (
check_build_number,
check_dir_content,
check_license_family,
check_name,
check_requirements,
check_source,
check_url,
check_version,
FIELDS,
get_field,
RecipeError,
render_jinja2,
yamlize,
)
from planemo.exit_codes import (
EXIT_CODE_GENERIC_FAILURE,
EXIT_CODE_OK,
)
from planemo.io import (
coalesce_return_codes,
find_matching_directories,
info,
)
from planemo.lint_util import handle_lint_complete, setup_lint


def lint_recipes_on_paths(ctx, paths, **kwds):
"""Apply conda linting procedure to recipes on supplied paths."""
assert_tools = kwds.get("assert_recipes", True)
recursive = kwds.get("recursive", False)
exit_codes = []
for recipe_dir in yield_recipes_on_paths(ctx, paths, recursive):
if lint_conda_recipe(ctx, recipe_dir, **kwds) != 0:
exit_codes.append(EXIT_CODE_GENERIC_FAILURE)
else:
exit_codes.append(EXIT_CODE_OK)
return coalesce_return_codes(exit_codes, assert_at_least_one=assert_tools)


def lint_conda_recipe(ctx, recipe_dir, **kwds):
info("Linting conda recipe %s" % recipe_dir)
lint_args, lint_ctx = setup_lint(ctx, **kwds)

def apply(f):
lint_ctx.lint(f.__name__, f, recipe_dir)

apply(lint_name)
apply(lint_version)
apply(lint_summary)
apply(lint_build_number)
apply(lint_directory_content)
apply(lint_license_family)
apply(lint_about_urls)
apply(lint_source)
apply(lint_fields)
apply(lint_requirements)

return handle_lint_complete(lint_ctx, lint_args)


def wraps_recipe_error(is_error=True):

def outer_wrapper(f):

@wraps(f)
def wrapper(recipe_dir, lint_ctx):
try:
f(recipe_dir, lint_ctx)
except RecipeError as e:
if is_error:
lint_ctx.error(str(e))
else:
lint_ctx.warn(str(e))
except TypeError as e: # Errors in recipe checking code from YAML.
lint_ctx.error(str(e))

return wrapper

return outer_wrapper


def lints_metadata(f):

@wraps(f)
def wrapper(recipe_dir, lint_ctx):
meta_path = os.path.join(recipe_dir, 'meta.yaml')
with open(meta_path, 'rb') as fi:
data = fi.read()
if b'{{' in data:
data = render_jinja2(recipe_dir)
meta = dict(yamlize(data))
f(meta, lint_ctx)

return wrapper


@wraps_recipe_error(is_error=False)
def lint_directory_content(recipe_dir, lint_ctx):
check_dir_content(recipe_dir)
lint_ctx.info("Directory content seems okay.")


@lints_metadata
@wraps_recipe_error(is_error=False)
def lint_license_family(meta, lint_ctx):
check_license_family(meta)
lint_ctx.info("License from vaild license family.")


@lints_metadata
def lint_summary(meta, lint_ctx):
summary = get_field(meta, 'about/summary')

if not summary:
lint_ctx.warn("No summary supplied in about metadata.")

if summary and len(summary) > 80:
msg = "summary exceeds 80 characters"
lint_ctx.warn(msg)


@lints_metadata
@wraps_recipe_error(is_error=False)
def lint_about_urls(meta, lint_ctx):
for field in ('about/home', 'about/dev_url', 'about/doc_url',
'about/license_url'):
url = get_field(meta, field)
if url:
check_url(url)
lint_ctx.info("About urls (if present) are valid")


@lints_metadata
@wraps_recipe_error(is_error=True)
def lint_source(meta, lint_ctx):
check_source(meta)
lint_ctx.info("Source (if present) is valid")


@lints_metadata
@wraps_recipe_error(is_error=True)
def lint_build_number(meta, lint_ctx):
build_number = get_field(meta, 'build/number', 0)
check_build_number(build_number)
lint_ctx.info("Valid build number [%s]" % build_number)


@lints_metadata
@wraps_recipe_error(is_error=True)
def lint_version(meta, lint_ctx):
version = get_field(meta, 'package/version')
check_version(version)
lint_ctx.info("Valid version number [%s]" % version)


@lints_metadata
@wraps_recipe_error(is_error=True)
def lint_name(meta, lint_ctx):
name = get_field(meta, 'package/name')
check_name(name)
lint_ctx.info("Valid recipe name [%s]" % name)


@lints_metadata
@wraps_recipe_error(is_error=False)
def lint_fields(meta, lint_ctx):
# Taken from validate_meta
for section in meta:
if section not in FIELDS:
raise RecipeError("Unknown section: %s" % section)
submeta = meta.get(section)
if submeta is None:
submeta = {}
for key in submeta:
# Next two lines added for planemo since we don't do the
# select lines thing.
if key == "skip":
continue

if key not in FIELDS[section]:
raise RecipeError("in section %r: unknown key %r" %
(section, key))


@lints_metadata
@wraps_recipe_error(is_error=False)
def lint_requirements(meta, lint_ctx):
check_requirements(meta)
lint_ctx.info("Reference recipe files appear valid")


def yield_recipes_on_paths(ctx, paths, recursive):
for path in paths:
recipe_dirs = find_matching_directories(
path, "meta.yaml", recursive=recursive
)
for recipe_dir in recipe_dirs:
yield recipe_dir


__all__ = [
"lint_recipes_on_paths",
]
@@ -0,0 +1,24 @@
Copyright (c) 2016, Continuum Analytics, Inc.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of Continuum Analytics, Inc. nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL CONTINUUM ANALYTICS BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1 @@
__version__ = '1.2.1'
@@ -0,0 +1,49 @@
LICENSE_FAMILIES = set("""
AGPL
GPL2
GPL3
LGPL
BSD
MIT
Apache
PSF
Public-Domain
Proprietary
Other
""".split())

FIELDS = {
'package': {'name', 'version'},
'source': {'fn', 'url', 'md5', 'sha1', 'sha256',
'git_url', 'git_tag', 'git_branch',
'patches', 'hg_url', 'hg_tag'},
'build': {'features', 'track_features',
'number', 'entry_points', 'osx_is_app', 'noarch',
'preserve_egg_dir', 'win_has_prefix', 'no_link',
'ignore_prefix_files', 'msvc_compiler',
'detect_binary_files_with_prefix',
'always_include_files'},
'requirements': {'build', 'run'},
'app': {'entry', 'icon', 'summary', 'type', 'cli_opts'},
'test': {'requires', 'commands', 'files', 'imports'},
'about': {'license', 'license_url', 'license_family', 'license_file',
'summary', 'description', 'home', 'doc_url', 'dev_url'},
}

MAGIC_HEADERS = {
'\xca\xfe\xba\xbe': 'MachO-universal',
'\xce\xfa\xed\xfe': 'MachO-i386',
'\xcf\xfa\xed\xfe': 'MachO-x86_64',
'\xfe\xed\xfa\xce': 'MachO-ppc',
'\xfe\xed\xfa\xcf': 'MachO-ppc64',
'MZ\x90\x00': 'DLL',
'\x7fELF': 'ELF',
}

DLL_TYPES = {
0x0: 'UNKNOWN', 0x1d3: 'AM33', 0x8664: 'AMD64', 0x1c0: 'ARM',
0xebc: 'EBC', 0x14c: 'I386', 0x200: 'IA64', 0x9041: 'M32R',
0x266: 'MIPS16', 0x366: 'MIPSFPU', 0x466: 'MIPSFPU16', 0x1f0: 'POWERPC',
0x1f1: 'POWERPCFP', 0x166: 'R4000', 0x1a2: 'SH3', 0x1a3: 'SH3DSP',
0x1a6: 'SH4', 0x1a8: 'SH5', 0x1c2: 'THUMB', 0x169: 'WCEMIPSV2',
}
Oops, something went wrong.

0 comments on commit 6a6f164

Please sign in to comment.