Skip to content
This repository has been archived by the owner on Jan 12, 2021. It is now read-only.

Commit

Permalink
Merge branch 'quickstart'
Browse files Browse the repository at this point in the history
  • Loading branch information
orsinium committed Apr 1, 2019
2 parents c5ea181 + 8572ba4 commit 863af7e
Show file tree
Hide file tree
Showing 26 changed files with 362 additions and 87 deletions.
3 changes: 3 additions & 0 deletions dephell/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ def main(argv: List[str]) -> int:
except Exception as e:
logger.exception('{}: {}'.format(type(e).__name__, e))
return ReturnCodes.UNKNOWN_EXCEPTION.value
except KeyboardInterrupt:
logger.exception('stopped by user')
return ReturnCodes.UNKNOWN_EXCEPTION.value
if not result:
return ReturnCodes.COMMAND_ERROR.value
return ReturnCodes.OK.value
Expand Down
4 changes: 3 additions & 1 deletion dephell/commands/deps_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def get_parser(cls):
def __call__(self):
loader = CONVERTERS[self.config['from']['format']]
dumper = CONVERTERS[self.config['to']['format']]
self.logger.info('converting...', extra={
self.logger.info('start converting...', extra={
'from-format': self.config['from']['format'],
'from-path': self.config['from']['path'],
'to-format': self.config['to']['format'],
Expand All @@ -48,6 +48,7 @@ def __call__(self):

# merge (without full graph building)
if not should_be_resolved:
self.logger.info('merging...')
resolved = resolver.resolve(level=1)
if not resolved:
conflict = analize_conflict(resolver=resolver)
Expand All @@ -58,6 +59,7 @@ def __call__(self):

# resolve (and merge)
if should_be_resolved:
self.logger.info('resolving...')
resolved = resolver.resolve()
if not resolved:
conflict = analize_conflict(resolver=resolver)
Expand Down
4 changes: 2 additions & 2 deletions dephell/commands/generate_editorconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
('*.{json,yml,yaml}', ('indent_style = space', 'indent_size = 2')),
('*.{html,j2}', ('indent_style = space', 'indent_size = 2')),

('Makefile', ('indent_style = tab')),
('*.go', ('indent_style = tab')),
('Makefile', ('indent_style = tab', )),
('*.go', ('indent_style = tab', )),
)


Expand Down
3 changes: 2 additions & 1 deletion dephell/config/builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def build_resolver(parser):
resolver_group = parser.add_argument_group('Resolver rules')
resolver_group.add_argument('--strategy', choices=STRATEGIES, help='Algorithm to select best release.')
resolver_group.add_argument('--prereleases', action='store_true', help='Allow prereleases')
resolver_group.add_argument('--mutations', type=int, help='Maximum mutations limit')


def build_api(parser):
Expand Down Expand Up @@ -62,7 +63,7 @@ def build_other(parser):
other_group = parser.add_argument_group('Other')

other_group.add_argument('--cache-path', help='path to dephell cache')
other_group.add_argument('--cache-ttl', help='Time to live for releases list cache')
other_group.add_argument('--cache-ttl', type=int, help='Time to live for releases list cache')

other_group.add_argument('--project', help='path to the current project')
other_group.add_argument('--bin', help='path to the dir for installing scripts')
Expand Down
1 change: 1 addition & 0 deletions dephell/config/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# resolver
prereleases=False,
strategy='max',
mutations=200,

# api
bitbucket='https://api.bitbucket.org/2.0',
Expand Down
12 changes: 11 additions & 1 deletion dephell/config/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,16 @@ def attach(self, data: dict, container: Optional[dict] = None) -> None:
for key, value in data.items():
if value is None:
continue

# convert `and` section from tomlkit types to python types
if isinstance(value, (list, tuple)):
new_value = []
for subvalue in value:
if isinstance(subvalue, dict):
subvalue = dict(subvalue)
new_value.append(subvalue)
value = new_value

if key not in container:
container[key] = value
elif isinstance(value, dict):
Expand Down Expand Up @@ -102,7 +112,7 @@ def attach_file(self, path: str, env: str, silent: bool = False) -> Optional[dic
def attach_cli(self, args, sep: str = '_') -> dict:
data = defaultdict(dict)
for name, value in args._get_kwargs():
if value is None:
if value is None or value is False:
continue
parsed = name.split(sep, maxsplit=1)
if len(parsed) == 1:
Expand Down
7 changes: 5 additions & 2 deletions dephell/config/scheme.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
'to': dict(required=False, **_TARGET),
'and': dict(
type='list',
schema=dict(required=True, **_TARGET),
schema=_TARGET,
required=False,
empty=True,
),
Expand All @@ -33,11 +33,14 @@
required=True,
),

'silent': dict(type='boolean', required=True),
'warehouse': dict(type='string', required=True),
'bitbucket': dict(type='string', required=True),

'strategy': dict(type='string', required=True, allowed=STRATEGIES),
'prereleases': dict(type='boolean', required=True),
'mutations': dict(type='integer', required=True),

'silent': dict(type='boolean', required=True),
'level': dict(type='string', required=True, allowed=LOG_LEVELS),
'format': dict(type='string', required=True, allowed=LOG_FORMATTERS),
'nocolors': dict(type='boolean', required=True),
Expand Down
30 changes: 19 additions & 11 deletions dephell/controllers/dependency.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# built-in
import re
from contextlib import suppress
from copy import copy
from logging import getLogger
from typing import List, Optional, Union

# external
Expand All @@ -16,6 +17,7 @@
from ..repositories import get_repo


logger = getLogger('dephell.controllers')
# regex for names generated by pipenv
rex_hash = re.compile(r'[a-f0-9]{7}')

Expand All @@ -25,7 +27,7 @@ class DependencyMaker:
extra_class = ExtraDependency

@classmethod
def from_requirement(cls, source, req, *, url=None, envs=None,
def from_requirement(cls, source, req, *, url=None, envs=None, marker: Union[Markers, str] = None,
editable=False) -> List[Union[Dependency, ExtraDependency]]:
if type(req) is str:
req = PackagingRequirement(req)
Expand All @@ -36,17 +38,21 @@ def from_requirement(cls, source, req, *, url=None, envs=None,
if isinstance(link, VCSLink) and link.rev:
constraint._specs[source.name] = GitSpecifier()

marker = None
if isinstance(marker, Markers):
marker = copy(marker)
else:
marker = Markers(marker)
if req.marker is not None:
# some libs uses `in` for python_version,
# but dephell_markers isn't ready for this
with suppress(ValueError):
marker = Markers(req.marker)
try:
marker &= Markers(req.marker)
except ValueError:
logger.warning('cannot parse marker', extra=dict(marker=req.marker))

if envs is None:
envs = {'main'}
if marker is not None:
envs.update(marker.extract('extra'))
envs.update(marker.extract('extra'))

base_dep = cls.dep_class(
raw_name=req.name,
Expand All @@ -66,7 +72,8 @@ def from_requirement(cls, source, req, *, url=None, envs=None,
@classmethod
def from_params(cls, *, raw_name: str, constraint,
url: Optional[str] = None, source: Optional['Dependency'] = None,
repo=None, marker=None, extras: Optional[List[str]] = None, envs=None,
repo=None, marker: Union[Markers, str] = None,
extras: Optional[List[str]] = None, envs=None,
**kwargs) -> List[Union[Dependency, ExtraDependency]]:

# make link
Expand All @@ -85,14 +92,15 @@ def from_params(cls, *, raw_name: str, constraint,
repo = get_repo(link)

# make marker
if marker is not None:
if isinstance(marker, Markers):
marker = copy(marker)
else:
marker = Markers(marker)

# make envs
if envs is None:
envs = {'main'}
if marker is not None:
envs.update(marker.extract('extra'))
envs.update(marker.extract('extra'))

base_dep = cls.dep_class(
link=link,
Expand Down
8 changes: 7 additions & 1 deletion dephell/controllers/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def add(self, dep, *, level: Optional[int] = None) -> None:
for parent in layer:
if parent.name in parents_names:
return self.add(dep, level=layer.level + 1)
raise KeyError('Can\'t find any parent for dependency')
raise KeyError('cannot find any parent for dependency: ' + str(dep.name))

def get_leafs(self, level: Optional[int] = None) -> tuple:
"""Get deps that isn't applied yet
Expand Down Expand Up @@ -132,6 +132,8 @@ def get_layer(self, dep_or_level) -> Layer:
raise KeyError('cannot find dep')

def get(self, name: str):
if name in self._deps:
return self._deps[name]
for layer in reversed(self._layers):
dep = layer.get(name)
if dep is not None:
Expand Down Expand Up @@ -162,13 +164,17 @@ def get_parents(self, *deps, avoid: Optional[list] = None) -> dict:
for dep in deps:
for layer in self._layers:
for parent in layer:
was_locked = parent.locked
for children in parent.dependencies:
if children.name != dep.name:
continue
if children.name in avoid:
continue
parents[parent.name] = parent
break
# if dependency hasn't been locked then unlock it after our accidental lock
if parent.locked and not was_locked:
parent.unlock()
if parents:
parents.update(self.get_parents(
*parents.values(),
Expand Down
104 changes: 81 additions & 23 deletions dephell/controllers/mutator.py
Original file line number Diff line number Diff line change
@@ -1,54 +1,112 @@
# built-in
from typing import Optional
from logging import getLogger
from typing import Optional, Tuple, Iterator, Iterable, Set, Sequence

import attr

# app
from ..config import config
from ..utils import lazy_product
from ..models import Dependency, Group, RootDependency
from .dependency import DependencyMaker
from .graph import Graph


logger = getLogger('dephell.controllers')


@attr.s()
class Mutator:
def __init__(self):
self._snapshots = set()
limit = attr.ib(type=int, factory=lambda: config['mutations'])
mutations = attr.ib(type=int, default=0, init=False)
_snapshots = attr.ib(type=Set[Tuple[str, ...]], factory=set, repr=False, init=False)

def mutate(self, graph) -> Optional[tuple]:
def mutate(self, graph: Graph) -> Optional[Tuple[Group, ...]]:
"""Get graph with conflict and mutate one dependency.
Mutation changes group for one from dependencies
from parents of conflicting dependency.
"""
parents = graph.get_parents(graph.conflict)
for groups in self.get_mutations(parents.values()):
if self.check(groups):
self.remember(groups)
return groups
if self.mutations >= self.limit:
logger.warning('mutations limit reached', extra=dict(limit=self.limit))
return None

parents = tuple(graph.get_parents(graph.conflict).values())
checker = (
self._check_in_conflict,
self._check_in_subgraph,
self._check_soft,
)
for check in checker:
for groups in self.get_mutations(deps=parents):
if check(groups=groups, deps=parents, conflict=graph.conflict):
self.remember(groups)
self.mutations += 1
return groups
return None # mypy wants it

def get_mutations(self, deps):
def get_mutations(self, deps: Iterable[Dependency]) -> Iterator[Tuple[Group, ...]]:
all_groups = []
for dep in deps:
all_groups.append(dep.groups)
for groups in lazy_product(*all_groups):
yield groups

@staticmethod
def _make_snapshot(groups) -> tuple:
snapshot = sorted(group.name + str(group.number) for group in groups)
def _make_snapshot(groups: Iterable[Group]) -> Tuple[str, ...]:
snapshot = sorted(group.name + '|' + str(group.number) for group in groups)
return tuple(snapshot)

def check(self, groups):
def _check_soft(self, groups: Sequence[Group], deps: Sequence[Dependency], conflict: Dependency) -> bool:
for group in groups:
if group.empty:
return False
return self._make_snapshot(groups) not in self._snapshots

def remember(self, groups):
self._snapshots.add(self._make_snapshot(groups))
def _check_in_subgraph(self, groups: Sequence[Group], deps: Sequence[Dependency],
conflict: Dependency) -> bool:
if not self._check_soft(groups=groups, deps=deps, conflict=conflict):
return False
# any new group has to change state of the subgraph
state = {dep.name: dict(dep.constraint.specs) for dep in deps if not isinstance(dep, RootDependency)}
state[conflict.name] = dict(conflict.constraint.specs)
print(state)
for group, dep in zip(groups, deps):
for subdep in group.dependencies:
if isinstance(subdep, Dependency):
if subdep.name not in state:
continue
if dep.name not in state[subdep.name]:
return True
if state[subdep.name][dep.name] != str(subdep.constraint):
return True
continue

@property
def mutations(self):
return len(self._snapshots)
for subdep in DependencyMaker.from_requirement(dep, subdep):
if subdep.name not in state:
continue
if dep.name not in state[subdep.name]:
return True
if state[subdep.name][dep.name] != str(subdep.constraint):
return True
return False

def __repr__(self):
return '{name}(mutations={mutations})'.format(
name=self.__class__.__name__,
mutations=self.mutations,
)
def _check_in_conflict(self, groups: Sequence[Group], deps: Sequence[Dependency],
conflict: Dependency) -> bool:
if not self._check_soft(groups=groups, deps=deps, conflict=conflict):
return False
state = {dep.name: dict(dep.constraint.specs) for dep in deps if not isinstance(dep, RootDependency)}
state[conflict.name] = dict(conflict.constraint.specs)
for group, dep in zip(groups, deps):
for subdep in group.dependencies:
if isinstance(subdep, Dependency):
if subdep.name == conflict.name:
return True
continue
for subdep in DependencyMaker.from_requirement(dep, subdep):
if subdep.name == conflict.name:
return True
return False

def remember(self, groups: Iterable[Group]) -> None:
self._snapshots.add(self._make_snapshot(groups))

0 comments on commit 863af7e

Please sign in to comment.