Merge pull request #401 from dephell/project-upload

dephell project upload

dephell/commands/discover.py

@@ -59,6 +59,7 @@
     'project bump',
     'project register',
     'project test',
+    'project upload',
     'project validate',
 
     'self auth',

dephell/commands/project_upload.py

@@ -0,0 +1,169 @@
+# built-in
+from argparse import ArgumentParser
+from pathlib import Path
+from typing import List, Optional
+
+# app
+from ..config import builders
+from ..controllers import Uploader
+from ..converters import CONVERTERS
+from ..models import Auth, Requirement
+from ..repositories import WarehouseAPIRepo
+from .base import BaseCommand
+
+
+class ProjectUploadCommand(BaseCommand):
+    """Upload project dist archives on pypi.org (or somewhere else).
+    """
+    @staticmethod
+    def build_parser(parser) -> ArgumentParser:
+        builders.build_config(parser)
+        builders.build_from(parser)
+
+        group = parser.add_argument_group('Release upload')
+        group.add_argument('--upload-sign', help='sign packages with GPG')
+        group.add_argument('--upload-identity', help='use name as the key to sign with')
+        group.add_argument('--upload-url', help='use name as the key to sign with')
+
+        builders.build_output(parser)
+        builders.build_other(parser)
+        return parser
+
+    def __call__(self) -> bool:
+        if 'from' not in self.config:
+            self.logger.error('`--from` is required for this command')
+            return False
+
+        uploader = Uploader(url=self.config['upload']['url'])
+        # auth
+        for cred in self.config['auth']:
+            if cred['hostname'] == uploader.hostname:
+                uploader.auth = Auth(**cred)
+        if uploader.auth is None:
+            self.logger.error('no credentials found', extra=dict(hostname=uploader.hostname))
+            return False
+
+        # metainfo
+        loader = CONVERTERS[self.config['from']['format']]
+        loader = loader.copy(project_path=Path(self.config['project']))
+        resolver = loader.load_resolver(path=self.config['from']['path'])
+        root = resolver.graph.metainfo
+        reqs = Requirement.from_graph(resolver.graph, lock=False)
+        self.logger.info('uploading release', extra=dict(
+            release_name=root.raw_name,
+            release_version=root.version,
+            upload_url=uploader.url,
+        ))
+
+        # get release info to check uploaded files
+        release = None
+        if uploader.hostname in {'pypi.org', 'test.pypi.org'}:
+            repo = WarehouseAPIRepo(name='pypi', url='https://{}/'.format(uploader.hostname))
+            releases = repo.get_releases(dep=root)
+            if not releases:
+                self.logger.debug('cannot find releases', extra=dict(
+                    release_name=root.name,
+                ))
+            releases = [r for r in releases if str(r.version) == root.version]
+            if len(releases) == 1:
+                release = releases[0]
+            else:
+                self.logger.debug('cannot find release', extra=dict(
+                    release_name=root.name,
+                    version=root.version,
+                    count=len(releases),
+                ))
+
+        # files to upload
+        paths = self._get_paths(loader=loader, root=root)
+        if not paths:
+            self.logger.error('no release files found')
+            return False
+
+        # do upload
+        uploaded = False
+        for path in paths:
+            url = self._uploaded(release=release, path=path)
+            if url:
+                self.logger.info('dist already uploaded', extra=dict(path=str(path), url=url))
+                continue
+            uploaded = True
+            self.logger.info('uploading dist...', extra=dict(path=str(path)))
+            if self.config['upload']['sign']:
+                uploader.sign(
+                    path=path,
+                    identity=self.config['upload'].get('identity'),
+                )
+            uploader.upload(path=path, root=root, reqs=reqs)
+
+        if not uploaded:
+            self.logger.warning('all dists already uploaded, nothing to do.')
+            return True
+
+        # show release url
+        if uploader.hostname in {'pypi.org', 'test.pypi.org'}:
+            url = 'https://{h}/project/{n}/{v}/'.format(
+                h=uploader.hostname,
+                n=root.name,
+                v=root.version,
+            )
+            self.logger.info('release uploaded', extra=dict(url=url))
+        else:
+            self.logger.info('release uploaded')
+        return True
+
+    def _get_paths(self, loader, root) -> List[Path]:
+        if self.config['from']['path'].endswith(('.tar.gz', '.whl')):
+            return [Path(self.config['from']['path'])]
+
+        # check dist dir if from-format is sdist or wheel
+        fmt = self.config['from']['format']
+        if fmt in {'sdist', 'wheel'}:
+            path = Path(self.config['from']['path'])
+            if path.is_dir():
+                if fmt == 'sdist':
+                    dists = list(path.glob('*.tar.gz'))
+                else:
+                    dists = list(path.glob('*.whl'))
+                if len(dists) == 1:
+                    return dists
+                # TODO: can we infer the name without knowing metadata?
+                raise FileNotFoundError('please, specify full path to the file')
+            raise IOError('invalid file extension: {}'.format(path.suffix))
+
+        path = Path(self.config['project']) / 'dist'
+        if not path.is_dir():
+            raise FileNotFoundError('cannot find ./dist/ dir')
+
+        # look for one-release archives in the dir
+        sdists = list(path.glob('*.tar.gz'))
+        wheels = list(path.glob('*.whl'))
+        if len(sdists) == 1 and len(wheels) == 1:
+            return sdists + wheels
+
+        result = []
+
+        # find sdist
+        file_name = '{name}-{version}.tar.gz'.format(
+            name=root.raw_name,
+            version=root.pep_version,
+        )
+        if (path / file_name).exists():
+            result.append(path / file_name)
+
+        # find wheel
+        glob = '{name}-{version}-py*-*-*.whl'.format(
+            name=root.raw_name.replace('-', '_'),
+            version=root.pep_version,
+        )
+        result.extend(list(path.glob(glob)))
+
+        return result
+
+    def _uploaded(self, release, path: Path) -> Optional[str]:
+        if release is None:
+            return None
+        for url in release.urls:
+            if url.rsplit('/', maxsplit=1)[-1] == path.name:
+                return url
+        return None

dephell/config/builders.py

@@ -74,10 +74,10 @@ def build_venv(parser):
 
 
 def build_docker(parser):
-    venv_group = parser.add_argument_group('Docker container')
-    venv_group.add_argument('--docker-repo', help='image name without tag')
-    venv_group.add_argument('--docker-tag', help='image tag')
-    venv_group.add_argument('--docker-container', help='container name')
+    docker_group = parser.add_argument_group('Docker container')
+    docker_group.add_argument('--docker-repo', help='image name without tag')
+    docker_group.add_argument('--docker-tag', help='image tag')
+    docker_group.add_argument('--docker-container', help='container name')
 
 
 def build_other(parser):

dephell/config/defaults.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 # app
-from ..constants import DEFAULT_WAREHOUSE
+from ..constants import DEFAULT_UPLOAD, DEFAULT_WAREHOUSE
 from .app_dirs import get_cache_dir, get_data_dir
 
 
@@ -41,6 +41,11 @@
         tag='latest',
     ),
 
+    upload=dict(
+        url=DEFAULT_UPLOAD,
+        sign=False,
+    ),
+
     # other
     cache=dict(
         path=str(get_cache_dir()),

dephell/config/scheme.py

@@ -91,6 +91,16 @@
         },
     ),
 
+    # project upload
+    'upload': dict(
+        type='dict',
+        required=True,
+        schema={
+            'url':      dict(type='string', required=True),
+            'sign':     dict(type='boolean', required=True),
+            'identity': dict(type='string', required=False),
+        },
+    ),
 
     # other
     'owner':    dict(type='string', required=False),

dephell/constants.py

@@ -24,6 +24,9 @@ class ReturnCodes(Enum):
 DEFAULT_WAREHOUSE = 'https://pypi.org/pypi/'
 WAREHOUSE_DOMAINS = {'pypi.org', 'pypi.python.org', 'test.pypi.org'}
 
+DEFAULT_UPLOAD = 'https://upload.pypi.org/legacy/'
+TEST_UPLOAD = 'https://test.pypi.org/legacy/'
+
 FORMATS = (
     'conda',
     'egginfo',

dephell/controllers/__init__.py

@@ -9,6 +9,7 @@
 from ._resolver import Resolver
 from ._safety import Safety, SafetyVulnInfo
 from ._snyk import Snyk, SnykVulnInfo
+from ._uploader import Uploader
 
 
 __all__ = [
@@ -25,4 +26,5 @@
     'SafetyVulnInfo',
     'Snyk',
     'SnykVulnInfo',
+    'Uploader',
 ]