Merge pull request #35 from chriskuehl/json

Add basic json api endpoint

dumb_pypi/main.py

@@ -109,7 +109,8 @@ def sort_key(self) -> Tuple[str, packaging.version.Version, str]:
     @property
     def formatted_upload_time(self) -> str:
         assert self.upload_timestamp is not None
-        return _format_datetime(datetime.fromtimestamp(self.upload_timestamp))
+        dt = datetime.utcfromtimestamp(self.upload_timestamp)
+        return _format_datetime(dt)
 
     @property
     def info_string(self) -> str:
@@ -126,10 +127,22 @@ def info_string(self) -> str:
             info += f', {self.uploaded_by}'
         return info
 
-    def url(self, base_url: str) -> str:
-        hash_part = f'#{self.hash}' if self.hash else ''
+    def url(self, base_url: str, *, include_hash: bool = True) -> str:
+        hash_part = f'#{self.hash}' if self.hash and include_hash else ''
         return f'{base_url.rstrip("/")}/{self.filename}{hash_part}'
 
+    def json_info(self, base_url: str) -> Dict[str, Any]:
+        ret: Dict[str, Any] = {
+            'filename': self.filename,
+            'url': self.url(base_url, include_hash=False),
+        }
+        if self.upload_timestamp is not None:
+            ret['upload_time'] = self.formatted_upload_time
+        if self.hash is not None:
+            algo, h = self.hash.split('=')
+            ret['digests'] = {algo: h}
+        return ret
+
     @classmethod
     def create(
             cls,
@@ -173,6 +186,20 @@ def _format_datetime(dt: datetime) -> str:
     return dt.strftime('%Y-%m-%d %H:%M:%S')
 
 
+def _package_json(files: List[Package], base_url: str) -> Dict[str, Any]:
+    # note: the full api contains much more, we only output the info we have
+    by_version: Dict[str, List[Dict[str, Any]]] = collections.defaultdict(list)
+    for file in files:
+        if file.version is not None:
+            by_version[file.version].append(file.json_info(base_url))
+
+    return {
+        'info': {'name': files[0].name, 'version': files[0].version},
+        'releases': by_version,
+        'urls': by_version[files[0].version] if files[0].version else [],
+    }
+
+
 class Settings(NamedTuple):
     output_dir: str
     packages_url: str
@@ -184,8 +211,10 @@ class Settings(NamedTuple):
 def build_repo(packages: Dict[str, Set[Package]], settings: Settings) -> None:
     simple = os.path.join(settings.output_dir, 'simple')
     os.makedirs(simple, exist_ok=True)
+    pypi = os.path.join(settings.output_dir, 'pypi')
+    os.makedirs(pypi, exist_ok=True)
 
-    current_date = _format_datetime(datetime.now())
+    current_date = _format_datetime(datetime.utcnow())
 
     # /index.html
     with atomic_write(os.path.join(settings.output_dir, 'index.html')) as f:
@@ -209,23 +238,29 @@ def build_repo(packages: Dict[str, Set[Package]], settings: Settings) -> None:
             package_names=sorted(packages),
         ))
 
-    for package_name, versions in packages.items():
+    for package_name, files in packages.items():
+        sorted_files = sorted(
+            # Newer versions should sort first.
+            files, key=operator.attrgetter('sort_key'), reverse=True,
+        )
+
         # /simple/{package}/index.html
         simple_package_dir = os.path.join(simple, package_name)
         os.makedirs(simple_package_dir, exist_ok=True)
         with atomic_write(os.path.join(simple_package_dir, 'index.html')) as f:
             f.write(jinja_env.get_template('package.html').render(
                 date=current_date,
                 package_name=package_name,
-                versions=sorted(
-                    versions,
-                    key=operator.attrgetter('sort_key'),
-                    # Newer versions should sort first.
-                    reverse=True,
-                ),
+                files=sorted_files,
                 packages_url=settings.packages_url,
             ))
 
+        # /pypi/{package}/json
+        pypi_package_dir = os.path.join(pypi, package_name)
+        os.makedirs(pypi_package_dir, exist_ok=True)
+        with atomic_write(os.path.join(pypi_package_dir, 'json')) as f:
+            json.dump(_package_json(sorted_files, settings.packages_url), f)
+
 
 def _lines_from_path(path: str) -> List[str]:
     f = sys.stdin if path == '-' else open(path)

dumb_pypi/templates/package.html

@@ -5,11 +5,11 @@
     </head>
     <body>
         <h1>{{package_name}}</h1>
-        {% if versions %}<p>Latest version: {{(versions|first).version}}{% endif %}</p>
+        <p>Latest version: {{(files|first).version}}</p>
         <p>Generated on {{date}}.</p>
         <ul>
-            {% for version in versions %}
-                <li><a href="{{version.url(packages_url)}}">{{version.filename}}</a> ({{version.info_string}})</li>
+            {% for file in files %}
+                <li><a href="{{file.url(packages_url)}}">{{file.filename}}</a> ({{file.info_string}})</li>
             {% endfor %}
         </ul>
     </body>

tests/main_test.py

@@ -89,6 +89,72 @@ def test_package_url_with_hash():
     assert package.url('/prefix') == '/prefix/f.tar.gz#sha256=badf00d'
 
 
+def test_package_info_all_info():
+    package = main.Package.create(
+        filename='f-1.0.tar.gz',
+        hash='sha256=deadbeef',
+        upload_timestamp=1528586805,
+    )
+    ret = package.json_info('/prefix')
+    assert ret == {
+        'digests': {'sha256': 'deadbeef'},
+        'filename': 'f-1.0.tar.gz',
+        'url': '/prefix/f-1.0.tar.gz',
+        'upload_time': '2018-06-09 23:26:45',
+    }
+
+
+def test_package_info_minimal_info():
+    ret = main.Package.create(filename='f-1.0.tar.gz').json_info('/prefix')
+    assert ret == {'filename': 'f-1.0.tar.gz', 'url': '/prefix/f-1.0.tar.gz'}
+
+
+def test_package_json_excludes_non_versioned_packages():
+    pkgs = [main.Package.create(filename='f.tar.gz')]
+    ret = main._package_json(pkgs, '/prefix')
+    assert ret == {
+        'info': {'name': 'f', 'version': None},
+        'releases': {},
+        'urls': [],
+    }
+
+
+def test_package_json_packages_with_info():
+    pkgs = [
+        main.Package.create(filename='f-2.0.tar.gz'),
+        main.Package.create(filename='f-1.0-py2.py3-none-any.whl'),
+        main.Package.create(filename='f-1.0.tar.gz'),
+    ]
+    ret = main._package_json(pkgs, '/prefix')
+    assert ret == {
+        'info': {'name': 'f', 'version': '2.0'},
+        'releases': {
+            '2.0': [
+                {
+                    'filename': 'f-2.0.tar.gz',
+                    'url': '/prefix/f-2.0.tar.gz',
+                },
+            ],
+            '1.0': [
+                {
+                    'filename': 'f-1.0-py2.py3-none-any.whl',
+                    'url': '/prefix/f-1.0-py2.py3-none-any.whl',
+                },
+                {
+                    'filename': 'f-1.0.tar.gz',
+                    'url': '/prefix/f-1.0.tar.gz',
+                },
+            ],
+        },
+        'urls': [
+            {
+                'filename': 'f-2.0.tar.gz',
+                'url': '/prefix/f-2.0.tar.gz',
+            },
+        ],
+    }
+
+
 def test_build_repo_smoke_test(tmpdir):
     package_list = tmpdir.join('package-list')
     package_list.write('ocflib-2016.12.10.1.48-py2.py3-none-any.whl\n')
@@ -111,7 +177,7 @@ def test_build_repo_json_smoke_test(tmpdir):
                 'filename': 'ocflib-2016.12.10.1.48-py2.py3-none-any.whl',
                 'uploaded_by': 'ckuehl',
                 'upload_timestamp': 1515783971,
-                'hash': 'md5-b1946ac92492d2347c6235b4d2611184',
+                'hash': 'md5=b1946ac92492d2347c6235b4d2611184',
             },
             {
                 'filename': 'numpy-1.11.0rc1.tar.gz',