Push: Get reponame from package name in manifest bundle

OLM Marketplace has requirement that packages must be pushed to
repositories with the same name as package name.

Incompatible API change was introduced due this request, so now it is
part of API v2. `<repo>` part has been dropped from operators push API.

Edpoints:
* [POST] `/v2/<organization>/zipfile`
* [POST] `/v2/<organization>/zipfile/<version>`
* [POST] `/v2/<organization>/koji/<nvr>`
* [POST] `/v2/<organization>/koji/<nvr>/<version>`

* OSBS-7128

Signed-off-by: Martin Bašti <mbasti@redhat.com>

Dockerfile

@@ -15,6 +15,7 @@ RUN dnf -y install \
     python3-flask \
     python3-jsonschema \
     python3-koji \
+    python3-pyyaml \
     python3-requests \
     python3-operator-courier \
     && dnf -y clean all \

docs/usage/v2.md

@@ -6,8 +6,8 @@ Operator manifests files must be added to zip archive
 
 #### Endpoints
 
-* [POST] `/v2/<organization>/<repository>/zipfile/<version>`
-* [POST] `/v2/<organization>/<repository>/zipfile`
+* [POST] `/v2/<organization>/zipfile/<version>`
+* [POST] `/v2/<organization>/zipfile`
 
 Zip file must be attached as `content_type='multipart/form-data'` assigned to
 field `file`. See `curl` examples bellow.
@@ -18,6 +18,9 @@ If `<version>` is omitted:
 
 `<version>` must be unique for repository. Quay doesn't support overwriting of releases.
 
+Target `repository` name is taken from the `packageName` attribute specified in the uploaded operator manifest.
+
+
 #### Replies
 
 **OK**
@@ -79,8 +82,8 @@ archive in koji.
 
 #### Endpoints
 
-* [POST] `/v2/<organization>/<repository>/koji/<nvr>/<version>`
-* [POST] `/v2/<organization>/<repository>/koji/<nvr>`
+* [POST] `/v2/<organization>/koji/<nvr>/<version>`
+* [POST] `/v2/<organization>/koji/<nvr>`
 
 Operator image build must be specified by N-V-R value from koji.
 
@@ -90,6 +93,8 @@ If `<version>` is omitted:
 
 `<version>` must be unique for repository. Quay doesn't support overwriting of releases.
 
+Target `repository` name is taken from the `packageName` attribute specified in the uploaded operator manifest.
+
 #### Replies
 
 **OK**

omps/api/v1/push.py

@@ -24,6 +24,7 @@
     QuayPackageNotFound,
 )
 from omps.koji_util import KOJI
+from omps.manifests_util import ManifestBundle
 from omps.quay import ReleaseVersion, ORG_MANAGER
 
 logger = logging.getLogger(__name__)
@@ -133,7 +134,18 @@ def extract_zip_file_from_koji(
                           max_uncompressed_size=max_uncompressed_size)
 
 
-def _get_package_version(quay_org, repo, version=None):
+def get_package_version(quay_org, repo, version=None):
+    """Returns version of new release.
+    If version is passed, it will be validated
+    If no version is passed then quay repo is queried for versions and latest
+    version is incremented by 1 on major position.
+
+    :param QuayOrganization quay_org: Quay organization object
+    :param str repo: repository name
+    :param str|None version:
+    :rtype: str
+    :return: version to be used as release
+    """
     if version is None:
         try:
             latest_ver = quay_org.get_latest_release_version(repo)
@@ -150,11 +162,17 @@ def _get_package_version(quay_org, repo, version=None):
     return version
 
 
+def _get_reponame_from_manifests(source_dir):
+    bundle = ManifestBundle.from_dir(source_dir)
+    return bundle.package_name
+
+
 def _zip_flow(*, organization, repo, version, extract_manifest_func,
               extras_data=None):
     """
     :param str organization: quay.io organization
-    :param str repo: target repository
+    :param str|None repo: target repository (if not specified, will be taken
+         from manifest data)
     :param str|None version: version of operator manifest
     :param Callable[str, int] extract_manifest_func: function to retrieve operator
         manifests zip file. First argument of function is path to target dir
@@ -166,26 +184,31 @@ def _zip_flow(*, organization, repo, version, extract_manifest_func,
     cnr_token = extract_auth_token(request)
     quay_org = ORG_MANAGER.get_org(organization, cnr_token)
 
-    version = _get_package_version(quay_org, repo, version)
-    logger.info("Using release version: %s", version)
-
-    data = {
-        'organization': organization,
-        'repo': repo,
-        'version': version,
-    }
-    if extras_data:
-        data.update(extras_data)
+    data = {}
 
     with TemporaryDirectory() as tmpdir:
         max_size = current_app.config['ZIPFILE_MAX_UNCOMPRESSED_SIZE']
         extract_manifest_func(tmpdir, max_uncompressed_size=max_size)
-        extracted_files = os.listdir(tmpdir)
+        extracted_files = sorted(os.listdir(tmpdir))
         logger.info("Extracted files: %s", extracted_files)
         data['extracted_files'] = extracted_files
 
+        if repo is None:
+            repo = _get_reponame_from_manifests(tmpdir)
+
+        version = get_package_version(quay_org, repo, version)
+        logger.info("Using release version: %s", version)
+
         quay_org.push_operator_manifest(repo, version, tmpdir)
 
+    data.update({
+        'organization': organization,
+        'repo': repo,
+        'version': version,
+    })
+    if extras_data:
+        data.update(extras_data)
+
     resp = jsonify(data)
     resp.status_code = 200
     return resp

omps/api/v2/push.py

@@ -3,23 +3,40 @@
 # see the LICENSE file for license
 #
 
+from functools import partial
+
+from flask import request
+
 from . import API
 from omps.api.v1.push import (
-    push_zipfile as push_zipfile_v1,
-    push_koji_nvr as push_koji_nvr_v1,
+    extract_zip_file_from_koji,
+    extract_zip_file_from_request,
+    _zip_flow,
 )
 
-@API.route("/<organization>/<repo>/zipfile", defaults={"version": None},
+
+@API.route("/<organization>/zipfile", defaults={"version": None},
            methods=('POST',))
-@API.route("/<organization>/<repo>/zipfile/<version>", methods=('POST',))
-def push_zipfile(organization, repo, version=None):
+@API.route("/<organization>/zipfile/<version>", methods=('POST',))
+def push_zipfile(organization, version=None):
     # V2 has the same implementation as V1
-    return push_zipfile_v1(organization, repo, version)
+    return _zip_flow(
+        organization=organization,
+        repo=None,
+        version=version,
+        extract_manifest_func=partial(extract_zip_file_from_request, request)
+    )
 
 
-@API.route("/<organization>/<repo>/koji/<nvr>", defaults={"version": None},
+@API.route("/<organization>/koji/<nvr>", defaults={"version": None},
            methods=('POST',))
-@API.route("/<organization>/<repo>/koji/<nvr>/<version>", methods=('POST',))
-def push_koji_nvr(organization, repo, nvr, version):
+@API.route("/<organization>/koji/<nvr>/<version>", methods=('POST',))
+def push_koji_nvr(organization, nvr, version):
     # V2 has the same implementation as V1
-    return push_koji_nvr_v1(organization, repo, nvr, version)
+    return _zip_flow(
+        organization=organization,
+        repo=None,
+        version=version,
+        extract_manifest_func=partial(extract_zip_file_from_koji, nvr),
+        extras_data={'nvr': nvr}
+    )

omps/manifests_util.py

@@ -0,0 +1,54 @@
+#
+# Copyright (C) 2019 Red Hat, Inc
+# see the LICENSE file for license
+#
+
+"""Helper classes/functions for operator manifests"""
+
+import logging
+import io
+
+import yaml
+
+from operatorcourier.api import build_and_verify
+from omps.errors import QuayCourierError
+
+logger = logging.getLogger(__name__)
+
+
+class ManifestBundle:
+    """Provides metadata of operator bundle"""
+
+    @classmethod
+    def from_dir(cls, source_dir_path):
+        """Build bundle from specified directory
+
+        :param source_dir_path: path to directory with operator manifest
+        :rtype: ManifestBundle
+        :return: object
+        """
+        try:
+            bundle = build_and_verify(source_dir_path)
+        except Exception as e:
+            msg = "Operator courier failed: {}".format(e)
+            raise QuayCourierError(msg)
+        return cls(bundle)
+
+    def __init__(self, bundle):
+        """
+        :param bundle: bundle built by operator-courier
+        """
+        self._bundle = bundle
+
+    @property
+    def bundle(self):
+        """Returns operator bundle"""
+        return self._bundle
+
+    @property
+    def package_name(self):
+        """Returns defined package name from operator bundle"""
+        #  op. courier do verification, this should be never empty
+        pkgs_yaml = self.bundle['data']['packages']
+        pkgs = yaml.safe_load(io.StringIO(pkgs_yaml))
+        return pkgs[0]['packageName']

setup.py

@@ -60,6 +60,7 @@ def get_project_version(version_file='omps/__init__.py'):
         'Flask==1.0.*',
         'jsonschema',
         'koji',
+        'PyYAML',
         'requests',
         'operator-courier',
     ],

tests/api/v1/test_api_push.py

@@ -15,7 +15,7 @@ def test_push_zipfile(
         client, valid_manifests_archive, endpoint_push_zipfile,
         mocked_quay_io, mocked_op_courier_push, auth_header):
     """Test REST API for pushing operators form zipfile"""
-    with open(valid_manifests_archive, 'rb') as f:
+    with open(valid_manifests_archive.path, 'rb') as f:
         data = {
             'file': (f, f.name),
         }
@@ -31,7 +31,7 @@ def test_push_zipfile(
         'organization': endpoint_push_zipfile.org,
         'repo': endpoint_push_zipfile.repo,
         'version': endpoint_push_zipfile.version or constants.DEFAULT_RELEASE_VERSION,
-        'extracted_files': ['empty.yml'],
+        'extracted_files': valid_manifests_archive.files,
     }
     assert rv.get_json() == expected
 
@@ -108,6 +108,7 @@ def test_push_koji_nvr(
         client, endpoint_push_koji, mocked_quay_io, mocked_op_courier_push,
         auth_header, mocked_koji_archive_download):
     """Test REST API for pushing operators form koji by NVR"""
+    archive = mocked_koji_archive_download
     rv = client.post(
         endpoint_push_koji.url_path,
         headers=auth_header
@@ -118,7 +119,7 @@ def test_push_koji_nvr(
         'repo': endpoint_push_koji.repo,
         'version': endpoint_push_koji.version or constants.DEFAULT_RELEASE_VERSION,
         'nvr': endpoint_push_koji.nvr,
-        'extracted_files': ['empty.yml'],
+        'extracted_files': archive.files,
     }
     assert rv.get_json() == expected
 

tests/api/v2/conftest.py

@@ -18,16 +18,15 @@
 def endpoint_push_zipfile(request, release_version):
     """Returns URL for zipfile endpoints"""
     organization = 'testorg'
-    repo = 'repo-Y'
     version = release_version if request.param else None
 
-    url_path = '/v2/{}/{}/zipfile'.format(organization, repo)
+    url_path = '/v2/{}/zipfile'.format(organization)
     if version:
         url_path = '{}/{}'.format(url_path, version)
 
     yield EntrypointMeta(
         url_path=url_path, org=organization,
-        repo=repo, version=version, nvr=None,
+        repo=None, version=version, nvr=None,
     )
 
 
@@ -38,17 +37,16 @@ def endpoint_push_zipfile(request, release_version):
 def endpoint_push_koji(request, release_version):
     """Returns URL for koji endpoints"""
     organization = 'testorg'
-    repo = 'repo-Y'
     nvr = 'build-1.0.1-2'
     version = release_version if request.param else None
 
-    url_path = '/v2/{}/{}/koji/{}'.format(organization, repo, nvr)
+    url_path = '/v2/{}/koji/{}'.format(organization, nvr)
     if version:
         url_path = '{}/{}'.format(url_path, version)
 
     yield EntrypointMeta(
         url_path=url_path, org=organization,
-        repo=repo, version=version, nvr=nvr,
+        repo=None, version=version, nvr=nvr,
     )