Koji NVR endpoint

Allows to download extracted manifests archive from koji (build by OSBS)
and uploads manifest artifact to quay.io

* OSBS-6687

Signed-off-by: Martin Bašti <mbasti@redhat.com>

.travis.yml

@@ -13,5 +13,7 @@ matrix:
     - dist: xenial
       python: "3.7"
       env: TOXENV=flake8
+before_install:
+  - sudo apt-get install -y rpm  # for rpm-py-installer (koji requirement)
 install: pip install tox-travis coveralls
 script: tox

Dockerfile

@@ -14,6 +14,7 @@ RUN dnf -y install \
     python3-gunicorn \
     python3-flask \
     python3-jsonschema \
+    python3-koji \
     python3-requests \
     python3-operator-courier \
     && dnf -y clean all \

README.md

@@ -22,6 +22,10 @@ class ProdConfig:
     LOG_LEVEL = "INFO"
     LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     DEFAULT_RELEASE_VERSION = "1.0.0"  # default operator manifest version
+
+    # configuration of Koji URLs
+    KOJIHUB_URL = 'https://koji.fedoraproject.org/kojihub'
+    KOJIROOT_URL = 'https://kojipkgs.fedoraproject.org/'
 ```
 
 ## Running service
@@ -140,6 +144,82 @@ curl \
   -F "file=@manifests.zip"
 ```
 
+### Uploading operators manifests from koji
+
+Downloads operator manifest archive from koji build specified by N-V-R.
+Build must be done by [OSBS](https://osbs.readthedocs.io)
+service which extracts operator manifests from images and stores them as a zip
+archive in koji.
+
+#### Endpoints
+
+* [POST] `/v1/<organization>/<repository>/koji/<nvr>/<version>`
+* [POST] `/v1/<organization>/<repository>/koji/<nvr>`
+
+Operator image build must be specified by N-V-R value from koji.
+
+If `<version>` is omitted:
+* the latest release version will be incremented and used (for example from `2.5.1` to `3.0.0`)
+* for new repository a default initial version will be used (`DEFAULT_RELEASE_VERSION` config option)
+
+`<version>` must be unique for repository. Quay doesn't support overwriting of releases.
+
+#### Replies
+
+**OK**
+
+HTTP code: 200
+
+```json
+{
+  "organization": "organization name",
+  "repo": "repository name",
+  "version": "0.0.1",
+  "nvr": "n-v-r",
+  "extracted_files": ["packages.yml", "..."]
+}
+
+```
+
+**Failures**
+
+Error messages have following format:
+```
+{
+  "status": <http numeric code>,
+  "error": "<error ID string>",
+  "message": "<detailed error description>",
+}
+```
+
+
+| HTTP Code / `status` |  `error`    |  Explanation        |
+|-----------|------------------------|---------------------|
+|400| OMPSUploadedFileError | Uploaded file didn't meet expectations (not a zip file, too big after unzip, corrupted zip file) |
+|400| OMPSInvalidVersionFormat | Invalid version format in URL |
+|400| KojiNotAnOperatorImage | Requested build is not an operator image |
+|403| OMPSAuthorizationHeaderRequired| No `Authorization` header found in request|
+|404| KojiNVRBuildNotFound | Requested build not found in koji |
+|500| KojiManifestsArchiveNotFound | Manifest archive not found in koji build |
+|500| KojiError | Koji generic error (connection failures, etc.) |
+|500| QuayCourierError | operator-courier module raised exception during building and pushing manifests to quay|
+|500| QuayPackageError | Failed to get information about application packages from quay |
+
+
+#### Example
+```bash
+curl \
+  -H "Authorization: ${TOKEN}" \
+  -X POST https://example.com/v1/myorg/myrepo/koji/image-1.2-5
+```
+or with explicit release version
+```bash
+curl \
+  -H "Authorization: ${TOKEN}" \
+  -X POST https://example.com/v1/myorg/myrepo/koji/image-1.2-5/1.1.5
+```
+
+
 ### Removing released operators manifests
 
 
@@ -220,10 +300,19 @@ pip install '.[test]'
 ### Running tests
 
 Project is integrated with tox:
+
+* please install `rpm-devel` (Fedora) or `rpm` (Ubuntu) package to be able
+build `koji` dependency `rpm-py-installer` in `tox`:
+```bash
+sudo dnf install -y rpm-devel
+```
+* run:
 ```bash
 tox
 ```
 
+
+
 To run tests manually, you can use pytest directly:
 ```bash
 py.test tests/

omps/api/v1/push.py

@@ -2,6 +2,8 @@
 # Copyright (C) 2019 Red Hat, Inc
 # see the LICENSE file for license
 #
+
+from functools import partial
 import logging
 import os
 from tempfile import NamedTemporaryFile, TemporaryDirectory
@@ -21,6 +23,7 @@
     OMPSExpectedFileError,
     QuayPackageNotFound,
 )
+from omps.koji_util import KOJI
 from omps.quay import QuayOrganization, ReleaseVersion
 
 logger = logging.getLogger(__name__)
@@ -35,7 +38,56 @@ def validate_allowed_extension(filename):
                 extension, ALLOWED_EXTENSIONS))
 
 
-def extract_zip_file(
+def _extract_zip_file(
+    filepath, target_dir,
+    max_uncompressed_size=DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE
+):
+    """Extract zip file into target directory
+
+    :param filepath: path to zip archive file
+    :param target_dir: directory where extracted files will be stored
+    :param max_uncompressed_size: size in Bytes how big data can be accepted
+        after uncompressing
+    """
+    try:
+        archive = zipfile.ZipFile(filepath)
+    except zipfile.BadZipFile as e:
+        raise OMPSUploadedFileError(str(e))
+
+    if logger.isEnabledFor(logging.DEBUG):
+        # log content of zipfile
+        logger.debug(
+            'Content of zip archive:\n%s',
+            '\n'.join(
+                "name={zi.filename}, compress_size={zi.compress_size}, "
+                "file_size={zi.file_size}".format(zi=zipinfo)
+                for zipinfo in archive.filelist
+            )
+        )
+
+    uncompressed_size = sum(zi.file_size for zi in archive.filelist)
+    if uncompressed_size > max_uncompressed_size:
+        raise OMPSUploadedFileError(
+            "Uncompressed archive is larger than limit "
+            "({}B>{}B)".format(
+                uncompressed_size, max_uncompressed_size
+            ))
+
+    try:
+        bad_file = archive.testzip()
+    except RuntimeError as e:
+        # trying to open an encrypted zip file without a password
+        raise OMPSUploadedFileError(str(e))
+
+    if bad_file is not None:
+        raise OMPSUploadedFileError(
+            "CRC check failed for file {} in archive".format(bad_file)
+        )
+    archive.extractall(target_dir)
+    archive.close()
+
+
+def extract_zip_file_from_request(
     req, target_dir,
     max_uncompressed_size=DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE
 ):
@@ -60,43 +112,25 @@ def extract_zip_file(
 
     with NamedTemporaryFile('w', suffix='.zip', dir=target_dir) as tmpf:
         uploaded_file.save(tmpf.name)
-        try:
-            archive = zipfile.ZipFile(tmpf.name)
-        except zipfile.BadZipFile as e:
-            raise OMPSUploadedFileError(str(e))
-
-        if logger.isEnabledFor(logging.DEBUG):
-            # log content of zipfile
-            logger.debug(
-                'Content of uploaded zip archive "%s":\n%s',
-                uploaded_file.filename, '\n'.join(
-                    "name={zi.filename}, compress_size={zi.compress_size}, "
-                    "file_size={zi.file_size}".format(zi=zipinfo)
-                    for zipinfo in archive.filelist
-                )
-            )
+        _extract_zip_file(tmpf.name, target_dir,
+                          max_uncompressed_size=max_uncompressed_size)
 
-        uncompressed_size = sum(zi.file_size for zi in archive.filelist)
-        if uncompressed_size > max_uncompressed_size:
-            raise OMPSUploadedFileError(
-                "Uncompressed archive is larger than limit "
-                "({}B>{}B)".format(
-                    uncompressed_size, max_uncompressed_size
-                ))
 
-        try:
-            bad_file = archive.testzip()
-        except RuntimeError as e:
-            # trying to open an encrypted zip file without a password
-            raise OMPSUploadedFileError(str(e))
-
-        if bad_file is not None:
-            raise OMPSUploadedFileError(
-                "CRC check failed for file {} in archive".format(bad_file)
-            )
+def extract_zip_file_from_koji(
+    nvr, target_dir,
+    max_uncompressed_size=DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE
+):
+    """Store content of operators_manifests zipfile in target_dir
 
-        archive.extractall(target_dir)
-        archive.close()
+    :param nvr: N-V-R of koji build
+    :param target_dir: directory where extracted files will be stored
+    :param max_uncompressed_size: size in Bytes how big data can be accepted
+        after uncompressing
+    """
+    with NamedTemporaryFile('wb', suffix='.zip', dir=target_dir) as tmpf:
+        KOJI.download_manifest_archive(nvr, tmpf)
+        _extract_zip_file(tmpf.name, target_dir,
+                          max_uncompressed_size=max_uncompressed_size)
 
 
 def _get_package_version(quay_org, repo, version=None):
@@ -116,17 +150,18 @@ def _get_package_version(quay_org, repo, version=None):
     return version
 
 
-@API.route("/<organization>/<repo>/zipfile", defaults={"version": None},
-           methods=('POST',))
-@API.route("/<organization>/<repo>/zipfile/<version>", methods=('POST',))
-def push_zipfile(organization, repo, version=None):
+def _zip_flow(*, organization, repo, version, extract_manifest_func,
+              extras_data=None):
     """
-    Push the particular version of operator manifest to registry from
-    the uploaded zipfile
-
-    :param organization: quay.io organization
-    :param repo: target repository
-    :param version: version of operator manifest
+    :param str organization: quay.io organization
+    :param str repo: target repository
+    :param str|None version: version of operator manifest
+    :param Callable[str, int] extract_manifest_func: function to retrieve operator
+        manifests zip file. First argument of function is path to target dir
+        where zip archive content will be extracted, second argument max size
+        of extracted files
+    :param extras_data: extra data added to response
+    :return: JSON response
     """
     token = extract_auth_token(request)
     quay_org = QuayOrganization(organization, token)
@@ -139,11 +174,12 @@ def push_zipfile(organization, repo, version=None):
         'repo': repo,
         'version': version,
     }
+    if extras_data:
+        data.update(extras_data)
 
     with TemporaryDirectory() as tmpdir:
         max_size = current_app.config['ZIPFILE_MAX_UNCOMPRESSED_SIZE']
-        extract_zip_file(request, tmpdir,
-                         max_uncompressed_size=max_size)
+        extract_manifest_func(tmpdir, max_uncompressed_size=max_size)
         extracted_files = os.listdir(tmpdir)
         logger.info("Extracted files: %s", extracted_files)
         data['extracted_files'] = extracted_files
@@ -155,21 +191,41 @@ def push_zipfile(organization, repo, version=None):
     return resp
 
 
-@API.route("/<organization>/<repo>/koji/<nvr>", methods=('POST',))
-def push_koji_nvr(organization, repo, nvr):
+@API.route("/<organization>/<repo>/zipfile", defaults={"version": None},
+           methods=('POST',))
+@API.route("/<organization>/<repo>/zipfile/<version>", methods=('POST',))
+def push_zipfile(organization, repo, version=None):
+    """
+    Push the particular version of operator manifest to registry from
+    the uploaded zipfile
+
+    :param organization: quay.io organization
+    :param repo: target repository
+    :param version: version of operator manifest
+    """
+    return _zip_flow(
+        organization=organization,
+        repo=repo,
+        version=version,
+        extract_manifest_func=partial(extract_zip_file_from_request, request)
+    )
+
+
+@API.route("/<organization>/<repo>/koji/<nvr>", defaults={"version": None},
+           methods=('POST',))
+@API.route("/<organization>/<repo>/koji/<nvr>/<version>", methods=('POST',))
+def push_koji_nvr(organization, repo, nvr, version):
     """
     Get operator manifest from koji by specified NVR and upload operator
     manifest to registry
     :param organization: quay.io organization
     :param repo: target repository
     :param nvr: image NVR from koji
     """
-    data = {
-        'organization': organization,
-        'repo': repo,
-        'nvr': nvr,
-        'msg': 'Not Implemented. Testing only'
-    }
-    resp = jsonify(data)
-    resp.status_code = 200
-    return resp
+    return _zip_flow(
+        organization=organization,
+        repo=repo,
+        version=version,
+        extract_manifest_func=partial(extract_zip_file_from_koji, nvr),
+        extras_data={'nvr': nvr}
+    )

omps/app.py

@@ -9,6 +9,7 @@
 
 from .api.v1 import API as API_V1
 from .errors import init_errors_handling
+from .koji_util import KOJI
 from .logger import init_logging
 from .settings import init_config
 
@@ -31,6 +32,7 @@ def _load_config(app):
     conf = init_config(app)
     init_logging(conf)
     logger.debug('Config loaded. Logging initialized')
+    KOJI.initialize(conf)
 
 
 def _init_errors_handling(app):

omps/constants.py

@@ -13,3 +13,5 @@
 DEFAULT_RELEASE_VERSION = '1.0.0'
 
 ALLOWED_EXTENSIONS = {".zip", }
+
+KOJI_OPERATOR_MANIFESTS_ARCHIVE_KEY = 'operator_manifests_archive'