This repository has been archived by the owner on May 24, 2023. It is now read-only.
/
push.py
231 lines (196 loc) · 7.49 KB
/
push.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
#
# Copyright (C) 2019 Red Hat, Inc
# see the LICENSE file for license
#
from functools import partial
import logging
import os
from tempfile import NamedTemporaryFile, TemporaryDirectory
import zipfile
from flask import jsonify, current_app, request
from . import API
from omps.api.common import extract_auth_token
from omps.constants import (
ALLOWED_EXTENSIONS,
DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE,
)
from omps.errors import (
OMPSInvalidVersionFormat,
OMPSUploadedFileError,
OMPSExpectedFileError,
QuayPackageNotFound,
)
from omps.koji_util import KOJI
from omps.quay import QuayOrganization, ReleaseVersion
logger = logging.getLogger(__name__)
def validate_allowed_extension(filename):
"""Check file extension"""
_, extension = os.path.splitext(filename)
if extension.lower() not in ALLOWED_EXTENSIONS:
raise OMPSUploadedFileError(
'Uploaded file extension "{}" is not any of {}'.format(
extension, ALLOWED_EXTENSIONS))
def _extract_zip_file(
filepath, target_dir,
max_uncompressed_size=DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE
):
"""Extract zip file into target directory
:param filepath: path to zip archive file
:param target_dir: directory where extracted files will be stored
:param max_uncompressed_size: size in Bytes how big data can be accepted
after uncompressing
"""
try:
archive = zipfile.ZipFile(filepath)
except zipfile.BadZipFile as e:
raise OMPSUploadedFileError(str(e))
if logger.isEnabledFor(logging.DEBUG):
# log content of zipfile
logger.debug(
'Content of zip archive:\n%s',
'\n'.join(
"name={zi.filename}, compress_size={zi.compress_size}, "
"file_size={zi.file_size}".format(zi=zipinfo)
for zipinfo in archive.filelist
)
)
uncompressed_size = sum(zi.file_size for zi in archive.filelist)
if uncompressed_size > max_uncompressed_size:
raise OMPSUploadedFileError(
"Uncompressed archive is larger than limit "
"({}B>{}B)".format(
uncompressed_size, max_uncompressed_size
))
try:
bad_file = archive.testzip()
except RuntimeError as e:
# trying to open an encrypted zip file without a password
raise OMPSUploadedFileError(str(e))
if bad_file is not None:
raise OMPSUploadedFileError(
"CRC check failed for file {} in archive".format(bad_file)
)
archive.extractall(target_dir)
archive.close()
def extract_zip_file_from_request(
req, target_dir,
max_uncompressed_size=DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE
):
"""Store uploaded file in target_directory
:param req: Flask request object
:param target_dir: directory where file will be stored
:param max_uncompressed_size: size in Bytes how big data can be accepted
after uncompressing
"""
assert req.method == 'POST'
if 'file' not in req.files:
raise OMPSExpectedFileError('No field "file" in uploaded data')
uploaded_file = req.files['file']
if not uploaded_file.filename:
# from Flask docs:
# if user does not select file, browser also
# submit an empty part without filename
raise OMPSExpectedFileError('No selected "file" in uploaded data')
validate_allowed_extension(uploaded_file.filename)
with NamedTemporaryFile('w', suffix='.zip', dir=target_dir) as tmpf:
uploaded_file.save(tmpf.name)
_extract_zip_file(tmpf.name, target_dir,
max_uncompressed_size=max_uncompressed_size)
def extract_zip_file_from_koji(
nvr, target_dir,
max_uncompressed_size=DEFAULT_ZIPFILE_MAX_UNCOMPRESSED_SIZE
):
"""Store content of operators_manifests zipfile in target_dir
:param nvr: N-V-R of koji build
:param target_dir: directory where extracted files will be stored
:param max_uncompressed_size: size in Bytes how big data can be accepted
after uncompressing
"""
with NamedTemporaryFile('wb', suffix='.zip', dir=target_dir) as tmpf:
KOJI.download_manifest_archive(nvr, tmpf)
_extract_zip_file(tmpf.name, target_dir,
max_uncompressed_size=max_uncompressed_size)
def _get_package_version(quay_org, repo, version=None):
if version is None:
try:
latest_ver = quay_org.get_latest_release_version(repo)
except QuayPackageNotFound:
version = current_app.config['DEFAULT_RELEASE_VERSION']
else:
latest_ver.increment()
version = str(latest_ver)
else:
try:
ReleaseVersion.validate_version(version)
except ValueError as e:
raise OMPSInvalidVersionFormat(str(e))
return version
def _zip_flow(*, organization, repo, version, extract_manifest_func,
extras_data=None):
"""
:param str organization: quay.io organization
:param str repo: target repository
:param str|None version: version of operator manifest
:param Callable[str, int] extract_manifest_func: function to retrieve operator
manifests zip file. First argument of function is path to target dir
where zip archive content will be extracted, second argument max size
of extracted files
:param extras_data: extra data added to response
:return: JSON response
"""
token = extract_auth_token(request)
quay_org = QuayOrganization(organization, token)
version = _get_package_version(quay_org, repo, version)
logger.info("Using release version: %s", version)
data = {
'organization': organization,
'repo': repo,
'version': version,
}
if extras_data:
data.update(extras_data)
with TemporaryDirectory() as tmpdir:
max_size = current_app.config['ZIPFILE_MAX_UNCOMPRESSED_SIZE']
extract_manifest_func(tmpdir, max_uncompressed_size=max_size)
extracted_files = os.listdir(tmpdir)
logger.info("Extracted files: %s", extracted_files)
data['extracted_files'] = extracted_files
quay_org.push_operator_manifest(repo, version, tmpdir)
resp = jsonify(data)
resp.status_code = 200
return resp
@API.route("/<organization>/<repo>/zipfile", defaults={"version": None},
methods=('POST',))
@API.route("/<organization>/<repo>/zipfile/<version>", methods=('POST',))
def push_zipfile(organization, repo, version=None):
"""
Push the particular version of operator manifest to registry from
the uploaded zipfile
:param organization: quay.io organization
:param repo: target repository
:param version: version of operator manifest
"""
return _zip_flow(
organization=organization,
repo=repo,
version=version,
extract_manifest_func=partial(extract_zip_file_from_request, request)
)
@API.route("/<organization>/<repo>/koji/<nvr>", defaults={"version": None},
methods=('POST',))
@API.route("/<organization>/<repo>/koji/<nvr>/<version>", methods=('POST',))
def push_koji_nvr(organization, repo, nvr, version):
"""
Get operator manifest from koji by specified NVR and upload operator
manifest to registry
:param organization: quay.io organization
:param repo: target repository
:param nvr: image NVR from koji
"""
return _zip_flow(
organization=organization,
repo=repo,
version=version,
extract_manifest_func=partial(extract_zip_file_from_koji, nvr),
extras_data={'nvr': nvr}
)