/
docker_publisher.py
executable file
·476 lines (382 loc) · 21.7 KB
/
docker_publisher.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
#!/usr/bin/python3
#
# Copyright (c) 2022 SUSE LLC
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# This script's job is to listen for new releases of products with docker images
# and publish those.
import argparse
import json
import os
import re
import requests
import subprocess
import sys
import tempfile
from lxml import etree as xml
import docker_registry
REPOMD_NAMESPACES = {'md': "http://linux.duke.edu/metadata/common",
'repo': "http://linux.duke.edu/metadata/repo",
'rpm': "http://linux.duke.edu/metadata/rpm"}
class DockerImagePublisher:
"""Base class for handling the publishing of docker images.
This handles multiple architectures, which have different layers
and therefore versions."""
def releasedDockerImageVersion(self, arch):
"""This function returns an identifier for the released docker
image's version."""
raise Exception("pure virtual")
def prepareReleasing(self):
"""Prepare the environment to allow calls to releaseDockerImage."""
raise Exception("pure virtual")
def addImage(self, version, arch, image_path):
"""This function adds the docker image with the image manifest, config layers
in image_path."""
raise Exception("pure virtual")
def finishReleasing(self):
"""This function publishes the released layers."""
raise Exception("pure virtual")
class DockerPublishException(Exception):
pass
class DockerImageFetcher:
"""Base class for handling the acquiring of docker images."""
def currentVersion(self):
"""This function returns the version of the latest available version
of the image for the product."""
raise Exception("pure virtual")
def getDockerImage(self, callback):
"""This function downloads the root fs layer and calls callback
with its path as argument."""
raise Exception("pure virtual")
class DockerFetchException(Exception):
pass
class DockerImagePublisherRegistry(DockerImagePublisher):
"""The DockerImagePublisherRegistry class works by using a manifest list to
describe a tag. The list contains a manifest for each architecture.
The manifest will be edited instead of replaced, which means if you don't
call addImage for an architecture, the existing released image stays in place."""
MAP_ARCH_RPM_DOCKER = {'i586': ("386", None),
'x86_64': ("amd64", None),
'armv6l': ("arm", "v6"),
'armv7l': ("arm", "v7"),
'aarch64': ("arm64", "v8"),
'ppc64le': ("ppc64le", None),
's390x': ("s390x", None)}
def __init__(self, dhc, tag, aliases=[]):
"""Construct a DIPR by passing a DockerRegistryClient instance as dhc
and a name for a tag as tag.
Optionally, add tag aliases as aliases. Those will only be written to,
never read."""
self.dhc = dhc
self.tag = tag
self.aliases = aliases
# The manifestlist for the tag is only downloaded if this cache is empty,
# so needs to be set to None to force a redownload.
self.cached_manifestlist = None
# Construct a new manifestlist for the tag.
self.new_manifestlist = None
def getDockerArch(self, arch):
if arch not in self.MAP_ARCH_RPM_DOCKER:
raise DockerPublishException("Unknown arch %s" % arch)
return self.MAP_ARCH_RPM_DOCKER[arch]
def _getManifestlist(self):
if self.cached_manifestlist is None:
self.cached_manifestlist = self.dhc.getManifest(self.tag)
return self.cached_manifestlist
def releasedDockerImageVersion(self, arch):
docker_arch, docker_variant = self.getDockerArch(arch)
manifestlist = self._getManifestlist()
if manifestlist is None:
# No manifest -> force outdated version
return "0"
for manifest in manifestlist['manifests']:
if docker_variant is not None:
if 'variant' not in manifest['platform'] or manifest['platform']['variant'] != docker_variant:
continue
if manifest['platform']['architecture'] == docker_arch:
if 'vnd-opensuse-version' in manifest:
return manifest['vnd-opensuse-version']
# Arch not in the manifest -> force outdated version
return "0"
def prepareReleasing(self):
if self.new_manifestlist is not None:
raise DockerPublishException("Did not finish publishing")
self.new_manifestlist = self._getManifestlist()
# Generate an empty manifestlist
if not self.new_manifestlist:
self.new_manifestlist = {'schemaVersion': 2,
'tag': self.tag,
'mediaType': "application/vnd.docker.distribution.manifest.list.v2+json",
'manifests': []}
return True
def getV2ManifestEntry(self, path, filename, mediaType):
"""For V1 -> V2 schema conversion. filename has to contain the digest"""
digest = filename
if re.match(r"^[a-f0-9]{64}", digest):
digest = "sha256:" + os.path.splitext(digest)[0]
if not digest.startswith("sha256"):
raise DockerPublishException("Invalid manifest contents")
return {'mediaType': mediaType,
'size': os.path.getsize(path + "/" + filename),
'digest': digest,
'x-osdp-filename': filename}
def convertV1ToV2Manifest(self, path, manifest_v1):
"""Converts the v1 manifest in manifest_v1 to a V2 manifest and returns it"""
layers = []
# The order of layers changed in V1 -> V2
for layer_filename in manifest_v1['Layers'][::-1]:
layers += [self.getV2ManifestEntry(path, layer_filename,
"application/vnd.docker.image.rootfs.diff.tar.gzip")]
return {'schemaVersion': 2,
'mediaType': "application/vnd.docker.distribution.manifest.v2+json",
'config': self.getV2ManifestEntry(path, manifest_v1['Config'],
"application/vnd.docker.container.image.v1+json"),
'layers': layers}
def addImage(self, version, arch, image_path):
docker_arch, docker_variant = self.getDockerArch(arch)
manifest = None
with open(image_path + "/manifest.json") as manifest_file:
manifest = json.load(manifest_file)
manifest_v2 = self.convertV1ToV2Manifest(image_path, manifest[0])
# Upload blobs
if not self.dhc.uploadBlob(image_path + "/" + manifest_v2['config']['x-osdp-filename'],
manifest_v2['config']['digest']):
raise DockerPublishException("Could not upload the image config")
for layer in manifest_v2['layers']:
if not self.dhc.uploadBlob(image_path + "/" + layer['x-osdp-filename'],
layer['digest']):
raise DockerPublishException("Could not upload an image layer")
# Upload the manifest
manifest_content = json.dumps(manifest_v2).encode("utf-8")
manifest_digest = self.dhc.uploadManifest(manifest_content)
if manifest_digest is False:
raise DockerPublishException("Could not upload the manifest")
# Register the manifest in the list
replaced = False
for manifest in self.new_manifestlist['manifests']:
if 'variant' in manifest['platform'] and manifest['platform']['variant'] != docker_variant:
continue
if manifest['platform']['architecture'] == docker_arch:
manifest['mediaType'] = manifest_v2['mediaType']
manifest['size'] = len(manifest_content)
manifest['digest'] = manifest_digest
manifest['vnd-opensuse-version'] = version
if docker_variant is not None:
manifest['platform']['variant'] = docker_variant
replaced = True
if not replaced:
# Add it instead
manifest = {'mediaType': manifest_v2['mediaType'],
'size': len(manifest_content),
'digest': manifest_digest,
'vnd-opensuse-version': version,
'platform': {
'architecture': docker_arch,
'os': "linux"}
}
if docker_variant is not None:
manifest['platform']['variant'] = docker_variant
self.new_manifestlist['manifests'] += [manifest]
return True
def finishReleasing(self):
# Generate the manifest content
manifestlist_content = json.dumps(self.new_manifestlist).encode('utf-8')
# Push the aliases
for alias in self.aliases:
if not self.dhc.uploadManifest(manifestlist_content, alias):
raise DockerPublishException("Could not push an manifest list alias")
# Push the new manifest list
if not self.dhc.uploadManifest(manifestlist_content, self.tag):
raise DockerPublishException("Could not upload the new manifest list")
self.new_manifestlist = None
self.cached_manifestlist = None # force redownload
return True
class DockerImageFetcherURL(DockerImageFetcher):
"""A trivial implementation. It downloads a (compressed) tar archive and passes
the decompressed contents to the callback.
The version number can't be determined automatically (it would need to extract
the image and look at /etc/os-release each time - too expensive.) so it
has to be passed manually."""
def __init__(self, version, url):
self.version = version
self.url = url
def currentVersion(self):
return self.version
def getDockerImage(self, callback):
"""Download the tar and extract it"""
with tempfile.NamedTemporaryFile() as tar_file:
tar_file.write(requests.get(self.url).content)
with tempfile.TemporaryDirectory() as tar_dir:
# Extract the .tar.xz into the dir
subprocess.call("tar -xaf '%s' -C '%s'" % (tar_file.name, tar_dir), shell=True)
return callback(tar_dir)
class DockerImageFetcherOBS(DockerImageFetcher):
"""Uses the OBS API to access the build artifacts.
Url has to be https://build.opensuse.org/public/build/<project>/<repo>/<arch>/<pkgname>
If maintenance_release is True, it picks the buildcontainer released last with that name.
e.g. for "foo" it would pick "foo.2019" instead of "foo" or "foo.2018"."""
def __init__(self, url, maintenance_release=False):
self.url = url
self.newest_release_url = None
if not maintenance_release:
self.newest_release_url = url
def _isMaintenanceReleaseOf(self, release, source):
"""Returns whether release describes a maintenance release of source.
E.g. "foo.2019", "foo" -> True, "foo-asdf", "foo" -> False"""
sourcebuildflavor = source.split(":")[1] if ":" in source else None
releasebuildflavor = release.split(":")[1] if ":" in release else None
return sourcebuildflavor == releasebuildflavor and release.startswith(source.split(":")[0] + ".")
def _getNewestReleaseUrl(self):
if self.newest_release_url is None:
buildcontainername = self.url.split("/")[-1]
prjurl = self.url + "/.."
buildcontainerlist_req = requests.get(prjurl)
buildcontainerlist = xml.fromstring(buildcontainerlist_req.content)
releases = [entry for entry in buildcontainerlist.xpath("entry/@name") if
self._isMaintenanceReleaseOf(entry, buildcontainername)]
releases.sort()
# Pick the first one with binaries
for release in releases[::-1] + [buildcontainername]:
self.newest_release_url = prjurl + "/" + release
try:
self._getFilename()
break
except DockerFetchException:
continue
return self.newest_release_url
def _getFilename(self):
"""Return the name of the binary at the URL with the filename ending in
.docker.tar."""
binarylist_req = requests.get(self._getNewestReleaseUrl())
binarylist = xml.fromstring(binarylist_req.content)
for binary in binarylist.xpath("binary/@filename"):
if binary.endswith(".docker.tar"):
return binary
raise DockerFetchException("No docker image built in the repository")
def currentVersion(self):
"""Return {version}-?({flavor}-)Build{build} of the docker file."""
filename = self._getFilename()
# Capture everything between arch and filename suffix
return re.match(r'[^.]*\.[^.]+-(.*)\.docker\.tar$', filename).group(1)
def getDockerImage(self, callback):
"""Download the tar and extract it"""
filename = self._getFilename()
with tempfile.NamedTemporaryFile() as tar_file:
tar_file.write(requests.get(self.newest_release_url + "/" + filename).content)
with tempfile.TemporaryDirectory() as tar_dir:
# Extract the .tar into the dir
subprocess.call("tar -xaf '%s' -C '%s'" % (tar_file.name, tar_dir), shell=True)
return callback(tar_dir)
def run():
drc_tw = docker_registry.DockerRegistryClient(os.environ['REGISTRY'], os.environ['REGISTRY_USER'], os.environ['REGISTRY_PASSWORD'],
os.environ['REGISTRY_REPO_TW'])
drc_leap = docker_registry.DockerRegistryClient(os.environ['REGISTRY'], os.environ['REGISTRY_USER'], os.environ['REGISTRY_PASSWORD'],
os.environ['REGISTRY_REPO_LEAP'])
config = {
'tumbleweed': {
'fetchers': {
'i586': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/i586/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
'x86_64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/x86_64/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
'aarch64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/aarch64/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
'armv7l': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/armv7l/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
'armv6l': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/armv6l/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
'ppc64le': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/ppc64le/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
's390x': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Tumbleweed/containers/s390x/opensuse-tumbleweed-image:docker", maintenance_release=True), # noqa: E501
},
'publisher': DockerImagePublisherRegistry(drc_tw, "latest"),
},
'leap-15.5': {
'fetchers': {
'x86_64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.5/containers/x86_64/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
'aarch64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.5/containers/aarch64/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
'armv7l': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.5/containers_armv7/armv7l/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
'ppc64le': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.5/containers/ppc64le/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
's390x': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.5/containers/s390x/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
},
'publisher': DockerImagePublisherRegistry(drc_leap, "latest", ["15.5", "15"]),
},
'leap-15.6': {
'fetchers': {
'x86_64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/x86_64/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
'aarch64': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/aarch64/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
'armv7l': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers_armv7/armv7l/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
'ppc64le': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/ppc64le/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
's390x': DockerImageFetcherOBS(url="https://build.opensuse.org/public/build/openSUSE:Containers:Leap:15.6/containers/s390x/opensuse-leap-image:docker", maintenance_release=True), # noqa: E501
},
'publisher': DockerImagePublisherRegistry(drc_leap, "15.6"),
},
}
# Parse args after defining the config - the available distros are included
# in the help output
parser = argparse.ArgumentParser(description="Docker image publish script",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("distros", metavar="distro", type=str, nargs="*",
default=[key for key in config],
help="Which distros to check for images to publish.")
args = parser.parse_args()
success = True
for distro in args.distros:
print("Handling %s" % distro)
archs_to_update = {}
fetchers = config[distro]['fetchers']
publisher = config[distro]['publisher']
for arch in fetchers:
print("\tArchitecture %s" % arch)
try:
current = fetchers[arch].currentVersion()
print("\t\tAvailable version: %s" % current)
released = publisher.releasedDockerImageVersion(arch)
print("\t\tReleased version: %s" % released)
if current != released:
archs_to_update[arch] = current
except Exception as e:
print("\t\tException during version fetching: %s" % e)
if not archs_to_update:
print("\tNothing to do.")
continue
if not publisher.prepareReleasing():
print("\tCould not prepare the publishing")
success = False
continue
need_to_upload = False
for arch, version in archs_to_update.items():
print("\tUpdating %s image to version %s" % (arch, version))
try:
fetchers[arch].getDockerImage(lambda image_path: publisher.addImage(version=version,
arch=arch,
image_path=image_path))
need_to_upload = True
except DockerFetchException as dfe:
print("\t\tCould not fetch the image: %s" % dfe)
success = False
continue
except DockerPublishException as dpe:
print("\t\tCould not publish the image: %s" % dpe)
success = False
continue
# If nothing got added to the publisher, don't try to upload it.
# For docker hub it'll just update the "last pushed" time without any change
if not need_to_upload:
continue
if not publisher.finishReleasing():
print("\tCould not publish the image")
continue
return 0 if success else 1
if __name__ == "__main__":
sys.exit(run())