This repository has been archived by the owner on Oct 13, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 26
/
runtime.py
1650 lines (1371 loc) · 73.7 KB
/
runtime.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
from multiprocessing.pool import MapResult
from future import standard_library
standard_library.install_aliases()
from future.utils import as_native_str
from multiprocessing.dummy import Pool as ThreadPool
from contextlib import contextmanager
from collections import namedtuple
import os
import sys
import tempfile
import threading
import shutil
import atexit
import datetime
import json
import yaml
import click
import logging
import functools
import traceback
import urllib.parse
import signal
import io
import pathlib
from typing import Optional, List, Dict, Tuple, Union
import time
import re
from jira import JIRA, Issue
from doozerlib import gitdata
from . import logutil
from . import assertion
from . import exectools
from . import dblib
from .pushd import Dir
from .image import ImageMetadata
from .rpmcfg import RPMMetadata
from .metadata import Metadata, RebuildHint
from doozerlib import state
from .model import Model, Missing
from multiprocessing import Lock, RLock, Semaphore
from .repos import Repos
from doozerlib.exceptions import DoozerFatalError
from doozerlib import constants
from doozerlib import util
from doozerlib import brew
from doozerlib.assembly import assembly_group_config, assembly_basis_event, assembly_type, AssemblyTypes, assembly_streams_config
from doozerlib.build_status_detector import BuildStatusDetector
# Values corresponds to schema for group.yml: freeze_automation. When
# 'yes', doozer itself will inhibit build/rebase related activity
# (exiting with an error if someone tries). Other values can
# be interpreted & enforced by the build pipelines (e.g. by
# invoking config:read-config).
FREEZE_AUTOMATION_YES = 'yes'
FREEZE_AUTOMATION_SCHEDULED = 'scheduled' # inform the pipeline that only manually run tasks should be permitted
FREEZE_AUTOMATION_NO = 'no'
# doozer cancel brew builds on SIGINT (Ctrl-C)
# but Jenkins sends a SIGTERM when cancelling a job.
def handle_sigterm(*_):
raise KeyboardInterrupt()
signal.signal(signal.SIGTERM, handle_sigterm)
# Registered atexit to close out debug/record logs
def close_file(f):
f.close()
def remove_tmp_working_dir(runtime):
if runtime.remove_tmp_working_dir:
shutil.rmtree(runtime.working_dir)
else:
click.echo("Temporary working directory preserved by operation: %s" % runtime.working_dir)
class WrapException(Exception):
""" https://bugs.python.org/issue13831 """
def __init__(self):
super(WrapException, self).__init__()
exc_type, exc_value, exc_tb = sys.exc_info()
self.exception = exc_value
self.formatted = "".join(
traceback.format_exception(exc_type, exc_value, exc_tb))
@as_native_str()
def __str__(self):
return "{}\nOriginal traceback:\n{}".format(Exception.__str__(self), self.formatted)
def wrap_exception(func):
""" Decorate a function, wrap exception if it occurs. """
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception:
raise WrapException()
return wrapper
def _unpack_tuple_args(func):
""" Decorate a function for unpacking the tuple argument `args`
This is used to workaround Python 3 lambda not unpacking tuple arguments (PEP-3113)
"""
@functools.wraps(func)
def wrapper(args):
return func(*args)
return wrapper
# A named tuple for caching the result of Runtime._resolve_source.
SourceResolution = namedtuple('SourceResolution', [
'source_path', 'url', 'branch', 'public_upstream_url', 'public_upstream_branch'
])
class Runtime(object):
# Use any time it is necessary to synchronize feedback from multiple threads.
mutex = RLock()
# Serialize access to the shared koji session
koji_lock = RLock()
# Build status detector lock
bs_lock = RLock()
# Serialize access to the console, and record log
log_lock = Lock()
def __init__(self, **kwargs):
# initialize defaults in case no value is given
self.verbose = False
self.quiet = False
self.load_wip = False
self.load_disabled = False
self.data_path = None
self.data_dir = None
self.latest_parent_version = False
self.rhpkg_config = None
self._koji_client_session = None
self.db = None
self.session_pool = {}
self.session_pool_available = {}
self.brew_event = None
self.assembly_basis_event = None
self.assembly_type = None
self.releases_config = None
self.assembly = 'test'
self._build_status_detector = None
self.disable_gssapi = False
self._build_data_product_cache: Model = None
self.stream: List[str] = [] # Click option. A list of image stream overrides from the command line.
self.stream_overrides: Dict[str, str] = {} # Dict of stream name -> pullspec from command line.
self.upstreams: List[str] = [] # Click option. A list of upstream source commit to use.
self.upstream_commitish_overrides: Dict[str, str] = {} # Dict from distgit key name to upstream source commit to use.
self.downstreams: List[str] = [] # Click option. A list of distgit commits to checkout.
self.downstream_commitish_overrides: Dict[str, str] = {} # Dict from distgit key name to distgit commit to check out.
# See get_named_semaphore. The empty string key serves as a lock for the data structure.
self.named_semaphores = {'': Lock()}
for key, val in kwargs.items():
self.__dict__[key] = val
if self.latest_parent_version:
self.ignore_missing_base = True
self._remove_tmp_working_dir = False
self.group_config = None
self.cwd = os.getcwd()
# If source needs to be cloned by oit directly, the directory in which it will be placed.
self.sources_dir = None
self.distgits_dir = None
self.record_log = None
self.record_log_path = None
self.debug_log_path = None
self.brew_logs_dir = None
self.flags_dir = None
# Map of dist-git repo name -> ImageMetadata object. Populated when group is set.
self.image_map: Dict[str, ImageMetadata] = {}
# Map of dist-git repo name -> RPMMetadata object. Populated when group is set.
self.rpm_map: Dict[str, RPMMetadata] = {}
# Maps component name to the Image or RPM Metadata responsible for the component
self.component_map: Dict[str, Union[ImageMetadata, RPMMetadata]] = dict()
# Map of source code repo aliases (e.g. "ose") to a tuple representing the source resolution cache.
# See registry_repo.
self.source_resolutions = {}
# Map of source code repo aliases (e.g. "ose") to a (public_upstream_url, public_upstream_branch) tuple.
# See registry_repo.
self.public_upstreams = {}
self.initialized = False
# Will be loaded with the streams.yml Model
self.streams = Model(dict_to_model={})
self.uuid = None
# Optionally available if self.fetch_rpms_for_tag() is called
self.rpm_list = None
self.rpm_search_tree = None
# Used for image build ordering
self.image_tree = {}
self.image_order = []
# allows mapping from name or distgit to meta
self.image_name_map = {}
# allows mapping from name in bundle to meta
self.name_in_bundle_map: Dict[str, ImageMetadata] = {}
# holds untouched group config
self.raw_group_config = {}
# Used to capture missing packages for 4.x build
self.missing_pkgs = set()
# Whether to prevent builds for this group. Defaults to 'no'.
self.freeze_automation = FREEZE_AUTOMATION_NO
self.rhpkg_config_lst = []
if self.rhpkg_config:
if not os.path.isfile(self.rhpkg_config):
raise DoozerFatalError('--rhpkg-config option given is not a valid file! {}'.format(self.rhpkg_config))
self.rhpkg_config = ' --config {} '.format(self.rhpkg_config)
self.rhpkg_config_lst = self.rhpkg_config.split()
else:
self.rhpkg_config = ''
def get_named_semaphore(self, lock_name, is_dir=False, count=1):
"""
Returns a semaphore (which can be used as a context manager). The first time a lock_name
is received, a new semaphore will be established. Subsequent uses of that lock_name will
receive the same semaphore.
:param lock_name: A unique name for resource threads are contending over. If using a directory name
as a lock_name, provide an absolute path.
:param is_dir: The lock_name is a directory (method will ignore things like trailing slashes)
:param count: The number of times the lock can be claimed. Default=1, which is a full mutex.
:return: A semaphore associated with the lock_name.
"""
with self.named_semaphores['']:
if is_dir:
p = '_dir::' + str(pathlib.Path(str(lock_name)).absolute()) # normalize (e.g. strip trailing /)
else:
p = lock_name
if p in self.named_semaphores:
return self.named_semaphores[p]
else:
new_semaphore = Semaphore(count)
self.named_semaphores[p] = new_semaphore
return new_semaphore
def get_releases_config(self):
if self.releases_config is not None:
return self.releases_config
load = self.gitdata.load_data(key='releases')
data = load.data if load else {}
if self.releases: # override filename specified on command line.
rcp = pathlib.Path(self.releases)
data = yaml.safe_load(rcp.read_text())
if load:
self.releases_config = Model(data)
else:
self.releases_config = Model()
return self.releases_config
def get_group_config(self) -> Model:
# group.yml can contain a `vars` section which should be a
# single level dict containing keys to str.format(**dict) replace
# into the YAML content. If `vars` found, the format will be
# preformed and the YAML model will reloaded from that result
tmp_config = Model(self.gitdata.load_data(key='group').data)
replace_vars = self._get_replace_vars(tmp_config)
try:
group_yml = yaml.safe_dump(tmp_config.primitive(), default_flow_style=False)
raw_group_config = yaml.full_load(group_yml.format(**replace_vars))
tmp_config = Model(dict(raw_group_config))
except KeyError as e:
raise ValueError('group.yml contains template key `{}` but no value was provided'.format(e.args[0]))
return assembly_group_config(self.get_releases_config(), self.assembly, tmp_config)
def _get_replace_vars(self, group_config: Model):
replace_vars = group_config.vars or Model()
# If assembly mode is enabled, `runtime_assembly` will become the assembly name.
replace_vars['runtime_assembly'] = ''
# If running against an assembly for a named release, release_name will become the release name.
replace_vars['release_name'] = ''
if self.assembly:
replace_vars['runtime_assembly'] = self.assembly
if self.assembly_type is not AssemblyTypes.STREAM:
replace_vars['release_name'] = util.get_release_name_for_assembly(self.group, self.get_releases_config(), self.assembly)
return replace_vars
def init_state(self):
self.state = dict(state.TEMPLATE_BASE_STATE)
if os.path.isfile(self.state_file):
with io.open(self.state_file, 'r', encoding='utf-8') as f:
self.state = yaml.full_load(f)
self.state.update(state.TEMPLATE_BASE_STATE)
def save_state(self):
with io.open(self.state_file, 'w', encoding='utf-8') as f:
yaml.safe_dump(self.state, f, default_flow_style=False)
def initialize(self, mode='images', clone_distgits=True,
validate_content_sets=False,
no_group=False, clone_source=None, disabled=None,
prevent_cloning: bool = False, config_only: bool = False):
if self.initialized:
return
if self.quiet and self.verbose:
click.echo("Flags --quiet and --verbose are mutually exclusive")
exit(1)
self.mode = mode
# We could mark these as required and the click library would do this for us,
# but this seems to prevent getting help from the various commands (unless you
# specify the required parameters). This can probably be solved more cleanly, but TODO
if not no_group and self.group is None:
click.echo("Group must be specified")
exit(1)
if self.lock_runtime_uuid:
self.uuid = self.lock_runtime_uuid
else:
self.uuid = datetime.datetime.now().strftime("%Y%m%d.%H%M%S")
if self.working_dir is None:
self.working_dir = tempfile.mkdtemp(".tmp", "oit-")
# This can be set to False by operations which want the working directory to be left around
self.remove_tmp_working_dir = True
atexit.register(remove_tmp_working_dir, self)
else:
self.working_dir = os.path.abspath(os.path.expanduser(self.working_dir))
if not os.path.isdir(self.working_dir):
os.makedirs(self.working_dir)
self.distgits_dir = os.path.join(self.working_dir, "distgits")
self.distgits_diff_dir = os.path.join(self.working_dir, "distgits-diffs")
self.sources_dir = os.path.join(self.working_dir, "sources")
self.record_log_path = os.path.join(self.working_dir, "record.log")
self.brew_logs_dir = os.path.join(self.working_dir, "brew-logs")
self.flags_dir = os.path.join(self.working_dir, "flags")
self.state_file = os.path.join(self.working_dir, 'state.yaml')
self.debug_log_path = os.path.join(self.working_dir, "debug.log")
if self.upcycle:
# A working directory may be upcycle'd numerous times.
# Don't let anything grow unbounded.
shutil.rmtree(self.brew_logs_dir, ignore_errors=True)
shutil.rmtree(self.flags_dir, ignore_errors=True)
for path in (self.record_log_path, self.state_file, self.debug_log_path):
if os.path.exists(path):
os.unlink(path)
if not os.path.isdir(self.distgits_dir):
os.mkdir(self.distgits_dir)
if not os.path.isdir(self.distgits_diff_dir):
os.mkdir(self.distgits_diff_dir)
if not os.path.isdir(self.sources_dir):
os.mkdir(self.sources_dir)
if disabled is not None:
self.load_disabled = disabled
self.initialize_logging()
self.init_state()
try:
self.db = dblib.DB(self, self.datastore)
except Exception as err:
self.logger.warning('Cannot connect to the DB: %s\n%s', str(err), traceback.format_exc())
self.logger.info(f'Initial execution (cwd) directory: {os.getcwd()}')
if no_group:
return # nothing past here should be run without a group
if '@' in self.group:
self.group, self.group_commitish = self.group.split('@', 1)
else:
self.group_commitish = self.group
# For each "--stream alias image" on the command line, register its existence with
# the runtime.
for s in self.stream:
self.register_stream_override(s[0], s[1])
for upstream in self.upstreams:
override_distgit_key = upstream[0]
override_commitish = upstream[1]
self.logger.warning(f'Upstream source for {override_distgit_key} being set to {override_commitish}')
self.upstream_commitish_overrides[override_distgit_key] = override_commitish
for upstream in self.downstreams:
override_distgit_key = upstream[0]
override_commitish = upstream[1]
self.logger.warning(f'Downstream distgit for {override_distgit_key} will be checked out to {override_commitish}')
self.downstream_commitish_overrides[override_distgit_key] = override_commitish
self.resolve_metadata()
self.record_log = io.open(self.record_log_path, 'a', encoding='utf-8')
atexit.register(close_file, self.record_log)
# Directory where brew-logs will be downloaded after a build
if not os.path.isdir(self.brew_logs_dir):
os.mkdir(self.brew_logs_dir)
# Directory for flags between invocations in the same working-dir
if not os.path.isdir(self.flags_dir):
os.mkdir(self.flags_dir)
if self.cache_dir:
self.cache_dir = os.path.abspath(self.cache_dir)
# get_releases_config also inits self.releases_config
self.assembly_type = assembly_type(self.get_releases_config(), self.assembly)
self.group_dir = self.gitdata.data_dir
self.group_config = self.get_group_config()
self.hotfix = False # True indicates builds should be tagged with associated hotfix tag for the artifacts branch
if self.group_config.assemblies.enabled or self.enable_assemblies:
if re.fullmatch(r'[\w.]+', self.assembly) is None or self.assembly[0] == '.' or self.assembly[-1] == '.':
raise ValueError('Assembly names may only consist of alphanumerics, ., and _, but not start or end with a dot (.).')
else:
# If assemblies are not enabled for the group,
# ignore this argument throughout doozer.
self.assembly = None
replace_vars = self._get_replace_vars(self.group_config).primitive()
# only initialize group and assembly configs and nothing else
if config_only:
return
# Read in the streams definition for this group if one exists
streams_data = self.gitdata.load_data(key='streams', replace_vars=replace_vars)
if streams_data:
org_stream_model = Model(dict_to_model=streams_data.data)
self.streams = assembly_streams_config(self.get_releases_config(), self.assembly, org_stream_model)
self.assembly_basis_event = assembly_basis_event(self.get_releases_config(), self.assembly)
if self.assembly_basis_event:
if self.brew_event:
raise IOError(f'Cannot run with assembly basis event {self.assembly_basis_event} and --brew-event at the same time.')
# If the assembly has a basis event, we constrain all brew calls to that event.
self.brew_event = self.assembly_basis_event
self.logger.warning(f'Constraining brew event to assembly basis for {self.assembly}: {self.brew_event}')
# This flag indicates builds should be tagged with associated hotfix tag for the artifacts branch
self.hotfix = self.assembly_type is not AssemblyTypes.STREAM
if not self.brew_event:
self.logger.info("Basis brew event is not set. Using the latest event....")
with self.shared_koji_client_session() as koji_session:
# If brew event is not set as part of the assembly and not specified on the command line,
# lock in an event so that there are no race conditions.
self.logger.info("Getting the latest event....")
event_info = koji_session.getLastEvent()
self.brew_event = event_info['id']
# register the sources
# For each "--source alias path" on the command line, register its existence with
# the runtime.
for r in self.source:
self.register_source_alias(r[0], r[1])
if self.sources:
with io.open(self.sources, 'r', encoding='utf-8') as sf:
source_dict = yaml.full_load(sf)
if not isinstance(source_dict, dict):
raise ValueError('--sources param must be a yaml file containing a single dict.')
for key, val in source_dict.items():
self.register_source_alias(key, val)
with Dir(self.group_dir):
# Flattens multiple comma/space delimited lists like [ 'x', 'y,z' ] into [ 'x', 'y', 'z' ]
def flatten_list(names):
if not names:
return []
# split csv values
result = []
for n in names:
result.append([x for x in n.replace(' ', ',').split(',') if x != ''])
# flatten result and remove dupes using set
return list(set([y for x in result for y in x]))
def filter_wip(n, d):
return d.get('mode', 'enabled') in ['wip', 'enabled']
def filter_enabled(n, d):
return d.get('mode', 'enabled') == 'enabled'
def filter_disabled(n, d):
return d.get('mode', 'enabled') in ['enabled', 'disabled']
cli_arches_override = flatten_list(self.arches)
if cli_arches_override: # Highest priority overrides on command line
self.arches = cli_arches_override
elif self.group_config.arches_override: # Allow arches_override in group.yaml to temporarily override GA architectures
self.arches = self.group_config.arches_override
else:
self.arches = self.group_config.get('arches', ['x86_64'])
# If specified, signed repo files will be generated to enforce signature checks.
self.gpgcheck = self.group_config.build_profiles.image.signed.gpgcheck
if self.gpgcheck is Missing:
# We should only really be building the latest release with unsigned RPMs, so default to True
self.gpgcheck = True
self.repos = Repos(self.group_config.repos, self.arches, self.gpgcheck)
self.freeze_automation = self.group_config.freeze_automation or FREEZE_AUTOMATION_NO
if validate_content_sets:
self.repos.validate_content_sets()
if self.group_config.name != self.group:
raise IOError(
f"Name in group.yml ({self.group_config.name}) does not match group name ({self.group}). Someone "
"may have copied this group without updating group.yml (make sure to check branch)")
if self.branch is None:
if self.group_config.branch is not Missing:
self.branch = self.group_config.branch
self.logger.info("Using branch from group.yml: %s" % self.branch)
else:
self.logger.info("No branch specified either in group.yml or on the command line; all included images will need to specify their own.")
else:
self.logger.info("Using branch from command line: %s" % self.branch)
scanner = self.group_config.image_build_log_scanner
if scanner is not Missing:
# compile regexen and fail early if they don't
regexen = []
for val in scanner.matches:
try:
regexen.append(re.compile(val))
except Exception as e:
raise ValueError(
"could not compile image build log regex for group:\n{}\n{}"
.format(val, e)
)
scanner.matches = regexen
exclude_keys = flatten_list(self.exclude)
image_ex = list(exclude_keys)
rpm_ex = list(exclude_keys)
image_keys = flatten_list(self.images)
rpm_keys = flatten_list(self.rpms)
filter_func = None
if self.load_wip and self.load_disabled:
pass # use no filter, load all
elif self.load_wip:
filter_func = filter_wip
elif self.load_disabled:
filter_func = filter_disabled
else:
filter_func = filter_enabled
# pre-load the image data to get the names for all images
# eventually we can use this to allow loading images by
# name or distgit. For now this is used elsewhere
image_name_data = self.gitdata.load_data(path='images')
def _register_name_in_bundle(name_in_bundle: str, distgit_key: str):
if name_in_bundle in self.name_in_bundle_map:
raise ValueError(f"Image {distgit_key} has name_in_bundle={name_in_bundle}, which is already taken by image {self.name_in_bundle_map[name_in_bundle]}")
self.name_in_bundle_map[name_in_bundle] = img.key
for img in image_name_data.values():
name = img.data.get('name')
short_name = name.split('/')[1]
self.image_name_map[name] = img.key
self.image_name_map[short_name] = img.key
name_in_bundle = img.data.get('name_in_bundle')
if name_in_bundle:
_register_name_in_bundle(name_in_bundle, img.key)
else:
short_name_without_ose = short_name[4:] if short_name.startswith("ose-") else short_name
_register_name_in_bundle(short_name_without_ose, img.key)
short_name_with_ose = "ose-" + short_name_without_ose
_register_name_in_bundle(short_name_with_ose, img.key)
image_data = self.gitdata.load_data(path='images', keys=image_keys,
exclude=image_ex,
replace_vars=replace_vars,
filter_funcs=None if len(image_keys) else filter_func)
try:
rpm_data = self.gitdata.load_data(path='rpms', keys=rpm_keys,
exclude=rpm_ex,
replace_vars=replace_vars,
filter_funcs=None if len(rpm_keys) else filter_func)
except gitdata.GitDataPathException:
# some older versions have no RPMs, that's ok.
rpm_data = {}
missed_include = set(image_keys + rpm_keys) - set(list(image_data.keys()) + list(rpm_data.keys()))
if len(missed_include) > 0:
raise DoozerFatalError('The following images or rpms were either missing or filtered out: {}'.format(', '.join(missed_include)))
if mode in ['images', 'both']:
for i in image_data.values():
if i.key not in self.image_map:
metadata = ImageMetadata(self, i, self.upstream_commitish_overrides.get(i.key), clone_source=clone_source, prevent_cloning=prevent_cloning)
self.image_map[metadata.distgit_key] = metadata
self.component_map[metadata.get_component_name()] = metadata
if not self.image_map:
self.logger.warning("No image metadata directories found for given options within: {}".format(self.group_dir))
for image in self.image_map.values():
image.resolve_parent()
# now that ancestry is defined, make sure no cyclic dependencies
for image in self.image_map.values():
for child in image.children:
if image.is_ancestor(child):
raise DoozerFatalError('{} cannot be both a parent and dependent of {}'.format(child.distgit_key, image.distgit_key))
self.generate_image_tree()
if mode in ['rpms', 'both']:
for r in rpm_data.values():
if clone_source is None:
# Historically, clone_source defaulted to True for rpms.
clone_source = True
metadata = RPMMetadata(self, r, self.upstream_commitish_overrides.get(r.key), clone_source=clone_source, prevent_cloning=prevent_cloning)
self.rpm_map[metadata.distgit_key] = metadata
self.component_map[metadata.get_component_name()] = metadata
if not self.rpm_map:
self.logger.warning("No rpm metadata directories found for given options within: {}".format(self.group_dir))
# Make sure that the metadata is not asking us to check out the same exact distgit & branch.
# This would almost always indicate someone has checked in duplicate metadata into a group.
no_collide_check = {}
for meta in list(self.rpm_map.values()) + list(self.image_map.values()):
key = '{}/{}/#{}'.format(meta.namespace, meta.name, meta.branch())
if key in no_collide_check:
raise IOError('Complete duplicate distgit & branch; something wrong with metadata: {} from {} and {}'.format(key, meta.config_filename, no_collide_check[key].config_filename))
no_collide_check[key] = meta
if clone_distgits:
self.clone_distgits()
self.initialized = True
def initialize_logging(self):
if self.initialized:
return
# Three flags control the output modes of the command:
# --verbose prints logs to CLI as well as to files
# --debug increases the log level to produce more detailed internal
# behavior logging
# --quiet opposes both verbose and debug
if self.debug:
log_level = logging.DEBUG
elif self.quiet:
log_level = logging.WARN
else:
log_level = logging.INFO
default_log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
root_logger = logging.getLogger()
root_logger.setLevel(logging.WARN)
root_stream_handler = logging.StreamHandler()
root_stream_handler.setFormatter(default_log_formatter)
root_logger.addHandler(root_stream_handler)
# If in debug mode, let all modules log
if not self.debug:
# Otherwise, only allow children of ocp to log
root_logger.addFilter(logging.Filter("ocp"))
# Get a reference to the logger for doozer
self.logger = logutil.getLogger()
self.logger.propagate = False
# levels will be set at the handler level. Make sure master level is low.
self.logger.setLevel(logging.DEBUG)
main_stream_handler = logging.StreamHandler()
main_stream_handler.setFormatter(default_log_formatter)
main_stream_handler.setLevel(log_level)
self.logger.addHandler(main_stream_handler)
debug_log_handler = logging.FileHandler(self.debug_log_path)
# Add thread information for debug log
debug_log_handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s (%(thread)d) %(message)s'))
debug_log_handler.setLevel(logging.DEBUG)
self.logger.addHandler(debug_log_handler)
def build_jira_client(self) -> JIRA:
"""
:return: Returns a JIRA client setup for the server in bug.yaml
"""
major, minor = self.get_major_minor_fields()
if major == 4 and minor < 6:
raise ValueError("ocp-build-data/bug.yml is not expected to be available for 4.X versions < 4.6")
bug_config = Model(self.gitdata.load_data(key='bug').data)
server = bug_config.jira_config.server or 'https://issues.redhat.com'
token_auth = os.environ.get("JIRA_TOKEN")
if not token_auth:
raise ValueError(f"Jira activity requires login credentials for {server}. Set a JIRA_TOKEN env var")
client = JIRA(server, token_auth=token_auth)
return client
def build_retrying_koji_client(self):
"""
:return: Returns a new koji client instance that will automatically retry
methods when it receives common exceptions (e.g. Connection Reset)
Honors doozer --brew-event.
"""
return brew.KojiWrapper([self.group_config.urls.brewhub], brew_event=self.brew_event)
@contextmanager
def shared_koji_client_session(self):
"""
Context manager which offers a shared koji client session. You hold a koji specific lock in this context
manager giving your thread exclusive access. The lock is reentrant, so don't worry about
call a method that acquires the same lock while you hold it.
Honors doozer --brew-event.
Do not rerun gssapi_login on this client. We've observed client instability when this happens.
"""
with self.koji_lock:
if self._koji_client_session is None:
self._koji_client_session = self.build_retrying_koji_client()
if not self.disable_gssapi:
self.logger.info("Authenticating to Brew...")
self._koji_client_session.gssapi_login()
yield self._koji_client_session
@contextmanager
def shared_build_status_detector(self) -> 'BuildStatusDetector':
"""
Yields a shared build status detector within context.
"""
with self.bs_lock:
if self._build_status_detector is None:
self._build_status_detector = BuildStatusDetector(self, self.logger)
yield self._build_status_detector
@contextmanager
def pooled_koji_client_session(self, caching: bool = False):
"""
Context manager which offers a koji client session from a limited pool. You hold a lock on this
session until you return. It is not recommended to call other methods that acquire their
own pooled sessions, because that may lead to deadlock if the pool is exhausted.
Honors doozer --brew-event.
:param caching: Set to True in order for your instance to place calls/results into
the global KojiWrapper cache. This is equivalent to passing
KojiWrapperOpts(caching=True) in each call within the session context.
"""
session = None
session_id = None
while True:
with self.mutex:
if len(self.session_pool_available) == 0:
if len(self.session_pool) < 30:
# pool has not grown to max size;
new_session = self.build_retrying_koji_client()
session_id = len(self.session_pool)
self.session_pool[session_id] = new_session
session = new_session # This is what we wil hand to the caller
break
else:
# Caller is just going to have to wait and try again
pass
else:
session_id, session = self.session_pool_available.popitem()
break
time.sleep(5)
# Arriving here, we have a session to use.
try:
session.force_instance_caching = caching
yield session
finally:
session.force_instance_caching = False
# Put it back into the pool
with self.mutex:
self.session_pool_available[session_id] = session
@staticmethod
def timestamp():
return datetime.datetime.utcnow().isoformat()
def assert_mutation_is_permitted(self):
"""
In group.yml, it is possible to instruct doozer to prevent all builds / mutation of distgits.
Call this method if you are about to mutate anything. If builds are disabled, an exception will
be thrown.
"""
if self.freeze_automation == FREEZE_AUTOMATION_YES:
raise DoozerFatalError('Automation (builds / mutations) for this group is currently frozen (freeze_automation set to {}). Coordinate with the group owner to change this if you believe it is incorrect.'.format(FREEZE_AUTOMATION_YES))
def image_metas(self) -> List[ImageMetadata]:
return list(self.image_map.values())
def ordered_image_metas(self) -> List[ImageMetadata]:
return [self.image_map[dg] for dg in self.image_order]
def get_global_arches(self):
"""
:return: Returns a list of architectures that are enabled globally in group.yml.
"""
return list(self.arches)
def get_product_config(self) -> Model:
"""
Returns a Model of the product.yml in ocp-build-data main branch.
"""
if self._build_data_product_cache:
return self._build_data_product_cache
url = 'https://raw.githubusercontent.com/openshift/ocp-build-data/main/product.yml'
req = urllib.request.Request(url)
req.add_header('Accept', 'application/yaml')
self._build_data_product_cache = Model(yaml.safe_load(exectools.urlopen_assert(req).read()))
return self._build_data_product_cache
def filter_failed_image_trees(self, failed):
for i in self.ordered_image_metas():
if i.parent and i.parent.distgit_key in failed:
failed.append(i.distgit_key)
for f in failed:
if f in self.image_map:
del self.image_map[f]
# regen order and tree
self.generate_image_tree()
return failed
def generate_image_tree(self):
self.image_tree = {}
image_lists = {0: []}
def add_child_branch(child, branch, level=1):
if level not in image_lists:
image_lists[level] = []
for sub_child in child.children:
if sub_child.distgit_key not in self.image_map:
continue # don't add images that have been filtered out
branch[sub_child.distgit_key] = {}
image_lists[level].append(sub_child.distgit_key)
add_child_branch(sub_child, branch[sub_child.distgit_key], level + 1)
for image in self.image_map.values():
if not image.parent:
self.image_tree[image.distgit_key] = {}
image_lists[0].append(image.distgit_key)
add_child_branch(image, self.image_tree[image.distgit_key])
levels = list(image_lists.keys())
levels.sort()
self.image_order = []
for level in levels:
for i in image_lists[level]:
if i not in self.image_order:
self.image_order.append(i)
def image_distgit_by_name(self, name):
"""Returns image meta by full name, short name, or distgit"""
return self.image_name_map.get(name, None)
def rpm_metas(self) -> List[RPMMetadata]:
return list(self.rpm_map.values())
def all_metas(self) -> List[Union[ImageMetadata, RPMMetadata]]:
return self.image_metas() + self.rpm_metas()
def get_payload_image_metas(self) -> List[ImageMetadata]:
"""
:return: Returns a list of ImageMetadata that are destined for the OCP release payload. Payload images must
follow the correct naming convention or an exception will be thrown.
"""
payload_images = []
for image_meta in self.image_metas():
if image_meta.is_payload:
"""
<Tim Bielawa> note to self: is only for `ose-` prefixed images
<Clayton Coleman> Yes, Get with the naming system or get out of town
"""
if not image_meta.image_name_short.startswith("ose-"):
raise ValueError(f"{image_meta.distgit_key} does not conform to payload naming convention with image name: {image_meta.image_name_short}")
payload_images.append(image_meta)
return payload_images
def get_for_release_image_metas(self) -> List[ImageMetadata]:
"""
:return: Returns a list of ImageMetada which are configured to be released by errata.
"""
return filter(lambda meta: meta.for_release, self.image_metas())
def get_non_release_image_metas(self) -> List[ImageMetadata]:
"""
:return: Returns a list of ImageMetada which are not meant to be released by errata.
"""
return filter(lambda meta: not meta.for_release, self.image_metas())
def register_source_alias(self, alias, path):
self.logger.info("Registering source alias %s: %s" % (alias, path))
path = os.path.abspath(path)
assertion.isdir(path, "Error registering source alias %s" % alias)
with Dir(path):
url = None
origin_url = "?"
rc1, out_origin, err_origin = exectools.cmd_gather(
["git", "config", "--get", "remote.origin.url"])
if rc1 == 0:
url = out_origin.strip()
origin_url = url
# Usually something like "git@github.com:openshift/origin.git"
# But we want an https hyperlink like http://github.com/openshift/origin
if origin_url.startswith("git@"):
origin_url = origin_url[4:] # remove git@
origin_url = origin_url.replace(":", "/", 1) # replace first colon with /
if origin_url.endswith(".git"):
origin_url = origin_url[:-4] # remove .git
origin_url = "https://%s" % origin_url
else:
self.logger.error("Failed acquiring origin url for source alias %s: %s" % (alias, err_origin))
branch = None
rc2, out_branch, err_branch = exectools.cmd_gather(
["git", "rev-parse", "--abbrev-ref", "HEAD"])
if rc2 == 0:
branch = out_branch.strip()
else:
self.logger.error("Failed acquiring origin branch for source alias %s: %s" % (alias, err_branch))
if self.group_config.public_upstreams:
if not (url and branch):
raise DoozerFatalError(f"Couldn't detect source URL or branch for local source {path}. Is it a valid Git repo?")
public_upstream_url, public_upstream_branch = self.get_public_upstream(url)
if branch == 'HEAD':
# If branch == HEAD, our source is a detached HEAD.
public_upstream_url = None
public_upstream_branch = None
else:
if not public_upstream_branch:
public_upstream_branch = branch
self.source_resolutions[alias] = SourceResolution(path, url, branch, public_upstream_url, public_upstream_branch)
else:
self.source_resolutions[alias] = SourceResolution(path, url, branch, None, None)
if 'source_alias' not in self.state:
self.state['source_alias'] = {}
self.state['source_alias'][alias] = {
'url': origin_url,
'branch': branch or '?',
'path': path