-
Notifications
You must be signed in to change notification settings - Fork 36
/
merge-all-kits.py
executable file
·848 lines (724 loc) · 40.5 KB
/
merge-all-kits.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
#!/usr/bin/python3
from merge_utils import *
from datetime import datetime
import json
from collections import defaultdict
from enum import Enum
import os
# KIT DESIGN AND DEVELOPER DOCS
# The maintainable model for kits is to have several source repositories that contain most of our source ebuilds/
# catpkgs, which are identified by SHA1 to point to a specific snapshot. Then, we combine that with a Funtoo 'kit-fixups'
# repository that contains only our forked ebuilds. Then this script, merge-all-kits.py, is used to automatically
# generate the kits. We don't commit directly to kits themselves -- this script automatically generates commits with
# updated ebuilds.
# A kit is generated from:
# 1. a collection of repositories and SHA1 commit to specify a snapshot of each repository to serve as a source for catpkgs,
# eclasses, licenses. It is also possible to specify a branch name instead of SHA1 (typically 'master') although this
# shouldn't ever be done for 'prime' branches of kits.
# 1. a selection of catpkgs (ebuilds) that are selected from source repositories. Each kit has a package-set file located
# in ../package-sets/*-kit relative to this file which contains patterns of catpkgs to select from each source
# repository and copy into the kit when regenerating it.
# 3. a collection of fix-ups (from the kit-fixups repository) that can be used to replace catpkgs in various kits globally,
# or in a specific branch of a kit. There is also an option to provide eclasses that get copied globally to each kit,
# to a particular kit, or to a branch of a particular kit. This is the where we fork ebuilds to fix specific issues.
# Below, the kits and branches should be defined in a way that includes all this information. It is also possible to
# have a kit that simply is a collection of ebuilds but tracks the latest gentoo-staging. It may or may not have
# additional fix-ups.
# When setting up a kit repository, the 'master' branch may used to store an 'unfrozen' kit that just tracks upstream
# Gentoo. Kits are not required to have a master branch -- we only create one if the kit is designed to offer unfrozen
# ebuilds to Funtoo users. Examples below are: science-kit, games-kit, text-kit, net-kit. These track gentoo.
# If we have a frozen enterprise branch that we are backporting security fixes to only, we want this to be an
# 'x.y-prime' branch. This branch's overlays' source SHA1s are not supposed to change and we will just augment it with
# fix-ups as needed.
# As kits are maintained, the following things may change:
#
# 1. The package-set files may change. This can result in different packages being selected for the kit the next time it
# is regenerated by this script. We can add mising packages, decide to move packages to other kits, etc. This script
# takes care of ensuring that all necessary eclasses and licenses are included when the kit is regenerated.
#
# 2. The fix-ups may change. This allows us to choose to 'fork' various ebuilds that we may need to fix, while keeping
# our changes separate from the source packages. We can also choose to unfork packages.
#
# 3. Kits can be added or removed.
#
# 4. Kit branches can be created, or alternatively deprecated. We need a system for gracefully deprecating a kit that does
# not involve deleting the branch. A user may decide to continue using the branch even if it has been deprecated.
#
# 5. Kits can be tagged by Funtoo as being mandatory or optional. Typically, most kits will be mandatory but some effort
# will be made as we progress to make things like the games-kit or the science-kit optional.
#
# HOW KITS ARE GENERATED
# Currently, kits are regenerated in a particluar order, such as: "first, core-kit, then security-kit, then perl-kit",
# etc. This script keeps a running list of catpkgs that are inserted into each kit. Once a catpkg is inserted into a
# kit, it is not available to be inserted into successive kits. This design is intended to prevent multiple copies of
# catpkgs existing in multiple kits in parallel that are designed to work together as a set. At the end of kit
# generation, this master list of inserted catpkgs is used to prune the 'nokit' repository of catpkgs, so that 'nokit'
# contains the set of all ebuilds that were not inserted into kits.
# Below, you will see how the sources for kits are defined.
# 1. OVERLAYS - lists sources for catpkgs, along with properties which can include "select" - a list of catpkgs to
# include. When "select" is specified, only these catpkgs will be available for selection by the package-set rules. .
# If no "select" is specified, then by default all available catpkgs could be included, if they match patterns, etc. in
# package-sets. Note that we do not specify branch or SHA1 here. This may vary based on kit, so it's specified elsewhere
# (see KIT SOURCES, below.)
overlays = {
# use gentoo-staging-2017 dirname to avoid conflicts with ports-2012 generation
"gentoo-staging" : { "type" : GitTree, "url" : "repos@git.funtoo.org:ports/gentoo-staging.git", "dirname" : "gentoo-staging-2017" },
"faustoo" : { "type" : GitTree, "url" : "https://github.com/fmoro/faustoo.git", "eclasses" : [
"waf",
"googlecode"
],
# SKIP any catpkgs that also exist in gentoo-staging (like nvidia-drivers). All others will be copied.
"filter" : [ "gentoo-staging" ]
},
"fusion809" : { "type" : GitTree, "url" : "https://github.com/fusion809/fusion809-overlay.git", "select" : [
"app-editors/atom-bin",
"app-editors/notepadqq",
"app-editors/bluefish",
"app-editors/textadept",
"app-editors/scite",
"app-editors/gvim",
"app-editors/vim",
"app-editors/vim-core",
"app-editors/sublime-text"
]
}, # FL-3633, FL-3663, FL-3776
"plex" : { "type" : GitTree, "url" : "https://github.com/Ghent/funtoo-plex.git", "select" : [
"media-tv/plex-media-server",
],
},
# Ryan Harris glassfish overlay. FL-3985:
"rh1" : { "type" : GitTree, "url" : "https://github.com/x48rph/glassfish.git", "select" : [
"www-servers/glassfish-bin",
],
},
# damex's deadbeef (music player like foobar2000) overlay
"deadbeef" : { "type" : GitTree, "url" : "https://github.com/damex/deadbeef-overlay.git", "copyfiles" : {
"profiles/package.mask": "profiles/package.mask/deadbeef.mask"
},
},
# damex's wmfs (window manager from scratch) overlay
"wmfs" : { "type" : GitTree, "url" : "https://github.com/damex/wmfs-overlay.git", "copyfiles" : {
"profiles/package.mask": "profiles/package.mask/wmfs.mask"
},
},
"flora" : { "type" : GitTree, "url" : "https://github.com/funtoo/flora.git", "copyfiles" : {
"licenses/renoise-EULA": "licenses/renoise-EULA"
},
},
}
# SUPPLEMENTAL REPOSITORIES: These are overlays that we are using but are not in KIT SOURCES. merge_scripts is something
# we are using only for profiles and other misc. things and may get phased out in the future:
merge_scripts = GitTree("merge-scripts", "master", "git@github.com:funtoo/merge-scripts.git")
fixup_repo = GitTree("kit-fixups", "master", "git@github.com:funtoo/kit-fixups.git")
# OUTPUT META-REPO: This is the master repository being written to.
meta_repo = GitTree("meta-repo", "master", "repos@git.funtoo.org:meta-repo.git", root="/var/git/dest-trees/meta-repo")
# 2. KIT SOURCES - kit sources are a combination of overlays, arranged in a python list [ ]. A KIT SOURCE serves as a
# unified collection of source catpkgs for a particular kit. Each kit can have one KIT SOURCE. KIT SOURCEs MAY be
# shared among kits to avoid duplication and to help organization. Note that this is where we specify branch or SHA1
# for each overlay.
# Each kit source can be used as a source of catpkgs for a kit. Order is important -- package-set rules are applied in
# the same order that the overlay appears in the kit_source_defs list -- so for "funtoo_current", package-set rules will
# be applied to gentoo-staging first, then flora, then faustoo, then fusion809. Once a particular catpkg matches and is
# copied into a dest-kit, a matching capkg in a later overlay, if one exists, will be ignored.
# It is important to note that we support two kinds of kit sources -- the first is the gentoo-staging master repository
# which contains a master set of eclasses and contains everything it needs for all the catpkgs it contains. The second
# kind of repository we support is an overlay that is designed to be used with the gentoo-staging overlay, so it may
# need some catpkgs (as dependencies) or eclasses from gentoo-staging. The gentoo-staging repository should always
# appear as the first item in kit_source_defs, with the overlays appearing after.
kit_source_defs = {
"funtoo_current" : [
# allow overlays to override gentoo
{ "repo" : "flora" },
{ "repo" : "faustoo" },
{ "repo" : "fusion809" },
{ "repo" : "rh1" },
{ "repo" : "gentoo-staging" }
],
"funtoo_mk2_prime" : [
# allow overlays to override gentoo
{ "repo" : "flora", },
{ "repo" : "faustoo" },
{ "repo" : "fusion809", "src_sha1" : "489b46557d306e93e6dc58c11e7c1da52abd34b0", 'date' : '31 Aug 2017' },
{ "repo" : "rh1", },
{ "repo" : "gentoo-staging", "src_sha1" : '80d2f3782e7f351855664919d679e94a95793a06', 'date' : '31 Aug 2017'},
],
"funtoo_mk3_prime" : [
# allow overlays to override gentoo
{ "repo" : "flora", },
{ "repo" : "faustoo", },
{ "repo" : "fusion809", "src_sha1" : "8733034816d3932486cb593db2dfbfbc7577e28b", 'date' : '09 Oct 2017' },
{ "repo" : "rh1", },
{ "repo" : "gentoo-staging", "src_sha1" : '2de4b388863ab0dbbd291422aa556c9de646f1ff', 'date' : '10 Oct 2017'},
],
"funtoo_mk3_late_prime": [
# allow overlays to override gentoo
{"repo": "flora", },
{"repo": "faustoo", },
{"repo": "fusion809", "src_sha1": "574f9f6f69b30f4eec7aa2eb53f55059d3c05b6a", 'date': '23 Oct 2017'},
{"repo": "rh1", },
{"repo": "gentoo-staging", "src_sha1": 'aa03020139bc129af2ad5f454640c102afa712e6', 'date': '22 Oct 2017'},
],
"funtoo_prime" : [
# allow overlays to override gentoo
{ "repo" : "flora", },
{ "repo" : "faustoo", },
{ "repo" : "fusion809", "src_sha1" : "8322bcd79d47ef81f7417c324a1a2b4772020985" },
{ "repo" : "rh1", },
{ "repo" : "gentoo-staging", "src_sha1" : '06a1fd99a3ce1dd33724e11ae9f81c5d0364985e', 'date' : '21 Apr 2017'},
],
"gentoo_prime_mk3_protected" : [
# lock down core-kit and security-kit
{ "repo" : "gentoo-staging", "src_sha1" : '2de4b388863ab0dbbd291422aa556c9de646f1ff', 'date' : '10 Oct 2017'},
],
"gentoo_prime_protected" : [
# lock down core-kit and security-kit
{ "repo" : "gentoo-staging", "src_sha1" : '06a1fd99a3ce1dd33724e11ae9f81c5d0364985e', 'date' : '21 Apr 2017'},
],
"gentoo_current_protected" : [
# lock down core-kit and security-kit
{ "repo" : "gentoo-staging" },
],
"funtoo_prime_xorg" : [
# specific snapshot for xorg-kit
{ "repo" : "gentoo-staging", 'src_sha1' : 'a56abf6b7026dae27f9ca30ed4c564a16ca82685', 'date' : '18 Nov 2016' }
],
"funtoo_prime_gnome" : [
# specific snapshot for gnome-kit
{ "repo" : "gentoo-staging", 'src_sha1' : '44677858bd088805aa59fd56610ea4fb703a2fcd', 'date' : '18 Sep 2016' }
],
"funtoo_prime_media" : [
# specific snapshot for media-kit, though we should bump and expand this soon
{ "repo" : "gentoo-staging", 'src_sha1' : '355a7986f9f7c86d1617de98d6bf11906729f108', 'date' : '25 Feb 2017' }
],
"funtoo_prime_perl" : [
# specific snapshot for perl-kit
{ "repo" : "gentoo-staging", 'src_sha1' : 'fc74d3206fa20caa19b7703aa051ff6de95d5588', 'date' : '11 Jan 2017' }
]
}
# 2. KIT GROUPS - this is where kits are actually defined. They are organized by GROUP: 'prime', 'current', or 'shared'.
# 'prime' kits are production-quality kits. Current kits are bleeding-edge kits. 'shared' kits are used by both 'prime'
# and 'current' -- they can have some "prime" kits as well as some "current" kits depending on what we want to stabilize.
# Note that we specify a 'source' which points to a name of a kit_source to use as a source of ebuilds. A kit is defined
# by a GROUP such as 'prime', a NAME, such as 'core-kit', a BRANCH, such as '1.0-prime', and a source (kit source) such
# as 'funtoo_prime'.
class KitStabilityRating(Enum):
PRIME = 0 # Kit is enterprise-quality
NEAR_PRIME = 1 # Kit is approaching enterprise-quality
BETA = 2 # Kit is in beta
ALPHA = 3 # Kit is in alpha
DEV = 4 # Kit is newly created and in active development
CURRENT = 10 # Kit follows Gentoo currrent
def KitRatingString(kit_enum):
if kit_enum is KitStabilityRating.PRIME:
return "prime"
elif kit_enum is KitStabilityRating.NEAR_PRIME:
return "near-prime"
elif kit_enum is KitStabilityRating.BETA:
return "beta"
elif kit_enum is KitStabilityRating.ALPHA:
return "alpha"
elif kit_enum is KitStabilityRating.DEV:
return "dev"
elif kit_enum is KitStabilityRating.CURRENT:
return "current"
kit_groups = {
'prime' : [
{ 'name' : 'core-kit', 'branch' : '1.0-prime', 'source': 'gentoo_prime_protected', 'default' : True },
{ 'name' : 'core-kit', 'branch' : '1.1-prime', 'source': 'gentoo_prime_mk3_protected', 'stability' : KitStabilityRating.DEV },
{ 'name' : 'core-hw-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True },
{ 'name' : 'security-kit', 'branch' : '1.0-prime', 'source': 'gentoo_prime_protected', 'default' : True },
{ 'name' : 'security-kit', 'branch' : '1.1-prime', 'source': 'gentoo_prime_mk3_protected', 'stability' : KitStabilityRating.DEV },
{ 'name' : 'xorg-kit', 'branch' : '1.17-prime', 'source': 'funtoo_prime_xorg', 'default' : False, 'stability' : KitStabilityRating.PRIME },
{ 'name' : 'xorg-kit', 'branch' : '1.19-prime', 'source': 'funtoo_mk2_prime', 'default' : True, 'stability' : KitStabilityRating.PRIME }, # MK2
{ 'name' : 'gnome-kit', 'branch' : '3.20-prime', 'source': 'funtoo_prime_gnome', 'default' : True },
{ 'name' : 'kde-kit', 'branch' : '5.10-prime', 'source': 'funtoo_mk3_prime', 'default' : True },
{ 'name' : 'media-kit', 'branch' : '1.0-prime', 'source': 'funtoo_prime_media', 'default' : True },
{ 'name' : 'media-kit', 'branch' : '1.1-prime', 'source': 'funtoo_mk3_prime', 'default' : False, 'stability' : KitStabilityRating.DEV }, # MK2
{ 'name' : 'perl-kit', 'branch' : '5.24-prime', 'source': 'funtoo_prime_perl', 'default' : True },
{ 'name' : 'perl-kit', 'branch' : '5.26-prime', 'source': 'funtoo_mk3_prime', 'default' : False, 'stability' : KitStabilityRating.DEV },
{ 'name' : 'python-kit', 'branch' : '3.4-prime', 'source': 'funtoo_prime', 'default' : True },
{ 'name' : 'python-kit', 'branch' : '3.6-prime', 'source': 'funtoo_mk2_prime', 'default' : False, 'stability' : KitStabilityRating.NEAR_PRIME }, # MK2
{ 'name' : 'python-kit', 'branch' : '3.6.3-prime', 'source': 'funtoo_mk3_prime', 'default': False, 'stability' : KitStabilityRating.DEV }, # MK3
{ 'name' : 'php-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True }, # We will freeze when 7.2.0 is released...
{ 'name' : 'java-kit', 'branch' : '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default' : True },
{ 'name' : 'ruby-kit', 'branch': '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default': True},
{ 'name' : 'haskell-kit', 'branch': '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default': True },
{ 'name' : 'ml-lang-kit', 'branch': '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default': True },
{ 'name' : 'lisp-scheme-kit', 'branch': '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default': True },
{ 'name' : 'lang-kit', 'branch': '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default': True },
{ 'name' : 'dev-kit', 'branch' : '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default' : True },
{ 'name' : 'xfce-kit', 'branch': '4.12-prime', 'source': 'funtoo_mk3_late_prime', 'default': True },
{ 'name' : 'desktop-kit', 'branch' : '1.1-prime', 'source': 'funtoo_mk3_late_prime', 'default' : True },
{ 'name' : 'editors-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True },
{ 'name' : 'net-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True },
{ 'name' : 'text-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True },
{ 'name' : 'science-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True },
{ 'name' : 'games-kit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True },
{ 'name' : 'nokit', 'branch' : 'master', 'source': 'funtoo_current', 'default' : True }
]
}
python_kit_settings = {
# branch / primary python / alternate python / python mask (if any)
'master' : {
"primary" : "python3_6",
"alternate" : "python2_7",
"mask" : None
},
'3.4-prime' : {
"primary" : "python3_4",
"alternate" : "python2_7",
"mask" : ">=dev-lang/python-3.5"
},
'3.6-prime' : {
"primary" : "python3_6",
"alternate" : "python2_7",
"mask" : ">=dev-lang/python-3.7"
},
'3.6.3-prime' : {
"primary" : "python3_6",
"alternate" : "python2_7",
"mask" : ">=dev-lang/python-3.7"
}
}
# It has already been explained how when we apply package-set rules, we process the kit_source repositories in order and
# after we find a catpkg that matches, any matches in successive repositories for catpkgs that we have already copied
# over to the destination kit are *ignored*. This is implemented using a dictionary called "kitted_catpkgs". Once a
# catpkg is inserted into a kit, it's no longer 'available' to be inserted into successive kits, to avoid duplicates.
kit_order = [ 'prime' ]
# We want to reset 'kitted_catpkgs' at certain points. The 'kit_order' variable below is used to control this, and we
# normally don't need to touch it. 'kitted_order' above tells the code to generate 'prime', then 'shared' (without
# resetting kitted_catpkgs to empty), then the None tells the code to reset kitted_catpkgs, so when 'current' kits are
# generated, they can include from all possible catpkgs. This is done because prime+shared is designed to be our
# primary enterprise-set of Funtoo kits. current+shared is also supported as a more bleeding edge option.
# 3. KIT PREP STEPS. To rebuild kits from scratch, we need to perform some initial actions to initialize an empty git
# repository, as well as some final actions. In the kit_steps dictionary below, indexed by kit, 'pre' dict lists the
# initial actions, and 'post' lists the final actions for the kit. There is also a special top-level key called
# 'regular-kits'. These actions are appended to any kit that is not core-kit or nokit. In addition to 'pre' and 'post'
# steps, there is also a 'copy' step that is not currently used (but is supported by getKitPrepSteps()).
def getKitPrepSteps(repos, kit_dict, gentoo_staging, fixup_repo):
kit_steps = {
'core-kit' : { 'pre' : [
GenerateRepoMetadata("core-kit", aliases=["gentoo"], priority=1000),
# core-kit has special logic for eclasses -- we want all of them, so that third-party overlays can reference the full set.
# All other kits use alternate logic (not in kit_steps) to only grab the eclasses they actually use.
SyncDir(gentoo_staging.root, "eclass"),
],
'post' : [
# We copy files into funtoo's profile structure as post-steps because we rely on kit-fixups step to get the initial structure into place
CopyAndRename("profiles/funtoo/1.0/linux-gnu/arch/x86-64bit/subarch", "profiles/funtoo/1.0/linux-gnu/arch/pure64/subarch", lambda x: os.path.basename(x) + "-pure64"),
# news items are not included here anymore
SyncDir(fixup_repo.root, "profiles/base"),
SyncDir(fixup_repo.root, "profiles/arch/base"),
SyncDir(fixup_repo.root, "profiles/updates"),
SyncDir(fixup_repo.root, "metadata", exclude=["cache","md5-cache","layout.conf"]),
SyncFiles(fixup_repo.root, {
"profiles/package.mask":"profiles/package.mask/00-gentoo",
"profiles/arch/amd64/package.use.mask":"profiles/funtoo/1.0/linux-gnu/arch/x86-64bit/package.use.mask/01-gentoo",
"profiles/arch/amd64/use.mask":"profiles/funtoo/1.0/linux-gnu/arch/x86-64bit/use.mask/01-gentoo",
"profiles/arch/x86/package.use.mask":"profiles/funtoo/1.0/linux-gnu/arch/x86-32bit/package.use.mask/01-gentoo",
"profiles/arch/x86/use.mask":"profiles/funtoo/1.0/linux-gnu/arch/x86-32bit/use.mask/01-gentoo",
"profiles/default/linux/package.use.mask":"profiles/funtoo/1.0/linux-gnu/package.use.mask/01-gentoo",
"profiles/default/linux/use.mask":"profiles/funtoo/1.0/linux-gnu/use.mask/01-gentoo",
"profiles/arch/amd64/no-multilib/package.use.mask":"profiles/funtoo/1.0/linux-gnu/arch/pure64/package.use.mask/01-gentoo",
"profiles/arch/amd64/no-multilib/package.mask":"profiles/funtoo/1.0/linux-gnu/arch/pure64/package.mask/01-gentoo",
"profiles/arch/amd64/no-multilib/use.mask":"profiles/funtoo/1.0/linux-gnu/arch/pure64/use.mask/01-gentoo"
}),
SyncFiles(fixup_repo.root, {
"profiles/info_pkgs" : "profiles/info_pkgs",
"profiles/thirdpartymirrors" : "profiles/thirdpartymirrors",
"profiles/license_groups" : "profiles/license_groups",
"profiles/use.desc" : "profiles/use.desc",
}),
# add funtoo stuff to thirdpartymirrors
ThirdPartyMirrors(),
RunSed(["profiles/base/make.defaults"], ["/^PYTHON_TARGETS=/d", "/^PYTHON_SINGLE_TARGET=/d"]),
]
},
# masters of core-kit for regular kits and nokit ensure that masking settings set in core-kit for catpkgs in other kits are applied
# to the other kits. Without this, mask settings in core-kit apply to core-kit only.
'regular-kits' : { 'pre' : [
GenerateRepoMetadata(kit_dict['name'], masters=[ "core-kit" ], priority=500),
]
},
'all-kits' : { 'pre' : [
SyncFiles(fixup_repo.root, {
"COPYRIGHT.txt":"COPYRIGHT.txt",
"LICENSE.txt":"LICENSE.txt",
}),
]
},
'nokit' : { 'pre' : [
GenerateRepoMetadata("nokit", masters=[ "core-kit" ], priority=-2000),
]
}
}
out_pre_steps = []
out_copy_steps = []
out_post_steps = []
kd = kit_dict['name']
if kd in kit_steps:
if 'pre' in kit_steps[kd]:
out_pre_steps += kit_steps[kd]['pre']
if 'post' in kit_steps[kd]:
out_post_steps += kit_steps[kd]['post']
if 'copy' in kit_steps[kd]:
out_copy_steps += kit_steps[kd]['copy']
# a 'regular kit' is not core-kit or nokit -- if we have pre or post steps for them, append these steps:
if kit_dict['name'] not in [ 'core-kit', 'nokit' ] and 'regular-kits' in kit_steps:
if 'pre' in kit_steps['regular-kits']:
out_pre_steps += kit_steps['regular-kits']['pre']
if 'post' in kit_steps['regular-kits']:
out_post_steps += kit_steps['regular-kits']['post']
if 'all-kits' in kit_steps:
if 'pre' in kit_steps['all-kits']:
out_pre_steps += kit_steps['all-kits']['pre']
if 'post' in kit_steps['all-kits']:
out_post_steps += kit_steps['all-kits']['post']
return ( out_pre_steps, out_copy_steps, out_post_steps )
# GET KIT SOURCE INSTANCE. This function returns a list of GitTree objects for each of repositories specified for
# a particular kit's kit_source, in the order that they should be processed (in the order they are defined in
# kit_source_defs, in other words.)
def getKitSourceInstance(kit_dict):
global kit_source_defs
source_name = kit_dict['source']
repos = []
source_defs = kit_source_defs[source_name]
for source_def in source_defs:
repo_name = source_def['repo']
repo_branch = source_def['src_branch'] if "src_branch" in source_def else "master"
repo_sha1 = source_def["src_sha1"] if "src_sha1" in source_def else None
repo_obj = overlays[repo_name]["type"]
repo_url = overlays[repo_name]["url"]
if "dirname" in overlays[repo_name]:
path = overlays[repo_name]["dirname"]
else:
path = repo_name
repo = repo_obj(repo_name, url=repo_url, root="/var/git/source-trees/%s" % path, branch=repo_branch, commit_sha1=repo_sha1)
repos.append( { "name" : repo_name, "repo" : repo, "overlay_def" : overlays[repo_name] } )
return repos
# UPDATE KIT. This function does the heavy lifting of taking a kit specification included in a kit_dict, and
# regenerating it. The kitted_catpkgs argument is a dictionary which is also written to and used to keep track of
# catpkgs copied between runs of updateKit.
def updateKit(kit_dict, prev_kit_dict, kit_group, cpm_logger, db=None, create=False, push=False, now=None, fixup_repo=None):
if prev_kit_dict != None and kit_dict['name'] != prev_kit_dict['name']:
# We are advancing to the next kit. For example, we just processed an xorg-kit and are now processing a python-kit. So we want to apply all our accumulated matches.
# If we are processing an xorg-kit again, this won't run, which is what we want. We want to keep accumulating catpkg names/matches.
cpm_logger.nextKit()
# get set of source repos used to grab catpkgs from:
repos = kit_dict["repo_obj"] = getKitSourceInstance(kit_dict)
# get a handy variable reference to gentoo_staging:
gentoo_staging = None
for x in repos:
if x["name"] == "gentoo-staging":
gentoo_staging = x["repo"]
break
if gentoo_staging == None:
print("Couldn't find source gentoo staging repo")
elif gentoo_staging.name != "gentoo-staging":
print("Gentoo staging mismatch -- name is %s" % gentoo_staging["name"])
# we should now be OK to use the repo and the local branch:
kit_dict['tree'] = tree = GitTree(kit_dict['name'], kit_dict['branch'],
"git@github.com:funtoo/%s" % kit_dict['name'], create=create,
root="/var/git/dest-trees/%s" % kit_dict['name'], pull=True)
# Phase 1: prep the kit
pre_steps = [
GitCheckout(kit_dict['branch']),
CleanTree()
]
prep_steps = getKitPrepSteps(repos, kit_dict, gentoo_staging, fixup_repo)
pre_steps += prep_steps[0]
copy_steps = prep_steps[1]
post_steps = prep_steps[2]
tree.run(pre_steps)
# Phase 2: copy core set of ebuilds
# Here we generate our main set of ebuild copy steps, based on the contents of the package-set file for the kit. The logic works as
# follows. We apply our package-set logic to each repo in succession. If copy ebuilds were actually copied (we detect this by
# looking for changed catpkg count in our dest_kit,) then we also run additional steps: "copyfiles" and "eclasses". "copyfiles"
# specifies files like masks to copy over to the dest_kit, and "eclasses" specifies eclasses from the overlay that we need to
# copy over to the dest_kit. We don't need to specify eclasses that we need from gentoo_staging -- these are automatically detected
# and copied, but if there are any special eclasses from the overlay then we want to copy these over initially.
copycount = cpm_logger.copycount
for repo_dict in repos:
steps = []
select_clause = "all"
overlay_def = repo_dict["overlay_def"]
if "select" in overlay_def:
select_clause = overlay_def["select"]
# If the repo has a "filter" : [ "foo", "bar", "oni" ], then construct a list of repos with those names and put
# them in filter_repos. We will pass this list of repo objects to InsertEbuilds inside generateKitSteps, and if
# a catpkg exists in any of these repos, then it will NOT be copied if it is scheduled to be copied for this
# repo. This is a way we can lock down overlays to not insert any catpkgs that are already defined in gentoo --
# just add: filter : [ "gentoo-staging" ] and if the catpkg exists in gentoo-staging, it won't get copied. This
# way we can more safely choose to include all ebuilds from 'potpurri' overlays like faustoo without exposing
# ourself to too much risk from messing stuff up.
filter_repos = []
if "filter" in overlay_def:
for filter_repo_name in overlay_def["filter"]:
for x in repos:
if x["name"] == filter_repo_name:
filter_repos.append(x["repo"])
if kit_dict["name"] == "nokit":
# grab all ebuilds to put in nokit
steps += [ InsertEbuilds(repo_dict["repo"], select_only=select_clause, skip=None, replace=False, cpm_logger=cpm_logger) ]
else:
steps += generateKitSteps(kit_dict['name'], repo_dict["repo"], tree, gentoo_staging, fixup_repo=fixup_repo, select_only=select_clause,
pkgdir=merge_scripts.root+"/funtoo/scripts", filter_repos=filter_repos, cpm_logger=cpm_logger)
tree.run(steps)
if copycount != cpm_logger.copycount:
# this means some catpkgs were installed from the repo we are currently processing. This means we also want to execute
# 'copyfiles' and 'eclasses' copy logic:
ov = overlays[repo_dict["name"]]
if "copyfiles" in ov and len(ov["copyfiles"]):
# since we copied over some ebuilds, we also want to make sure we copy over things like masks, etc:
steps += [ SyncFiles(repo_dict["repo"].root, ov["copyfiles"]) ]
if "eclasses" in ov:
# we have eclasses to copy over, too:
ec_files = {}
for eclass in ov["eclasses"]:
ecf = "/eclass/" + eclass + ".eclass"
ec_files[ecf] = ecf
steps += [ SyncFiles(repo_dict["repo"].root, ec_files) ]
copycount = cpm_logger.copycount
# Phase 3: copy eclasses, licenses, profile info, and ebuild/eclass fixups from the kit-fixups repository.
# First, we are going to process the kit-fixups repository and look for ebuilds and eclasses to replace. Eclasses can be
# overridden by using the following paths inside kit-fixups:
# kit-fixups/eclass <--------------------- global eclasses, get installed to all kits unconditionally (overrides those above)
# kit-fixups/<kit>/global/eclass <-------- global eclasses for a particular kit, goes in all branches (overrides those above)
# kit-fixups/<kit>/global/profiles <------ global profile info for a particular kit, goes in all branches (overrides those above)
# kit-fixups/<kit>/<branch>/eclass <------ eclasses to install in just a specific branch of a specific kit (overrides those above)
# kit-fixups/<kit>/<branch>/profiles <---- profile info to install in just a specific branch of a specific kit (overrides those above)
# Note that profile repo_name and categories files are excluded from any copying.
# Ebuilds can be installed to kits by putting them in the following location(s):
# kit-fixups/<kit>/global/cat/pkg <------- install cat/pkg into all branches of a particular kit
# kit-fixups/<kit>/<branch>/cat/pkg <----- install cat/pkg into a particular branch of a kit
# Remember that at this point, we may be missing a lot of eclasses and licenses from Gentoo. We will then perform a final sweep
# of all catpkgs in the dest_kit and auto-detect missing eclasses from Gentoo and copy them to our dest_kit. Remember that if you
# need a custom eclass from a third-party overlay, you will need to specify it in the overlay's overlays["ov_name"]["eclasses"]
# list. Or alternatively you can copy the eclasses you need to kit-fixups and maintain them there :)
steps = []
# Here is the core logic that copies all the fix-ups from kit-fixups (eclasses and ebuilds) into place:
if os.path.exists(fixup_repo.root + "/eclass"):
steps += [ InsertEclasses(fixup_repo, select="all", skip=None) ]
if kit_dict["branch"] == "master":
fixup_dirs = [ "global", "master" ]
else:
fixup_dirs = [ "global", "curated", kit_dict["branch"] ]
for fixup_dir in fixup_dirs:
fixup_path = kit_dict['name'] + "/" + fixup_dir
if os.path.exists(fixup_repo.root + "/" + fixup_path):
if os.path.exists(fixup_repo.root + "/" + fixup_path + "/eclass"):
steps += [
InsertFilesFromSubdir(fixup_repo, "eclass", ".eclass", select="all", skip=None, src_offset=fixup_path)
]
if os.path.exists(fixup_repo.root + "/" + fixup_path + "/licenses"):
steps += [
InsertFilesFromSubdir(fixup_repo, "licenses", None, select="all", skip=None, src_offset=fixup_path)
]
if os.path.exists(fixup_repo.root + "/" + fixup_path + "/profiles"):
steps += [
InsertFilesFromSubdir(fixup_repo, "profiles", None, select="all", skip=["repo_name", "categories"] , src_offset=fixup_path)
]
# copy appropriate kit readme into place:
readme_path = fixup_path + "/README.rst"
if os.path.exists(fixup_repo.root + "/" + readme_path ):
steps += [
SyncFiles(fixup_repo.root, {
readme_path : "README.rst"
})
]
steps += [
# here we log catpkgs to the CatPkgMatchLogger, but cpm_ignore=True tells our code to not consult the
# cpm_logger to determine whether to copy. Always copy -- these are fixups!
InsertEbuilds(fixup_repo, ebuildloc=fixup_path, select="all", skip=None, replace=True,
cpm_logger=cpm_logger, is_fixup=True )
]
tree.run(steps)
# Now we want to perform a scan of any eclasses in the Gentoo repo that we need to copy over to our dest_kit so that it contains all
# eclasses and licenses it needs within itself, without having to reference any in the Gentoo repo.
copy_steps = []
# For eclasses we perform a much more conservative scan. We will only scour missing eclasses from gentoo-staging, not
# eclasses. If you need a special eclass, you need to specify it in the eclasses list for the overlay explicitly.
last_count = None
iterations = 0
max_iterations = 16
keep_going = True
missing_eclasses = []
# This loop is rather complicated to handle the case where we copy an eclass into our dest-kit, and this eclass in
# turn requires additional eclasses, thus creating additional missing eclasses. So we need a super-loop to drive the
# whole thing, and we keep going until the number of missing eclasses is zero.
if tree.name != "core-kit":
# for core-kit, we are going to include a COMPLETE set of eclasses, even unused ones, so that third-party overlays that
# depend on any of these eclasses will find them and be able to use them.
# All other kits get only the eclasses they actually use, so they have local copies of the versions of the eclasses with
# which they were tested.
while keep_going:
for repo, elist in getAllEclasses(tree, gentoo_staging).items():
repo_obj = None
if repo == "dest_kit":
# This means the eclass already exists in the kit -- probably copied from fixups
continue
elif repo == "parent_repo":
# This means that the eclass is in gentoo_staging and needs to be copied
repo_obj = gentoo_staging
elif repo == None:
if len(elist) == 0:
keep_going = False
elif len(elist) == last_count:
# If there is something in elist, this means that the listed eclasses was nowhere to be found.
iterations += 1
if iterations > max_iterations:
missing_eclasses = elist
keep_going = False
if repo_obj != None:
copy_steps += [ InsertEclasses(repo_obj, select=elist) ]
if iterations > max_iterations and len(missing_eclasses):
print("!!! Error: The following eclasses were not found:")
print("!!! : " + " ".join(missing_eclasses))
print("!!! : Please be sure to use kit-fixups or the overlay's eclass list to copy these necessary eclasses into place.")
sys.exit(1)
# we need to get all the eclasses in place right away so that the following license extraction code has a
# chance of working correctly:
tree.run(copy_steps)
copy_steps = []
# for licenses, we are going to aggressively scan all source repos for any missing licenses. We don't need to
# be as anal about licenses as eclasses where slight differences can cause problems:
not_found_licenses = {}
for repo_dict in repos:
for repo, elist in getAllLicenses(tree, repo_dict["repo"]).items():
repo_obj = None
if repo == "dest_kit":
# already where we want them
continue
elif repo == "parent_repo":
# found license in the parent repo/overlay
repo_obj = repo_dict["repo"]
elif repo == None:
for license in elist:
not_found_licenses[license] = repo_dict["repo"]
for license in elist:
if license in not_found_licenses:
del not_found_licenses[license]
if repo_obj:
copy_steps += [ InsertLicenses(repo_obj, select=elist) ]
if len(list(not_found_licenses)):
# we scoured all our source repositories and these licenses were not found:
print("!!! Error: The following eclasses were not found:")
for license, repo in not_found_licenses.items():
print("!!! %s license in %s" % (license, repo.name))
print("!!! : Please be sure to install these licenses in the source repository.")
sys.exit(1)
tree.run(copy_steps)
# QA check: all eclasses should be in place. Let's confirm. if egencache is run without all eclasses in place, it hangs.
result = getAllEclasses(tree)
if None in result and len(result[None]):
# we have some missing eclasses
print("!!! Error: QA check on kit failed -- missing eclasses:")
print("!!! : " + " ".join(result[None]))
print("!!! : Please be sure to use kit-fixups or the overlay's eclass list to copy these necessary eclasses into place.")
sys.exit(1)
# Phase 4: finalize and commit
post_steps += [
ELTSymlinkWorkaround(),
CreateCategories(gentoo_staging),
# multi-plex this and store in different locations so that different selections can be made based on which python-kit is enabled.
# python-kit itself only needs one set which will be enabled by default.
]
if kit_dict["name"] == "python_kit":
# on the python-kit itself, we only need settings for ourselves (not other branches)
python_settings = python_kit_settings[kit_dict["name"]]
else:
# all other kits -- generate multiple settings, depending on what version of python-kit is active -- epro will select the right one for us.
python_settings = python_kit_settings
for branch, py_settings in python_settings.items():
post_steps += [ GenPythonUse(py_settings, "funtoo/kits/python-kit/%s" % branch) ]
post_steps += [
Minify(),
GenUseLocalDesc(),
GenCache( cache_dir="/var/cache/edb/%s-%s" % ( kit_dict['name'], kit_dict['branch'] ) ),
]
if kit_group in [ "prime" ]:
# doing to record distfiles in mysql only for prime, not current, at least for now
post_steps += [
CatPkgScan(now=now, db=db)
]
tree.run(post_steps)
tree.gitCommit(message="updates",branch=kit_dict['branch'],push=push)
return tree.head()
if __name__ == "__main__":
# one global timestamp for each run of this tool -- for mysql db
now = datetime.utcnow()
if len(sys.argv) < 2 or sys.argv[1] not in [ "push", "nopush" ]:
print("Please specify push or nopush as an argument.")
sys.exit(1)
else:
push = True if "push" in sys.argv else False
if "db" in sys.argv:
db = getMySQLDatabase()
else:
db = None
cpm_logger = CatPkgMatchLogger(log_xml=push)
output_sha1s = {}
output_order = []
output_settings = defaultdict(dict)
for kit_group in kit_order:
if kit_group == None:
if push:
cpm_logger.writeXML()
cpm_logger = CatPkgMatchLogger(log_xml=False)
else:
prev_kit_dict = None
# kits that are officially tagged as 'prime' -- this means that they are really prime -- branch name doesn't guarantee
kit_labels = defaultdict(list)
for kit_dict in kit_groups[kit_group]:
print("Regenerating kit ",kit_dict)
head = updateKit(kit_dict, prev_kit_dict, kit_group, cpm_logger, db=db, create=not push, push=push, now=now, fixup_repo=fixup_repo)
kit_name = kit_dict["name"]
kit_branch = kit_dict["branch"]
if 'default' in kit_dict:
kit_stability = KitStabilityRating.PRIME
elif 'stability' in kit_dict:
kit_stability = kit_dict['stability']
elif kit_group == 'current':
kit_stability = KitStabilityRating.CURRENT
else:
print("Unknown kit stability")
sys.exit(1)
if kit_group in [ "prime" ] and kit_name not in output_order:
output_order.append(kit_name)
if 'default' in kit_dict and kit_dict['default'] == True:
output_settings[kit_name]["default"] = kit_branch
# specific keywords that can be set for each branch to identify its current quality level
if 'stability' not in output_settings[kit_name]:
output_settings[kit_name]['stability'] = {}
output_settings[kit_name]['stability'][kit_branch] = KitRatingString(kit_stability)
if kit_name not in output_sha1s:
output_sha1s[kit_name] = {}
output_sha1s[kit_name][kit_branch] = head
prev_kit_dict = kit_dict
if not os.path.exists(meta_repo.root + "/metadata"):
os.makedirs(meta_repo.root + "/metadata")
with open(meta_repo.root + "/metadata/kit-sha1.json", "w") as a:
a.write(json.dumps(output_sha1s, sort_keys=True, indent=4, ensure_ascii=False))
outf = meta_repo.root + "/metadata/kit-info.json"
# read first to preserve any metadata we added manually
k_info = {}
if os.path.exists(outf):
a = open(outf, 'r')
k_info = json.loads(a.read())
a.close()
k_info["kit_order"] = output_order
k_info["kit_settings"] = output_settings
with open(meta_repo.root + "/metadata/kit-info.json", "w") as a:
a.write(json.dumps(k_info, sort_keys=True, indent=4, ensure_ascii=False))
print("Checking out default versions of kits.")
for kit_dict in kit_groups['prime']:
if 'default' not in kit_dict or kit_dict['default'] != True:
# skip non-default kits
continue
kit_dict["tree"].run([GitCheckout(branch=kit_dict['branch'])])
if push:
# use the github url for the submodule, for public consumption.
meta_repo.gitSubmoduleAddOrUpdate(kit_dict["tree"], "kits/%s" % kit_dict["name"], "https://github.com/funtoo/%s.git" % kit_dict["name"])
if push:
meta_repo.gitCommit(message="kit updates", branch="master", push=push)
# vim: ts=4 sw=4 noet tw=140