From 489957edab347d4280cf5118f9587316188d678b Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Fri, 2 May 2025 13:57:14 +0200 Subject: [PATCH 01/11] archive: add src upload/download support --- pym/bob/archive.py | 63 ++++++++++++++++++++++++++----- pym/bob/input.py | 3 +- test/black-box/bundle/bundle.zip | Bin 0 -> 30482 bytes test/unit/test_archive.py | 6 +++ 4 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 test/black-box/bundle/bundle.zip diff --git a/pym/bob/archive.py b/pym/bob/archive.py index 658290e0..03ae857b 100644 --- a/pym/bob/archive.py +++ b/pym/bob/archive.py @@ -95,9 +95,15 @@ def wantUploadJenkins(self, enable): def canDownload(self): return False + def canDownloadSrc(self): + return False + def canUpload(self): return False + def canUploadSrc(self, step, freshCheckout=None): + return False + def canCache(self): return False @@ -194,13 +200,13 @@ def _extractAudit(self, filename=None, fileobj=None): return Audit.fromByteStream(auditJson, filename) - def _pack(self, name, fileobj, audit, content): + def _pack(self, name, fileobj, audit, content, filter): pax = { 'bob-archive-vsn' : "1" } with gzip.open(name or fileobj, 'wb', 6) as gzf: with tarfileOpen(name, "w", fileobj=gzf, format=tarfile.PAX_FORMAT, pax_headers=pax) as tar: tar.add(audit, "meta/" + os.path.basename(audit)) - tar.add(content, arcname="content") + tar.add(content, arcname="content", filter=filter) class JenkinsArchive(TarHelper): @@ -224,9 +230,15 @@ def wantUploadJenkins(self, enable): def canDownload(self): return True + def canDownloadSrc(self): + return False + def canUpload(self): return True + def canUploadSrc(self, step, freshCheckout=None): + return False + def canCache(self): return True @@ -263,7 +275,7 @@ def _uploadPackage(self, name, buildId, audit, content): # Needed to gracefully handle ctrl+c. signal.signal(signal.SIGINT, signal.default_int_handler) try: - self._pack(name, None, audit, content) + self._pack(name, None, audit, content, None) except (tarfile.TarError, OSError) as e: raise BuildError("Cannot pack artifact: " + str(e)) finally: @@ -375,6 +387,9 @@ def __init__(self, spec): self.__wantDownloadJenkins = False self.__wantUploadLocal = False self.__wantUploadJenkins = False + self.__srcUpload = "src-upload" in flags + self.__srcDownload = "src-download" in flags + self.__srcUploadVCS = spec.get("src-upload-vcs", False) @property def ignoreUploadErrors(self): @@ -396,16 +411,31 @@ def canDownload(self): return self.__useDownload and ((self.__wantDownloadLocal and self.__useLocal) or (self.__wantDownloadJenkins and self.__useJenkins)) + def canDownloadSrc(self): + return self.__srcDownload and ((self.__wantDownloadLocal and self.__useLocal) or + (self.__wantDownloadJenkins and self.__useJenkins)) + def canUpload(self): return self.__useUpload and ((self.__wantUploadLocal and self.__useLocal) or (self.__wantUploadJenkins and self.__useJenkins)) + def canUploadSrc(self, step, freshCheckout=None): + return (self.__srcUpload and (True if freshCheckout is None else freshCheckout) and + ((self.__wantUploadLocal and self.__useLocal) or + (self.__wantUploadJenkins and self.__useJenkins))) + def canCache(self): return self.__useCache def _openDownloadFile(self, buildId, suffix): raise ArtifactNotFoundError() + def _srcUploadVcsFilter(tarinfo): + if tarinfo.isdir() and (".git" in tarinfo.name or + ".svn" in tarinfo.name): + return None + return tarinfo + def canManage(self): return self.__managed and self._canManage() @@ -421,7 +451,8 @@ def _namedErrorString(self, err): async def downloadPackage(self, step, buildId, audit, content, caches=[], executor=None): - if not self.canDownload(): + if not ((self.canDownload() and not step.isCheckoutStep()) or (self.canDownloadSrc() + and step.isCheckoutStep())): return False loop = asyncio.get_event_loop() @@ -536,8 +567,13 @@ def _openUploadFile(self, buildId, suffix, overwrite): raise ArtifactUploadError("not implemented") async def uploadPackage(self, step, buildId, audit, content, executor=None): - if not self.canUpload(): + if step.isPackageStep() and not self.canUpload() or \ + step.isCheckoutStep() and not self.canUploadSrc(step): + return + + if step.isCheckoutStep() and not step.isDeterministic(): return + if not audit: stepMessage(step, "UPLOAD", "skipped (no audit trail)", SKIPPED, IMPORTANT) @@ -549,19 +585,20 @@ async def uploadPackage(self, step, buildId, audit, content, executor=None): with stepAction(step, "UPLOAD", content, details=details) as a: try: msg, kind = await loop.run_in_executor(executor, BaseArchive._uploadPackage, - self, buildId, suffix, audit, content) + self, buildId, suffix, audit, content, + (BaseArchive._srcUploadVcsFilter if step.isCheckoutStep() and not self.__srcUploadVCS else None)) a.setResult(msg, kind) except (concurrent.futures.CancelledError, concurrent.futures.process.BrokenProcessPool): raise BuildError(self._namedErrorString("Upload of package interrupted.")) - def _uploadPackage(self, buildId, suffix, audit, content): + def _uploadPackage(self, buildId, suffix, audit, content, filter): # Set default signal handler so that KeyboardInterrupt is raised. # Needed to gracefully handle ctrl+c. signal.signal(signal.SIGINT, signal.default_int_handler) try: with self._openUploadFile(buildId, suffix, False) as (name, fileobj): - self._pack(name, fileobj, audit, content) + self._pack(name, fileobj, audit, content, filter) except (ArtifactExistsError, HttpAlreadyExistsError): return (self._namedErrorString("skipped ({} exists in archive)".format(content)), SKIPPED) except (ArtifactUploadError, HttpUploadError, tarfile.TarError, OSError) as e: @@ -1272,17 +1309,23 @@ def wantUploadJenkins(self, enable): def canDownload(self): return any(i.canDownload() for i in self.__archives) + def canDownloadSrc(self): + return any(i.canDownloadSrc() for i in self.__archives) + def canUpload(self): return any(i.canUpload() for i in self.__archives) + def canUploadSrc(self, step, freshCheckout=None): + return any(i.canUploadSrc(step, freshCheckout) for i in self.__archives) + async def uploadPackage(self, step, buildId, audit, content, executor=None): for i in self.__archives: - if not i.canUpload(): continue + if not (i.canUpload() or i.canUploadSrc(step)): continue await i.uploadPackage(step, buildId, audit, content, executor=executor) async def downloadPackage(self, step, buildId, audit, content, executor=None): for i in self.__archives: - if not i.canDownload(): continue + if not (i.canDownload() or i.canDownloadSrc()): continue caches = [ a for a in self.__archives if (a is not i) and a.canCache() ] if await i.downloadPackage(step, buildId, audit, content, caches, executor): return True diff --git a/pym/bob/input.py b/pym/bob/input.py index e57396f4..4b1f4e45 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -3084,7 +3084,8 @@ def __init__(self): 'backend' : str, schema.Optional('name') : str, schema.Optional('flags') : schema.Schema(["download", "upload", "managed", - "nofail", "nolocal", "nojenkins", "cache", "strictdownload"]) + "nofail", "nolocal", "nojenkins", "cache", "strictdownload", + "src-upload", "src-download"]), } fileArchive = baseArchive.copy() fileArchive["path"] = str diff --git a/test/black-box/bundle/bundle.zip b/test/black-box/bundle/bundle.zip new file mode 100644 index 0000000000000000000000000000000000000000..246045839a93bab9ac3811d5083bf96fa2815d84 GIT binary patch literal 30482 zcmagFb97~Uo35QyB^6g}+vbYRif!9=Qn77UY*%dCwrxA>tKDz+>D}+yU!OI`v*sLY z+~fIoUGutsQ%>qD7z)T=A9Q_y=6~G$=hxRiR|6Azx&Pn5w|@mV4E`+x@VW@0hXeun zYZ(4dhJ}ei-^75!z);_aje&y!z{tXG$iWU^Hf8}B>9ZR%0T}feX&LF9Og-=;AR((! z7R*3^PS$qJMmDS_#wIKv7b~7D^X~W%*DDN{f>aL$X^G1W=qBT-Mktk1f=Y>!%4n%-?FGZF(d>HZs0+V$3ha`f9Hizby5fcE^Ik z)Mv%eKE5Nz;qltwO{39(%90%K-8(~4Ut&s=lf zK7lAif`YrP3fQ=8Z$ly=XHBj~Wa*R>W}B3@50-nlv88pGkSE>pL`ErzS1Z=Cs#Q7t zlw7}L?i@`36HGqkcjUKEd`GshOZriDo3J{wSH`BsaD8tMW=I|9h3chPs?sW3bm}Ma z_z5%Ox@aw*Grf-yc1oEGyaZ zM%M7wcH`;r#jRTMrg`PXWH^&2g`9qm+=^YVp)1!Fx3_R_vRb;$?Gan_0xDk#CeEi@ zotZXL2PT(6C<#a>A=Iwle2M6g7z45r&GX+J8K$d)17^Xi?jHsr&B^YI=shiemTnvy zj<4cJ5~F6idP>}&5=?2`-%cI}D-jwLSmk?CAw$sDeCP;#NZ&L(N_t0}j+KpVrovEp z7aIJYdV6YNUiKOI!6_8PLdi?iz`b3x;QWT9eS$!aerNrDMDjk0fLa~^S8b^g#gT^U zGVr;BndCA(Fv+x)+tH*`#*H(v5}3?N?i@Ai=2=?S`e_UmjmVdb=4{KJ6?w7)u~O{_ z`2$bDwlgy|g;j(aVV`WXD-60^1!>TVBS&t@&&K0IalVXcZhBtH+lFoC0&a)7DBej%m%q(qIG;r7OWRk1kKwe6P zm=cDDBmLGtl;bQ(zV0mLIlJSbHlB@%qGf26j#DufhSN1ef?qK=rEml+a+V4>=h{D! z#nxyGmYzX7L{j)RJ#)TiXdw4Bn@#PUCRtRdP^rz1(%vLK&bG*|yp2rm$$=QfH-k=n z%jG3$4>_QT@?bXbJyQNOTne(F_+$KQe7HP%zeI+Qv zph>;9#6vt|=`}9fzW3o#V%=GPx&yI_#5wA`7A`?v8&qlC{)a%w4A1@#?Bddd35Do7 zT(nFfM|zZr;c|Joe#b4O!?{G5Nk$p$4x}|ztfvXc0Wv%SoiOG-4|f(W_TITeC&}2734|{7OMoJp*!)a>%3l6ABRJeUqo`ZIKB`Q0=Xy`z_8CqUTdY>8N<}}T;g43(6?t&dQ0J@zxK8)l zTPjYdm`&`il}EZ+5p~)_w+-~d%h|}w$v?pbvP;QIxg{RTLqvFSeEh^r@$@St%U!$r zTCnZj%{CazanTcHo=T+26x>(#*LBGyBWHT@%{OsmhOBiUm_XG1rH*VYe?@<@k1G|H zjt~1`Buq1Mh??&%fxFr%s%Z6aW9nBoQ@oi@5S);>On4ohu?ER*qM6PllyX@i+bTh- z0udHc@msUj=`6TwW~@xw#p?vLq6))fUbMEvT{{oZsxYb-BrxAqFZMJ zQ3fi7Mhl{ryassh$>MXmzjb4B*t=I`!*HnT5LOgg2&A*&B-C7H2_yF2P+hWn9sk{6QC@S_(D8-P$cO~vw#NKC&3(5VT6!OoP?xic&Af&9HAaWoe{|zYY|3ENC9Dk#e ze+)4EFM+=h=g$UqM$!ARvFuUl?Tc9~lDx8wZOCGk}>1pl`%% z%wYfk7&3F{v$C`Q#Y6xj4kmV!|Hh!R|H2>(3kySL8yjnoi&Re*lvTW;Be#rUx1)vO zJrv{dIfN)3yc3kNzMF9d_&o#e;2Sc44k;<80Y^MV(rr4oWJ=OYIRbF(8ih~G`PXlLee)Y5mmbR?@UjXFI`kaVt`M-#uIvLx z<)ANU_c8pRO`kgq9Q-^+&Z{q(4OPBE&iK$H5M5t`MKSZDVnu_^Afe5T^vukXv+mKS6%dGW4aLY9jnjwR{q}(bqO+1PvXWfKy%92B)pF^{$k6H= zY+#sM(7|ptJ-K+`UnAHYF6>)DQ_f+*yEcG*G1$tt2^dReG(5bHZ0P}0X;v~-=}C^i z?i1RvEwDB;#cAm8=u42vZUDm%?NabkmXp?GceR}uEUyEnl&<^!Xls8BC*t~JPm7w3 z3(V9=LpKHoT|k zG804GDP4z6k@>rc?{wx}sHtK@Prmz1FN=tM zSrJzItf99&!u$lOFp_#B4fEfXa`y-|fvFAOX$RA{uKcW{9LoEyh7 zP1*~sOy;9>c$wuiVw+27rLP|%d?Z!!p#qk_&XrG}IE(b7RHbyiEWSR1MXQ%XrkgL3 zjylv&XQF>0{z|};TT(^fFp0NL$7i?k^_8NP?NeY1JV%@aGtnWU4{ngzRN#sVuXk>k zXQ9xx`Yq})3yEh*+m`!w{bEC)@ZoEG^WAtlKe*sRod%d+HS#Yfx&~Qhm?4uk+nGTNk}WJV z5D*-rtE;7&c_`_uEX9Z&NdN_FP9suFueGW`^#}@F-_;@`eNBL_#O5nh`G67iiP-AR zVqmUyTRC!w2t(LYmXfseWbhHjGI_Q;_Y6zL>=dU`)O5dt$q z+#g%Qc}v(j=)W#xogGl%UqV>9dE=mfpH8%96bmcL742=IicUpOQm4!51o=a;>!nH- zrYznd?s0e==6yicO6xhquQ{)aRX-?fpC3b2dHqBwq1#$}7VuR*09BM6B~WeC>U;Nk zP3u2)nZ?^_db{8DC1=)?Nfip@ zP%23gr+m4tGO}z}${_V-4|`w4IlJz0z*4s=<)pEZtH4_2+~&cpM=Efv8Bf~lb?wt# zV#;$dh^F|^JvkGd+;tSTjdiaTl6tQecUmHTgtG>Ucl!=s= zDa~-6Hnq!@apV!nss!y)&!IlRLH)`-tx%*kl^C}xS?S0`HqV0;(dT#hP&^n2 ztc2no`Z}Jl0Jw;3c8) z1c;ai=d49kIWGB*W_mUE;6NOstMdw{WN^P~`wKsD5N2@^9#l+q{-6%6Wo9?A zPBQb6d-p(5afoP>M%KkJ#-9T6F7co|rTvLUL1AG*uf&_i1U#d{A?wm~&- zjx6XUUDi5aRPTwN+# zI4Hg|LU}kBuhFe{(X~T#ygqd$<$U9b=oF^_-H6_XC+2KH>*q4482q)U4nL>+b6_5ydl>Av#$UjcQ$a=c3c zLvFKtC$2|;l+XOoU?GJSqEUP=zyRM%U?}QG9~`XQJCKjp>-l=8F9aT+*Xz^b6MPFa zWC4WY-@)agaXFCVk1_dc{sR~O13Qe^^w|xJ7yyPW07C{A21YgjBbz>pu>mt1D}ar~ zkj;eAgyA1>!QIWnjLpOLZ`Q>8Vo|&yA}$x!9pkAbjgk3MII^-I>xm&1gOQjyP#B7| zz@T)nHMw$yPD+YV&^Rbbxka-n)Zj+}5+>e5<|ME4B=1N4YWCjS>k)g(7GUcN(6#k; zmo{blmossKpj}Uwxho9ePX;^W`#cTp*@pD(^tl6|1&|sLO8Q9EXFUbg0e;609PkZj z(pWxoxY3M#YK`QgiwJmd} zTZ;bY+cVn-2GKWvQoVb~h|(oen(UB;fnBsWAo<9C+k43L7uS#KJKo0+AmsoFg%j-J zDZKjg9obJoLdY#ZpWpE=NvE8nBECX3^W-Y|Vf-e0Hz(SP z8+lr^21?qf^$G#n;Od+Pl z(#Oy%pROcs@(c|E;R~J(Jsjao@k0MuUvt!CeKdy4Hg2sg#1$s@No}-Bx$hufuFaLG zRck{Lt>*>)o9n1gjK7BSR@Jl6dXMVzGFcB@AHYNlbsig^y&LMc&V>zp<*!wT=Cn1U zxHp?|Uu53*g?;h!PMTBzI&X6DLA7(2;_B8@Mmwp9*;Bg=TQB!knO&2g!J=!20w&xD z#L|m2%sGBpM(Otu)4PX5w)cvlVvGJ#@p5e4P*pB_rZ^HSHRb)BKMz{Y@tj7}DnGBJ z_ZmheTG~?X%f2yZOggkwY6Vf9e)Z1f!w@A`K)R2k}%=Y9Q zVCMDC+2x=BuSLD|o9}*4Ja-uTYICrtHb`Y_EW8SbeNAJ8>5?%-&%I=%Y{~!RW4)<= z*y+jZjy||hPHgy-Ym3t)o@d3&zzA51&04K-jI4YeVr-tZDL8eHQ`hhOiSl+fi{xdZ zc!u$stC$scQog)v&*51kC95QD_xquL0Z5HYMYjpq*j;=83Br?(e{DFb)3Dsc3Lp=z z&n~dNLik7*3S1!=e7*5@lI@L+(Efe)Gr8>=fThQ%lO6Z9mME2JL`k0 zo71LX5CRoIlppUpKiDav(gYkuoe7301b8K{1XE>6)5e86To6b~h(803+VgEvcj4lq5RmLguj@O=+V$a5BEmzIg~u`jnWb}at+*6S3b zYIrpKL)mc1Bu2-=@p~Cf2DOs)l0F~NNH|J}NV*&u#d7r}ud1OR(Y_xM7j; zdiK{PTa;>ivKXs*KCXE+1M>(O*wa7yl_Z=5{XMqJy(*{uExqCqM3L)|x_Hc-P8-2{ z@VRJZRicX*}UX zspte$vrF*|90k4jA3fYL=4(;SjG7M3I|I%3UJ7vsfu-H}RF2oF#xde6<4!R{W5n@p z^U3n{o}`zvTS_gRqJ8g#1BQh~P@X}Z=(@`=Cici3J$dQw)CMvPU+9lq>FxDwRQ&dv zlR&*c+pk>VYxh6o&vw)AY6lq}3Z_?9XcJ9Q&r9($JYa7Py$r%2SLPAq!1zIZsv^1jaE^8KaJFoOG~qcx zCq$F00nKdrd$(fi{`Sx(n(W=QQL=mdP9u`bqOHdC{)k$7R zDwY`&SGqc>M02CSi;s(tP;e9Q@K_Vj*J3xZn$N9M>vc-25HZkro=?c2X>{}Y6IA9F zH`C8`x8Y2;4!e!1I+dm(kRJ`LpCD-mdKTvF^P~#=UAy}o;;;+;hz}{Opo!zF|BEf= zwLT-ji43A6(Xt(iHIDze{N>8}a4QE`!+`^uC#*Iq-BCY&#P^pwfq3wJ@lXZ2)~Zp+ zaF|LgJ`dk@i9Exf@a6ZDN~KmuQP*x8}y&O$$$RJ>kIt$=tj`>KlO?7Kj;%S#=lvOe+n2g{V##P z^vRzMfd5zK&k_#Sf7|1yY{4P$hlT!{zj)|h8Dr-N8)X=}(K~DmQ121#vBo7ziFWotKy@U=_QM zj^tNZAx=RANn-#43PlnOCQgBvha(L}2~ib_(>f|Cj{ph-t#B?5@P~_|Qce)@1EVA& zp!3)}KiS9z(p0dw=lE{^h);XUa^Jdo+kXL%5-?<2O8}@SF$H0PWuPzn{&6S~1BgDp zA3NTVFed^GB-TF3cP$+~Ru0qa_b`l0?2r(I9cnt9`vkW59+1l9B&p`G5+4f@U(jG* zHHW+CKF8ko2ykFW(sB;GY-*2m@lU3T-V~z{o(h`hzafw@qHx-Um?3wW zVPkn;0+vNh^zgyX33wn%fX;%P7SS7e9>)Jvms z^%q7Xe?7%A3x%Y1n)QoPZTaYM2d^yOpOpnIl25tHi)UixZgo>IH2w9NC2XcAn4jbZ znPCtnk1gg`&g?D@Ey;Ui(w5`n#phhH3Q_gulv?be?b>viR@&+5-i&Q`aUI?`UgSZ1 z3n|R&KG)jwe1*@Dx1O|Cs^sbo;A2uV+uy0BowNHVk}o|8)?@RVMt57>t>_nA$o)nu zPp?=V1DDP@Pku4-)X5Zn>J19?VgeeR!s>)0?GCm|{LVy?a zk{zs4IB}YYqg~q95ziJ+ zd!#&zXygJpO8Mk{6Pjz^#>+fXAMuhl4@vk}T9e6piY!FinX)v>-^ii!$*Vs)OKa?o zxdNlEC7D{;zu0zi9&=^MC%d4fIIdPm|E85V$J@X}731l=Q9p>6WFIoLfpAvzh~{)m zT+E5UPgjcRYLd!*3+e3T+7L?wE9D%3&3el{41*2o&AGXnTzWe+3F1o-srp$XSQ%{H z@)@HHVX6Fy__L1vh#fm$u{^x`*^@<+v#Nb+nQ)ewhziwt0#?6}--VoG*)-p%J&d>N zd%N;Ve__&!{YA7>aTSGFi~esl#R^_qxplUy(L<&T(^h3T<2YrWTBJ1n!JQG}1GwzC z>4Lr0q9(1rC_b$Nh`{OF`G$az)c`o-HgfuR6f{y{IqdA&qG)-*3}6l(Hs}dP`Z@vPYzOOc3Mr&MXDM~sxxc%Li^ial zG9#R|L89puQGy?$fK{PcI?iJM$dbDGL25&;zhMimVuR##AA2V;nLB2X*Dd zx9~uuIUn%Z_&J3P^O#6o5ziG|+f%c)5$lJ?VS+L)xOF(-_(~_Izn$9NvghguXpjiQVCBB^8 zS=(s@;J}_SABSmC++I^NzAf;hdBcjv{rG|~g1i%zrPejLArmX2uL6>glj+uGXVQMK z_U!9Iw%Bh$opu}_HU*xZOS^NO4GvyC&9hE!dFi}FgzvZ z5l9a(uphLc3qkvg^Q2s5duD0qt=Z;f}k zAaepaAOp=PcayFZa}%08WV?jitFMJ8#NyIQGCx~n;(Pm6zi7vW%$CrUAD|NQ1gB{AIv$53>UHFjkT zJZ%Mk2Xsb#%0BVE@j>4E!hjY(>ArTwcz#*!8tBd#Fj7pj8uyyK+2J@rg!G*UdXK*! zec%W7o%F^AzG%&pN3;XKKJhgJdp^3{yjy`!HEL#TAw2YsUI#B*IPd6Bf5-@83W(Of z-Wl+V-r(0~z$^SIaQC6~^G)vK{qPLETlI`j`1uydcTT1)2SoQ@2=Nw+R}=6_Tp+ml zU%3cmt8)v)+)hgC{|2M@y{MkQw>kg98UPD}u^~X;*o2jtiHTL8g_X@fpTo$Aoz)P) z%mFZAX5ujZo1XbUIAZ9)=0se(tFRSgHm0FT=&-Q~6-k_M zN)JO|5A0q|3k!wxqLTn2S&IBz(GOBG>$==5=Po)2b+V^1s=di~tFE~f-LA>Y&-&Fz zmTKS5DfiAPK+Y6k>geM$`w2Y{W`?z4AV}|q!_|wxOa$a}48IZtWDfZ8()1(;5+3NU z#I0JC1rJ3OXYV(5z$a2Ug@Boe}_gTmzT* zV4=U323Gvw!oMv%!!O*Gdb}OL7K(eNDDdM22@!;ajUDR(rTpzj52DMBSsVf5@I)lk zE+NbpYI?k-yT>k1ynV&e$!**Xth$e~RM7fB-{%%xfq27&Uy|AH`OpO_l1fWomFZQSIkFX$kJ#olY*)+k9;T3mN+ zva7-qMj&fdrkg>~u7|%?;bUmihxz?|+RRr>Q21H(?ha|=wp!VX@pjIXOyDQvXK_V@ z3Pv^L+Z=|4)!=GOljX_3`diW>gn=H;7U3Kr~#Ra+=j!kxD_r4qz*2bBlX43-VgYZJdYLSEuo{O?eWR^{Mm%N8FR~=b5 zk9z-lJ{x9z$MNZE3s6_5`LlT$PAduHU`tpHCpd|fYIL);``;*8p-*}=6%X`dvg1Nq zEmrb&_sVHDK0^du1m$#k1>%4UCnF)$^5w8OsI8l*i1pJfTG!J#Mjuh+L9$bc?UiSV z`#3rp%kQ+dCp_z!`V62MiN9D-z*;1dd2^^}cyOJ2EN+InJ0*ik6@0qPsDm@*Z19AZ zY0z{^+&X2e=9&hxQdu??H4EzYcj+a$*a+sZKjjjTrg6WeWtrsfoe9@sGG}HrI?@jh z`eP#|O-@~8>NQMl2}h}ivxjDS0Q80J`?!7LT!HuAoPE)Z3ADEPF0?j>E$QiJzc?H@ zcamjk<74M8g`~|leTSah!n|IrvKJLH0w((|ufzrx*uA3+!2mmuuIA`Iyv?PFrt(Od zNmwRVC(}MGNe4;Y^NsYe$+ROH`4nr!IYx1mmv7rY%javv**ZSkXrqhO(JWU1STPQF zitUW%OkFf-5?qFD)=Z~f(=1!*Vr#5`op<++YPqgtg(?ziswpX2^K@)^y`{=}Gg~9# z=>@}t>XMHI_HZJ!5WMmJ>se{j$-1{S_^FfxCiy2+2V1lHZoPVnH2+BA)D2~tNySPb z?We_MOh=f6@>2gZvbi-LPYZ`L6Onl$wzYzXaA8fpaKx4~!a*B{b`=EM(TQ?e?nKJdo{Shs+1u*6HAWieAmQ{+qnlhz@quh^O}#Lq$-MmIl=4fWoXR@J4_>( zkr~D+oZeG^031EMo@3p(CbPuNg6tucMrYLu!5gfgJi`zzEyneZ*l0`JH%h6?iYO%i zQOe~`tZWls+%>< znX}%_ZJRVlE;oszzD4=hS@8w-QvR+-dzs@-#LL8NI>wjEeTwDgBZ&(M`_oeCME(0# zex2io%!kPRL*=x75kZh1|54brGyGpaXYb?EF)i^^El;Sd3~;-~E}(;gxg*a)zkkog zIEZkGWR)>|=>4)LF)Y56USe-~+7o=w)5K+s5>PnAy8BbLJ^}rym$`H*J|gK-X(vDC zq)Z0;UNo+$DNazT5Owvqq!z|OU25y);Jc^`U4iz_?Mw4l>}hPBs>vzkpd4~b_$c$i zFl}c@#)j3Uv-A#Vn(+ISWBF%u0?~d>1qmx`o)T!C(s1601qvH?@O_w5pZ;b&MY?p@ zl$Q|tACwmcy#pMM!c#-#qfgBhaivpA-V41Y^WU@3n=&cLIx0)pcuytqc$uN}`Qgkz zSu;Vwqr{}|?m%n{?v(VYDJC)W)31Vq8LQGnR}_SpU7MM7 zi%4$V;>UY_xW2aJpJaFP zo!l=#;Kz?bLZ{8G(Ylkvn;%$#)B8?7?u&oK$Da!9L|6~?F>1$GcO|uK&mi;u;2ZV5 zVgc%ZYDW8d5TJe}S)KCex_!RUyyFKyF9RO}KY^4#dw3sx`H$aqVXr=NBFWoA^#L$ z`};YSe-HfSQT}W&HvF3-`KQdEB_{0umQfm#J=*$n1m&;!%cJ}&W5Dp|qzR*uAv*xT zZp6l7z+%YAVaUSDz{JMJ4A5s{VgGaY-yS9EzdQ=7sRa|egSqJ+aB=6Df5RIZYQSro zhRp>e4;!g4_X!NAvn81I7lhR^r(!az3$Yv1kXp`{WE94+QbMhy1cZmiveG8c(30$^ zJe>fV)*LTij*gvj4j-nTp5A@Bj@*4epF6ky6iYBc0{Wth!oj3r$AZux6)@Mncb?QZ z(ZWENj~m`E(cMHKdQA_QM^a`eRE*mJ?@iOM-vx-e#YV&$LO~}l@%%_C`NOM01)r<& z!pGRZzVbZd7k+GgiV?+vFc^9dzQp5i06F#W%R=BsK>S3`^qglPt+gqjEAs>92OSKz zA-Y3^Nqa+C(BY}(69gZFTERbFu<)g$WF?-R$Us z_z4oogIu%U?3)515IVGhOP;>>@4&=~AB4FaD|oKmtF{>+4EC}7lQm#ff*Mq9Up%pd z_$M*byMu8UA#6twxo~>2RhIS|{SJ|H zs?2(aFPOEhq@MaYf9R~z;C48`pVPBC$lt6hfKP010nZI^ufBhl$IoH8%sF+@}f1dwL8+18sa`l z(=Q+O`uhC1q&@+6sXvUgajtXYblbn2Cy_*omPLFJfU%g$f5onaI7rU(~`bii&z0!P#9h%1W zH%UrkKxjR~>s~Idbpc6Wa-J&fX!rq00VJqF}UDh4@=V$FV`s<3E=WnboK4$8Z zl`dy$E#Ki{kn)mTq57$(2r$X9&&jpekx4Ph+^-CjRXj`pqW1jvtHVYR5QA=;x2twd z367!UWqzcs^9zz2bPCIjw&JdKC48DSFAO?H_H|#I!P4Zugrq*axfbAO^e*v=E_BIQ z$AHE?W>8mS^=H9)5lKNebl}X2bvRH^!aDRALwFV1qrE6^;a*{>T4C^=g^YCKDfIS(lVKJ! zHjT6G#3wh`dl_*VIXNnN?;R?&CO|DGNg(d0tBbuug0`dip1%a(g%oD1K&#%)1?nHd7cfy(SakAjmuZ0LC zG@cl(Zx|C#b&xxLir*6^U?R}p`bef+$q`Ukk2gHx?{RxN13`{S6WWP2c>M%SI;~-d z7-Xm%&k`dSQ@0rik9*_hd~E#rxKx%~R_|I(w?aLtLQo|`$Ign-pt7{u#Vwg)$d<&P zgDcN_(A&>vw5VgF%J{Za`F)yF+swE9Vr(fCu`{HW@~ABmsn<`fbQhmI^qF+=6p(v< zp0{hlw$l-9Pnsg?IQ>{+Gn9n%UN&-3O@LM|OmQfon`pq(V50fD(q_On47C5jv9HF4qAB;{R`}=DCW;#iw-NK)TKHvSeC2vkgz4fh^?3e6Ybss ztzEGt{y3>fI2`F<{^i*OFO}e~Gv4`A>pyRZxI$M^?_E2_!?L~Nhg)O#G`w^l9>Z3C z*hUab!qtRr|CRua6%LDscyCzllW#_ivz^=8WbczTZrV-KC{$&SO^l>nc4|1FbQoPA zwfmy{Q99_-(_$YADO-lmU7vRKx`BmaYx8P}o9Z4m3*-8Md-a3(kakaJ=PA?Dc}bQT zHF>L;wg~NBja;=@zG!1>tkmP7k^rQpaS(>%i1SRB{8QC8uepMF{d4HTSNDmJMBEAQ*Hyt!BHOThlvw zvhGpM{~KGnLr}7L{uC~M&0nzaFX&)qGG;JgU^Ft)XESDDHDP3AXJ%wE;ox9l`*ViX z2w=#;#>D&&u)yZV;o@Ln!2G9lF_lw1`BS=3P+j^vnU^CVpi)Z6i06XKgTl=ghC(BW z%f*s>`ywSTELR|`MXW&6{DsQj|JSe3{v;VN^I5P$Dt{VrDKIUy`{;UA)5lk1`;NZ1 zx7Y*;rn^q&qxYO6ZgNPHFXm<`i?tZT^U0iu`X6L$Bd|uyk3An)mt}=2i zye5}IByRt_xuaR}fGa3ur`lRhHnL{j6@o7^Hx(lK)91a39o&mK^okIukxG9T6Fkk( z41Bjba)jXluG&$}1xOO|yH&91K)~obA-IXd6cHV^go z1HNP73r6*MjX$p|2ftqJ4c+HoYcN1Y^y(ie#01zmzAX74+CXh@{nfc#N#8mEb-UhH zpMb?E`6={#Ep_&MR*%&YAd6=;89qK4^N}7Re$syABzKA#3uF1TuHrM)XiepX<@A&; zwXj^T@s-|=WdXVe4%TtmW}^9^tm?|;Q``O5;FXN*wH!3|^_3JRCGB(k0jbj^_!6^k z14+jI$;rQ?PXQPjHf2AoIHGFpx_AtJ=b0AYT!?$WK+vbC+m`yKkRelQnkQ(GHVDY@ zaAk4jSvy*HHtDnLRY-kb}AT=xHy}%D53W=ms)`Ct7YIaFr> z$etC$x|DFQRC?x$byoP0RNTAHbsTwRJL#Qp!rJ&cuVWexRmEFr#gnI?{w@-CYI6_F z3D58wwkDFa2{|hxTQt%2y>VfU4kv#TJ?Iu-oh#XwlS%LjdX3x*nmqKJ3fvRtmWIno zjrdWCR;RXWH^gM*;%rWQB$-t$mA5+*sVNG6=o8eOF_&xm_`-bTJeCE2!n{N2B(uR@g3NI4?Oi6$a>#NisYA9h7bfv2tz+*fRjs~w05X42 z0%;HR$HYlZ-};7DhHy!vB?F}zMI}-`Xi`kW(RUtr`&O?FB?-WcU0_KmKBlG0uB-U( z0u_=%{#fg!@+Bq9>s=?01XKv~`eKMndA{rhI%+Ph`6hyX$je(8RyrTkHRY`2${~Zw z{;L)l_>XdM^I2rO1th$>;NN$xVrN%ovxFVT*VwF&kX&V`x3 z$Gq0GO>eRkhM>WczU?r+e3HRAq&K8yo{`(5u4R6qFmcRS-Cyd=c4aY_-}`$5wag{)G1<>03+6QntzW1NXsq{G6?iJrsosGOD`1j@Ef9eZyEr{ei{Z)~_FV z%t?(*2HaM|$V<`K{_^V4z@#z({Q0z$nsh(cObQx_nl>fP2($>%cN zUxxLruWvs2qkF8R?&SI2`$Qs#o}ZbrI+a^XJ1VsDNSWeS(cVuF@cou8P2-!|ue@Bg z(<4RECn^rX-rxW-B>e`ea$w{c4CT0p%|ZCDXL0Ox$+XpCWiRwtA@!Hay52iUdv@|F zLQN!dC+>tjlVi7+6O@QCXRJ;ex%}k<7n-}x+Hq8AK!-QvM&rwL% zNY4tqUuLAxlD_?}l26{~^T0|zFZ%N=WnhbdRF9uL=QzYLQZa|jY*%PpaNhK57iz>@ zge#zmr_M|#+Q`0p-{OLdIp$z*50{k%ba4lQNVLRgm_MAvTQ9Te)dHwy{7~r32QSvn z0g!j+>o-!%7w5lbC(lT(;S}Vu=_or;lL-!Bci`68rjt$WsxPyXo1WHoIgD5M(o>g= z?D30PhtZ14^SpNknP^J&cO|_dKaLirQ;Vy0IoOr|a`%pO{f>3#2yPr)oM~k?owGhy z?yK=)O!Z=89>PV%!vzTg`oy}Dj6^A&jPedvmUVl!+`>dGLEcc_8d(7RhK{kTh~<-OzrH>82}IF|9piK!Yk? zP^-)lkhQuZ8LeEY+Q2?5&d79A5x%n0L9$6YXz$^aS8Z6OJw6HDU8nxLN32G%L^1RWBLAVX!59tSAF{q zzXbSKkJ4r5T?V{#;`Be6%1mMHynVO*ei-o@GdTHy!wr1c zyaIi~QAPcf;R&Q+!xK!QH*drkpt%Cof9@FN$6tVc#gUE3&J2(V_%< z8{#}xIn`?5D!@L4!y$SDPA{bVfEa?`e_q8r9}T(TTi{;6W9n)c4+Fb=#Q79dvHg3A zoRiV;HC!p5#ZkTm4f5;hk|yBa;hziKzO-$5pS%Gnf=C2W zJp}Rrx8qaI`&xwMR8eSITHiKHA{4>x<|sb3?|6k0|*t=+rIAhWD`F;Fme6idJHB6@jN zWW^I*kfCkh-ORenY;R{D@I^Sqncs4Wwt3PT97kfnrysP1aA*{+RQ)vc;hJ|V18-cV zNMG}<#%eFV-|wK^az449)R~qwo3>KHD24HPA7*3@b_FXIli6s%tPReqBfIzkrTqvZ z?Up)*qRd8%b-uH@!4+H^!s&Rpd{8~D*e)sOX1C|xLPD#^8Y%K#h}BYkmGU^QiPR86 zR<76RrP``8hndeZ7)<`ql~w6Hq3-s9ps-TFhwH@IkhtRJ^#J&2Mjx7)&xs+8pJ#KY zj4;&B#9EpiQGG$K%p+0WdVPooW0}o!A0siLsUCV>vqGvW_lT}+R<86_ZX;55=}a7@ zUKYyEw1%CC$B|LBfH87*!C&VFK%{B4eUkOW1%(+;3Puc<;u`jzkJ>JspK_w31}FJK&K9Lz zyrcI`#XMX<)GvkMlEJUgBwCyN8U!-_;olX6&J!aHMSd zvi2USzfFE@KoUVv?OY&SHR#30vnp54aEh_Rn8v6pESy@|HNs5N=Y%rtmO;K3gMoP} zET>#T7D%&Rt1&rTWE8%UVqii2Irh%=XvW$uaU}%}J&K`JD?+`MFw@SyWFSIc|C36U ziSD(V(>KcgkEmED!M5dgdU?Y7!nInIN=klArcBs3o>{KrQR*CTKVh46iAVh4ios-< zwA}V>XCLOK;mYRO-s{0g;cb2y%5}^Z4+EbV|nB>#IDcOnNiVm?hv_n@>QE7 z$+;$LmCQfqbEc#J__)NKkXOT3!*bS=6KThmVtI4AA8C#=b-(q%_+061!jvx+vZCXo z!0`{6$u9PD3gWj2H4EXY(odG{impH693wD?VGJH1h(yeC?Mn@J)L1Vg^#*84p1t*Q zF>oTv--Uc3huPRAqk8>9w$Ye(j{$WNf>+b@=!ym=9BoZI)-Df_2bhfGl;xT{L?O7M zamK9`X~*Dmn>XrH5xu*_jb9}D@bzYi;bcmT?)`pA(8{M8Ve|&TGvCu{u7n2*51AF8 z5r2|HUU}gC?##kq5Ijflm>i=~UQfU+8*wj9@6I7qZe)J6kc$m)Gw2D7Kyl?KJ&%kd zrimwbk8D%@96%M^rOlr`JtGSqAJ|&WFl%%lQO5EuJ zsw$sh7aZu=%MGrrkU#PBps7nJpO!MHF+}#%#3>fHx~;|mlsD6^-{cRxw*`3yIroHw zj!jE2)%oW@bYWZiVX;kM?pI})wxHE9;aq9nEQbyP=ODSGpex}$XN8^o$4iia8!*is$oU>&E4>rBBHsAZT3;QjDe1Q&)Tzd2U zk31HjG`a2v0UGzaf!}H#2RzFIdf$E{eg^X%263kjT23r2#e#%e`?H)_HX1T2wMoD!lzO7qMsVmL0-?=Z-X%dI6&d|nIx(DF=616yaw9Et`+_kiVrt^E zBn2!yVSq@UtFlxDY8mYH5oHD!Q};wD{5k3)(#}?e>l;p?aDsfW^j8MekZ+*V_)84F z_oOgUAIeZjR6NALA;R1zRVLli99y%vd9WIdc{>dRp58!@aXyNQdXTj)kq$eE!K68V zfyYjhcFE{B_RxBPPwd@ShBoGbhQ1%X&{-{ZhS(qX=ny07dIy2m_9aXv(BS1<2la)8 zn}M#Dj|*tcB#S+)gwFrTLSM?Zs`J-f1@1(Gvc)Lo_ja(yleiw5oBQtZw)`6$=X_)O zp=`zpRz_qt8c93VZYwl(`!06saEi~#HUtDLFx7*Gttu=eWT?7{&EUU0)FR&thlE7ke8UfxG0xW+-3*shcW z&c;6TQ!ms*sE2z)4SjIB*ms{$Km8(aRWIG0Ok=3A>R~TA&yKn`yiFNH6YCLx``DMU zgLk92_$s&4=Q>{Jw&zBM8Cy~RG8Js3>UY`lT%FkPmLy7NEZ?3_Nxzkv3g^yEe4V6 z4)kU6vCL7-@z^j&6eS~*4Y z*t5J#Pj+f;gq_@@3f7~Xh!#77lMhkcWjdu*YR)a$jLIKx9YxwQwHAFn`^7yw52Nu; zljAG4LEA();!nLFbquPxH{d01E!4WGCzsBZVR~%ZazNfiiiT|Giis=AaN05R0?cIW z8~tFfRKBHz1Ne3%^(!pVvodo>q@w!18ixvs~*?9>D0aL zbWV!#;I%1x#f}s^gT2Iii7-W+>Ya~2W?jt6J2O{QSChojQypMxwa;`h?1nrfjW(p+ z>GU{lHZ|pnI$6uXrqCr_aN=vOiqZB!@E15u+*KkXsugol=JoI!^vr_p1A6mozRgvO z7jn8L*YCnKt=f2MYZ0jTqcq-J`|NnFbG7;A9<4;-Qqk29zM3giYC|B3xJR)plh2E) zb!5e9WgdU%c*2)-l^`#r*zJSmpsf}x!ux@2cI6%d31@2jr1#l~o@{mpjD#j;U-|o40;=p!XcO`c|EibH~x7fCg!vAi$)tW+vJa1u@UWlYd*P;<+k;l8eLQj31E z-+C)0m!DybM5T+?@TH_HXE8kK@_i8no(pHX_?>4K1v9NLn<#r}iHZ zFBz`i`t#xPg?KJZ#cpb~T@Xc*AwQ$islnXKSyUHCWiv7!Y7VHX-rlzZn52mznLp7{ z3}?(-uZtGqL&<#>$v%m1b zZ{ylcKb;ZzB`ph(Td`0yst`np&mHvbg)e zU@bPQNn~uNhqAeBSw~0d?!+^2I2W8^f}@-;c2t(v1pPt1 zSn73)@~M81Mmsy8Ac4kvi_;ZdDKU?dfJ*IQ7K0LLVbow7pFIE95YA!r!vwiDbavAC zZ)%J8kTtHY%r4FE>+;F4zEKpRzE^2Q`)%)40N%(+!V^)s9Ya=rqA7i$_M$=}db{k{ z6mP=!lU{4M)*;myt!V|{3g-QvN0xmcKJ3U0Qdo&RPO>5pTrtMhMS=g~I2Axn%^0rR z4cV3t|5Va31YEd254{eRs3e?GUbt>RI7B3kJA58@W@zAa^!fKW(b%Hfm!Rj z4mB8hD^$}9UglnfNDY$Qko25FDxv2%{1C~lE)2a*Lw27tAf`|X{CRh*Mt-m10`w^7 z1ZX4cse;Je2=5 z25Q9YpBa+xV@%+!Y1W~E$@lwvaPo5ut2viBHw!zj2@fX^n~6Cu7aI$^IX5c{RR4{Y zhm8{|1jqZ2;G~H7U$r{6?q2L{R!)E)I*`B|0x)X|h44sv=)gp+sW+bgN(i+|Rn%7cSlRNShYU z26<+IpuAX;k*k}Gb2!bwQX+v39FvzG{84Byw#Z?SLrrLGV(bT5(t%P%v?yUDX0n`w zsA*q+!Fe}(t9o*YB0^?-VCJFT0E5PC!eYS5^SU5n4bl*50{-X^d~_+94=Dm-JF7y- z=Glk)e6}z9I(_J#8@4cEMsyf+R_*Y>6Bb9S_8HAE9a61C6gCz&-Or>}Sy>7&!_0Vg zv*~U7N=HoJx70q1ts6MKJQ<_Z2lE;)2mv9uJFk8!){^oes=qI$yx%C$KRsp>atQ*R zK@?{*`#mcjA|Hev9-%Y~5h-h|hs?>dn@vNOngxj{T!Jt?ZQA(in3& z>;l^|XFg|mBKAJPQdN^?4gBoo)H-T!+4xakWm>~Qp-L~Wo#7t)B6Zi`;(OUK^??R0 zDZ_Sf-uPc%qQ*OHiaF1}$sK?#k}Xbtz^s#NTi3t&l>QtYj+iegdU^ci0o+T;D+Pk5 zAEMXLNNn^EJ5N-Qtz_waj%|3wFD2nen$2FPw}Q$8lNf+#}mW z90>uY=Zcbeb77M=N#G-Djk+AH7qD#yYSgN`n2n^oulu$y5h<(iLbJjWhTE=a_lmUH z$gBiaZ;9rH70*R9qJ%2QBoTZive?1J`9cZpi3K`ZthB6If{g5~ex6IT?jH3VDYoMT zWH#7er+3E&J;`?;t-qTb)y*-6=-g{v+~(7$&jea;6c%|oDj+X6RmcgD)xRXPQY<-KmGGUjrZld~SSih)v_E zsFg!#B0)%d99j^l5U6 z(!zMXJQkHlRE3E2#ar&0_|!%HXt~NE~k>I6ZFQ4~WTrUth_&9TM~&Sts^LN^j5j z%&ATsG1|seG+UGTAkuUACKKo0OGTwsGy<{dRLi2NU9Z)I(l}mxeq2fvdu65Vja}gu z94g0bL0R9i?vxa~5+r@@g_X+HRktdWX96vu2o8G{oX7K0)S}3?axom-;~6Bb)1Pt2 z>^+LvqUUfkuz7sKRMo#76a7F)GXjP*tI`ORrs^}(X7{jS+qJhbc{XC| z>Ap{$$dO~2K0TG;u`N+y%y^TINx*5i90eF9d#M`$G4f$e^_ws3&KG&S88nhYjY zGOp1?2<(OU?k0V{xQrLID`mRfyj;#c@e@)3wdKZ?jykURBwk$H19H!{K6<^aWNH2* zqC-PHr&m5%%8Z)!$R`EuHBq|8Dp)L2LptI1*rw#4i38&SIFaNBs-UEExQU)&9mYdl zij+4P45blkLlFTz_wJRH0&+ZpZuc3&hqyY2fqU2Io7bV8IZn_AsudAx`J z3@*;jj?h6+xs#cHiNwB|w9Hn~?KG3AO>@gE)dk=;$>T@P3_Sgc+11E_+f>VzM?X$R(pU2G?W?>+D@{DQ7zofLYFtQz+%&J+ z;b9zUsMY~r5uuUU(zc%m^=)A>i0O?~tf9P1U!r>=hAF?;ho0Vy$SW%)i&jKk^f7B$ zyjt7BsS|nHc{7NFMC3Vh0Pn4a`QuuGP}1^nsSvOIb-p;2(?)Hs%`M&yv;#JEqz93i z!n52x=TyQB{&PE0?2H`Uc9e=4$Vc%8V+hh1SpO3zVj$ zQKC@Bo3*c^XugRR>o0GZtw4)#uB<3BJvMpdMmJs5nT0EqZ}zC=T3o78^)1cXYUTko z<04*VN^^%GYUa2Nj#C)5{DP0cQO?oFDH@uI!Y|=HXnvYF65`3J+B5EIm=}p;A^R!4 zji^-8|;kRlMff1u* zSQC0wfaZNAfd{jG*Wwq$J2I!EjoA+AJ>#>Aiaep^mDR<(q?tp-i*IL4<6OE1*9xQB zJT!R|ZPjMdJ)gKI6e^@0ZFU-j>@nQX2ge84PECl{C#fJ+?&M&+OSFSHtt*LelKIKY*#ZwKAvIc=YNX4k6!qHe4_@><;eL8% zw`G}KVvFus70Rhwoj#xl4ndi(V0y5%4@8C zIF?YPsIegxOC(M11QEagJfCKxk=D4dp#q;R!HW_r zRMMQP;T(UfR*HyFQbA`4o0Y@cpi+ie5ty7%;Og3o`yPTH3p&xGvyC%$=zO0abl~8p z2QQf`na><%LL^Y3Gy1L65ND0J;!LsdY#Iqe*pVTQt!3TztAE{vJK=2vfxhKTYa5v7URuTUHxFH? z2D=p=*?wp m;lCoH%$pRDSOtxaG*lT?i~cVC-~r{NJut(NCYl9{2%{H*W$3NI56BlSU{$0kgUPiaN@GUO` zQ-nn17BBQnnIh zJaAydH{{L!|2tGk1})c{X|Vi$s1orX5+ocP|KOPXrvbeG%#Pe0@I8Y9wZO&xKa6}I z12qD;Yo{{mMVJhfU-JF_oGu(5d(^^n0ZaJcG$;JEDXH+_1uGUj zQxpJR@@S7eca!bMH&NY@D$@3eL9Q6Dq@@tknQD=?(WwTFCh!G-C?*ZNpYNqmG=mkn7S?U>1-Ozyu+K zhC_-_(fqF%t~iMbWG06!u$8YuowO1VA_I&Wi|$3hW_FV2e-t5B!x;$ucqrx>d;|JU zazZ6M=yE!s54^qj=6y-l5`vBp!ePXSP#N7dvTjI}UqKe6^;*tv5J);Z4HmvRkX?pk zt3+bpf;25paWJo)Fc1YB-ms2KSJzm3EQ7XcW6e$G&`RlvSqt^8Bp3QO4cPrnLW@T& zbF3t8)ke&^mKLby!o80HOiqoc_@HTkwbx?y0^p|qYv|rPMnFtT!5g}B(e`4;P1vXd z{U(8C{znUqqWG!~Pd(>s7V5ie=rzs=4Pfd&uPG_c)>pu1Y<$Q>rS2K`)ZmIy_idbf zW`Am$?;5wLlNK{v4|etx;$Pl#d|7<0DL?`{FL7+$ve*T4pTUBjPR5!*W)GiLJIxa1 zlf!V61_331xWjR}?I1Y%M z+ePH&YE>NblNW`+Fk%KpruS>xYW-n8$~T#~3f{>mRNB^cVCHsI>fqAFI1cuhEG6gZ zC?EZAGzu}`tBQ@Q_cIc0(n80*hLx}|*3H3X z1-pfrbgLt2dh}Jfim@+!dZ6h#ahR0x>Ll05t8ZqYAz_eRCeyXdJNs*VNS<4q?w72Z z^tyeq#MtxN{Wl})5UrQZzO%AH`*Q#voi7@T@Ie6N%3JZ~HTjweZfb8%Fk2{B65Vo} z-#hx5CJpk)dF|C^(fY=dH)i$8pU##0OY0_2lX=A7+sF)%z+cD=Y8ceYd9^JWowu1b zhaMWK6RWvHs`dnwq|JH#j) zFK+=Y@)E=3${KlySI2kE->7eXrCiXZss8e;KZ=vmO4fqs!@+h)W1@v)BE9LZZ6j+) zQb^ZkuR1>W>o2y_foFCKwX=a)uLC{22jWk({k&p6(Q1|)31SnUU+VcfVAs#dH5F8r z%BaVrei`wLEm@Jx*Bg}TYu3P}3<8np%Wrp|Os`E8R$JvoRcN)DkE>7?njYtV{mP2% z#p3WeGc{tMeD+iA1J|V{>*CK}S`V%w8|fE}gfHtvB$eB5f@hK*2rW||miAB8cS;Y> za&qaGO3E9yQfxg%HT-N5 zThDnRFhh->T(@st(X)aPP`Hx6yRi)9EpBP8a~8KA*HYBRC_j1j$xu*7OT1p!wt>7( z@m>-Q8(lbeoS|2N<=*l-x?guN`ZmZ^&69co@BP4<_eFHOFE-OwOvytz?#jKxqKEDb z1u}{4+Go~g+1}skRuAx_o7@c6O+?>u53;kO2%#N%q|9$eq2ftrb7_s%eyc5;kOj;r;o(~Kk8dMWGxNFSdO_Lc%-sCFJi`;tgtPL zHObu92iJpqHP(#E_)g{n4?5ja9_L?y=kl>s1?wBIri8DzSN4widQ#f#Lp>~p)Dci~ z_=2&%vi40R*$|<1xPMz@?din_w$mAPZHPE!H5LkGxSus{8PgUzA{|o;%EYvbH+>#J zeE4K*d?vSk;`Pxa3sz_X-W8XrMZj0%O5$M#L)3aul} zbEcK{bGJcYBQO*44jL9AYIU}!LMB<$UbG%??Lsn7_W4YIPB-WrHn>QKCeDg4G|V(w$1)K8PZX&UWDD@Guf@ z&az*)moN*6EkdW(WL^&@w;5>{=FG${jChHD@0?}^e)zc+Je+YAE6RZXV=3I&d+&|Z ztD%4X04b{4bzaB~!G=;z$_TqXpLt}v@DMsmA1&gr(JHN|kz8j25T!%zf+SY%(S{+l zlNeQ}1v zR_O^#ABJ%gs4t6KMb@~jN}lfm6lu%6PFSvt`{EzOFCyzw_jeBj#%HS>^ZK39P-6&@_4GZBXa~TF2$E&?RgL@K6+F-s{ zb;P9WGg%^3DK$RCzphip$VTbzuOJ}@l*DRR_j$L!biRJ3kbRq!*+3Uwy;V!@IIUYw zb3UY#0Bq{7o5+9Ya`TEVzNT78*j>L_h|B~at|2^sUP9A8>(=w7EMy>WE;{1)Ne4rx zeWNR6OO>hC_mBk;{CezU&UShO(`_Qri6>G;hK#rrK9uqP|c>9!qF1Gs*>HJTEndf(4P3Y(zvVa-k}1bh-PLgJF$=-#x(u$i_P{TZ36tiq);r#r0tl(YQ>8aBK@_4eXYcrS^y z2H~T9KLW<FiSq=_ zq6x##h2Z!tXu{7@)z2lpC1V=4Up$Gb$IjPRlI=>bnqb63yx*E&V~jrntR*SpicE6?`JIswIB8nk~d*7i^9cXd{N{Um(< zc1kn+D3bnjfHf5GL+j<&>fH74`z}QM^zcLbUu;`l$U-{ph-z6-H2d3y5W!P@&5w+zM`y9z+Y+WU%PPkQ~i!n|LNpM8vA$5-9+}U ub@Kch^XF;pyR*3aI{M2xhUH(g`16-jCE5EwrbP`y4O0&d1tjdg+xvg#)rKPg literal 0 HcmV?d00001 diff --git a/test/unit/test_archive.py b/test/unit/test_archive.py index 5823da05..80f50311 100644 --- a/test/unit/test_archive.py +++ b/test/unit/test_archive.py @@ -61,6 +61,12 @@ def getPackage(self): return DummyPackage() def getWorkspacePath(self): return "unused" + def isCheckoutStep(self): + return False + def isDeterministic(self): + return False + def isPackageStep(self): + return True def run(coro): with patch('bob.archive.signal.signal'): From 4d0ce489816cdeb2295357a8ff8e086404e937b0 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Fri, 2 May 2025 13:57:40 +0200 Subject: [PATCH 02/11] builder: up/download sources from archive --- pym/bob/builder.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 4b1925eb..91a70857 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -1233,11 +1233,26 @@ async def _cookCheckoutStep(self, checkoutStep, depth): oldCheckoutHash = datetime.datetime.now() BobState().setResultHash(prettySrcPath, oldCheckoutHash) - with stepExec(checkoutStep, "CHECKOUT", - "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a: - await self._runShell(checkoutStep, "checkout", a, created) - self.__statistic.checkouts += 1 - checkoutExecuted = True + wasDownloaded = False + if self.__archive.canDownloadSrc() and created: + audit= os.path.join(os.path.dirname(checkoutStep.getWorkspacePath()), + "audit.json.gz") + wasDownloaded = await self.__archive.downloadPackage(checkoutStep, + checkoutDigest, audit, prettySrcPath, executor=self.__executor) + + if wasDownloaded: + if not os.path.exists(audit): + raise BuildError("Downloaded artifact misses its audit trail!") + checkoutHash = hashWorkspace(checkoutStep) + if Audit.fromFile(audit).getArtifact().getResultHash() != checkoutHash: + raise BuildError("Corrupt downloaded artifact! Extracted content hash does not match audit trail.") + + if not wasDownloaded: + with stepExec(checkoutStep, "CHECKOUT", + "{} ({}) {}".format(prettySrcPath, checkoutReason, overridesString)) as a: + await self._runShell(checkoutStep, "checkout", a, created) + self.__statistic.checkouts += 1 + checkoutExecuted = True currentResultHash.invalidate() # force recalculation # reflect new checkout state BobState().setDirectoryState(prettySrcPath, checkoutState) @@ -1280,6 +1295,13 @@ async def _cookCheckoutStep(self, checkoutStep, depth): assert predicted, "Non-predicted incorrect Build-Id found!" self.__handleChangedBuildId(checkoutStep, checkoutHash) + if self.__archive.canUploadSrc(checkoutStep, isFreshCheckout): + auditPath = os.path.join(os.path.dirname(checkoutStep.getWorkspacePath()), + "audit.json.gz") + await self.__archive.uploadPackage(checkoutStep, checkoutDigest, + auditPath, + checkoutStep.getStoragePath(), executor=self.__executor) + async def _cookBuildStep(self, buildStep, depth, buildBuildId): # Add the execution path of the build step to the buildDigest to # detect changes between sandbox and non-sandbox builds. This is From 5a42a73a38b0d2c58686ae7a7c1487dd104d84e6 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Sun, 18 May 2025 11:03:25 +0200 Subject: [PATCH 03/11] archive: add finish method This method is called by the builder once the build has been finished successfully. --- pym/bob/archive.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pym/bob/archive.py b/pym/bob/archive.py index 03ae857b..3b9616df 100644 --- a/pym/bob/archive.py +++ b/pym/bob/archive.py @@ -129,6 +129,9 @@ async def downloadLocalFingerprint(self, step, key, executor=None): def getArchiveName(self): return "Dummy" + def finish(self, success): + return True + class ArtifactNotFoundError(Exception): pass @@ -734,6 +737,8 @@ def _getArchiveUri(self): def getArchiveName(self): return self.__name + def finish(self, success): + return True class Tee: def __init__(self, fileName, fileObj, buildId, caches, workspace): @@ -1357,6 +1362,9 @@ async def downloadLocalFingerprint(self, step, key, executor=None): if ret is not None: break return ret + def finish(self, success): + for i in self.__archives: + i.finish(success) def getSingleArchiver(recipes, archiveSpec): archiveBackend = archiveSpec.get("backend", "none") From 2a36c505108993e8f097f06110f34e86cbcc5d31 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Sun, 18 May 2025 11:04:39 +0200 Subject: [PATCH 04/11] archive: add BundleArchive The bundle archive is for up/download of sources only. It uses a temporary directory in the workspace to collect all sources needed to build the package. The temporary directory is in the workspace since the typical temp-dir localtion might not have enought free space. The finish method puts all the sources in the temp-dir into an uncomporessed zip file. When downloading sources from this Archive the sources are extracted from the zip-file again. --- pym/bob/archive.py | 99 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/pym/bob/archive.py b/pym/bob/archive.py index 3b9616df..309e01ef 100644 --- a/pym/bob/archive.py +++ b/pym/bob/archive.py @@ -25,15 +25,17 @@ SKIPPED, EXECUTED, WARNING, INFO, TRACE, ERROR, IMPORTANT from .utils import asHexStr, removePath, isWindows, getBashPath, tarfileOpen, binStat, removePrefix from .webdav import WebDav, HTTPException, HttpDownloadError, HttpUploadError, HttpNotFoundError, HttpAlreadyExistsError -from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile, gettempdir +from tempfile import mkstemp, NamedTemporaryFile, TemporaryDirectory, TemporaryFile, gettempdir import asyncio import concurrent.futures import concurrent.futures.process import errno import gzip import io +import re import os import os.path +import pathlib import shutil import signal import socket @@ -42,6 +44,7 @@ import tarfile import urllib.parse import hashlib +import zipfile ARCHIVE_GENERATION = '-1' ARTIFACT_SUFFIX = ".tgz" @@ -944,6 +947,88 @@ def __exit__(self, exc_type, exc_value, traceback): os.unlink(self.tmp.name) return False +class BundleArchiveDownloader: + def __init__(self, bundle, name): + self.__bundle = bundle + self.__name = name + + def __enter__(self): + try: + self._zip = zipfile.ZipFile(self.__bundle, mode='r') + self.fd = self._zip.open(self.__name) + except KeyError as e: + raise ArtifactDownloadError(f"{self.__name} not found in {self.__bundle}.") + except OSError as e: + raise ArtifactDownloadError(str(e)) + return (None, self.fd) + + def __exit__(self, exc_type, exc_value, traceback): + self.fd.close() + return False + +class BundleArchive(LocalArchive): + def __init__(self, spec): + self.__file = spec.get("path") + self.__mode = spec.get("mode") + self.__bundle = self.__mode == "bundle" + self.__exclude = spec.get("exclude") + spec["flags"] = ["src-upload" if self.__bundle else "src-download"] + if self.__bundle: + self.__tempdir = spec.get("tempdir") + spec["path"] = spec.get("tempdir") + super().__init__(spec) + + def canDownload(self): + return False + + def canUpload(self): + return False + + def canDownloadSrc(self): + return not self.__bundle + + def _canUploadSrc(self, step): + if self.__exclude is not None: + for p in self.__exclude: + if re.match(p, step.getPackage().getName()): + return False + return self.__bundle + + def canUploadSrc(self, step, freshCheckout=None): + return self._canUploadSrc(step) + + def _getZipPath(self, buildId, suffix): + packageResultId = buildIdToName(buildId) + return "/".join([packageResultId[0:2], + packageResultId[2:4], + packageResultId[4:]]) + suffix + + def _openDownloadFile(self, buildId, suffix): + packageResultFile = self._getZipPath(buildId, suffix) + return BundleArchiveDownloader(self.__file, + self._getZipPath(buildId, suffix)) + def finish(self, success): + if not self.__bundle: + return True + try: + if success: + print(f"Finalizing bundle: {self.__file}") + bundleDir = pathlib.Path(self.__tempdir) + with zipfile.ZipFile(self.__file, mode="a") as bundleZip: + names = bundleZip.namelist() + for file_path in bundleDir.rglob("*"): + rpath = str(file_path.relative_to(bundleDir)) + if os.path.isdir(file_path): + rpath += os.sep + if rpath in names: + if not os.path.isdir(file_path): + print(f"Not adding {rpath}. Exists in bundle!") + continue + else: + bundleZip.write(file_path, + arcname=rpath) + except OSError as e: + raise BuildError("Unable to create bundle zip file" + str(e)) class HttpArchive(BaseArchive): def __init__(self, spec): @@ -1380,10 +1465,12 @@ def getSingleArchiver(recipes, archiveSpec): return DummyArchive() elif archiveBackend == "__jenkins": return JenkinsArchive(archiveSpec) + elif archiveBackend == "__bundle": + return BundleArchive(archiveSpec) else: raise BuildError("Invalid archive backend: "+archiveBackend) -def getArchiver(recipes, jenkins=None): +def getArchiver(recipes, jenkins=None, bundle=None): archiveSpec = recipes.archiveSpec() if jenkins is not None: jenkins = jenkins.copy() @@ -1393,6 +1480,14 @@ def getArchiver(recipes, jenkins=None): else: archiveSpec = [jenkins, archiveSpec] + if bundle is not None and bundle.get("path") is not None: + bundle = bundle.copy() + bundle["backend"] = "__bundle" + if isinstance(archiveSpec, list): + archiveSpec = [bundle] + archiveSpec + else: + archiveSpec = [bundle, archiveSpec] + if isinstance(archiveSpec, list): if len(archiveSpec) == 0: return DummyArchive() From 38affe258af772710bb5eedf39fb9c94ea0de035 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 2 Jan 2025 18:19:06 +0100 Subject: [PATCH 05/11] cmds-build: add bundle arguments And use the builder functions to enable and finish bundling. --- pym/bob/cmds/build/build.py | 43 +++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/pym/bob/cmds/build/build.py b/pym/bob/cmds/build/build.py index afc9607b..fe45fa64 100644 --- a/pym/bob/cmds/build/build.py +++ b/pym/bob/cmds/build/build.py @@ -22,6 +22,7 @@ import stat import sys import time +import tempfile from .state import DevelopDirOracle @@ -224,6 +225,16 @@ def _downloadLayerArgument(arg): help="Move scm to attic if inline switch is not possible (default).") group.add_argument('--no-attic', action='store_false', default=None, dest='attic', help="Do not move to attic, instead fail the build.") + + group = parser.add_mutually_exclusive_group() + group.add_argument('--bundle', metavar='BUNDLE', default=None, + help="Bundle sources to BUNDLE") + group.add_argument('--unbundle', metavar='BUNDLE', default=None, + help="Prefer sources from BUNDLE.") + parser.add_argument('--bundle-exclude', action='append', default=[], + help="Do not add matching packages to bundle.") + parser.add_argument('--bundle-vcs', default=False, action='store_true', + help="Do not strip version control system informations from bundle.") args = parser.parse_args(argv) defines = processDefines(args.defines) @@ -315,15 +326,38 @@ def _downloadLayerArgument(arg): sandboxMode.stablePaths) if develop: developPersister.prime(packages) + if args.bundle and args.build_mode == 'build-only': + parser.error("--bundle can't be used with --build-only") + + bundleSpec = None + bundleTemp = None + if args.bundle is not None: + bundleTemp = tempfile.TemporaryDirectory(dir=os.getcwd(), + prefix=".bundle") + bundleSpec = {"path" : args.bundle, + "mode" : "bundle", + "flags" : ["src-upload"], + "src-upload-vcs" : args.bundle_vcs, + "exclude" : args.bundle_exclude, + "tempdir" : bundleTemp.name} + args.always_checkout += ['.*'] + args.clean_checkout = True + if args.unbundle is not None: + bundleSpec = {"path" : args.unbundle, + "flags" : ["src-download"], + "mode" : "unbundle"} + + + archivers = getArchiver(recipes, bundle=bundleSpec) verbosity = cfg.get('verbosity', 0) + args.verbose - args.quiet setVerbosity(verbosity) builder = LocalBuilder(verbosity, args.force, args.no_deps, True if args.build_mode == 'build-only' else False, args.preserve_env, envWhiteList, bobRoot, args.clean, - args.no_logfiles) + args.no_logfiles, args.unbundle is not None) builder.setExecutor(executor) - builder.setArchiveHandler(getArchiver(recipes)) + builder.setArchiveHandler(archivers) builder.setLocalUploadMode(args.upload) builder.setLocalDownloadMode(args.download) builder.setLocalDownloadLayerMode(args.download_layer) @@ -339,6 +373,7 @@ def _downloadLayerArgument(arg): builder.setShareMode(args.shared, args.install) builder.setAtticEnable(args.attic) builder.setSlimSandbox(sandboxMode.slimSandbox) + if args.resume: builder.loadBuildState() backlog = [] @@ -380,10 +415,14 @@ def _downloadLayerArgument(arg): finally: if args.jobs > 1: setTui(1) builder.saveBuildState() + archivers.finish(success) + if bundleTemp is not None: + bundleTemp.cleanup() runHook(recipes, 'postBuildHook', ["success" if success else "fail"] + results) # tell the user if results: + if len(results) == 1: print("Build result is in", results[0]) else: From eb75749fef700cdfabf46570c62bf575f7fe28c7 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Sun, 18 May 2025 10:57:54 +0200 Subject: [PATCH 06/11] test-lib: handle process error Since 27d791fb the error codes of subprocesses are returned if such a command fails. Make expect_fail happy with anything but zero. --- test/test-lib.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test-lib.sh b/test/test-lib.sh index b2f26991..5667d7f3 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -144,14 +144,12 @@ exec_blackbox_test() expect_fail() { - "$@" 2>&1 || if [[ $? -ne 1 ]] ; then - echo "Unexpected return code: $*" >&2 + "$@" 2>&1 || if [[ $? -eq 0 ]] ; then + echo "Expected command to fail: $*" >&2 return 1 else return 0 fi - echo "Expected command to fail: $*" >&2 - return 1 } expect_output() From c14b3e351eef624edf668d57ff6a9335a25ab731 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Fri, 9 May 2025 11:32:11 +0200 Subject: [PATCH 07/11] test: add inital bundle test --- test/black-box/bundle/.gitignore | 1 + test/black-box/bundle/config.yaml | 1 + test/black-box/bundle/indeterministic.yaml | 4 + test/black-box/bundle/recipes/git.yaml | 9 ++ test/black-box/bundle/recipes/root.yaml | 8 ++ test/black-box/bundle/recipes/tar.yaml | 7 + test/black-box/bundle/run.sh | 145 +++++++++++++++++++++ 7 files changed, 175 insertions(+) create mode 100644 test/black-box/bundle/.gitignore create mode 100644 test/black-box/bundle/config.yaml create mode 100644 test/black-box/bundle/indeterministic.yaml create mode 100644 test/black-box/bundle/recipes/git.yaml create mode 100644 test/black-box/bundle/recipes/root.yaml create mode 100644 test/black-box/bundle/recipes/tar.yaml create mode 100755 test/black-box/bundle/run.sh diff --git a/test/black-box/bundle/.gitignore b/test/black-box/bundle/.gitignore new file mode 100644 index 00000000..75f416e7 --- /dev/null +++ b/test/black-box/bundle/.gitignore @@ -0,0 +1 @@ +bundle.zip diff --git a/test/black-box/bundle/config.yaml b/test/black-box/bundle/config.yaml new file mode 100644 index 00000000..9ad56157 --- /dev/null +++ b/test/black-box/bundle/config.yaml @@ -0,0 +1 @@ +bobMinimumVersion: "1.0.0" diff --git a/test/black-box/bundle/indeterministic.yaml b/test/black-box/bundle/indeterministic.yaml new file mode 100644 index 00000000..0e23fd0a --- /dev/null +++ b/test/black-box/bundle/indeterministic.yaml @@ -0,0 +1,4 @@ +scmOverrides: + - match: + url: "*" + del: [digestSHA1, commit] diff --git a/test/black-box/bundle/recipes/git.yaml b/test/black-box/bundle/recipes/git.yaml new file mode 100644 index 00000000..049965d9 --- /dev/null +++ b/test/black-box/bundle/recipes/git.yaml @@ -0,0 +1,9 @@ +checkoutSCM: + scm: git + url: ${GIT_URL} + commit: ${GIT_COMMIT} + +buildScript: | + cp -r $1/* . +packageScript: | + cp -r $1/* . diff --git a/test/black-box/bundle/recipes/root.yaml b/test/black-box/bundle/recipes/root.yaml new file mode 100644 index 00000000..e80db7cd --- /dev/null +++ b/test/black-box/bundle/recipes/root.yaml @@ -0,0 +1,8 @@ +root: True + +depends: + - git + - tar + +buildScript: "true" +packageScript: "true" diff --git a/test/black-box/bundle/recipes/tar.yaml b/test/black-box/bundle/recipes/tar.yaml new file mode 100644 index 00000000..a98e8fb5 --- /dev/null +++ b/test/black-box/bundle/recipes/tar.yaml @@ -0,0 +1,7 @@ +checkoutSCM: + scm: url + url: ${TAR_URL} + digestSHA1: ${TAR_SHA1} + +buildScript: "true" +packageScript: "true" diff --git a/test/black-box/bundle/run.sh b/test/black-box/bundle/run.sh new file mode 100755 index 00000000..d3509a92 --- /dev/null +++ b/test/black-box/bundle/run.sh @@ -0,0 +1,145 @@ +#!/bin/bash -e +. ../../test-lib.sh 2>/dev/null || { echo "Must run in script directory!" ; exit 1 ; } + +cleanup +rm -rf default.yaml + +# trap 'rm -rf "${archiveDir}" "${srcDir}" "${srcDirTmp}" default.yaml bundle.zip' EXIT +archiveDir=$(mktemp -d) +srcDir=$(mktemp -d) +srcDirTemp=$(mktemp -d) + +rm -rf $srcDir/* +# setup sources for checkouts +pushd $srcDir +mkdir -p git_scm +pushd git_scm +git init -b master . +git config user.email "bob@bob.bob" +git config user.name test +echo "Hello World!" > hello.txt +git add hello.txt +git commit -m "hello" +echo "foo" > foo.txt +git add foo.txt +git commit -m "foo" + +GIT_URL=$(pwd) +GIT_COMMIT=$(git rev-parse HEAD) +popd #git_scm + +mkdir -p tar +pushd tar +dd if=/dev/zero of=test.dat bs=1K count=1 +tar cvf test.tar test.dat +TAR_URL=$(pwd)/test.tar +TAR_SHA1=$(sha1sum test.tar | cut -d ' ' -f1) +popd #tar +popd # srcDir + +function run_src_upload_tests () { + # cleanup + rm -rf work dev $archiveDir/* + + cat > default.yaml < content/foo.dat + tar --pax-option bob-archive-vsn=1 -zcf "$(basename $A)" meta content + rm content meta -rf + popd + popd + + expect_fail run_bob dev root -DTAR_URL=${TAR_URL} -DTAR_SHA1=${TAR_SHA1} -DGIT_URL=${GIT_URL} -DGIT_COMMIT=${GIT_COMMIT} --download yes + + rm default.yaml +} + +function _run_bob() { + run_bob dev root -DTAR_URL=${TAR_URL} -DTAR_SHA1=${TAR_SHA1} \ + -DGIT_URL=${GIT_URL} -DGIT_COMMIT=${GIT_COMMIT} \ + -v \ + "$@" +} + +function _run_bundle () { + _run_bob --bundle bundle.zip "$@" +} + +function _run_unbundle () { + cleanup + run_bob dev root -DTAR_SHA1=${TAR_SHA1} \ + -DGIT_COMMIT=${GIT_COMMIT} \ + --unbundle bundle.zip "$@" +} + +function run_bundle_tests () { + cleanup + _run_bundle + _run_unbundle -DTAR_URL="/nonexisting/test.tar" -DGIT_URL="/nonexisting/test.git" + expect_not_exist dev/src/git/1/workspace/.git + + # editing code + build should work as usual + echo "hello" > dev/src/git/1/workspace/hello.txt + _run_bob -b + expect_exist dev/dist/git/1/workspace/hello.txt + + rm dev/dist/git/1/workspace/hello.txt + _run_unbundle -DTAR_URL="/nonexisting/test.tar" -DGIT_URL="/nonexisting/test.git" + expect_exist dev/dist/git/1/workspace/hello.txt + + # switching from bundle mode to normal mode should move to attic + touch dev/src/git/1/workspace/canary.txt + _run_bob + expect_not_exist dev/src/git/1/workspace/canary.txt + + # switching from normal mode to bundle mode should move to attic + touch dev/src/git/1/workspace/canary.txt + _run_unbundle -DTAR_URL="/nonexisting/test.tar" -DGIT_URL="/nonexisting/test.git" + expect_not_exist dev/src/git/1/workspace/canary.txt + + # we always bundle clean sources + cleanup + _run_bob + echo "hello" > dev/src/git/1/workspace/new.txt + # XXX: clean checkout does not clean url scms :/ + # echo "hello" > dev/src/tar/1/workspace/new.txt + _run_bundle + expect_not_exist dev/src/git/1/workspace/new.txt + # expect_not_exist dev/src/tar/1/workspace/new.txt + _run_unbundle -DTAR_URL="/nonexisting/test.tar" -DGIT_URL="/nonexisting/test.git" + + # test bundle-vcs option + cleanup bundle.zip + _run_bundle --bundle-vcs + _run_unbundle -DTAR_URL="/nonexisting/test.tar" -DGIT_URL="/nonexisting/test.git" + expect_exist dev/src/git/1/workspace/.git + + # test exclude + cleanup bundle.zip + _run_bundle --bundle-exclude "ta*" + expect_fail _run_unbundle -DTAR_URL="/nonexisting/test.tar" -DGIT_URL="/nonexisting/test.git" + expect_not_exist dev/src/tar/1/workspace/test.dat +} + +run_src_upload_tests +run_bundle_tests From 68309926864543bac964f6aeb33293d3eeb75879 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Wed, 25 Jun 2025 13:48:23 +0200 Subject: [PATCH 08/11] bash-completion: add bob complete file --- contrib/bash-completion/bob | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/contrib/bash-completion/bob b/contrib/bash-completion/bob index 11222065..33da852c 100644 --- a/contrib/bash-completion/bob +++ b/contrib/bash-completion/bob @@ -29,6 +29,17 @@ __bob_complete_dir() compgen -d -P "$2" -S / -- "$1" ) ) } +# Complete file +__bob_complete_file() +{ + local IFS=$'\n' + COMPREPLY=( $(for i in "${chroot[@]}" ; do eval ls "$i" || exit ; done + compgen -f -P "$2" -- "$1" ) ) + for ((i=0; i < ${#COMPREPLY[@]}; i++)); do + [ -d "${COMPREPLY[$i]}" ] && COMPREPLY[$i]=${COMPREPLY[$i]}/ + done +} + __bob_commands="build dev clean graph help init jenkins ls project status \ query-scm query-recipe query-path query-meta show layers \ ls-recipes" From 44f2560ca6dcad3861494f790fd8a01c74cc5ede Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 2 Jan 2025 18:19:24 +0100 Subject: [PATCH 09/11] bash-completion: add bundle --- contrib/bash-completion/bob | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/bash-completion/bob b/contrib/bash-completion/bob index 33da852c..1a6de593 100644 --- a/contrib/bash-completion/bob +++ b/contrib/bash-completion/bob @@ -132,14 +132,18 @@ __bob_cook() { if [[ "$prev" = "--destination" ]] ; then __bob_complete_dir "$cur" + elif [[ "$prev" == "--bundle" || "$prev" == "--unbundle" ]]; then + __bob_complete_file "$cur" elif [[ "$prev" = "--download" ]] ; then __bob_complete_words "yes no deps forced forced-deps forced-fallback" elif [[ "$prev" = "--download-layer" ]] ; then __bob_complete_words "yes= no= forced=" elif [[ "$prev" = "--always-checkout" ]] ; then COMPREPLY=( ) + elif [[ "$prev" = "--bundle-indeterministic" ]] ; then + __bob_complete_words "yes no fail" else - __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --clean-checkout --attic --no-attic" + __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --slim-sandbox --dev-sandbox --strict-sandbox --clean-checkout --attic --no-attic --bundle --bundle-exclude --bundle-indeterministic --bundle-vcs --unbundle" fi } From 6cc4dbc4cc5d6131fb2212b8ecebb0c75dc73a0c Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Thu, 2 Jan 2025 18:26:29 +0100 Subject: [PATCH 10/11] doc: document bundle --- doc/manpages/bob-build-dev.rst | 20 ++++++++++++++++++++ doc/manpages/bob-build.rst | 2 ++ doc/manpages/bob-dev.rst | 2 ++ doc/tutorial/compile.rst | 27 +++++++++++++++++++++++++++ 4 files changed, 51 insertions(+) diff --git a/doc/manpages/bob-build-dev.rst b/doc/manpages/bob-build-dev.rst index a57a6c6f..1a1d61dc 100644 --- a/doc/manpages/bob-build-dev.rst +++ b/doc/manpages/bob-build-dev.rst @@ -96,6 +96,23 @@ Options This is the default unless the user changed it in ``default.yaml``. +``--bundle BUNDLE`` + Bundle all the sources needed to build the package. The output of this is + a zip-file containing all sources required to build the package. This also + enables `--always-checkout` and `--clean-checkout` and can not be used + along with `--build-only`. + +``--bundle-exclude BUNDLE_EXCLUDE`` + Do not add packages matching a given regular expression (regex) to the + bundle. Can be specified multiple times. + +``--bundle-vcs`` + Add files used by version control systems to the bundle. + +``--unbundle BUNDLE`` + Try to download sources from BUNDLE first. The BUNDLE should be the zip + file created with the ``--bundle`` option. + ``--clean`` Do clean builds by clearing the build directory before executing the build commands. It will *not* clean all build results (e.g. like ``make clean``) @@ -363,6 +380,9 @@ Options ``-q, --quiet`` Decrease verbosity (may be specified multiple times) +``--unbundle`` + Use bundle specified by ``--bundle`` as source input. + ``-v, --verbose`` Increase verbosity (may be specified multiple times) diff --git a/doc/manpages/bob-build.rst b/doc/manpages/bob-build.rst index aa88335a..10c386c7 100644 --- a/doc/manpages/bob-build.rst +++ b/doc/manpages/bob-build.rst @@ -26,6 +26,8 @@ Synopsis [--install | --no-install] [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] + [--bundle BUNDLE | --unbundle BUNDLE] + [--bundle-exclude BUNDLE_EXCLUDE] [--bundle-vcs] PACKAGE [PACKAGE ...] Description diff --git a/doc/manpages/bob-dev.rst b/doc/manpages/bob-dev.rst index 23fadc4c..1b7dd3c5 100644 --- a/doc/manpages/bob-dev.rst +++ b/doc/manpages/bob-dev.rst @@ -26,6 +26,8 @@ Synopsis [--install | --no-install] [--sandbox | --slim-sandbox | --dev-sandbox | --strict-sandbox | --no-sandbox] [--clean-checkout] [--attic | --no-attic] + [--bundle BUNDLE | --unbundle BUNDLE] + [--bundle-exclude BUNDLE_EXCLUDE] [--bundle-vcs] PACKAGE [PACKAGE ...] Description diff --git a/doc/tutorial/compile.rst b/doc/tutorial/compile.rst index c101286c..cbb2960d 100644 --- a/doc/tutorial/compile.rst +++ b/doc/tutorial/compile.rst @@ -350,3 +350,30 @@ the zlib packages: :: .. raw:: html + +Using source bundles +==================== + +A source code bundle is a zip file containing all the sources required to build a +package. Such a bundle can be used to compile on a air gapped system, to +archive the build input, to transfer the sources to a reviewer, ... + +To create a bundle for a given package you need to build this package using +:ref:`manpage-dev` or :ref:`manpage-build` with the ``--bundle`` option. This +option takes one argument specifying the name of the bundle file. Several other +`bundle` arguments are available to control what goes into the bundle. Refer to +the manpages for a detailed description of those. + +For example to bundle the sources needed to build `my_package` use: :: + + $ bob build my_package --bundle my_package_bundle.zip + +After that you can take the my_package_bundle.tar to another system and use: :: + + $ bob build my_package --unbundle my_package_bundle.zip + +to build `my_package` from the bundled-sources. + +.. note:: + The recipes and `bob` are not part of the bundle and need to be handled + separately. It's strongly recommended to use matching recipes. From 4cf0a21c06a4eb5384f64e83a29250b992d24767 Mon Sep 17 00:00:00 2001 From: Ralf Hubert Date: Mon, 9 Jun 2025 11:04:46 +0200 Subject: [PATCH 11/11] builder: handle bundle state changes Move the affected sources to attic whenever the bundle state changes. --- pym/bob/builder.py | 21 +++++++++++++++++---- pym/bob/cmds/jenkins/exec.py | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 91a70857..e1992cce 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -68,9 +68,11 @@ def invalidate(self): CHECKOUT_STATE_VARIANT_ID = None # Key in checkout directory state for step variant-id CHECKOUT_STATE_BUILD_ONLY = 1 # Key for checkout state of build-only builds +CHECKOUT_STATE_BUNDLE = 2 # Store whether the checkout origin was a bundle or not # Keys in checkout getDirectoryState that are not directories -CHECKOUT_NON_DIR_KEYS = {CHECKOUT_STATE_VARIANT_ID, CHECKOUT_STATE_BUILD_ONLY} +CHECKOUT_NON_DIR_KEYS = {CHECKOUT_STATE_VARIANT_ID, CHECKOUT_STATE_BUILD_ONLY, + CHECKOUT_STATE_BUNDLE} def compareDirectoryState(left, right): """Compare two directory states while ignoring the SCM specs. @@ -88,6 +90,10 @@ def compareDirectoryState(left, right): right = { d : v[0] for d, v in right.items() if d != CHECKOUT_STATE_BUILD_ONLY } return left == right +def compareBundleState(left, right): + _r = right[CHECKOUT_STATE_BUNDLE] if CHECKOUT_STATE_BUNDLE in right else False + return left[CHECKOUT_STATE_BUNDLE] == _r + def checkoutsFromState(state): """Return only the tuples related to SCMs from the checkout state. @@ -356,7 +362,7 @@ def fmt(step, props): return fmt def __init__(self, verbose, force, skipDeps, buildOnly, preserveEnv, - envWhiteList, bobRoot, cleanBuild, noLogFile): + envWhiteList, bobRoot, cleanBuild, noLogFile, unbundle): self.__wasRun= {} self.__wasSkipped = {} self.__wasDownloadTried = {} @@ -397,6 +403,7 @@ def __init__(self, verbose, force, skipDeps, buildOnly, preserveEnv, self.__executor = None self.__attic = True self.__slimSandbox = False + self.__unbundle = unbundle def setExecutor(self, executor): self.__executor = executor @@ -1102,6 +1109,7 @@ async def _cookCheckoutStep(self, checkoutStep, depth): checkoutState = checkoutStep.getScmDirectories().copy() checkoutState[CHECKOUT_STATE_VARIANT_ID] = (checkoutDigest, None) checkoutState[CHECKOUT_STATE_BUILD_ONLY] = checkoutBuildOnlyState(checkoutStep, checkoutInputHashes) + checkoutState[CHECKOUT_STATE_BUNDLE] = (self.__unbundle, None) currentResultHash = HashOnce(checkoutStep) if self.__buildOnly and (BobState().getResultHash(prettySrcPath) is not None): inputChanged = checkoutBuildOnlyStateChanged(checkoutState, oldCheckoutState) @@ -1147,7 +1155,10 @@ async def _cookCheckoutStep(self, checkoutStep, depth): elif not checkoutStep.isDeterministic(): checkoutReason = "indeterministic" elif not compareDirectoryState(checkoutState, oldCheckoutState): - checkoutReason = "recipe changed" + if not compareBundleState(checkoutState, oldCheckoutState): + checkoutReason = "bundle mode changed" + else: + checkoutReason = "recipe changed" elif (checkoutInputHashes != BobState().getInputHashes(prettySrcPath)): checkoutReason = "dependency changed" elif (checkoutStep.getMainScript() or checkoutStep.getPostRunCmds()) \ @@ -1171,8 +1182,10 @@ async def _cookCheckoutStep(self, checkoutStep, depth): BobState().setAtticDirectoryState(atticPath, scmSpec) del oldCheckoutState[scmDir] BobState().setDirectoryState(prettySrcPath, oldCheckoutState) - elif scmDigest != checkoutState.get(scmDir, (None, None))[0]: + elif (scmDigest != checkoutState.get(scmDir, (None, None))[0]) or \ + not compareBundleState(checkoutState, oldCheckoutState): canSwitch = (scmDir in scmMap) and scmDigest and \ + compareBundleState(checkoutState, oldCheckoutState) and \ scmSpec is not None and \ scmMap[scmDir].canSwitch(getScm(scmSpec)) and \ os.path.exists(scmPath) diff --git a/pym/bob/cmds/jenkins/exec.py b/pym/bob/cmds/jenkins/exec.py index b07f42f4..4109bbec 100644 --- a/pym/bob/cmds/jenkins/exec.py +++ b/pym/bob/cmds/jenkins/exec.py @@ -223,7 +223,7 @@ def doJenkinsExecuteRun(argv, bobRoot): with EventLoopWrapper() as (loop, executor): setVerbosity(TRACE) builder = LocalBuilder(TRACE, False, False, False, False, envWhiteList, - bobRoot, False, True) + bobRoot, False, True, False) builder.setBuildDistBuildIds(dependencyBuildIds) builder.setExecutor(executor) builder.setArchiveHandler(getArchiver(