From d71442b04bbaf4742ede5720943b8b1d74943af5 Mon Sep 17 00:00:00 2001 From: Bart Maertens Date: Fri, 15 Jul 2022 10:10:01 +0200 Subject: [PATCH] HOP-4061 - Hop in Apache Airflow initial version --- .../airflow-dag-run.png | Bin 0 -> 16819 bytes docs/hop-user-manual/modules/ROOT/nav.adoc | 1 + .../ROOT/pages/how-to-guides/index.adoc | 3 +- .../run-hop-in-apache-airflow.adoc | 99 ++++++++++++++++++ 4 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 docs/hop-user-manual/modules/ROOT/assets/images/how-to-guides/run-hop-in-apache-airflow/airflow-dag-run.png create mode 100644 docs/hop-user-manual/modules/ROOT/pages/how-to-guides/run-hop-in-apache-airflow.adoc diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/how-to-guides/run-hop-in-apache-airflow/airflow-dag-run.png b/docs/hop-user-manual/modules/ROOT/assets/images/how-to-guides/run-hop-in-apache-airflow/airflow-dag-run.png new file mode 100644 index 0000000000000000000000000000000000000000..6799ded7c64b1f7f14d0279eec3199c0d6526ed5 GIT binary patch literal 16819 zcmd_SXH-+s_brMI1qBR9m7<|ZM`_YU14x$=kPgzj^cIRB5W4gplqv)Wy%*^nq)V3) zdJi?Ud+>Mfe~kOaedB$2<9@jBePHa2!`bKTvi4eYuDO!n4~o(R_*D2fI5-3{?;$EU zIJd8HaBj&wxD8zKUmWuV8obZ%H63wq2-|M{ZpE;Z_tKjKR5&YlYrSPB7t-tel@%#QS z{Wxrt9*6|G;gdWu(xnr--X{XTH@)5BpRplKufKeQu#ihi`Q z$I69i=pmBQ{KaQ_-kxz?33A^R|)!pw*#3tpWw4h1GD^_LVBoP$dl~i`L$gz z3FD@u;68R=&s#DZ3D(Z0#`h1rSsCs{c#~e1_PhaXOu13375-w4vzzzHxMhg0^S!cI zf5YNp%st?);cB+If*8|P?ey{D&v$aWJ^6!(YG=)`gkPWkTW1PGzi5=CX&7i?x+7W> zKf8E&j*Oxdp3Vl0W5(NisHCaKe)yf`y8+e7DK)U7k&dBaQcPY11zL3bn7hzW5km5a zbmM0G8d*v`?S8oX_pfi)W~ODMr!0FMN3f5;U@(e9wJTR;$X03$vuC*a@MeC`8{>_1 zEMmyDiE6xV&7Zv!pD4=-@lzm;c6g?2WFpDI!#Vi&u!g1h7?|ju!nb_+Wa`cqhY34} zI-c;4pX9kjEQFRP<0h+KNy56nH7eK$t0wiHSOooOp1N6Hfd#d+vViq?V=4C4pL`Z! zd1_H;F$jb~o#t~%drsfZ8hv^NxoA?+^xro6??JdX2KG1QTWX7=e7si!hXS;6BX%p? zRH3*k9R9O?;k88IqdQ6vZ`ZM$0wrFK)wqyz4zD@5RP$N&Zq7=}rjLWK9ufRAheG-q zBH^~Kiq!8DC7nBj#8I+5NKZDlwNjh%*B@6_Y$^}@Bj|O0vLhAF$z~BR!FW+-7 zD8p?PlFS;>-;fKxD|HBHFL)lg&E3qO<2g-*CZg<51XB_bZ!9Ms*GN|cL7=2CEa?jS zZWakFMQWvi&pU+d-&VR_%pqFE-_|J3rb)oFB4m`6uN$1MTVKXtSAxGO+8T6YYM;ya z1bYx%}s67Hqi`mK=+KwXBpM+9fBuIdfEpXkf+agia`z<-AP|Y}3Qy<&hS0mDz4S@s zNe$ya7h+3wNmceT3HFLa#dFGdy?jJ@`!k>3=DJ9!RaLf|v~q-Lt?ds9SCrO?JR`@n z$+ZH))OP_77w_Lrt|TRC6uWoM>Djp)|N8W32<8640_Vt9bkJ+ll9WZb5%1rGK+_sHOU z#B9kYuXJas$wnqoE7v!PtR-O%oda7%8|mNc*hjwUXLA0uc;WpO-gS8ZZtW17t;Euc zk3a^3vaK?LDkhL_eLmSXN_vVp>|?_%E1a&)Bm&t}VFnd$*KE6J zR7;fcPIIdwWwNbE?>2MzRRG@Ew>g?-bEGUHX_IKv1}n+PkyNJ>)Tes>=<`nDr%=a= zhKfN5WK^wkZSEp4H_}ay?|9pH8sF;1q5Grl)3f`I+^OcwC!>+kTats*bwp@u1GrA&K8K!OeadxG6qaKP0PatiGD{6H{8-S5G<=B;{b;e(PU526a_4 zeu!`F8On3M71b<&^RGjqacAwJB9}qZkc^pRGRw2gF!`SzxHvcz{7~C{CVHRBnbKKCK0#Ua zjjw{5+w*U5cS|ADxo-QyCaH`v(&~YHZ&3 z=wX@BJFNgho_vhbp92l@sR-BE61{^`4pfVBZMo9f0M7fJ%Q(egc%to5(c)_Vi#^Kj ze_s9NmXqedA~eo;3c88+_4X>!lQlutNsrKm^s@J;ZEML!<;-3V625zJoo)2fATKGQ zo2UubR2fr8rj_;*#wXJ$zx>oP{ApTB)XL#cBukK<(yOy6g7`ZG1TI#`TS(foJM=^@ zjX+FRzc=fw$dzKmD`o9yK}I!d9CN?oIJ1WX)eq%wc@TJGC~zosdJ9iS6jz3V?eXbr ziO1mNXB#)6{0;lJhA~FT^R~wD7!DN;)*EU`u>XyK*m+^v3ET z;x)CuziFCl+T|S_Tda}CU&Aj+7{V_SSDWALTax>K_yt%$=7e%^f-4hx z?pWNOS23L2s2-qyfKCqSVR|mCf!caTTuvpnRyd`yRX+fu_~#j(^>ow4#t8;oG{nW< z>uZ=+ad<2EJ!R{U>-%Jro}}q%hP0nQg;+^I+6H+dn))JzJ?f#SJR8q%CK2mMQ|$Tb zpbt~-&b4A;Y_`vol5vq6s$+(V|H_y&wO5%ip;_7SJ<0Bs-9x*CmxejxuOrEo+_8Or z$`5YLE+f@Dt-?A+MzW2znjuoC%qko(i?0J zo;z@I1p9Oa)ro(&#|=E<-jK;BHn4BZsi7M0Iu2;M}MFqW0=an$4qq}p|nHwDxVN82Nv*_Ir}5h zSlQer3%9^vVx+gGioj%f?|RX6QZC(=p+2|l-!NCe%rCX_<)>2d<6W0O=IsZXS)b9I zVaVeujCUMpUMYo}Cv2Tfnp|vr>cuCBK*DWr-t(BCO`mJH5J_@XQ)C-@95^XCWj4I5 z&LHai^Xf>xCMU}vGn{`sj;qPg*6+oaG%mn?a9cJOTilnYl+A^aTeVb8GnUC9(T=fyL{ghv!Y4dKenEu#2fv(pXIyTQTx?QMhs=Q3k zX{SfqHz1wjAAHPAJU21I8ci;Em{tDyq?>cFQ})rasPP9fXn88F>3t^8b_`EC8%o?g z>(GM3PpRMr68S32mx0B3#8=o2=xYG5uB?Di~&#U26N0yRtL#>H(9}STbY_x{VB7myM2dG z3v=?Ky{wd>a&6xqz{PxTqB8xrtr>If0(KHUQw%+9ck)S|P~n?7FET4*6QU9A%H+FE ztyow<0{m2d6Eg{m=`SrzUlqLb_~uv}d=;{)s^QDYzAdbrHKMJ<^Kc`AbdwSY#H|-^ zC)lmh933a@vmlTR4%MoBt+W$cFALnv?{y6CG~C{5!wQNF)Z6%O!5`c>d+xXWeY{+x z6rg^N7kV5Y1}C4C9mtMbE-$sm&qgs+Bt~0fmf-Iqi;Z>uGkGakD(EyI&$8%tF zKuU4i_1#@ne?k#ea*){M26t7a-NIsP=R>vQYOB;kC;Mg*J{djs(_cr1+raUYx-)4q z^fW~A;lHRtac@?VQm9wid4~->p552bsa37OJ2q;EK&IaR{Ew7S&}6kDos|BFP-YYC zuHZoq@=8m~`77GK;E4%ENC8{A-LDb6#uV2gn4jRJW{+x}E; zsLgc$lrfsySK^=TMkyQ%A!!JR<@6Ktl#F`xe4F3>cQ#cExa$8Un zyW%lITva30cWZ;_GNF!+WBWb!KsOwLX&VJ`ecZRgsRxM2sR5tKye;}<@72Uy+V?Q^ z!5#S!)nfp8;mj~p?IKX6^ZJu&H0BM&$*=i$g4Xefd{R=}&0q%~5A=VO>AS#ASO5C( zmiu@y@)}ytjJM0U6NN(uZf09eX{(BL&)FD$T2vWqw4}#fYDUgo)Z~z9z`XsM^}6Lb z(C1LqW^r?JDUSYTSb^IvS?THZ&g%lXQ?yM2hO0y!3U!%He@PZFD4~gqRkZ6sxwxS4 z{`CdLtp_+b*grlVKH(o7S9*Cg{4^T*uC$5fF8>nkTsM2g?5IZ61p*7d_PKXtu*3Cs zJ6)IHyv)o=j07YJeZ8uJ?z9PrEVhYcW;ut$yGm?5252MYA*mfIx=rqtYeJmZsbKhv z=PbeS?EJ61U(CF6S<0)^az$fPi_(y_$8RP*Gm3Vy*7keDvVZQ5QIVOSA{_F**S%;p z5PePplH*HAXX&pmj8ntGdAU3UMns%6T)ti(kKll>Y_ zeYLCRxZUn_HGk=P&qry?wQ5t#FYU15`sfHxN%}}BhP?m%2N#mbi7eyoA)BHpYh#%f zSi|p+StD9H&5i!i(5d#E`fxJL?M6H4L~H#R#8LA5yh-#<&y1RhKot8<)ruW!S$aHJ zJ4x2gqcO{^5J40vsKtzHyc#MKA&m2M0zcQ@L1rje&#qo?G`_ult(RcPYq}P&T!ibbY0-de zoP+B76zp_;IanOSt*JB~d#fB@&W>q)3#S=C`0aQv!#FT)zDi#|?HrH3YPUA5X>OANj4 zTYns9YlxlPwPntTb0mqQxkH9{+Bony+azI!^S49BcJ_f-udswh`zE2gOQppReqoJ4 z#Xr{4^a(-6ok;%NzExY60R+_$b-X2e(b4HlUFrEG`yCwL6l>o}#;qzdA1iUANNdU! zC%9}C_vI)*?$VNZx7VfG_M_j?=fsOkKP&BjS&3g7V&rpb~{%ScT~`d z;ANA?Wr}awSqm4JsLE8&TJB5Dz9f4DTC)3~NC3Ep_FQuDLW=_BO1%kfx~4JA5n!n0 zwE^q7KWdC3SiOr$#iOcU@=BA=|DO4=ML``d)%u>Sy#qnk-B_n$j5~r)02)$@amU-8 zwL@+MOG_wQGsu$MHKxY#wTmt5&a!pyk&y(tTaLgy;-Fznr13B`n=Sif$@7@ zfsAC>4?5`uW%-X;h*UTxt#vsM8*zcf+Qy*7H27z`(wnf3|9RhrdUkpg$j^}-%GOtiGf6c&EoGCydM<9`s) zOb+#lOKv5*@4MCYw7>Tow5^vrQjCp{@kM)^y4qRV)(BT^Zk90y3I_Dlbpg!YG{vQp z*JLCdb{=7O0uu#nsW7mP$C+k5W=Y9I*srxqyjf9beLFEus6c>ahcnL_-2W=(3kcLb zR(VjCbv7A=Sv(WFe$8(sSOQb3KGuw@8U1CPYh!l$Lk!6M^#4{GV@t7Jj#kbnXT;C0>T2mW zU<9SpdZq6kaqYkF|KwFM>#demzBPPVxwTi8__Wt1h%w%f|7gqXh*)09L@R&0((BVs z*=@$Bh36sd*#VVtwY6TR7!`l1_Aahq=FP49z1z$T(2kYGBP}%F&0@3hJ|Vm)I@%q) z@Lp+1isAswQhA*$hwiefCTv{?h!lLQorDYagM6$0adcJdqd(SR-{5oCSxN9BI^SiY z9{f5xHREnp!^(m3ifRL!%AE5&(IQd)pme}abLSmdPEA&*p>c(E$4UJINDO)ze1XB_^wbNK{r>R!TM(HdmB9LZ^9!*A;X%`MUmMAnBOdjM>&xgk{F%U$`p}up%Va5-EZ}8H1B~Ofn9UNt!4bAd_cWR@N+}^zzsR zuQp_5lbsn3e=!zz!>fFwcKNgEz^vz&aSCH!#@T)VxW+pXvA0)MljPdL9Lac1Ds*nr z8RSEFCH?MvLgOt-M~97WYV67Hk7!s%rNQd-5xExn8HnSu2y2_ld!t79yPgjdfxIXm zY=7?Y6tfh*HW9B79@(YFjNSz0TkyuWTlpqF4G-M zG}lU^tX*7|b1nhDdHJ1}tRqCA{yJup;Zq3Mvfk~Ci`Bd~YFQuQw&ffICq-62X7DHz$)&0U2}oxF#p{rEtN)0@ucrRluPj?Z-?@@Oa6Xzr@} z^xjB@q2puV86FTQ)*{Z|ZbJWo_}S-m*D=w1Z=TNMJ@NVNEPjECFE1Ru=b58ZI7EK( z)_JkcY%$pU;I#UrpM*E&zSF4)0O|4Vo7Qv_>EOQ~dWlU00)9b7ZF@`JKxgpI_j!xc z#f+wK_?%NkKqk@P+v^gQM(@Mt8RfCRo5bDGN$wJ0%f=_KcAE;QUxuiyDk;fyPjawK z-@?h5Fj$HJz-;T!$53Ebhnd28`Vt`!o92N#J_oR4blmFPdI%Q^T`tON^Zov(C!%we zTf!1xA*}mv?9~TE<_ns_Vj8lzLjU3A;%yf<-~;&nMJY9P&-r~E3cSE1$(wOP%YlOj zoYF-vSH>Gd#Ly99leq%mYu9Hf$I`<>vF}iu*O%Vs%lps4SZLvlhe`3nAEiri^%o7C zCKv-ZA~LjHMX10>_wBzEUUplev5c~!2E%tVJNKAv7B%P{N&aopSFnncJ z{0Z-tS3Bu_9l74lCXYaur(d2IrC;bz8N3w%4f)ifydmGVy!&H|)N{FXVq;QbOhy&b z`9(Jq2!r8@q-GWhm1PWceG~mMgHt<9V9Qc+&L=9ofdz4$BrR&}2So zvpJ(Gc0~b^D5EQVl=ds9vvUT^d~S;o=fOX?fHk9N)joD~^-IF*c0ilDrK)r>u_zEM4LqeUk z)iv`?lFjnST}@tn>h1Mvd~=D>0^7CYs&?i< zB~|ZiXb(y~I&P`R-tpL^4;wxpnxD$%9RqOMt?yiRz3@@}mxrEGREE<>}n-lO_ldw)`QgcAU zVjn_)|-o13nrRr_}E&c+uqVSSs*{22I@&{|)6Be~#G(FxCXG_?&Y2(+KPWrKF{ zbVtO4UUo3QjDWvHn1z8rd1TrJ({dcWcclO(gbrAlTrPC3Zdv~-2L@LeznDx)sCk1_ zSfU&(y94A!6GI|oZ@#KN94vQ)m! z&-O$nIEU(yh3JS(N?NA-5PHW3SW7tWP^!0WVR`)lex$pX%ur80ID&kIk|(|EC2C+> z3%@o5)}mgWmsQ`vAxSp$6WquotDF}^?fpI)CLt@c?bybao=m*2LUyA0uxe_HxjxG7 zho|o#JxESrf&M!Dy=tD|+nY?~3zB|e{k)|`_?i!AyEEaj%h3b<)7{wJFaM(!ppGnp zDO~lcKuxY7Xuxe?LOuR!CW|q_CPqd;Tw2+Drd``muQY!vOK7(ri|POJ!SltFgIG>l zkI{8H>09kmQ10KDY#rU|7gb$RsYXcAT`PjGZ-0FF{;F;lkz86!Ijsi-n()~4wV~y! z(IOI7v4cmKk?uXhfpFrbyunHhEtlJb2VIb+=;zOBV7p8UVUhk&SEK2A)e6t+rBpLE@b@>`ewE3Kton=OCA>zUP ze8wa#sm3l@gu#TRY;{YZ(zcgngi~?B;voT<1_kIIdTuObQ%+$bD+<=Puy?}iz8%y` z+7DrnD~0HU)Vdlex-@=5#(2*8e@|y*d4tt(a zB}VnhedKy|aY`5tAKtZQyKY@w@NaF2zHml!p_9hrCY9iNC=C&Gu?>4=Z+;&mE*RXA z%N-pFjraNMfH=rsvAU<|Qg2z2e(dp7_c%d=vPxxz7;$mr+{svT&G-FDy!>(@a%$I! z#(DlUi&N1~FV8F-9kAI$%?#O(ZLJim+j9I-7ZGH!`ZQ6;Y+4I#c@UCIU3t_YVrP0R zA3Zoa#wX+OjWYd}MHjTv13GD#yZN)V@_BBtGKB{zP@D1VJ!wA>7W**7!D}rO_SeUT z6J z^dNZG;uFS*3<#t!IyE+h<5gTfQ7%{~8+B#X2piAJb9P?;DAXDrgQYxPMC{Iu(xW}@F*~)?^VyH~=x|gDi zJCI|l=QDPa5QoqgQCefpefOf~+4#@kkH~gmj*0 zr`6%}D)^^|*+1_>2RT#qqbUc$DiK==n$_Gt^Q?y&&Ipv~_UkRTwsCQ)!vkI9E!F~x zA}av`Wlo9)_5gr3m*pG8pndkkju;JGx4!cgS7Lqf0nm`2h0bg;9z+czV4hv6RV57}-u#IGBdEdU$WT|DF@h`mjbx9K$tPz?&G3n`A!~V~M%6fz>8*nTKR+~F z42L`U5l`JCN1-tMp{8S!W1e%gkE0%*&t-!}8k=)-fo5~pHZ2%?8v62F(c$|1M7PjY zi5hdP)WMZFF8q_@kLFd-V5@+S=v7+XFq8AWOzi=?!NKBDUFI@-KX*HmIe7pzbptY) zwk*<8Cj)Y!7H(LQXCU_lv;f4147q|VI`EYJVUIX%E%jM$8i0RC|7ZM*%#55*+1q69 z$lmYC-wVB}tlH#ytPB?DIBc*0-FpLgbZ()x4*<01As5!a*!}hNs6n7eUx7!^yS3G* zpC|(A;v{^Pv>TZ&^Q7V5?L!=u!%BbtAHlu*lIIc;Y}_Ko0?-s<+Xp~?rWg1(Y{ua3yUCsp@&+^Yo+@x)ZO*Q8`~T4;isorL66`L?tT~U*A*; z2|c@fBsX}avDDrhPR_-4Lt6}%C9sr5L3NS!KhLr#{}Cj5y5Ygk3bKl@Jz4}DAjEWt zx}RHsBJK;qU0-ENbXDrf+7E^tu9JIj`2bG{`|LKg(Esuerb!18aVK|L{DR%Bwk8wp z0Eeq|w)ZDfU+;;|kdzUZZ@8gvip8pBzIodwBQQ+>F!#I=c4eunYj7>{(%z0c8K__r z28#&M8!ok!L=IH%Cjk|2INTrKiwlfQ`_zBlS9z*?et|j);!L1z7{)}K*%SR-f_59h zyEb<(x<;ONH#9|D+%wYerkcP26)8Bj_*v;Gd8-#(U@fff(X7bkdhIBgmb!hOS@&wE zBXPPrxBWg8DLl41rGITSt<7Ag_`}CZs^w~Ce~DxX8*ziof4sxibW?HZd9N;Ut>xeT z-Z{4ylsR~rw~xB886K-)RynFUKDs77&C2$N)22o3?I^|EhzFmfX1H)q1b+P&tr;c= zaWpIjX8ds1z2crQ(M30j-hDtTba;)uaHcw0NXDCp6oZtzNwox7F#%VkA-ic-`ab5o zGtw^g?-Qa+*G#HPFHP~LFafc#)+t$A6!`X?8L_cWy~^O)ZVg;tyP5w%dVCKZF_fnA zvMIa~a8B$r;YGLO=}6`0!kbaY)pWZu5v(;Kg<}?39bgn70P9s)3Iz7e|#-b6V>Z?G_k4F0SjE{S~uVZC4XOF^YO^a%bidK&hE& zKhT|aIkrsO=L;g>-Xgi_RI5+5MCccQjhPpkJf5p@NyyEdU&ZO>uuYIV>~3SSa>Crz z+h;opEi$q#ku;?d{Y6QwCVDPLQ<7ar0bt{Bw&uKv6BPpCZvGRz`9OBu{$MG$$| z2dCc67iQ^^FC5wo*oyNl`QADE3;)b3v3sm^xwCv#Pw^`QH$`` z9nqAamvN1MeRJ!CKU~XJG9|%{V620$GmI5>1_Fza<#gRCvexZI^xEO95ibGQy+TI4 z&l@YAZAfiI4T`dqv3Td{cJV_8@#jESx{?A)9nh(o&yUJKpQf2ESV;7?dN+!j#%Q94FUz&ZxTw zqQS034T6u&kH#)_l$d)C5)y6l7X!!Nd_RJrNBpX$v{YuHy)kX?i#U2*se0SrJbQdV z!`83p5?8o+<>K`|fY^!oAU z*E_%FEjs$`rXshhC}M))#yP#ypExGfzbi0;Gt|7Ar{wiq1TE=zY@&HrwoN&d+V-)0 zg1Ag|CjQQHIlthE-!A1V->9^*D7Cr8oyGNcj^t;Z|k%D>$im*|Y(Yxk{f zT(s9VnR&2(jCuno3rBlTn}*ZQmnvC z*#ciDVm!_a^hQ?u1enB$Ihw08ZXMb&w1 zFGqJ6{p9Xu&FvJEwI0CSk?s+HsQvltk&+M?{$RI*6{I@Jf;Z==k@z2oPFI^lbJ97E zo+a7-Ky)|UZIF1Tl%}foXvcs3RcjvDx~TbA%(9F0GU~wut;nIk9)KzQk%8v*o3M1U+d9CnZZ>~)uI{oH7ZrB zkN;*pZMbEEyxkdHl)pmw7}QOR3iA6jbdd!%5#O~?=f4Hi7k@o;w91YMwZaN5AGxv$ z1()fa?#|S$YdNRzcO*}(rj&F0&PlP53YR@F+hD2KTz0As^ zpwA!BFq?{RWXoxuB_i`pNJrw(HEXDg^Lo)(j}$oL`@|BX?56wre);1Rl)58CqX}b0 z)VT1J;?NpgoQ+=ZaZ_0udK4Ld}v?2|b{g)+UyL+B^jU4uK#?~kU` z$Af@dherZX-;s*c>Id)Vd?ky(CC(s+JpJui6Cu_|70V0m(!f))NkGv>0s@WL#= zr{mq3r^1L8CL*lN%gS4q(>nC~z2Gt`r92%P6dEF6UN@SxhFDX9h zM6gq&PeB_ws;=*`0-NN%`1U#4YxJ`t08uYX1TG*>K9&~%M0<4c6UBAsN04i!_wcX; zBnUy}lUH%Jn9UCcaM#ZM?#!4l=L^rLlIimDj1Tg97{GkBqOVkQ@kDb)x}xPTj%>F! z^nW*6YN-P-r_*-DKl-E~chwxe8J#UPL|^tK?t5+X$2_pMO@4QgHRs@pah~!afKhm% zr%@vsl&62cG<}hXX0(gxlxEq;%ePAVeha5npxj9I9hZ|=If0`P1nDit`FF*`N$TyD zUrDw!3N}0(rMf`(_vJ27XCp{@_4%+{pWCI|E7(Tj2#}lr?S)UT9(KVqU+-CK6)u>Q zoL4j6awk1U8Maoo;G^zXZ%O7;xci5^4omRp*LPE0pS>f`pfCIRNp~)uCU^c z1MHHS3P4CgGy!p?Rj&qMuN0)k$BrpmgEUPAi^HC)Q{oouZWjdCk*f^H+PG?7MVCk7 zl>t_O@;F3^ckbAEa>(RhM61-ad80`Ij2Ux7qz;fpAzxe(lP(!RX!_|PG1YgT&+CDYwQ5^@U*I%t~-cluD=gcQFI6gz-Z85ySb3PFa z559=+3h%D=aX8(q#gL!70FQrj4cuCVxI{I*Kj?m_|Bw6qWqmK6sW*6a!x#%4KKvt+ zUe7PXJOF&YNACkrzg?fHf`8Pn@tb!>b1DnTm@79J8_M8w5TO6>b`WqK0x(J-XYl75 zcuh5-$TqjfqlF8Ul>X{&ir;GE)`ct~V_S?zOQN7_`!$qGQSly*uX*4h3DjfY`c+>d zFQB8DxPDFRL~Rw>onNkA77*Wvq%nfQR&-S7z8kprZ*;;isrTZ_*-*kpU7rQ)s?X)I z{JD4o$?G%8`yh_Ss-*=_p&dDbeVHG}gqB##KD@!ilYzF-MgV3?xBl6d!wMSH0#sM+ z9X6OBkk=R;jO6E2$T)E~_J{8u0MB`rF*g73>hj&TD%0`x^fj03MU!{p-afezZF0x9 zu9z6dy(2>)4Jryu_5(1}1!g){eBa6|#mai}L=WjTbd!xILt&2Ihoaj^;(TGRd9xL9Xl5v*f05yoi?Z0?{5 z08&cIx7L}3ME89?ZUaUV!*#LiGd$zLc*x`F3-9rEW8^S->{B>dTrBEp``ccu`pC7k zbax!DD|%iaZ2B1Os~_)QwNG&87Y9!t`>UG@4`8mUV1bqW!@3qp@pRaNK6G;N$kn#T zd}3}kkJ^L230APJ-Ti#?yp)R@{s%yP+@BV@IhCzc92QCDp>(G)xg3AoPpvPiqTdZt zO~v`+8tB|E)c$?Nwad7JU5Zi@VL&m!2h@RW3&&9Sq z+bRJg_A4M+MfkW!=2K0bD~Qi+fH73ahTu~*1Ouv#w?@0y#;PI={=U-gcTz+Wo4!T@ zT8o-E^Siu6j;8L;pCpD*Y7R%N0uEa4wuo;KsZ&FN1LX?l@9k33b z!e}E=H9ugK(89rP@foO7-|GyrIuli|V=<*?k00<&Xp{4t*vhT%e{T9%$=Q@4qrF$e z{CjJkFSO1SO1cg5METJFFO{dc;TB1C) z+(r$MJc5vCrLV2p0;8YI|B*`vmTh8#K+?-!a4r7PT->w5c}#d^M;iDJty; zG;wnX35Pdz&G9DyzXYK;KBjfwmrUrDgE-Y@46`8PwF^R_5Qx0Vj7ln!>T|QhZ_UO* z1)l!#)k6_!i2O%AYt0t$D)PS+tA$(tU9a+kK<50>$7H56ALE5nKp^o6>uH3HUO~uh z90(6E#tE1pZ#+Fhx5Ao-BBnD4{u5mazN}8GpiQ>q9*Fg{7rKNo(K(Kv7(Z4t)A^$v z-m0Ri@;dtR7C2gozuGunOn;&-v%`V_dnN0NMHjmAflz$ro~`(R#st|K%nL(4k0gsn82ULqDVLQ|ga+Vl0`>%SC<4ECj(}B-CNuC% zdN^11PlH=tel?N-IsZ1i1qMemaDnnv&fC(Sc)yV@CkDwxbG>KDgBc4F1KQG=ikKnW zjGBtxQarZcfwQwL2XT~Q9Mw=eUW=Mk1KZ6TY=A>Pzie->;m6Qz->P|K(fC9flDG6E zG(aYdJ~>Gno3EL9p~1rF^@ir1&J^&E-EwnTlgbpZPxW;B(#=7?5(CyEl0=WQ5>uR)d zJxCRAgd$Czn2aj?P6S&NaO6NTdPGsbXPH73Xftj}GRr~%n7}dDRn5Tf;kg*5rmBfH z>CosY?5(rg$M+CHAiO(r^I6s~9-iv{aARfb*!BE&0J_1cex3(=N6xf@l>5(qTo(a3 z@Ycaay5tT83Z$E5zCqXTZ%#k+g$_RAp)S#2?eL#EL+eB-nIqgajpLKT`lWRFG+k<7 zujs+b78yHZG->Lsezwv!ySk=x4V69d|Jl00tKbKj?ZcXQ zF&*sd&#f&_g|Fzav5NY7gKu8@Wx-ac}jN{_$qIjYh8qb-Ca z^=K$d!N+S>{+YWZO3BpndEp?<$2$z3lFAved487W_XMIf4zVwwk5$gkSG-Ywmg)5# zdkhpk=!xz-?b>DRebq@J{lrOgWi!Ow@$#irTlgVRo=7qEGW!R4t=QBJkR%=V{?zid z#GcEH38Xyd6J~Qruy}oM+KPl#@pH%e3_%7FXrI+EUWz9Rs7aP*G&d#S!+w}>-4(|^ zRxXuI%w7){gS0tI)1ey7!!5=L$zOJ~Xs#Bt+9#`NwL)D>Zv7H4NR%*69C@y_;Imh< z2qZ%Xy(TT(xr*yk8!oDB?$lgwo^7;ET>ZMqtM3Xf#d%dpd~n*b*E#76eKK2Wz5UCU zur5dKOU-trsA6R9FJ65E4rlbVTx#lEtr&(%1|LVN){qJD4=xDnm!PwWmVG>!-$0N} zG+dc&L>;#Bj^`xBtxlwXese~Hm|dOWn~DNNg#yVN-P zIQ@y`?)My2N9taf`>uTbS}1*n4{z9Pqj9POP!Ia+MbIHhrVDLTX!bYYmH^xIketfG`2* zj8lS6`DqGBA2_Eg{~!J8dB*%-ci@);6mEVOKpdFp{~x|;|G#&Rz1Kt$_xCzmQE&-h PJ#l0t6(L1$4gCHa<}b%Z literal 0 HcmV?d00001 diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 71560aac566..1bb64e4a15d 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -446,3 +446,4 @@ under the License. * xref:hop-usps.adoc[Unique Selling Propositions] * xref:how-to-guides/index.adoc[How-to guides] ** xref:how-to-guides/apache-hop-web-services-docker.adoc[Hop web services in Docker] +** xref:how-to-guides/run-hop-in-apache-airflow.adoc[Run Hop workflows and pipelines in Apache Airflow] \ No newline at end of file diff --git a/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/index.adoc b/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/index.adoc index 67bff8bf766..e477b2c57aa 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/index.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/index.adoc @@ -22,4 +22,5 @@ under the License. This page contains a collection of how-to guides to perform a variety of tasks, configurations etc with Apache Hop. -* xref:how-to-guides/apache-hop-web-services-docker.adoc[using Apache Hop web services in Docker] \ No newline at end of file +* xref:how-to-guides/apache-hop-web-services-docker.adoc[using Apache Hop web services in Docker] +* xref:how-to-guides/run-hop-in-apache-airflow.adoc[Run Pipelines and Workflows from Apache Airflow] \ No newline at end of file diff --git a/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/run-hop-in-apache-airflow.adoc b/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/run-hop-in-apache-airflow.adoc new file mode 100644 index 00000000000..6827c869b98 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/how-to-guides/run-hop-in-apache-airflow.adoc @@ -0,0 +1,99 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +[[HopServer]] +:imagesdir: ../../assets/images +:description: This tutorial explains how to run Apache Hop workflows and pipelines in Apache Airflow with the DockerOperator + += Run workflows and pipelines from Apache Airflow + +== Introduction + +Apache Airflow is an open-source workflow management platform for data engineering pipelines. + +Airflow uses directed acyclic graphs (DAGs) to manage workflow orchestration. Tasks and dependencies are defined in Python and then Airflow manages the scheduling and execution. DAGs can be run either on a defined schedule (e.g. hourly or daily) or based on external event triggers (e.g. a file appearing in an AWS S3 bucket). DAGs can often be written in one Python file. + +Apache Hop workflows and pipelines can be used in Airflow through the https://airflow.apache.org/docs/apache-airflow-providers-docker/stable/_api/airflow/providers/docker/operators/docker/index.html[DockerOperator^] . +Alternatively, the https://airflow.apache.org/docs/apache-airflow/stable/howto/operator/bash.html[BashOperator^] to call xref:hop-run/index.adoc[Hop Run] could also be used. + +== Sample Dag + +Running a Hop workflow or pipeline through the Airflow DockerOperator uses Docker to run a workflow or pipeline through a Docker container. + +TIP: Check the xref:tech-manual::docker-container.adoc[Docker] docs for more information on how to run Apache Hop workflows and pipelines with Docker. Check xref:projects/index.adoc[Projects and environments] for more information and best practices to set up your project . + +In the example below, we'll run a sample pipeline. The project and environment will be provided as mounted volumes to the container (`LOCAL_PATH_TO_PROJECT_FOLDER` and `LOCAL_PATH_TO_ENV_FOLDER`). + +Since your Airflow workflows probably will do more than just run a pipeline (e.g. perform a `git clone` or `git pull` first), two DummyOperators (start and end) were added to the sample. + +[code,python] +---- +from datetime import datetime, timedelta +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.docker_operator import DockerOperator +from airflow.operators.python_operator import BranchPythonOperator +from airflow.operators.dummy_operator import DummyOperator +from docker.types import Mount +default_args = { +'owner' : 'airflow', +'description' : 'sample-pipeline', +'depend_on_past' : False, +'start_date' : datetime(2022, 1, 1), +'email_on_failure' : False, +'email_on_retry' : False, +'retries' : 1, +'retry_delay' : timedelta(minutes=5) +} + +with DAG('sample-pipeline', default_args=default_args, schedule_interval=None, catchup=False, is_paused_upon_creation=False) as dag: + start_dag = DummyOperator( + task_id='start_dag' + ) + end_dag = DummyOperator( + task_id='end_dag' + ) + hop = DockerOperator( + task_id='sample-pipeline', +# use the Apache Hop Docker image. Add your tags here in the default apache/hop: syntax + image='apache/hop', + api_version='auto', + auto_remove=True, + environment= { + 'HOP_RUN_PARAMETERS': 'INPUT_DIR=', + 'HOP_LOG_LEVEL': 'Basic', + 'HOP_FILE_PATH': '${PROJECT_HOME}/etl/sample-pipeline.hpl', + 'HOP_PROJECT_DIRECTORY': '/project', + 'HOP_PROJECT_NAME': 'hop-airflow-sample', + 'HOP_ENVIRONMENT_NAME': 'env-hop-airflow-sample.json', + 'HOP_ENVIRONMENT_CONFIG_FILE_NAME_PATHS': '/project-config/env-hop-airflow-sample.json', + 'HOP_RUN_CONFIG': 'local' + }, + docker_url="unix://var/run/docker.sock", + network_mode="bridge", + mounts=[Mount(source='', target='/project', type='bind'), Mount(source='LOCAL_PATH_TO_ENV_FOLDER', target='/project-config', type='bind')], + force_pull=False + ) + start_dag >> hop >> end_dag +---- + +After you deploy this DAG to your Airflow dags folder (e.g. as `hop-airflow-sample.py`), it will be picked up by Apache Airflow and is ready to run. + +Check the Airflow logs for the `sample-pipeline` task for the full Hop logs of the pipeline execution. + +image:how-to-guides/run-hop-in-apache-airflow/airflow-dag-run.png[Apache Airflow - Hop DAG run, width="45%"] + +