From fa9536806e2032b0546fb41167c5622cc02f8671 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 17 Jan 2022 12:15:05 +0100 Subject: [PATCH 01/11] Bump dev version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index de90417..518e37b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,7 +15,7 @@ plugins { } group = "org.radarbase" -version = "2.0.2" +version = "2.0.3-SNAPSHOT" repositories { mavenCentral() From e1a460883bd9806d127173cb9ee2d783eabf6ff1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 2 Mar 2022 19:14:53 +0100 Subject: [PATCH 02/11] Allow non-record keys --- .../output/path/RecordPathFactory.kt | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/radarbase/output/path/RecordPathFactory.kt b/src/main/java/org/radarbase/output/path/RecordPathFactory.kt index 16aa2ce..dd8d7f0 100644 --- a/src/main/java/org/radarbase/output/path/RecordPathFactory.kt +++ b/src/main/java/org/radarbase/output/path/RecordPathFactory.kt @@ -16,7 +16,9 @@ package org.radarbase.output.path +import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord +import org.apache.avro.generic.GenericRecordBuilder import org.radarbase.output.Plugin import org.radarbase.output.util.TimeUtil import org.slf4j.LoggerFactory @@ -55,7 +57,7 @@ abstract class RecordPathFactory : Plugin { */ open fun getRecordOrganization(topic: String, record: GenericRecord, attempt: Int): RecordOrganization { - val keyField = record.get("key") as? GenericRecord + val keyField = record.get("key") val valueField = record.get("value") as? GenericRecord if (keyField == null || valueField == null) { @@ -63,11 +65,21 @@ abstract class RecordPathFactory : Plugin { throw IllegalArgumentException("Failed to process $record; no key or value") } - val time = TimeUtil.getDate(keyField, valueField) + val keyRecord: GenericRecord = if (keyField is GenericRecord) { + keyField + } else { + GenericRecordBuilder(observationKeySchema) + .set("projectId", valueField.getOrNull("projectId")) + .set("userId", keyField.toString()) + .set("sourceId", valueField.getOrNull("sourceId") ?: "unknown") + .build() + } + + val time = TimeUtil.getDate(keyRecord, valueField) - val relativePath = getRelativePath(topic, keyField, valueField, time, attempt) + val relativePath = getRelativePath(topic, keyRecord, valueField, time, attempt) val outputPath = root.resolve(relativePath) - val category = getCategory(keyField, valueField) + val category = getCategory(keyRecord, valueField) return RecordOrganization(outputPath, category, time) } @@ -119,5 +131,21 @@ abstract class RecordPathFactory : Plugin { ?.let { ILLEGAL_CHARACTER_PATTERN.matcher(it.toString()).replaceAll("") } ?.takeIf { it.isNotEmpty() } ?: defaultValue + + private val observationKeySchema = Schema.Parser().parse(""" + { + "namespace": "org.radarcns.kafka", + "type": "record", + "name": "ObservationKey", + "doc": "Key of an observation.", + "fields": [ + {"name": "projectId", "type": ["null", "string"], "doc": "Project identifier. Null if unknown or the user is not enrolled in a project.", "default": null}, + {"name": "userId", "type": "string", "doc": "User Identifier created during the enrolment."}, + {"name": "sourceId", "type": "string", "doc": "Unique identifier associated with the source."} + ] + } + """.trimIndent()) + + private fun GenericRecord.getOrNull(fieldName: String): Any? = if (hasField(fieldName)) get(fieldName) else null } } From cfa1ed91ae71ac9e81b09caf25dd37599eb20ab6 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 2 Mar 2022 19:14:53 +0100 Subject: [PATCH 03/11] Allow non-record keys --- .../output/path/RecordPathFactory.kt | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/radarbase/output/path/RecordPathFactory.kt b/src/main/java/org/radarbase/output/path/RecordPathFactory.kt index 5465b04..96ee0e1 100644 --- a/src/main/java/org/radarbase/output/path/RecordPathFactory.kt +++ b/src/main/java/org/radarbase/output/path/RecordPathFactory.kt @@ -16,7 +16,9 @@ package org.radarbase.output.path +import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord +import org.apache.avro.generic.GenericRecordBuilder import org.radarbase.output.Plugin import org.radarbase.output.util.TimeUtil import org.slf4j.LoggerFactory @@ -58,7 +60,7 @@ abstract class RecordPathFactory : Plugin { record: GenericRecord, attempt: Int, ): RecordOrganization { - val keyField = record.get("key") as? GenericRecord + val keyField = record.get("key") val valueField = record.get("value") as? GenericRecord if (keyField == null || valueField == null) { @@ -66,11 +68,21 @@ abstract class RecordPathFactory : Plugin { throw IllegalArgumentException("Failed to process $record; no key or value") } - val time = TimeUtil.getDate(keyField, valueField) + val keyRecord: GenericRecord = if (keyField is GenericRecord) { + keyField + } else { + GenericRecordBuilder(observationKeySchema) + .set("projectId", valueField.getOrNull("projectId")) + .set("userId", keyField.toString()) + .set("sourceId", valueField.getOrNull("sourceId") ?: "unknown") + .build() + } + + val time = TimeUtil.getDate(keyRecord, valueField) - val relativePath = getRelativePath(topic, keyField, valueField, time, attempt) + val relativePath = getRelativePath(topic, keyRecord, valueField, time, attempt) val outputPath = root.resolve(relativePath) - val category = getCategory(keyField, valueField) + val category = getCategory(keyRecord, valueField) return RecordOrganization(outputPath, category, time) } @@ -127,5 +139,21 @@ abstract class RecordPathFactory : Plugin { ?.let { ILLEGAL_CHARACTER_PATTERN.matcher(it.toString()).replaceAll("") } ?.takeIf { it.isNotEmpty() } ?: defaultValue + + private val observationKeySchema = Schema.Parser().parse(""" + { + "namespace": "org.radarcns.kafka", + "type": "record", + "name": "ObservationKey", + "doc": "Key of an observation.", + "fields": [ + {"name": "projectId", "type": ["null", "string"], "doc": "Project identifier. Null if unknown or the user is not enrolled in a project.", "default": null}, + {"name": "userId", "type": "string", "doc": "User Identifier created during the enrolment."}, + {"name": "sourceId", "type": "string", "doc": "Unique identifier associated with the source."} + ] + } + """.trimIndent()) + + private fun GenericRecord.getOrNull(fieldName: String): Any? = if (hasField(fieldName)) get(fieldName) else null } } From 4b55e73dc8e2c3dbf0f6487efeb8c624396fd40f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 3 Mar 2022 14:19:10 +0100 Subject: [PATCH 04/11] Bump gradle --- Dockerfile | 2 +- build.gradle.kts | 2 +- gradle/wrapper/gradle-wrapper.jar | Bin 59536 -> 59821 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e338187..5e8da1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gradle:7.3-jdk17 AS builder +FROM --platform=$BUILDPLATFORM gradle:7.4-jdk17 AS builder RUN mkdir /code WORKDIR /code diff --git a/build.gradle.kts b/build.gradle.kts index 518e37b..7984408 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -299,5 +299,5 @@ tasks.named("dependencyUpdates").configure { } tasks.wrapper { - gradleVersion = "7.3.3" + gradleVersion = "7.4" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7454180f2ae8848c63b8b4dea2cb829da983f2fa..41d9927a4d4fb3f96a785543079b8df6723c946b 100644 GIT binary patch delta 8958 zcmY+KWl$VIlZIh&f(Hri?gR<$?iyT!TL`X;1^2~W7YVSq1qtqM!JWlDxLm%}UESUM zndj}Uny%^UnjhVhFb!8V3s(a#fIy>`VW15{5nuy;_V&a5O#0S&!a4dSkUMz_VHu3S zGA@p9Q$T|Sj}tYGWdjH;Mpp8m&yu&YURcrt{K;R|kM~(*{v%QwrBJIUF+K1kX5ZmF zty3i{d`y0;DgE+de>vN@yYqFPe1Ud{!&G*Q?iUc^V=|H%4~2|N zW+DM)W!`b&V2mQ0Y4u_)uB=P@-2`v|Wm{>CxER1P^ z>c}ZPZ)xxdOCDu59{X^~2id7+6l6x)U}C4Em?H~F`uOxS1?}xMxTV|5@}PlN%Cg$( zwY6c}r60=z5ZA1L zTMe;84rLtYvcm?M(H~ZqU;6F7Evo{P7!LGcdwO|qf1w+)MsnvK5^c@Uzj<{ zUoej1>95tuSvDJ|5K6k%&UF*uE6kBn47QJw^yE&#G;u^Z9oYWrK(+oL97hBsUMc_^ z;-lmxebwlB`Er_kXp2$`&o+rPJAN<`WX3ws2K{q@qUp}XTfV{t%KrsZ5vM!Q#4{V& zq>iO$MCiLq#%wXj%`W$_%FRg_WR*quv65TdHhdpV&jlq<=K^K`&!Kl5mA6p4n~p3u zWE{20^hYpn1M}}VmSHBXl1*-)2MP=0_k)EPr#>EoZukiXFDz?Di1I>2@Z^P$pvaF+ zN+qUy63jek2m59;YG)`r^F3-O)0RDIXPhf)XOOdkmu`3SMMSW(g+`Ajt{=h1dt~ks ztrhhP|L4G%5x79N#kwAHh5N){@{fzE7n&%dnisCm65Za<8r_hKvfx4Bg*`%-*-Mvn zFvn~)VP@}1sAyD+B{{8l{EjD10Av&Mz9^Xff*t`lU=q=S#(|>ls520;n3<}X#pyh& z*{CJf7$*&~!9jMnw_D~ikUKJ2+UnXmN6qak{xx%W;BKuXt7@ky!LPI1qk?gDwG@@o zkY+BkIie>{{q==5)kXw(*t#I?__Kwi>`=+s?Gq6X+vtSsaAO&Tf+Bl$vKnzc&%BHM z=loWOQq~n}>l=EL(5&6((ESsQC3^@4jlO5Od{qN#sWV)vqXw}aA>*uvwZopNN(|-T zRTF%5Y_k1R$;(d-)n;hWex{;7b6KgdAVE@&0pd(*qDzBO#YZV%kh%pYt1`hnQ(Fa& zYiDrOTDqk5M7hzp9kI2h!PxNnuJ&xl*zF8sx6!67bA49R1bmUF5bpK&&{eI0U~cH}PM z3aW1$lRb|ItkG5~_eBNu$|I|vYIdAA9a!pVq<+UTx*M}fG`23zxXp&E=FfnY- zEzKj;Cu_s4v>leO7M2-mE(UzKHL4c$c`3dS*19OpLV^4NI*hWWnJQ9lvzP4c;c?do zqrcsKT*i~eIHl0D3r4N{)+RsB6XhrC^;sp2cf_Eq#6*CV;t8v=V!ISe>>9kPgh}NI z=1UZutslxcT$Ad;_P^;Oouoa(cs!Ctpvi>%aQ+Zp=1d|h{W9Wmf7JWxa(~<#tSZ?C%wu4_5F!fc!<@PIBeJ)Nr^$bB6!_Gic_7}c3J{QI~Gg5g5jTp9}V6KYgrgaX>pJt}7$!wOht&KO|+z{Iw@YL|@~D zMww}+lG}rm2^peNx>58ME||ZQxFQeVSX8iogHLq_vXb`>RnoEKaTWBF-$JD#Q4BMv zt2(2Qb*x-?ur1Y(NsW8AdtX0#rDB?O(Vs4_xA(u-o!-tBG03OI!pQD+2UytbL5>lG z*(F)KacHqMa4?dxa(Vcrw>IIAeB$3cx#;;5r2X;HE8|}eYdAgCw#tpXNy7C3w1q`9 zGxZ6;@1G%8shz9e+!K2MO*{_RjO}Jo6eL3{TSZ>nY7)Qs`Dhi5><@oh0r)gT7H-?3 zLDsd^@m%JvrS8sta5`QiZNs^*GT}Hiy^zjK2^Ni%`Z|ma)D2 zuyumbvw$M8$haCTI~6M%d4+P)uX%u{Sfg4Al+F7c6;O-*)DKI7E8izSOKB#FcV{M+ zEvY0FBkq!$J0EW$Cxl}3{JwV^ki-T?q6C30Y5e&p@8Rd?$ST-Ghn*-`tB{k54W<>F z5I)TFpUC!E9298=sk>m#FI4sUDy_!8?51FqqW!9LN1(zuDnB3$!pEUjL>N>RNgAG~-9Xm|1lqHseW(%v&6K(DZ3Pano(1-Qe?3%J&>0`~w^Q-p&@ zg@HjvhJk?*hpF7$9P|gkzz`zBz_5Z!C4_-%fCcAgiSilzFQef!@amHDrW!YZS@?7C zs2Y9~>yqO+rkih?kXztzvnB^6W=f52*iyuZPv$c42$WK7>PHb z6%MYIr5D32KPdwL1hJf{_#jn?`k(taW?mwmZVvrr=y~fNcV$`}v(8};o9AjOJumS4 z`889O91^pkF+|@$d9wVoZ3;^j;^sUs&Ubo_qD&MTL%O z&*SE0ujG~zm;?x)8TLC&ft))nyI zcg44@*Q{cYT+qGrA=In_X{NNCD+B0w#;@g)jvBU;_8od6U>;7HIo@F*=g8CQUo(u^ z3r4FJ7#<@)MXO&5+DgKE&^>^`r!loe7CWE*1k0*0wLFzSOV8jvlX~WOQ?$1v zk$Or}!;ix0g78^6W;+<=J>z@CBs!<<)HvF(Ls-&`matpesJ5kkjC)6nGB@b{ii6-Uoho$BT%iJgugTOeZ$5Xo4D7Pd< zC*LJh5V@2#5%aBZCgzlQi3@<_!VfiL07ywc)ZbwKPfcR|ElQoS(8x|a7#IR}7#Io= zwg4$8S{egr-NffD)Fg&X9bJSoM25pF&%hf>(T&9bI}=#dPQyNYz;ZZ7EZ=u1n701SWKkZ9n(-qU ztN`sdWL1uxQ1mKS@x11;O|@^AD9!NeoPx}?EKIr!2>1Qq4gjfGU)tr6?Z5l7JAS3j zZeq{vG{rb%DFE4%$szK}d2UzB{4>L?Tv+NAlE*&Nq6g+XauaSI+N2Y8PJLw+aNg1p zbxr|hI8wcMP&&+(Cu|%+Jq|r>+BHk@{AvfBXKiVldN)@}TBS0LdIpnANCVE26WL-} zV}HJ^?m&$Rkq;Zf*i-hoasnpJVyTH__dbGWrB_R55d*>pTyl6(?$EO@>RCmTX1Hzr zT2)rOng?D4FfZ_C49hjMV*UonG2DlG$^+k=Y%|?Dqae4}JOU=8=fgY4Uh!pa9eEqf zFX&WLPu!jArN*^(>|H>dj~g`ONZhaaD%h_HHrHkk%d~TR_RrX{&eM#P@3x=S^%_6h zh=A)A{id16$zEFq@-D7La;kTuE!oopx^9{uA3y<}9 z^bQ@U<&pJV6kq7LRF47&!UAvgkBx=)KS_X!NY28^gQr27P=gKh0+E>$aCx&^vj2uc}ycsfSEP zedhTgUwPx%?;+dESs!g1z}5q9EC+fol}tAH9#fhZQ?q1GjyIaR@}lGCSpM-014T~l zEwriqt~ftwz=@2tn$xP&-rJt?nn5sy8sJ5Roy;pavj@O+tm}d_qmAlvhG(&k>(arz z;e|SiTr+0<&6(-An0*4{7akwUk~Yf4M!!YKj^swp9WOa%al`%R>V7mi z+5+UodFAaPdi4(8_FO&O!Ymb#@yxkuVMrog(7gkj$G@FLA#ENMxG)4f<}S%Fn?Up$+C%{02AgMKa^ z4SFGWp6U>{Q6VRJV}yjxXT*e`1XaX}(dW1F&RNhpTzvCtzuu;LMhMfJ2LBEy?{^GHG!OF!! zDvs64TG)?MX&9NCE#H3(M0K>O>`ca0WT2YR>PTe&tn?~0FV!MRtdb@v?MAUG&Ef7v zW%7>H(;Mm)RJkt18GXv!&np z?RUxOrCfs;m{fBz5MVlq59idhov21di5>WXWD-594L-X5;|@kyWi@N+(jLuh=o+5l zGGTi~)nflP_G}Yg5Pi%pl88U4+^*ihDoMP&zA*^xJE_X*Ah!jODrijCqQ^{=&hD7& z^)qv3;cu?olaT3pc{)Kcy9jA2E8I)#Kn8qO>70SQ5P8YSCN=_+_&)qg)OYBg|-k^d3*@jRAeB?;yd-O1A0wJ z?K*RDm|wE<(PBz~+C%2CTtzCTUohxP2*1kE8Of~{KRAvMrO_}NN&@P7SUO{;zx0iK z@or9R8ydYOFZf(cHASCAatL%;62IL27~SmASr(7F&NMr+#gNw@z1VM z_ALFwo3)SoANEwRerBdRV`>y`t72#aF2ConmWQp(Xy|msN9$yxhZ1jAQ67lq{vbC5 zujj|MlGo`6Bfn0TfKgi(k=gq0`K~W+X(@GzYlPI4g0M;owH3yG14rhK>lG8lS{`!K z+Nc@glT-DGz?Ym?v#Hq|_mEdPAlHH5jZuh*6glq!+>Lk$S%ED2@+ea6CE@&1-9a?s znglt|fmIK}fg<9@XgHe4*q!aO<-;Xj$T?IzB-{&2`#eA6rdtCi80mpP&vw(Uytxu$#YzNI_cB>LS zmim>ys;ir;*Dzbr22ZDxO2s;671&J0U<9(n1yj)J zHFNz=ufPcQVEG+ePjB<5C;=H0{>Mi*xD>hQq8`Vi7TjJ$V04$`h3EZGL|}a07oQdR z?{cR(z+d>arn^AUug&voOzzi$ZqaS)blz-z3zr;10x;oP2)|Cyb^WtN2*wNn`YX!Y z+$Pji<7|!XyMCEw4so}xXLU)p)BA~2fl>y2Tt}o9*BPm?AXA8UE8a;>rOgyCwZBFa zyl42y`bc3}+hiZL_|L_LY29vVerM+BVE@YxK>TGm@dHi@Uw*7AIq?QA9?THL603J% zIBJ4y3n8OFzsOI;NH%DZ!MDwMl<#$)d9eVVeqVl(5ZX$PPbt*p_(_9VSXhaUPa9Qu z7)q4vqYKX7ieVSjOmVEbLj4VYtnDpe*0Y&+>0dS^bJ<8s*eHq3tjRAw^+Mu4W^-E= z4;&namG4G;3pVDyPkUw#0kWEO1;HI6M51(1<0|*pa(I!sj}F^)avrE`ShVMKBz}nE zzKgOPMSEp6M>hJzyTHHcjV%W*;Tdb}1xJjCP#=iQuBk_Eho6yCRVp&e!}4IBJ&?ksVc&u#g3+G$oNlJ?mWfADjeBS-Ph3`DKk-~Z70XugH8sq2eba@4 zIC1H_J$`9b$K`J)sGX3d!&>OmC@@rx1TL~NinQOYy72Q_+^&Mg>Ku(fTgaXdr$p_V z#gav1o{k~c>#)u3r@~6v^o)Lf=C{rAlL@!s457pq)pO;Cojx7U{urO4cvXP|E>+dV zmr2?!-5)tk-&*ap^D^2x7NG6nOop2zNFQ9v8-EZ{WCz-h36C)<^|f{V#R_WE^@(T0+d-at5hXX{U?zak*ac-XnyINo+yBD~~3O1I=a z99|CI>502&s-Qi5bv>^2#cQ%ut<4d7KgQ^kE|=%6#VlGiY8$rdJUH{sra;P~cyb_i zeX(kS%w0C?mjhJl9TZp8RS;N~y3(EXEz13oPhOSE4WaTljGkVXWd~|#)vsG6_76I)Kb z8ro?;{j^lxNsaxE-cfP;g(e;mhh3)&ba}li?woV2#7ByioiD>s%L_D;?#;C#z;a(N z-_WY<=SH42m9bFQ>Nb z@4K$@4l8pD7AKxCR>t0%`Qoy9=hA?<<^Vcj8;-E+oBe3ReW1`el8np8E$k{LgFQ}2 z2t8a`wOXFdJ9!5$&mEfD1CnJ)TB+RJih88-Zos9@HZ# zL#{qfbF0ARTXkR@G{lwlOH~nnL)1jcyu!qv2`57S&%oKz0}r{~l9U_UHaJ5!8#nrs z?2FrL`mxnzu&{bweD&62)ilz*?pYIvt`T!XFVVA78})p1YEy7 z8fK#s?b~Yo$n7&_a?EBdXH-_W)Z44?!;DFx6pZ?~RArtBI*Qm4~6nX6Z_T*i$bQPE;Qz?DAPstpGSqr-AJ zo%m9cA`oDDm?&dTaoh_>@F>a?!y4qt_;NGN9Z<%SS;fX-cSu|>+Pba22`CRb#|HZa z;{)yHE>M-pc1C0mrnT~80!u&dvVTYFV8xTQ#g;6{c<9d!FDqU%TK5T6h*w*p980D~ zUyCb`y3{-?(mJFP)0*-Nt;mI$-gc4VQumh|rs&j_^R{sgTPF`1Xja2YWstsKFuQ(d zmZMxV$p$|qQUXchu&8%J(9|)B?`~rIx&)LqDS>ob5%gTeTP#Sbny#y*rnJ&?(l=!( zoV~}LJ1DPLnF8oyM(2ScrQ0{Q4m4-BWnS4wilgCW-~~;}pw=&<+HggRD_3c@3RQIr z9+-%!%}u_{`YS=&>h%kPO3ce}>y!d-zqiniNR-b5r97u;+K6HA2tS>Z#cV{+eFI`* zd8RMGAUtX1KWfPV;q<-5JAykS+2sY$2~UX+4461a(%{P#{rwFPu0xpIuYlbgD{C7C z=U{FUarVTYX6ZUq3wE@G^QT4H2Re;n$Fz9cJ>hABl)9T8pozqbA1)H-%1=WKm^QMu zjnUZ&Pu>q+X&6Co*y#@pxc-4waKMInEPGmE_>3@Ym3S*dedSradmc5mlJn`i0vMW6 zhBnGQD^Z;&S0lnS0curqDO@({J7kTtRE+Ra?nl^HP9<)W&C>~`!258f$XDbyQOQXG zP8hhySnarOpgu8xv8@WlXnm(Uk~)_3$Sg0vTbU3 z{W!5B(L3{Yy3K5PN<@jEarAtja`}@KYva&zFRF*s+_%jIXh$T(S=an8?=Ry3H*NRqWgsM`&!#|@kf1>=4q%bFw7^Rhz!z5I zyI^zU8_R1WN9`88Z=n>pIZQ`Ixr~_9G%Q}@A7rd#*%y7G zXl^Id=^ZL?Rx}}gWXCqzj9C6;x(~mAH|$JteXa1MH<6UQig@!Hf~t}B%tP0I|H&;y zO6N0}svOa1a^PyP9N5?4W6VF%=Bj{qHUgc8@siw4bafT=UPFSoQqKgyUX>sXTBZ=x zOh^Ad!{kOM9v{%5y}`-8u*T&C7Vq6mD%GR}UeU(*epO&qgC-CkD;%=l)ZuinSzHM` z{@`j&_vC6dDe{Yb9k@1zeV_K6!l(@=6ucoI=R^cH=6{i71%4W3$J-?<8Qn#$-DMtA z6Qqi)t?4ifrt%3jSA#6ji#{f(($KBL-iQh-xrC||3U3lq`9>r)>X%oLvtimuHW-)} zy}>9~|M>w4eES`g7;iBM%Se5-OP%1U6gNWp3AZqT8C6OlFFfQ$|7LL;tBV)(qlp4K zruar^K8FnJN3@_}B;G`a~H`t|3+6d>q3#`ctTkE-D^1#d9NalQ04lH*qUW2!V zhk7#z8OwHhSl8w14;KctfO8ubZJ4$dEdpXE78wABz=n5*=q9ex3S}`e7x~~V-jmHOhtX2*n+pBslo3uosdE7xABK=V#-t{1Hd~?i z{i~%Bw6NYF+F$aK$M`r#xe=NxhA5=p%i7!$);sd>Q}#`G?Q~fygrMXmZw?0#5#17W}6Tj+&kFexG{!mYl5FoA99}3G9l;3lVQ^ z48^~gsVppE*x91WheqI(A%F0Z#$#1UJP1R12Mj9r)y(A?a+iquX+d8WD4WAQJ_!oq z9rTISr7bPd(GTP57xm$}C}&kjMivi;zi^Y9g3&X0A;ovdJ?{%_wHgt%%9P&N4H z^XzV(uNA4 zAP`hgP6BEN5`YXh|DF~6Pud?~gWfhUKoPX4>z|}0aocC&K+AoV%|SX*N!wGq3|y< zg4lP(04XIPmt6}$N!dTk+pZv>u;MTB{L4hp9uXk7>aS!6jqM2lVr%{)H3$O127TSZ z0x9hi0k-P?nWFdQ0K`pykqUIT&jD~B0tHP{ffS(}fZ(aW$oBWTSfHO!A^><6vA?qar%tzN-5NQO zL&|F{nGiQyzNJ+bM$Y`n=Lx^3wTG^o2bGB@cwr1eb+6c-1tN=U+Db;bc~eJ!hwM{SbI=#g?$!PjDB+) zPgU_2EIxocr*EOJG52-~!gml&|D|C2OQ3Y(zAhL}iae4-Ut0F*!z!VEdfw8#`LAi# zhJ_EM*~;S|FMV6y%-SduHjPOI3cFM(GpH|HES<}*=vqY+64%dJYc|k?n6Br7)D#~# zEqO(xepfaf2F{>{E2`xb=AO%A<7RtUq6kU_Iu0m?@0K(+<}u3gVw5fy=Y4CC*{IE3 zLP3YBJ7x+U(os5=&NT%gKi23bbaZ`@;%ln)wp4GpDUT$J8NtFDHJzIe_-t}{!HAsh zJ4<^WovY};)9IKAskSebdQiXv$y5}THuJZ}ouoElIZRui=6lrupV|_Jz=9^&;@HwL;J#@23k?A;k`0Bgf;ioO>W`IQ+4? z7A)eKoY4%+g%=w;=Vm8}H>@U*=*AWNtPqgWRqib#5RTGA@Q=43FrQn3J`GkTUV5yp0U`EOTqjfp+-9;0F8!dMEwwcK%(6`8sDD^aR04 zd6O5vh|Xk?&3dy4f|1QK&Ulf{h6Iq;d-&*ti#Ck>wZFG;GHwc?b;X~eBITx49>2d8 z4HcK&1&DvEGT6kXdzAm4oO8%c}8OBt~8H956_;YP-ss*uMf==a+%w~F>Qkm7r)IAuxuoX}h92$gHqbFUun#8m zWHdy`Zrm#=Pa98x8cO0vd@Tgkr*lm0{dky+Gocr0P8y%HGEI#c3qLqIRc`Oq_C%*; zG+QTr(#Q|yHKv6R@!DmLlwJQ3FAB)Yor-I4zyDyqM4yp5n2TrQH>gRt*Zw0+WI-Sj`EgmYHh=t9! zF6lz^xpqGGpo6!5`sc0a^FVhy_Uxq|@~(1@IIzV)nTpY9sY`CV!?8e&bB8=M&sYEb z2i}fvKdhp9Hs68Y-!QJ<=wE(iQ5+49tqt;Rh|jhYrI5VW-mIz|UY{h8E=rC5sh#DU z?wGgk-Tn!I?+Zer7pHlF_Z^!Kd1qkS3&lv#%s6-<5Y%jQL${cge5=G5Ab?D&|9$Y~ zf%rJC2+=2vg;y0-SJb3<@3%}BO$T$C66q$L_H33a`VUbgW~N(4B=v5(<=My|#|J7q z*Ox4wL4kbJd_~EjLTABSu4U7Jk#`y(6O*U6(k6XxM}CtGZB(H@3~kh*zaGRXM}Iwp zQ%xFk2>@wiZrVCV_G4G~v;NebCQ%T7{SDyPpSv&dT@Cn)Mx@IK*IdNrj{*4pkV4wv z)y0J538h>cpB7iPSzA~x24T`{dzNkpvGIqvt1Dvdq@o-`B=$hkczX8$yFMhsWNK-X zxr$kR$tMD0@W)Vxe1^t9qVmsg&K^F@u84)(n2dttIEAZFN6VD$&tskpG%SI7whGL3 z)DeRiwe&?8m7U{G`oW8!SCi*dM>oYL%UKQnKxV_0RXAEBQg1kStExGEUVwLJ0orGGwb7uv+kPDl7_E2*iD|J*=8A@;XCvwq0aw5oJYN*Yh&o=l} z2z8YKb-fIAH5spql4eXqp*)o2*b>#1@DSt?zZi{GPj0gH&Nm+EI<3^z0w%YTEV4xw zI6$+=Faa|Y4o5i0zm5lOg|&tmnJ806DBovU@Ll6XsA;NRrTK~t*AAJIAS=v-UZ%Pr z$oddI@NRir&erzCwq|)ciJemr-E061j{0Vc@Ys7K(mW|JYj*$+i1Q8XlIK8T?TYS(AXu$`2U zQ@fHxc=AVHl_}cRZQ)w0anMEoqRKKIvS^`<-aMf*FM`NsG&Uowneo+Ji$7DUDYc7*Hjg;-&aHM%3 zXO6cz$$G};Uqh+iY7Wpme>PHG4cu(q;xyskNLs$^uRRMfEg?8Cj~aE-ajM%CXkx0F z>C?g3tIA#9sBQOpe`J+04{q7^TqhFk^F1jFtk4JDRO*`d-fx`GYHb=&(JiaM1b?Y^ zO3Kj3sj76ieol|N$;>j@t#tKj=@*gP+mv}KwlTcPYgR$+)2(gk)2JNE=jSauPq!$< z<|?Sb%W)wS)b>b6i{8!x!^!xIdU3{CJFVnTcw0j{M%DUCF=_>eYYEUWnA-|B(+KYL z_W_`JI&&u^@t0})@DH^1LDuT0s3dMpCHIbYBgOT4Zh_4yHbSqRbtIKndeT4Q*Jg91 z@>rO!^t-G~*AIW;FQ$3J=b;oGg8?CTa~qNCb>&cgp@e;?0AqA&paz~(%PYO+QBo4( zp?}ZdSMWx0iJm7HVNk9A#^9Osa#GPJ!_pYEW}($8>&2}fbr@&ygZ?${A7_9?X$(&5 z#~-hxdPQwCNEpf=^+WH-3`2LxrrBMTa}~qJC9S;VzhG!On^JLyW6WkF{8aAE$sM+( zxr8xLW(KIjI`Rm(24r3OJBk<3GF=G!uSP0-G&AY32mLm8q=#Xom&Pqv=1C{d3>1^ zAjsmV@XZ%BKq^eUfBpa8KvO8ob|F3hAjJv*yo2Bhl0)KUus{qA9m8jf)KnOGGTa6~4>3@J_VzkL|vYPl*uL+Ot*Q7W!f5rJw5+AsjP_IfL+-S*2p| zB7!FhjvkUTxQkGWGSg{X;h~dK>gAJivW?88Nu!3o>ySDaABn$rAYt086#27fbjPQS zhq>55ASvm*60qRdVOY9=bU^+{Pi#!OaZwENN;zy5?EztOHK-Q5;rCuiFl}BSc1YaQ zC-S{=KsGDz@Ji9O5W;XxE0xI|@3o6(2~i4b8Ii9VT;^G$*dRw(V?=br)D&q^XkeBX z+gl~+R@rVD-Hwv@7RHV?Bip5KMI)aV^&snt?H<$Nt=OPx#VxF&BGi?2A2+lNOYywNUGMeGL;|(=UjGDtLG0sN&LpGx;|U;xa13s z;W_|SPk^G}!M9_^pO zA3bt3-tca%^42sHeDtfcC0S3w3H1ny!Bxpa=*k?XRPpx9Bb-gx1J9Yvx)4J(8cG+q z(iCPZ9dsf3#QVyZgD_MW#G#qgV)olu$59&3(PzQfw@%4uZ~<5J=ABvdY43(Qnp{;G zHg3>@T#>DbTuhFl3)fb3TFqdh)V2aq7!;&JOHseTWukvA7}(iGUq;v-{2J0iHSNHq z;+)h!p6Ok^+Sp8-jgL($n6Qu47xyE`cFO5SdZR6;R!FET`tm#0D37z339Suxjpv+s z*=%2-N$N?X&0?x_uut3erF@aBGj;9$k9?3FlbDO{RQa1_qtxrh4!4#fjp4x~akvdTp@ zos?^Q&XE;3N93s4rHQGPrV7+au1$$aB6$hLy*Yz_kN$~dweb9PcB!eYVQTGjFuJP> zZCEwBtb>TIgIO^qAzq@Bv-qud_ZD-2W<_at&ml-gv`tPt$@DF5`HlA zM>DmmMkpv&Zm-8)Y#0bLQf4MpD4_-7M8eu6rh(tL8dq8onHs#R9J~dGd2IaXXMC~h z91pKhnQa%Fsn29nAA1;x(%oC zhca~qQDJaMf?wFrl-Pj;e$bZMYmMF!Y3Lv&Sb?Sjn#!NVx&NDyc^$b4uYyo2OmERa zRz;yDGd@JTykzFLe|Wk-y7#3x`6$wt$zR8r48mdUvfbeL+4D|Z``~7$PrE@qc7rZe zVsIoIbCwzjLZ@_M1*bD{HaYn();Z1-q*-I{tEnTZ(}Zmk&%MXSNBX>o| z-u*RNkAyKC-Srp7c-=@5f)xMWg>o2WWl}j6j9=8+D8;T z>0*0q#;qw8%U8i;6s0fu#I*%(g*@@a2Er@@nyI}{=@W{Z-;`=wN4N~>6Xrh&z#g}l zN1g5}0-#(nHUTv_rl2{yUZ;h#t&Fd?tY!7L%ClY)>uH-Ny2ET$lW$S)IQiN79H)D^ zb&0AXYkupy0~w8)*>Sj_p9}4L?lGTq%VG|2p`nWGhnM^!g|j-|O{%9Q%swOq63|*W zw$(N_laI}`ilB+o!a-wl?er~;;3+)$_akSQ!8YO_&-e*SI7n^(QQ;X0ZE`{4f!gAl z5$d+9CKVNonM!NO_frREICIAxOv)wm>}-k?iRisM`R7;=lyo|E_YR~FpS&PS`Lg0f zl-ON<0S%Uix8J%#yZdkCz4YNhcec<|7*P(JsM#>-L>+tYg_71q9~70FAc^6KW5jql zw!crdgVLH1G_eET=|SEc977;)ezVC|{PJZfra|}@rD;0s&@61mTEBJtILllg{%{vN zfhb&lq0yChaLhnJ-Qb62MB7`>M;|_ceHKZAeeh@#8tbrK!ArP6oXIhMK;dhEJTY`@ z0Tq>MIe0`7tGv)N*F0IGYSJv0vN?Az8g+4K9S!pW2~9F4W(_U_T=jCZrzuZ3*|__T zONp_UWmyePv8C~rckc?Xji;Z5OEqg zC*Um)i;Wh4TEwqReQdVVbUKT^2>Tpi6z_^-uF*adUFug4i@JhzpWT^Sk&E>CyP2?H zWf6x}ehuTs6wvzCnTU&gYzT029Nz19(In1WC z`(1IGmi!O%2AR|BjQa4Q0~u)kM%}?xQyjWuQ16^Gp++;`vr7!k--UZWM*~7Zl|ceO@I3`OpaRhD;YoCuo5IC0uHx>9 z478hu@H|e0Zlo)Zj@01#;8BDs@991xe~^9uG2}UXLM(m7fa}AMwX*tjioBeV&Q8Gx zSq$6wZFkRBK`cMI>R(@W@+lo2t)L+4q-negWRLWZBz*|%=W4v62JrmzNuOtA*x)QE z5L%=OH#@KMdB%Jp^r?0tE}5-*6oP`-lO7Sf)0)n*e<{HA=&qhLR)oD8-+V}Z4=md) z+k9lKf64DB2hAT)UaCP~di?-V3~JBH7itYyk~L6hrnxM%?RKntqd`=!b|e7eFnAcu z3*V;g{xr7TSTm$}DY%~SMpl>m{Sj!We+WfxSEor?YeiAxYUy25pn(?T()E>ByP^c@ zipwvWrhIK((R((VU+;@LmOnDu)ZXB3YArzzin!Z^0;PyJWnlfflo|q8(QY;o1*5CO z##hnkO{uynTMdk`~DOC#1 zdiYxQoy}=@7(ke#A8$YZZVtk4wo$8x28&I;cY3Ro-|kW=*yiiHgCLZeAr)UtVx>Tu z|LvL0hq|1-jC0I4x#>&QZCfrVB=zT!nR|~Uz`9%~2 znl{uZ{VEszW`Fad^q_HB!K9*|U-stK%?~;g?&&+12A}Rq$z($Bzuk^2X(Y=hF?-dQ ztc3DsQKI;qhWIV`99Q#R3xnU0AvY!i*BECj-z9l74|%O=V@nlv|qqC^r^-~C?E zGW%c|uYgnfJ(gjsTm_cIqcv*mYM{+i+&@F@+69ZQOK&u#v4oxUSQJ=tvqQ3W=*m;| z>SkBi8LYb-qRY7Sthh*0%3XAC%$z1rhOJzuX=PkTOa=DlocZUpE#KxVNH5)_4n=T( zGi3YrH7e~sPNYVBd~Grcq#CF~rN{p9Zza-Ntnwfma@TB)=3g36*0lSZg#ixEjFe%+ zX=&LDZ5zqculZ`=RYc^ln(~;nN|Qh6gN=!6f9-N2h+3NWbIxYud&;4SX*tWf5slk4 z{q@@l71UAZgj~*6edXb57fBUxvAS7s(RI=X868JM0+^DCn2yC>;v%S;qPOjB>YVsz(Zx9a>>BK&M zIQK>7_n)4ud0X5YM}^i*keH{ehLsiy9@NvOpsFeQjdI6anLGvVbBw_*fU1TzdVS$i z*4j7z!I5RF#rSz|8ibi$;qE{4`aqWYik7QB5U&F5C*;TO_x+gtzPGpzNt!7~nsBT7)Ckc(K~%uv&{{6A`mmBJVAk-{s~52Vu|HbCH7_W1~ZCX^RflOakGg=jo2Z z<*s;5-J+2@^LRDZ-7EV&Pq+FTErw@pfFqvx^i%E7Fx#^n(E`m2(c>K-O5`M`Yek9el zzTGs5qD6*G;y#~xu3>qWuO?-amKYtvRA}I9z#UspEeM;wOERYeot_n_EUMJf$4_u?E!6X~?q)tPoZb^_;8Y_Ox2h1m<+Le-fsRd|T8db<8#$bqez zua^Z|>h%zdnuU^ww$#-dZ9NTM`FN+!IlLkz*FqWb!x^Z|C{KyGjZ+>G;;7Mb@LY|H zc+Gp`L((Dw7pnDlHNm&;SfHedhx*kad$I^uGz{`0BYelq0yEUHpNKSkvj$|dpvY3{7*YGyhXA^LP0&wOw9oNoC=QoVx1<2Dne8qqZL zm>nFh5DX(-RnQwvHCZQwn^#Z=E!SPVlaRJ78Bo@}!!9dRt^qZy?-*`Pt4WSmgucJv zV1yFkcjlEM^uz-;b#Q7ZCP@Lk)m}uPX={R4B=56k7WNh11BN~0T*vr@!!ow^B0hOR zQ)4)&(e%>bNNL%bm<&8H{*l_L7s0$2GUgX2Vd;=4d9Dm2v3TaL+;L>{K7h7 zV#k?xDPm(NDE31$ z<}|X)pEY6myjK+^gaIMk&Yj2~F0rSKemNqlsVm4c|N7mp_C*L01s;GNx#D-*&gk!qQr}^?_r@q!8fuXw!)fA7xkd} zb>vHvdx~H$5qqAWrow7}+8zBM65-JOt5z za=T6f7MK`XJuQog8kIEboPdhcaVJeHy)5z7EBLK5NRr()E|#K0L0N^JD@pUA^Czb` zbUZ_558y+vqAGeyHCbrvOvLD67Ph}06959VzQ_|>RrXQAqE+AQ(-AaKdxoWaF8hdt z{O3W@b^*o#-f1VuU>YMV03ELF7zkCN4Q&b#prz%3Nne0lSbRo@@ z^ihv%oIl~Qyl6Q;a#$*jOC%x0_;eis*)J7=f@Ct*)xF5 zo}u~@-I}2|$b%5L7>@+Z?4o+1r&v6ceIy+vroK&jCQ<4q&45HP2wCol4hVm3pZtjf zHz1D7oyaSKJ~T{Gx}7ONLA)D5k(%%`WswrDyzX*rn}i}}TB4^y#@mAwPzoC)`?rYv zHgx|trUN#mu*VzUV~8TnJM2Qh*ZM5B{x&y>5An`(M7=Z*Q>TdiH@j*2=moNuOtvpz z+G`@~-`%~+AgPKgke@XiRPgndh@bp*-HRsh;HTtz@-y_uhb%7ylVOTqG0#u?Vn5c5 zEp*XRo|8hcgG^$#{$O9CJ&NE;TrfRpSnLmes&MO{m=N%zc`}gb!eQ7odl$oy1%PI} z#AIxx%oRVy&{O~9xnK4$EY>(eQj}!HKIV$Fz*H=-=Kn)N0D6u`(;iO|VraI4fu_W` z;b5{7;Lyx4za}DU#+U7}=H0dAS#YJJ&g2!P@Htu-AL&w=-)*%P9h2{wR|@?Ff9~)b z^+e_3Hetq7W%ls{!?<6&Y$Z;NNB41pvrv)|MET6AZXFXJeFqbFW5@i5WGzl?bP+~? z*&_puH;wKv2)9T_d+P`bLvJFqX#j&xa*-;0nGBbQf0DC>o~=J_Wmtf*2SZQr?{i~X z9-IbRH8{iy?<0v9Ir1?$66+igy|yDQ5J~A9sFX@Pe<*kCY8+MwH?I z`P}zfQ6l^AO8ehZ=l^ZR;R%uu4;BK*=?W9t|0{+-at(MQZ(CtG=EJFNaFMlKCMXu30(gJUqj5+ z`GM|!keqcj;FKTa_qq;{*dHRXAq157hlB@kL#8%yAm2AgfU|*rDKX@FLlp=HL8ddv zAWLCHe@DcDeB2}fl7#=0+#<05c3=VqM*O3bkr@9X4nO|)q0hU;Gye{L8ZN*NH8Id@mP-u;Fmb8YuorjLrW&ndip8CN%_qp982r w1WEnz9^$&s1hkp_3#lPJQ~!HI7WYYjA7>z!`?f%npAh2%rB@vD|Lau$2O)#1n*aa+ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2e6e589..41dfb87 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 3322cd33bf934222a57a7dc76d271b2bc3bbab7f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 3 Mar 2022 16:43:27 +0100 Subject: [PATCH 05/11] Fix build --- Dockerfile | 2 +- Dockerfile.hdfs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bcdd963..8213c9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gradle:6.6.1-jdk11 AS builder +FROM --platform=$BUILDPLATFORM gradle:6.6.1-jdk11 AS builder RUN mkdir /code WORKDIR /code diff --git a/Dockerfile.hdfs b/Dockerfile.hdfs index 850d192..a0f00d5 100644 --- a/Dockerfile.hdfs +++ b/Dockerfile.hdfs @@ -10,7 +10,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM openjdk:8-alpine AS builder +FROM --platform=$BUILDPLATFORM openjdk:8-alpine AS builder RUN mkdir /code WORKDIR /code From 47c08c89cb0ea0077c8d51f5c7d429220837d7c4 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 3 Mar 2022 16:43:42 +0100 Subject: [PATCH 06/11] Case insensitive field matching --- .../output/path/RecordPathFactory.kt | 8 +- .../org/radarbase/output/util/TimeUtil.kt | 100 ++++++++++-------- 2 files changed, 65 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/radarbase/output/path/RecordPathFactory.kt b/src/main/java/org/radarbase/output/path/RecordPathFactory.kt index dd8d7f0..64462b0 100644 --- a/src/main/java/org/radarbase/output/path/RecordPathFactory.kt +++ b/src/main/java/org/radarbase/output/path/RecordPathFactory.kt @@ -146,6 +146,12 @@ abstract class RecordPathFactory : Plugin { } """.trimIndent()) - private fun GenericRecord.getOrNull(fieldName: String): Any? = if (hasField(fieldName)) get(fieldName) else null + fun GenericRecord.getFieldOrNull(fieldName: String): Schema.Field? { + return schema.fields + .find { it.name().equals(fieldName, ignoreCase = true) } + } + + fun GenericRecord.getOrNull(fieldName: String): Any? = getFieldOrNull(fieldName) + ?.let { get(it.pos()) } } } diff --git a/src/main/java/org/radarbase/output/util/TimeUtil.kt b/src/main/java/org/radarbase/output/util/TimeUtil.kt index 1b2018a..0614d59 100644 --- a/src/main/java/org/radarbase/output/util/TimeUtil.kt +++ b/src/main/java/org/radarbase/output/util/TimeUtil.kt @@ -3,6 +3,8 @@ package org.radarbase.output.util import com.fasterxml.jackson.databind.JsonNode import org.apache.avro.Schema import org.apache.avro.generic.GenericRecord +import org.radarbase.output.path.RecordPathFactory.Companion.getFieldOrNull +import org.radarbase.output.path.RecordPathFactory.Companion.getOrNull import java.math.RoundingMode import java.time.* import java.time.format.DateTimeParseException @@ -21,73 +23,74 @@ object TimeUtil { fun getDate(key: GenericRecord?, value: GenericRecord?): Instant? { value?.timeOrNull("time") - ?.let { return it } + ?.let { return it } key?.run { timeOrNull("timeStart") - ?.let { return it } - + ?.let { return it } - schema.getField("start") - ?.takeIf { it.schema().type == Schema.Type.LONG } - ?.let { return Instant.ofEpochMilli(get(it.pos()) as Long) } + getFieldOrNull("start") + ?.takeIf { it.hasType(Schema.Type.LONG) } + ?.let { get(it.pos()) as? Long } + ?.let { return Instant.ofEpochMilli(it) } } value?.run { dateTimeOrNull("dateTime") - ?.let { return it } + ?.let { return it } dateOrNull("date") - ?.let { return it } + ?.let { return it } timeOrNull("timeReceived") - ?.let { return it } + ?.let { return it } timeOrNull("timeCompleted") - ?.let { return it } + ?.let { return it } } return null } fun getDate(key: JsonNode?, value: JsonNode?): Double? { - value?.get("time") - ?.takeIf { it.isNumber } - ?.let { return it.asDouble() } + value?.getOrNull("time") + ?.takeIf { it.isNumber } + ?.let { return it.asDouble() } key?.run { - get("timeStart") - ?.takeIf { it.isNumber } - ?.let { return it.asDouble() } + getOrNull("timeStart") + ?.takeIf { it.isNumber } + ?.let { return it.asDouble() } - get("start") - ?.takeIf { it.isNumber } - ?.let { return it.asLong() / 1000.0 } + getOrNull("start") + ?.takeIf { it.isNumber } + ?.let { return it.asLong() / 1000.0 } } value?.run { - get("dateTime") - ?.takeIf { it.isTextual } - ?.let { node -> return node.asText().parseDateTime()?.toDouble() } + getOrNull("dateTime") + ?.takeIf { it.isTextual } + ?.let { node -> return node.asText().parseDateTime()?.toDouble() } - get("date") - ?.takeIf { it.isTextual } - ?.let { node -> return node.asText().parseDate()?.toDouble() } + getOrNull("date") + ?.takeIf { it.isTextual } + ?.let { node -> return node.asText().parseDate()?.toDouble() } - get("timeReceived") - ?.takeIf { it.isNumber } - ?.let { return it.asDouble() } + getOrNull("timeReceived") + ?.takeIf { it.isNumber } + ?.let { return it.asDouble() } - get("timeCompleted") - ?.takeIf { it.isNumber } - ?.let { return it.asDouble() } + getOrNull("timeCompleted") + ?.takeIf { it.isNumber } + ?.let { return it.asDouble() } } return null } - private fun GenericRecord.timeOrNull(fieldName: String): Instant? = schema.getField(fieldName) - ?.takeIf { it.schema().type == Schema.Type.DOUBLE } - ?.let { (get(it.pos()) as Double).toInstant() } + private fun GenericRecord.timeOrNull(fieldName: String): Instant? = getFieldOrNull(fieldName) + ?.takeIf { it.hasType(Schema.Type.DOUBLE) } + ?.let { (get(it.pos()) as? Double) } + ?.toInstant() /** * Parse the dateTime field of a record, if present. @@ -96,9 +99,11 @@ object TimeUtil { * @return `Instant` representing the dateTime or `null` if the field cannot be * found or parsed. */ - private fun GenericRecord.dateTimeOrNull(fieldName: String): Instant? = schema.getField(fieldName) - ?.takeIf { it.schema().type == Schema.Type.STRING } - ?.let { get(it.pos()).toString().parseDateTime() } + private fun GenericRecord.dateTimeOrNull(fieldName: String): Instant? = getFieldOrNull(fieldName) + ?.takeIf { it.hasType(Schema.Type.STRING) } + ?.let { get(it.pos()) } + ?.toString() + ?.parseDateTime() /** * Parse the date field of a record, if present. @@ -107,9 +112,11 @@ object TimeUtil { * @return `Instant` representing the start of given date or `null` if the field * cannot be found or parsed. */ - private fun GenericRecord.dateOrNull(fieldName: String): Instant? = schema.getField(fieldName) - ?.takeIf { it.schema().type == Schema.Type.STRING } - ?.let { get(it.pos()).toString().parseDate() } + private fun GenericRecord.dateOrNull(fieldName: String): Instant? = getFieldOrNull(fieldName) + ?.takeIf { it.hasType(Schema.Type.STRING) } + ?.let { get(it.pos()) } + ?.toString() + ?.parseDate() private fun Double.toInstant(): Instant { val time = toBigDecimal() @@ -122,8 +129,8 @@ object TimeUtil { fun String.parseDate(): Instant? = try { LocalDate.parse(this) - .atStartOfDay(ZoneOffset.UTC) - .toInstant() + .atStartOfDay(ZoneOffset.UTC) + .toInstant() } catch (ex: DateTimeParseException) { null } @@ -142,6 +149,15 @@ object TimeUtil { + (nano.toBigDecimal().divide(NANO_MULTIPLIER, 9, RoundingMode.HALF_UP)) ).toDouble() + private fun JsonNode.getOrNull(fieldName: String): JsonNode? = fields().asSequence() + .find { (name, _) -> name.equals(fieldName, ignoreCase = true) } + ?.value + + private fun Schema.Field.hasType(type: Schema.Type): Boolean { + val s = schema() + return s.type == type + || (s.type == Schema.Type.UNION && s.types.any { it.type == type }) + } fun Temporal.durationSince(): Duration = Duration.between(this, Instant.now()) } From 0e9e43207b7a04ee6b00293d46694ded04e6bb49 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 14 Mar 2022 09:37:18 +0100 Subject: [PATCH 07/11] Ignore snyk issue not vulnerable --- .snyk | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.snyk b/.snyk index b01dc39..ee762fd 100644 --- a/.snyk +++ b/.snyk @@ -7,4 +7,9 @@ ignore: reason: Cannot resolve for Azure expires: 2023-01-02T13:02:10.075Z created: 2022-01-03T13:02:10.078Z + SNYK-JAVA-ORGJETBRAINSKOTLIN-2393744: + - '*': + reason: Not using Kotlin Files interface + expires: 2022-12-13T08:35:54.696Z + created: 2022-03-14T08:35:54.706Z patch: {} From 7f3286606a37c50e57ceefc0141e0d2119a9dd48 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 21 Mar 2022 10:15:39 +0100 Subject: [PATCH 08/11] Bumped dependencies --- build.gradle.kts | 2 +- gradle.properties | 26 ++++++++++++------------ gradle/wrapper/gradle-wrapper.properties | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7984408..ff72520 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -299,5 +299,5 @@ tasks.named("dependencyUpdates").configure { } tasks.wrapper { - gradleVersion = "7.4" + gradleVersion = "7.4.1" } diff --git a/gradle.properties b/gradle.properties index e0696e7..fcc1db6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,26 +2,26 @@ kotlin.code.style=official kotlinVersion=1.6.10 dokkaVersion=1.6.10 -dockerComposeVersion=0.14.11 -dependencyUpdateVersion=0.40.0 +dockerComposeVersion=0.15.2 +dependencyUpdateVersion=0.42.0 nexusPublishVersion=1.1.0 jsoupVersion=1.14.3 avroVersion=1.11.0 snappyVersion=1.1.8.4 -jacksonVersion=2.13.1 -jCommanderVersion=1.81 +jacksonVersion=2.13.2 +jCommanderVersion=1.82 almworksVersion=1.1.2 -minioVersion=8.3.4 -guavaVersion=31.0.1-jre -opencsvVersion=5.5.2 +minioVersion=8.3.7 +guavaVersion=31.1-jre +opencsvVersion=5.6 okhttpVersion=4.9.3 -jedisVersion=4.0.1 -slf4jVersion=1.7.32 -log4jVersion=2.17.1 -azureStorageVersion=12.14.2 -nettyVersion=4.1.72.Final +jedisVersion=4.1.1 +slf4jVersion=1.7.36 +log4jVersion=2.17.2 +azureStorageVersion=12.15.0 +nettyVersion=4.1.75.Final junitVersion=5.8.2 mockitoKotlinVersion=4.0.0 -radarSchemasVersion=0.7.5 +radarSchemasVersion=0.7.7 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 41dfb87..00e33ed 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From b56d68ac008ec048eea0c75df1d58693a992fc48 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 21 Mar 2022 10:16:56 +0100 Subject: [PATCH 09/11] Bump version --- README.md | 4 ++-- build.gradle.kts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e384d0a..5a46888 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example: ```shell -docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.0.2 -o /output /myTopic +docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.1.0 -o /output /myTopic ``` ## Command line usage @@ -182,7 +182,7 @@ This package requires at least Java JDK 8. Build the distribution with and install the package into `/usr/local` with for example ```shell sudo mkdir -p /usr/local -sudo tar -xzf build/distributions/radar-output-restructure-2.0.2.tar.gz -C /usr/local --strip-components=1 +sudo tar -xzf build/distributions/radar-output-restructure-2.1.0.tar.gz -C /usr/local --strip-components=1 ``` Now the `radar-output-restructure` command should be available. diff --git a/build.gradle.kts b/build.gradle.kts index ff72520..9073066 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,7 +15,7 @@ plugins { } group = "org.radarbase" -version = "2.0.3-SNAPSHOT" +version = "2.1.0" repositories { mavenCentral() From 9952de69654fb6fb1161c8b17d49464f75ac571d Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 21 Mar 2022 11:37:14 +0100 Subject: [PATCH 10/11] Simplify GA --- .github/workflows/main.yml | 16 ++++----------- .github/workflows/publish_snapshots.yml | 16 ++++----------- .github/workflows/release.yml | 17 +++++----------- .github/workflows/scheduled_snyk.yaml | 26 +++++++++++++------------ .github/workflows/snyk.yaml | 19 +++++++++++++++--- 5 files changed, 43 insertions(+), 51 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 08791da..f03f7c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,21 +22,13 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: 17 - - name: Gradle cache - uses: actions/cache@v2 - with: - # Cache gradle directories - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # An explicit key for restoring and saving the cache - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle.kts', 'gradle.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 # Compile the code - name: Compile code diff --git a/.github/workflows/publish_snapshots.yml b/.github/workflows/publish_snapshots.yml index d5c6f51..7462ae2 100644 --- a/.github/workflows/publish_snapshots.yml +++ b/.github/workflows/publish_snapshots.yml @@ -21,21 +21,13 @@ jobs: id: is-snapshot run: grep 'version = ".*-SNAPSHOT"' build.gradle.kts - - uses: actions/setup-java@v1 + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: 17 - - name: Cache - uses: actions/cache@v2 - with: - # Cache gradle directories - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # Key for restoring and saving the cache - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle.kts', 'gradle.properties') }} - restore-keys: | - ${{ runner.os }}-gradle + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 - name: Install gpg secret key run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 536d808..aadd2c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,21 +17,14 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 + + - uses: actions/setup-java@v2 with: + distribution: zulu java-version: 17 - - name: Gradle cache - uses: actions/cache@v2 - with: - # Cache gradle directories - path: | - ~/.gradle/caches - ~/.gradle/wrapper - # An explicit key for restoring and saving the cache - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle.kts', 'gradle.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 # Compile code - name: Compile code diff --git a/.github/workflows/scheduled_snyk.yaml b/.github/workflows/scheduled_snyk.yaml index 2e71ca2..adb630e 100644 --- a/.github/workflows/scheduled_snyk.yaml +++ b/.github/workflows/scheduled_snyk.yaml @@ -9,19 +9,21 @@ jobs: REPORT_FILE: test.json steps: - uses: actions/checkout@master + - uses: snyk/actions/setup@master - - name: Run Snyk to check for vulnerabilities - uses: snyk/actions/gradle-jdk11@master - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + - uses: actions/setup-java@v2 with: - command: test - args: --json-file-output=${{ env.REPORT_FILE }} + distribution: zulu + java-version: 17 - - name: Report new vulnerabilities - uses: thehyve/report-vulnerability@master - with: - report-file: ${{ env.REPORT_FILE }} + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - name: Run Snyk env: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: ${{ failure() }} + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + run: > + snyk test + --configuration-matching='^runtimeClasspath$' + --json-file-output=${{ env.REPORT_FILE }} + --org=radar-base diff --git a/.github/workflows/snyk.yaml b/.github/workflows/snyk.yaml index 0b9d65a..4fc013a 100644 --- a/.github/workflows/snyk.yaml +++ b/.github/workflows/snyk.yaml @@ -8,9 +8,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@master + - uses: snyk/actions/setup@master + + - uses: actions/setup-java@v2 + with: + distribution: zulu + java-version: 17 + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + - name: Run Snyk to check for vulnerabilities - uses: snyk/actions/gradle-jdk11@master env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - args: --severity-threshold=high + run: > + snyk test + --severity-threshold=high + --configuration-matching='^runtimeClasspath$' + --fail-on=upgradable + --org=radar-base From f68dbd7632d0a5c5772f5b9a371cbb7b102a2462 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 24 Mar 2022 17:03:39 +0100 Subject: [PATCH 11/11] Fix GA main branch references --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f03f7c4..a0526a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,12 +1,12 @@ # Continuous integration, including test and integration test name: CI -# Run in master and dev branches and in all pull requests to those branches +# Run in main and dev branches and in all pull requests to those branches on: push: - branches: [ master, dev ] + branches: [ main, dev ] pull_request: - branches: [ master, dev ] + branches: [ main, dev ] env: DOCKER_IMAGE: radarbase/radar-output-restructure