From afdadaf26ee1014b9f9417e9bba4bf983fb3c578 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 20:24:07 -0600 Subject: [PATCH 01/16] lookup join docs --- .../query-languages/esql/esql-commands.md | 51 ++++++++- .../query-languages/esql/esql-enrich-data.md | 8 ++ .../query-languages/esql/esql-lookup-join.md | 105 ++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 docs/reference/query-languages/esql/esql-lookup-join.md diff --git a/docs/reference/query-languages/esql/esql-commands.md b/docs/reference/query-languages/esql/esql-commands.md index 4202a92986fd1..a4f076f510561 100644 --- a/docs/reference/query-languages/esql/esql-commands.md +++ b/docs/reference/query-languages/esql/esql-commands.md @@ -6,7 +6,6 @@ mapped_pages: # {{esql}} commands [esql-commands] - ## Source commands [esql-source-commands] An {{esql}} source command produces a table, typically with data from {{es}}. An {{esql}} query must start with a source command. @@ -39,6 +38,7 @@ An {{esql}} source command produces a table, typically with data from {{es}}. An * [`GROK`](#esql-grok) * [`KEEP`](#esql-keep) * [`LIMIT`](#esql-limit) +* [preview] [`LOOKUP JOIN`](#esql-lookup-join) * [preview] [`MV_EXPAND`](#esql-mv_expand) * [`RENAME`](#esql-rename) * [`SORT`](#esql-sort) @@ -663,6 +663,55 @@ FROM employees | LIMIT 5 ``` +## `LOOKUP JOIN` [esql-lookup-join] + +`LOOKUP JOIN` is useful for any scenario where you need to pull in information from a lookup index to streamline data enrichment and analysis. + +**Syntax** + +```esql +FROM firewall_logs +| LOOKUP JOIN threat_list ON source.IP +| WHERE threat_level IS NOT NULL +``` + +**Parameters** + +TBD + +**Description** + +TBD + +**Examples** + +**IP Threat correlation**: This query would allow you to see if any source IPs match known malicious addresses. + +```esql +FROM firewall_logs +| LOOKUP JOIN threat_list ON source.IP +``` + +**Host metadata correlation**: This query pulls in environment or ownership details for each host to correlate with your metrics data. + +```esql +FROM system_metrics +| LOOKUP JOIN host_inventory ON host.name +| LOOKUP JOIN employees ON host.name +``` + +**Service ownership mapping**: This query would show logs with the owning team or escalation information for faster triage and incident response. + +```esql +FROM app_logs +| LOOKUP JOIN service_owners ON service_id +``` + +In case of name collisions, the newly created columns will override existing columns. + +```esql + +``` ## `MV_EXPAND` [esql-mv_expand] diff --git a/docs/reference/query-languages/esql/esql-enrich-data.md b/docs/reference/query-languages/esql/esql-enrich-data.md index a47eb97e4a41a..f087164b0c962 100644 --- a/docs/reference/query-languages/esql/esql-enrich-data.md +++ b/docs/reference/query-languages/esql/esql-enrich-data.md @@ -15,6 +15,14 @@ For example, you can use `ENRICH` to: * Add product information to retail orders based on product IDs * Supplement contact information based on an email address +[`ENRICH`](/reference/query-languages/esql/esql-commands.md#esql-enrich) is similar to [`LOOKUP join`](/reference/query-languages/esql/esql-commands.md#esql-lookup-join) in the fact that they both help you join data together. You should use `ENRICH` when: + +* Enrichment data doesn't change frequently +* You can accept index-time overhead +* You are working with structured enrichment patterns +* You can accept having multiple matches combined into multi-values +* You can accept being limited to predefined match fields +* `ENRICH` has a simplified security model. There are no restirctions to specific enrich policies or document and field level security. ### How the `ENRICH` command works [esql-how-enrich-works] diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md new file mode 100644 index 0000000000000..82d42ea54ccad --- /dev/null +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -0,0 +1,105 @@ +--- +navigation_title: "Correlate data with LOOKUP JOIN" +mapped_pages: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-enrich-data.html +--- + +# LOOKUP JOIN [esql-lookup-join] + +The {{esql}} [`LOOKUP join`](/reference/query-languages/esql/esql-commands.md#esql-lookup-join) processing command combines, at query-time, data from one or more source indexes with correlated information found in an input table. Teams often have data scattered across multiple indices – like logs, IPs, user IDs, hosts, employees etc. Without a direct way to enrich or correlate each event with reference data, root-cause analysis, security checks, and operational insights become time-consuming. + +For example, you can use `LOOKUP JOIN` to: + +* Pull in environment or ownership details for each host to correlate your metrics data. +* Quickly see if any source IPs match known malicious addresses. +* Tag logs with the owning team or escalation info for faster triage and incident response. + +[`LOOKUP join`](/reference/query-languages/esql/esql-commands.md#esql-lookup-join) is similar to [`ENRICH`](/reference/query-languages/esql/esql-commands.md#esql-enrich) in the fact that they both help you join data together. You should use `LOOKUP JOIN` when: + +* Enrichment data changes frequently +* You want to avoid index time processing +* Working with regular indices +* Need to preserve distinct matches +* Need to match on any field in a lookup index +* You use document or field level security + +## How the `LOOKUP JOIN` command works [esql-how-lookup-join-works] + +The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} indices. + +:::{image} ../../../images/esql-lookup-join.png +:alt: esql lookup join +::: + +$$$esql-source-index$$$ + +Source index +: An index which stores data that the `LOOKUP` command can add to input tables. You can create and manage these indices just like a regular {{es}} index. You also can use the same source index in multiple lookup joins. + +## Example + +In the case where there are multiple matches on the index `LOOKUP JOIN` the output rows is the combination of each match from the left with each match on the right. + +Imagine you have the two tables: + +**Left** + +|Key|Entry| +| --- | --- | +|1|A| +|2|B| + +**Right** + +|Key|Language|TLD| +|---|---|---| +|1|English|CA| +|1|English|null| +|1|null|UK| +|1|English|US| +|2|German|AT,DE| +|2|German|CH| +|2|German|null| + +Running the following query would provide the results shown below. + +```esql +FROM employees +| WHERE Language IS NOT NULL // works and filter TLD UK +| LOOKUP JOIN Right ON Key +``` + +|Key|Entry|Language|TLD| +|---|---|---|---| +|1|A|English|CA| +|1|A|English|null| +|1|A|null|UK| +|1|A|English|US| +|2|A|German|AT,DE| +|2|B|German|CH| +|2|B|German|null| + +::::{important} +`LOOKUP JOIN` does not guarantee the output to be in any particular order. If a certain order is required, users should use a [`SORT`](/reference/query-languages/esql/esql-commands.md#esql-sort) somewhere after the `LOOKUP JOIN`. + +:::: + +## Prerequisites [esql-enrich-prereqs] + +To use `LOOKUP JOIN`, you must have: + +* Data types of join key and join field in the lookup index need to generally be the same. As long as the field data type is structurally capable of handling the data that you are passing through. For example `short,byte` is considered equal to `integer`. Also, text fields can be used on the left hand side if and only if there is an exact subfield whose name is suffixed with `.keyword`. + +## Limitations + +The following are the current limitations with `LOOKUP JOIN` + +* `LOOKUP JOIN` will be sucessfull if both left and right type of the join are both `KEYWORD` types or if the left type is of `TEXT` and the right type is `KEYWORD`. +* Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded. +* Cross cluster search is unsupported. Both source and lookup indicies must be local. +* `LOOKUP JOIN` can only use a single match field, and can only use a single index. Wildcards, aliases, datemath, and datastreams are not supported. +* The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require renames or evals to achieve. +* The query will circuit break if many documents from the lookup index have the same key. A large heap is needed to manage results of multiple megabytes per key. + * This limit is per page of data which is about about 10,000 rows. + * Matching many rows per incoming row will count against this limit. + * This limit is approximately the same as for [`ENRICH`](/reference/query-languages/esql/esql-commands.md#esql-enrich). \ No newline at end of file From 294003e1dfdbd9c40e3e0e296be26146799a8e9a Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 20:26:37 -0600 Subject: [PATCH 02/16] adding image --- docs/images/esql-lookup-join.png | Bin 0 -> 13034 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/images/esql-lookup-join.png diff --git a/docs/images/esql-lookup-join.png b/docs/images/esql-lookup-join.png new file mode 100644 index 0000000000000000000000000000000000000000..de220b0638a06627f1036772b08f4b48eef50ebc GIT binary patch literal 13034 zcmd6NcUV)~_U#6w2nu>YP`aX`VnC%C5C}G;_ufRLhN6TXN&vBO00n8%MWhCV5RlLm z6-2r~C?Q0p1PBm%4|yBB?|1Jx_uTT{`{(8R_+Ybk)?Rz9ImaAx%=tu5N9`axKRW;b z2i0#~Hvj;JB>-SFVq*dS!?4qd0)Oqlcgq|B036)AKMcUjw4=LY2m`gNKuL$-6nL@E zNmW}F0Lo%Hw(Xb!;8LjibyXvOhFPMnv!GK3eIeHO+Ru~lVF3}h$9$%up&@T?yGY*m zDzikgeF!Ui*fP^@pYm+RjxmmlBjs6&$HjhTO|6OYL+8b-PDRazCASH_N|drka>X48 zyD#|jw}EGZZdLYg(GfYe!9|oZjofJyO)pKFQG@NgTR~AqVaeR!dQYcer(vIXv7EORjefDQ7hbL-DnDhHSA>u+A`Zbj_$P{*R`N!zMG*FonLCg3}?Q4jz+4^IGq zf6N6|;63626aX^B+8BU&LBk$B|9HL$nNUj1=Gx4+u6ombz>RCkIrB{&>8Adel5af( znPpr0S>OymZcyLZNy_#~#BeWFMQ?Vx7RYjDL+}ixR7)(BZ+jh*YOYV_1$YkH#Wb>e zjyx$%N{v?w5%$A$+R%@?W=pXFPaj-Rv+FB)xZt&p4Pow_?b4%5+&A4SIQg9+>kK#( zz91@Tbo|Fo6WeNzetxPnB9sr{*>4wfwOn#_NIp9}Jy)%#`>WJ!bj^_ap)>&=R^SFK zIp^7FSH96)X8)miN;Dp7J-hdqRmAp5%VuY4`=TAMZT8DR(+6+pJ9$ z5giOX`~6VYe!P?P=9u9Np*JTS+2oJH=u?ygR)Ozgje?P*xB<$V@Tc5ksS!2-l-s=q zp&Gzi~UgF9p~OsPh8iQt!t7f4$_FFfl}g;c_YW&-b>tfbYl-&K!`MU+++lX3mx zbU6uNUI^Tidx+L{*ZxN90fOjd4=Ve6zXwqEkHq@E@Zp=~Rd>XOh#yT)*W*Ni`D5VC z$@#JqF;3(}dFcx3xPJ?eEKpD3Ho+XFXMVo1@v5|RL8T~(yq!mC;R|lq_?V-@3|Kx+ z{QIfeaRMkZ8))5E)RO;X02^gROHSo-mF#pOh5>_$JtK&PxpVuWDCaw=04rem^gm5> zQTAZur2spfrK-6x4?+#*br{*t<#wke1MK;si8C(@`J{?gwFk;|4n=PEeksiv=VX8# z1_xgM=->32AHO|_+elq`H!gg$^}`cga1+;z0yobxps1h)7(m$^KjBI#nYnf%arU7|+;4Y!r0>Lr#?pl|aj!RN%)jid zaSw66eBXJaI%g&kEIJ)hN(#*y>eBkLyKfGSdOYEt z0C3F2nKLpJM^@A;djK@7zlE8<1f+{9zCJrz_$HGd4PyTrc1GV^bpOeDFTJlo@x711 zH0M3wZSH>)#ExQ%mR`naWb7!_@LVzd2CLYIdUYH$E8ej_sW+DR{l1AtO5QLvTM8Ib zm_dL&+fZSuW-H>qj0>UGPk4@WgB}l>Fnk@O0$<(aQdn3O03ic3%**&q;gJT<&w(7& z77m2M&W5e5vd==d!r$QNVb0u2LATffiFXovx_^M-z4!xYV8{Cm;{E(o3<3$@GmD2m>ws6 zKq&45ly@g+O!fOo8E^8bVnUVf!pTtI1sh!0y*>L;a+w+4@NZEXV%PzP^$Ikg9%k@mW3MYNsRR=m09a2ank zb2Slv1oRy+Tn@aDmY;1a14ITenvd4%US;Q-fl%@E)n6Hlxa~4+f-8212XH&i@}A+} zbB;Y@NEWG42t5tAFPi@!5CSRJA3twz($BcP$`47@-6Hmf|fORE6MtY2_5cX&sV|JLfUuBI1#>md&b#UdwHblHKx0q#ae{@AZ> z1~}tG2^x!j==NZixZtZkYfen4t0&szZM6a5z2$hmm0?Z|@t8KA*|oyMv#8rA$WjwB zR?*|?-IEkIe#caf#u>xFYem6}7&1oW!K_m=)dA zp|QEs1YBK#&(vl>Iicfbu5EM|X3k6CHD4jtutiz!T`6w@6hd{a(yP2zD(-VlR+)US zeA~R^BG>6=#e^BYlPj(I$9kv9cS#Pi(Ad6u5dgryF1{&I_R_+%&!APIJ$%*I;F}d( z+VfC4!9)k0s)c8UOqcq`6df~J3WK)?vJ_ebl+4wk56D1Em&u$Jh|h43tZ(|ON?7)ktO$?IAWS4!zxBl#;1@HZ&-=NYpl;oV zEP8B)0JlF*I#PvKuHQ#MQW17&BIF#p9Bn+{ zd0H{4$AUlK?`HHNKM1+NM^M)y7)Wy$;Q^M}z{z-B@ju&rv8?LQR0~tv@yqFBxZc^# zm3Kp-o;6w*QZV%@<`3KbqSd|=zO3MIWVM5UHxJYIZDW_RIW1S~_nC5_`$_7L)&225!+qoe z8gg8NiGg!KMb!e4sR)$!x)(w+E*H>(G7S6e!mJ2S{e1}cj3b!ir+?K22%iH00l zI~8!+B6S3iU;yBDk#C{{W-v6x%T-Z%fumw|ZWY!Zs`mn0xh93S&#YZ@U)y2;ZX4eH zC)w^`(Ka{!M_X&f1RfM|8Z-^Az(St%K-H% zA6#XicJ=nR5si&;Sbcsl_jKOpbcOYwonui`o=)BiElw-F4Z_c)c#7Pc>m@^+w(l(1XZ-~cQ8OI(%f#wStMmktVxga13%~ zqd2I1ekGu9B8R#$DzJEC_8ZY(eJ`KrzQo#vm76(6@I6)f&7E#B;n(yLhp-2&Lndj6EH{ylH8cjX6)yT>A-?Ic`$jXSj?jwOSX z3#Q@wex3Pb@8{N{ksFeauLir0^>=$cLg|1C$L;iowv_%4Nf8c|2~YfJBJwDNRb08R z&68v}H3go|Z{X=r?)t?k;zCJJSHPs*_1?B2$fH%cZLRl`6NM)X_kj6ikh{Ne{p~yN zBrFd+=`IGvS1|dNEv|ju{NMya_s-!wJlsSDyIeqJ<%%P>A>XL#J>Ic;rSyX#cK|E$ zVCN#Y{;?>HcN#>|b80yTIyRQHC;4ramsf8YvBWh{^q~R?EwX751d@Bvbq8hg(e@Z= z;e?st)T#X5icNC6>*`3@(3Z_t^*Vctkovs1rIzwe_e}abev;7J)1$UGvT0ShbopYE zt)72T$z~lZztFG<*Ml5~n08oe?b1xcn4K(RK|*$Z+jR$-a|reQaXtxYo2rHN%-!CL zHsXD3Z+$yH#3*LenkE-ZsO1iJ5o$*TO~iP5d}iJmTF|7O5)dPVnDiZ+H+|d4{c%gJ z8B~|?%qT)fL#0{LAca%}(s=!{e%`T(wsDDA+)$WKj>Cu$qwLiIi$2M;bXNl_cz=q< zH0``$L-9?2qWUV_@#LTfdgi`+n>i%3)W9obHH8Hm62_mI+qZUyZjs?KY9^q>cRRAh zR>bC_Y&#?I=G)kG#Zp(J10XQDRJ7GZU@t)OA7aaEstRvD&a=!mk$U`9%4^!vni5|5 z)&u|Ze9__W_O3$f6bJHMYXZ~I(X|wesz;$~5dJW_L)i%mpzJF)i)dWNG4{uur7N=UCkRe7XC(MI>RmP>>i!RHG+nLZRpzLakyRMIhv zvK%G?GbVkbPUMC}YOav)d5Zem&Je5jEWd~B@EJFlji@{RLB8m6v67~8dWiOSr@4pm z=k)95;;ZkqriRW<1n3q9v<@&5ZZ>j_34RPYBWSeaGx@RQQg72>FZM?p4iPpzz+6a{Dv54j$cdHsH|BLYpJe4h}ZZU$Js}= zEHFVbrELs%)-JuyoeCPYP+YTj)OfF;6)kwUOPg(}9dW|RT3eGiYz!gl_*^!3%C<-M z)V}A}t<0m;*jFO!hGKraDWqv>=6q+3Us6gbgi~VSqw>l3V@@?^eidv;EHE{uP;-ft zadTNndowFgdUzt8?`CdkJ1$;*zrj|eA?cuhk%Y={BQN>F@ar9^3K>%+qa{RuDuugn zA;%7jkdJqzBJZz!qjpc%h#24ZIB8F^GoK|MqoE&LbFH}3wREETq^k(Qd}$D)RE zYZl@Xp+S!FvC!<1pW#+5ixx(_N)mvuw8g}HN_$upj4Vts<3@J)k9RR_jQtPq(U)qR zStSnbtUg*)59s$Rtfkl19$q=oD@nW6Dmbr&vm_?^_GBn`vZI=Rzp(ZQ3(M=BG-*fP zdJ^YT>3-TgQJOI8;UsR*TppN^7L2j(U&R$Rp>?f=`~-TRkNIqx4I&hXKWlYQQoc1S z7bNI9Be-t5tZX1v^K)Tb6}1E*SnR7*A{Xx+W7)Y=AsW%oD&J!i^U~G(j%iOnP#5A0 z%!^aNLxL|1PGsCN=`)zj)R-Q)Kj%F4L$b1i=+#$8_b)&15aXlPnmO%AX?o=)2aTyA zZQY$7xYh#A{AS?HYO&*wPq_5X)Z@f6l#=0!iSi8mSy6?{Baq)Q9 zeJM(D%UXpU7a@_ai|n`3`V8jH&2XXX=nhWV@O5SgS96ZV&TBaW)v3UqYAp8H_7)n^ z>_vuVLd8#Q9dy8}OJo+!Ww1GsWzqua7gj?0uV-;j0+WtwD5SURs*3|YP&ski{fBe5 zTuOoc){Bbmg(8f_4%bea$%u%OlqmU9DH(g|a~p9T+B|-WWd>cvxl65W5#X!Jv7-ZFNL$zMXN6ujN}uPGOVJ7WOS|b)uMSA9vy5 z$<@)R-`~14ruu+1@&-r?^=KnZ2^oWL7B!JR^SQ7O9YVV#afEh$9}!asFIZVs(901Z z;0dU=3=mH=XKqZ{AR|O6PMWquvrwe`tSVWxPg#M}^!4s$zq6@#nO@`d-VUX|0%aYE z%4(m!Ypg~AQFOk~6QXT3wve-O?8D6V(12h$L$ddq80K05V*TJ%S&LpZ?DEcP743b# zi+fXi@T_#~sb*|v(~6xNZhk`OpiU5f)mQW><=O^Y8ok^SRB&GLZmzwIvY6tX>_d=` zjNo#>s)%G(4;wr!?HSqpQ5j?ty4iZTu8D# z)TREQ+V*_!HLP$KUMIcXq-T^e&Gi{;=k^?w`zY~0 zLMOe@Rlp}K8f45_+Pc*#zwY<+Y-;y%F@b!>7oPV@9jmJlp zb{5Hr1b2#%o=LeT%{6KMBKe6>REA*)RPX-)V}V_b@~`TId+SMl$pXOjY7|PR^2eq~ z7|joeQwOhW-j?1?epvolX?=SPJTfvU zXK{Nr9N%>Zz>F0h3~pZ4ow~?1kmn15IyiqOZrxd^OC8)ERI}UVX`4)brQhi8VK#8D zXd!Ek@zfK@`SgmxY0YhXhVKO|OYucTk1pfmvRcs+?ZO zfU{qgAvid&?+02D%$y2b1Tbd+=WH-@t38lGI1Uajtx&juibEr{+_qBkB{f2m`*4E{ zS=U@mKl)FxG%}=LO=f~U`aNAyM58#_ryI*|UYq4L39~|ZH2zmL9>FCrYfA&`ra^beTRlImr&@LQ#EFXe zG0b91V?WtAHMX7{>-l9@ml!?OImXfj|EkXL^wsGjji z9C#8aG%veABe=pWMYGn5-x3HGxK9qmP9}Je2qBJGqF6{+MvPl;4 zeWBk90sUO@z^Iz`s;+B^;yr_0zm zU#|{hh;>c}i??*&>k!X%ML;NGv=Afu{|wtzIk(W{gPLq%`*j^p_a5I z796J?=amp+^nobH5B&LuC3#Qg_6G8u$bqhSif_H9IDM=g+)}52ab{5Xfmf8hgK>h$ zL;%@L2cOwe?3a1b*=uH|!QrDTE!`<}`*+t3(K-fMI%il3XlJpu#u6W|NL>-+J2CaW z1AfXeR;71r9UX3E5!!ksmSZf=qNp$+ncNa$TVkv{{JI|A?iAZSqv%ZvfMP;4G*ya+ zp~b6%a?D@~e;dxM4kg^(x?4V@o9H0RcsDR%b3DzRcgz^#b4^7Op^dNU z?=~;5hdW~tRBwvxms*nR@WzY2!1~Z<0q!cit}V)yS8^m(lbPP*+2b8B?^{c3A)*m? z1H_$DQ#*rCoNewkPwitCDk%?Ei(iR%-OhJ8y)`PTC-#M6v3Xq2kZ%_%;gPNnu7$?W zj^BkQ6?$^QH)Jm6nI~api!(~4Ukrlf1yuvM{7IQEb^<)BH;5UoV~XxFr&dVf{t zF=uzgFmV8-DvKMMW$o_V=M4oZ?e6Zfnnp{6zE8Msm>b@UD>%Kcard3Z~~O7|q;VP8<7NmlMzc%MoV z((38mV^-(1V~ASD0vrezmwlVqvD$|fGG3%~q{XJ9N=n+=v&1*F+|UR6zS24KK3zfh zqxPP(X!#&4(W0Tm>}U+gHv2MNF(EdG)HM;6kl!_-?T1E)`oRliisB!1_$rVwX+Z zw)Qhnm|?4W`8f+{cv!}{}-R-tor;H@r(dmUy~gLS-o!b z^|^-8Sd;~tSE^2tEYNQh0tMqa>A|MU{XEY4b?H6P%{pk&Txq40bu0>u`jnN>mxddC zvwO;0{`OI@Psg&O@e1c>jocF?f4>fTRsPrGp;w!(1g*bP4qOrn7;V*+sR^#CY@lTa z8H}u?pQEZsc?Jc2jD(^j=l8maezgVt<)J7Y_xJpx6bt}$kr$*;WxK?-xM+DB~azfX3xUjQ|7H5qxnHbA`I#zRMVBVf24 zjFXS@BkM&07*(`Q$>cAkUtpP~ao9dj(k3S9(UySNKY zq2tApagKO2IpT5=T>-R`?IO=OvlC>+K1X8s0by<W@vX|S#Sm&3 zVP%IWgzr~*VnwT-&!7h$`>xF=JSmlA1N5xGn@_t&SC0->kGCI~MM7;Z-8cw5jb^wf z3`!wH**=kD!yZE>qpJC0ocYxBn zwm@yeYy7?42zVNMVXb!_PE%O-dh15qZVw#x;>sES{weR}#w_xj672;5e7>2CEEHtk z=%#I%6pI`SGL9I{bOkH_FW0BT&v^>*09gY^0n-b)c~u23_q}6Q0J7HEmKcW)Gvf+K zp`e8>9G+nI{+sT*Hx{0fj62E|juBm(-R};dR_uVZyrzbySGPPdu{QlwsWY|4reFda z%{dp#+6W#9Y6O;y1fRI3T?I1YQyFnhw|s%ggAc0nPKO5jf9{px@E7<1_KDOM>R;Ta z02*+A8^_(-opbAsm-0wk#@j=TL>!4>_dWQYMFv8v6^94&TZMKlC_t6DQ+BS^gX!nq zYw{V*P6yQvw|+DMvgUW+>CbW>Xds6tGWcoRQiUNG>)g`8rZk_TB#^IiHw0Y+P(Q`m zlpM0t{gj5WP1P<3er>v;nA$!7#t*7UU=!hAqQyVt5erwAEYPPj*L~|xE0YC4pi zs~hE)E2Z*W>PcVm3#S8)#8OjCE|%?y?Lt$#3QIV)xAo>XULLQb@4v{S~k;S z%g0Aj(>d~_=kl*?9{f|KOlzg#eD8!|dv!3V7KsEUCyP)z0B=XJlHK~`i7pm^0TAG55Bw)%2!khpbBKCni)%o7f zo_EF`kcA&K+B7|h$euwsjO}XZx9ZiXf|X_{md@ja(yuHc820S)%Ru$i^2Vm zt!)XY3rnnKb=yoSdvHYZf5EL(^MARt)hvmM*zJb5NySABDO{k&*CqM zWa5tU)$k#~V$ILI*VPPOtJkrhRYnN30kBvwM%Z_r6B%ic5*~Jp7@q0Qi$R_*^l9N9 z1G*GJTsGGuV7{Wwk}cuHH>HQa16`>gCY#GE0$EiT0nUHVGJY1gU@f$j7iNRBgF57| zZoFRA&d=nfYS@OHu+<7x%!w4wOwiq}?hXYlD}@GKtIxN_NdUx%Jt&K^oEY~qPtdqg zzc80Z_tgJyi(ol`H|syt=l-Eh_wOFoyws*-@aGi#S9HCjYD2q&b4b7Kqz;N%-pkk! z1|X2hZZ7-3$iVjiO}HGtHop@rfw%P6wMp*T6;T!nqGIu&;es)RSz6GiR5x1q3B3|l zDqG#e37p|kXL)a41@7?x+fsPBsu$RtscOW#Z`WjM-@9d{ zm37a7kaD683G=7KfcSJHna_+i65)RgEb#6PaGd)^39S6UNs_@y3at;0<_fUwSzT=w zSZ|I^8qN{g^(Cn+`%D_NUJV&UFa;wRdn^*ecOC4dbo2}oAteQa*m_P0qYa{E0{9@c zWaO8O7?UlUND>;8_|6{U#o<+hme|E^1lQnS_B!o3xDlt&V-5gt`HvFKU&c^%P%l@3 zj3m@=U$0Nz@x95Rt1OXuJ}B4*yD^E}S$`O$U*l~G;M8o5z}y>!%|8V;n*aHuL>6K8 z&NMJmzO}VFa?G%<_$MF1<$UTaO}ZmLPEZ@ci?qn_c^{`C9vdz(ZAt zSr6WrgnU6f-i)$)0|)Td1r$C2)R%ikj=huoHdevt# zJm3!U0e3PT;1Vq%8Q$2jqm3TkDvrmExd{~rt3`Om2I~q-(a_cte;6hO^Xaoas)_? zn~_?<82Q1xL_rn3(Isn5WlL@JAXjov%YE@Uhla!w3KxjXOXstSv3?3pguCVF-DVf_ zB(%tDJaY>!9x`p)^EGkWzuf&Y*5lo0q$y&=5;9Vg0&8b28B~BjcS5p!ZBfpFHn)`c z5v*EOAcRX?6}>snXbkadGyqlg&awWYSj(b+M`d?gxnc3Rh$G&(&pA@qcrE|tcqZ0s z+AR6}7Gkx`eH2P5P7umpfz@~!kdsX+*Na+k=$EGeAod7xM!+7e#gTWx$x>+$U)u4~ zeQ`wA$H}Z(lEwh>NtHK*m(|A!=9mlhGaWk9z&wW=W_XW$u+e08Arew)dHX;u{pf=vR6l~Org)YsYbBnNC0Al`T8G#HcGLyZyW0t~KJL*%(~=%MIG^X4 z`Z;RZCAn}yz^a@%sI-7h`Rp<92?6^sp8O5Ah4Mtdh<-yt`Ag=0*RP-)pb40(cHngU z>DHO7cMRD2N@3?&4Y(O;niJvc;efs(B5zIAiV7BwcNn3|h4I%h z7&8E1x_rmi-S^B{ajbVlybK2hUf^yl`vyzIe~Z=*Zv!*tW|8&)Qo-vOB$eCZWqIO3 zUB9J-l(9P9lhoky7No9h0M(=UNW^+0Qfplt1`Av? zrfaBoE|ZTsfD!O$|F>FaKUNvbr2gLyR`8sbbS`FADA~yeabB%IK)2zgV>F zt@JpJ$lxvPlN&+mqia5;hqZ0IiB0g!aFFF%D&#*N1VMBJsGMJHw1RHch)C~Fol`uX zgSEWh|D15ddE>q~_6o{6$y Date: Mon, 10 Mar 2025 20:28:18 -0600 Subject: [PATCH 03/16] fixing file --- .../query-languages/esql/esql-lookup-join.md | 112 +++++++++++------- 1 file changed, 66 insertions(+), 46 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 82d42ea54ccad..660886aa6b337 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -1,27 +1,30 @@ --- navigation_title: "Correlate data with LOOKUP JOIN" mapped_pages: - - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-enrich-data.html + - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-enrich-data.html --- -# LOOKUP JOIN [esql-lookup-join] +# LOOKUP JOIN [esql-lookup-join-reference] -The {{esql}} [`LOOKUP join`](/reference/query-languages/esql/esql-commands.md#esql-lookup-join) processing command combines, at query-time, data from one or more source indexes with correlated information found in an input table. Teams often have data scattered across multiple indices – like logs, IPs, user IDs, hosts, employees etc. Without a direct way to enrich or correlate each event with reference data, root-cause analysis, security checks, and operational insights become time-consuming. +The {{esql}} [`LOOKUP JOIN`](/reference/query-languages/esql/esql-commands.md#esql-lookup-join) processing command combines data from your {esql} query results table with matching records from a specified lookup index. It adds fields from the lookup index as new columns to your results table based on matching values in the join field. + +Teams often have data scattered across multiple indices – like logs, IPs, user IDs, hosts, employees etc. Without a direct way to enrich or correlate each event with reference data, root-cause analysis, security checks, and operational insights become time-consuming. For example, you can use `LOOKUP JOIN` to: -* Pull in environment or ownership details for each host to correlate your metrics data. +* Retrieve environment or ownership details for each host to correlate your metrics data. * Quickly see if any source IPs match known malicious addresses. * Tag logs with the owning team or escalation info for faster triage and incident response. [`LOOKUP join`](/reference/query-languages/esql/esql-commands.md#esql-lookup-join) is similar to [`ENRICH`](/reference/query-languages/esql/esql-commands.md#esql-enrich) in the fact that they both help you join data together. You should use `LOOKUP JOIN` when: -* Enrichment data changes frequently -* You want to avoid index time processing -* Working with regular indices -* Need to preserve distinct matches -* Need to match on any field in a lookup index +* Your enrichment data changes frequently +* You want to avoid index-time processing +* You're working with regular indices +* You need to preserve distinct matches +* You need to match on any field in a lookup index * You use document or field level security +* You want to restrict users to a specific lookup indices that they can you ## How the `LOOKUP JOIN` command works [esql-how-lookup-join-works] @@ -38,68 +41,85 @@ Source index ## Example -In the case where there are multiple matches on the index `LOOKUP JOIN` the output rows is the combination of each match from the left with each match on the right. +`LOOKUP JOIN` has left-join behavior. If no rows match in the looked index, `LOOKUP JOIN` retains the incoming row and adds `null`s. If many rows in the lookedup index match, `LOOKUP JOIN` adds one row per match. -Imagine you have the two tables: +In this example, we have two sample tables: -**Left** +**employees** -|Key|Entry| -| --- | --- | -|1|A| -|2|B| +| birth_date|emp_no:|first_name|gender|hire_date|language| +|1955-10-04T00:00:00Z|10091|Amabile |M|1992-11-18T00:00:00Z|3| +|1964-10-18T00:00:00Z|10092|Valdiodio |F|1989-09-22T00:00:00Z|1| +|1964-06-11T00:00:00Z|10093|Sailaja |M|1996-11-05T00:00:00Z|3| +|1957-05-25T00:00:00Z|10094|Arumugam |F|1987-04-18T00:00:00Z|5| +|1965-01-03T00:00:00Z|10095|Hilari |M|1986-07-15T00:00:00Z|4| -**Right** +**languages_non_unique_key** -|Key|Language|TLD| -|---|---|---| -|1|English|CA| -|1|English|null| -|1|null|UK| -|1|English|US| -|2|German|AT,DE| -|2|German|CH| -|2|German|null| +|language_code|language_name|country| +|1|English|Canada| +|1|English| +|1||United Kingdom| +|1|English|United States of America| +|2|German|[Germany|Austria]| +|2|German|Switzerland| +|2|German| +|4|Quenya| +|5||Atlantis| +|[6|7]|Mv-Lang|Mv-Land| +|[7|8]|Mv-Lang2|Mv-Land2| +|Null-Lang|Null-Land| +|Null-Lang2|Null-Land2| Running the following query would provide the results shown below. ```esql FROM employees -| WHERE Language IS NOT NULL // works and filter TLD UK -| LOOKUP JOIN Right ON Key +| EVAL language_code = emp_no % 10 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| WHERE emp_no > 10090 AND emp_no < 10096 +| SORT emp_no, country +| KEEP emp_no, language_code, language_name, country; ``` -|Key|Entry|Language|TLD| +|emp_no|language_code|language_name|country| |---|---|---|---| -|1|A|English|CA| -|1|A|English|null| -|1|A|null|UK| -|1|A|English|US| -|2|A|German|AT,DE| -|2|B|German|CH| -|2|B|German|null| +| 10091 | 1 | English | Canada| +| 10091 | 1 | null | United Kingdom| +| 10091 | 1 | English | United States of America| +| 10091 | 1 | English | null| +| 10092 | 2 | German | [Germany, Austria]| +| 10092 | 2 | German | Switzerland| +| 10092 | 2 | German | null| +| 10093 | 3 | null | null| +| 10094 | 4 | Quenya | null| +| 10095 | 5 | null | Atlantis| ::::{important} `LOOKUP JOIN` does not guarantee the output to be in any particular order. If a certain order is required, users should use a [`SORT`](/reference/query-languages/esql/esql-commands.md#esql-sort) somewhere after the `LOOKUP JOIN`. :::: -## Prerequisites [esql-enrich-prereqs] +## Prerequisites for LOOKUP JOIN + +To use `LOOKUP JOIN`, the following requirements must be met: -To use `LOOKUP JOIN`, you must have: +* **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means: + * The data types must either be identical or be internally represented as the same type in Elasticsearch's type system + * Numeric types follow these compatibility rules: + * `short` and `byte` are compatible with `integer` (all represented as `int`) + * `float`, `half_float`, and `scaled_float` are compatible with `double` (all represented as `double`) + * For text fields: You can use text fields on the left-hand side of the join only if they have a `.keyword` subfield -* Data types of join key and join field in the lookup index need to generally be the same. As long as the field data type is structurally capable of handling the data that you are passing through. For example `short,byte` is considered equal to `integer`. Also, text fields can be used on the left hand side if and only if there is an exact subfield whose name is suffixed with `.keyword`. +For a complete list of supported data types and their internal representations, see the [Supported Field Types documentation](/reference/query-languages/esql/limitations.md#_supported_types). ## Limitations The following are the current limitations with `LOOKUP JOIN` -* `LOOKUP JOIN` will be sucessfull if both left and right type of the join are both `KEYWORD` types or if the left type is of `TEXT` and the right type is `KEYWORD`. +* `LOOKUP JOIN` will be successful if the join field in the lookup index is a `KEYWORD` type. If the main index's join field is `TEXT` type, it must have an exact `.keyword` subfield that can be matched with the lookup index's `KEYWORD` field. * Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded. -* Cross cluster search is unsupported. Both source and lookup indicies must be local. -* `LOOKUP JOIN` can only use a single match field, and can only use a single index. Wildcards, aliases, datemath, and datastreams are not supported. +* Cross cluster search is unsupported. Both source and lookup indices must be local. +* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported. * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require renames or evals to achieve. -* The query will circuit break if many documents from the lookup index have the same key. A large heap is needed to manage results of multiple megabytes per key. - * This limit is per page of data which is about about 10,000 rows. - * Matching many rows per incoming row will count against this limit. - * This limit is approximately the same as for [`ENRICH`](/reference/query-languages/esql/esql-commands.md#esql-enrich). \ No newline at end of file +* The query will circuit break if there are too many matching documents in the lookup index, or if the documents are too large. More precisely, `LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large amount of heap space is needed if the matching documents from the lookup index for a batch are multiple megabytes or larger. This is roughly the same as for `ENRICH`. From d90068a895dc40d9154a54960b7ef716fb7c3770 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 20:33:27 -0600 Subject: [PATCH 04/16] fixing toc --- docs/reference/query-languages/toc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/query-languages/toc.yml b/docs/reference/query-languages/toc.yml index 99e286ab0b8e0..3c75f07d37ee2 100644 --- a/docs/reference/query-languages/toc.yml +++ b/docs/reference/query-languages/toc.yml @@ -91,6 +91,7 @@ toc: - file: query-languages/esql/esql-multivalued-fields.md - file: query-languages/esql/esql-process-data-with-dissect-grok.md - file: query-languages/esql/esql-enrich-data.md + - file: query-languages/esql/esql-lookup-join.md - file: query-languages/esql/esql-implicit-casting.md - file: query-languages/esql/esql-time-spans.md - file: query-languages/esql/limitations.md From bbfd63a9b5a4d8b2784833b502cc2794df72336d Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 20:38:09 -0600 Subject: [PATCH 05/16] fixing toc --- docs/reference/toc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/toc.yml b/docs/reference/toc.yml index 739290c91462f..c232a38cef98b 100644 --- a/docs/reference/toc.yml +++ b/docs/reference/toc.yml @@ -513,6 +513,7 @@ toc: - file: query-languages/esql/esql-multivalued-fields.md - file: query-languages/esql/esql-process-data-with-dissect-grok.md - file: query-languages/esql/esql-enrich-data.md + - file: query-languages/esql/esql-lookup-join.md - file: query-languages/esql/esql-implicit-casting.md - file: query-languages/esql/esql-time-spans.md - file: query-languages/esql/limitations.md From 27f6548cb08faea4a99a4f22edf8bfc9df63440d Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 20:47:25 -0600 Subject: [PATCH 06/16] fixing tables --- docs/reference/query-languages/esql/esql-lookup-join.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 660886aa6b337..9b893121c77b4 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -47,7 +47,8 @@ In this example, we have two sample tables: **employees** -| birth_date|emp_no:|first_name|gender|hire_date|language| +| birth_date|emp_no|first_name|gender|hire_date|language| +|---|---|---|---|---|---| |1955-10-04T00:00:00Z|10091|Amabile |M|1992-11-18T00:00:00Z|3| |1964-10-18T00:00:00Z|10092|Valdiodio |F|1989-09-22T00:00:00Z|1| |1964-06-11T00:00:00Z|10093|Sailaja |M|1996-11-05T00:00:00Z|3| @@ -57,6 +58,7 @@ In this example, we have two sample tables: **languages_non_unique_key** |language_code|language_name|country| +|---|---|---| |1|English|Canada| |1|English| |1||United Kingdom| From 1cc18a1834770d6c92f7767ced464dc929ec19e5 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 21:34:24 -0600 Subject: [PATCH 07/16] updates --- .../query-languages/esql/esql-commands.md | 22 ++++++++++++++++--- .../query-languages/esql/esql-lookup-join.md | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-commands.md b/docs/reference/query-languages/esql/esql-commands.md index a4f076f510561..4bfc7ca93639e 100644 --- a/docs/reference/query-languages/esql/esql-commands.md +++ b/docs/reference/query-languages/esql/esql-commands.md @@ -665,10 +665,19 @@ FROM employees ## `LOOKUP JOIN` [esql-lookup-join] -`LOOKUP JOIN` is useful for any scenario where you need to pull in information from a lookup index to streamline data enrichment and analysis. +::::{warning} +This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. +:::: + +`LOOKUP JOIN` enables you to add data from another index, AKA a 'lookup' index, to your {esql} query results, simplifying data enrichment and analysis workflows. **Syntax** +``` +FROM +| LOOKUP JOIN ON +``` + ```esql FROM firewall_logs | LOOKUP JOIN threat_list ON source.IP @@ -677,11 +686,18 @@ FROM firewall_logs **Parameters** -TBD +`` +: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. +`` +: The field to join on. This field must exist in both your current query results and in the lookup index. If the field contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows). **Description** -TBD +The `LOOKUP JOIN` command adds new columns to your {esql} query results table by finding documents in a lookup index that share the same join field value as your result rows. + +For each row in your results table that matches a document in the lookup index based on the join field, all fields from the matching document are added as new columns to that row. + +If multiple documents in the lookup index match a single row in your results, the output will contain one row for each matching combination. **Examples** diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 9b893121c77b4..521b614197e26 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -102,7 +102,7 @@ FROM employees :::: -## Prerequisites for LOOKUP JOIN +## Prerequisites [esql-lookup-join-prereqs] To use `LOOKUP JOIN`, the following requirements must be met: From 0c5501b0fb0da20f90c00dc75b56cc75f2558174 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 21:51:15 -0600 Subject: [PATCH 08/16] fixing feilds --- docs/reference/query-languages/esql/esql-lookup-join.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index 521b614197e26..d7f7bcbbe42fc 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -34,10 +34,11 @@ The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} ind :alt: esql lookup join ::: -$$$esql-source-index$$$ +`` +: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. -Source index -: An index which stores data that the `LOOKUP` command can add to input tables. You can create and manage these indices just like a regular {{es}} index. You also can use the same source index in multiple lookup joins. +`` +: The field to join on. This field must exist in both your current query results and in the lookup index. If the field contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows). ## Example From 03ca01a2074a89068d333eae6f70c63e105d07c3 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 21:54:17 -0600 Subject: [PATCH 09/16] escaping characters in table --- docs/reference/query-languages/esql/esql-lookup-join.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d7f7bcbbe42fc..f111d6259bc8b 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -64,13 +64,13 @@ In this example, we have two sample tables: |1|English| |1||United Kingdom| |1|English|United States of America| -|2|German|[Germany|Austria]| +|2|German|[Germany\|Austria]| |2|German|Switzerland| |2|German| |4|Quenya| |5||Atlantis| -|[6|7]|Mv-Lang|Mv-Land| -|[7|8]|Mv-Lang2|Mv-Land2| +|[6\|7]|Mv-Lang|Mv-Land| +|[7\|8]|Mv-Lang2|Mv-Land2| |Null-Lang|Null-Land| |Null-Lang2|Null-Land2| From f004e8c6e4a5272167f236f597462db6e8941ac5 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 21:57:51 -0600 Subject: [PATCH 10/16] adding index mode for lookup --- docs/reference/elasticsearch/index-settings/index-modules.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/reference/elasticsearch/index-settings/index-modules.md b/docs/reference/elasticsearch/index-settings/index-modules.md index 7c6aaab2ecfac..b099313953671 100644 --- a/docs/reference/elasticsearch/index-settings/index-modules.md +++ b/docs/reference/elasticsearch/index-settings/index-modules.md @@ -72,6 +72,10 @@ Index mode supports the following values: `standard` : Standard indexing with default settings. +`lookup` +: Index that can be used for lookup joins in ES|QL. Limited to 1 shard. + + `time_series` : *(data streams only)* Index mode optimized for storage of metrics. For more information, see [Time series index settings](time-series.md). From 2ad2296f84c43c4daae999882c4cd2fdc1e3ccee Mon Sep 17 00:00:00 2001 From: George Wallace Date: Mon, 10 Mar 2025 21:59:04 -0600 Subject: [PATCH 11/16] fixing substituion --- docs/reference/query-languages/esql/esql-commands.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-commands.md b/docs/reference/query-languages/esql/esql-commands.md index 4bfc7ca93639e..83fae20a94a9e 100644 --- a/docs/reference/query-languages/esql/esql-commands.md +++ b/docs/reference/query-languages/esql/esql-commands.md @@ -669,7 +669,7 @@ FROM employees This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. :::: -`LOOKUP JOIN` enables you to add data from another index, AKA a 'lookup' index, to your {esql} query results, simplifying data enrichment and analysis workflows. +`LOOKUP JOIN` enables you to add data from another index, AKA a 'lookup' index, to your {{esql}} query results, simplifying data enrichment and analysis workflows. **Syntax** @@ -688,6 +688,7 @@ FROM firewall_logs `` : The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. + `` : The field to join on. This field must exist in both your current query results and in the lookup index. If the field contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows). From dc3cb7cb1107949658f510741305c0eb295d20d1 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Wed, 12 Mar 2025 08:39:54 -0600 Subject: [PATCH 12/16] updates --- .../query-languages/esql/esql-commands.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-commands.md b/docs/reference/query-languages/esql/esql-commands.md index 83fae20a94a9e..0c2e8fff4f176 100644 --- a/docs/reference/query-languages/esql/esql-commands.md +++ b/docs/reference/query-languages/esql/esql-commands.md @@ -702,6 +702,10 @@ If multiple documents in the lookup index match a single row in your results, th **Examples** +::::{tip} +In case of name collisions, the newly created columns will override existing columns. +:::: + **IP Threat correlation**: This query would allow you to see if any source IPs match known malicious addresses. ```esql @@ -724,10 +728,18 @@ FROM app_logs | LOOKUP JOIN service_owners ON service_id ``` -In case of name collisions, the newly created columns will override existing columns. +`LOOKUP JOIN` is generally faster when there are fewer rows to join with. ES|QL will try and perform any `WHERE` clause before the `LOOKUP JOIN` where possible. The first example below has the `WHERE` clause before the `LOOKUP JOIN`. This is not needed, the optimizer will move the filter before the lookup. Both examples will have the same results. ```esql +FROM Left +| WHERE Language IS NOT NULL +| LOOKUP JOIN Right ON Key +``` +```esql +FROM Left +| LOOKUP JOIN Right ON Key +| WHERE Language IS NOT NULL ``` ## `MV_EXPAND` [esql-mv_expand] From be42837bf26ebfd1781ce1bf51ab46f761f4e10c Mon Sep 17 00:00:00 2001 From: George Wallace Date: Wed, 12 Mar 2025 08:42:57 -0600 Subject: [PATCH 13/16] updates --- docs/reference/query-languages/esql/esql-commands.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-commands.md b/docs/reference/query-languages/esql/esql-commands.md index 0c2e8fff4f176..18556a2504aa2 100644 --- a/docs/reference/query-languages/esql/esql-commands.md +++ b/docs/reference/query-languages/esql/esql-commands.md @@ -728,7 +728,9 @@ FROM app_logs | LOOKUP JOIN service_owners ON service_id ``` -`LOOKUP JOIN` is generally faster when there are fewer rows to join with. ES|QL will try and perform any `WHERE` clause before the `LOOKUP JOIN` where possible. The first example below has the `WHERE` clause before the `LOOKUP JOIN`. This is not needed, the optimizer will move the filter before the lookup. Both examples will have the same results. +`LOOKUP JOIN` is generally faster when there are fewer rows to join with. {{esql}} will try and perform any `WHERE` clause before the `LOOKUP JOIN` where possible. + +The two following examples will have the same results. The two examples have the `WHERE` clause before and after the `LOOKUP JOIN`. It does not matter how you write your query, our optimizer will move the filter before the lookup when ran. ```esql FROM Left From 2ea1689bbb79dc2f92957b585a920eb6c517c3b0 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Thu, 13 Mar 2025 07:34:14 -0600 Subject: [PATCH 14/16] Update docs/reference/query-languages/esql/esql-commands.md Co-authored-by: Alexander Spies --- docs/reference/query-languages/esql/esql-commands.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/query-languages/esql/esql-commands.md b/docs/reference/query-languages/esql/esql-commands.md index 18556a2504aa2..4925050c822e6 100644 --- a/docs/reference/query-languages/esql/esql-commands.md +++ b/docs/reference/query-languages/esql/esql-commands.md @@ -730,7 +730,7 @@ FROM app_logs `LOOKUP JOIN` is generally faster when there are fewer rows to join with. {{esql}} will try and perform any `WHERE` clause before the `LOOKUP JOIN` where possible. -The two following examples will have the same results. The two examples have the `WHERE` clause before and after the `LOOKUP JOIN`. It does not matter how you write your query, our optimizer will move the filter before the lookup when ran. +The two following examples will have the same results. The two examples have the `WHERE` clause before and after the `LOOKUP JOIN`. It does not matter how you write your query, our optimizer will move the filter before the lookup when possible. ```esql FROM Left From 2d0c5643580340a6e3920110b473f15ecf188782 Mon Sep 17 00:00:00 2001 From: George Wallace Date: Thu, 13 Mar 2025 07:36:43 -0600 Subject: [PATCH 15/16] Update esql-lookup-join.md --- docs/reference/query-languages/esql/esql-lookup-join.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index f111d6259bc8b..d460233512e3c 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -67,8 +67,8 @@ In this example, we have two sample tables: |2|German|[Germany\|Austria]| |2|German|Switzerland| |2|German| -|4|Quenya| -|5||Atlantis| +|4|Spanish| +|5||France| |[6\|7]|Mv-Lang|Mv-Land| |[7\|8]|Mv-Lang2|Mv-Land2| |Null-Lang|Null-Land| @@ -95,8 +95,8 @@ FROM employees | 10092 | 2 | German | Switzerland| | 10092 | 2 | German | null| | 10093 | 3 | null | null| -| 10094 | 4 | Quenya | null| -| 10095 | 5 | null | Atlantis| +| 10094 | 4 | Spanish | null| +| 10095 | 5 | null | France| ::::{important} `LOOKUP JOIN` does not guarantee the output to be in any particular order. If a certain order is required, users should use a [`SORT`](/reference/query-languages/esql/esql-commands.md#esql-sort) somewhere after the `LOOKUP JOIN`. From 7e7efc87c8ff8d39fdf4990725c95ef2f9e3305b Mon Sep 17 00:00:00 2001 From: George Wallace Date: Thu, 13 Mar 2025 07:40:06 -0600 Subject: [PATCH 16/16] Fix table formatting in esql-lookup-join.md --- docs/reference/query-languages/esql/esql-lookup-join.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/query-languages/esql/esql-lookup-join.md b/docs/reference/query-languages/esql/esql-lookup-join.md index d460233512e3c..a3bc909be1ccc 100644 --- a/docs/reference/query-languages/esql/esql-lookup-join.md +++ b/docs/reference/query-languages/esql/esql-lookup-join.md @@ -71,8 +71,8 @@ In this example, we have two sample tables: |5||France| |[6\|7]|Mv-Lang|Mv-Land| |[7\|8]|Mv-Lang2|Mv-Land2| -|Null-Lang|Null-Land| -|Null-Lang2|Null-Land2| +||Null-Lang|Null-Land| +||Null-Lang2|Null-Land2| Running the following query would provide the results shown below.