[ES|QL] Reduce number of iterations on null columns (#176174)

## Summary

Follow up on #174585

Less code, less iterations.

---------

Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
Co-authored-by: Stratoula Kalafateli <stratoula1@gmail.com>
Co-authored-by: Julia Rechkunova <julia.rechkunova@gmail.com>

src/plugins/data/common/search/expressions/esql.ts

@@ -11,7 +11,6 @@ import { castEsToKbnFieldTypeName, ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/
 import { i18n } from '@kbn/i18n';
 import type {
   Datatable,
-  DatatableColumn,
   DatatableColumnType,
   ExpressionFunctionDefinition,
 } from '@kbn/expressions-plugin/common';
@@ -238,37 +237,28 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
           );
         }),
         map(({ rawResponse: body, warning }) => {
-          const columns =
-            body.columns?.map(({ name, type }) => ({
-              id: name,
-              name,
-              meta: { type: normalizeType(type) },
-            })) ?? [];
           // all_columns in the response means that there is a separation between
           // columns with data and empty columns
           // columns contain only columns with data while all_columns everything
           const hasEmptyColumns =
             body.all_columns && body.all_columns?.length > body.columns.length;
+          const lookup = new Set(
+            hasEmptyColumns ? body.columns?.map(({ name }) => name) || [] : []
+          );
+          const allColumns =
+            (body.all_columns ?? body.columns)?.map(({ name, type }) => ({
+              id: name,
+              name,
+              meta: { type: normalizeType(type) },
+              isNull: hasEmptyColumns ? !lookup.has(name) : false,
+            })) ?? [];
 
-          let emptyColumns: DatatableColumn[] = [];
-
+          // sort only in case of empty columns to correctly align columns to items in values array
           if (hasEmptyColumns) {
-            const difference =
-              body.all_columns?.filter((col1) => {
-                return !body.columns.some((col2) => {
-                  return col1.name === col2.name;
-                });
-              }) ?? [];
-            emptyColumns =
-              difference?.map(({ name, type }) => ({
-                id: name,
-                name,
-                meta: { type: normalizeType(type) },
-                isNull: true,
-              })) ?? [];
+            allColumns.sort((a, b) => Number(a.isNull) - Number(b.isNull));
           }
-          const allColumns = [...columns, ...emptyColumns];
-          const columnNames = allColumns.map(({ name }) => name);
+          const columnNames = allColumns?.map(({ name }) => name);
+
           const rows = body.values.map((row) => zipObject(columnNames, row));
 
           return {