fix(insights): Fall back to dataNodeLogic results and add dashboard concurrency

frontend/src/queries/nodes/DataNode/dataNodeLogic.ts

@@ -127,14 +127,17 @@ export const dataNodeLogic = kea<dataNodeLogicType>([
             actions.clearResponse()
         }
         if (
-            !(props.cachedResults && props.key.includes('dashboard')) && // Don't load data on dashboard if cached results are available
+            !(
+                (props.cachedResults?.['result'] || props.cachedResults?.['results']) &&
+                props.key.includes('dashboard')
+            ) && // Don't load data on dashboard if cached results are available
             ((!values.response?.['result'] && !values.response?.['results']) ||
                 !queryEqual(props.query, oldProps.query)) &&
             (!props.cachedResults ||
                 (isInsightQueryNode(props.query) && !props.cachedResults['result'] && !props.cachedResults['results']))
         ) {
             actions.loadData()
-        } else if (props.cachedResults) {
+        } else if (props.cachedResults && !values.response?.['result'] && !values.response?.['results']) {
             // Use cached results if available, otherwise this logic will load the data again
             actions.setResponse(props.cachedResults)
         }

frontend/src/scenes/dashboard/dashboardLogic.test.ts

@@ -459,7 +459,7 @@ describe('dashboardLogic', () => {
             })
                 .toFinishAllListeners()
                 .toMatchValues({
-                    refreshStatus: { 1001: { error: true, timer: expect.anything() } },
+                    refreshStatus: { 1001: { refreshed: true, timer: expect.anything() } },
                 })
         })
     })

frontend/src/scenes/dashboard/dashboardLogic.tsx

@@ -84,6 +84,39 @@ export interface RefreshStatus {
 
 export const AUTO_REFRESH_INITIAL_INTERVAL_SECONDS = 1800
 
+async function runWithLimit<T>(tasks: (() => Promise<T>)[], limit: number): Promise<T[]> {
+    const results: T[] = []
+    const activePromises: Promise<void>[] = []
+    const remainingTasks = [...tasks] // Clone the tasks array to manage remaining tasks
+
+    const enqueueTask = (): void => {
+        while (activePromises.length < limit && remainingTasks.length > 0) {
+            const task = remainingTasks.shift() // Get the next task and remove it from remaining
+            if (task) {
+                const promise = task()
+                    .then((result) => {
+                        results.push(result)
+                        void activePromises.splice(activePromises.indexOf(promise), 1) // Remove promise when done
+                    })
+                    .catch((error) => {
+                        void activePromises.splice(activePromises.indexOf(promise), 1) // Handle errors and remove promise
+                        console.error('Error executing task:', error)
+                    })
+                activePromises.push(promise)
+            }
+        }
+    }
+
+    enqueueTask() // Initial population of the activePromises array
+
+    while (activePromises.length > 0) {
+        await Promise.race(activePromises)
+        enqueueTask() // Check and enqueue new tasks if there's room
+    }
+
+    return results
+}
+
 // to stop kea typegen getting confused
 export type DashboardTileLayoutUpdatePayload = Pick<DashboardTile, 'id' | 'layouts'>
 
@@ -1010,7 +1043,7 @@ export const dashboardLogic = kea<dashboardLogicType>([
                         }
                         cancelled = true
                     } else {
-                        actions.setRefreshError(insight.short_id)
+                        actions.setRefreshStatus(insight.short_id)
                     }
                 }
 
@@ -1043,17 +1076,7 @@ export const dashboardLogic = kea<dashboardLogicType>([
                 }
             })
 
-            async function loadNextPromise(): Promise<void> {
-                if (!cancelled && fetchItemFunctions.length > 0) {
-                    const nextPromise = fetchItemFunctions.shift()
-                    if (nextPromise) {
-                        await nextPromise()
-                        await loadNextPromise()
-                    }
-                }
-            }
-
-            void loadNextPromise()
+            await runWithLimit(fetchItemFunctions, 2)
 
             eventUsageLogic.actions.reportDashboardRefreshed(dashboardId, values.newestRefreshed)
         },