[CORE-1198] Add a timeout to loki requests from debug dump (#8329)

* loki timeout and line num change

* move timout to debug dump server

src/internal/lokiutil/lokiutil.go

@@ -13,7 +13,7 @@ import (
 const (
 	// maxLogMessages is the maximum number of log messages the loki server
 	// will send us, it will error if this is made higher.
-	maxLogMessages = 5000
+	maxLogMessages = 2000
 )
 
 func forEachLine(resp loki.QueryResponse, f func(t time.Time, line string) error) error {

src/server/debug/server/server.go

@@ -777,6 +777,9 @@ func (s *debugServer) collectPipelineDumpFunc(limit int64) collectPipelineFunc {
 		}
 		if s.env.Config().LokiHost != "" {
 			if err := s.forEachWorkerLoki(ctx, pipelineInfo, func(pod string) error {
+				// Loki requests can hang if the size of the log lines is too big, so we set a timeout
+				ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+				defer cancel()
 				workerPrefix := join(podPrefix, pod)
 				if len(prefix) > 0 {
 					workerPrefix = join(prefix[0], workerPrefix)