fix high IO after sudden filebeat stop (elastic#35893)

In case of corrupted log file (which has good chances to happen in case
of sudden unclean system shutdown), we set a flag which causes us to
checkpoint immediately, but never do anything else besides that. This
causes filebeat to just checkpoint on each log operation (therefore
causing a high IO load on the server and also causing filebeat to fall
behind).

This change resets the logInvalid flag after a successful checkpointing.

libbeat/statestore/backend/memlog/diskstore.go

@@ -213,6 +213,10 @@ func (s *diskstore) mustCheckpoint() bool {
 	return s.logInvalid || s.checkpointPred(s.logFileSize)
 }
 
+func (s *diskstore) resetLogState() {
+	s.logInvalid = false
+}
+
 func (s *diskstore) Close() error {
 	if s.logFile != nil {
 		// always sync log file on ordinary shutdown.

libbeat/statestore/backend/memlog/store.go

@@ -227,6 +227,9 @@ func (s *store) logOperation(op op) error {
 			// appending the log operation.
 			// idea: make append configurable and retry checkpointing with backoff.
 			_ = s.disk.LogOperation(op)
+		} else {
+			// after successfully checkpointing, reset the logInvalid flag.
+			s.disk.resetLogState()
 		}
 
 		return err