osd: consider high/low mode when putting agent to sleep

If we are in low flush mode, we may only get up to max_low_ops in flight,
in which case we may never go to sleep here.

Fix it by using the max_low_ops threshold when appropriate.

Note that agent_work() might start up *more* than this many ops (if there
are lots of evicts to do) currently, but I think it is fine if evicts go
a bit slower if we are in low mode.  (Really, the high/low shouldn't be
tied to flushing specifically.)

Fixes: #14752
Reported-by: Markus Blank-Burian <burian@muenster.de>
Signed-off-by: Sage Weil <sage@redhat.com>

src/osd/OSD.cc

@@ -535,8 +535,11 @@ void OSDService::agent_entry()
 	     << (agent_active ? " active" : " NOT ACTIVE")
 	     << dendl;
     dout(20) << __func__ << " oids " << agent_oids << dendl;
-    if (agent_ops >= g_conf->osd_agent_max_ops || top.empty() ||
-	!agent_active) {
+    int max = g_conf->osd_agent_max_ops - agent_ops;
+    int agent_flush_quota = max;
+    if (!flush_mode_high_count)
+      agent_flush_quota = g_conf->osd_agent_max_low_ops - agent_ops;
+    if (agent_flush_quota <= 0 || top.empty() || !agent_active) {
       agent_cond.Wait(agent_lock);
       continue;
     }
@@ -546,11 +549,9 @@ void OSDService::agent_entry()
       agent_valid_iterator = true;
     }
     PGRef pg = *agent_queue_pos;
-    int max = g_conf->osd_agent_max_ops - agent_ops;
-    int agent_flush_quota = max;
-    if (!flush_mode_high_count)
-      agent_flush_quota = g_conf->osd_agent_max_low_ops - agent_ops;
-    dout(10) << "high_count " << flush_mode_high_count << " agent_ops " << agent_ops << " flush_quota " << agent_flush_quota << dendl;
+    dout(10) << "high_count " << flush_mode_high_count
+	     << " agent_ops " << agent_ops
+	     << " flush_quota " << agent_flush_quota << dendl;
     agent_lock.Unlock();
     if (!pg->agent_work(max, agent_flush_quota)) {
       dout(10) << __func__ << " " << pg->get_pgid()