rgw: RGWSI_SysObj_Cache::remove() invalidates after successful delete

invalidating the cache before the librados delete means that a racing call
to `RGWSI_SysObj_Cache::read()` may succeed and repopulate the cache. in
that case, subsequent reads will continue to return cached data even after
the librados delete succeeds

Fixes: https://tracker.ceph.com/issues/64480

Signed-off-by: Casey Bodley <cbodley@redhat.com>

src/rgw/services/svc_sys_obj_cache.cc

@@ -89,6 +89,11 @@ int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp,
                                optional_yield y)
 
 {
+  int r = RGWSI_SysObj_Core::remove(dpp, objv_tracker, obj, y);
+  if (r < 0) {
+    return r;
+  }
+
   rgw_pool pool;
   string oid;
   normalize_pool_and_obj(obj.pool, obj.oid, pool, oid);
@@ -97,12 +102,12 @@ int RGWSI_SysObj_Cache::remove(const DoutPrefixProvider *dpp,
   cache.invalidate_remove(dpp, name);
 
   ObjectCacheInfo info;
-  int r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y);
+  r = distribute_cache(dpp, name, obj, info, INVALIDATE_OBJ, y);
   if (r < 0) {
     ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to distribute cache: r=" << r << dendl;
-  }
+  } // not fatal
 
-  return RGWSI_SysObj_Core::remove(dpp, objv_tracker, obj, y);
+  return 0;
 }
 
 int RGWSI_SysObj_Cache::read(const DoutPrefixProvider *dpp,