Skip to content

DAOS-19386 dfuse: move notify calls to an asynchronous queue #18806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mjmac wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

DAOS-19386 dfuse: move notify calls to an asynchronous queue #18806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a21a3c8
DAOS-19386 test: Add stall watchdog to NLT FI tests
mjmac Aug 2, 2026
e3242d1
DAOS-19386 dfuse: move notify calls to an asynchronous queue
mjmac Aug 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ci/unit/test_nlt_node.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ mkdir -p nlt_logs
sudo mount -t tmpfs tmpfs nlt_logs
sudo chown jenkins:jenkins nlt_logs

# Unbuffered so the console shows exactly where a hang occurs.
TMPDIR="$(pwd)/nlt_logs" \
PYTHONUNBUFFERED=1 \
HTTPS_PROXY="${DAOS_HTTPS_PROXY:-}" \
NO_PROXY="${DAOS_NO_PROXY:-}" \
exec ./utils/node_local_test.py "$@"
89 changes: 76 additions & 13 deletions src/client/dfs/duns.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/**
* (C) Copyright 2019-2024 Intel Corporation.
* (C) Copyright 2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -899,6 +900,69 @@ duns_link_lustre_path(const char *pool, const char *cont, daos_cont_layout_t typ
}
#endif

/* Bounds queued-notification delivery latency, normally microseconds; does not wait out
* dentry expiry on mounts with long timeouts.
*/
#define DUNS_RESOLVE_TIMEOUT_MS 10000
#define DUNS_RESOLVE_BACKOFF_MS_MAX 50

/* Poll until dfuse binds the entry point to the new container, or we
* hit the timeout.
*/
static int
duns_wait_for_resolution(const char *path, uuid_t cont_uuid)
{
struct timespec start, now;
int backoff_ms = 1;
int rc;

clock_gettime(CLOCK_MONOTONIC, &start);

while (1) {
struct dfuse_il_reply il_reply = {};
int fd;
int64_t elapsed_ms;

fd = open(path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
if (fd == -1) {
rc = errno;
/* ENOLINK: dfuse attempted entry-point resolution and the container
* connect failed - expected during rebinding, and terminal here since a
* failed connect will not heal within the poll window.
*/
if (rc != ENOLINK)
D_ERROR("Failed to open %s to verify resolution: %d (%s)\n", path,
rc, strerror(rc));
return rc;
}

rc = ioctl(fd, DFUSE_IOCTL_IL, &il_reply);
close(fd);
if (rc == -1) {
rc = errno;
D_ERROR("Dfuse IL ioctl failed on %s: %d (%s)\n", path, rc, strerror(rc));
return rc;
}

if (uuid_compare(il_reply.fir_cont, cont_uuid) == 0)
return 0;

clock_gettime(CLOCK_MONOTONIC, &now);
elapsed_ms =
(now.tv_sec - start.tv_sec) * 1000 + (now.tv_nsec - start.tv_nsec) / 1000000;
if (elapsed_ms >= DUNS_RESOLVE_TIMEOUT_MS) {
D_ERROR("Entry point %s still bound to container " DF_UUIDF " after %dms, "
"expected " DF_UUIDF "\n",
path, DP_UUID(il_reply.fir_cont), DUNS_RESOLVE_TIMEOUT_MS,
DP_UUID(cont_uuid));
return ENOLINK;
}

usleep(backoff_ms * 1000);
backoff_ms = min(backoff_ms * 2, DUNS_RESOLVE_BACKOFF_MS_MAX);
}
}

int
duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
{
Expand Down Expand Up @@ -1099,20 +1163,12 @@ duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
goto err_cont;
}
if (backend_dfuse) {
struct stat finfo;
/*
* This next stat will cause dfuse to lookup the entry point and perform a
* container connect, therefore this data will be read from root of the new
* container, not the directory.
*
* TODO: This could call getxattr to verify success.
/* Confirm dfuse has looked up the entry point and connected to the new
* container.
*/
rc = stat(path, &finfo);
if (rc) {
rc = errno;
D_ERROR("Failed to access new container: %d (%s)\n", rc, strerror(rc));
goto err_link;
}
rc = duns_wait_for_resolution(path, attrp->da_cuuid);
if (rc)
goto err_verify;
Comment on lines -1102 to +1171

@mchaarawi mchaarawi Aug 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i understand now why you are doing this now.
technically this stat was there (as the comment says) to force the lookup. but with the new async invalidation, there is a chance it might hit a stale dentry. maybe for now we don't even do anything here.
but i guess your test exposes something inthis case.

your PR actually exposes this issue more bec you are also moving the inval_inode and notify_delete() as async, so maybe you do hit this issue in your test. but normally maybe o

}

return rc;
Expand All @@ -1126,6 +1182,13 @@ duns_create_path(daos_handle_t poh, const char *path, struct duns_attr_t *attrp)
else if (attrp->da_type != DAOS_PROP_CO_LAYOUT_UNKNOWN)
unlink(path);
return rc;
err_verify:
/* clean up the path before destroying the linked container */
rmdir(path);
rc2 = daos_cont_destroy(poh, attrp->da_cont, 1, NULL);
if (rc2)
D_ERROR("Failed to cleanup created container %s (%d)\n", attrp->da_cont, rc2);
return rc;
}

int
Expand Down
3 changes: 3 additions & 0 deletions src/client/dfuse/SConscript
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,9 @@ def scons():

cenv.Install(os.path.join("$PREFIX", 'bin'), dfuse_bin)

if prereqs.test_requested():
SConscript('tests/SConscript', exports={'denv': cenv})


if __name__ == "SCons.Script":
scons()
21 changes: 21 additions & 0 deletions src/client/dfuse/dfuse.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
#include <fused/fuse.h>
#include <fused/fuse_lowlevel.h>

/* Only inval.c may call these directly. */
#ifndef DFUSE_NOTIFY_RAW_OK
#pragma GCC poison fuse_lowlevel_notify_inval_entry fuse_lowlevel_notify_expire_entry fuse_lowlevel_notify_delete fuse_lowlevel_notify_inval_inode
#endif

#include <gurt/list.h>
#include <gurt/hash.h>
#include <gurt/atomic.h>
Expand Down Expand Up @@ -76,6 +81,11 @@ struct dfuse_info {
ATOMIC uint64_t di_fh_count;
ATOMIC uint64_t di_pool_count;
ATOMIC uint64_t di_container_count;

ATOMIC uint64_t di_notify_enqueued;
ATOMIC uint64_t di_notify_coalesced;
ATOMIC uint64_t di_notify_delivered;
ATOMIC uint64_t di_notify_dropped;
};

struct dfuse_eq {
Expand Down Expand Up @@ -1201,6 +1211,17 @@ ival_thread_stop();
void
ival_fini();

/* Fire-and-forget reverse notifications, delivered by the notify thread in inval.c */
void
dfuse_notify_inval_entry(struct dfuse_info *dfuse_info, fuse_ino_t parent, const char *name);

void
dfuse_notify_delete(struct dfuse_info *dfuse_info, fuse_ino_t parent, fuse_ino_t ino,
const char *name);

void
dfuse_notify_inval_inode(struct dfuse_info *dfuse_info, fuse_ino_t ino);

/* Data caching functions */

/* Mark the data cache as up-to-date from now */
Expand Down
11 changes: 10 additions & 1 deletion src/client/dfuse/dfuse_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* (C) Copyright 2016-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025 Google LLC
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
Expand Down Expand Up @@ -977,6 +977,15 @@ main(int argc, char **argv)
}
}

if (dfuse_info)
DFUSE_TRA_INFO(dfuse_info,
"Notify: enqueued=" DF_U64 " coalesced=" DF_U64 " delivered=" DF_U64
" dropped=" DF_U64,
atomic_load_relaxed(&dfuse_info->di_notify_enqueued),
atomic_load_relaxed(&dfuse_info->di_notify_coalesced),
atomic_load_relaxed(&dfuse_info->di_notify_delivered),
atomic_load_relaxed(&dfuse_info->di_notify_dropped));

DFUSE_TRA_DOWN(dfuse_info);
daos_fini();
out_debug:
Expand Down
Loading
Loading