Merge tag 'nfs-for-7.4-2' of git://git.linux-nfs.org/projects/anna/linux-nfs

Pull NFS client fixes from Anna Schumaker:

 - Various NFS and SunRPC fixes from Avi Weiss

 - Various localio fixes from Mike Snitzer

 - Delegated timestamp fixes from Jeff Layton

 - CB_NOTIFY_LOCK fixes from Olga Kornievskaia

 - Various sunrpc, flexfiles, and xprtrdma fixes from Tim Menninger

* tag 'nfs-for-7.4-2' of git://git.linux-nfs.org/projects/anna/linux-nfs: (26 commits)
  NFSv4: Prevent indefinite waits for out-of-order OPEN stateids
  NFSv4/flexfiles: limit transport disconnects when cancelling I/O
  SUNRPC: add request-scoped disconnects when cancelling tasks
  pNFS: avoid repeatedly cancelling I/O for the same lseg
  xprtrdma: serialize unmap_sync with xprt_disconnect
  NFSv4: remember that the server may send CB_NOTIFY_LOCK
  NFSv4: wake blocking lock waiters when a delegated lock is released
  NFS: stamp after the server's ctime following a CLONE
  NFS: stamp after the server's ctime when truncating under delegated times
  NFS: don't let delegated c/mtime stamps go backward
  NFS/localio: give the LOCALIO probe a short timeout of its own
  NFS/localio: probe again for LOCALIO when localio_enabled is set to Y
  NFS/localio: hold the client across an asynchronous LOCALIO probe
  SUNRPC: Balance queue accounting when relocating a task
  SUNRPC: Balance switch queue length on transport removal
  SUNRPC: Fix TCP-with-TLS transport class identification
  SUNRPC: preserve transport security policy when cloning clients
  SUNRPC: preserve timeout policy when cloning clients
  SUNRPC: preserve fatal network errors during RPC binding
  sunrpc: instantiate rpc_pipefs dentry before publishing it
  ...
This commit is contained in:
Linus Torvalds
2026-11-15 13:48:31 -08:00
22 changed files with 529 additions and 107 deletions
+12
View File
@@ -312,6 +312,18 @@ Module Parameters
/sys/module/nfs/parameters/localio_enabled (bool)
controls if LOCALIO is enabled, defaults to Y. If client and server are
local but 'localio_enabled' is set to N then LOCALIO will not be used.
It is checked on every I/O: setting it to N sends the next I/O of every
client over the network, and setting it back to Y resumes LOCALIO. A
client that probed while it was N is probed again when it is set to Y.
/sys/module/nfs/parameters/localio_timeout (uint)
sets how many seconds the LOCALIO probe waits for each reply from the
server, defaults to 10, at most 600. A server that does not support
LOCALIO may not answer the probe at all. If no reply comes in time,
the probe gives up without a log message. The client then does not use
LOCALIO. The mount's timeouts do not apply to the probe. Set to 0, the
probe uses the mount's timeouts instead, as an NFS RPC would. A change
applies to the next probe.
/sys/module/nfs/parameters/localio_O_DIRECT_semantics (bool)
controls if O_DIRECT extends down to the underlying filesystem, defaults
+2
View File
@@ -95,6 +95,8 @@ void nfs4_inode_make_writeable(struct inode *inode);
void nfs_update_delegated_atime(struct inode *inode);
void nfs_update_delegated_mtime(struct inode *inode);
void nfs_update_delegated_mtime_locked(struct inode *inode);
void nfs_delegated_ctime_raise(struct inode *inode,
const struct nfs_fattr *fattr);
static inline int nfs_have_read_or_write_delegation(struct inode *inode)
{
+3 -1
View File
@@ -1253,6 +1253,8 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx)
spin_lock(&file->f_lock);
desc->dir_cookie = dir_ctx->dir_cookie;
desc->folio_index = dir_ctx->page_index;
desc->current_index = (loff_t)desc->folio_index *
nfs_readdir_array_maxentries();
desc->last_cookie = dir_ctx->last_cookie;
desc->attr_gencount = dir_ctx->attr_gencount;
desc->eof = dir_ctx->eof;
@@ -2735,7 +2737,7 @@ static bool nfs_rename_is_unsafe_cross_dir(struct dentry *old_dentry,
{
struct nfs_server *server = NFS_SB(old_dentry->d_sb);
if (old_dentry->d_parent != new_dentry->d_parent)
if (old_dentry->d_parent == new_dentry->d_parent)
return false;
if (server->fh_expire_type & NFS_FH_RENAME_UNSAFE)
return !(server->fh_expire_type & NFS_FH_NOEXPIRE_WITH_OPEN);
+15 -4
View File
@@ -11,6 +11,7 @@
#include <linux/nfs_mount.h>
#include <linux/nfs_page.h>
#include <linux/module.h>
#include <linux/bitmap.h>
#include <linux/file.h>
#include <linux/sched/mm.h>
@@ -340,6 +341,7 @@ static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls)
{
if (fls) {
ff_layout_free_mirror_array(fls);
bitmap_free(fls->cancelled_stripes);
kfree(fls);
}
}
@@ -653,6 +655,12 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
fls->mirror_array[i]->report_interval = be32_to_cpup(p);
out_sort_mirrors:
fls->cancelled_stripes = bitmap_zalloc(
fls->mirror_array_cnt * dss_count, gfp_flags);
if (!fls->cancelled_stripes) {
rc = -ENOMEM;
goto out_err_free;
}
ff_layout_sort_mirrors(fls);
ret = &fls->generic_hdr;
dprintk("<-- %s (success)\n", __func__);
@@ -2582,6 +2590,10 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg,
if (devid && memcmp(&mirror->dss[dss_id].devid, devid,
sizeof(*devid)) != 0)
continue;
/* Do not cancel a stripe again while this lseg drains. */
if (test_and_set_bit(idx * mirror->dss_count + dss_id,
flseg->cancelled_stripes))
continue;
rcu_read_lock();
mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds);
if (IS_ERR_OR_NULL(mirror_ds) ||
@@ -2599,10 +2611,9 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg,
clnt = ds_clp->cl_rpcclient;
if (!clnt)
goto next;
if (!rpc_cancel_tasks(clnt, -ECANCELED,
ff_layout_match_io, lseg))
goto next;
rpc_clnt_disconnect(clnt);
rpc_cancel_tasks_and_disconnect(clnt, -ECANCELED,
ff_layout_match_io,
lseg);
next:
nfs4_ff_layout_put_deviceid(mirror_ds);
}
+1
View File
@@ -109,6 +109,7 @@ struct nfs4_ff_layout_segment {
u64 stripe_unit;
u32 flags;
u32 mirror_array_cnt;
unsigned long *cancelled_stripes;
struct nfs4_ff_layout_mirror *mirror_array[] __counted_by(mirror_array_cnt);
};
+52 -17
View File
@@ -629,28 +629,52 @@ nfs_fattr_fixup_delegated(struct inode *inode, struct nfs_fattr *fattr)
}
}
/*
* With delegated timestamps, the client's coarse clock can trail the
* server's fine-grained ctime cached from an OPEN or SETATTR reply. The
* server won't move ctime backward when the times are returned, so a
* new stamp must land after the cached ctime.
*/
static struct timespec64 nfs_delegated_stamp(struct inode *inode)
{
struct timespec64 ctime = inode_get_ctime(inode);
struct timespec64 now = current_time(inode);
if (timespec64_compare(&now, &ctime) <= 0) {
now = ctime;
timespec64_add_ns(&now, 1);
}
return now;
}
/*
* The server stamped c/mtime itself (e.g. for a size change). Raise the
* cached ctime to the server's so that the next delegated stamp lands
* after it.
*/
void nfs_delegated_ctime_raise(struct inode *inode,
const struct nfs_fattr *fattr)
{
struct timespec64 ctime = inode_get_ctime(inode);
if ((fattr->valid & NFS_ATTR_FATTR_CTIME) &&
timespec64_compare(&fattr->ctime, &ctime) > 0)
inode_set_ctime_to_ts(inode, fattr->ctime);
}
EXPORT_SYMBOL_GPL(nfs_delegated_ctime_raise);
static void nfs_set_timestamps_to_ts(struct inode *inode, struct iattr *attr)
{
unsigned int cache_flags = 0;
if (attr->ia_valid & ATTR_MTIME_SET) {
struct timespec64 ctime = inode_get_ctime(inode);
struct timespec64 mtime = inode_get_mtime(inode);
struct timespec64 now;
bool updated = false;
now = inode_set_ctime_current(inode);
if (!timespec64_equal(&now, &ctime))
updated = true;
/* The stamp is always after the old ctime */
inode_set_ctime_to_ts(inode, nfs_delegated_stamp(inode));
inode_set_mtime_to_ts(inode, attr->ia_mtime);
if (!timespec64_equal(&now, &mtime))
updated = true;
inode_maybe_inc_iversion(inode, updated);
inode_maybe_inc_iversion(inode, true);
cache_flags |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME;
}
if (attr->ia_valid & ATTR_ATIME_SET) {
if (attr->ia_valid & ATTR_ATIME) {
inode_set_atime_to_ts(inode, attr->ia_atime);
cache_flags |= NFS_INO_INVALID_ATIME;
}
@@ -665,7 +689,11 @@ static void nfs_update_atime(struct inode *inode)
static void nfs_update_mtime(struct inode *inode)
{
inode_update_time(inode, FS_UPD_CMTIME, 0);
struct timespec64 now = nfs_delegated_stamp(inode);
/* Not inode_update_time(): lockless readers would see its stamp */
inode_set_ctime_to_ts(inode, now);
inode_set_mtime_to_ts(inode, now);
NFS_I(inode)->cache_validity &=
~(NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME);
}
@@ -684,6 +712,7 @@ void nfs_update_delegated_mtime_locked(struct inode *inode)
nfs_have_directory_delegation(inode))
nfs_update_mtime(inode);
}
EXPORT_SYMBOL_GPL(nfs_update_delegated_mtime_locked);
void nfs_update_delegated_mtime(struct inode *inode)
{
@@ -746,7 +775,7 @@ nfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
} else if (nfs_have_delegated_atime(inode) &&
attr->ia_valid & ATTR_ATIME &&
!(attr->ia_valid & ATTR_MTIME)) {
if (!(attr->ia_valid & ATTR_ATIME_SET)) {
if (!(attr->ia_valid & (ATTR_ATIME_SET | ATTR_TIMES_SET))) {
nfs_update_delegated_atime(inode);
attr->ia_valid &= ~ATTR_ATIME;
}
@@ -808,10 +837,14 @@ static int nfs_vmtruncate(struct inode * inode, loff_t offset)
nfs_ooo_clear(NFS_I(inode));
}
NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
/*
* Stamp in the same i_lock section that checked the delegation and
* raised ctime, before a delegation return can start.
*/
nfs_update_delegated_mtime_locked(inode);
spin_unlock(&inode->i_lock);
truncate_pagecache(inode, offset);
nfs_update_delegated_mtime_locked(inode);
spin_lock(&inode->i_lock);
out:
return err;
@@ -837,6 +870,8 @@ void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr,
if ((attr->ia_valid & ATTR_SIZE) != 0) {
if (!nfs_have_delegated_mtime(inode))
nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
else
nfs_delegated_ctime_raise(inode, fattr);
nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS);
nfs_inc_stats(inode, NFSIOS_SETATTRTRUNC);
nfs_vmtruncate(inode, attr->ia_size);
+114 -7
View File
@@ -25,6 +25,7 @@
#include <linux/nfs_xdr.h>
#include "internal.h"
#include "netns.h"
#include "pnfs.h"
#include "nfstrace.h"
@@ -55,7 +56,50 @@ struct nfs_local_fsync_ctx {
};
static bool localio_enabled __read_mostly = true;
module_param(localio_enabled, bool, 0644);
static void nfs_local_probe_all(void);
/*
* localio_enabled is checked on every I/O, so turning it off stops LOCALIO
* at once and turning it back on resumes it for clients that use it. A
* client that probed while it was off was disabled, and nothing probes it
* again: turning it on probes every client that does not use LOCALIO.
*/
static int localio_enabled_set(const char *val, const struct kernel_param *kp)
{
bool was = READ_ONCE(localio_enabled);
int ret;
ret = param_set_bool(val, kp);
if (!ret && !was && READ_ONCE(localio_enabled))
nfs_local_probe_all();
return ret;
}
static const struct kernel_param_ops localio_enabled_ops = {
.set = localio_enabled_set,
.get = param_get_bool,
};
module_param_cb(localio_enabled, &localio_enabled_ops, &localio_enabled, 0644);
/*
* Seconds the LOCALIO probe waits for each reply. With 0, the probe uses
* the NFS client's timeouts instead.
*/
static unsigned int localio_timeout __read_mostly = 10;
static int localio_timeout_set(const char *val, const struct kernel_param *kp)
{
return param_set_uint_minmax(val, kp, 0, 600);
}
static const struct kernel_param_ops localio_timeout_ops = {
.set = localio_timeout_set,
.get = param_get_uint,
};
module_param_cb(localio_timeout, &localio_timeout_ops, &localio_timeout, 0644);
MODULE_PARM_DESC(localio_timeout,
"LOCALIO probe timeout in seconds (default 10), 0 to use the mount's default");
static void nfs_local_do_read(struct nfs_local_kiocb *iocb,
const struct rpc_call_ops *call_ops);
@@ -131,13 +175,45 @@ const struct rpc_program nfslocalio_program = {
/*
* nfs_init_localioclient - Initialise an NFS localio client connection
*
* A server that does not support LOCALIO may not answer its RPCs at all.
* The probe shares the NFS client's transport and runs beside its RPCs.
* With localio_timeout set, it gets a quiet, soft timeout of that many
* seconds of its own, whatever timeouts the mount uses. With 0, it uses
* the NFS client's timeouts, as an NFS RPC would.
*/
static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp)
{
struct rpc_clnt *rpcclient_localio;
unsigned int timeout_secs = READ_ONCE(localio_timeout);
struct rpc_clnt *parent = clp->cl_rpcclient;
struct rpc_clnt *probe = NULL, *rpcclient_localio;
rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient,
&nfslocalio_program, 1);
/*
* rpc_bind_new_program() pings the new program with the timeout of
* the client it clones, so set the probe's timeout on a private
* clone first. The clone's cl_timeout points at the transport's
* timeout: give it its own copy rather than write through it.
*/
if (timeout_secs) {
probe = rpc_clone_client(parent);
if (IS_ERR(probe))
return probe;
probe->cl_timeout_default = (struct rpc_timeout) {
.to_initval = timeout_secs * HZ,
.to_maxval = timeout_secs * HZ,
.to_retries = 0,
};
probe->cl_timeout = &probe->cl_timeout_default;
probe->cl_softrtry = 1;
probe->cl_noretranstimeo = 0;
probe->cl_chatty = 0;
parent = probe;
}
rpcclient_localio = rpc_bind_new_program(parent, &nfslocalio_program, 1);
/* The new client holds a reference to its parent, the probe clone */
if (probe)
rpc_shutdown_client(probe);
dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n",
__func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR),
@@ -203,18 +279,49 @@ void nfs_local_probe_async_work(struct work_struct *work)
struct nfs_client *clp =
container_of(work, struct nfs_client, cl_local_probe_work);
if (!refcount_inc_not_zero(&clp->cl_count))
return;
nfs_local_probe(clp);
/* The reference nfs_local_probe_async() took for this work */
nfs_put_client(clp);
}
/*
* The work is part of the client, so it holds a reference on the client
* from when it is queued until it has run: the last nfs_put_client()
* cannot free the client under a queued probe.
*/
void nfs_local_probe_async(struct nfs_client *clp)
{
queue_work(nfsiod_workqueue, &clp->cl_local_probe_work);
if (!refcount_inc_not_zero(&clp->cl_count))
return;
if (!queue_work(nfsiod_workqueue, &clp->cl_local_probe_work))
nfs_put_client(clp); /* already queued, with its reference */
}
EXPORT_SYMBOL_GPL(nfs_local_probe_async);
/* Probe every client, in every namespace, that does not use LOCALIO. */
static void nfs_local_probe_all(void)
{
struct nfs_client *clp;
struct net *net;
/* Set on the module command line: there are no clients yet. */
if (!nfs_net_id)
return;
down_read(&net_rwsem);
for_each_net(net) {
struct nfs_net *nn = net_generic(net, nfs_net_id);
spin_lock(&nn->nfs_client_lock);
list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link)
if (clp->cl_cons_state == NFS_CS_READY &&
!nfs_client_is_local(clp))
nfs_local_probe_async(clp);
spin_unlock(&nn->nfs_client_lock);
}
up_read(&net_rwsem);
}
static inline void nfs_local_file_put(struct nfsd_file *localio)
{
/* nfs_to_nfsd_file_put_local() expects an __rcu pointer
+20 -5
View File
@@ -380,6 +380,7 @@ out:
* @pos: destination offset
* @len: copy length
* @oldsize: length of the file prior to clone/copy
* @fattr: destination attributes from the reply, or NULL
*
* Punch a hole in the inode page cache, so that the NFS client will
* know to retrieve new data.
@@ -387,7 +388,7 @@ out:
* invalid cached values for change attribute, ctime, mtime and space used.
*/
static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len,
loff_t oldsize)
loff_t oldsize, const struct nfs_fattr *fattr)
{
struct inode *inode = file_inode(file);
struct address_space *mapping = file->f_mapping;
@@ -405,8 +406,12 @@ static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len,
NFS_INO_INVALID_CTIME |
NFS_INO_INVALID_MTIME |
NFS_INO_INVALID_BLOCKS);
/* The server may have stamped c/mtime; stamp after it */
if (fattr && nfs_have_delegated_mtime(inode))
nfs_delegated_ctime_raise(inode, fattr);
/* Before a refresh can see the times invalid and take the server's */
nfs_update_delegated_mtime_locked(inode);
spin_unlock(&inode->i_lock);
nfs_update_delegated_mtime(inode);
}
static ssize_t _nfs42_proc_copy(struct file *src,
@@ -506,7 +511,8 @@ static ssize_t _nfs42_proc_copy(struct file *src,
goto out;
}
nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst);
nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst,
NULL);
nfs_invalidate_atime(src_inode);
status = res->write_res.count;
out:
@@ -1311,8 +1317,17 @@ static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f,
/* a zero-length count means clone to EOF in src */
if (count == 0 && res.dst_fattr->valid & NFS_ATTR_FATTR_SIZE)
count = nfs_size_to_loff_t(res.dst_fattr->size) - dst_offset;
nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst);
status = nfs_post_op_update_inode(dst_inode, res.dst_fattr);
nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst,
res.dst_fattr);
/*
* nfs_post_op_update_inode() would mark ctime invalid, so the
* reply's ctime would overwrite the delegated stamp.
*/
if (nfs_have_delegated_mtime(dst_inode))
status = nfs_refresh_inode(dst_inode, res.dst_fattr);
else
status = nfs_post_op_update_inode(dst_inode,
res.dst_fattr);
}
kfree(res.dst_fattr);
+22 -4
View File
@@ -1722,6 +1722,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state,
{
DEFINE_WAIT(wait);
int status = 0;
unsigned long deadline = jiffies + 5 * HZ;
signed long remaining;
for (;;) {
if (nfs_stateid_is_sequential(state, stateid))
@@ -1756,7 +1758,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state,
if (!fatal_signal_pending(current) &&
!nfs_current_task_exiting()) {
if (schedule_timeout(5*HZ) == 0)
remaining = (long)(deadline - jiffies);
if (remaining <= 0 || schedule_timeout(remaining) == 0)
status = -EAGAIN;
else
status = 0;
@@ -3093,6 +3096,8 @@ static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata,
if (server->caps & NFS_CAP_POSIX_LOCK)
set_bit(NFS_STATE_POSIX_LOCKS, &state->flags);
if (opendata->o_res.rflags & NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK)
server->nfs_client->cl_may_notify_lock = true;
if (server->nfs_client->cl_may_notify_lock)
set_bit(NFS_STATE_MAY_NOTIFY_LOCK, &state->flags);
if (opendata->o_res.rflags & NFS4_OPEN_RESULT_PRESERVE_UNLINKED)
set_bit(NFS_INO_PRESERVE_UNLINKED, &NFS_I(state->inode)->flags);
@@ -7287,8 +7292,15 @@ static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock *
if (status != 0)
goto out;
/* Is this a delegated lock? */
if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0)
if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0) {
struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
struct cb_notify_lock_args cbnl = { .cbnl_valid = false };
/* The server never saw it, so it will not notify waiters */
nfs_copy_fh(&cbnl.cbnl_fh, NFS_FH(inode));
__wake_up(&clp->cl_lock_waitq, TASK_NORMAL, 0, &cbnl);
goto out;
}
alloc_seqid = NFS_SERVER(inode)->nfs_client->cl_mvops->alloc_seqid;
seqid = alloc_seqid(&lsp->ls_seqid, GFP_KERNEL);
status = -ENOMEM;
@@ -7708,8 +7720,14 @@ nfs4_wake_lock_waiter(wait_queue_entry_t *wait, unsigned int mode, int flags, vo
struct nfs_lowner *lowner = &cbnl->cbnl_owner,
*wowner = &waiter->owner;
/* Only wake if the callback was for the same owner. */
if (lowner->id != wowner->id || lowner->s_dev != wowner->s_dev)
/*
* Only wake the owner a CB_NOTIFY_LOCK names. A lock cached
* under a delegation is released locally, and the releasing
* thread cannot tell which waiter wants it, so those wakeups
* (cbnl_valid false) name only the inode and wake them all.
*/
if (cbnl->cbnl_valid &&
(lowner->id != wowner->id || lowner->s_dev != wowner->s_dev))
return 0;
/* Make sure it's for the right inode */
+1 -1
View File
@@ -2086,7 +2086,7 @@ static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
break;
}
if (size_set && io_set) {
if (size && io)
if (size || io)
ret = true;
} else if (size || io)
ret = true;
+1
View File
@@ -120,6 +120,7 @@ struct nfs_client {
#define NFS_SP4_MACH_CRED_COMMIT 6 /* COMMIT */
#define NFS_SP4_MACH_CRED_PNFS_CLEANUP 7 /* LAYOUTRETURN */
wait_queue_head_t cl_lock_waitq;
bool cl_may_notify_lock;
#endif /* CONFIG_NFS_V4 */
/* Our own IP address, as a null-terminated string.
+4
View File
@@ -230,6 +230,10 @@ unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
bool (*fnmatch)(const struct rpc_task *,
const void *),
const void *data);
unsigned long rpc_cancel_tasks_and_disconnect(struct rpc_clnt *clnt, int error,
bool (*fnmatch)(const struct rpc_task *,
const void *),
const void *data);
void rpc_execute(struct rpc_task *);
void rpc_init_priority_wait_queue(struct rpc_wait_queue *, const char *);
void rpc_init_wait_queue(struct rpc_wait_queue *, const char *);
+5
View File
@@ -112,6 +112,11 @@ struct rpc_rqst {
ktime_t rq_xtime; /* transmit time stamp */
int rq_ntrans;
bool rq_disconnect_on_release;
/*
* protected by rq_xprt->transport_lock
* while the request is active
*/
#if defined(CONFIG_SUNRPC_BACKCHANNEL)
struct lwq_node rq_bc_list; /* Callback service list */
+87 -21
View File
@@ -653,6 +653,9 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args,
}
args->servername = xprt->servername;
args->nodename = clnt->cl_nodename;
args->xprtsec = clnt->cl_xprtsec;
if (!args->timeout)
args->timeout = clnt->cl_timeout;
new = rpc_new_client(args, xps, xprt, clnt);
if (IS_ERR(new))
@@ -901,6 +904,59 @@ void rpc_killall_tasks(struct rpc_clnt *clnt)
}
EXPORT_SYMBOL_GPL(rpc_killall_tasks);
struct rpc_cancel_disconnect_ctx {
bool (*fnmatch)(const struct rpc_task *task, const void *data);
const void *data;
};
static int rpc_mark_xprt_disconnect_on_release(struct rpc_clnt *clnt,
struct rpc_xprt *xprt,
void *arg)
{
struct rpc_cancel_disconnect_ctx *ctx = arg;
xprt_mark_matching_reqs_disconnect_on_release(xprt, clnt,
ctx->fnmatch,
ctx->data);
return 0;
}
static unsigned long
__rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
bool (*fnmatch)(const struct rpc_task *,
const void *),
const void *data,
bool disconnect)
{
struct rpc_task *task;
unsigned long count = 0;
if (list_empty(&clnt->cl_tasks))
return 0;
spin_lock(&clnt->cl_lock);
list_for_each_entry(task, &clnt->cl_tasks, tk_task) {
if (!RPC_IS_ACTIVATED(task))
continue;
if (!fnmatch(task, data))
continue;
rpc_task_try_cancel(task, error);
count++;
}
spin_unlock(&clnt->cl_lock);
if (disconnect && count) {
struct rpc_cancel_disconnect_ctx ctx = {
.fnmatch = fnmatch,
.data = data,
};
rpc_clnt_iterate_for_each_xprt(clnt,
rpc_mark_xprt_disconnect_on_release,
&ctx);
}
return count;
}
/**
* rpc_cancel_tasks - try to cancel a set of RPC tasks
* @clnt: Pointer to RPC client
@@ -916,28 +972,35 @@ unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
const void *),
const void *data)
{
struct rpc_task *task;
unsigned long count = 0;
if (list_empty(&clnt->cl_tasks))
return 0;
/*
* Spin lock all_tasks to prevent changes...
*/
spin_lock(&clnt->cl_lock);
list_for_each_entry(task, &clnt->cl_tasks, tk_task) {
if (!RPC_IS_ACTIVATED(task))
continue;
if (!fnmatch(task, data))
continue;
rpc_task_try_cancel(task, error);
count++;
}
spin_unlock(&clnt->cl_lock);
return count;
return __rpc_cancel_tasks(clnt, error, fnmatch, data, false);
}
EXPORT_SYMBOL_GPL(rpc_cancel_tasks);
/**
* rpc_cancel_tasks_and_disconnect - cancel matching RPC tasks
* @clnt: Pointer to RPC client
* @error: RPC task error value to set
* @fnmatch: Pointer to selector function
* @data: User data
*
* Like rpc_cancel_tasks(), but after cancelling matching tasks, marks
* matching in-flight requests so that xprt_release() conditionally
* disconnects the connection generation on which each request was sent.
*
* @fnmatch is called while holding spinlocks and must not sleep.
*
* Returns the number of active tasks matched by @fnmatch.
*/
unsigned long
rpc_cancel_tasks_and_disconnect(struct rpc_clnt *clnt, int error,
bool (*fnmatch)(const struct rpc_task *,
const void *),
const void *data)
{
return __rpc_cancel_tasks(clnt, error, fnmatch, data, true);
}
EXPORT_SYMBOL_GPL(rpc_cancel_tasks_and_disconnect);
static int rpc_clnt_disconnect_xprt(struct rpc_clnt *clnt,
struct rpc_xprt *xprt, void *dummy)
{
@@ -1194,7 +1257,7 @@ void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt)
(task->tk_flags & RPC_TASK_MOVEABLE)))
return;
xprt_release(task);
xprt_put(task->tk_xprt);
rpc_task_release_xprt(clnt, task->tk_xprt);
}
if (task->tk_flags & RPC_TASK_NO_ROUND_ROBIN)
task->tk_xprt = rpc_task_get_first_xprt(clnt);
@@ -2125,8 +2188,10 @@ call_bind_status(struct rpc_task *task)
goto retry_timeout;
case -ENETDOWN:
case -ENETUNREACH:
if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL)
if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL) {
status = task->tk_status;
break;
}
fallthrough;
case -ECONNREFUSED: /* connection problems */
case -ECONNRESET:
@@ -2257,6 +2322,7 @@ call_connect_status(struct rpc_task *task)
xprt_release(task);
value = atomic_long_dec_return(&xprt->queuelen);
atomic_long_dec(&xps->xps_queuelen);
if (value == 0)
rpc_xprt_switch_remove_xprt(xps, saved,
true);
+2 -1
View File
@@ -191,6 +191,7 @@ rpc_close_pipes(struct dentry *dentry)
cancel_delayed_work_sync(&pipe->queue_timeout);
rpc_inode_setowner(inode, NULL);
RPC_I(inode)->pipe = NULL;
wake_up(&RPC_I(inode)->waitq);
inode_unlock(inode);
}
@@ -665,8 +666,8 @@ int rpc_mkpipe_dentry(struct dentry *parent, const char *name,
rpci->private = private;
rpci->pipe = pipe;
rpc_inode_setowner(inode, private);
pipe->dentry = dentry; // borrowed
d_make_persistent(dentry, inode);
pipe->dentry = dentry; // borrowed
fsnotify_create(dir, dentry);
simple_done_creating(dentry);
return 0;
+8
View File
@@ -38,9 +38,17 @@ static inline int sock_is_loopback(struct sock *sk)
struct svc_serv;
struct svc_rqst;
struct rpc_clnt;
struct rpc_task;
struct rpc_xprt;
int rpc_clients_notifier_register(void);
void rpc_clients_notifier_unregister(void);
void auth_domain_cleanup(void);
void svc_sock_update_bufs(struct svc_serv *serv);
enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp);
void xprt_mark_matching_reqs_disconnect_on_release(struct rpc_xprt *xprt,
const struct rpc_clnt *clnt,
bool (*fnmatch)(const struct rpc_task *,
const void *),
const void *data);
#endif /* _NET_SUNRPC_SUNRPC_H */
+101 -6
View File
@@ -812,6 +812,17 @@ xprt_request_retransmit_after_disconnect(struct rpc_task *task)
!xprt_connected(xprt);
}
static void xprt_conditional_disconnect_locked(struct rpc_xprt *xprt,
unsigned int cookie)
__must_hold(&xprt->transport_lock)
{
if (cookie != xprt->connect_cookie)
return;
if (test_bit(XPRT_CLOSING, &xprt->state))
return;
xprt_schedule_autoclose_locked(xprt);
}
/**
* xprt_conditional_disconnect - force a transport to disconnect
* @xprt: transport to disconnect
@@ -827,12 +838,7 @@ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
{
/* Don't race with the test_bit() in xprt_clear_locked() */
spin_lock(&xprt->transport_lock);
if (cookie != xprt->connect_cookie)
goto out;
if (test_bit(XPRT_CLOSING, &xprt->state))
goto out;
xprt_schedule_autoclose_locked(xprt);
out:
xprt_conditional_disconnect_locked(xprt, cookie);
spin_unlock(&xprt->transport_lock);
}
@@ -1434,6 +1440,79 @@ xprt_request_dequeue_transmit(struct rpc_task *task)
spin_unlock(&xprt->queue_lock);
}
static bool
xprt_request_matches(const struct rpc_rqst *req,
const struct rpc_clnt *clnt,
bool (*fnmatch)(const struct rpc_task *, const void *),
const void *data)
{
const struct rpc_task *task = req->rq_task;
return task && task->tk_client == clnt && fnmatch(task, data);
}
static void
xprt_request_mark_disconnect_on_release(struct rpc_xprt *xprt,
struct rpc_rqst *req)
__must_hold(&xprt->queue_lock)
{
spin_lock(&xprt->transport_lock);
req->rq_disconnect_on_release = true;
spin_unlock(&xprt->transport_lock);
}
/**
* xprt_mark_matching_reqs_disconnect_on_release - mark matching in-flight
* requests for connection-generation
* teardown at release
* @xprt: transport whose queues to walk
* @clnt: RPC client whose requests are eligible
* @fnmatch: selector called with the request's owning rpc_task
* @data: caller data forwarded to @fnmatch
*
* @fnmatch must not sleep and must be safe to call while xprt->queue_lock
* is held.
*
* Walks the transmit queue (including per-owner rq_xmit2 chains) and the
* receive tree under xprt->queue_lock and marks each matching rpc_rqst so
* that xprt_release() will conditionally disconnect the transport against
* that request's rq_connect_cookie. Requests remain eligible for marking
* while they are present on a transport queue. The marker is set under
* xprt->transport_lock, nested inside xprt->queue_lock, and xprt_release()
* consumes it with xprt->transport_lock held after removing queue membership.
*
* The @clnt qualification preserves the selection domain of
* rpc_cancel_tasks(). An rpc_xprt may be shared by multiple rpc_clnt
* instances, so requests belonging to other clients must not be marked.
*/
void
xprt_mark_matching_reqs_disconnect_on_release(struct rpc_xprt *xprt,
const struct rpc_clnt *clnt,
bool (*fnmatch)(const struct rpc_task *,
const void *),
const void *data)
{
struct rpc_rqst *req, *pos;
struct rb_node *n;
spin_lock(&xprt->queue_lock);
list_for_each_entry(req, &xprt->xmit_queue, rq_xmit) {
if (xprt_request_matches(req, clnt, fnmatch, data))
xprt_request_mark_disconnect_on_release(xprt, req);
list_for_each_entry(pos, &req->rq_xmit2, rq_xmit2) {
if (xprt_request_matches(pos, clnt, fnmatch, data))
xprt_request_mark_disconnect_on_release(xprt,
pos);
}
}
for (n = rb_first(&xprt->recv_queue); n; n = rb_next(n)) {
req = rb_entry(n, struct rpc_rqst, rq_recv);
if (xprt_request_matches(req, clnt, fnmatch, data))
xprt_request_mark_disconnect_on_release(xprt, req);
}
spin_unlock(&xprt->queue_lock);
}
/**
* xprt_request_dequeue_xprt - remove a task from the transmit+receive queue
* @task: pointer to rpc_task
@@ -1915,6 +1994,7 @@ xprt_request_init(struct rpc_task *task)
req->rq_rcv_buf.bvec = NULL;
req->rq_release_snd_buf = NULL;
req->rq_seqno_count = 0;
req->rq_disconnect_on_release = false;
xprt_init_majortimeo(task, req, task->tk_client->cl_timeout);
trace_xprt_reserve(req);
@@ -1991,6 +2071,20 @@ void xprt_release(struct rpc_task *task)
xprt = req->rq_xprt;
xprt_request_dequeue_xprt(task);
spin_lock(&xprt->transport_lock);
if (unlikely(req->rq_disconnect_on_release)) {
req->rq_disconnect_on_release = false;
/*
* Dequeue may skip queue_lock. Pair with the fully ordered
* test_and_clear_bit(RPC_TASK_NEED_XMIT) in
* xprt_request_dequeue_transmit_locked() to order the runstate
* check before reading the sender's rq_ntrans and
* rq_connect_cookie updates.
*/
smp_rmb();
if (req->rq_ntrans > 0)
xprt_conditional_disconnect_locked(xprt,
req->rq_connect_cookie);
}
xprt->ops->release_xprt(xprt, task);
if (xprt->ops->release_request)
xprt->ops->release_request(task);
@@ -2020,6 +2114,7 @@ xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task,
task->tk_rqstp = req;
req->rq_task = task;
xprt_init_connect_cookie(req, req->rq_xprt);
req->rq_disconnect_on_release = false;
/*
* Set up the xdr_buf length.
* This also indicates that the buffer is XDR encoded already.
+55 -34
View File
@@ -27,13 +27,17 @@
/* Transport recovery
*
* frwr_map and frwr_unmap_* cannot run at the same time the transport
* connect worker is running. The connect worker holds the transport
* send lock, just as ->send_request does. This prevents frwr_map and
* the connect worker from running concurrently. When a connection is
* closed, the Receive completion queue is drained before the allowing
* the connect worker to get control. This prevents frwr_unmap and the
* connect worker from running concurrently.
* frwr_map() is serialized with the connect worker by the transport
* send lock.
*
* frwr_unmap_async() runs from Receive completion processing.
* rpcrdma_xprt_drain() drains the Receive Queue before the Send Queue,
* so asynchronous LOCAL_INV Work Requests are submitted before Send
* Queue drain.
*
* frwr_unmap_sync() runs after the transport send lock is released.
* rx_unmap_rwsem orders its LOCAL_INV submission before disconnect
* drains the QP.
*
* When the underlying transport disconnects, MRs that are in flight
* are flushed and are likely unusable. Thus all MRs are destroyed.
@@ -139,7 +143,6 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr)
mr->mr_ibmr = frmr;
mr->mr_device = NULL;
INIT_LIST_HEAD(&mr->mr_list);
init_completion(&mr->mr_linv_done);
frwr_cid_init(ep, mr);
sg_init_table(sg, depth);
@@ -539,40 +542,60 @@ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc)
* @cq: completion queue
* @wc: WCE for a completed LocalInv WR
*
* Awaken anyone waiting for an MR to finish being fenced.
* Wake frwr_unmap_sync() after the final LOCAL_INV completion.
*/
static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc)
{
struct ib_cqe *cqe = wc->wr_cqe;
struct rpcrdma_mr *mr = container_of(cqe, struct rpcrdma_mr, mr_cqe);
struct rpcrdma_req *req = mr->mr_req;
/* WARNING: Only wr_cqe and status are reliable at this point */
trace_xprtrdma_wc_li_wake(wc, &mr->mr_cid);
frwr_mr_done(wc, mr);
complete(&mr->mr_linv_done);
complete(&req->rl_linv_done);
rpcrdma_flush_disconnect(cq->cq_context, wc);
}
/**
* frwr_unmap_sync - invalidate memory regions that were registered for @req
* frwr_unmap_sync - synchronously invalidate MRs registered for @req
* @r_xprt: controlling transport instance
* @req: rpcrdma_req with a non-empty list of MRs to process
* @req: request whose registered MRs are to be invalidated
*
* Sleeps until it is safe for the host CPU to access the previously mapped
* memory regions. This guarantees that registered MRs are properly fenced
* from the server before the RPC consumer accesses the data in them. It
* also ensures proper Send flow control: waking the next RPC waits until
* this RPC has relinquished all its Send Queue entries.
* If @req still owns registered MRs after synchronizing with transport
* disconnect, post a chain of LOCAL_INV Work Requests and wait for the
* final completion. On successful completion, this fences the mapped
* regions from remote access and preserves Send Queue flow control.
*
* A concurrent disconnect can reset @req while this function waits for
* the read side. In that case the MRs have already been released and no
* LOCAL_INV Work Requests are posted.
*
* Context: Process context. Takes and releases
* @r_xprt->rx_unmap_rwsem for read. May sleep.
*/
void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
{
struct ib_send_wr *first, **prev, *last;
struct rpcrdma_ep *ep = r_xprt->rx_ep;
const struct ib_send_wr *bad_wr;
struct rpcrdma_ep *ep;
struct rpcrdma_mr *mr;
int rc;
/* serialize against rpcrdma_xprt_disconnect() */
down_read(&r_xprt->rx_unmap_rwsem);
/*
* Disconnect can reset this request while we wait for the read side.
* Recheck rl_registered in case rx_ep now refers to a new connection.
*/
ep = r_xprt->rx_ep;
if (!ep || list_empty(&req->rl_registered)) {
up_read(&r_xprt->rx_unmap_rwsem);
return;
}
/* ORDER: Invalidate all of the MRs first
*
* Chain the LOCAL_INV Work Requests and post them with
@@ -599,37 +622,35 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
prev = &last->next;
} while ((mr = rpcrdma_mr_pop(&req->rl_registered)));
mr = container_of(last, struct rpcrdma_mr, mr_invwr);
/* Strong send queue ordering guarantees that when the
* last WR in the chain completes, all WRs in the chain
* are complete.
*/
last->wr_cqe->done = frwr_wc_localinv_wake;
reinit_completion(&mr->mr_linv_done);
reinit_completion(&req->rl_linv_done);
/* Transport disconnect drains the receive CQ before it
* replaces the QP. The RPC reply handler won't call us
* unless re_id->qp is a valid pointer.
/*
* The read side prevents disconnect from draining this QP until the
* LOCAL_INV chain has been submitted.
*/
bad_wr = NULL;
rc = ib_post_send(ep->re_id->qp, first, &bad_wr);
if (rc) {
/* On error, the MRs get destroyed once the QP has drained. */
trace_xprtrdma_post_linv_err(req, rc);
/* Force a connection loss to ensure complete recovery.
*/
rpcrdma_force_disconnect(ep);
}
up_read(&r_xprt->rx_unmap_rwsem);
/* The final LOCAL_INV WR in the chain is supposed to
* do the wake. If it was never posted, the wake will
* not happen, so don't wait in that case.
*/
if (bad_wr != first)
wait_for_completion(&mr->mr_linv_done);
if (!rc)
return;
/* On error, the MRs get destroyed once the QP has drained. */
trace_xprtrdma_post_linv_err(req, rc);
/* Force a connection loss to ensure complete recovery.
*/
rpcrdma_force_disconnect(ep);
wait_for_completion(&req->rl_linv_done);
}
/**
+2
View File
@@ -364,6 +364,8 @@ xprt_setup_rdma(struct xprt_create *args)
INIT_DELAYED_WORK(&new_xprt->rx_connect_worker,
xprt_rdma_connect_worker);
init_rwsem(&new_xprt->rx_unmap_rwsem);
xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT;
return xprt;
+12 -4
View File
@@ -572,20 +572,24 @@ out:
* rpcrdma_xprt_disconnect - Disconnect underlying transport
* @r_xprt: controlling transport instance
*
* Caller serializes. Either the transport send lock is held,
* or we're being called to destroy the transport.
* Context: Caller serializes. Either the transport send lock is held,
* or the transport is being destroyed. Takes and releases
* @r_xprt->rx_unmap_rwsem for write. May sleep.
*
* On return, @r_xprt is completely divested of all hardware
* resources and prepared for the next ->connect operation.
*/
void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)
{
struct rpcrdma_ep *ep = r_xprt->rx_ep;
struct rpcrdma_ep *ep;
struct rdma_cm_id *id;
int rc;
down_write(&r_xprt->rx_unmap_rwsem);
ep = r_xprt->rx_ep;
if (!ep)
return;
goto out_unlock;
/* Prevent teardown-generated events from requesting another close. */
atomic_xchg(&ep->re_force_disconnect, 1);
@@ -604,6 +608,9 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)
rdma_destroy_id(id);
r_xprt->rx_ep = NULL;
out_unlock:
up_write(&r_xprt->rx_unmap_rwsem);
}
/* Fixed-size circular FIFO queue. This implementation is wait-free and
@@ -927,6 +934,7 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,
INIT_LIST_HEAD(&req->rl_free_mrs);
INIT_LIST_HEAD(&req->rl_registered);
init_completion(&req->rl_linv_done);
spin_lock(&buffer->rb_lock);
list_add(&req->rl_all, &buffer->rb_allreqs);
spin_unlock(&buffer->rb_lock);
+9 -1
View File
@@ -46,6 +46,7 @@
#include <linux/spinlock.h> /* spinlock_t, etc */
#include <linux/atomic.h> /* atomic_t, etc */
#include <linux/kref.h> /* struct kref */
#include <linux/rwsem.h> /* struct rw_semaphore */
#include <linux/workqueue.h> /* struct work_struct */
#include <linux/llist.h>
@@ -249,7 +250,6 @@ struct rpcrdma_mr {
int mr_nents;
enum dma_data_direction mr_dir;
struct ib_cqe mr_cqe;
struct completion mr_linv_done;
union {
struct ib_reg_wr mr_regwr;
struct ib_send_wr mr_invwr;
@@ -348,6 +348,7 @@ struct rpcrdma_req {
struct list_head rl_free_mrs;
struct list_head rl_registered;
struct completion rl_linv_done;
};
static inline struct rpcrdma_req *
@@ -449,6 +450,13 @@ struct rpcrdma_xprt {
struct delayed_work rx_connect_worker;
struct rpc_timeout rx_timeout;
struct rpcrdma_stats rx_stats;
/*
* Orders frwr_unmap_sync()'s LOCAL_INV submission ahead of the
* QP drain. Disconnect holds the write side across the drain
* and resource teardown.
*/
struct rw_semaphore rx_unmap_rwsem;
};
#define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt)
+1 -1
View File
@@ -3501,7 +3501,7 @@ static struct rpc_xprt *xs_setup_tcp_tls(struct xprt_create *args)
transport = container_of(xprt, struct sock_xprt, xprt);
xprt->prot = IPPROTO_TCP;
xprt->xprt_class = &xs_tcp_transport;
xprt->xprt_class = &xs_tcp_tls_transport;
xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
xprt->bind_timeout = XS_BIND_TO;