mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'nfs-for-7.4-2' of git://git.linux-nfs.org/projects/anna/linux-nfs
Pull NFS client fixes from Anna Schumaker: - Various NFS and SunRPC fixes from Avi Weiss - Various localio fixes from Mike Snitzer - Delegated timestamp fixes from Jeff Layton - CB_NOTIFY_LOCK fixes from Olga Kornievskaia - Various sunrpc, flexfiles, and xprtrdma fixes from Tim Menninger * tag 'nfs-for-7.4-2' of git://git.linux-nfs.org/projects/anna/linux-nfs: (26 commits) NFSv4: Prevent indefinite waits for out-of-order OPEN stateids NFSv4/flexfiles: limit transport disconnects when cancelling I/O SUNRPC: add request-scoped disconnects when cancelling tasks pNFS: avoid repeatedly cancelling I/O for the same lseg xprtrdma: serialize unmap_sync with xprt_disconnect NFSv4: remember that the server may send CB_NOTIFY_LOCK NFSv4: wake blocking lock waiters when a delegated lock is released NFS: stamp after the server's ctime following a CLONE NFS: stamp after the server's ctime when truncating under delegated times NFS: don't let delegated c/mtime stamps go backward NFS/localio: give the LOCALIO probe a short timeout of its own NFS/localio: probe again for LOCALIO when localio_enabled is set to Y NFS/localio: hold the client across an asynchronous LOCALIO probe SUNRPC: Balance queue accounting when relocating a task SUNRPC: Balance switch queue length on transport removal SUNRPC: Fix TCP-with-TLS transport class identification SUNRPC: preserve transport security policy when cloning clients SUNRPC: preserve timeout policy when cloning clients SUNRPC: preserve fatal network errors during RPC binding sunrpc: instantiate rpc_pipefs dentry before publishing it ...
This commit is contained in:
@@ -312,6 +312,18 @@ Module Parameters
|
||||
/sys/module/nfs/parameters/localio_enabled (bool)
|
||||
controls if LOCALIO is enabled, defaults to Y. If client and server are
|
||||
local but 'localio_enabled' is set to N then LOCALIO will not be used.
|
||||
It is checked on every I/O: setting it to N sends the next I/O of every
|
||||
client over the network, and setting it back to Y resumes LOCALIO. A
|
||||
client that probed while it was N is probed again when it is set to Y.
|
||||
|
||||
/sys/module/nfs/parameters/localio_timeout (uint)
|
||||
sets how many seconds the LOCALIO probe waits for each reply from the
|
||||
server, defaults to 10, at most 600. A server that does not support
|
||||
LOCALIO may not answer the probe at all. If no reply comes in time,
|
||||
the probe gives up without a log message. The client then does not use
|
||||
LOCALIO. The mount's timeouts do not apply to the probe. Set to 0, the
|
||||
probe uses the mount's timeouts instead, as an NFS RPC would. A change
|
||||
applies to the next probe.
|
||||
|
||||
/sys/module/nfs/parameters/localio_O_DIRECT_semantics (bool)
|
||||
controls if O_DIRECT extends down to the underlying filesystem, defaults
|
||||
|
||||
@@ -95,6 +95,8 @@ void nfs4_inode_make_writeable(struct inode *inode);
|
||||
void nfs_update_delegated_atime(struct inode *inode);
|
||||
void nfs_update_delegated_mtime(struct inode *inode);
|
||||
void nfs_update_delegated_mtime_locked(struct inode *inode);
|
||||
void nfs_delegated_ctime_raise(struct inode *inode,
|
||||
const struct nfs_fattr *fattr);
|
||||
|
||||
static inline int nfs_have_read_or_write_delegation(struct inode *inode)
|
||||
{
|
||||
|
||||
+3
-1
@@ -1253,6 +1253,8 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx)
|
||||
spin_lock(&file->f_lock);
|
||||
desc->dir_cookie = dir_ctx->dir_cookie;
|
||||
desc->folio_index = dir_ctx->page_index;
|
||||
desc->current_index = (loff_t)desc->folio_index *
|
||||
nfs_readdir_array_maxentries();
|
||||
desc->last_cookie = dir_ctx->last_cookie;
|
||||
desc->attr_gencount = dir_ctx->attr_gencount;
|
||||
desc->eof = dir_ctx->eof;
|
||||
@@ -2735,7 +2737,7 @@ static bool nfs_rename_is_unsafe_cross_dir(struct dentry *old_dentry,
|
||||
{
|
||||
struct nfs_server *server = NFS_SB(old_dentry->d_sb);
|
||||
|
||||
if (old_dentry->d_parent != new_dentry->d_parent)
|
||||
if (old_dentry->d_parent == new_dentry->d_parent)
|
||||
return false;
|
||||
if (server->fh_expire_type & NFS_FH_RENAME_UNSAFE)
|
||||
return !(server->fh_expire_type & NFS_FH_NOEXPIRE_WITH_OPEN);
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <linux/nfs_mount.h>
|
||||
#include <linux/nfs_page.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/bitmap.h>
|
||||
#include <linux/file.h>
|
||||
#include <linux/sched/mm.h>
|
||||
|
||||
@@ -340,6 +341,7 @@ static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls)
|
||||
{
|
||||
if (fls) {
|
||||
ff_layout_free_mirror_array(fls);
|
||||
bitmap_free(fls->cancelled_stripes);
|
||||
kfree(fls);
|
||||
}
|
||||
}
|
||||
@@ -653,6 +655,12 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
|
||||
fls->mirror_array[i]->report_interval = be32_to_cpup(p);
|
||||
|
||||
out_sort_mirrors:
|
||||
fls->cancelled_stripes = bitmap_zalloc(
|
||||
fls->mirror_array_cnt * dss_count, gfp_flags);
|
||||
if (!fls->cancelled_stripes) {
|
||||
rc = -ENOMEM;
|
||||
goto out_err_free;
|
||||
}
|
||||
ff_layout_sort_mirrors(fls);
|
||||
ret = &fls->generic_hdr;
|
||||
dprintk("<-- %s (success)\n", __func__);
|
||||
@@ -2582,6 +2590,10 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg,
|
||||
if (devid && memcmp(&mirror->dss[dss_id].devid, devid,
|
||||
sizeof(*devid)) != 0)
|
||||
continue;
|
||||
/* Do not cancel a stripe again while this lseg drains. */
|
||||
if (test_and_set_bit(idx * mirror->dss_count + dss_id,
|
||||
flseg->cancelled_stripes))
|
||||
continue;
|
||||
rcu_read_lock();
|
||||
mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds);
|
||||
if (IS_ERR_OR_NULL(mirror_ds) ||
|
||||
@@ -2599,10 +2611,9 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg,
|
||||
clnt = ds_clp->cl_rpcclient;
|
||||
if (!clnt)
|
||||
goto next;
|
||||
if (!rpc_cancel_tasks(clnt, -ECANCELED,
|
||||
ff_layout_match_io, lseg))
|
||||
goto next;
|
||||
rpc_clnt_disconnect(clnt);
|
||||
rpc_cancel_tasks_and_disconnect(clnt, -ECANCELED,
|
||||
ff_layout_match_io,
|
||||
lseg);
|
||||
next:
|
||||
nfs4_ff_layout_put_deviceid(mirror_ds);
|
||||
}
|
||||
|
||||
@@ -109,6 +109,7 @@ struct nfs4_ff_layout_segment {
|
||||
u64 stripe_unit;
|
||||
u32 flags;
|
||||
u32 mirror_array_cnt;
|
||||
unsigned long *cancelled_stripes;
|
||||
struct nfs4_ff_layout_mirror *mirror_array[] __counted_by(mirror_array_cnt);
|
||||
};
|
||||
|
||||
|
||||
+52
-17
@@ -629,28 +629,52 @@ nfs_fattr_fixup_delegated(struct inode *inode, struct nfs_fattr *fattr)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* With delegated timestamps, the client's coarse clock can trail the
|
||||
* server's fine-grained ctime cached from an OPEN or SETATTR reply. The
|
||||
* server won't move ctime backward when the times are returned, so a
|
||||
* new stamp must land after the cached ctime.
|
||||
*/
|
||||
static struct timespec64 nfs_delegated_stamp(struct inode *inode)
|
||||
{
|
||||
struct timespec64 ctime = inode_get_ctime(inode);
|
||||
struct timespec64 now = current_time(inode);
|
||||
|
||||
if (timespec64_compare(&now, &ctime) <= 0) {
|
||||
now = ctime;
|
||||
timespec64_add_ns(&now, 1);
|
||||
}
|
||||
return now;
|
||||
}
|
||||
|
||||
/*
|
||||
* The server stamped c/mtime itself (e.g. for a size change). Raise the
|
||||
* cached ctime to the server's so that the next delegated stamp lands
|
||||
* after it.
|
||||
*/
|
||||
void nfs_delegated_ctime_raise(struct inode *inode,
|
||||
const struct nfs_fattr *fattr)
|
||||
{
|
||||
struct timespec64 ctime = inode_get_ctime(inode);
|
||||
|
||||
if ((fattr->valid & NFS_ATTR_FATTR_CTIME) &&
|
||||
timespec64_compare(&fattr->ctime, &ctime) > 0)
|
||||
inode_set_ctime_to_ts(inode, fattr->ctime);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(nfs_delegated_ctime_raise);
|
||||
|
||||
static void nfs_set_timestamps_to_ts(struct inode *inode, struct iattr *attr)
|
||||
{
|
||||
unsigned int cache_flags = 0;
|
||||
|
||||
if (attr->ia_valid & ATTR_MTIME_SET) {
|
||||
struct timespec64 ctime = inode_get_ctime(inode);
|
||||
struct timespec64 mtime = inode_get_mtime(inode);
|
||||
struct timespec64 now;
|
||||
bool updated = false;
|
||||
|
||||
now = inode_set_ctime_current(inode);
|
||||
if (!timespec64_equal(&now, &ctime))
|
||||
updated = true;
|
||||
|
||||
/* The stamp is always after the old ctime */
|
||||
inode_set_ctime_to_ts(inode, nfs_delegated_stamp(inode));
|
||||
inode_set_mtime_to_ts(inode, attr->ia_mtime);
|
||||
if (!timespec64_equal(&now, &mtime))
|
||||
updated = true;
|
||||
|
||||
inode_maybe_inc_iversion(inode, updated);
|
||||
inode_maybe_inc_iversion(inode, true);
|
||||
cache_flags |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME;
|
||||
}
|
||||
if (attr->ia_valid & ATTR_ATIME_SET) {
|
||||
if (attr->ia_valid & ATTR_ATIME) {
|
||||
inode_set_atime_to_ts(inode, attr->ia_atime);
|
||||
cache_flags |= NFS_INO_INVALID_ATIME;
|
||||
}
|
||||
@@ -665,7 +689,11 @@ static void nfs_update_atime(struct inode *inode)
|
||||
|
||||
static void nfs_update_mtime(struct inode *inode)
|
||||
{
|
||||
inode_update_time(inode, FS_UPD_CMTIME, 0);
|
||||
struct timespec64 now = nfs_delegated_stamp(inode);
|
||||
|
||||
/* Not inode_update_time(): lockless readers would see its stamp */
|
||||
inode_set_ctime_to_ts(inode, now);
|
||||
inode_set_mtime_to_ts(inode, now);
|
||||
NFS_I(inode)->cache_validity &=
|
||||
~(NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME);
|
||||
}
|
||||
@@ -684,6 +712,7 @@ void nfs_update_delegated_mtime_locked(struct inode *inode)
|
||||
nfs_have_directory_delegation(inode))
|
||||
nfs_update_mtime(inode);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(nfs_update_delegated_mtime_locked);
|
||||
|
||||
void nfs_update_delegated_mtime(struct inode *inode)
|
||||
{
|
||||
@@ -746,7 +775,7 @@ nfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry,
|
||||
} else if (nfs_have_delegated_atime(inode) &&
|
||||
attr->ia_valid & ATTR_ATIME &&
|
||||
!(attr->ia_valid & ATTR_MTIME)) {
|
||||
if (!(attr->ia_valid & ATTR_ATIME_SET)) {
|
||||
if (!(attr->ia_valid & (ATTR_ATIME_SET | ATTR_TIMES_SET))) {
|
||||
nfs_update_delegated_atime(inode);
|
||||
attr->ia_valid &= ~ATTR_ATIME;
|
||||
}
|
||||
@@ -808,10 +837,14 @@ static int nfs_vmtruncate(struct inode * inode, loff_t offset)
|
||||
nfs_ooo_clear(NFS_I(inode));
|
||||
}
|
||||
NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
|
||||
/*
|
||||
* Stamp in the same i_lock section that checked the delegation and
|
||||
* raised ctime, before a delegation return can start.
|
||||
*/
|
||||
nfs_update_delegated_mtime_locked(inode);
|
||||
|
||||
spin_unlock(&inode->i_lock);
|
||||
truncate_pagecache(inode, offset);
|
||||
nfs_update_delegated_mtime_locked(inode);
|
||||
spin_lock(&inode->i_lock);
|
||||
out:
|
||||
return err;
|
||||
@@ -837,6 +870,8 @@ void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr,
|
||||
if ((attr->ia_valid & ATTR_SIZE) != 0) {
|
||||
if (!nfs_have_delegated_mtime(inode))
|
||||
nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME);
|
||||
else
|
||||
nfs_delegated_ctime_raise(inode, fattr);
|
||||
nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS);
|
||||
nfs_inc_stats(inode, NFSIOS_SETATTRTRUNC);
|
||||
nfs_vmtruncate(inode, attr->ia_size);
|
||||
|
||||
+114
-7
@@ -25,6 +25,7 @@
|
||||
#include <linux/nfs_xdr.h>
|
||||
|
||||
#include "internal.h"
|
||||
#include "netns.h"
|
||||
#include "pnfs.h"
|
||||
#include "nfstrace.h"
|
||||
|
||||
@@ -55,7 +56,50 @@ struct nfs_local_fsync_ctx {
|
||||
};
|
||||
|
||||
static bool localio_enabled __read_mostly = true;
|
||||
module_param(localio_enabled, bool, 0644);
|
||||
|
||||
static void nfs_local_probe_all(void);
|
||||
|
||||
/*
|
||||
* localio_enabled is checked on every I/O, so turning it off stops LOCALIO
|
||||
* at once and turning it back on resumes it for clients that use it. A
|
||||
* client that probed while it was off was disabled, and nothing probes it
|
||||
* again: turning it on probes every client that does not use LOCALIO.
|
||||
*/
|
||||
static int localio_enabled_set(const char *val, const struct kernel_param *kp)
|
||||
{
|
||||
bool was = READ_ONCE(localio_enabled);
|
||||
int ret;
|
||||
|
||||
ret = param_set_bool(val, kp);
|
||||
if (!ret && !was && READ_ONCE(localio_enabled))
|
||||
nfs_local_probe_all();
|
||||
return ret;
|
||||
}
|
||||
|
||||
static const struct kernel_param_ops localio_enabled_ops = {
|
||||
.set = localio_enabled_set,
|
||||
.get = param_get_bool,
|
||||
};
|
||||
module_param_cb(localio_enabled, &localio_enabled_ops, &localio_enabled, 0644);
|
||||
|
||||
/*
|
||||
* Seconds the LOCALIO probe waits for each reply. With 0, the probe uses
|
||||
* the NFS client's timeouts instead.
|
||||
*/
|
||||
static unsigned int localio_timeout __read_mostly = 10;
|
||||
|
||||
static int localio_timeout_set(const char *val, const struct kernel_param *kp)
|
||||
{
|
||||
return param_set_uint_minmax(val, kp, 0, 600);
|
||||
}
|
||||
|
||||
static const struct kernel_param_ops localio_timeout_ops = {
|
||||
.set = localio_timeout_set,
|
||||
.get = param_get_uint,
|
||||
};
|
||||
module_param_cb(localio_timeout, &localio_timeout_ops, &localio_timeout, 0644);
|
||||
MODULE_PARM_DESC(localio_timeout,
|
||||
"LOCALIO probe timeout in seconds (default 10), 0 to use the mount's default");
|
||||
|
||||
static void nfs_local_do_read(struct nfs_local_kiocb *iocb,
|
||||
const struct rpc_call_ops *call_ops);
|
||||
@@ -131,13 +175,45 @@ const struct rpc_program nfslocalio_program = {
|
||||
|
||||
/*
|
||||
* nfs_init_localioclient - Initialise an NFS localio client connection
|
||||
*
|
||||
* A server that does not support LOCALIO may not answer its RPCs at all.
|
||||
* The probe shares the NFS client's transport and runs beside its RPCs.
|
||||
* With localio_timeout set, it gets a quiet, soft timeout of that many
|
||||
* seconds of its own, whatever timeouts the mount uses. With 0, it uses
|
||||
* the NFS client's timeouts, as an NFS RPC would.
|
||||
*/
|
||||
static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp)
|
||||
{
|
||||
struct rpc_clnt *rpcclient_localio;
|
||||
unsigned int timeout_secs = READ_ONCE(localio_timeout);
|
||||
struct rpc_clnt *parent = clp->cl_rpcclient;
|
||||
struct rpc_clnt *probe = NULL, *rpcclient_localio;
|
||||
|
||||
rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient,
|
||||
&nfslocalio_program, 1);
|
||||
/*
|
||||
* rpc_bind_new_program() pings the new program with the timeout of
|
||||
* the client it clones, so set the probe's timeout on a private
|
||||
* clone first. The clone's cl_timeout points at the transport's
|
||||
* timeout: give it its own copy rather than write through it.
|
||||
*/
|
||||
if (timeout_secs) {
|
||||
probe = rpc_clone_client(parent);
|
||||
if (IS_ERR(probe))
|
||||
return probe;
|
||||
probe->cl_timeout_default = (struct rpc_timeout) {
|
||||
.to_initval = timeout_secs * HZ,
|
||||
.to_maxval = timeout_secs * HZ,
|
||||
.to_retries = 0,
|
||||
};
|
||||
probe->cl_timeout = &probe->cl_timeout_default;
|
||||
probe->cl_softrtry = 1;
|
||||
probe->cl_noretranstimeo = 0;
|
||||
probe->cl_chatty = 0;
|
||||
parent = probe;
|
||||
}
|
||||
|
||||
rpcclient_localio = rpc_bind_new_program(parent, &nfslocalio_program, 1);
|
||||
/* The new client holds a reference to its parent, the probe clone */
|
||||
if (probe)
|
||||
rpc_shutdown_client(probe);
|
||||
|
||||
dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n",
|
||||
__func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR),
|
||||
@@ -203,18 +279,49 @@ void nfs_local_probe_async_work(struct work_struct *work)
|
||||
struct nfs_client *clp =
|
||||
container_of(work, struct nfs_client, cl_local_probe_work);
|
||||
|
||||
if (!refcount_inc_not_zero(&clp->cl_count))
|
||||
return;
|
||||
nfs_local_probe(clp);
|
||||
/* The reference nfs_local_probe_async() took for this work */
|
||||
nfs_put_client(clp);
|
||||
}
|
||||
|
||||
/*
|
||||
* The work is part of the client, so it holds a reference on the client
|
||||
* from when it is queued until it has run: the last nfs_put_client()
|
||||
* cannot free the client under a queued probe.
|
||||
*/
|
||||
void nfs_local_probe_async(struct nfs_client *clp)
|
||||
{
|
||||
queue_work(nfsiod_workqueue, &clp->cl_local_probe_work);
|
||||
if (!refcount_inc_not_zero(&clp->cl_count))
|
||||
return;
|
||||
if (!queue_work(nfsiod_workqueue, &clp->cl_local_probe_work))
|
||||
nfs_put_client(clp); /* already queued, with its reference */
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(nfs_local_probe_async);
|
||||
|
||||
/* Probe every client, in every namespace, that does not use LOCALIO. */
|
||||
static void nfs_local_probe_all(void)
|
||||
{
|
||||
struct nfs_client *clp;
|
||||
struct net *net;
|
||||
|
||||
/* Set on the module command line: there are no clients yet. */
|
||||
if (!nfs_net_id)
|
||||
return;
|
||||
|
||||
down_read(&net_rwsem);
|
||||
for_each_net(net) {
|
||||
struct nfs_net *nn = net_generic(net, nfs_net_id);
|
||||
|
||||
spin_lock(&nn->nfs_client_lock);
|
||||
list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link)
|
||||
if (clp->cl_cons_state == NFS_CS_READY &&
|
||||
!nfs_client_is_local(clp))
|
||||
nfs_local_probe_async(clp);
|
||||
spin_unlock(&nn->nfs_client_lock);
|
||||
}
|
||||
up_read(&net_rwsem);
|
||||
}
|
||||
|
||||
static inline void nfs_local_file_put(struct nfsd_file *localio)
|
||||
{
|
||||
/* nfs_to_nfsd_file_put_local() expects an __rcu pointer
|
||||
|
||||
+20
-5
@@ -380,6 +380,7 @@ out:
|
||||
* @pos: destination offset
|
||||
* @len: copy length
|
||||
* @oldsize: length of the file prior to clone/copy
|
||||
* @fattr: destination attributes from the reply, or NULL
|
||||
*
|
||||
* Punch a hole in the inode page cache, so that the NFS client will
|
||||
* know to retrieve new data.
|
||||
@@ -387,7 +388,7 @@ out:
|
||||
* invalid cached values for change attribute, ctime, mtime and space used.
|
||||
*/
|
||||
static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len,
|
||||
loff_t oldsize)
|
||||
loff_t oldsize, const struct nfs_fattr *fattr)
|
||||
{
|
||||
struct inode *inode = file_inode(file);
|
||||
struct address_space *mapping = file->f_mapping;
|
||||
@@ -405,8 +406,12 @@ static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len,
|
||||
NFS_INO_INVALID_CTIME |
|
||||
NFS_INO_INVALID_MTIME |
|
||||
NFS_INO_INVALID_BLOCKS);
|
||||
/* The server may have stamped c/mtime; stamp after it */
|
||||
if (fattr && nfs_have_delegated_mtime(inode))
|
||||
nfs_delegated_ctime_raise(inode, fattr);
|
||||
/* Before a refresh can see the times invalid and take the server's */
|
||||
nfs_update_delegated_mtime_locked(inode);
|
||||
spin_unlock(&inode->i_lock);
|
||||
nfs_update_delegated_mtime(inode);
|
||||
}
|
||||
|
||||
static ssize_t _nfs42_proc_copy(struct file *src,
|
||||
@@ -506,7 +511,8 @@ static ssize_t _nfs42_proc_copy(struct file *src,
|
||||
goto out;
|
||||
}
|
||||
|
||||
nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst);
|
||||
nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst,
|
||||
NULL);
|
||||
nfs_invalidate_atime(src_inode);
|
||||
status = res->write_res.count;
|
||||
out:
|
||||
@@ -1311,8 +1317,17 @@ static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f,
|
||||
/* a zero-length count means clone to EOF in src */
|
||||
if (count == 0 && res.dst_fattr->valid & NFS_ATTR_FATTR_SIZE)
|
||||
count = nfs_size_to_loff_t(res.dst_fattr->size) - dst_offset;
|
||||
nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst);
|
||||
status = nfs_post_op_update_inode(dst_inode, res.dst_fattr);
|
||||
nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst,
|
||||
res.dst_fattr);
|
||||
/*
|
||||
* nfs_post_op_update_inode() would mark ctime invalid, so the
|
||||
* reply's ctime would overwrite the delegated stamp.
|
||||
*/
|
||||
if (nfs_have_delegated_mtime(dst_inode))
|
||||
status = nfs_refresh_inode(dst_inode, res.dst_fattr);
|
||||
else
|
||||
status = nfs_post_op_update_inode(dst_inode,
|
||||
res.dst_fattr);
|
||||
}
|
||||
|
||||
kfree(res.dst_fattr);
|
||||
|
||||
+22
-4
@@ -1722,6 +1722,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state,
|
||||
{
|
||||
DEFINE_WAIT(wait);
|
||||
int status = 0;
|
||||
unsigned long deadline = jiffies + 5 * HZ;
|
||||
signed long remaining;
|
||||
for (;;) {
|
||||
|
||||
if (nfs_stateid_is_sequential(state, stateid))
|
||||
@@ -1756,7 +1758,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state,
|
||||
|
||||
if (!fatal_signal_pending(current) &&
|
||||
!nfs_current_task_exiting()) {
|
||||
if (schedule_timeout(5*HZ) == 0)
|
||||
remaining = (long)(deadline - jiffies);
|
||||
if (remaining <= 0 || schedule_timeout(remaining) == 0)
|
||||
status = -EAGAIN;
|
||||
else
|
||||
status = 0;
|
||||
@@ -3093,6 +3096,8 @@ static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata,
|
||||
if (server->caps & NFS_CAP_POSIX_LOCK)
|
||||
set_bit(NFS_STATE_POSIX_LOCKS, &state->flags);
|
||||
if (opendata->o_res.rflags & NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK)
|
||||
server->nfs_client->cl_may_notify_lock = true;
|
||||
if (server->nfs_client->cl_may_notify_lock)
|
||||
set_bit(NFS_STATE_MAY_NOTIFY_LOCK, &state->flags);
|
||||
if (opendata->o_res.rflags & NFS4_OPEN_RESULT_PRESERVE_UNLINKED)
|
||||
set_bit(NFS_INO_PRESERVE_UNLINKED, &NFS_I(state->inode)->flags);
|
||||
@@ -7287,8 +7292,15 @@ static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock *
|
||||
if (status != 0)
|
||||
goto out;
|
||||
/* Is this a delegated lock? */
|
||||
if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0)
|
||||
if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0) {
|
||||
struct nfs_client *clp = NFS_SERVER(inode)->nfs_client;
|
||||
struct cb_notify_lock_args cbnl = { .cbnl_valid = false };
|
||||
|
||||
/* The server never saw it, so it will not notify waiters */
|
||||
nfs_copy_fh(&cbnl.cbnl_fh, NFS_FH(inode));
|
||||
__wake_up(&clp->cl_lock_waitq, TASK_NORMAL, 0, &cbnl);
|
||||
goto out;
|
||||
}
|
||||
alloc_seqid = NFS_SERVER(inode)->nfs_client->cl_mvops->alloc_seqid;
|
||||
seqid = alloc_seqid(&lsp->ls_seqid, GFP_KERNEL);
|
||||
status = -ENOMEM;
|
||||
@@ -7708,8 +7720,14 @@ nfs4_wake_lock_waiter(wait_queue_entry_t *wait, unsigned int mode, int flags, vo
|
||||
struct nfs_lowner *lowner = &cbnl->cbnl_owner,
|
||||
*wowner = &waiter->owner;
|
||||
|
||||
/* Only wake if the callback was for the same owner. */
|
||||
if (lowner->id != wowner->id || lowner->s_dev != wowner->s_dev)
|
||||
/*
|
||||
* Only wake the owner a CB_NOTIFY_LOCK names. A lock cached
|
||||
* under a delegation is released locally, and the releasing
|
||||
* thread cannot tell which waiter wants it, so those wakeups
|
||||
* (cbnl_valid false) name only the inode and wake them all.
|
||||
*/
|
||||
if (cbnl->cbnl_valid &&
|
||||
(lowner->id != wowner->id || lowner->s_dev != wowner->s_dev))
|
||||
return 0;
|
||||
|
||||
/* Make sure it's for the right inode */
|
||||
|
||||
+1
-1
@@ -2086,7 +2086,7 @@ static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
|
||||
break;
|
||||
}
|
||||
if (size_set && io_set) {
|
||||
if (size && io)
|
||||
if (size || io)
|
||||
ret = true;
|
||||
} else if (size || io)
|
||||
ret = true;
|
||||
|
||||
@@ -120,6 +120,7 @@ struct nfs_client {
|
||||
#define NFS_SP4_MACH_CRED_COMMIT 6 /* COMMIT */
|
||||
#define NFS_SP4_MACH_CRED_PNFS_CLEANUP 7 /* LAYOUTRETURN */
|
||||
wait_queue_head_t cl_lock_waitq;
|
||||
bool cl_may_notify_lock;
|
||||
#endif /* CONFIG_NFS_V4 */
|
||||
|
||||
/* Our own IP address, as a null-terminated string.
|
||||
|
||||
@@ -230,6 +230,10 @@ unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
|
||||
bool (*fnmatch)(const struct rpc_task *,
|
||||
const void *),
|
||||
const void *data);
|
||||
unsigned long rpc_cancel_tasks_and_disconnect(struct rpc_clnt *clnt, int error,
|
||||
bool (*fnmatch)(const struct rpc_task *,
|
||||
const void *),
|
||||
const void *data);
|
||||
void rpc_execute(struct rpc_task *);
|
||||
void rpc_init_priority_wait_queue(struct rpc_wait_queue *, const char *);
|
||||
void rpc_init_wait_queue(struct rpc_wait_queue *, const char *);
|
||||
|
||||
@@ -112,6 +112,11 @@ struct rpc_rqst {
|
||||
|
||||
ktime_t rq_xtime; /* transmit time stamp */
|
||||
int rq_ntrans;
|
||||
bool rq_disconnect_on_release;
|
||||
/*
|
||||
* protected by rq_xprt->transport_lock
|
||||
* while the request is active
|
||||
*/
|
||||
|
||||
#if defined(CONFIG_SUNRPC_BACKCHANNEL)
|
||||
struct lwq_node rq_bc_list; /* Callback service list */
|
||||
|
||||
+87
-21
@@ -653,6 +653,9 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args,
|
||||
}
|
||||
args->servername = xprt->servername;
|
||||
args->nodename = clnt->cl_nodename;
|
||||
args->xprtsec = clnt->cl_xprtsec;
|
||||
if (!args->timeout)
|
||||
args->timeout = clnt->cl_timeout;
|
||||
|
||||
new = rpc_new_client(args, xps, xprt, clnt);
|
||||
if (IS_ERR(new))
|
||||
@@ -901,6 +904,59 @@ void rpc_killall_tasks(struct rpc_clnt *clnt)
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rpc_killall_tasks);
|
||||
|
||||
struct rpc_cancel_disconnect_ctx {
|
||||
bool (*fnmatch)(const struct rpc_task *task, const void *data);
|
||||
const void *data;
|
||||
};
|
||||
|
||||
static int rpc_mark_xprt_disconnect_on_release(struct rpc_clnt *clnt,
|
||||
struct rpc_xprt *xprt,
|
||||
void *arg)
|
||||
{
|
||||
struct rpc_cancel_disconnect_ctx *ctx = arg;
|
||||
|
||||
xprt_mark_matching_reqs_disconnect_on_release(xprt, clnt,
|
||||
ctx->fnmatch,
|
||||
ctx->data);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static unsigned long
|
||||
__rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
|
||||
bool (*fnmatch)(const struct rpc_task *,
|
||||
const void *),
|
||||
const void *data,
|
||||
bool disconnect)
|
||||
{
|
||||
struct rpc_task *task;
|
||||
unsigned long count = 0;
|
||||
|
||||
if (list_empty(&clnt->cl_tasks))
|
||||
return 0;
|
||||
|
||||
spin_lock(&clnt->cl_lock);
|
||||
list_for_each_entry(task, &clnt->cl_tasks, tk_task) {
|
||||
if (!RPC_IS_ACTIVATED(task))
|
||||
continue;
|
||||
if (!fnmatch(task, data))
|
||||
continue;
|
||||
rpc_task_try_cancel(task, error);
|
||||
count++;
|
||||
}
|
||||
spin_unlock(&clnt->cl_lock);
|
||||
|
||||
if (disconnect && count) {
|
||||
struct rpc_cancel_disconnect_ctx ctx = {
|
||||
.fnmatch = fnmatch,
|
||||
.data = data,
|
||||
};
|
||||
rpc_clnt_iterate_for_each_xprt(clnt,
|
||||
rpc_mark_xprt_disconnect_on_release,
|
||||
&ctx);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* rpc_cancel_tasks - try to cancel a set of RPC tasks
|
||||
* @clnt: Pointer to RPC client
|
||||
@@ -916,28 +972,35 @@ unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error,
|
||||
const void *),
|
||||
const void *data)
|
||||
{
|
||||
struct rpc_task *task;
|
||||
unsigned long count = 0;
|
||||
|
||||
if (list_empty(&clnt->cl_tasks))
|
||||
return 0;
|
||||
/*
|
||||
* Spin lock all_tasks to prevent changes...
|
||||
*/
|
||||
spin_lock(&clnt->cl_lock);
|
||||
list_for_each_entry(task, &clnt->cl_tasks, tk_task) {
|
||||
if (!RPC_IS_ACTIVATED(task))
|
||||
continue;
|
||||
if (!fnmatch(task, data))
|
||||
continue;
|
||||
rpc_task_try_cancel(task, error);
|
||||
count++;
|
||||
}
|
||||
spin_unlock(&clnt->cl_lock);
|
||||
return count;
|
||||
return __rpc_cancel_tasks(clnt, error, fnmatch, data, false);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rpc_cancel_tasks);
|
||||
|
||||
/**
|
||||
* rpc_cancel_tasks_and_disconnect - cancel matching RPC tasks
|
||||
* @clnt: Pointer to RPC client
|
||||
* @error: RPC task error value to set
|
||||
* @fnmatch: Pointer to selector function
|
||||
* @data: User data
|
||||
*
|
||||
* Like rpc_cancel_tasks(), but after cancelling matching tasks, marks
|
||||
* matching in-flight requests so that xprt_release() conditionally
|
||||
* disconnects the connection generation on which each request was sent.
|
||||
*
|
||||
* @fnmatch is called while holding spinlocks and must not sleep.
|
||||
*
|
||||
* Returns the number of active tasks matched by @fnmatch.
|
||||
*/
|
||||
unsigned long
|
||||
rpc_cancel_tasks_and_disconnect(struct rpc_clnt *clnt, int error,
|
||||
bool (*fnmatch)(const struct rpc_task *,
|
||||
const void *),
|
||||
const void *data)
|
||||
{
|
||||
return __rpc_cancel_tasks(clnt, error, fnmatch, data, true);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(rpc_cancel_tasks_and_disconnect);
|
||||
|
||||
static int rpc_clnt_disconnect_xprt(struct rpc_clnt *clnt,
|
||||
struct rpc_xprt *xprt, void *dummy)
|
||||
{
|
||||
@@ -1194,7 +1257,7 @@ void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt)
|
||||
(task->tk_flags & RPC_TASK_MOVEABLE)))
|
||||
return;
|
||||
xprt_release(task);
|
||||
xprt_put(task->tk_xprt);
|
||||
rpc_task_release_xprt(clnt, task->tk_xprt);
|
||||
}
|
||||
if (task->tk_flags & RPC_TASK_NO_ROUND_ROBIN)
|
||||
task->tk_xprt = rpc_task_get_first_xprt(clnt);
|
||||
@@ -2125,8 +2188,10 @@ call_bind_status(struct rpc_task *task)
|
||||
goto retry_timeout;
|
||||
case -ENETDOWN:
|
||||
case -ENETUNREACH:
|
||||
if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL)
|
||||
if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL) {
|
||||
status = task->tk_status;
|
||||
break;
|
||||
}
|
||||
fallthrough;
|
||||
case -ECONNREFUSED: /* connection problems */
|
||||
case -ECONNRESET:
|
||||
@@ -2257,6 +2322,7 @@ call_connect_status(struct rpc_task *task)
|
||||
|
||||
xprt_release(task);
|
||||
value = atomic_long_dec_return(&xprt->queuelen);
|
||||
atomic_long_dec(&xps->xps_queuelen);
|
||||
if (value == 0)
|
||||
rpc_xprt_switch_remove_xprt(xps, saved,
|
||||
true);
|
||||
|
||||
@@ -191,6 +191,7 @@ rpc_close_pipes(struct dentry *dentry)
|
||||
cancel_delayed_work_sync(&pipe->queue_timeout);
|
||||
rpc_inode_setowner(inode, NULL);
|
||||
RPC_I(inode)->pipe = NULL;
|
||||
wake_up(&RPC_I(inode)->waitq);
|
||||
inode_unlock(inode);
|
||||
}
|
||||
|
||||
@@ -665,8 +666,8 @@ int rpc_mkpipe_dentry(struct dentry *parent, const char *name,
|
||||
rpci->private = private;
|
||||
rpci->pipe = pipe;
|
||||
rpc_inode_setowner(inode, private);
|
||||
pipe->dentry = dentry; // borrowed
|
||||
d_make_persistent(dentry, inode);
|
||||
pipe->dentry = dentry; // borrowed
|
||||
fsnotify_create(dir, dentry);
|
||||
simple_done_creating(dentry);
|
||||
return 0;
|
||||
|
||||
@@ -38,9 +38,17 @@ static inline int sock_is_loopback(struct sock *sk)
|
||||
|
||||
struct svc_serv;
|
||||
struct svc_rqst;
|
||||
struct rpc_clnt;
|
||||
struct rpc_task;
|
||||
struct rpc_xprt;
|
||||
int rpc_clients_notifier_register(void);
|
||||
void rpc_clients_notifier_unregister(void);
|
||||
void auth_domain_cleanup(void);
|
||||
void svc_sock_update_bufs(struct svc_serv *serv);
|
||||
enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp);
|
||||
void xprt_mark_matching_reqs_disconnect_on_release(struct rpc_xprt *xprt,
|
||||
const struct rpc_clnt *clnt,
|
||||
bool (*fnmatch)(const struct rpc_task *,
|
||||
const void *),
|
||||
const void *data);
|
||||
#endif /* _NET_SUNRPC_SUNRPC_H */
|
||||
|
||||
+101
-6
@@ -812,6 +812,17 @@ xprt_request_retransmit_after_disconnect(struct rpc_task *task)
|
||||
!xprt_connected(xprt);
|
||||
}
|
||||
|
||||
static void xprt_conditional_disconnect_locked(struct rpc_xprt *xprt,
|
||||
unsigned int cookie)
|
||||
__must_hold(&xprt->transport_lock)
|
||||
{
|
||||
if (cookie != xprt->connect_cookie)
|
||||
return;
|
||||
if (test_bit(XPRT_CLOSING, &xprt->state))
|
||||
return;
|
||||
xprt_schedule_autoclose_locked(xprt);
|
||||
}
|
||||
|
||||
/**
|
||||
* xprt_conditional_disconnect - force a transport to disconnect
|
||||
* @xprt: transport to disconnect
|
||||
@@ -827,12 +838,7 @@ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
|
||||
{
|
||||
/* Don't race with the test_bit() in xprt_clear_locked() */
|
||||
spin_lock(&xprt->transport_lock);
|
||||
if (cookie != xprt->connect_cookie)
|
||||
goto out;
|
||||
if (test_bit(XPRT_CLOSING, &xprt->state))
|
||||
goto out;
|
||||
xprt_schedule_autoclose_locked(xprt);
|
||||
out:
|
||||
xprt_conditional_disconnect_locked(xprt, cookie);
|
||||
spin_unlock(&xprt->transport_lock);
|
||||
}
|
||||
|
||||
@@ -1434,6 +1440,79 @@ xprt_request_dequeue_transmit(struct rpc_task *task)
|
||||
spin_unlock(&xprt->queue_lock);
|
||||
}
|
||||
|
||||
static bool
|
||||
xprt_request_matches(const struct rpc_rqst *req,
|
||||
const struct rpc_clnt *clnt,
|
||||
bool (*fnmatch)(const struct rpc_task *, const void *),
|
||||
const void *data)
|
||||
{
|
||||
const struct rpc_task *task = req->rq_task;
|
||||
|
||||
return task && task->tk_client == clnt && fnmatch(task, data);
|
||||
}
|
||||
|
||||
static void
|
||||
xprt_request_mark_disconnect_on_release(struct rpc_xprt *xprt,
|
||||
struct rpc_rqst *req)
|
||||
__must_hold(&xprt->queue_lock)
|
||||
{
|
||||
spin_lock(&xprt->transport_lock);
|
||||
req->rq_disconnect_on_release = true;
|
||||
spin_unlock(&xprt->transport_lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* xprt_mark_matching_reqs_disconnect_on_release - mark matching in-flight
|
||||
* requests for connection-generation
|
||||
* teardown at release
|
||||
* @xprt: transport whose queues to walk
|
||||
* @clnt: RPC client whose requests are eligible
|
||||
* @fnmatch: selector called with the request's owning rpc_task
|
||||
* @data: caller data forwarded to @fnmatch
|
||||
*
|
||||
* @fnmatch must not sleep and must be safe to call while xprt->queue_lock
|
||||
* is held.
|
||||
*
|
||||
* Walks the transmit queue (including per-owner rq_xmit2 chains) and the
|
||||
* receive tree under xprt->queue_lock and marks each matching rpc_rqst so
|
||||
* that xprt_release() will conditionally disconnect the transport against
|
||||
* that request's rq_connect_cookie. Requests remain eligible for marking
|
||||
* while they are present on a transport queue. The marker is set under
|
||||
* xprt->transport_lock, nested inside xprt->queue_lock, and xprt_release()
|
||||
* consumes it with xprt->transport_lock held after removing queue membership.
|
||||
*
|
||||
* The @clnt qualification preserves the selection domain of
|
||||
* rpc_cancel_tasks(). An rpc_xprt may be shared by multiple rpc_clnt
|
||||
* instances, so requests belonging to other clients must not be marked.
|
||||
*/
|
||||
void
|
||||
xprt_mark_matching_reqs_disconnect_on_release(struct rpc_xprt *xprt,
|
||||
const struct rpc_clnt *clnt,
|
||||
bool (*fnmatch)(const struct rpc_task *,
|
||||
const void *),
|
||||
const void *data)
|
||||
{
|
||||
struct rpc_rqst *req, *pos;
|
||||
struct rb_node *n;
|
||||
|
||||
spin_lock(&xprt->queue_lock);
|
||||
list_for_each_entry(req, &xprt->xmit_queue, rq_xmit) {
|
||||
if (xprt_request_matches(req, clnt, fnmatch, data))
|
||||
xprt_request_mark_disconnect_on_release(xprt, req);
|
||||
list_for_each_entry(pos, &req->rq_xmit2, rq_xmit2) {
|
||||
if (xprt_request_matches(pos, clnt, fnmatch, data))
|
||||
xprt_request_mark_disconnect_on_release(xprt,
|
||||
pos);
|
||||
}
|
||||
}
|
||||
for (n = rb_first(&xprt->recv_queue); n; n = rb_next(n)) {
|
||||
req = rb_entry(n, struct rpc_rqst, rq_recv);
|
||||
if (xprt_request_matches(req, clnt, fnmatch, data))
|
||||
xprt_request_mark_disconnect_on_release(xprt, req);
|
||||
}
|
||||
spin_unlock(&xprt->queue_lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* xprt_request_dequeue_xprt - remove a task from the transmit+receive queue
|
||||
* @task: pointer to rpc_task
|
||||
@@ -1915,6 +1994,7 @@ xprt_request_init(struct rpc_task *task)
|
||||
req->rq_rcv_buf.bvec = NULL;
|
||||
req->rq_release_snd_buf = NULL;
|
||||
req->rq_seqno_count = 0;
|
||||
req->rq_disconnect_on_release = false;
|
||||
xprt_init_majortimeo(task, req, task->tk_client->cl_timeout);
|
||||
|
||||
trace_xprt_reserve(req);
|
||||
@@ -1991,6 +2071,20 @@ void xprt_release(struct rpc_task *task)
|
||||
xprt = req->rq_xprt;
|
||||
xprt_request_dequeue_xprt(task);
|
||||
spin_lock(&xprt->transport_lock);
|
||||
if (unlikely(req->rq_disconnect_on_release)) {
|
||||
req->rq_disconnect_on_release = false;
|
||||
/*
|
||||
* Dequeue may skip queue_lock. Pair with the fully ordered
|
||||
* test_and_clear_bit(RPC_TASK_NEED_XMIT) in
|
||||
* xprt_request_dequeue_transmit_locked() to order the runstate
|
||||
* check before reading the sender's rq_ntrans and
|
||||
* rq_connect_cookie updates.
|
||||
*/
|
||||
smp_rmb();
|
||||
if (req->rq_ntrans > 0)
|
||||
xprt_conditional_disconnect_locked(xprt,
|
||||
req->rq_connect_cookie);
|
||||
}
|
||||
xprt->ops->release_xprt(xprt, task);
|
||||
if (xprt->ops->release_request)
|
||||
xprt->ops->release_request(task);
|
||||
@@ -2020,6 +2114,7 @@ xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task,
|
||||
task->tk_rqstp = req;
|
||||
req->rq_task = task;
|
||||
xprt_init_connect_cookie(req, req->rq_xprt);
|
||||
req->rq_disconnect_on_release = false;
|
||||
/*
|
||||
* Set up the xdr_buf length.
|
||||
* This also indicates that the buffer is XDR encoded already.
|
||||
|
||||
@@ -27,13 +27,17 @@
|
||||
|
||||
/* Transport recovery
|
||||
*
|
||||
* frwr_map and frwr_unmap_* cannot run at the same time the transport
|
||||
* connect worker is running. The connect worker holds the transport
|
||||
* send lock, just as ->send_request does. This prevents frwr_map and
|
||||
* the connect worker from running concurrently. When a connection is
|
||||
* closed, the Receive completion queue is drained before the allowing
|
||||
* the connect worker to get control. This prevents frwr_unmap and the
|
||||
* connect worker from running concurrently.
|
||||
* frwr_map() is serialized with the connect worker by the transport
|
||||
* send lock.
|
||||
*
|
||||
* frwr_unmap_async() runs from Receive completion processing.
|
||||
* rpcrdma_xprt_drain() drains the Receive Queue before the Send Queue,
|
||||
* so asynchronous LOCAL_INV Work Requests are submitted before Send
|
||||
* Queue drain.
|
||||
*
|
||||
* frwr_unmap_sync() runs after the transport send lock is released.
|
||||
* rx_unmap_rwsem orders its LOCAL_INV submission before disconnect
|
||||
* drains the QP.
|
||||
*
|
||||
* When the underlying transport disconnects, MRs that are in flight
|
||||
* are flushed and are likely unusable. Thus all MRs are destroyed.
|
||||
@@ -139,7 +143,6 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr)
|
||||
mr->mr_ibmr = frmr;
|
||||
mr->mr_device = NULL;
|
||||
INIT_LIST_HEAD(&mr->mr_list);
|
||||
init_completion(&mr->mr_linv_done);
|
||||
frwr_cid_init(ep, mr);
|
||||
|
||||
sg_init_table(sg, depth);
|
||||
@@ -539,40 +542,60 @@ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc)
|
||||
* @cq: completion queue
|
||||
* @wc: WCE for a completed LocalInv WR
|
||||
*
|
||||
* Awaken anyone waiting for an MR to finish being fenced.
|
||||
* Wake frwr_unmap_sync() after the final LOCAL_INV completion.
|
||||
*/
|
||||
static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc)
|
||||
{
|
||||
struct ib_cqe *cqe = wc->wr_cqe;
|
||||
struct rpcrdma_mr *mr = container_of(cqe, struct rpcrdma_mr, mr_cqe);
|
||||
struct rpcrdma_req *req = mr->mr_req;
|
||||
|
||||
/* WARNING: Only wr_cqe and status are reliable at this point */
|
||||
trace_xprtrdma_wc_li_wake(wc, &mr->mr_cid);
|
||||
frwr_mr_done(wc, mr);
|
||||
complete(&mr->mr_linv_done);
|
||||
complete(&req->rl_linv_done);
|
||||
|
||||
rpcrdma_flush_disconnect(cq->cq_context, wc);
|
||||
}
|
||||
|
||||
/**
|
||||
* frwr_unmap_sync - invalidate memory regions that were registered for @req
|
||||
* frwr_unmap_sync - synchronously invalidate MRs registered for @req
|
||||
* @r_xprt: controlling transport instance
|
||||
* @req: rpcrdma_req with a non-empty list of MRs to process
|
||||
* @req: request whose registered MRs are to be invalidated
|
||||
*
|
||||
* Sleeps until it is safe for the host CPU to access the previously mapped
|
||||
* memory regions. This guarantees that registered MRs are properly fenced
|
||||
* from the server before the RPC consumer accesses the data in them. It
|
||||
* also ensures proper Send flow control: waking the next RPC waits until
|
||||
* this RPC has relinquished all its Send Queue entries.
|
||||
* If @req still owns registered MRs after synchronizing with transport
|
||||
* disconnect, post a chain of LOCAL_INV Work Requests and wait for the
|
||||
* final completion. On successful completion, this fences the mapped
|
||||
* regions from remote access and preserves Send Queue flow control.
|
||||
*
|
||||
* A concurrent disconnect can reset @req while this function waits for
|
||||
* the read side. In that case the MRs have already been released and no
|
||||
* LOCAL_INV Work Requests are posted.
|
||||
*
|
||||
* Context: Process context. Takes and releases
|
||||
* @r_xprt->rx_unmap_rwsem for read. May sleep.
|
||||
*/
|
||||
void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
|
||||
{
|
||||
struct ib_send_wr *first, **prev, *last;
|
||||
struct rpcrdma_ep *ep = r_xprt->rx_ep;
|
||||
const struct ib_send_wr *bad_wr;
|
||||
struct rpcrdma_ep *ep;
|
||||
struct rpcrdma_mr *mr;
|
||||
int rc;
|
||||
|
||||
/* serialize against rpcrdma_xprt_disconnect() */
|
||||
down_read(&r_xprt->rx_unmap_rwsem);
|
||||
|
||||
/*
|
||||
* Disconnect can reset this request while we wait for the read side.
|
||||
* Recheck rl_registered in case rx_ep now refers to a new connection.
|
||||
*/
|
||||
ep = r_xprt->rx_ep;
|
||||
if (!ep || list_empty(&req->rl_registered)) {
|
||||
up_read(&r_xprt->rx_unmap_rwsem);
|
||||
return;
|
||||
}
|
||||
|
||||
/* ORDER: Invalidate all of the MRs first
|
||||
*
|
||||
* Chain the LOCAL_INV Work Requests and post them with
|
||||
@@ -599,37 +622,35 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)
|
||||
prev = &last->next;
|
||||
} while ((mr = rpcrdma_mr_pop(&req->rl_registered)));
|
||||
|
||||
mr = container_of(last, struct rpcrdma_mr, mr_invwr);
|
||||
|
||||
/* Strong send queue ordering guarantees that when the
|
||||
* last WR in the chain completes, all WRs in the chain
|
||||
* are complete.
|
||||
*/
|
||||
last->wr_cqe->done = frwr_wc_localinv_wake;
|
||||
reinit_completion(&mr->mr_linv_done);
|
||||
reinit_completion(&req->rl_linv_done);
|
||||
|
||||
/* Transport disconnect drains the receive CQ before it
|
||||
* replaces the QP. The RPC reply handler won't call us
|
||||
* unless re_id->qp is a valid pointer.
|
||||
/*
|
||||
* The read side prevents disconnect from draining this QP until the
|
||||
* LOCAL_INV chain has been submitted.
|
||||
*/
|
||||
bad_wr = NULL;
|
||||
rc = ib_post_send(ep->re_id->qp, first, &bad_wr);
|
||||
if (rc) {
|
||||
/* On error, the MRs get destroyed once the QP has drained. */
|
||||
trace_xprtrdma_post_linv_err(req, rc);
|
||||
|
||||
/* Force a connection loss to ensure complete recovery.
|
||||
*/
|
||||
rpcrdma_force_disconnect(ep);
|
||||
}
|
||||
up_read(&r_xprt->rx_unmap_rwsem);
|
||||
|
||||
/* The final LOCAL_INV WR in the chain is supposed to
|
||||
* do the wake. If it was never posted, the wake will
|
||||
* not happen, so don't wait in that case.
|
||||
*/
|
||||
if (bad_wr != first)
|
||||
wait_for_completion(&mr->mr_linv_done);
|
||||
if (!rc)
|
||||
return;
|
||||
|
||||
/* On error, the MRs get destroyed once the QP has drained. */
|
||||
trace_xprtrdma_post_linv_err(req, rc);
|
||||
|
||||
/* Force a connection loss to ensure complete recovery.
|
||||
*/
|
||||
rpcrdma_force_disconnect(ep);
|
||||
wait_for_completion(&req->rl_linv_done);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -364,6 +364,8 @@ xprt_setup_rdma(struct xprt_create *args)
|
||||
INIT_DELAYED_WORK(&new_xprt->rx_connect_worker,
|
||||
xprt_rdma_connect_worker);
|
||||
|
||||
init_rwsem(&new_xprt->rx_unmap_rwsem);
|
||||
|
||||
xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT;
|
||||
|
||||
return xprt;
|
||||
|
||||
@@ -572,20 +572,24 @@ out:
|
||||
* rpcrdma_xprt_disconnect - Disconnect underlying transport
|
||||
* @r_xprt: controlling transport instance
|
||||
*
|
||||
* Caller serializes. Either the transport send lock is held,
|
||||
* or we're being called to destroy the transport.
|
||||
* Context: Caller serializes. Either the transport send lock is held,
|
||||
* or the transport is being destroyed. Takes and releases
|
||||
* @r_xprt->rx_unmap_rwsem for write. May sleep.
|
||||
*
|
||||
* On return, @r_xprt is completely divested of all hardware
|
||||
* resources and prepared for the next ->connect operation.
|
||||
*/
|
||||
void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)
|
||||
{
|
||||
struct rpcrdma_ep *ep = r_xprt->rx_ep;
|
||||
struct rpcrdma_ep *ep;
|
||||
struct rdma_cm_id *id;
|
||||
int rc;
|
||||
|
||||
down_write(&r_xprt->rx_unmap_rwsem);
|
||||
|
||||
ep = r_xprt->rx_ep;
|
||||
if (!ep)
|
||||
return;
|
||||
goto out_unlock;
|
||||
|
||||
/* Prevent teardown-generated events from requesting another close. */
|
||||
atomic_xchg(&ep->re_force_disconnect, 1);
|
||||
@@ -604,6 +608,9 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)
|
||||
rdma_destroy_id(id);
|
||||
|
||||
r_xprt->rx_ep = NULL;
|
||||
|
||||
out_unlock:
|
||||
up_write(&r_xprt->rx_unmap_rwsem);
|
||||
}
|
||||
|
||||
/* Fixed-size circular FIFO queue. This implementation is wait-free and
|
||||
@@ -927,6 +934,7 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,
|
||||
|
||||
INIT_LIST_HEAD(&req->rl_free_mrs);
|
||||
INIT_LIST_HEAD(&req->rl_registered);
|
||||
init_completion(&req->rl_linv_done);
|
||||
spin_lock(&buffer->rb_lock);
|
||||
list_add(&req->rl_all, &buffer->rb_allreqs);
|
||||
spin_unlock(&buffer->rb_lock);
|
||||
|
||||
@@ -46,6 +46,7 @@
|
||||
#include <linux/spinlock.h> /* spinlock_t, etc */
|
||||
#include <linux/atomic.h> /* atomic_t, etc */
|
||||
#include <linux/kref.h> /* struct kref */
|
||||
#include <linux/rwsem.h> /* struct rw_semaphore */
|
||||
#include <linux/workqueue.h> /* struct work_struct */
|
||||
#include <linux/llist.h>
|
||||
|
||||
@@ -249,7 +250,6 @@ struct rpcrdma_mr {
|
||||
int mr_nents;
|
||||
enum dma_data_direction mr_dir;
|
||||
struct ib_cqe mr_cqe;
|
||||
struct completion mr_linv_done;
|
||||
union {
|
||||
struct ib_reg_wr mr_regwr;
|
||||
struct ib_send_wr mr_invwr;
|
||||
@@ -348,6 +348,7 @@ struct rpcrdma_req {
|
||||
|
||||
struct list_head rl_free_mrs;
|
||||
struct list_head rl_registered;
|
||||
struct completion rl_linv_done;
|
||||
};
|
||||
|
||||
static inline struct rpcrdma_req *
|
||||
@@ -449,6 +450,13 @@ struct rpcrdma_xprt {
|
||||
struct delayed_work rx_connect_worker;
|
||||
struct rpc_timeout rx_timeout;
|
||||
struct rpcrdma_stats rx_stats;
|
||||
|
||||
/*
|
||||
* Orders frwr_unmap_sync()'s LOCAL_INV submission ahead of the
|
||||
* QP drain. Disconnect holds the write side across the drain
|
||||
* and resource teardown.
|
||||
*/
|
||||
struct rw_semaphore rx_unmap_rwsem;
|
||||
};
|
||||
|
||||
#define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt)
|
||||
|
||||
@@ -3501,7 +3501,7 @@ static struct rpc_xprt *xs_setup_tcp_tls(struct xprt_create *args)
|
||||
transport = container_of(xprt, struct sock_xprt, xprt);
|
||||
|
||||
xprt->prot = IPPROTO_TCP;
|
||||
xprt->xprt_class = &xs_tcp_transport;
|
||||
xprt->xprt_class = &xs_tcp_tls_transport;
|
||||
xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
|
||||
|
||||
xprt->bind_timeout = XS_BIND_TO;
|
||||
|
||||
Reference in New Issue
Block a user