diff --git a/Documentation/filesystems/nfs/localio.rst b/Documentation/filesystems/nfs/localio.rst index 79808b37d745..45c6981da0d8 100644 --- a/Documentation/filesystems/nfs/localio.rst +++ b/Documentation/filesystems/nfs/localio.rst @@ -312,6 +312,18 @@ Module Parameters /sys/module/nfs/parameters/localio_enabled (bool) controls if LOCALIO is enabled, defaults to Y. If client and server are local but 'localio_enabled' is set to N then LOCALIO will not be used. +It is checked on every I/O: setting it to N sends the next I/O of every +client over the network, and setting it back to Y resumes LOCALIO. A +client that probed while it was N is probed again when it is set to Y. + +/sys/module/nfs/parameters/localio_timeout (uint) +sets how many seconds the LOCALIO probe waits for each reply from the +server, defaults to 10, at most 600. A server that does not support +LOCALIO may not answer the probe at all. If no reply comes in time, +the probe gives up without a log message. The client then does not use +LOCALIO. The mount's timeouts do not apply to the probe. Set to 0, the +probe uses the mount's timeouts instead, as an NFS RPC would. A change +applies to the next probe. /sys/module/nfs/parameters/localio_O_DIRECT_semantics (bool) controls if O_DIRECT extends down to the underlying filesystem, defaults diff --git a/fs/nfs/delegation.h b/fs/nfs/delegation.h index fba4699952b8..cd6142939d34 100644 --- a/fs/nfs/delegation.h +++ b/fs/nfs/delegation.h @@ -95,6 +95,8 @@ void nfs4_inode_make_writeable(struct inode *inode); void nfs_update_delegated_atime(struct inode *inode); void nfs_update_delegated_mtime(struct inode *inode); void nfs_update_delegated_mtime_locked(struct inode *inode); +void nfs_delegated_ctime_raise(struct inode *inode, + const struct nfs_fattr *fattr); static inline int nfs_have_read_or_write_delegation(struct inode *inode) { diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 08a940e63152..49437037f948 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1253,6 +1253,8 @@ static int nfs_readdir(struct file *file, struct dir_context *ctx) spin_lock(&file->f_lock); desc->dir_cookie = dir_ctx->dir_cookie; desc->folio_index = dir_ctx->page_index; + desc->current_index = (loff_t)desc->folio_index * + nfs_readdir_array_maxentries(); desc->last_cookie = dir_ctx->last_cookie; desc->attr_gencount = dir_ctx->attr_gencount; desc->eof = dir_ctx->eof; @@ -2735,7 +2737,7 @@ static bool nfs_rename_is_unsafe_cross_dir(struct dentry *old_dentry, { struct nfs_server *server = NFS_SB(old_dentry->d_sb); - if (old_dentry->d_parent != new_dentry->d_parent) + if (old_dentry->d_parent == new_dentry->d_parent) return false; if (server->fh_expire_type & NFS_FH_RENAME_UNSAFE) return !(server->fh_expire_type & NFS_FH_NOEXPIRE_WITH_OPEN); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 9492d6bc1d1a..5c289737366f 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -340,6 +341,7 @@ static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls) { if (fls) { ff_layout_free_mirror_array(fls); + bitmap_free(fls->cancelled_stripes); kfree(fls); } } @@ -653,6 +655,12 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, fls->mirror_array[i]->report_interval = be32_to_cpup(p); out_sort_mirrors: + fls->cancelled_stripes = bitmap_zalloc( + fls->mirror_array_cnt * dss_count, gfp_flags); + if (!fls->cancelled_stripes) { + rc = -ENOMEM; + goto out_err_free; + } ff_layout_sort_mirrors(fls); ret = &fls->generic_hdr; dprintk("<-- %s (success)\n", __func__); @@ -2582,6 +2590,10 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg, if (devid && memcmp(&mirror->dss[dss_id].devid, devid, sizeof(*devid)) != 0) continue; + /* Do not cancel a stripe again while this lseg drains. */ + if (test_and_set_bit(idx * mirror->dss_count + dss_id, + flseg->cancelled_stripes)) + continue; rcu_read_lock(); mirror_ds = rcu_dereference(mirror->dss[dss_id].mirror_ds); if (IS_ERR_OR_NULL(mirror_ds) || @@ -2599,10 +2611,9 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg, clnt = ds_clp->cl_rpcclient; if (!clnt) goto next; - if (!rpc_cancel_tasks(clnt, -ECANCELED, - ff_layout_match_io, lseg)) - goto next; - rpc_clnt_disconnect(clnt); + rpc_cancel_tasks_and_disconnect(clnt, -ECANCELED, + ff_layout_match_io, + lseg); next: nfs4_ff_layout_put_deviceid(mirror_ds); } diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h index 09c3cd6964fd..5d5587fded76 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.h +++ b/fs/nfs/flexfilelayout/flexfilelayout.h @@ -109,6 +109,7 @@ struct nfs4_ff_layout_segment { u64 stripe_unit; u32 flags; u32 mirror_array_cnt; + unsigned long *cancelled_stripes; struct nfs4_ff_layout_mirror *mirror_array[] __counted_by(mirror_array_cnt); }; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index e0a2e92952af..5a682f74d7d1 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -629,28 +629,52 @@ nfs_fattr_fixup_delegated(struct inode *inode, struct nfs_fattr *fattr) } } +/* + * With delegated timestamps, the client's coarse clock can trail the + * server's fine-grained ctime cached from an OPEN or SETATTR reply. The + * server won't move ctime backward when the times are returned, so a + * new stamp must land after the cached ctime. + */ +static struct timespec64 nfs_delegated_stamp(struct inode *inode) +{ + struct timespec64 ctime = inode_get_ctime(inode); + struct timespec64 now = current_time(inode); + + if (timespec64_compare(&now, &ctime) <= 0) { + now = ctime; + timespec64_add_ns(&now, 1); + } + return now; +} + +/* + * The server stamped c/mtime itself (e.g. for a size change). Raise the + * cached ctime to the server's so that the next delegated stamp lands + * after it. + */ +void nfs_delegated_ctime_raise(struct inode *inode, + const struct nfs_fattr *fattr) +{ + struct timespec64 ctime = inode_get_ctime(inode); + + if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && + timespec64_compare(&fattr->ctime, &ctime) > 0) + inode_set_ctime_to_ts(inode, fattr->ctime); +} +EXPORT_SYMBOL_GPL(nfs_delegated_ctime_raise); + static void nfs_set_timestamps_to_ts(struct inode *inode, struct iattr *attr) { unsigned int cache_flags = 0; if (attr->ia_valid & ATTR_MTIME_SET) { - struct timespec64 ctime = inode_get_ctime(inode); - struct timespec64 mtime = inode_get_mtime(inode); - struct timespec64 now; - bool updated = false; - - now = inode_set_ctime_current(inode); - if (!timespec64_equal(&now, &ctime)) - updated = true; - + /* The stamp is always after the old ctime */ + inode_set_ctime_to_ts(inode, nfs_delegated_stamp(inode)); inode_set_mtime_to_ts(inode, attr->ia_mtime); - if (!timespec64_equal(&now, &mtime)) - updated = true; - - inode_maybe_inc_iversion(inode, updated); + inode_maybe_inc_iversion(inode, true); cache_flags |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME; } - if (attr->ia_valid & ATTR_ATIME_SET) { + if (attr->ia_valid & ATTR_ATIME) { inode_set_atime_to_ts(inode, attr->ia_atime); cache_flags |= NFS_INO_INVALID_ATIME; } @@ -665,7 +689,11 @@ static void nfs_update_atime(struct inode *inode) static void nfs_update_mtime(struct inode *inode) { - inode_update_time(inode, FS_UPD_CMTIME, 0); + struct timespec64 now = nfs_delegated_stamp(inode); + + /* Not inode_update_time(): lockless readers would see its stamp */ + inode_set_ctime_to_ts(inode, now); + inode_set_mtime_to_ts(inode, now); NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME); } @@ -684,6 +712,7 @@ void nfs_update_delegated_mtime_locked(struct inode *inode) nfs_have_directory_delegation(inode)) nfs_update_mtime(inode); } +EXPORT_SYMBOL_GPL(nfs_update_delegated_mtime_locked); void nfs_update_delegated_mtime(struct inode *inode) { @@ -746,7 +775,7 @@ nfs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, } else if (nfs_have_delegated_atime(inode) && attr->ia_valid & ATTR_ATIME && !(attr->ia_valid & ATTR_MTIME)) { - if (!(attr->ia_valid & ATTR_ATIME_SET)) { + if (!(attr->ia_valid & (ATTR_ATIME_SET | ATTR_TIMES_SET))) { nfs_update_delegated_atime(inode); attr->ia_valid &= ~ATTR_ATIME; } @@ -808,10 +837,14 @@ static int nfs_vmtruncate(struct inode * inode, loff_t offset) nfs_ooo_clear(NFS_I(inode)); } NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; + /* + * Stamp in the same i_lock section that checked the delegation and + * raised ctime, before a delegation return can start. + */ + nfs_update_delegated_mtime_locked(inode); spin_unlock(&inode->i_lock); truncate_pagecache(inode, offset); - nfs_update_delegated_mtime_locked(inode); spin_lock(&inode->i_lock); out: return err; @@ -837,6 +870,8 @@ void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr, if ((attr->ia_valid & ATTR_SIZE) != 0) { if (!nfs_have_delegated_mtime(inode)) nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME); + else + nfs_delegated_ctime_raise(inode, fattr); nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); nfs_inc_stats(inode, NFSIOS_SETATTRTRUNC); nfs_vmtruncate(inode, attr->ia_size); diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index cdada6cbb13b..7f8d794581a3 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -25,6 +25,7 @@ #include #include "internal.h" +#include "netns.h" #include "pnfs.h" #include "nfstrace.h" @@ -55,7 +56,50 @@ struct nfs_local_fsync_ctx { }; static bool localio_enabled __read_mostly = true; -module_param(localio_enabled, bool, 0644); + +static void nfs_local_probe_all(void); + +/* + * localio_enabled is checked on every I/O, so turning it off stops LOCALIO + * at once and turning it back on resumes it for clients that use it. A + * client that probed while it was off was disabled, and nothing probes it + * again: turning it on probes every client that does not use LOCALIO. + */ +static int localio_enabled_set(const char *val, const struct kernel_param *kp) +{ + bool was = READ_ONCE(localio_enabled); + int ret; + + ret = param_set_bool(val, kp); + if (!ret && !was && READ_ONCE(localio_enabled)) + nfs_local_probe_all(); + return ret; +} + +static const struct kernel_param_ops localio_enabled_ops = { + .set = localio_enabled_set, + .get = param_get_bool, +}; +module_param_cb(localio_enabled, &localio_enabled_ops, &localio_enabled, 0644); + +/* + * Seconds the LOCALIO probe waits for each reply. With 0, the probe uses + * the NFS client's timeouts instead. + */ +static unsigned int localio_timeout __read_mostly = 10; + +static int localio_timeout_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, 0, 600); +} + +static const struct kernel_param_ops localio_timeout_ops = { + .set = localio_timeout_set, + .get = param_get_uint, +}; +module_param_cb(localio_timeout, &localio_timeout_ops, &localio_timeout, 0644); +MODULE_PARM_DESC(localio_timeout, + "LOCALIO probe timeout in seconds (default 10), 0 to use the mount's default"); static void nfs_local_do_read(struct nfs_local_kiocb *iocb, const struct rpc_call_ops *call_ops); @@ -131,13 +175,45 @@ const struct rpc_program nfslocalio_program = { /* * nfs_init_localioclient - Initialise an NFS localio client connection + * + * A server that does not support LOCALIO may not answer its RPCs at all. + * The probe shares the NFS client's transport and runs beside its RPCs. + * With localio_timeout set, it gets a quiet, soft timeout of that many + * seconds of its own, whatever timeouts the mount uses. With 0, it uses + * the NFS client's timeouts, as an NFS RPC would. */ static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp) { - struct rpc_clnt *rpcclient_localio; + unsigned int timeout_secs = READ_ONCE(localio_timeout); + struct rpc_clnt *parent = clp->cl_rpcclient; + struct rpc_clnt *probe = NULL, *rpcclient_localio; - rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient, - &nfslocalio_program, 1); + /* + * rpc_bind_new_program() pings the new program with the timeout of + * the client it clones, so set the probe's timeout on a private + * clone first. The clone's cl_timeout points at the transport's + * timeout: give it its own copy rather than write through it. + */ + if (timeout_secs) { + probe = rpc_clone_client(parent); + if (IS_ERR(probe)) + return probe; + probe->cl_timeout_default = (struct rpc_timeout) { + .to_initval = timeout_secs * HZ, + .to_maxval = timeout_secs * HZ, + .to_retries = 0, + }; + probe->cl_timeout = &probe->cl_timeout_default; + probe->cl_softrtry = 1; + probe->cl_noretranstimeo = 0; + probe->cl_chatty = 0; + parent = probe; + } + + rpcclient_localio = rpc_bind_new_program(parent, &nfslocalio_program, 1); + /* The new client holds a reference to its parent, the probe clone */ + if (probe) + rpc_shutdown_client(probe); dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n", __func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR), @@ -203,18 +279,49 @@ void nfs_local_probe_async_work(struct work_struct *work) struct nfs_client *clp = container_of(work, struct nfs_client, cl_local_probe_work); - if (!refcount_inc_not_zero(&clp->cl_count)) - return; nfs_local_probe(clp); + /* The reference nfs_local_probe_async() took for this work */ nfs_put_client(clp); } +/* + * The work is part of the client, so it holds a reference on the client + * from when it is queued until it has run: the last nfs_put_client() + * cannot free the client under a queued probe. + */ void nfs_local_probe_async(struct nfs_client *clp) { - queue_work(nfsiod_workqueue, &clp->cl_local_probe_work); + if (!refcount_inc_not_zero(&clp->cl_count)) + return; + if (!queue_work(nfsiod_workqueue, &clp->cl_local_probe_work)) + nfs_put_client(clp); /* already queued, with its reference */ } EXPORT_SYMBOL_GPL(nfs_local_probe_async); +/* Probe every client, in every namespace, that does not use LOCALIO. */ +static void nfs_local_probe_all(void) +{ + struct nfs_client *clp; + struct net *net; + + /* Set on the module command line: there are no clients yet. */ + if (!nfs_net_id) + return; + + down_read(&net_rwsem); + for_each_net(net) { + struct nfs_net *nn = net_generic(net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link) + if (clp->cl_cons_state == NFS_CS_READY && + !nfs_client_is_local(clp)) + nfs_local_probe_async(clp); + spin_unlock(&nn->nfs_client_lock); + } + up_read(&net_rwsem); +} + static inline void nfs_local_file_put(struct nfsd_file *localio) { /* nfs_to_nfsd_file_put_local() expects an __rcu pointer diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index ab86246fc364..09b4a6ecad88 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -380,6 +380,7 @@ out: * @pos: destination offset * @len: copy length * @oldsize: length of the file prior to clone/copy + * @fattr: destination attributes from the reply, or NULL * * Punch a hole in the inode page cache, so that the NFS client will * know to retrieve new data. @@ -387,7 +388,7 @@ out: * invalid cached values for change attribute, ctime, mtime and space used. */ static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len, - loff_t oldsize) + loff_t oldsize, const struct nfs_fattr *fattr) { struct inode *inode = file_inode(file); struct address_space *mapping = file->f_mapping; @@ -405,8 +406,12 @@ static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len, NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME | NFS_INO_INVALID_BLOCKS); + /* The server may have stamped c/mtime; stamp after it */ + if (fattr && nfs_have_delegated_mtime(inode)) + nfs_delegated_ctime_raise(inode, fattr); + /* Before a refresh can see the times invalid and take the server's */ + nfs_update_delegated_mtime_locked(inode); spin_unlock(&inode->i_lock); - nfs_update_delegated_mtime(inode); } static ssize_t _nfs42_proc_copy(struct file *src, @@ -506,7 +511,8 @@ static ssize_t _nfs42_proc_copy(struct file *src, goto out; } - nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst); + nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst, + NULL); nfs_invalidate_atime(src_inode); status = res->write_res.count; out: @@ -1311,8 +1317,17 @@ static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f, /* a zero-length count means clone to EOF in src */ if (count == 0 && res.dst_fattr->valid & NFS_ATTR_FATTR_SIZE) count = nfs_size_to_loff_t(res.dst_fattr->size) - dst_offset; - nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst); - status = nfs_post_op_update_inode(dst_inode, res.dst_fattr); + nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst, + res.dst_fattr); + /* + * nfs_post_op_update_inode() would mark ctime invalid, so the + * reply's ctime would overwrite the delegated stamp. + */ + if (nfs_have_delegated_mtime(dst_inode)) + status = nfs_refresh_inode(dst_inode, res.dst_fattr); + else + status = nfs_post_op_update_inode(dst_inode, + res.dst_fattr); } kfree(res.dst_fattr); diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index e1018b03c1cb..6759fa61b9b4 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -1722,6 +1722,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state, { DEFINE_WAIT(wait); int status = 0; + unsigned long deadline = jiffies + 5 * HZ; + signed long remaining; for (;;) { if (nfs_stateid_is_sequential(state, stateid)) @@ -1756,7 +1758,8 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state, if (!fatal_signal_pending(current) && !nfs_current_task_exiting()) { - if (schedule_timeout(5*HZ) == 0) + remaining = (long)(deadline - jiffies); + if (remaining <= 0 || schedule_timeout(remaining) == 0) status = -EAGAIN; else status = 0; @@ -3093,6 +3096,8 @@ static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata, if (server->caps & NFS_CAP_POSIX_LOCK) set_bit(NFS_STATE_POSIX_LOCKS, &state->flags); if (opendata->o_res.rflags & NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK) + server->nfs_client->cl_may_notify_lock = true; + if (server->nfs_client->cl_may_notify_lock) set_bit(NFS_STATE_MAY_NOTIFY_LOCK, &state->flags); if (opendata->o_res.rflags & NFS4_OPEN_RESULT_PRESERVE_UNLINKED) set_bit(NFS_INO_PRESERVE_UNLINKED, &NFS_I(state->inode)->flags); @@ -7287,8 +7292,15 @@ static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock * if (status != 0) goto out; /* Is this a delegated lock? */ - if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0) + if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0) { + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + struct cb_notify_lock_args cbnl = { .cbnl_valid = false }; + + /* The server never saw it, so it will not notify waiters */ + nfs_copy_fh(&cbnl.cbnl_fh, NFS_FH(inode)); + __wake_up(&clp->cl_lock_waitq, TASK_NORMAL, 0, &cbnl); goto out; + } alloc_seqid = NFS_SERVER(inode)->nfs_client->cl_mvops->alloc_seqid; seqid = alloc_seqid(&lsp->ls_seqid, GFP_KERNEL); status = -ENOMEM; @@ -7708,8 +7720,14 @@ nfs4_wake_lock_waiter(wait_queue_entry_t *wait, unsigned int mode, int flags, vo struct nfs_lowner *lowner = &cbnl->cbnl_owner, *wowner = &waiter->owner; - /* Only wake if the callback was for the same owner. */ - if (lowner->id != wowner->id || lowner->s_dev != wowner->s_dev) + /* + * Only wake the owner a CB_NOTIFY_LOCK names. A lock cached + * under a delegation is released locally, and the releasing + * thread cannot tell which waiter wants it, so those wakeups + * (cbnl_valid false) name only the inode and wake them all. + */ + if (cbnl->cbnl_valid && + (lowner->id != wowner->id || lowner->s_dev != wowner->s_dev)) return 0; /* Make sure it's for the right inode */ diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 6dc00233e837..14f4a4b12ffd 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -2086,7 +2086,7 @@ static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, break; } if (size_set && io_set) { - if (size && io) + if (size || io) ret = true; } else if (size || io) ret = true; diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 416c6f39f31d..6e1b0332f2de 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -120,6 +120,7 @@ struct nfs_client { #define NFS_SP4_MACH_CRED_COMMIT 6 /* COMMIT */ #define NFS_SP4_MACH_CRED_PNFS_CLEANUP 7 /* LAYOUTRETURN */ wait_queue_head_t cl_lock_waitq; + bool cl_may_notify_lock; #endif /* CONFIG_NFS_V4 */ /* Our own IP address, as a null-terminated string. diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index 0dbdf3722537..dbaad8b5f181 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -230,6 +230,10 @@ unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error, bool (*fnmatch)(const struct rpc_task *, const void *), const void *data); +unsigned long rpc_cancel_tasks_and_disconnect(struct rpc_clnt *clnt, int error, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data); void rpc_execute(struct rpc_task *); void rpc_init_priority_wait_queue(struct rpc_wait_queue *, const char *); void rpc_init_wait_queue(struct rpc_wait_queue *, const char *); diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index 005730623f3b..780eaa157a9f 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -112,6 +112,11 @@ struct rpc_rqst { ktime_t rq_xtime; /* transmit time stamp */ int rq_ntrans; + bool rq_disconnect_on_release; + /* + * protected by rq_xprt->transport_lock + * while the request is active + */ #if defined(CONFIG_SUNRPC_BACKCHANNEL) struct lwq_node rq_bc_list; /* Callback service list */ diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 6cedc824cf82..2f076b4bfd54 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -653,6 +653,9 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args, } args->servername = xprt->servername; args->nodename = clnt->cl_nodename; + args->xprtsec = clnt->cl_xprtsec; + if (!args->timeout) + args->timeout = clnt->cl_timeout; new = rpc_new_client(args, xps, xprt, clnt); if (IS_ERR(new)) @@ -901,6 +904,59 @@ void rpc_killall_tasks(struct rpc_clnt *clnt) } EXPORT_SYMBOL_GPL(rpc_killall_tasks); +struct rpc_cancel_disconnect_ctx { + bool (*fnmatch)(const struct rpc_task *task, const void *data); + const void *data; +}; + +static int rpc_mark_xprt_disconnect_on_release(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, + void *arg) +{ + struct rpc_cancel_disconnect_ctx *ctx = arg; + + xprt_mark_matching_reqs_disconnect_on_release(xprt, clnt, + ctx->fnmatch, + ctx->data); + return 0; +} + +static unsigned long +__rpc_cancel_tasks(struct rpc_clnt *clnt, int error, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data, + bool disconnect) +{ + struct rpc_task *task; + unsigned long count = 0; + + if (list_empty(&clnt->cl_tasks)) + return 0; + + spin_lock(&clnt->cl_lock); + list_for_each_entry(task, &clnt->cl_tasks, tk_task) { + if (!RPC_IS_ACTIVATED(task)) + continue; + if (!fnmatch(task, data)) + continue; + rpc_task_try_cancel(task, error); + count++; + } + spin_unlock(&clnt->cl_lock); + + if (disconnect && count) { + struct rpc_cancel_disconnect_ctx ctx = { + .fnmatch = fnmatch, + .data = data, + }; + rpc_clnt_iterate_for_each_xprt(clnt, + rpc_mark_xprt_disconnect_on_release, + &ctx); + } + return count; +} + /** * rpc_cancel_tasks - try to cancel a set of RPC tasks * @clnt: Pointer to RPC client @@ -916,28 +972,35 @@ unsigned long rpc_cancel_tasks(struct rpc_clnt *clnt, int error, const void *), const void *data) { - struct rpc_task *task; - unsigned long count = 0; - - if (list_empty(&clnt->cl_tasks)) - return 0; - /* - * Spin lock all_tasks to prevent changes... - */ - spin_lock(&clnt->cl_lock); - list_for_each_entry(task, &clnt->cl_tasks, tk_task) { - if (!RPC_IS_ACTIVATED(task)) - continue; - if (!fnmatch(task, data)) - continue; - rpc_task_try_cancel(task, error); - count++; - } - spin_unlock(&clnt->cl_lock); - return count; + return __rpc_cancel_tasks(clnt, error, fnmatch, data, false); } EXPORT_SYMBOL_GPL(rpc_cancel_tasks); +/** + * rpc_cancel_tasks_and_disconnect - cancel matching RPC tasks + * @clnt: Pointer to RPC client + * @error: RPC task error value to set + * @fnmatch: Pointer to selector function + * @data: User data + * + * Like rpc_cancel_tasks(), but after cancelling matching tasks, marks + * matching in-flight requests so that xprt_release() conditionally + * disconnects the connection generation on which each request was sent. + * + * @fnmatch is called while holding spinlocks and must not sleep. + * + * Returns the number of active tasks matched by @fnmatch. + */ +unsigned long +rpc_cancel_tasks_and_disconnect(struct rpc_clnt *clnt, int error, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data) +{ + return __rpc_cancel_tasks(clnt, error, fnmatch, data, true); +} +EXPORT_SYMBOL_GPL(rpc_cancel_tasks_and_disconnect); + static int rpc_clnt_disconnect_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt, void *dummy) { @@ -1194,7 +1257,7 @@ void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt) (task->tk_flags & RPC_TASK_MOVEABLE))) return; xprt_release(task); - xprt_put(task->tk_xprt); + rpc_task_release_xprt(clnt, task->tk_xprt); } if (task->tk_flags & RPC_TASK_NO_ROUND_ROBIN) task->tk_xprt = rpc_task_get_first_xprt(clnt); @@ -2125,8 +2188,10 @@ call_bind_status(struct rpc_task *task) goto retry_timeout; case -ENETDOWN: case -ENETUNREACH: - if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL) + if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { + status = task->tk_status; break; + } fallthrough; case -ECONNREFUSED: /* connection problems */ case -ECONNRESET: @@ -2257,6 +2322,7 @@ call_connect_status(struct rpc_task *task) xprt_release(task); value = atomic_long_dec_return(&xprt->queuelen); + atomic_long_dec(&xps->xps_queuelen); if (value == 0) rpc_xprt_switch_remove_xprt(xps, saved, true); diff --git a/net/sunrpc/rpc_pipe.c b/net/sunrpc/rpc_pipe.c index 9d349cfbc483..f14cae295280 100644 --- a/net/sunrpc/rpc_pipe.c +++ b/net/sunrpc/rpc_pipe.c @@ -191,6 +191,7 @@ rpc_close_pipes(struct dentry *dentry) cancel_delayed_work_sync(&pipe->queue_timeout); rpc_inode_setowner(inode, NULL); RPC_I(inode)->pipe = NULL; + wake_up(&RPC_I(inode)->waitq); inode_unlock(inode); } @@ -665,8 +666,8 @@ int rpc_mkpipe_dentry(struct dentry *parent, const char *name, rpci->private = private; rpci->pipe = pipe; rpc_inode_setowner(inode, private); - pipe->dentry = dentry; // borrowed d_make_persistent(dentry, inode); + pipe->dentry = dentry; // borrowed fsnotify_create(dir, dentry); simple_done_creating(dentry); return 0; diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h index e3c6e3b63f0b..11b9e5cead8e 100644 --- a/net/sunrpc/sunrpc.h +++ b/net/sunrpc/sunrpc.h @@ -38,9 +38,17 @@ static inline int sock_is_loopback(struct sock *sk) struct svc_serv; struct svc_rqst; +struct rpc_clnt; +struct rpc_task; +struct rpc_xprt; int rpc_clients_notifier_register(void); void rpc_clients_notifier_unregister(void); void auth_domain_cleanup(void); void svc_sock_update_bufs(struct svc_serv *serv); enum svc_auth_status svc_authenticate(struct svc_rqst *rqstp); +void xprt_mark_matching_reqs_disconnect_on_release(struct rpc_xprt *xprt, + const struct rpc_clnt *clnt, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data); #endif /* _NET_SUNRPC_SUNRPC_H */ diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index 48a3618cbb29..c110806989c5 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -812,6 +812,17 @@ xprt_request_retransmit_after_disconnect(struct rpc_task *task) !xprt_connected(xprt); } +static void xprt_conditional_disconnect_locked(struct rpc_xprt *xprt, + unsigned int cookie) + __must_hold(&xprt->transport_lock) +{ + if (cookie != xprt->connect_cookie) + return; + if (test_bit(XPRT_CLOSING, &xprt->state)) + return; + xprt_schedule_autoclose_locked(xprt); +} + /** * xprt_conditional_disconnect - force a transport to disconnect * @xprt: transport to disconnect @@ -827,12 +838,7 @@ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie) { /* Don't race with the test_bit() in xprt_clear_locked() */ spin_lock(&xprt->transport_lock); - if (cookie != xprt->connect_cookie) - goto out; - if (test_bit(XPRT_CLOSING, &xprt->state)) - goto out; - xprt_schedule_autoclose_locked(xprt); -out: + xprt_conditional_disconnect_locked(xprt, cookie); spin_unlock(&xprt->transport_lock); } @@ -1434,6 +1440,79 @@ xprt_request_dequeue_transmit(struct rpc_task *task) spin_unlock(&xprt->queue_lock); } +static bool +xprt_request_matches(const struct rpc_rqst *req, + const struct rpc_clnt *clnt, + bool (*fnmatch)(const struct rpc_task *, const void *), + const void *data) +{ + const struct rpc_task *task = req->rq_task; + + return task && task->tk_client == clnt && fnmatch(task, data); +} + +static void +xprt_request_mark_disconnect_on_release(struct rpc_xprt *xprt, + struct rpc_rqst *req) + __must_hold(&xprt->queue_lock) +{ + spin_lock(&xprt->transport_lock); + req->rq_disconnect_on_release = true; + spin_unlock(&xprt->transport_lock); +} + +/** + * xprt_mark_matching_reqs_disconnect_on_release - mark matching in-flight + * requests for connection-generation + * teardown at release + * @xprt: transport whose queues to walk + * @clnt: RPC client whose requests are eligible + * @fnmatch: selector called with the request's owning rpc_task + * @data: caller data forwarded to @fnmatch + * + * @fnmatch must not sleep and must be safe to call while xprt->queue_lock + * is held. + * + * Walks the transmit queue (including per-owner rq_xmit2 chains) and the + * receive tree under xprt->queue_lock and marks each matching rpc_rqst so + * that xprt_release() will conditionally disconnect the transport against + * that request's rq_connect_cookie. Requests remain eligible for marking + * while they are present on a transport queue. The marker is set under + * xprt->transport_lock, nested inside xprt->queue_lock, and xprt_release() + * consumes it with xprt->transport_lock held after removing queue membership. + * + * The @clnt qualification preserves the selection domain of + * rpc_cancel_tasks(). An rpc_xprt may be shared by multiple rpc_clnt + * instances, so requests belonging to other clients must not be marked. + */ +void +xprt_mark_matching_reqs_disconnect_on_release(struct rpc_xprt *xprt, + const struct rpc_clnt *clnt, + bool (*fnmatch)(const struct rpc_task *, + const void *), + const void *data) +{ + struct rpc_rqst *req, *pos; + struct rb_node *n; + + spin_lock(&xprt->queue_lock); + list_for_each_entry(req, &xprt->xmit_queue, rq_xmit) { + if (xprt_request_matches(req, clnt, fnmatch, data)) + xprt_request_mark_disconnect_on_release(xprt, req); + list_for_each_entry(pos, &req->rq_xmit2, rq_xmit2) { + if (xprt_request_matches(pos, clnt, fnmatch, data)) + xprt_request_mark_disconnect_on_release(xprt, + pos); + } + } + for (n = rb_first(&xprt->recv_queue); n; n = rb_next(n)) { + req = rb_entry(n, struct rpc_rqst, rq_recv); + if (xprt_request_matches(req, clnt, fnmatch, data)) + xprt_request_mark_disconnect_on_release(xprt, req); + } + spin_unlock(&xprt->queue_lock); +} + /** * xprt_request_dequeue_xprt - remove a task from the transmit+receive queue * @task: pointer to rpc_task @@ -1915,6 +1994,7 @@ xprt_request_init(struct rpc_task *task) req->rq_rcv_buf.bvec = NULL; req->rq_release_snd_buf = NULL; req->rq_seqno_count = 0; + req->rq_disconnect_on_release = false; xprt_init_majortimeo(task, req, task->tk_client->cl_timeout); trace_xprt_reserve(req); @@ -1991,6 +2071,20 @@ void xprt_release(struct rpc_task *task) xprt = req->rq_xprt; xprt_request_dequeue_xprt(task); spin_lock(&xprt->transport_lock); + if (unlikely(req->rq_disconnect_on_release)) { + req->rq_disconnect_on_release = false; + /* + * Dequeue may skip queue_lock. Pair with the fully ordered + * test_and_clear_bit(RPC_TASK_NEED_XMIT) in + * xprt_request_dequeue_transmit_locked() to order the runstate + * check before reading the sender's rq_ntrans and + * rq_connect_cookie updates. + */ + smp_rmb(); + if (req->rq_ntrans > 0) + xprt_conditional_disconnect_locked(xprt, + req->rq_connect_cookie); + } xprt->ops->release_xprt(xprt, task); if (xprt->ops->release_request) xprt->ops->release_request(task); @@ -2020,6 +2114,7 @@ xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task, task->tk_rqstp = req; req->rq_task = task; xprt_init_connect_cookie(req, req->rq_xprt); + req->rq_disconnect_on_release = false; /* * Set up the xdr_buf length. * This also indicates that the buffer is XDR encoded already. diff --git a/net/sunrpc/xprtrdma/frwr_ops.c b/net/sunrpc/xprtrdma/frwr_ops.c index e83cef19e656..97034a3b5141 100644 --- a/net/sunrpc/xprtrdma/frwr_ops.c +++ b/net/sunrpc/xprtrdma/frwr_ops.c @@ -27,13 +27,17 @@ /* Transport recovery * - * frwr_map and frwr_unmap_* cannot run at the same time the transport - * connect worker is running. The connect worker holds the transport - * send lock, just as ->send_request does. This prevents frwr_map and - * the connect worker from running concurrently. When a connection is - * closed, the Receive completion queue is drained before the allowing - * the connect worker to get control. This prevents frwr_unmap and the - * connect worker from running concurrently. + * frwr_map() is serialized with the connect worker by the transport + * send lock. + * + * frwr_unmap_async() runs from Receive completion processing. + * rpcrdma_xprt_drain() drains the Receive Queue before the Send Queue, + * so asynchronous LOCAL_INV Work Requests are submitted before Send + * Queue drain. + * + * frwr_unmap_sync() runs after the transport send lock is released. + * rx_unmap_rwsem orders its LOCAL_INV submission before disconnect + * drains the QP. * * When the underlying transport disconnects, MRs that are in flight * are flushed and are likely unusable. Thus all MRs are destroyed. @@ -139,7 +143,6 @@ int frwr_mr_init(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr *mr) mr->mr_ibmr = frmr; mr->mr_device = NULL; INIT_LIST_HEAD(&mr->mr_list); - init_completion(&mr->mr_linv_done); frwr_cid_init(ep, mr); sg_init_table(sg, depth); @@ -539,40 +542,60 @@ static void frwr_wc_localinv(struct ib_cq *cq, struct ib_wc *wc) * @cq: completion queue * @wc: WCE for a completed LocalInv WR * - * Awaken anyone waiting for an MR to finish being fenced. + * Wake frwr_unmap_sync() after the final LOCAL_INV completion. */ static void frwr_wc_localinv_wake(struct ib_cq *cq, struct ib_wc *wc) { struct ib_cqe *cqe = wc->wr_cqe; struct rpcrdma_mr *mr = container_of(cqe, struct rpcrdma_mr, mr_cqe); + struct rpcrdma_req *req = mr->mr_req; /* WARNING: Only wr_cqe and status are reliable at this point */ trace_xprtrdma_wc_li_wake(wc, &mr->mr_cid); frwr_mr_done(wc, mr); - complete(&mr->mr_linv_done); + complete(&req->rl_linv_done); rpcrdma_flush_disconnect(cq->cq_context, wc); } /** - * frwr_unmap_sync - invalidate memory regions that were registered for @req + * frwr_unmap_sync - synchronously invalidate MRs registered for @req * @r_xprt: controlling transport instance - * @req: rpcrdma_req with a non-empty list of MRs to process + * @req: request whose registered MRs are to be invalidated * - * Sleeps until it is safe for the host CPU to access the previously mapped - * memory regions. This guarantees that registered MRs are properly fenced - * from the server before the RPC consumer accesses the data in them. It - * also ensures proper Send flow control: waking the next RPC waits until - * this RPC has relinquished all its Send Queue entries. + * If @req still owns registered MRs after synchronizing with transport + * disconnect, post a chain of LOCAL_INV Work Requests and wait for the + * final completion. On successful completion, this fences the mapped + * regions from remote access and preserves Send Queue flow control. + * + * A concurrent disconnect can reset @req while this function waits for + * the read side. In that case the MRs have already been released and no + * LOCAL_INV Work Requests are posted. + * + * Context: Process context. Takes and releases + * @r_xprt->rx_unmap_rwsem for read. May sleep. */ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) { struct ib_send_wr *first, **prev, *last; - struct rpcrdma_ep *ep = r_xprt->rx_ep; const struct ib_send_wr *bad_wr; + struct rpcrdma_ep *ep; struct rpcrdma_mr *mr; int rc; + /* serialize against rpcrdma_xprt_disconnect() */ + down_read(&r_xprt->rx_unmap_rwsem); + + /* + * Disconnect can reset this request while we wait for the read side. + * Recheck rl_registered in case rx_ep now refers to a new connection. + */ + ep = r_xprt->rx_ep; + if (!ep || list_empty(&req->rl_registered)) { + up_read(&r_xprt->rx_unmap_rwsem); + return; + } + /* ORDER: Invalidate all of the MRs first * * Chain the LOCAL_INV Work Requests and post them with @@ -599,37 +622,35 @@ void frwr_unmap_sync(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req) prev = &last->next; } while ((mr = rpcrdma_mr_pop(&req->rl_registered))); - mr = container_of(last, struct rpcrdma_mr, mr_invwr); - /* Strong send queue ordering guarantees that when the * last WR in the chain completes, all WRs in the chain * are complete. */ last->wr_cqe->done = frwr_wc_localinv_wake; - reinit_completion(&mr->mr_linv_done); + reinit_completion(&req->rl_linv_done); - /* Transport disconnect drains the receive CQ before it - * replaces the QP. The RPC reply handler won't call us - * unless re_id->qp is a valid pointer. + /* + * The read side prevents disconnect from draining this QP until the + * LOCAL_INV chain has been submitted. */ bad_wr = NULL; rc = ib_post_send(ep->re_id->qp, first, &bad_wr); + if (rc) { + /* On error, the MRs get destroyed once the QP has drained. */ + trace_xprtrdma_post_linv_err(req, rc); + + /* Force a connection loss to ensure complete recovery. + */ + rpcrdma_force_disconnect(ep); + } + up_read(&r_xprt->rx_unmap_rwsem); /* The final LOCAL_INV WR in the chain is supposed to * do the wake. If it was never posted, the wake will * not happen, so don't wait in that case. */ if (bad_wr != first) - wait_for_completion(&mr->mr_linv_done); - if (!rc) - return; - - /* On error, the MRs get destroyed once the QP has drained. */ - trace_xprtrdma_post_linv_err(req, rc); - - /* Force a connection loss to ensure complete recovery. - */ - rpcrdma_force_disconnect(ep); + wait_for_completion(&req->rl_linv_done); } /** diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index d4e6746d8ecd..e38637c2ec78 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -364,6 +364,8 @@ xprt_setup_rdma(struct xprt_create *args) INIT_DELAYED_WORK(&new_xprt->rx_connect_worker, xprt_rdma_connect_worker); + init_rwsem(&new_xprt->rx_unmap_rwsem); + xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; return xprt; diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 54957295a631..620afff79eff 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -572,20 +572,24 @@ out: * rpcrdma_xprt_disconnect - Disconnect underlying transport * @r_xprt: controlling transport instance * - * Caller serializes. Either the transport send lock is held, - * or we're being called to destroy the transport. + * Context: Caller serializes. Either the transport send lock is held, + * or the transport is being destroyed. Takes and releases + * @r_xprt->rx_unmap_rwsem for write. May sleep. * * On return, @r_xprt is completely divested of all hardware * resources and prepared for the next ->connect operation. */ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) { - struct rpcrdma_ep *ep = r_xprt->rx_ep; + struct rpcrdma_ep *ep; struct rdma_cm_id *id; int rc; + down_write(&r_xprt->rx_unmap_rwsem); + + ep = r_xprt->rx_ep; if (!ep) - return; + goto out_unlock; /* Prevent teardown-generated events from requesting another close. */ atomic_xchg(&ep->re_force_disconnect, 1); @@ -604,6 +608,9 @@ void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt) rdma_destroy_id(id); r_xprt->rx_ep = NULL; + +out_unlock: + up_write(&r_xprt->rx_unmap_rwsem); } /* Fixed-size circular FIFO queue. This implementation is wait-free and @@ -927,6 +934,7 @@ struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt, INIT_LIST_HEAD(&req->rl_free_mrs); INIT_LIST_HEAD(&req->rl_registered); + init_completion(&req->rl_linv_done); spin_lock(&buffer->rb_lock); list_add(&req->rl_all, &buffer->rb_allreqs); spin_unlock(&buffer->rb_lock); diff --git a/net/sunrpc/xprtrdma/xprt_rdma.h b/net/sunrpc/xprtrdma/xprt_rdma.h index 4cbc941e4a3e..b3016e1eb172 100644 --- a/net/sunrpc/xprtrdma/xprt_rdma.h +++ b/net/sunrpc/xprtrdma/xprt_rdma.h @@ -46,6 +46,7 @@ #include /* spinlock_t, etc */ #include /* atomic_t, etc */ #include /* struct kref */ +#include /* struct rw_semaphore */ #include /* struct work_struct */ #include @@ -249,7 +250,6 @@ struct rpcrdma_mr { int mr_nents; enum dma_data_direction mr_dir; struct ib_cqe mr_cqe; - struct completion mr_linv_done; union { struct ib_reg_wr mr_regwr; struct ib_send_wr mr_invwr; @@ -348,6 +348,7 @@ struct rpcrdma_req { struct list_head rl_free_mrs; struct list_head rl_registered; + struct completion rl_linv_done; }; static inline struct rpcrdma_req * @@ -449,6 +450,13 @@ struct rpcrdma_xprt { struct delayed_work rx_connect_worker; struct rpc_timeout rx_timeout; struct rpcrdma_stats rx_stats; + + /* + * Orders frwr_unmap_sync()'s LOCAL_INV submission ahead of the + * QP drain. Disconnect holds the write side across the drain + * and resource teardown. + */ + struct rw_semaphore rx_unmap_rwsem; }; #define rpcx_to_rdmax(x) container_of(x, struct rpcrdma_xprt, rx_xprt) diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 027b37de694b..173e3b0fd7a3 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -3501,7 +3501,7 @@ static struct rpc_xprt *xs_setup_tcp_tls(struct xprt_create *args) transport = container_of(xprt, struct sock_xprt, xprt); xprt->prot = IPPROTO_TCP; - xprt->xprt_class = &xs_tcp_transport; + xprt->xprt_class = &xs_tcp_tls_transport; xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; xprt->bind_timeout = XS_BIND_TO;