mirror of
https://github.com/torvalds/linux.git
synced 2026-11-20 13:03:48 +00:00
Merge tag 'block-7.4-20261029' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux
Pull more block updates from Jens Axboe:
"Bigger than usual for an end-of-merge-window pull request, main reason
being that the MD side I had missed a pull on, and both NVMe and block
got a good pile of LLM driven fixes where some of them may as well
just go in now rather than wait a full cycle. This contains:
- NVMe fixes via Keith:
- nvme-tcp fallout from the parallel queue allocation: initialize
pf_cache_lock and the header digest before requests are set up,
and don't dereference a NULL queue->ctrl on reconnect
- nvme-fabrics queue mapping fixes: reset io_queues on reconnect,
limit each queue type to the online CPUs, and clear a poll map
with no poll queues
- nvme-tcp io_work priority inversion with send_mutex held
(Seiji), error recovery on read_sock failure and skipping TLS
control records while waiting for the ICResp (Martin B), TLS
handshake timeout validation, and an address attribute race
- nvme-rdma NULL metadata_sgl dereference with T10-PI
- nvme-pci early boot hang with the quirks parameter
- nvme-multipath ANA log bounds underflow, and an nvme-fc
discovery reference leak
- nvmet hardening: rdma inline SGL out-of-bounds write and short
connect requests, fc use-after-free in LS create connection and
teardown on admin queue failure, pci-epf transport error
reporting and SGL length checks, zns NUMD wrap, fc stale read
data, auth state and status fixes, ports list and qid_max
locking, rdma double puts and irq-safe locking, and a double
free on metadata SGL allocation failure
- nvmet fixes from Josef: big discard length truncation, NULL
derefs for PCI endpoint targets and debugfs host_traddr, stop
listening when a tcp port is removed, and don't BUG() in
nvme-loop when a scan runs beside a reset
- MD fixes via Yu Kuai:
- raid5 validation: reject too few disks, invalid reshape disks, a
per-device size below one chunk, and an inconsistent reshape
state from the superblock instead of BUG(), plus journal payload
size checks
- raid5 error path and lifetime fixes: extra_page leak and NULL
derefs in free_conf/setup_conf, draining released_stripes before
destroying the cache, RCU for the big_stripe_tree lookup, and
per-array io_unit slab names
- md core: reject v0.90 arrays with too many devices, validate
bblog_size, take reconfig_mutex when enabling PPL, fix the
mddev->private NULL deref when stopping an array under IO, and
don't hand out the array before mddev->kobj is added
- raid10 missing barrier wakeup, raid0 device count validation, a
bogus mddev_unlock() in lbs_store(), and shared helpers
- blk-mq fixes from Josef:
- don't trust the bio cookie in blk_mq_poll()
- keep tag sharing fair after a queue update
- don't bring back flags of a completed request from the timeout
handler
- honor a restarted timer
- initialize the dma state for P2P
- quiesce a queue joining a quiesced tag set
- set up and exit hctxs under tag_list_lock
- free the tag bitmaps in the SRCU callback
- blk-cgroup and friends from Josef:
- keep the old blkg while reassociating a bio
- free struct ioc after a grace period
- don't truncate v1 iops limits
- fix root io.stat on 32-bit
- don't free a blkg with its punt work queued
- flush iolatency enable_work after going offline
- pass dead group I/O counts on and make reset_stats reset them
- don't enter reclaim under rq_qos_mutex
- Other core fixes from Josef:
- a disk use-after-free in sync_bdevs()
- don't losing the status of a failed flush request
- 64-bit sector math for mac partitions
- apply the segment size to the integrity buffer
- blk-iocost: abort a weight transfer on invalid donation weights,
and publish hweights as one atomic value
- Fix a NULL deref and use-after-free in __blk_crypto_evict_key(),
reachable via dm-inlinecrypt
- Fix recursive i_rwsem locking in blkdev_write_iter(), from two
fixes for the same race landing in the window
- Advance the iterator after copy-in for SG_DXFER_TO_FROM_DEV, fixing
an infinite loop in blk_rq_map_user_iov()
- Check the open mode for LOOP_SET_STATUS in the compat ioctl
- rnbd-srv use-after-free and kobject fixes
- drbd timer and req_lock fixes
- zloop cleanups and error path fixes
- cdrom header and kernel-doc cleanups"
* tag 'block-7.4-20261029' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (109 commits)
blk-crypto: fix NULL deref and UAF in __blk_crypto_evict_key()
zloop: check that the zone backing file support ->{read,write}_iter
zloop: fput file after failed zone init
zloop: share more code for opening a zone backing file
zloop: remove a duplicate call to zloop_get_block_size
block: fix kernel-doc warnings in blk-map.c and blk-mq-dma.c
blk-iocost: publish hweights as one atomic value
blk-iocost: abort weight transfer on invalid donation weights
rnbd-srv: publish the sess_dev at the end of OPEN
rnbd-srv: fix use-after-free on two CLOSE messages
block/rnbd-srv: drop the sess_dev kobject unlocked
block/rnbd-srv: don't free the device on a sysfs error
drbd: take req_lock to skip the initial sync as peer
drbd: stop the start_resync timer of a dying device
block: apply the segment size to the integrity buffer
blk-mq: free the bitmaps of tags in the SRCU callback
block: partitions/mac: compute the sectors in 64 bits
blk-mq: set up a queue under tag_list_lock
blk-mq: exit the hctxs of a queue under tag_list_lock
blk-mq: quiesce a queue that joins a quiesced tag set
...
This commit is contained in:
@@ -4792,6 +4792,16 @@ Kernel parameters
|
||||
Example:
|
||||
nvme.quirks=7710:2267:bogus_nid,^identify_cns-9900:7711:broken_msi
|
||||
|
||||
nvme_tcp.tls_handshake_timeout=
|
||||
[NVME] NVMe/TCP host TLS handshake timeout in seconds.
|
||||
Valid values range from 0 to the smaller of
|
||||
UINT_MAX / 1000 and MAX_JIFFY_OFFSET / HZ. Default: 10
|
||||
|
||||
nvmet_tcp.tls_handshake_timeout=
|
||||
[NVME] NVMe/TCP target TLS handshake timeout in seconds.
|
||||
Valid values range from 0 to the smaller of
|
||||
UINT_MAX / 1000 and MAX_JIFFY_OFFSET / HZ. Default: 10
|
||||
|
||||
ohci1394_dma=early [HW,EARLY] enable debugging via the ohci1394 driver.
|
||||
See Documentation/core-api/debugging-via-ohci1394.rst for more
|
||||
info.
|
||||
|
||||
@@ -1366,6 +1366,15 @@ void sync_bdevs(bool wait)
|
||||
old_inode = inode;
|
||||
bdev = I_BDEV(inode);
|
||||
|
||||
/*
|
||||
* The inode does not keep the gendisk of a partition alive, the
|
||||
* device does.
|
||||
*/
|
||||
if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) {
|
||||
spin_lock(&blockdev_superblock->s_inode_list_lock);
|
||||
continue;
|
||||
}
|
||||
|
||||
mutex_lock(&bdev->bd_disk->open_mutex);
|
||||
if (!atomic_read(&bdev->bd_openers)) {
|
||||
; /* skip */
|
||||
@@ -1381,6 +1390,7 @@ void sync_bdevs(bool wait)
|
||||
filemap_fdatawrite(inode->i_mapping);
|
||||
}
|
||||
mutex_unlock(&bdev->bd_disk->open_mutex);
|
||||
put_device(&bdev->bd_device);
|
||||
|
||||
spin_lock(&blockdev_superblock->s_inode_list_lock);
|
||||
}
|
||||
|
||||
+29
-10
@@ -121,6 +121,11 @@ static void blkg_free_workfn(struct work_struct *work)
|
||||
struct request_queue *q = blkg->q;
|
||||
int i;
|
||||
|
||||
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
|
||||
/* It may have been queued for bios which an earlier run has taken */
|
||||
cancel_work_sync(&blkg->async_bio_work);
|
||||
#endif
|
||||
|
||||
/*
|
||||
* pd_free_fn() can also be called from blkcg_deactivate_policy(),
|
||||
* in order to make sure pd_free_fn() is called in order, the deletion
|
||||
@@ -218,7 +223,7 @@ static void blkg_async_bio_workfn(struct work_struct *work)
|
||||
struct blk_plug plug;
|
||||
bool need_plug = false;
|
||||
|
||||
/* as long as there are pending bios, @blkg can't go away */
|
||||
/* blkg_free_workfn() waits for us */
|
||||
spin_lock(&blkg->async_bio_lock);
|
||||
bio_list_merge_init(&bios, &blkg->async_bios);
|
||||
spin_unlock(&blkg->async_bio_lock);
|
||||
@@ -245,10 +250,16 @@ void blkcg_punt_bio_submit(struct bio *bio)
|
||||
struct blkcg_gq *blkg = bio->bi_blkg;
|
||||
|
||||
if (blkg->parent) {
|
||||
/*
|
||||
* Once it is on the list, a run which is under way can submit
|
||||
* @bio, and its reference goes when it completes.
|
||||
*/
|
||||
blkg_get(blkg);
|
||||
spin_lock(&blkg->async_bio_lock);
|
||||
bio_list_add(&blkg->async_bios, bio);
|
||||
spin_unlock(&blkg->async_bio_lock);
|
||||
queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
|
||||
blkg_put(blkg);
|
||||
} else {
|
||||
/* never bounce for the root cgroup */
|
||||
submit_bio(bio);
|
||||
@@ -873,7 +884,11 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
|
||||
parent = blkcg_parent(parent);
|
||||
}
|
||||
|
||||
/* Drop locks to do new blkg allocation with GFP_KERNEL. */
|
||||
/*
|
||||
* Drop the spinlock to allocate the new blkg. rq_qos_mutex and
|
||||
* blkcg_mutex stay held, and both are taken with the queue
|
||||
* frozen elsewhere: GFP_NOIO.
|
||||
*/
|
||||
spin_unlock_irq(&q->queue_lock);
|
||||
|
||||
new_blkg = blkg_alloc(pos, disk, GFP_NOIO);
|
||||
@@ -882,7 +897,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
|
||||
goto fail_exit;
|
||||
}
|
||||
|
||||
if (radix_tree_preload(GFP_KERNEL)) {
|
||||
if (radix_tree_preload(GFP_NOIO)) {
|
||||
blkg_free(new_blkg);
|
||||
ret = -ENOMEM;
|
||||
goto fail_exit;
|
||||
@@ -1118,11 +1133,11 @@ static void blkcg_fill_root_iostats(void)
|
||||
cpu_dkstats->ios[STAT_DISCARD];
|
||||
// convert sectors to bytes
|
||||
tmp.bytes[BLKG_IOSTAT_READ] +=
|
||||
cpu_dkstats->sectors[STAT_READ] << 9;
|
||||
(u64)cpu_dkstats->sectors[STAT_READ] << 9;
|
||||
tmp.bytes[BLKG_IOSTAT_WRITE] +=
|
||||
cpu_dkstats->sectors[STAT_WRITE] << 9;
|
||||
(u64)cpu_dkstats->sectors[STAT_WRITE] << 9;
|
||||
tmp.bytes[BLKG_IOSTAT_DISCARD] +=
|
||||
cpu_dkstats->sectors[STAT_DISCARD] << 9;
|
||||
(u64)cpu_dkstats->sectors[STAT_DISCARD] << 9;
|
||||
}
|
||||
|
||||
flags = u64_stats_update_begin_irqsave(&blkg->iostat.sync);
|
||||
@@ -1637,7 +1652,8 @@ retry:
|
||||
if (!pd) {
|
||||
/*
|
||||
* GFP_NOWAIT failed. Free the existing one and
|
||||
* prealloc for @blkg w/ GFP_KERNEL.
|
||||
* prealloc for @blkg w/ GFP_NOIO: blkcg_mutex is held,
|
||||
* and a bio based queue has not been frozen above.
|
||||
*/
|
||||
if (pinned_blkg)
|
||||
blkg_put(pinned_blkg);
|
||||
@@ -1649,7 +1665,7 @@ retry:
|
||||
if (pd_prealloc)
|
||||
pol->pd_free_fn(pd_prealloc);
|
||||
pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,
|
||||
GFP_KERNEL);
|
||||
GFP_NOIO);
|
||||
if (pd_prealloc)
|
||||
goto retry;
|
||||
else
|
||||
@@ -2139,8 +2155,7 @@ static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
|
||||
void bio_associate_blkg_from_css(struct bio *bio,
|
||||
struct cgroup_subsys_state *css)
|
||||
{
|
||||
if (bio->bi_blkg)
|
||||
blkg_put(bio->bi_blkg);
|
||||
struct blkcg_gq *old_blkg = bio->bi_blkg;
|
||||
|
||||
if (css && css->parent) {
|
||||
bio->bi_blkg = blkg_tryget_closest(bio, css);
|
||||
@@ -2148,6 +2163,10 @@ void bio_associate_blkg_from_css(struct bio *bio,
|
||||
blkg_get(bdev_get_queue(bio->bi_bdev)->root_blkg);
|
||||
bio->bi_blkg = bdev_get_queue(bio->bi_bdev)->root_blkg;
|
||||
}
|
||||
|
||||
/* @old_blkg may be the only thing which pins @css */
|
||||
if (old_blkg)
|
||||
blkg_put(old_blkg);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
|
||||
|
||||
|
||||
+2
-4
@@ -76,11 +76,9 @@ struct blkcg_gq {
|
||||
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
|
||||
spinlock_t async_bio_lock;
|
||||
struct bio_list async_bios;
|
||||
struct work_struct async_bio_work;
|
||||
#endif
|
||||
union {
|
||||
struct work_struct async_bio_work;
|
||||
struct work_struct free_work;
|
||||
};
|
||||
struct work_struct free_work;
|
||||
|
||||
atomic_t use_delay;
|
||||
atomic64_t delay_nsec;
|
||||
|
||||
@@ -524,6 +524,12 @@ bool blk_crypto_fallback_bio_prep(struct bio *bio)
|
||||
|
||||
int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key)
|
||||
{
|
||||
/*
|
||||
* Pairs with smp_store_release() in
|
||||
* blk_crypto_fallback_start_using_mode()
|
||||
*/
|
||||
if (!smp_load_acquire(&tfms_inited[key->crypto_cfg.crypto_mode]))
|
||||
return 0;
|
||||
return __blk_crypto_evict_key(blk_crypto_fallback_profile, key);
|
||||
}
|
||||
|
||||
|
||||
+7
-1
@@ -212,7 +212,13 @@ static enum rq_end_io_ret flush_end_io(struct request *flush_rq,
|
||||
spin_lock_irqsave(&fq->mq_flush_lock, flags);
|
||||
|
||||
if (!req_ref_put_and_test(flush_rq)) {
|
||||
fq->rq_status = error;
|
||||
/*
|
||||
* Remember the status of the flush for whoever drops the last
|
||||
* reference. blk_mq_put_rq_ref() gets here with BLK_STS_OK and
|
||||
* must not wipe it out.
|
||||
*/
|
||||
if (error != BLK_STS_OK)
|
||||
fq->rq_status = error;
|
||||
spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
|
||||
return RQ_END_IO_NONE;
|
||||
}
|
||||
|
||||
+96
-23
@@ -173,6 +173,7 @@
|
||||
*/
|
||||
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/timer.h>
|
||||
#include <linux/time64.h>
|
||||
@@ -474,8 +475,8 @@ struct ioc_gq {
|
||||
*
|
||||
* When an idle cgroup becomes active its `active` goes from 0 to
|
||||
* `weight`. `inuse` is the surplus adjusted active weight.
|
||||
* `active` and `inuse` are used to calculate `hweight_active` and
|
||||
* `hweight_inuse`.
|
||||
* `active` and `inuse` determine the hierarchical weights cached in
|
||||
* `hweight_cache`.
|
||||
*
|
||||
* `last_inuse` remembers `inuse` while an iocg is idle to persist
|
||||
* surplus adjustments.
|
||||
@@ -522,7 +523,9 @@ struct ioc_gq {
|
||||
u64 child_active_sum;
|
||||
u64 child_inuse_sum;
|
||||
u64 child_adjusted_sum;
|
||||
int hweight_gen;
|
||||
/* Generation in bits 63:32; active - 1 and inuse - 1 in 31:16/15:0. */
|
||||
atomic64_t hweight_cache;
|
||||
/* Donation workspace, protected by ioc->lock. */
|
||||
u32 hweight_active;
|
||||
u32 hweight_inuse;
|
||||
u32 hweight_donating;
|
||||
@@ -729,6 +732,54 @@ static void iocg_commit_bio(struct ioc_gq *iocg, struct bio *bio,
|
||||
put_cpu_ptr(gcs);
|
||||
}
|
||||
|
||||
/*
|
||||
* The hierarchical weights are in [1, WEIGHT_ONE] and the generation is a
|
||||
* 32-bit counter. Pack generation and both weights into one 64-bit value so
|
||||
* that a reader or a refresher always observes a complete snapshot:
|
||||
*
|
||||
* bits 63:32 generation
|
||||
* bits 31:16 hweight_active - 1
|
||||
* bits 15:0 hweight_inuse - 1
|
||||
*/
|
||||
#define HWEIGHT_GEN_MASK GENMASK_ULL(63, 32)
|
||||
#define HWEIGHT_ACTIVE_MASK GENMASK_ULL(31, 16)
|
||||
#define HWEIGHT_INUSE_MASK GENMASK_ULL(15, 0)
|
||||
|
||||
static inline u64 hweight_cache_pack(u32 gen, u32 active, u32 inuse)
|
||||
{
|
||||
return FIELD_PREP(HWEIGHT_GEN_MASK, gen) |
|
||||
FIELD_PREP(HWEIGHT_ACTIVE_MASK, active - 1) |
|
||||
FIELD_PREP(HWEIGHT_INUSE_MASK, inuse - 1);
|
||||
}
|
||||
|
||||
static inline void hweight_cache_unpack(u64 cache, u32 *genp, u32 *activep,
|
||||
u32 *inusep)
|
||||
{
|
||||
if (genp)
|
||||
*genp = FIELD_GET(HWEIGHT_GEN_MASK, cache);
|
||||
if (activep)
|
||||
*activep = FIELD_GET(HWEIGHT_ACTIVE_MASK, cache) + 1;
|
||||
if (inusep)
|
||||
*inusep = FIELD_GET(HWEIGHT_INUSE_MASK, cache) + 1;
|
||||
}
|
||||
|
||||
/* Decode the published weights without refreshing the cache. */
|
||||
static inline void iocg_hweight_cache(struct ioc_gq *iocg, u32 *activep,
|
||||
u32 *inusep)
|
||||
{
|
||||
hweight_cache_unpack(atomic64_read(&iocg->hweight_cache), NULL,
|
||||
activep, inusep);
|
||||
}
|
||||
|
||||
/* Decode the cached inuse weight; only used by tracepoints. */
|
||||
static inline u32 iocg_hweight_cache_inuse(struct ioc_gq *iocg)
|
||||
{
|
||||
u32 inuse;
|
||||
|
||||
iocg_hweight_cache(iocg, NULL, &inuse);
|
||||
return inuse;
|
||||
}
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/iocost.h>
|
||||
|
||||
@@ -1159,12 +1210,15 @@ static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep
|
||||
{
|
||||
struct ioc *ioc = iocg->ioc;
|
||||
int lvl;
|
||||
u32 hwa, hwi;
|
||||
int ioc_gen;
|
||||
u32 hwa, hwi, ioc_gen, cache_gen;
|
||||
u64 cache;
|
||||
|
||||
/* hot path - if uptodate, use cached */
|
||||
ioc_gen = atomic_read(&ioc->hweight_gen);
|
||||
if (ioc_gen == iocg->hweight_gen)
|
||||
ioc_gen = (u32)atomic_read(&ioc->hweight_gen);
|
||||
|
||||
/* hot path - if uptodate, use the cached snapshot */
|
||||
cache = atomic64_read(&iocg->hweight_cache);
|
||||
hweight_cache_unpack(cache, &cache_gen, &hwa, &hwi);
|
||||
if (cache_gen == ioc_gen)
|
||||
goto out;
|
||||
|
||||
/*
|
||||
@@ -1199,14 +1253,20 @@ static void current_hweight(struct ioc_gq *iocg, u32 *hw_activep, u32 *hw_inusep
|
||||
hwi = div64_u64((u64)hwi * inuse, inuse_sum);
|
||||
}
|
||||
|
||||
iocg->hweight_active = max_t(u32, hwa, 1);
|
||||
iocg->hweight_inuse = max_t(u32, hwi, 1);
|
||||
iocg->hweight_gen = ioc_gen;
|
||||
hwa = max_t(u32, hwa, 1);
|
||||
hwi = max_t(u32, hwi, 1);
|
||||
/*
|
||||
* Publish the generation and both weights as one atomic snapshot.
|
||||
* A concurrent refresher may overwrite this with its own snapshot,
|
||||
* but the generation travels with the weights so the next reader
|
||||
* detects the mismatch and refreshes again.
|
||||
*/
|
||||
atomic64_set(&iocg->hweight_cache, hweight_cache_pack(ioc_gen, hwa, hwi));
|
||||
out:
|
||||
if (hw_activep)
|
||||
*hw_activep = iocg->hweight_active;
|
||||
*hw_activep = hwa;
|
||||
if (hw_inusep)
|
||||
*hw_inusep = iocg->hweight_inuse;
|
||||
*hw_inusep = hwi;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1254,7 +1314,8 @@ static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
|
||||
{
|
||||
struct ioc *ioc = iocg->ioc;
|
||||
u64 __maybe_unused last_period, cur_period;
|
||||
u64 vtime, vtarget;
|
||||
u64 vtime, vtarget, cache;
|
||||
u32 active, inuse;
|
||||
int i;
|
||||
|
||||
/*
|
||||
@@ -1304,11 +1365,16 @@ static bool iocg_activate(struct ioc_gq *iocg, struct ioc_now *now)
|
||||
vtime = vtarget;
|
||||
|
||||
/*
|
||||
* Activate, propagate weight and start period timer if not
|
||||
* running. Reset hweight_gen to avoid accidental match from
|
||||
* wrapping.
|
||||
* Activate, propagate weight and start period timer if not running.
|
||||
* Invalidate the cached generation to avoid an accidental match from
|
||||
* wrapping. Keep the last published weights so that the activation
|
||||
* tracepoint can report them; the next current_hweight() recomputes.
|
||||
*/
|
||||
iocg->hweight_gen = atomic_read(&ioc->hweight_gen) - 1;
|
||||
cache = atomic64_read(&iocg->hweight_cache);
|
||||
hweight_cache_unpack(cache, NULL, &active, &inuse);
|
||||
atomic64_set(&iocg->hweight_cache,
|
||||
hweight_cache_pack((u32)(atomic_read(&ioc->hweight_gen) - 1),
|
||||
active, inuse));
|
||||
list_add(&iocg->active_list, &ioc->active_iocgs);
|
||||
|
||||
propagate_weights(iocg, iocg->weight,
|
||||
@@ -1961,6 +2027,7 @@ static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
|
||||
pr_cont(": active=%u donating=%u after=%u\n",
|
||||
iocg->hweight_active, iocg->hweight_donating,
|
||||
iocg->hweight_after_donation);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2053,12 +2120,13 @@ static void transfer_surpluses(struct list_head *surpluses, struct ioc_now *now)
|
||||
|
||||
TRACE_IOCG_PATH(inuse_transfer, iocg, now,
|
||||
iocg->inuse, inuse,
|
||||
iocg->hweight_inuse,
|
||||
iocg_hweight_cache_inuse(iocg),
|
||||
iocg->hweight_after_donation);
|
||||
|
||||
__propagate_weights(iocg, iocg->active, inuse, true, now);
|
||||
}
|
||||
|
||||
out:
|
||||
/* walk list should be dissolved after use */
|
||||
list_for_each_entry_safe(iocg, tiocg, &inner_walk, walk_list)
|
||||
list_del_init(&iocg->walk_list);
|
||||
@@ -2365,7 +2433,7 @@ static void ioc_timer_fn(struct timer_list *timer)
|
||||
*/
|
||||
TRACE_IOCG_PATH(inuse_shortage, iocg, &now,
|
||||
iocg->inuse, iocg->active,
|
||||
iocg->hweight_inuse, new_hwi);
|
||||
old_hwi, new_hwi);
|
||||
|
||||
__propagate_weights(iocg, iocg->active,
|
||||
iocg->active, true, &now);
|
||||
@@ -2920,7 +2988,8 @@ static void ioc_rqos_exit(struct rq_qos *rqos)
|
||||
|
||||
timer_shutdown_sync(&ioc->timer);
|
||||
free_percpu(ioc->pcpu_stat);
|
||||
kfree(ioc);
|
||||
/* blkcg_print_blkgs() gets here from a pd with only RCU held */
|
||||
kfree_rcu_mightsleep(ioc);
|
||||
}
|
||||
|
||||
static const struct rq_qos_ops ioc_rqos_ops = {
|
||||
@@ -2937,11 +3006,12 @@ static int blk_iocost_init(struct gendisk *disk)
|
||||
struct ioc *ioc;
|
||||
int i, cpu, ret;
|
||||
|
||||
ioc = kzalloc_obj(*ioc);
|
||||
/* The caller holds rq_qos_mutex, which is taken with the queue frozen */
|
||||
ioc = kzalloc_obj(*ioc, GFP_NOIO);
|
||||
if (!ioc)
|
||||
return -ENOMEM;
|
||||
|
||||
ioc->pcpu_stat = alloc_percpu(struct ioc_pcpu_stat);
|
||||
ioc->pcpu_stat = alloc_percpu_gfp(struct ioc_pcpu_stat, GFP_NOIO);
|
||||
if (!ioc->pcpu_stat) {
|
||||
kfree(ioc);
|
||||
return -ENOMEM;
|
||||
@@ -3052,6 +3122,9 @@ static void ioc_pd_init(struct blkg_policy_data *pd)
|
||||
INIT_LIST_HEAD(&iocg->active_list);
|
||||
INIT_LIST_HEAD(&iocg->walk_list);
|
||||
INIT_LIST_HEAD(&iocg->surplus_list);
|
||||
atomic64_set(&iocg->hweight_cache,
|
||||
hweight_cache_pack((u32)(atomic_read(&ioc->hweight_gen) - 1),
|
||||
WEIGHT_ONE, WEIGHT_ONE));
|
||||
iocg->hweight_active = WEIGHT_ONE;
|
||||
iocg->hweight_inuse = WEIGHT_ONE;
|
||||
|
||||
|
||||
@@ -637,8 +637,9 @@ static void blkcg_iolatency_exit(struct rq_qos *rqos)
|
||||
struct blk_iolatency *blkiolat = BLKIOLATENCY(rqos);
|
||||
|
||||
timer_shutdown_sync(&blkiolat->timer);
|
||||
flush_work(&blkiolat->enable_work);
|
||||
blkcg_deactivate_policy(rqos->disk, &blkcg_policy_iolatency);
|
||||
/* iolatency_pd_offline() schedules it for the last group with a target */
|
||||
flush_work(&blkiolat->enable_work);
|
||||
kfree(blkiolat);
|
||||
}
|
||||
|
||||
@@ -760,7 +761,8 @@ static int blk_iolatency_init(struct gendisk *disk)
|
||||
struct blk_iolatency *blkiolat;
|
||||
int ret;
|
||||
|
||||
blkiolat = kzalloc_obj(*blkiolat);
|
||||
/* The caller holds rq_qos_mutex, which is taken with the queue frozen */
|
||||
blkiolat = kzalloc_obj(*blkiolat, GFP_NOIO);
|
||||
if (!blkiolat)
|
||||
return -ENOMEM;
|
||||
|
||||
|
||||
+1
-1
@@ -236,6 +236,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data,
|
||||
ret = bio_copy_from_iter(bio, &iter2);
|
||||
if (ret)
|
||||
goto cleanup;
|
||||
iov_iter_advance(iter, bio->bi_iter.bi_size);
|
||||
} else {
|
||||
if (bmd->is_our_pages)
|
||||
zero_fill_bio(bio);
|
||||
@@ -356,7 +357,6 @@ static void bio_copy_kern_endio_read(struct bio *bio)
|
||||
* @rq: request to fill
|
||||
* @data: pointer to buffer to copy
|
||||
* @len: length in bytes
|
||||
* @op: bio/request operation
|
||||
* @gfp_mask: allocation flags for bio and page allocation
|
||||
*
|
||||
* copy the kernel address into a bio suitable for io to a block
|
||||
|
||||
+4
-3
@@ -181,6 +181,8 @@ static bool blk_dma_map_iter_start(struct request *req, struct device *dma_dev,
|
||||
switch (pci_p2pdma_state(&iter->p2pdma, dma_dev,
|
||||
phys_to_page(vec.paddr))) {
|
||||
case PCI_P2PDMA_MAP_BUS_ADDR:
|
||||
/* don't leave the state of an earlier mapping to the caller */
|
||||
memset(state, 0, sizeof(*state));
|
||||
return blk_dma_map_bus(iter, &vec);
|
||||
case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
|
||||
/*
|
||||
@@ -213,7 +215,7 @@ static bool blk_dma_map_iter_start(struct request *req, struct device *dma_dev,
|
||||
* at unmap time, @iter is only needed at map time.
|
||||
*
|
||||
* Returns %false if there is no segment to map, including due to an error, or
|
||||
* %true ft it did map a segment.
|
||||
* %true if it did map a segment.
|
||||
*
|
||||
* If a segment was mapped, the DMA address for it is returned in @iter.addr and
|
||||
* the length in @iter.len. If no segment was mapped the status code is
|
||||
@@ -243,7 +245,7 @@ EXPORT_SYMBOL_GPL(blk_rq_dma_map_iter_start);
|
||||
* arguments.
|
||||
*
|
||||
* Returns %false if there is no segment to map, including due to an error, or
|
||||
* %true ft it did map a segment.
|
||||
* %true if it did map a segment.
|
||||
*
|
||||
* If a segment was mapped, the DMA address for it is returned in @iter.addr and
|
||||
* the length in @iter.len. If no segment was mapped the status code is
|
||||
@@ -361,7 +363,6 @@ EXPORT_SYMBOL_GPL(blk_rq_integrity_dma_map_iter_start);
|
||||
* a request
|
||||
* @req: request to map
|
||||
* @dma_dev: device to map to
|
||||
* @state: DMA IOVA state
|
||||
* @iter: block layer DMA iterator
|
||||
*
|
||||
* Iterate to the next integrity mapping after a previous call to
|
||||
|
||||
+4
-4
@@ -611,17 +611,17 @@ static void blk_mq_free_tags_callback(struct rcu_head *head)
|
||||
kmemleak_free(page_address(page));
|
||||
__free_pages(page, page->private);
|
||||
}
|
||||
/* An iterator may walk the bitmaps for as long as it may see the tags */
|
||||
sbitmap_queue_free(&tags->bitmap_tags);
|
||||
sbitmap_queue_free(&tags->breserved_tags);
|
||||
kfree(tags);
|
||||
}
|
||||
|
||||
void blk_mq_free_tags(struct blk_mq_tags *tags)
|
||||
{
|
||||
sbitmap_queue_free(&tags->bitmap_tags);
|
||||
sbitmap_queue_free(&tags->breserved_tags);
|
||||
|
||||
/* if tags pages is not allocated yet, free tags directly */
|
||||
if (list_empty(&tags->page_list)) {
|
||||
kfree(tags);
|
||||
blk_mq_free_tags_callback(&tags->rcu_head);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
+70
-8
@@ -336,12 +336,15 @@ void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
|
||||
{
|
||||
struct request_queue *q;
|
||||
|
||||
spin_lock(&set->quiesce_lock);
|
||||
set->quiesce_depth++;
|
||||
rcu_read_lock();
|
||||
list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
|
||||
if (!blk_queue_skip_tagset_quiesce(q))
|
||||
blk_mq_quiesce_queue_nowait(q);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
spin_unlock(&set->quiesce_lock);
|
||||
|
||||
blk_mq_wait_quiesce_done(set);
|
||||
}
|
||||
@@ -351,12 +354,16 @@ void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
|
||||
{
|
||||
struct request_queue *q;
|
||||
|
||||
spin_lock(&set->quiesce_lock);
|
||||
if (set->quiesce_depth)
|
||||
set->quiesce_depth--;
|
||||
rcu_read_lock();
|
||||
list_for_each_entry_rcu(q, &set->tag_list, tag_set_list) {
|
||||
if (!blk_queue_skip_tagset_quiesce(q))
|
||||
blk_mq_unquiesce_queue(q);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
spin_unlock(&set->quiesce_lock);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
|
||||
|
||||
@@ -1639,7 +1646,7 @@ EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
|
||||
|
||||
static void blk_mq_rq_timed_out(struct request *req)
|
||||
{
|
||||
req->rq_flags |= RQF_TIMED_OUT;
|
||||
blk_rq_change_timed_out(req, true);
|
||||
if (req->q->mq_ops->timeout) {
|
||||
enum blk_eh_timer_return ret;
|
||||
|
||||
@@ -1710,8 +1717,11 @@ static bool blk_mq_handle_expired(struct request *rq, void *priv)
|
||||
{
|
||||
struct blk_expired_data *expired = priv;
|
||||
|
||||
if (blk_mq_req_expired(rq, expired))
|
||||
if (blk_mq_req_expired(rq, expired)) {
|
||||
blk_mq_rq_timed_out(rq);
|
||||
/* If its timer has been started again, count the new deadline */
|
||||
blk_mq_req_expired(rq, expired);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -4334,7 +4344,8 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)
|
||||
{
|
||||
struct blk_mq_tag_set *set = q->tag_set;
|
||||
|
||||
mutex_lock(&set->tag_list_lock);
|
||||
lockdep_assert_held(&set->tag_list_lock);
|
||||
|
||||
list_del_rcu(&q->tag_set_list);
|
||||
if (list_is_singular(&set->tag_list)) {
|
||||
/* just transitioned to unshared */
|
||||
@@ -4342,13 +4353,14 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)
|
||||
/* update existing queue */
|
||||
blk_mq_update_tag_set_shared(set, false);
|
||||
}
|
||||
mutex_unlock(&set->tag_list_lock);
|
||||
}
|
||||
|
||||
static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
|
||||
struct request_queue *q)
|
||||
{
|
||||
mutex_lock(&set->tag_list_lock);
|
||||
unsigned int i;
|
||||
|
||||
lockdep_assert_held(&set->tag_list_lock);
|
||||
|
||||
/*
|
||||
* Check to see if we're transitioning to shared (from 1 to 2 queues).
|
||||
@@ -4361,9 +4373,18 @@ static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
|
||||
}
|
||||
if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
|
||||
queue_set_hctx_shared(q, true);
|
||||
list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
|
||||
|
||||
mutex_unlock(&set->tag_list_lock);
|
||||
/*
|
||||
* A queue that joins a quiesced tag set is quiesced like the others,
|
||||
* and blk_mq_unquiesce_tagset() finds what it is going to undo.
|
||||
*/
|
||||
spin_lock(&set->quiesce_lock);
|
||||
list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
|
||||
if (!blk_queue_skip_tagset_quiesce(q)) {
|
||||
for (i = 0; i < set->quiesce_depth; i++)
|
||||
blk_mq_quiesce_queue_nowait(q);
|
||||
}
|
||||
spin_unlock(&set->quiesce_lock);
|
||||
}
|
||||
|
||||
/* All allocations will be freed in release handler of q->mq_kobj */
|
||||
@@ -4640,6 +4661,8 @@ static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
|
||||
int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
|
||||
struct request_queue *q)
|
||||
{
|
||||
unsigned int memflags;
|
||||
|
||||
/* mark the queue as mq asap */
|
||||
q->mq_ops = set->ops;
|
||||
|
||||
@@ -4658,6 +4681,16 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
|
||||
INIT_LIST_HEAD(&q->unused_hctx_list);
|
||||
spin_lock_init(&q->unused_hctx_lock);
|
||||
|
||||
/*
|
||||
* Keep blk_mq_update_nr_hw_queues() away until the queue is on
|
||||
* set->tag_list: it changes the number of hardware queues, the tags and
|
||||
* the maps, and blk_mq_map_swqueue() allocates and frees tags of the set.
|
||||
* An update can be part of the error handling of a driver, so it must
|
||||
* not have to wait for somebody who waits for I/O.
|
||||
*/
|
||||
memflags = memalloc_noio_save();
|
||||
mutex_lock(&set->tag_list_lock);
|
||||
|
||||
blk_mq_realloc_hw_ctxs(set, q);
|
||||
if (!q->nr_hw_queues)
|
||||
goto err_hctxs;
|
||||
@@ -4678,9 +4711,14 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
|
||||
blk_mq_init_cpu_queues(q, set->nr_hw_queues);
|
||||
blk_mq_map_swqueue(q);
|
||||
blk_mq_add_queue_tag_set(set, q);
|
||||
|
||||
mutex_unlock(&set->tag_list_lock);
|
||||
memalloc_noio_restore(memflags);
|
||||
return 0;
|
||||
|
||||
err_hctxs:
|
||||
mutex_unlock(&set->tag_list_lock);
|
||||
memalloc_noio_restore(memflags);
|
||||
blk_mq_release(q);
|
||||
err_exit:
|
||||
q->mq_ops = NULL;
|
||||
@@ -4693,10 +4731,16 @@ void blk_mq_exit_queue(struct request_queue *q)
|
||||
{
|
||||
struct blk_mq_tag_set *set = q->tag_set;
|
||||
|
||||
/*
|
||||
* As long as the queue is on set->tag_list, its hctxs belong to
|
||||
* blk_mq_update_nr_hw_queues() as well, which exits and replaces them.
|
||||
*/
|
||||
mutex_lock(&set->tag_list_lock);
|
||||
/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
|
||||
blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
|
||||
/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
|
||||
blk_mq_del_queue_tag_set(q);
|
||||
mutex_unlock(&set->tag_list_lock);
|
||||
}
|
||||
|
||||
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
|
||||
@@ -4930,6 +4974,8 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
|
||||
|
||||
mutex_init(&set->tag_list_lock);
|
||||
INIT_LIST_HEAD(&set->tag_list);
|
||||
spin_lock_init(&set->quiesce_lock);
|
||||
set->quiesce_depth = 0;
|
||||
|
||||
return 0;
|
||||
|
||||
@@ -5189,6 +5235,9 @@ fallback:
|
||||
goto fallback;
|
||||
}
|
||||
blk_mq_map_swqueue(q);
|
||||
/* The new hctxs don't know yet that they share their tags */
|
||||
if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
|
||||
queue_set_hctx_shared(q, true);
|
||||
}
|
||||
switch_back:
|
||||
/* The blk_mq_elv_switch_back unfreezes queue for us. */
|
||||
@@ -5250,9 +5299,22 @@ static int blk_hctx_poll(struct request_queue *q, struct blk_mq_hw_ctx *hctx,
|
||||
int blk_mq_poll(struct request_queue *q, blk_qc_t cookie,
|
||||
struct io_comp_batch *iob, unsigned int flags)
|
||||
{
|
||||
struct blk_mq_hw_ctx *hctx;
|
||||
|
||||
if (!blk_mq_can_poll(q))
|
||||
return 0;
|
||||
return blk_hctx_poll(q, q->queue_hw_ctx[cookie], iob, flags);
|
||||
|
||||
/*
|
||||
* The cookie is no more than a hint. iocb_bio_iopoll() may have read it
|
||||
* from a bio that was reused for another queue right afterwards, and
|
||||
* zone write plugging keeps the number of segments in the same field.
|
||||
*/
|
||||
if (cookie >= q->nr_hw_queues)
|
||||
return 0;
|
||||
hctx = q->queue_hw_ctx[cookie];
|
||||
if (hctx->type != HCTX_TYPE_POLL)
|
||||
return 0;
|
||||
return blk_hctx_poll(q, hctx, iob, flags);
|
||||
}
|
||||
|
||||
int blk_rq_poll(struct request *rq, struct io_comp_batch *iob,
|
||||
|
||||
+25
-3
@@ -1322,7 +1322,8 @@ static int blk_throtl_init(struct gendisk *disk)
|
||||
unsigned int memflags;
|
||||
int ret;
|
||||
|
||||
td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node);
|
||||
/* The caller holds rq_qos_mutex, which is taken with the queue frozen */
|
||||
td = kzalloc_node(sizeof(*td), GFP_NOIO, q->node);
|
||||
if (!td)
|
||||
return -ENOMEM;
|
||||
|
||||
@@ -1386,7 +1387,8 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of,
|
||||
if (is_u64)
|
||||
*(u64 *)((void *)tg + of_cft(of)->private) = v;
|
||||
else
|
||||
*(unsigned int *)((void *)tg + of_cft(of)->private) = v;
|
||||
*(unsigned int *)((void *)tg + of_cft(of)->private) =
|
||||
min_t(u64, v, UINT_MAX);
|
||||
|
||||
tg_conf_updated(tg, false);
|
||||
ret = 0;
|
||||
@@ -1669,7 +1671,26 @@ static void tg_flush_bios(struct throtl_grp *tg)
|
||||
|
||||
static void throtl_pd_offline(struct blkg_policy_data *pd)
|
||||
{
|
||||
tg_flush_bios(pd_to_tg(pd));
|
||||
struct throtl_grp *tg = pd_to_tg(pd);
|
||||
struct throtl_grp *parent = blkg_to_tg(tg_to_blkg(tg)->parent);
|
||||
|
||||
tg_flush_bios(tg);
|
||||
|
||||
/* the recursive files of the ancestors go on counting what it did */
|
||||
if (parent) {
|
||||
blkg_rwstat_add_aux(&parent->stat_bytes, &tg->stat_bytes);
|
||||
blkg_rwstat_add_aux(&parent->stat_ios, &tg->stat_ios);
|
||||
blkg_rwstat_reset(&tg->stat_bytes);
|
||||
blkg_rwstat_reset(&tg->stat_ios);
|
||||
}
|
||||
}
|
||||
|
||||
static void throtl_pd_reset_stats(struct blkg_policy_data *pd)
|
||||
{
|
||||
struct throtl_grp *tg = pd_to_tg(pd);
|
||||
|
||||
blkg_rwstat_reset(&tg->stat_bytes);
|
||||
blkg_rwstat_reset(&tg->stat_ios);
|
||||
}
|
||||
|
||||
struct blkcg_policy blkcg_policy_throtl = {
|
||||
@@ -1681,6 +1702,7 @@ struct blkcg_policy blkcg_policy_throtl = {
|
||||
.pd_online_fn = throtl_pd_online,
|
||||
.pd_offline_fn = throtl_pd_offline,
|
||||
.pd_free_fn = throtl_pd_free,
|
||||
.pd_reset_stats_fn = throtl_pd_reset_stats,
|
||||
};
|
||||
|
||||
static void tg_cancel_writeback_bios(struct throtl_grp *tg,
|
||||
|
||||
+2
-1
@@ -137,7 +137,8 @@ void blk_add_timer(struct request *req)
|
||||
if (!req->timeout)
|
||||
req->timeout = q->rq_timeout;
|
||||
|
||||
req->rq_flags &= ~RQF_TIMED_OUT;
|
||||
if (req->rq_flags & RQF_TIMED_OUT)
|
||||
blk_rq_change_timed_out(req, false);
|
||||
|
||||
expiry = jiffies + req->timeout;
|
||||
WRITE_ONCE(req->deadline, expiry);
|
||||
|
||||
+14
@@ -332,6 +332,20 @@ static inline void bio_integrity_free(struct bio *bio)
|
||||
unsigned long blk_rq_timeout(unsigned long timeout);
|
||||
void blk_add_timer(struct request *req);
|
||||
|
||||
/*
|
||||
* The timeout handler changes RQF_TIMED_OUT in a request which it does not
|
||||
* own: whoever completes it changes rq_flags at the same time, with plain
|
||||
* loads and stores. Never write back a flag that has changed under us.
|
||||
*/
|
||||
static inline void blk_rq_change_timed_out(struct request *rq, bool set)
|
||||
{
|
||||
req_flags_t old = READ_ONCE(rq->rq_flags), new;
|
||||
|
||||
do {
|
||||
new = set ? old | RQF_TIMED_OUT : old & ~RQF_TIMED_OUT;
|
||||
} while (!try_cmpxchg(&rq->rq_flags, &old, new));
|
||||
}
|
||||
|
||||
enum bio_merge_status {
|
||||
BIO_MERGE_OK,
|
||||
BIO_MERGE_NONE,
|
||||
|
||||
@@ -812,9 +812,7 @@ static ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
|
||||
} else {
|
||||
ssize_t ret2;
|
||||
|
||||
inode_lock_shared(bd_inode);
|
||||
ret2 = blkdev_buffered_write(iocb, from);
|
||||
inode_unlock_shared(bd_inode);
|
||||
ret = direct_write_fallback(iocb, from, ret,
|
||||
ret2);
|
||||
need_sync = true;
|
||||
|
||||
@@ -97,8 +97,8 @@ int mac_partition(struct parsed_partitions *state)
|
||||
if (be16_to_cpu(part->signature) != MAC_PARTITION_MAGIC)
|
||||
break;
|
||||
put_partition(state, slot,
|
||||
be32_to_cpu(part->start_block) * (secsize/512),
|
||||
be32_to_cpu(part->block_count) * (secsize/512));
|
||||
(sector_t)be32_to_cpu(part->start_block) * (secsize/512),
|
||||
(sector_t)be32_to_cpu(part->block_count) * (secsize/512));
|
||||
|
||||
if (!strncasecmp(part->type, "Linux_RAID", 10))
|
||||
state->parts[slot].flags = ADDPART_FLAG_RAID;
|
||||
|
||||
@@ -2163,6 +2163,7 @@ void drbd_destroy_device(struct kref *kref)
|
||||
struct drbd_peer_device *peer_device, *tmp_peer_device;
|
||||
|
||||
timer_shutdown_sync(&device->request_timer);
|
||||
timer_shutdown_sync(&device->start_resync_timer);
|
||||
|
||||
/* paranoia asserts */
|
||||
D_ASSERT(device, device->open_cnt == 0);
|
||||
|
||||
@@ -4096,8 +4096,10 @@ static int receive_uuids(struct drbd_connection *connection, struct packet_info
|
||||
BM_LOCKED_TEST_ALLOWED, NULL);
|
||||
_drbd_uuid_set(device, UI_CURRENT, p_uuid[UI_CURRENT]);
|
||||
_drbd_uuid_set(device, UI_BITMAP, 0);
|
||||
spin_lock_irq(&device->resource->req_lock);
|
||||
_drbd_set_state(_NS2(device, disk, D_UP_TO_DATE, pdsk, D_UP_TO_DATE),
|
||||
CS_VERBOSE, NULL);
|
||||
spin_unlock_irq(&device->resource->req_lock);
|
||||
drbd_md_sync(device);
|
||||
updated_uuids = 1;
|
||||
}
|
||||
|
||||
@@ -1604,8 +1604,10 @@ static int lo_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
|
||||
|
||||
switch(cmd) {
|
||||
case LOOP_SET_STATUS:
|
||||
err = loop_set_status_compat(lo,
|
||||
(const struct compat_loop_info __user *)arg);
|
||||
err = -EPERM;
|
||||
if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
|
||||
err = loop_set_status_compat(lo,
|
||||
(const struct compat_loop_info __user *)arg);
|
||||
break;
|
||||
case LOOP_GET_STATUS:
|
||||
err = loop_get_status_compat(lo,
|
||||
|
||||
@@ -44,12 +44,17 @@ int rnbd_srv_create_dev_sysfs(struct rnbd_srv_dev *dev,
|
||||
struct kobject *bdev_kobj;
|
||||
int ret;
|
||||
|
||||
ret = kobject_init_and_add(&dev->dev_kobj, &dev_ktype,
|
||||
rnbd_devs_kobj, "%pg", bdev);
|
||||
if (ret) {
|
||||
kobject_put(&dev->dev_kobj);
|
||||
/*
|
||||
* The release function frees the device, which the caller goes on
|
||||
* using and which others may share: never drop the reference of the
|
||||
* kobject here. That is for destroy_device().
|
||||
*/
|
||||
if (!dev->dev_kobj.state_initialized)
|
||||
kobject_init(&dev->dev_kobj, &dev_ktype);
|
||||
|
||||
ret = kobject_add(&dev->dev_kobj, rnbd_devs_kobj, "%pg", bdev);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
dev->dev_sessions_kobj = kobject_create_and_add("sessions",
|
||||
&dev->dev_kobj);
|
||||
@@ -69,7 +74,6 @@ put_sess_kobj:
|
||||
kobject_put(dev->dev_sessions_kobj);
|
||||
free_dev_kobj:
|
||||
kobject_del(&dev->dev_kobj);
|
||||
kobject_put(&dev->dev_kobj);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -189,6 +193,10 @@ static const struct kobj_type rnbd_srv_sess_dev_ktype = {
|
||||
.release = rnbd_srv_sess_dev_release,
|
||||
};
|
||||
|
||||
/*
|
||||
* On failure the caller has to drop the reference of the kobject. The release
|
||||
* function takes the lock of the device, which the caller holds here.
|
||||
*/
|
||||
int rnbd_srv_create_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev)
|
||||
{
|
||||
int ret;
|
||||
@@ -196,17 +204,13 @@ int rnbd_srv_create_dev_session_sysfs(struct rnbd_srv_sess_dev *sess_dev)
|
||||
ret = kobject_init_and_add(&sess_dev->kobj, &rnbd_srv_sess_dev_ktype,
|
||||
sess_dev->dev->dev_sessions_kobj, "%s",
|
||||
sess_dev->sess->sessname);
|
||||
if (ret) {
|
||||
kobject_put(&sess_dev->kobj);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = sysfs_create_group(&sess_dev->kobj,
|
||||
&rnbd_srv_default_dev_session_attr_group);
|
||||
if (ret) {
|
||||
if (ret)
|
||||
kobject_del(&sess_dev->kobj);
|
||||
kobject_put(&sess_dev->kobj);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -208,6 +208,9 @@ static void destroy_device(struct kref *kref)
|
||||
* Destroy kobj only if it was really created.
|
||||
*/
|
||||
rnbd_srv_destroy_dev_sysfs(dev);
|
||||
else if (dev->dev_kobj.state_initialized)
|
||||
/* The files could not be created. Frees the name, too. */
|
||||
kobject_put(&dev->dev_kobj);
|
||||
else
|
||||
kfree(dev);
|
||||
}
|
||||
@@ -357,14 +360,15 @@ static void process_msg_close(struct rnbd_srv_session *srv_sess,
|
||||
|
||||
trace_process_msg_close(srv_sess, close_msg);
|
||||
|
||||
sess_dev = rnbd_get_sess_dev(le32_to_cpu(close_msg->device_id),
|
||||
srv_sess);
|
||||
if (IS_ERR(sess_dev))
|
||||
return;
|
||||
|
||||
rnbd_put_sess_dev(sess_dev);
|
||||
/*
|
||||
* Each teardown of a sess_dev runs under srv_sess->lock and erases the
|
||||
* ID, so an entry that is found under the lock is alive.
|
||||
*/
|
||||
mutex_lock(&srv_sess->lock);
|
||||
rnbd_srv_destroy_dev_session_sysfs(sess_dev);
|
||||
sess_dev = xa_load(&srv_sess->index_idr,
|
||||
le32_to_cpu(close_msg->device_id));
|
||||
if (sess_dev)
|
||||
rnbd_srv_destroy_dev_session_sysfs(sess_dev);
|
||||
mutex_unlock(&srv_sess->lock);
|
||||
}
|
||||
|
||||
@@ -427,7 +431,7 @@ static struct rnbd_srv_sess_dev
|
||||
if (!sess_dev)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
error = xa_alloc(&srv_sess->index_idr, &sess_dev->device_id, sess_dev,
|
||||
error = xa_alloc(&srv_sess->index_idr, &sess_dev->device_id, NULL,
|
||||
xa_limit_32b, GFP_NOWAIT);
|
||||
if (error < 0) {
|
||||
pr_warn("Allocating idr failed, err: %d\n", error);
|
||||
@@ -587,6 +591,7 @@ rnbd_srv_create_set_sess_dev(struct rnbd_srv_session *srv_sess,
|
||||
return sdev;
|
||||
|
||||
kref_init(&sdev->kref);
|
||||
INIT_LIST_HEAD(&sdev->dev_list);
|
||||
|
||||
strscpy(sdev->pathname, open_msg->dev_name, sizeof(sdev->pathname));
|
||||
|
||||
@@ -786,12 +791,24 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess,
|
||||
rnbd_srv_err(srv_sess_dev,
|
||||
"Opening device failed, failed to create dev client sysfs files, err: %d\n",
|
||||
ret);
|
||||
goto free_srv_sess_dev;
|
||||
/*
|
||||
* rnbd_destroy_sess_dev() undoes everything down to the open
|
||||
* of the block device.
|
||||
*/
|
||||
kobject_put(&srv_sess_dev->kobj);
|
||||
goto free_path;
|
||||
}
|
||||
|
||||
list_add(&srv_sess_dev->dev_list, &srv_dev->sess_dev_list);
|
||||
mutex_unlock(&srv_dev->lock);
|
||||
|
||||
/*
|
||||
* Nothing fails from here on, so let I/O messages find the sess_dev.
|
||||
* The ID is reserved, so this store allocates nothing.
|
||||
*/
|
||||
xa_store(&srv_sess->index_idr, srv_sess_dev->device_id, srv_sess_dev,
|
||||
GFP_KERNEL);
|
||||
|
||||
rnbd_srv_info(srv_sess_dev, "Opened device '%s'\n", srv_dev->name);
|
||||
|
||||
kfree(full_path);
|
||||
@@ -803,7 +820,6 @@ fill_response:
|
||||
|
||||
free_srv_sess_dev:
|
||||
xa_erase(&srv_sess->index_idr, srv_sess_dev->device_id);
|
||||
synchronize_rcu();
|
||||
kfree(srv_sess_dev);
|
||||
srv_dev_put:
|
||||
if (open_msg->access_mode != RNBD_ACCESS_RO) {
|
||||
|
||||
+38
-42
@@ -1107,8 +1107,6 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,
|
||||
{
|
||||
struct zloop_zone *zone = &zlo->zones[zone_no];
|
||||
int oflags = O_RDWR;
|
||||
struct kstat stat;
|
||||
sector_t file_sectors;
|
||||
int ret;
|
||||
|
||||
mutex_init(&zone->lock);
|
||||
@@ -1122,38 +1120,49 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,
|
||||
if (!opts->buffered_io)
|
||||
oflags |= O_DIRECT;
|
||||
|
||||
zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/%s-%06u",
|
||||
zlo->base_dir, zlo->id,
|
||||
zone_no < zlo->nr_conv_zones ? "cnv" : "seq",
|
||||
zone_no);
|
||||
if (IS_ERR(zone->file)) {
|
||||
pr_err("Failed to open zone %u file %s/%u/%s-%06u (err=%ld)",
|
||||
zone_no, zlo->base_dir, zlo->id,
|
||||
zone_no < zlo->nr_conv_zones ? "cnv" : "seq",
|
||||
zone_no, PTR_ERR(zone->file));
|
||||
return PTR_ERR(zone->file);
|
||||
}
|
||||
|
||||
if (!zone->file->f_op->read_iter || !zone->file->f_op->write_iter) {
|
||||
ret = -EINVAL;
|
||||
goto out_fput;
|
||||
}
|
||||
|
||||
if (zlo->block_size) {
|
||||
ret = zloop_get_block_size(zlo, zone);
|
||||
if (ret)
|
||||
goto out_fput;
|
||||
}
|
||||
|
||||
if (zone_no < zlo->nr_conv_zones) {
|
||||
sector_t file_sectors;
|
||||
struct kstat stat;
|
||||
|
||||
/* Conventional zone file. */
|
||||
set_bit(ZLOOP_ZONE_CONV, &zone->flags);
|
||||
zone->cond = BLK_ZONE_COND_NOT_WP;
|
||||
zone->wp = U64_MAX;
|
||||
|
||||
zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/cnv-%06u",
|
||||
zlo->base_dir, zlo->id, zone_no);
|
||||
if (IS_ERR(zone->file)) {
|
||||
pr_err("Failed to open zone %u file %s/%u/cnv-%06u (err=%ld)",
|
||||
zone_no, zlo->base_dir, zlo->id, zone_no,
|
||||
PTR_ERR(zone->file));
|
||||
return PTR_ERR(zone->file);
|
||||
}
|
||||
|
||||
if (!zlo->block_size) {
|
||||
ret = zloop_get_block_size(zlo, zone);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0);
|
||||
if (ret < 0) {
|
||||
pr_err("Failed to get zone %u file stat\n", zone_no);
|
||||
return ret;
|
||||
goto out_fput;
|
||||
}
|
||||
file_sectors = stat.size >> SECTOR_SHIFT;
|
||||
|
||||
if (restore && file_sectors != zlo->zone_size) {
|
||||
pr_err("Invalid conventional zone %u file size (%llu sectors != %llu)\n",
|
||||
zone_no, file_sectors, zlo->zone_capacity);
|
||||
return ret;
|
||||
goto out_fput;
|
||||
}
|
||||
|
||||
ret = vfs_truncate(&zone->file->f_path,
|
||||
@@ -1161,34 +1170,21 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,
|
||||
if (ret < 0) {
|
||||
pr_err("Failed to truncate zone %u file (err=%d)\n",
|
||||
zone_no, ret);
|
||||
return ret;
|
||||
goto out_fput;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Sequential zone file. */
|
||||
zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/seq-%06u",
|
||||
zlo->base_dir, zlo->id, zone_no);
|
||||
if (IS_ERR(zone->file)) {
|
||||
pr_err("Failed to open zone %u file %s/%u/seq-%06u (err=%ld)",
|
||||
zone_no, zlo->base_dir, zlo->id, zone_no,
|
||||
PTR_ERR(zone->file));
|
||||
return PTR_ERR(zone->file);
|
||||
}
|
||||
|
||||
if (!zlo->block_size) {
|
||||
ret = zloop_get_block_size(zlo, zone);
|
||||
} else {
|
||||
/* Sequential zone file. */
|
||||
mutex_lock(&zone->lock);
|
||||
ret = zloop_update_seq_zone(zlo, zone_no);
|
||||
mutex_unlock(&zone->lock);
|
||||
if (ret)
|
||||
return ret;
|
||||
goto out_fput;
|
||||
}
|
||||
|
||||
zloop_get_block_size(zlo, zone);
|
||||
|
||||
mutex_lock(&zone->lock);
|
||||
ret = zloop_update_seq_zone(zlo, zone_no);
|
||||
mutex_unlock(&zone->lock);
|
||||
return 0;
|
||||
|
||||
out_fput:
|
||||
fput(zone->file);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
+49
-23
@@ -414,6 +414,20 @@ check_suspended:
|
||||
if (!percpu_ref_tryget_live(&mddev->active_io))
|
||||
goto check_suspended;
|
||||
}
|
||||
if (!mddev->pers) {
|
||||
/*
|
||||
* The __md_stop() sets 'mddev->private' to NULL during
|
||||
* the IO submitting, check 'mddev->pers' before the IO
|
||||
* being processed by specific driver to avoid the
|
||||
* null-ptr-deref of 'mddev-><member>'. The check is
|
||||
* safe because the IO has got the 'mddev->active_io'
|
||||
* reference, and all __md_stop() callers will wait for
|
||||
* the reference to be zero.
|
||||
*/
|
||||
bio_io_error(bio);
|
||||
percpu_ref_put(&mddev->active_io);
|
||||
return true;
|
||||
}
|
||||
if (!mddev->pers->make_request(mddev, bio)) {
|
||||
percpu_ref_put(&mddev->active_io);
|
||||
if (mddev_is_dm(mddev) && mddev->pers->prepare_suspend)
|
||||
@@ -1393,7 +1407,7 @@ static int super_90_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor
|
||||
goto abort;
|
||||
}
|
||||
|
||||
if (sb->raid_disks <= 0)
|
||||
if (sb->raid_disks <= 0 || sb->raid_disks > MD_SB_DISKS)
|
||||
goto abort;
|
||||
|
||||
if (md_csum_fold(calc_sb_csum(sb)) != md_csum_fold(sb->sb_csum)) {
|
||||
@@ -1934,8 +1948,11 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_
|
||||
if (!badblocks_set(&rdev->badblocks, sector, count, 1))
|
||||
return -EINVAL;
|
||||
}
|
||||
} else if (sb->bblog_offset != 0)
|
||||
} else if (sb->bblog_offset != 0) {
|
||||
if (le16_to_cpu(sb->bblog_size) > (PAGE_SIZE / 512))
|
||||
return -EINVAL;
|
||||
rdev->badblocks.shift = 0;
|
||||
}
|
||||
|
||||
if ((le32_to_cpu(sb->feature_map) &
|
||||
(MD_FEATURE_PPL | MD_FEATURE_MULTIPLE_PPLS))) {
|
||||
@@ -3586,10 +3603,7 @@ static ssize_t recovery_start_store(struct md_rdev *rdev, const char *buf, size_
|
||||
return -EBUSY;
|
||||
|
||||
rdev->recovery_offset = recovery_start;
|
||||
if (recovery_start == MaxSector)
|
||||
set_bit(In_sync, &rdev->flags);
|
||||
else
|
||||
clear_bit(In_sync, &rdev->flags);
|
||||
assign_bit(In_sync, &rdev->flags, recovery_start == MaxSector);
|
||||
return len;
|
||||
}
|
||||
|
||||
@@ -4670,7 +4684,8 @@ array_state_store(struct mddev *mddev, const char *buf, size_t len)
|
||||
case readonly:
|
||||
case inactive:
|
||||
case read_auto:
|
||||
if (!mddev->pers || !md_is_rdwr(mddev))
|
||||
if (!mddev->pers ||
|
||||
(st != clear && st != inactive && !md_is_rdwr(mddev)))
|
||||
break;
|
||||
/* write sysfs will not open mddev and opener should be 0 */
|
||||
err = mddev_set_closing_and_sync_blockdev(mddev, 0);
|
||||
@@ -5884,7 +5899,15 @@ consistency_policy_store(struct mddev *mddev, const char *buf, size_t len)
|
||||
else
|
||||
err = -EBUSY;
|
||||
} else if (mddev->external && strncmp(buf, "ppl", 3) == 0) {
|
||||
set_bit(MD_HAS_PPL, &mddev->flags);
|
||||
/* md_run() holds the lock until ->pers is set */
|
||||
err = mddev_lock(mddev);
|
||||
if (err)
|
||||
return err;
|
||||
if (mddev->pers)
|
||||
err = -EBUSY;
|
||||
else
|
||||
set_bit(MD_HAS_PPL, &mddev->flags);
|
||||
mddev_unlock(mddev);
|
||||
} else {
|
||||
err = -EINVAL;
|
||||
}
|
||||
@@ -5915,10 +5938,7 @@ fail_last_dev_store(struct mddev *mddev, const char *buf, size_t len)
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (value)
|
||||
set_bit(MD_FAILLAST_DEV, &mddev->flags);
|
||||
else
|
||||
clear_bit(MD_FAILLAST_DEV, &mddev->flags);
|
||||
assign_bit(MD_FAILLAST_DEV, &mddev->flags, value);
|
||||
|
||||
return len;
|
||||
}
|
||||
@@ -6048,11 +6068,10 @@ lbs_store(struct mddev *mddev, const char *buf, size_t len)
|
||||
|
||||
err = mddev_lock(mddev);
|
||||
if (err)
|
||||
goto unlock;
|
||||
return err;
|
||||
|
||||
err = mddev_set_logical_block_size(mddev, lbs);
|
||||
|
||||
unlock:
|
||||
mddev_unlock(mddev);
|
||||
return err ?: len;
|
||||
}
|
||||
@@ -6385,12 +6404,22 @@ struct mddev *md_alloc(dev_t dev, char *name)
|
||||
|
||||
disk->events |= DISK_EVENT_MEDIA_CHANGE;
|
||||
mddev->gendisk = disk;
|
||||
|
||||
/*
|
||||
* add_disk() makes the array openable before mddev->kobj exists.
|
||||
* Hold reconfig_mutex until kobject_add() is done so that ioctls
|
||||
* issued in between wait instead of using an uninitialised kobj.
|
||||
*/
|
||||
mddev_lock_nointr(mddev);
|
||||
error = add_disk(disk);
|
||||
if (error)
|
||||
if (error) {
|
||||
mddev_unlock(mddev);
|
||||
goto out_put_disk;
|
||||
}
|
||||
|
||||
kobject_init(&mddev->kobj, &md_ktype);
|
||||
error = kobject_add(&mddev->kobj, &disk_to_dev(disk)->kobj, "%s", "md");
|
||||
mddev_unlock(mddev);
|
||||
if (error) {
|
||||
/*
|
||||
* The disk is already live at this point. Clear the hold flag
|
||||
@@ -7574,14 +7603,10 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info)
|
||||
}
|
||||
|
||||
clear_bit(In_sync, &rdev->flags); /* just to be sure */
|
||||
if (info->state & (1<<MD_DISK_WRITEMOSTLY))
|
||||
set_bit(WriteMostly, &rdev->flags);
|
||||
else
|
||||
clear_bit(WriteMostly, &rdev->flags);
|
||||
if (info->state & (1<<MD_DISK_FAILFAST))
|
||||
set_bit(FailFast, &rdev->flags);
|
||||
else
|
||||
clear_bit(FailFast, &rdev->flags);
|
||||
assign_bit(WriteMostly, &rdev->flags,
|
||||
info->state & (1 << MD_DISK_WRITEMOSTLY));
|
||||
assign_bit(FailFast, &rdev->flags,
|
||||
info->state & (1 << MD_DISK_FAILFAST));
|
||||
|
||||
if (info->state & (1<<MD_DISK_JOURNAL)) {
|
||||
struct md_rdev *rdev2;
|
||||
@@ -8299,6 +8324,7 @@ static bool md_ioctl_need_suspend(unsigned int cmd)
|
||||
case HOT_REMOVE_DISK:
|
||||
case SET_BITMAP_FILE:
|
||||
case SET_ARRAY_INFO:
|
||||
case STOP_ARRAY:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -989,6 +989,13 @@ static inline void rdev_dec_pending(struct md_rdev *rdev, struct mddev *mddev)
|
||||
}
|
||||
}
|
||||
|
||||
static inline void rdev_record_write_error(struct md_rdev *rdev)
|
||||
{
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED, &rdev->mddev->recovery);
|
||||
}
|
||||
|
||||
static inline int mddev_is_clustered(struct mddev *mddev)
|
||||
{
|
||||
return mddev->cluster_info && mddev->bitmap_info.nodes > 1;
|
||||
|
||||
+11
-5
@@ -79,7 +79,10 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
|
||||
*private_conf = ERR_PTR(-ENOMEM);
|
||||
if (!conf)
|
||||
return -ENOMEM;
|
||||
|
||||
cnt = 0;
|
||||
rdev_for_each(rdev1, mddev) {
|
||||
cnt++;
|
||||
pr_debug("md/raid0:%s: looking at %pg\n",
|
||||
mdname(mddev),
|
||||
rdev1->bdev);
|
||||
@@ -144,6 +147,14 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
|
||||
}
|
||||
|
||||
err = -ENOMEM;
|
||||
|
||||
if (cnt != mddev->raid_disks) {
|
||||
pr_warn("md/raid0:%s: too few disks (%d of %d) - aborting!\n",
|
||||
mdname(mddev), cnt, mddev->raid_disks);
|
||||
err = -EINVAL;
|
||||
goto abort;
|
||||
}
|
||||
|
||||
conf->strip_zone = kvzalloc_objs(struct strip_zone, conf->nr_strip_zones);
|
||||
if (!conf->strip_zone)
|
||||
goto abort;
|
||||
@@ -200,11 +211,6 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
|
||||
smallest = rdev1;
|
||||
cnt++;
|
||||
}
|
||||
if (cnt != mddev->raid_disks) {
|
||||
pr_warn("md/raid0:%s: too few disks (%d of %d) - aborting!\n",
|
||||
mdname(mddev), cnt, mddev->raid_disks);
|
||||
goto abort;
|
||||
}
|
||||
zone->nb_dev = cnt;
|
||||
zone->zone_end = smallest->sectors * cnt;
|
||||
|
||||
|
||||
@@ -155,6 +155,16 @@ static inline bool raid1_add_bio_to_plug(struct mddev *mddev, struct bio *bio,
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Block size used when narrowing a write error down to badblocks granularity. */
|
||||
static inline int rdev_bb_block_sectors(struct md_rdev *rdev)
|
||||
{
|
||||
int lbs = bdev_logical_block_size(rdev->bdev) >> 9;
|
||||
|
||||
if (rdev->badblocks.shift < 0)
|
||||
return lbs;
|
||||
return roundup(1 << rdev->badblocks.shift, lbs);
|
||||
}
|
||||
|
||||
/*
|
||||
* current->bio_list will be set under submit_bio() context, in this case bitmap
|
||||
* io will be added to the list and wait for current io submission to finish,
|
||||
@@ -166,6 +176,23 @@ static inline void raid1_prepare_flush_writes(struct mddev *mddev)
|
||||
mddev->bitmap_ops->unplug(mddev, current->bio_list == NULL);
|
||||
}
|
||||
|
||||
static inline void flush_bio_list(struct mddev *mddev, struct bio *bio,
|
||||
wait_queue_head_t *wait)
|
||||
{
|
||||
/* flush any pending bitmap writes to disk before proceeding w/ I/O */
|
||||
raid1_prepare_flush_writes(mddev);
|
||||
if (wq_has_sleeper(wait))
|
||||
wake_up(wait);
|
||||
|
||||
while (bio) { /* submit pending writes */
|
||||
struct bio *next = bio->bi_next;
|
||||
|
||||
raid1_submit_write(bio);
|
||||
bio = next;
|
||||
cond_resched();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Used by fix_read_error() to decay the per rdev read_errors.
|
||||
* We halve the read error count for every hour that has elapsed
|
||||
|
||||
+6
-36
@@ -483,10 +483,7 @@ static void raid1_end_write_request(struct bio *bio)
|
||||
* 'one mirror IO has finished' event handler:
|
||||
*/
|
||||
if (bio->bi_status && !ignore_error) {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED, &
|
||||
conf->mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
|
||||
if (test_bit(FailFast, &rdev->flags) &&
|
||||
(bio->bi_opf & MD_FAILFAST) &&
|
||||
@@ -922,21 +919,6 @@ static void wake_up_barrier(struct r1conf *conf)
|
||||
wake_up(&conf->wait_barrier);
|
||||
}
|
||||
|
||||
static void flush_bio_list(struct r1conf *conf, struct bio *bio)
|
||||
{
|
||||
/* flush any pending bitmap writes to disk before proceeding w/ I/O */
|
||||
raid1_prepare_flush_writes(conf->mddev);
|
||||
wake_up_barrier(conf);
|
||||
|
||||
while (bio) { /* submit pending writes */
|
||||
struct bio *next = bio->bi_next;
|
||||
|
||||
raid1_submit_write(bio);
|
||||
bio = next;
|
||||
cond_resched();
|
||||
}
|
||||
}
|
||||
|
||||
static void flush_pending_writes(struct r1conf *conf)
|
||||
{
|
||||
/* Any writes that have been queued but are awaiting
|
||||
@@ -962,7 +944,7 @@ static void flush_pending_writes(struct r1conf *conf)
|
||||
*/
|
||||
__set_current_state(TASK_RUNNING);
|
||||
blk_start_plug(&plug);
|
||||
flush_bio_list(conf, bio);
|
||||
flush_bio_list(conf->mddev, bio, &conf->wait_barrier);
|
||||
blk_finish_plug(&plug);
|
||||
} else
|
||||
spin_unlock_irq(&conf->device_lock);
|
||||
@@ -1292,7 +1274,7 @@ static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)
|
||||
|
||||
/* we aren't scheduling, so we can do the write-out directly. */
|
||||
bio = bio_list_get(&plug->pending);
|
||||
flush_bio_list(conf, bio);
|
||||
flush_bio_list(conf->mddev, bio, &conf->wait_barrier);
|
||||
kfree(plug);
|
||||
}
|
||||
|
||||
@@ -2067,10 +2049,7 @@ static void end_sync_write(struct bio *bio)
|
||||
|
||||
if (bio->bi_status) {
|
||||
abort_sync_write(mddev, r1_bio);
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED, &
|
||||
mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
set_bit(R1BIO_WriteError, &r1_bio->state);
|
||||
} else if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) &&
|
||||
!rdev_has_badblock(conf->mirrors[r1_bio->read_disk].rdev,
|
||||
@@ -2088,11 +2067,7 @@ static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,
|
||||
/* success */
|
||||
return 1;
|
||||
if (rw == REQ_OP_WRITE) {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement,
|
||||
&rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED, &
|
||||
rdev->mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
}
|
||||
/* need to record an error - either for the block or the device */
|
||||
rdev_set_badblocks(rdev, sector, sectors, 0);
|
||||
@@ -2475,16 +2450,11 @@ static void narrow_write_error(struct r1bio *r1_bio, int i)
|
||||
* We currently own a reference on the rdev.
|
||||
*/
|
||||
|
||||
int block_sectors, lbs = bdev_logical_block_size(rdev->bdev) >> 9;
|
||||
int block_sectors = rdev_bb_block_sectors(rdev);
|
||||
sector_t sector;
|
||||
int sectors;
|
||||
int sect_to_write = r1_bio->sectors;
|
||||
|
||||
if (rdev->badblocks.shift < 0)
|
||||
block_sectors = lbs;
|
||||
else
|
||||
block_sectors = roundup(1 << rdev->badblocks.shift, lbs);
|
||||
|
||||
sector = r1_bio->sector;
|
||||
sectors = ((sector + block_sectors)
|
||||
& ~(sector_t)(block_sectors - 1))
|
||||
|
||||
+12
-47
@@ -496,10 +496,7 @@ static void raid10_end_write_request(struct bio *bio)
|
||||
*/
|
||||
md_error(rdev->mddev, rdev);
|
||||
else {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED,
|
||||
&rdev->mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
|
||||
dec_rdev = 0;
|
||||
if (test_bit(FailFast, &rdev->flags) &&
|
||||
@@ -898,16 +895,7 @@ static void flush_pending_writes(struct r10conf *conf)
|
||||
__set_current_state(TASK_RUNNING);
|
||||
|
||||
blk_start_plug(&plug);
|
||||
raid1_prepare_flush_writes(conf->mddev);
|
||||
wake_up(&conf->wait_barrier);
|
||||
|
||||
while (bio) { /* submit pending writes */
|
||||
struct bio *next = bio->bi_next;
|
||||
|
||||
raid1_submit_write(bio);
|
||||
bio = next;
|
||||
cond_resched();
|
||||
}
|
||||
flush_bio_list(conf->mddev, bio, &conf->wait_barrier);
|
||||
blk_finish_plug(&plug);
|
||||
} else
|
||||
spin_unlock_irq(&conf->device_lock);
|
||||
@@ -1010,7 +998,8 @@ static bool wait_barrier_nolock(struct r10conf *conf)
|
||||
if (!read_seqretry(&conf->resync_lock, seq))
|
||||
return true;
|
||||
|
||||
if (atomic_dec_and_test(&conf->nr_pending))
|
||||
if (atomic_dec_and_test(&conf->nr_pending) ||
|
||||
READ_ONCE(conf->array_freeze_pending))
|
||||
wake_up_barrier(conf);
|
||||
|
||||
return false;
|
||||
@@ -1037,7 +1026,7 @@ static void wait_barrier(struct r10conf *conf)
|
||||
static void allow_barrier(struct r10conf *conf)
|
||||
{
|
||||
if ((atomic_dec_and_test(&conf->nr_pending)) ||
|
||||
(conf->array_freeze_pending))
|
||||
READ_ONCE(conf->array_freeze_pending))
|
||||
wake_up_barrier(conf);
|
||||
}
|
||||
|
||||
@@ -1056,12 +1045,12 @@ static void freeze_array(struct r10conf *conf, int extra)
|
||||
* we continue.
|
||||
*/
|
||||
write_seqlock_irq(&conf->resync_lock);
|
||||
conf->array_freeze_pending++;
|
||||
WRITE_ONCE(conf->array_freeze_pending, conf->array_freeze_pending + 1);
|
||||
WRITE_ONCE(conf->barrier, conf->barrier + 1);
|
||||
conf->nr_waiting++;
|
||||
wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==
|
||||
conf->nr_queued + extra, flush_pending_writes(conf));
|
||||
conf->array_freeze_pending--;
|
||||
WRITE_ONCE(conf->array_freeze_pending, conf->array_freeze_pending - 1);
|
||||
write_sequnlock_irq(&conf->resync_lock);
|
||||
}
|
||||
|
||||
@@ -1104,16 +1093,7 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
|
||||
|
||||
/* we aren't scheduling, so we can do the write-out directly. */
|
||||
bio = bio_list_get(&plug->pending);
|
||||
raid1_prepare_flush_writes(mddev);
|
||||
wake_up_barrier(conf);
|
||||
|
||||
while (bio) { /* submit pending writes */
|
||||
struct bio *next = bio->bi_next;
|
||||
|
||||
raid1_submit_write(bio);
|
||||
bio = next;
|
||||
cond_resched();
|
||||
}
|
||||
flush_bio_list(mddev, bio, &conf->wait_barrier);
|
||||
kfree(plug);
|
||||
}
|
||||
|
||||
@@ -2299,10 +2279,7 @@ static void end_sync_write(struct bio *bio)
|
||||
if (repl)
|
||||
md_error(mddev, rdev);
|
||||
else {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED,
|
||||
&rdev->mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
set_bit(R10BIO_WriteError, &r10_bio->state);
|
||||
}
|
||||
} else if (rdev_has_badblock(rdev, r10_bio->devs[slot].addr,
|
||||
@@ -2500,11 +2477,7 @@ static void fix_recovery_read_error(struct r10bio *r10_bio)
|
||||
pages[idx],
|
||||
REQ_OP_WRITE, false);
|
||||
if (!ok) {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement,
|
||||
&rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED,
|
||||
&rdev->mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
@@ -2585,10 +2558,7 @@ static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
|
||||
/* success */
|
||||
return 1;
|
||||
if (op == REQ_OP_WRITE) {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED,
|
||||
&rdev->mddev->recovery);
|
||||
rdev_record_write_error(rdev);
|
||||
}
|
||||
/* need to record an error - either for the block or the device */
|
||||
rdev_set_badblocks(rdev, sector, sectors, 0);
|
||||
@@ -2775,16 +2745,11 @@ static void narrow_write_error(struct r10bio *r10_bio, int i)
|
||||
* We currently own a reference to the rdev.
|
||||
*/
|
||||
|
||||
int block_sectors, lbs = bdev_logical_block_size(rdev->bdev) >> 9;
|
||||
int block_sectors = rdev_bb_block_sectors(rdev);
|
||||
sector_t sector;
|
||||
int sectors;
|
||||
int sect_to_write = r10_bio->sectors;
|
||||
|
||||
if (rdev->badblocks.shift < 0)
|
||||
block_sectors = lbs;
|
||||
else
|
||||
block_sectors = roundup(1 << rdev->badblocks.shift, lbs);
|
||||
|
||||
sector = r10_bio->sector;
|
||||
sectors = ((r10_bio->sector + block_sectors)
|
||||
& ~(sector_t)(block_sectors - 1))
|
||||
|
||||
+41
-13
@@ -432,10 +432,8 @@ static inline void r5c_update_log_state(struct r5l_log *log)
|
||||
wake_reclaim = true;
|
||||
clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
|
||||
}
|
||||
if (free_space < 3 * reclaim_space)
|
||||
set_bit(R5C_LOG_TIGHT, &conf->cache_state);
|
||||
else
|
||||
clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
|
||||
assign_bit(R5C_LOG_TIGHT, &conf->cache_state,
|
||||
free_space < 3 * reclaim_space);
|
||||
|
||||
if (wake_reclaim)
|
||||
r5l_wake_reclaim(log, 0);
|
||||
@@ -2001,27 +1999,36 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
|
||||
if (!page)
|
||||
return -ENOMEM;
|
||||
|
||||
while (mb_offset < le32_to_cpu(mb->meta_size)) {
|
||||
while (mb_offset + sizeof(struct r5l_payload_header) <= le32_to_cpu(mb->meta_size)) {
|
||||
u32 payload_size;
|
||||
sector_t payload_len;
|
||||
|
||||
payload = (void *)mb + mb_offset;
|
||||
payload_flush = (void *)mb + mb_offset;
|
||||
|
||||
if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
|
||||
if (mb_offset + sizeof(struct r5l_payload_data_parity)
|
||||
> le32_to_cpu(mb->meta_size))
|
||||
goto mismatch;
|
||||
payload_size = le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9);
|
||||
payload_len = sizeof(struct r5l_payload_data_parity) +
|
||||
(sector_t)sizeof(__le32) *
|
||||
(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
|
||||
if (mb_offset + payload_len > le32_to_cpu(mb->meta_size))
|
||||
(sector_t)sizeof(__le32) * payload_size;
|
||||
if (mb_offset + payload_len > le32_to_cpu(mb->meta_size) ||
|
||||
payload_size < 1)
|
||||
goto mismatch;
|
||||
if (r5l_recovery_verify_data_checksum(
|
||||
log, ctx, page, log_offset,
|
||||
payload->checksum[0]) < 0)
|
||||
goto mismatch;
|
||||
} else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) {
|
||||
if (mb_offset + sizeof(struct r5l_payload_data_parity)
|
||||
> le32_to_cpu(mb->meta_size))
|
||||
goto mismatch;
|
||||
payload_size = le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9);
|
||||
payload_len = sizeof(struct r5l_payload_data_parity) +
|
||||
(sector_t)sizeof(__le32) *
|
||||
(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
|
||||
if (mb_offset + payload_len > le32_to_cpu(mb->meta_size))
|
||||
(sector_t)sizeof(__le32) * payload_size;
|
||||
if (mb_offset + payload_len > le32_to_cpu(mb->meta_size) ||
|
||||
payload_size < conf->max_degraded)
|
||||
goto mismatch;
|
||||
if (r5l_recovery_verify_data_checksum(
|
||||
log, ctx, page, log_offset,
|
||||
@@ -2035,6 +2042,9 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
|
||||
payload->checksum[1]) < 0)
|
||||
goto mismatch;
|
||||
} else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
|
||||
if (mb_offset + sizeof(struct r5l_payload_flush)
|
||||
> le32_to_cpu(mb->meta_size))
|
||||
goto mismatch;
|
||||
payload_len = sizeof(struct r5l_payload_flush) +
|
||||
(sector_t)le32_to_cpu(payload_flush->size);
|
||||
if (mb_offset + payload_len > le32_to_cpu(mb->meta_size))
|
||||
@@ -2096,7 +2106,7 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log,
|
||||
mb_offset = sizeof(struct r5l_meta_block);
|
||||
log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
|
||||
|
||||
while (mb_offset < le32_to_cpu(mb->meta_size)) {
|
||||
while (mb_offset + sizeof(struct r5l_payload_header) < le32_to_cpu(mb->meta_size)) {
|
||||
sector_t payload_len;
|
||||
int dd;
|
||||
|
||||
@@ -2106,6 +2116,9 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log,
|
||||
if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
|
||||
int i, count;
|
||||
|
||||
if (mb_offset + sizeof(struct r5l_payload_flush) >
|
||||
le32_to_cpu(mb->meta_size))
|
||||
return -EINVAL;
|
||||
payload_len = sizeof(struct r5l_payload_flush) +
|
||||
(sector_t)le32_to_cpu(payload_flush->size);
|
||||
if (mb_offset + payload_len >
|
||||
@@ -2130,6 +2143,9 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log,
|
||||
}
|
||||
|
||||
/* DATA or PARITY payload */
|
||||
if (mb_offset + sizeof(struct r5l_payload_data_parity) >
|
||||
le32_to_cpu(mb->meta_size))
|
||||
return -EINVAL;
|
||||
payload_len = sizeof(struct r5l_payload_data_parity) +
|
||||
(sector_t)sizeof(__le32) *
|
||||
(le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
|
||||
@@ -2952,7 +2968,9 @@ bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect)
|
||||
return false;
|
||||
|
||||
tree_index = r5c_tree_index(conf, sect);
|
||||
rcu_read_lock();
|
||||
slot = radix_tree_lookup(&log->big_stripe_tree, tree_index);
|
||||
rcu_read_unlock();
|
||||
return slot != NULL;
|
||||
}
|
||||
|
||||
@@ -3066,6 +3084,7 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
|
||||
{
|
||||
struct r5l_log *log;
|
||||
struct md_thread *thread;
|
||||
char cache_name[48];
|
||||
int ret;
|
||||
|
||||
pr_debug("md/raid:%s: using device %pg as journal\n",
|
||||
@@ -3105,7 +3124,16 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
|
||||
INIT_LIST_HEAD(&log->flushing_ios);
|
||||
INIT_LIST_HEAD(&log->finished_ios);
|
||||
|
||||
log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
|
||||
if (mddev_is_dm(conf->mddev))
|
||||
snprintf(cache_name, sizeof(cache_name), "r5l_io_unit-%p",
|
||||
conf->mddev);
|
||||
else
|
||||
snprintf(cache_name, sizeof(cache_name), "r5l_io_unit-%s",
|
||||
mdname(conf->mddev));
|
||||
|
||||
log->io_kc = kmem_cache_create(cache_name, sizeof(struct r5l_io_unit),
|
||||
__alignof__(struct r5l_io_unit),
|
||||
0, NULL);
|
||||
if (!log->io_kc)
|
||||
goto io_kc;
|
||||
|
||||
|
||||
+11
-1
@@ -1318,6 +1318,7 @@ int ppl_init_log(struct r5conf *conf)
|
||||
{
|
||||
struct ppl_conf *ppl_conf;
|
||||
struct mddev *mddev = conf->mddev;
|
||||
char cache_name[48];
|
||||
int ret = 0;
|
||||
int max_disks;
|
||||
int i;
|
||||
@@ -1360,7 +1361,16 @@ int ppl_init_log(struct r5conf *conf)
|
||||
|
||||
ppl_conf->mddev = mddev;
|
||||
|
||||
ppl_conf->io_kc = KMEM_CACHE(ppl_io_unit, 0);
|
||||
if (mddev_is_dm(mddev))
|
||||
snprintf(cache_name, sizeof(cache_name), "ppl_io_unit-%p", mddev);
|
||||
else
|
||||
snprintf(cache_name, sizeof(cache_name), "ppl_io_unit-%s",
|
||||
mdname(mddev));
|
||||
|
||||
ppl_conf->io_kc = kmem_cache_create(cache_name,
|
||||
sizeof(struct ppl_io_unit),
|
||||
__alignof__(struct ppl_io_unit),
|
||||
0, NULL);
|
||||
if (!ppl_conf->io_kc) {
|
||||
ret = -ENOMEM;
|
||||
goto err;
|
||||
|
||||
+79
-27
@@ -2642,6 +2642,7 @@ static int resize_stripes(struct r5conf *conf, int newsize)
|
||||
} else {
|
||||
kfree(conf->disks);
|
||||
conf->disks = ndisks;
|
||||
conf->disks_cnt = newsize;
|
||||
}
|
||||
} else
|
||||
err = -ENOMEM;
|
||||
@@ -2712,6 +2713,37 @@ static int drop_one_stripe(struct r5conf *conf)
|
||||
|
||||
static void shrink_stripes(struct r5conf *conf)
|
||||
{
|
||||
struct list_head temp[NR_STRIPE_HASH_LOCKS];
|
||||
int i;
|
||||
|
||||
/*
|
||||
* grow_one_stripe() parks new stripes on released_stripes when
|
||||
* mddev->thread is already set. Takeover leaves the previous
|
||||
* personality's thread there, so raid5d never moves them to
|
||||
* inactive_list. Drain them before the cache is destroyed.
|
||||
* slab_cache is assigned only after device_lock and the hash
|
||||
* lists exist; earlier abort paths have nothing to drain.
|
||||
*/
|
||||
if (conf->slab_cache) {
|
||||
for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
|
||||
INIT_LIST_HEAD(&temp[i]);
|
||||
|
||||
spin_lock_irq(&conf->device_lock);
|
||||
release_stripe_list(conf, temp);
|
||||
spin_unlock_irq(&conf->device_lock);
|
||||
|
||||
for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) {
|
||||
if (list_empty(&temp[i]))
|
||||
continue;
|
||||
spin_lock_irq(conf->hash_locks + i);
|
||||
if (list_empty(conf->inactive_list + i))
|
||||
atomic_dec(&conf->empty_inactive_list_nr);
|
||||
list_splice_tail_init(&temp[i],
|
||||
conf->inactive_list + i);
|
||||
spin_unlock_irq(conf->hash_locks + i);
|
||||
}
|
||||
}
|
||||
|
||||
while (conf->max_nr_stripes &&
|
||||
drop_one_stripe(conf))
|
||||
;
|
||||
@@ -2888,11 +2920,8 @@ static void raid5_end_write_request(struct bio *bi)
|
||||
set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
|
||||
} else {
|
||||
if (bi->bi_status) {
|
||||
set_bit(WriteErrorSeen, &rdev->flags);
|
||||
rdev_record_write_error(rdev);
|
||||
set_bit(R5_WriteError, &sh->dev[i].flags);
|
||||
if (!test_and_set_bit(WantReplacement, &rdev->flags))
|
||||
set_bit(MD_RECOVERY_NEEDED,
|
||||
&rdev->mddev->recovery);
|
||||
} else if (rdev_has_badblock(rdev, sh->sector,
|
||||
RAID5_STRIPE_SECTORS(conf))) {
|
||||
set_bit(R5_MadeGood, &sh->dev[i].flags);
|
||||
@@ -4748,10 +4777,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
|
||||
RAID5_STRIPE_SECTORS(conf)))
|
||||
set_bit(R5_ReadRepl, &dev->flags);
|
||||
else {
|
||||
if (rdev && !test_bit(Faulty, &rdev->flags))
|
||||
set_bit(R5_NeedReplace, &dev->flags);
|
||||
else
|
||||
clear_bit(R5_NeedReplace, &dev->flags);
|
||||
assign_bit(R5_NeedReplace, &dev->flags,
|
||||
rdev && !test_bit(Faulty, &rdev->flags));
|
||||
rdev = conf->disks[i].rdev;
|
||||
clear_bit(R5_ReadRepl, &dev->flags);
|
||||
}
|
||||
@@ -7538,7 +7565,8 @@ static void raid5_free_percpu(struct r5conf *conf)
|
||||
if (!conf->percpu)
|
||||
return;
|
||||
|
||||
cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
|
||||
if (!hlist_unhashed_lockless(&conf->node))
|
||||
cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
|
||||
free_percpu(conf->percpu);
|
||||
}
|
||||
|
||||
@@ -7552,7 +7580,7 @@ static void free_conf(struct r5conf *conf)
|
||||
free_thread_groups(conf);
|
||||
shrink_stripes(conf);
|
||||
raid5_free_percpu(conf);
|
||||
for (i = 0; i < conf->pool_size; i++)
|
||||
for (i = 0; i < conf->disks_cnt; i++)
|
||||
if (conf->disks[i].extra_page)
|
||||
put_page(conf->disks[i].extra_page);
|
||||
kfree(conf->disks);
|
||||
@@ -7630,10 +7658,15 @@ static unsigned long raid5_cache_count(struct shrinker *shrink,
|
||||
return max_stripes - min_stripes;
|
||||
}
|
||||
|
||||
static int min_raid_disks(int level)
|
||||
{
|
||||
return level == 6 ? 4 : 2;
|
||||
}
|
||||
|
||||
static struct r5conf *setup_conf(struct mddev *mddev)
|
||||
{
|
||||
struct r5conf *conf;
|
||||
int raid_disk, memory, max_disks;
|
||||
int raid_disk, memory, max_disks, min_disks;
|
||||
struct md_rdev *rdev;
|
||||
struct disk_info *disk;
|
||||
char pers_name[6];
|
||||
@@ -7657,9 +7690,11 @@ static struct r5conf *setup_conf(struct mddev *mddev)
|
||||
mdname(mddev), mddev->new_layout);
|
||||
return ERR_PTR(-EIO);
|
||||
}
|
||||
if (mddev->new_level == 6 && mddev->raid_disks < 4) {
|
||||
pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",
|
||||
mdname(mddev), mddev->raid_disks);
|
||||
min_disks = min_raid_disks(mddev->new_level);
|
||||
if (mddev->raid_disks < min_disks) {
|
||||
pr_warn("md/raid:%s: not enough configured devices (%d, minimum %d)\n",
|
||||
mdname(mddev), mddev->raid_disks,
|
||||
min_disks);
|
||||
return ERR_PTR(-EINVAL);
|
||||
}
|
||||
|
||||
@@ -7675,6 +7710,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
|
||||
if (conf == NULL)
|
||||
goto abort;
|
||||
|
||||
conf->mddev = mddev;
|
||||
#if PAGE_SIZE != DEFAULT_STRIPE_SIZE
|
||||
conf->stripe_size = DEFAULT_STRIPE_SIZE;
|
||||
conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
|
||||
@@ -7733,7 +7769,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
|
||||
|
||||
if (!conf->disks)
|
||||
goto abort;
|
||||
|
||||
conf->disks_cnt = max_disks;
|
||||
for (i = 0; i < max_disks; i++) {
|
||||
conf->disks[i].extra_page = alloc_page(GFP_KERNEL);
|
||||
if (!conf->disks[i].extra_page)
|
||||
@@ -7743,7 +7779,6 @@ static struct r5conf *setup_conf(struct mddev *mddev)
|
||||
ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
|
||||
if (ret)
|
||||
goto abort;
|
||||
conf->mddev = mddev;
|
||||
|
||||
ret = -ENOMEM;
|
||||
conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);
|
||||
@@ -8046,6 +8081,7 @@ static int raid5_run(struct mddev *mddev)
|
||||
sector_t here_new, here_old;
|
||||
int old_disks;
|
||||
int max_degraded = (mddev->level == 6 ? 2 : 1);
|
||||
int min_disks = min_raid_disks(mddev->level);
|
||||
int chunk_sectors;
|
||||
int new_data_disks;
|
||||
|
||||
@@ -8061,6 +8097,12 @@ static int raid5_run(struct mddev *mddev)
|
||||
return -EINVAL;
|
||||
}
|
||||
old_disks = mddev->raid_disks - mddev->delta_disks;
|
||||
if (mddev->raid_disks < min_disks || old_disks < min_disks) {
|
||||
pr_warn("md/raid:%s: invalid reshape disks (%d -> %d, min %d)\n",
|
||||
mdname(mddev), old_disks, mddev->raid_disks,
|
||||
min_disks);
|
||||
return -EINVAL;
|
||||
}
|
||||
/* reshape_position must be on a new-stripe boundary, and one
|
||||
* further up in new geometry must map after here in old
|
||||
* geometry.
|
||||
@@ -8110,11 +8152,17 @@ static int raid5_run(struct mddev *mddev)
|
||||
}
|
||||
pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
|
||||
/* OK, we should be able to continue; */
|
||||
} else {
|
||||
BUG_ON(mddev->level != mddev->new_level);
|
||||
BUG_ON(mddev->layout != mddev->new_layout);
|
||||
BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
|
||||
BUG_ON(mddev->delta_disks != 0);
|
||||
} else if (mddev->level != mddev->new_level ||
|
||||
mddev->layout != mddev->new_layout ||
|
||||
mddev->chunk_sectors != mddev->new_chunk_sectors ||
|
||||
mddev->delta_disks != 0) {
|
||||
/* No reshape is in progress, but the array describes one.
|
||||
* The superblock validators do not cross-check these against
|
||||
* reshape_position, so this is reachable from disk.
|
||||
*/
|
||||
pr_warn("md/raid:%s: inconsistent reshape state - aborting.\n",
|
||||
mdname(mddev));
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (test_bit(MD_HAS_JOURNAL, &mddev->flags) &&
|
||||
@@ -8203,6 +8251,12 @@ static int raid5_run(struct mddev *mddev)
|
||||
|
||||
/* device size must be a multiple of chunk size */
|
||||
mddev->dev_sectors &= ~((sector_t)mddev->chunk_sectors - 1);
|
||||
if (!mddev->dev_sectors) {
|
||||
pr_warn("md/raid:%s: device size is smaller than one chunk\n",
|
||||
mdname(mddev));
|
||||
ret = -EINVAL;
|
||||
goto abort;
|
||||
}
|
||||
mddev->resync_max_sectors = mddev->dev_sectors;
|
||||
|
||||
if (mddev->degraded > dirty_parity_disks &&
|
||||
@@ -8536,6 +8590,8 @@ static int raid5_resize(struct mddev *mddev, sector_t sectors)
|
||||
if (raid5_has_log(conf) || raid5_has_ppl(conf))
|
||||
return -EINVAL;
|
||||
sectors &= ~((sector_t)conf->chunk_sectors - 1);
|
||||
if (!sectors)
|
||||
return -EINVAL;
|
||||
newsize = raid5_size(mddev, sectors, mddev->raid_disks);
|
||||
if (mddev->external_size &&
|
||||
mddev->array_sectors > newsize)
|
||||
@@ -8600,13 +8656,9 @@ static int check_reshape(struct mddev *mddev)
|
||||
if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
|
||||
/* We might be able to shrink, but the devices must
|
||||
* be made bigger first.
|
||||
* For raid6, 4 is the minimum size.
|
||||
* Otherwise 2 is the minimum
|
||||
*/
|
||||
int min = 2;
|
||||
if (mddev->level == 6)
|
||||
min = 4;
|
||||
if (mddev->raid_disks + mddev->delta_disks < min)
|
||||
if (mddev->raid_disks + mddev->delta_disks <
|
||||
min_raid_disks(mddev->level))
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
|
||||
@@ -667,6 +667,7 @@ struct r5conf {
|
||||
unsigned long cache_state;
|
||||
struct shrinker *shrinker;
|
||||
int pool_size; /* number of disks in stripeheads in pool */
|
||||
int disks_cnt; /* number of entries in disks[] array */
|
||||
spinlock_t device_lock;
|
||||
struct disk_info *disks;
|
||||
struct bio_set bio_split;
|
||||
|
||||
@@ -85,10 +85,12 @@ static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
|
||||
qid == 0 ? NVME_QID_ANY : qid, flags);
|
||||
if (ret > 0)
|
||||
dev_warn(ctrl->device,
|
||||
"qid %d auth_send failed with status %d\n", qid, ret);
|
||||
"qid %d auth_%s failed with status %d\n",
|
||||
qid, auth_send ? "send" : "receive", ret);
|
||||
else if (ret < 0)
|
||||
dev_err(ctrl->device,
|
||||
"qid %d auth_send failed with error %d\n", qid, ret);
|
||||
"qid %d auth_%s failed with error %d\n",
|
||||
qid, auth_send ? "send" : "receive", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@@ -1125,16 +1125,27 @@ out:
|
||||
void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues,
|
||||
u32 io_queues[HCTX_MAX_TYPES])
|
||||
{
|
||||
if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
|
||||
/*
|
||||
* Each queue type is mapped to the CPUs separately, so a type with
|
||||
* more queues than online CPUs has hardware contexts without an online
|
||||
* CPU. Use the same limits as nvmf_nr_io_queues.
|
||||
*/
|
||||
u32 nr_io = min(opts->nr_io_queues, num_online_cpus());
|
||||
u32 nr_write = min(opts->nr_write_queues, num_online_cpus());
|
||||
u32 nr_poll = min(opts->nr_poll_queues, num_online_cpus());
|
||||
|
||||
memset(io_queues, 0, sizeof(*io_queues) * HCTX_MAX_TYPES);
|
||||
|
||||
if (nr_write && nr_io < nr_io_queues) {
|
||||
/*
|
||||
* separate read/write queues
|
||||
* hand out dedicated default queues only after we have
|
||||
* sufficient read queues.
|
||||
*/
|
||||
io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
|
||||
io_queues[HCTX_TYPE_READ] = nr_io;
|
||||
nr_io_queues -= io_queues[HCTX_TYPE_READ];
|
||||
io_queues[HCTX_TYPE_DEFAULT] =
|
||||
min(opts->nr_write_queues, nr_io_queues);
|
||||
min(nr_write, nr_io_queues);
|
||||
nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
|
||||
} else {
|
||||
/*
|
||||
@@ -1143,14 +1154,14 @@ void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues,
|
||||
* sufficient queue count to have dedicated default queues.
|
||||
*/
|
||||
io_queues[HCTX_TYPE_DEFAULT] =
|
||||
min(opts->nr_io_queues, nr_io_queues);
|
||||
min(nr_io, nr_io_queues);
|
||||
nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT];
|
||||
}
|
||||
|
||||
if (opts->nr_poll_queues && nr_io_queues) {
|
||||
if (nr_poll && nr_io_queues) {
|
||||
/* map dedicated poll queues only if we have queues left */
|
||||
io_queues[HCTX_TYPE_POLL] =
|
||||
min(opts->nr_poll_queues, nr_io_queues);
|
||||
min(nr_poll, nr_io_queues);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(nvmf_set_io_queues);
|
||||
@@ -1188,6 +1199,13 @@ void nvmf_map_queues(struct blk_mq_tag_set *set, struct nvme_ctrl *ctrl,
|
||||
io_queues[HCTX_TYPE_DEFAULT] +
|
||||
io_queues[HCTX_TYPE_READ];
|
||||
blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
|
||||
} else {
|
||||
/*
|
||||
* blk_mq_alloc_tag_set sets nr_queues of all maps to
|
||||
* nr_hw_queues. Clear the poll map, otherwise the queue claims
|
||||
* poll support and polled I/O ends up on hctx 0.
|
||||
*/
|
||||
set->map[HCTX_TYPE_POLL].nr_queues = 0;
|
||||
}
|
||||
|
||||
dev_info(ctrl->device,
|
||||
|
||||
@@ -3778,6 +3778,9 @@ static ssize_t nvme_fc_nvme_discovery_store(struct device *dev,
|
||||
restart:
|
||||
list_for_each_entry(lport, &nvme_fc_lport_list, port_list) {
|
||||
list_for_each_entry(rport, &lport->endp_list, endp_list) {
|
||||
/* skip rports already queued for discovery */
|
||||
if (!list_empty(&rport->disc_list))
|
||||
continue;
|
||||
if (!nvme_fc_lport_get(lport))
|
||||
continue;
|
||||
if (!nvme_fc_rport_get(rport)) {
|
||||
@@ -3799,9 +3802,7 @@ restart:
|
||||
"failures\n");
|
||||
goto process_local_list;
|
||||
}
|
||||
if (list_empty(&rport->disc_list))
|
||||
list_add_tail(&rport->disc_list,
|
||||
&local_disc_list);
|
||||
list_add_tail(&rport->disc_list, &local_disc_list);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -233,7 +233,7 @@ out_free_val:
|
||||
return err;
|
||||
}
|
||||
|
||||
static char quirks_param[128];
|
||||
static char quirks_param[256];
|
||||
static const struct kernel_param_ops quirks_param_ops = {
|
||||
.set = quirks_param_set,
|
||||
.get = param_get_string,
|
||||
@@ -244,7 +244,7 @@ static struct kparam_string quirks_param_string = {
|
||||
.string = quirks_param,
|
||||
};
|
||||
|
||||
module_param_cb(quirks, &quirks_param_ops, &quirks_param_string, 0444);
|
||||
device_param_cb(quirks, &quirks_param_ops, &quirks_param_string, 0444);
|
||||
MODULE_PARM_DESC(quirks, "Enable/disable NVMe quirks by specifying "
|
||||
"quirks=VID:DID:quirk_names");
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ static int nvme_rdma_init_request(struct blk_mq_tag_set *set,
|
||||
return -ENOMEM;
|
||||
|
||||
/* metadata nvme_rdma_sgl struct is located after command's data SGL */
|
||||
if (queue->pi_support)
|
||||
if (queue_idx && ctrl->ctrl.max_integrity_segments)
|
||||
req->metadata_sgl = (void *)nvme_req(rq) +
|
||||
sizeof(struct nvme_rdma_request) +
|
||||
NVME_RDMA_DATA_SGL_SIZE;
|
||||
|
||||
+244
-65
@@ -6,6 +6,7 @@
|
||||
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
||||
#include <linux/module.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/async.h>
|
||||
#include <linux/err.h>
|
||||
@@ -48,13 +49,34 @@ MODULE_PARM_DESC(wq_unbound, "Use unbound workqueue for nvme-tcp IO context (def
|
||||
/*
|
||||
* TLS handshake timeout
|
||||
*/
|
||||
static int tls_handshake_timeout = 10;
|
||||
static unsigned int tls_handshake_timeout = 10;
|
||||
#ifdef CONFIG_NVME_TCP_TLS
|
||||
module_param(tls_handshake_timeout, int, 0644);
|
||||
static int tls_handshake_timeout_set(const char *val,
|
||||
const struct kernel_param *kp)
|
||||
{
|
||||
return param_set_uint_minmax(val, kp, 0,
|
||||
min_t(unsigned long, UINT_MAX / MSEC_PER_SEC,
|
||||
MAX_JIFFY_OFFSET / HZ));
|
||||
}
|
||||
|
||||
static const struct kernel_param_ops tls_handshake_timeout_ops = {
|
||||
.set = tls_handshake_timeout_set,
|
||||
.get = param_get_uint,
|
||||
};
|
||||
|
||||
module_param_cb(tls_handshake_timeout, &tls_handshake_timeout_ops,
|
||||
&tls_handshake_timeout, 0644);
|
||||
__MODULE_PARM_TYPE(tls_handshake_timeout, "uint");
|
||||
MODULE_PARM_DESC(tls_handshake_timeout,
|
||||
"nvme TLS handshake timeout in seconds (default 10)");
|
||||
#endif
|
||||
|
||||
/* read a TLS control record in chunks of this size */
|
||||
#define NVME_TCP_TLS_REC_CHUNK 256
|
||||
|
||||
/* TLS records read while waiting for the ICResp, the ICResp included */
|
||||
#define NVME_TCP_ICRESP_MAX_RECORDS 8
|
||||
|
||||
static int nvme_tcp_cpu_queues[NR_CPUS];
|
||||
static DEFINE_SPINLOCK(nvme_tcp_cpu_queues_lock);
|
||||
|
||||
@@ -97,6 +119,9 @@ enum nvme_tcp_queue_flags {
|
||||
NVME_TCP_Q_IO_CPU_SET = 3,
|
||||
};
|
||||
|
||||
#define NVME_TCP_OPT_NR_QUEUES(opts) ((opts)->nr_io_queues + \
|
||||
(opts)->nr_write_queues + (opts)->nr_poll_queues + 1)
|
||||
|
||||
enum nvme_tcp_recv_state {
|
||||
NVME_TCP_RECV_PDU = 0,
|
||||
NVME_TCP_RECV_DATA,
|
||||
@@ -1430,6 +1455,19 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
|
||||
lock_sock(sk);
|
||||
queue->nr_cqe = 0;
|
||||
consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
|
||||
/*
|
||||
* nvme_tcp_recv_skb() handles its own errors, so a queue that is
|
||||
* still enabled failed inside read_sock(). Handle it before
|
||||
* release_sock() processes the backlog.
|
||||
*/
|
||||
if (unlikely(consumed < 0 && consumed != -EAGAIN &&
|
||||
queue->rd_enabled)) {
|
||||
dev_err(queue->ctrl->ctrl.device,
|
||||
"queue %d: receive failed: %d\n",
|
||||
nvme_tcp_queue_id(queue), consumed);
|
||||
queue->rd_enabled = false;
|
||||
nvme_tcp_error_recovery(&queue->ctrl->ctrl);
|
||||
}
|
||||
release_sock(sk);
|
||||
return consumed == -EAGAIN ? 0 : consumed;
|
||||
}
|
||||
@@ -1442,9 +1480,11 @@ static void nvme_tcp_io_work(struct work_struct *w)
|
||||
|
||||
do {
|
||||
bool pending = false;
|
||||
bool got_send_mutex = false;
|
||||
int result;
|
||||
|
||||
if (mutex_trylock(&queue->send_mutex)) {
|
||||
got_send_mutex = true;
|
||||
result = nvme_tcp_try_send(queue);
|
||||
mutex_unlock(&queue->send_mutex);
|
||||
if (result > 0)
|
||||
@@ -1459,8 +1499,13 @@ static void nvme_tcp_io_work(struct work_struct *w)
|
||||
else if (unlikely(result < 0))
|
||||
return;
|
||||
|
||||
/* did we get some space after spending time in recv? */
|
||||
if (nvme_tcp_queue_has_pending(queue) &&
|
||||
/*
|
||||
* did we get some space after spending time in recv?
|
||||
* Only check this if we could take send_mutex. Otherwise
|
||||
* the task holding it sends the requests and queues io_work
|
||||
* again after it releases the mutex.
|
||||
*/
|
||||
if (got_send_mutex && nvme_tcp_queue_has_pending(queue) &&
|
||||
sk_stream_is_writeable(queue->sock->sk))
|
||||
pending = true;
|
||||
|
||||
@@ -1523,23 +1568,159 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
|
||||
* nvme_reset_ctrl_work(), queue teardown is immediately followed by
|
||||
* re-allocation. fput() defers socket cleanup to delayed_fput_work
|
||||
* in workqueue context, which can race with new queue setup.
|
||||
*
|
||||
* Clear queue->sock under the queue lock so that a sysfs reader in
|
||||
* nvme_tcp_get_address() either observes the NVME_TCP_Q_LIVE bit
|
||||
* cleared or a still valid socket. The queue lock itself is owned
|
||||
* by the controller and only destroyed when the controller is
|
||||
* freed, so readers blocked on it cannot race its teardown.
|
||||
*/
|
||||
mutex_lock(&queue->queue_lock);
|
||||
__fput_sync(queue->sock->file);
|
||||
queue->sock = NULL;
|
||||
mutex_unlock(&queue->queue_lock);
|
||||
memalloc_noio_restore(noio_flag);
|
||||
|
||||
kfree(queue->pdu);
|
||||
mutex_destroy(&queue->send_mutex);
|
||||
mutex_destroy(&queue->queue_lock);
|
||||
mutex_destroy(&queue->pf_cache_lock);
|
||||
}
|
||||
|
||||
static int nvme_tcp_recv_tls_alert(struct nvme_tcp_queue *queue,
|
||||
struct msghdr *msg, int len)
|
||||
{
|
||||
struct sock *sk = queue->sock->sk;
|
||||
u8 level, description;
|
||||
|
||||
if (len < 2) {
|
||||
dev_err(queue->ctrl->ctrl.device,
|
||||
"queue %d: truncated TLS alert\n",
|
||||
nvme_tcp_queue_id(queue));
|
||||
return -ENOTCONN;
|
||||
}
|
||||
|
||||
/* tls_alert_recv() reads through the iterator */
|
||||
iov_iter_revert(&msg->msg_iter, len);
|
||||
tls_alert_recv(sk, msg, &level, &description);
|
||||
|
||||
if (level == TLS_ALERT_LEVEL_FATAL) {
|
||||
dev_err(queue->ctrl->ctrl.device,
|
||||
"queue %d: fatal TLS alert %u\n",
|
||||
nvme_tcp_queue_id(queue), description);
|
||||
return -ENOTCONN;
|
||||
}
|
||||
|
||||
/* a peer shutting down normally is not worth a warning */
|
||||
if (description == TLS_ALERT_DESC_CLOSE_NOTIFY)
|
||||
dev_dbg(queue->ctrl->ctrl.device,
|
||||
"queue %d: TLS close_notify\n",
|
||||
nvme_tcp_queue_id(queue));
|
||||
else
|
||||
dev_warn(queue->ctrl->ctrl.device,
|
||||
"queue %d: TLS alert %u\n",
|
||||
nvme_tcp_queue_id(queue), description);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Discard a TLS record that arrived in place of the ICResp. A peer may send
|
||||
* a NewSessionTicket as soon as the handshake completes.
|
||||
*
|
||||
* Returns 0 when the record was discarded and the ICResp can be read, or a
|
||||
* negative errno if the connection cannot continue.
|
||||
*/
|
||||
static int nvme_tcp_skip_tls_record(struct nvme_tcp_queue *queue,
|
||||
struct msghdr *msg, u8 ctype, int len)
|
||||
{
|
||||
u8 buf[NVME_TCP_TLS_REC_CHUNK];
|
||||
int ret;
|
||||
|
||||
switch (ctype) {
|
||||
case TLS_RECORD_TYPE_HANDSHAKE:
|
||||
break;
|
||||
case TLS_RECORD_TYPE_ALERT:
|
||||
ret = nvme_tcp_recv_tls_alert(queue, msg, len);
|
||||
return ret ? : -ENOTCONN;
|
||||
default:
|
||||
pr_err("queue %d: unhandled TLS record %u\n",
|
||||
nvme_tcp_queue_id(queue), ctype);
|
||||
return -ENOTCONN;
|
||||
}
|
||||
|
||||
/* the ICResp is in a later record, drain this one first */
|
||||
while (!(msg->msg_flags & MSG_EOR)) {
|
||||
struct kvec iov = {
|
||||
.iov_base = buf,
|
||||
.iov_len = sizeof(buf),
|
||||
};
|
||||
char cbuf[CMSG_LEN(sizeof(char))] = {};
|
||||
/* kTLS fails a control record read without a cmsg buffer */
|
||||
struct msghdr drain = {
|
||||
.msg_control = cbuf,
|
||||
.msg_controllen = sizeof(cbuf),
|
||||
};
|
||||
|
||||
ret = kernel_recvmsg(queue->sock, &drain, &iov, 1, sizeof(buf),
|
||||
0);
|
||||
if (ret <= 0) {
|
||||
pr_warn("queue %d: failed to drain TLS record: %d\n",
|
||||
nvme_tcp_queue_id(queue), ret);
|
||||
return ret ? : -ECONNRESET;
|
||||
}
|
||||
msg->msg_flags = drain.msg_flags;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Receive the ICResp. Over TLS, skip the records that a peer may send
|
||||
* ahead of it.
|
||||
*
|
||||
* Returns the length received, or a negative errno.
|
||||
*/
|
||||
static int nvme_tcp_recv_icresp(struct nvme_tcp_queue *queue,
|
||||
struct nvme_tcp_icresp_pdu *icresp)
|
||||
{
|
||||
char cbuf[CMSG_LEN(sizeof(char))];
|
||||
struct msghdr msg;
|
||||
struct kvec iov;
|
||||
u8 ctype;
|
||||
int i, ret;
|
||||
|
||||
for (i = 0; i < NVME_TCP_ICRESP_MAX_RECORDS; i++) {
|
||||
memset(&msg, 0, sizeof(msg));
|
||||
memset(cbuf, 0, sizeof(cbuf));
|
||||
iov.iov_base = icresp;
|
||||
iov.iov_len = sizeof(*icresp);
|
||||
if (nvme_tcp_queue_tls(queue)) {
|
||||
msg.msg_control = cbuf;
|
||||
msg.msg_controllen = sizeof(cbuf);
|
||||
}
|
||||
msg.msg_flags = MSG_WAITALL;
|
||||
ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
|
||||
iov.iov_len, msg.msg_flags);
|
||||
if (ret <= 0 || !nvme_tcp_queue_tls(queue))
|
||||
return ret;
|
||||
|
||||
ctype = tls_get_record_type(queue->sock->sk,
|
||||
(struct cmsghdr *)cbuf);
|
||||
if (ctype == TLS_RECORD_TYPE_DATA)
|
||||
return ret;
|
||||
|
||||
ret = nvme_tcp_skip_tls_record(queue, &msg, ctype, ret);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
pr_err("queue %d: too many TLS records before the ICResp\n",
|
||||
nvme_tcp_queue_id(queue));
|
||||
return -EPROTO;
|
||||
}
|
||||
|
||||
static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
|
||||
{
|
||||
struct nvme_tcp_icreq_pdu *icreq;
|
||||
struct nvme_tcp_icresp_pdu *icresp;
|
||||
char cbuf[CMSG_LEN(sizeof(char))] = {};
|
||||
u8 ctype;
|
||||
struct msghdr msg = {};
|
||||
struct kvec iov;
|
||||
bool ctrl_hdgst, ctrl_ddgst;
|
||||
@@ -1577,16 +1758,7 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
|
||||
goto free_icresp;
|
||||
}
|
||||
|
||||
memset(&msg, 0, sizeof(msg));
|
||||
iov.iov_base = icresp;
|
||||
iov.iov_len = sizeof(*icresp);
|
||||
if (nvme_tcp_queue_tls(queue)) {
|
||||
msg.msg_control = cbuf;
|
||||
msg.msg_controllen = sizeof(cbuf);
|
||||
}
|
||||
msg.msg_flags = MSG_WAITALL;
|
||||
ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
|
||||
iov.iov_len, msg.msg_flags);
|
||||
ret = nvme_tcp_recv_icresp(queue, icresp);
|
||||
if (ret >= 0 && ret < sizeof(*icresp))
|
||||
ret = -ECONNRESET;
|
||||
if (ret < 0) {
|
||||
@@ -1594,16 +1766,6 @@ static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
|
||||
nvme_tcp_queue_id(queue), ret);
|
||||
goto free_icresp;
|
||||
}
|
||||
ret = -ENOTCONN;
|
||||
if (nvme_tcp_queue_tls(queue)) {
|
||||
ctype = tls_get_record_type(queue->sock->sk,
|
||||
(struct cmsghdr *)cbuf);
|
||||
if (ctype != TLS_RECORD_TYPE_DATA) {
|
||||
pr_err("queue %d: unhandled TLS record %d\n",
|
||||
nvme_tcp_queue_id(queue), ctype);
|
||||
goto free_icresp;
|
||||
}
|
||||
}
|
||||
ret = -EINVAL;
|
||||
if (icresp->hdr.type != nvme_tcp_icresp) {
|
||||
pr_err("queue %d: bad type returned %d\n",
|
||||
@@ -1795,8 +1957,10 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
|
||||
{
|
||||
int qid = nvme_tcp_queue_id(queue);
|
||||
int ret;
|
||||
long wait_ret;
|
||||
struct tls_handshake_args args;
|
||||
unsigned long tmo = tls_handshake_timeout * HZ;
|
||||
unsigned int timeout = READ_ONCE(tls_handshake_timeout);
|
||||
unsigned long tmo = (unsigned long)timeout * HZ;
|
||||
key_serial_t keyring = nvme_keyring_id();
|
||||
|
||||
dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n",
|
||||
@@ -1810,7 +1974,7 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
|
||||
if (nctrl->opts->keyring)
|
||||
keyring = key_serial(nctrl->opts->keyring);
|
||||
args.ta_keyring = keyring;
|
||||
args.ta_timeout_ms = tls_handshake_timeout * 1000;
|
||||
args.ta_timeout_ms = timeout * MSEC_PER_SEC;
|
||||
queue->tls_err = -EOPNOTSUPP;
|
||||
init_completion(&queue->tls_complete);
|
||||
ret = tls_client_hello_psk(&args, GFP_KERNEL);
|
||||
@@ -1819,10 +1983,10 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
|
||||
qid, ret);
|
||||
return ret;
|
||||
}
|
||||
ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo);
|
||||
if (ret <= 0) {
|
||||
if (ret == 0)
|
||||
ret = -ETIMEDOUT;
|
||||
wait_ret = wait_for_completion_interruptible_timeout(&queue->tls_complete,
|
||||
tmo);
|
||||
if (wait_ret <= 0) {
|
||||
ret = wait_ret < 0 ? -ERESTARTSYS : -ETIMEDOUT;
|
||||
|
||||
dev_err(nctrl->device,
|
||||
"queue %d: TLS handshake failed, error %d\n",
|
||||
@@ -1842,6 +2006,28 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void nvme_tcp_init_queue(struct nvme_tcp_ctrl *ctrl, int qid)
|
||||
{
|
||||
struct nvme_tcp_queue *queue = &ctrl->queues[qid];
|
||||
|
||||
queue->ctrl = ctrl;
|
||||
mutex_init(&queue->queue_lock);
|
||||
mutex_init(&queue->send_mutex);
|
||||
mutex_init(&queue->pf_cache_lock);
|
||||
INIT_WORK(&queue->io_work, nvme_tcp_io_work);
|
||||
queue->hdr_digest = ctrl->ctrl.opts->hdr_digest;
|
||||
queue->data_digest = ctrl->ctrl.opts->data_digest;
|
||||
}
|
||||
|
||||
static void nvme_tcp_exit_queue(struct nvme_tcp_ctrl *ctrl, int qid)
|
||||
{
|
||||
struct nvme_tcp_queue *queue = &ctrl->queues[qid];
|
||||
|
||||
mutex_destroy(&queue->queue_lock);
|
||||
mutex_destroy(&queue->send_mutex);
|
||||
mutex_destroy(&queue->pf_cache_lock);
|
||||
}
|
||||
|
||||
static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
|
||||
key_serial_t pskid)
|
||||
{
|
||||
@@ -1850,13 +2036,8 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
|
||||
int ret, rcv_pdu_size;
|
||||
struct file *sock_file;
|
||||
|
||||
mutex_init(&queue->queue_lock);
|
||||
queue->ctrl = ctrl;
|
||||
init_llist_head(&queue->req_list);
|
||||
INIT_LIST_HEAD(&queue->send_list);
|
||||
mutex_init(&queue->send_mutex);
|
||||
INIT_WORK(&queue->io_work, nvme_tcp_io_work);
|
||||
mutex_init(&queue->pf_cache_lock);
|
||||
|
||||
if (qid > 0)
|
||||
queue->cmnd_capsule_len = nctrl->ioccsz * 16;
|
||||
@@ -1870,14 +2051,12 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
|
||||
if (ret) {
|
||||
dev_err(nctrl->device,
|
||||
"failed to create socket: %d\n", ret);
|
||||
goto err_destroy_mutex;
|
||||
return ret;
|
||||
}
|
||||
|
||||
sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL);
|
||||
if (IS_ERR(sock_file)) {
|
||||
ret = PTR_ERR(sock_file);
|
||||
goto err_destroy_mutex;
|
||||
}
|
||||
if (IS_ERR(sock_file))
|
||||
return PTR_ERR(sock_file);
|
||||
|
||||
sk_net_refcnt_upgrade(queue->sock->sk);
|
||||
|
||||
@@ -1939,9 +2118,6 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
|
||||
}
|
||||
}
|
||||
|
||||
queue->hdr_digest = nctrl->opts->hdr_digest;
|
||||
queue->data_digest = nctrl->opts->data_digest;
|
||||
|
||||
rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
|
||||
nvme_tcp_hdgst_len(queue);
|
||||
queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
|
||||
@@ -1986,10 +2162,6 @@ err_sock:
|
||||
/* Use sync variant - see nvme_tcp_free_queue() for explanation */
|
||||
__fput_sync(queue->sock->file);
|
||||
queue->sock = NULL;
|
||||
err_destroy_mutex:
|
||||
mutex_destroy(&queue->send_mutex);
|
||||
mutex_destroy(&queue->queue_lock);
|
||||
mutex_destroy(&queue->pf_cache_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -2676,6 +2848,9 @@ static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
|
||||
static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
|
||||
{
|
||||
struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
|
||||
/* the same count used to size and initialize the queue array */
|
||||
unsigned int nr_queues = NVME_TCP_OPT_NR_QUEUES(nctrl->opts);
|
||||
unsigned int i;
|
||||
|
||||
if (list_empty_careful(&ctrl->list))
|
||||
goto free_ctrl;
|
||||
@@ -2687,6 +2862,8 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
|
||||
nvmf_free_options(nctrl->opts);
|
||||
free_ctrl:
|
||||
put_net(ctrl->net);
|
||||
for (i = 0; i < nr_queues; i++)
|
||||
nvme_tcp_exit_queue(ctrl, i);
|
||||
kfree(ctrl->queues);
|
||||
kfree(ctrl);
|
||||
}
|
||||
@@ -2939,17 +3116,18 @@ static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
|
||||
|
||||
len = nvmf_get_address(ctrl, buf, size);
|
||||
|
||||
if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
|
||||
return len;
|
||||
|
||||
mutex_lock(&queue->queue_lock);
|
||||
|
||||
ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr);
|
||||
if (ret > 0) {
|
||||
if (len > 0)
|
||||
len--; /* strip trailing newline */
|
||||
len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n",
|
||||
(len) ? "," : "", &src_addr);
|
||||
if (test_bit(NVME_TCP_Q_LIVE, &queue->flags)) {
|
||||
ret = kernel_getsockname(queue->sock,
|
||||
(struct sockaddr *)&src_addr);
|
||||
if (ret > 0) {
|
||||
if (len > 0)
|
||||
len--; /* strip trailing newline */
|
||||
len += scnprintf(buf + len, size - len,
|
||||
"%ssrc_addr=%pISc\n",
|
||||
(len) ? "," : "", &src_addr);
|
||||
}
|
||||
}
|
||||
|
||||
mutex_unlock(&queue->queue_lock);
|
||||
@@ -3015,7 +3193,7 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,
|
||||
struct nvmf_ctrl_options *opts)
|
||||
{
|
||||
struct nvme_tcp_ctrl *ctrl;
|
||||
int ret;
|
||||
int ret, i;
|
||||
|
||||
ctrl = kzalloc_obj(*ctrl);
|
||||
if (!ctrl)
|
||||
@@ -3026,8 +3204,7 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,
|
||||
*/
|
||||
context_unsafe(INIT_LIST_HEAD(&ctrl->list));
|
||||
ctrl->ctrl.opts = opts;
|
||||
ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
|
||||
opts->nr_poll_queues + 1;
|
||||
ctrl->ctrl.queue_count = NVME_TCP_OPT_NR_QUEUES(opts);
|
||||
ctrl->ctrl.sqsize = opts->queue_size - 1;
|
||||
ctrl->ctrl.kato = opts->kato;
|
||||
|
||||
@@ -3085,8 +3262,10 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,
|
||||
goto out_free_ctrl;
|
||||
}
|
||||
|
||||
ctrl->net = get_net(current->nsproxy->net_ns);
|
||||
for (i = 0; i < ctrl->ctrl.queue_count; i++)
|
||||
nvme_tcp_init_queue(ctrl, i);
|
||||
|
||||
ctrl->net = get_net(current->nsproxy->net_ns);
|
||||
ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
|
||||
if (ret)
|
||||
goto out_kfree_queues;
|
||||
|
||||
@@ -237,6 +237,7 @@ void nvmet_auth_sq_free(struct nvmet_sq *sq)
|
||||
sq->dhchap_c2 = NULL;
|
||||
kfree(sq->dhchap_skey);
|
||||
sq->dhchap_skey = NULL;
|
||||
sq->dhchap_skey_len = 0;
|
||||
}
|
||||
|
||||
void nvmet_auth_sq_destroy(struct nvmet_sq *sq)
|
||||
|
||||
@@ -387,10 +387,15 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item,
|
||||
const char *page, size_t count)
|
||||
{
|
||||
struct nvmet_port *port = to_nvmet_port(item);
|
||||
ssize_t ret = count;
|
||||
int i;
|
||||
|
||||
if (nvmet_is_port_enabled(port, __func__))
|
||||
return -EACCES;
|
||||
down_write(&nvmet_config_sem);
|
||||
|
||||
if (nvmet_is_port_enabled(port, __func__)) {
|
||||
ret = -EACCES;
|
||||
goto out;
|
||||
}
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(nvmet_transport); i++) {
|
||||
if (sysfs_streq(page, nvmet_transport[i].name))
|
||||
@@ -398,7 +403,8 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item,
|
||||
}
|
||||
|
||||
pr_err("Invalid value '%s' for trtype\n", page);
|
||||
return -EINVAL;
|
||||
ret = -EINVAL;
|
||||
goto out;
|
||||
|
||||
found:
|
||||
memset(&port->disc_addr.tsas, 0, NVMF_TSAS_SIZE);
|
||||
@@ -407,7 +413,9 @@ found:
|
||||
nvmet_port_init_tsas_rdma(port);
|
||||
else if (port->disc_addr.trtype == NVMF_TRTYPE_TCP)
|
||||
nvmet_port_init_tsas_tcp(port, NVMF_TCP_SECTYPE_NONE);
|
||||
return count;
|
||||
out:
|
||||
up_write(&nvmet_config_sem);
|
||||
return ret;
|
||||
}
|
||||
|
||||
CONFIGFS_ATTR(nvmet_, addr_trtype);
|
||||
@@ -1783,8 +1791,10 @@ static ssize_t nvmet_subsys_attr_qid_max_store(struct config_item *item,
|
||||
subsys->max_qid = qid_max;
|
||||
|
||||
/* Force reconnect */
|
||||
mutex_lock(&subsys->lock);
|
||||
list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
|
||||
ctrl->ops->delete_ctrl(ctrl);
|
||||
mutex_unlock(&subsys->lock);
|
||||
up_write(&nvmet_config_sem);
|
||||
|
||||
return cnt;
|
||||
@@ -2104,7 +2114,9 @@ static void nvmet_port_release(struct config_item *item)
|
||||
|
||||
/* Let inflight controllers teardown complete */
|
||||
flush_workqueue(nvmet_wq);
|
||||
down_write(&nvmet_config_sem);
|
||||
list_del(&port->global_entry);
|
||||
up_write(&nvmet_config_sem);
|
||||
|
||||
key_put(port->keyring);
|
||||
kfree(port);
|
||||
@@ -2165,8 +2177,6 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
|
||||
port->ana_state[i] = NVME_ANA_INACCESSIBLE;
|
||||
}
|
||||
|
||||
list_add(&port->global_entry, &nvmet_ports_list);
|
||||
|
||||
INIT_LIST_HEAD(&port->entry);
|
||||
INIT_LIST_HEAD(&port->subsystems);
|
||||
INIT_LIST_HEAD(&port->referrals);
|
||||
@@ -2200,6 +2210,10 @@ static struct config_group *nvmet_ports_make(struct config_group *group,
|
||||
configfs_add_default_group(&port->ana_default_group.group,
|
||||
&port->ana_groups_group);
|
||||
|
||||
down_write(&nvmet_config_sem);
|
||||
list_add(&port->global_entry, &nvmet_ports_list);
|
||||
up_write(&nvmet_config_sem);
|
||||
|
||||
return &port->group;
|
||||
}
|
||||
|
||||
|
||||
@@ -393,11 +393,13 @@ void nvmet_disable_port(struct nvmet_port *port)
|
||||
|
||||
lockdep_assert_held(&nvmet_config_sem);
|
||||
|
||||
port->enabled = false;
|
||||
port->tr_ops = NULL;
|
||||
if (!port->enabled)
|
||||
return;
|
||||
|
||||
ops = nvmet_transports[port->disc_addr.trtype];
|
||||
ops = port->tr_ops;
|
||||
port->enabled = false;
|
||||
ops->remove_port(port);
|
||||
port->tr_ops = NULL;
|
||||
module_put(ops->owner);
|
||||
}
|
||||
|
||||
@@ -1101,6 +1103,9 @@ static u32 nvmet_io_cmd_transfer_len(struct nvmet_req *req)
|
||||
case nvme_cmd_dsm:
|
||||
return nvmet_dsm_len(req);
|
||||
case nvme_cmd_zone_mgmt_recv:
|
||||
/* The handler rejects a NUMD whose byte count would wrap. */
|
||||
if (le32_to_cpu(req->cmd->zmr.numd) >= U32_MAX / 4)
|
||||
return 0;
|
||||
return (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2;
|
||||
default:
|
||||
return 0;
|
||||
@@ -1362,7 +1367,7 @@ int nvmet_req_alloc_sgls(struct nvmet_req *req)
|
||||
|
||||
return 0;
|
||||
out_free:
|
||||
sgl_free(req->sg);
|
||||
nvmet_req_free_sgls(req);
|
||||
out:
|
||||
return -ENOMEM;
|
||||
}
|
||||
@@ -1823,6 +1828,9 @@ ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl,
|
||||
{
|
||||
if (!ctrl->ops->host_traddr)
|
||||
return -EOPNOTSUPP;
|
||||
/* They all take the address from the admin queue */
|
||||
if (!ctrl->sqs[0])
|
||||
return -ENOTCONN;
|
||||
return ctrl->ops->host_traddr(ctrl, traddr, traddr_len);
|
||||
}
|
||||
|
||||
|
||||
@@ -330,6 +330,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
|
||||
/* Restart negotiation */
|
||||
pr_debug("%s: ctrl %d qid %d reset negotiation\n",
|
||||
__func__, ctrl->cntlid, req->sq->qid);
|
||||
nvmet_auth_sq_free(req->sq);
|
||||
if (!req->sq->qid) {
|
||||
dhchap_status = nvmet_setup_auth(ctrl, req->sq,
|
||||
true);
|
||||
@@ -344,8 +345,9 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
|
||||
}
|
||||
req->sq->dhchap_step =
|
||||
NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
|
||||
} else if (data->auth_id != req->sq->dhchap_step)
|
||||
} else {
|
||||
goto done_failure1;
|
||||
}
|
||||
/* Validate negotiation parameters */
|
||||
dhchap_status = nvmet_auth_negotiate(req, d, tl);
|
||||
if (dhchap_status == 0)
|
||||
@@ -605,10 +607,11 @@ void nvmet_execute_auth_receive(struct nvmet_req *req)
|
||||
switch (req->sq->dhchap_step) {
|
||||
case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
|
||||
if (nvmet_auth_challenge(req, d, al) < 0) {
|
||||
pr_warn("ctrl %d qid %d: challenge error (%d)\n",
|
||||
ctrl->cntlid, req->sq->qid, status);
|
||||
pr_warn("ctrl %d qid %d: challenge error\n",
|
||||
ctrl->cntlid, req->sq->qid);
|
||||
status = NVME_SC_INTERNAL;
|
||||
break;
|
||||
kfree(d);
|
||||
goto done;
|
||||
}
|
||||
req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
|
||||
break;
|
||||
|
||||
@@ -186,8 +186,7 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
|
||||
pr_warn("queue size zero!\n");
|
||||
req->error_loc = offsetof(struct nvmf_connect_command, sqsize);
|
||||
req->cqe->result.u32 = IPO_IATTR_CONNECT_SQE(sqsize);
|
||||
ret = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR;
|
||||
goto err;
|
||||
return NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR;
|
||||
}
|
||||
|
||||
if (ctrl->sqs[qid] != NULL) {
|
||||
|
||||
@@ -1708,7 +1708,7 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
|
||||
be16_to_cpu(rqst->assoc_cmd.sqsize));
|
||||
if (!queue) {
|
||||
ret = VERR_QUEUE_ALLOC_FAIL;
|
||||
nvmet_fc_tgt_a_put(iod->assoc);
|
||||
nvmet_fc_schedule_delete_assoc(iod->assoc);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1799,9 +1799,6 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
|
||||
be16_to_cpu(rqst->connect_cmd.sqsize));
|
||||
if (!queue)
|
||||
ret = VERR_QUEUE_ALLOC_FAIL;
|
||||
|
||||
/* release get taken in nvmet_fc_find_target_assoc */
|
||||
nvmet_fc_tgt_a_put(iod->assoc);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1815,6 +1812,8 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
|
||||
FCNVME_RJT_RC_INV_ASSOC :
|
||||
FCNVME_RJT_RC_LOGIC,
|
||||
FCNVME_RJT_EXP_NONE, 0);
|
||||
if (iod->assoc)
|
||||
nvmet_fc_tgt_a_put(iod->assoc);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1836,6 +1835,9 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
|
||||
acc->connectid.connection_id =
|
||||
cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
|
||||
be16_to_cpu(rqst->connect_cmd.qid)));
|
||||
|
||||
/* release get taken in nvmet_fc_find_target_assoc */
|
||||
nvmet_fc_tgt_a_put(iod->assoc);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -2487,11 +2489,11 @@ __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
|
||||
} else {
|
||||
|
||||
/*
|
||||
* try to push the data even if the SQE status is non-zero.
|
||||
* There may be a status where data still was intended to
|
||||
* be moved
|
||||
* A failed command may not have filled its read buffer, so
|
||||
* only push data for success, as the tcp and rdma targets do.
|
||||
*/
|
||||
if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
|
||||
if (fod->io_dir == NVMET_FCP_READ && fod->data_sg_cnt &&
|
||||
!cqe->status) {
|
||||
/* push the data over before sending rsp */
|
||||
nvmet_fc_transfer_fcp_data(tgtport, fod,
|
||||
NVMET_FCOP_READDATA);
|
||||
|
||||
@@ -316,7 +316,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
|
||||
&prot_miter);
|
||||
if (unlikely(rc)) {
|
||||
bio_io_error(bio);
|
||||
return;
|
||||
goto out_plug;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -336,11 +336,12 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req)
|
||||
rc = nvmet_bdev_alloc_bip(req, bio, &prot_miter);
|
||||
if (unlikely(rc)) {
|
||||
bio_io_error(bio);
|
||||
return;
|
||||
goto out_plug;
|
||||
}
|
||||
}
|
||||
|
||||
submit_bio(bio);
|
||||
out_plug:
|
||||
blk_finish_plug(&plug);
|
||||
}
|
||||
|
||||
@@ -389,7 +390,8 @@ static void nvmet_bdev_execute_discard(struct nvmet_req *req)
|
||||
if (status)
|
||||
break;
|
||||
|
||||
nr_sects = le32_to_cpu(range.nlb) << (ns->blksize_shift - 9);
|
||||
nr_sects = (sector_t)le32_to_cpu(range.nlb) <<
|
||||
(ns->blksize_shift - 9);
|
||||
__blkdev_issue_discard(ns->bdev,
|
||||
nvmet_lba_to_sect(ns, range.slba), nr_sects,
|
||||
GFP_KERNEL, &bio);
|
||||
|
||||
@@ -221,8 +221,6 @@ static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
|
||||
struct nvme_loop_ctrl *ctrl = to_loop_ctrl(data);
|
||||
struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
|
||||
|
||||
BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
|
||||
|
||||
/*
|
||||
* flush_end_io() can be called recursively for us, so use our own
|
||||
* lock class key for avoiding lockdep possible recursive locking,
|
||||
|
||||
@@ -959,7 +959,8 @@ static inline void nvmet_req_bio_put(struct nvmet_req *req, struct bio *bio)
|
||||
#ifdef CONFIG_NVME_TARGET_TCP_TLS
|
||||
static inline key_serial_t nvmet_queue_tls_keyid(struct nvmet_sq *sq)
|
||||
{
|
||||
return sq->tls_key ? key_serial(sq->tls_key) : 0;
|
||||
/* A PCI endpoint controller is created without a queue */
|
||||
return sq && sq->tls_key ? key_serial(sq->tls_key) : 0;
|
||||
}
|
||||
static inline void nvmet_sq_put_tls_key(struct nvmet_sq *sq)
|
||||
{
|
||||
|
||||
@@ -1073,10 +1073,22 @@ static int nvmet_pci_epf_iod_parse_sgl_segments(struct nvmet_pci_epf_ctrl *ctrl,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static u64 nvmet_pci_epf_iod_data_segs_len(struct nvmet_pci_epf_iod *iod)
|
||||
{
|
||||
u64 len = 0;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < iod->nr_data_segs; i++)
|
||||
len += iod->data_segs[i].length;
|
||||
|
||||
return len;
|
||||
}
|
||||
|
||||
static int nvmet_pci_epf_iod_parse_sgls(struct nvmet_pci_epf_iod *iod)
|
||||
{
|
||||
struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl;
|
||||
struct nvme_sgl_desc *sgl = &iod->cmd.common.dptr.sgl;
|
||||
int ret;
|
||||
|
||||
if (sgl->type == (NVME_SGL_FMT_DATA_DESC << 4)) {
|
||||
/* Single data descriptor case. */
|
||||
@@ -1084,10 +1096,23 @@ static int nvmet_pci_epf_iod_parse_sgls(struct nvmet_pci_epf_iod *iod)
|
||||
iod->data_segs = &iod->data_seg;
|
||||
iod->data_seg.pci_addr = le64_to_cpu(sgl->addr);
|
||||
iod->data_seg.length = le32_to_cpu(sgl->length);
|
||||
return 0;
|
||||
} else {
|
||||
ret = nvmet_pci_epf_iod_parse_sgl_segments(ctrl, iod);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
|
||||
return nvmet_pci_epf_iod_parse_sgl_segments(ctrl, iod);
|
||||
/*
|
||||
* The command executes on data_len bytes, but each segment is
|
||||
* allocated and transferred in full, so they must describe exactly
|
||||
* data_len bytes.
|
||||
*/
|
||||
if (nvmet_pci_epf_iod_data_segs_len(iod) != iod->data_len) {
|
||||
iod->status = NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int nvmet_pci_epf_alloc_iod_data_buf(struct nvmet_pci_epf_iod *iod)
|
||||
@@ -1152,7 +1177,6 @@ static void nvmet_pci_epf_complete_iod(struct nvmet_pci_epf_iod *iod)
|
||||
unsigned long flags;
|
||||
|
||||
/* Print an error message for failed commands, except AENs. */
|
||||
iod->status = le16_to_cpu(iod->cqe.status) >> 1;
|
||||
if (iod->status && iod->cmd.common.opcode != nvme_admin_async_event)
|
||||
dev_err(iod->ctrl->dev,
|
||||
"CQ[%d]: Command %s (0x%x) status 0x%0x\n",
|
||||
|
||||
@@ -513,7 +513,9 @@ static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev,
|
||||
|
||||
static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue)
|
||||
{
|
||||
spin_lock(&queue->rsp_wr_wait_lock);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&queue->rsp_wr_wait_lock, flags);
|
||||
while (!list_empty(&queue->rsp_wr_wait_list)) {
|
||||
struct nvmet_rdma_rsp *rsp;
|
||||
bool ret;
|
||||
@@ -522,16 +524,16 @@ static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue)
|
||||
struct nvmet_rdma_rsp, wait_list);
|
||||
list_del(&rsp->wait_list);
|
||||
|
||||
spin_unlock(&queue->rsp_wr_wait_lock);
|
||||
spin_unlock_irqrestore(&queue->rsp_wr_wait_lock, flags);
|
||||
ret = nvmet_rdma_execute_command(rsp);
|
||||
spin_lock(&queue->rsp_wr_wait_lock);
|
||||
spin_lock_irqsave(&queue->rsp_wr_wait_lock, flags);
|
||||
|
||||
if (!ret) {
|
||||
list_add(&rsp->wait_list, &queue->rsp_wr_wait_list);
|
||||
break;
|
||||
}
|
||||
}
|
||||
spin_unlock(&queue->rsp_wr_wait_lock);
|
||||
spin_unlock_irqrestore(&queue->rsp_wr_wait_lock, flags);
|
||||
}
|
||||
|
||||
static u16 nvmet_rdma_check_pi_status(struct ib_mr *sig_mr)
|
||||
@@ -801,7 +803,10 @@ static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
|
||||
|
||||
if (unlikely(wc->status != IB_WC_SUCCESS)) {
|
||||
nvmet_rdma_rw_ctx_destroy(rsp);
|
||||
nvmet_req_uninit(&rsp->req);
|
||||
/*
|
||||
* Unlike in nvmet_rdma_read_data_done(), the request has been
|
||||
* completed and its references are put: no nvmet_req_uninit().
|
||||
*/
|
||||
nvmet_rdma_release_rsp(rsp);
|
||||
if (wc->status != IB_WC_WR_FLUSH_ERR) {
|
||||
pr_info("RDMA WRITE for CQE failed with status %s (%d).\n",
|
||||
@@ -864,7 +869,8 @@ static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp)
|
||||
return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
|
||||
}
|
||||
|
||||
if (off + len > rsp->queue->dev->inline_data_size) {
|
||||
if (off > (u64)rsp->queue->dev->inline_data_size ||
|
||||
len > (u64)rsp->queue->dev->inline_data_size - off) {
|
||||
pr_err("invalid inline data offset!\n");
|
||||
return NVME_SC_SGL_INVALID_OFFSET | NVME_STATUS_DNR;
|
||||
}
|
||||
@@ -976,6 +982,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp)
|
||||
static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue,
|
||||
struct nvmet_rdma_rsp *cmd)
|
||||
{
|
||||
unsigned long flags;
|
||||
u16 status;
|
||||
|
||||
ib_dma_sync_single_for_cpu(queue->dev->device,
|
||||
@@ -993,9 +1000,9 @@ static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue,
|
||||
goto out_err;
|
||||
|
||||
if (unlikely(!nvmet_rdma_execute_command(cmd))) {
|
||||
spin_lock(&queue->rsp_wr_wait_lock);
|
||||
spin_lock_irqsave(&queue->rsp_wr_wait_lock, flags);
|
||||
list_add_tail(&cmd->wait_list, &queue->rsp_wr_wait_list);
|
||||
spin_unlock(&queue->rsp_wr_wait_lock);
|
||||
spin_unlock_irqrestore(&queue->rsp_wr_wait_lock, flags);
|
||||
}
|
||||
|
||||
return;
|
||||
@@ -1412,7 +1419,7 @@ nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn,
|
||||
struct nvme_rdma_cm_req *req;
|
||||
|
||||
req = (struct nvme_rdma_cm_req *)conn->private_data;
|
||||
if (!req || conn->private_data_len == 0)
|
||||
if (!req || conn->private_data_len < sizeof(*req))
|
||||
return NVME_RDMA_CM_INVALID_LEN;
|
||||
|
||||
if (le16_to_cpu(req->recfmt) != NVME_RDMA_CM_FMT_1_0)
|
||||
@@ -2150,6 +2157,12 @@ static void __exit nvmet_rdma_exit(void)
|
||||
{
|
||||
nvmet_unregister_transport(&nvmet_rdma_ops);
|
||||
ib_unregister_client(&nvmet_rdma_ib_client);
|
||||
/*
|
||||
* nvmet_rdma_remove_one() does not flush for a device that has left
|
||||
* device_list, and the release work that took it off still has to
|
||||
* return from nvmet_rdma_free_dev().
|
||||
*/
|
||||
flush_workqueue(nvmet_wq);
|
||||
WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list));
|
||||
ida_destroy(&nvmet_rdma_queue_ida);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
||||
#include <linux/module.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/jiffies.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/crc32c.h>
|
||||
#include <linux/err.h>
|
||||
@@ -76,8 +77,24 @@ MODULE_PARM_DESC(idle_poll_period_usecs,
|
||||
/*
|
||||
* TLS handshake timeout
|
||||
*/
|
||||
static int tls_handshake_timeout = 10;
|
||||
module_param(tls_handshake_timeout, int, 0644);
|
||||
static unsigned int tls_handshake_timeout = 10;
|
||||
|
||||
static int tls_handshake_timeout_set(const char *val,
|
||||
const struct kernel_param *kp)
|
||||
{
|
||||
return param_set_uint_minmax(val, kp, 0,
|
||||
min_t(unsigned long, UINT_MAX / MSEC_PER_SEC,
|
||||
MAX_JIFFY_OFFSET / HZ));
|
||||
}
|
||||
|
||||
static const struct kernel_param_ops tls_handshake_timeout_ops = {
|
||||
.set = tls_handshake_timeout_set,
|
||||
.get = param_get_uint,
|
||||
};
|
||||
|
||||
module_param_cb(tls_handshake_timeout, &tls_handshake_timeout_ops,
|
||||
&tls_handshake_timeout, 0644);
|
||||
__MODULE_PARM_TYPE(tls_handshake_timeout, "uint");
|
||||
MODULE_PARM_DESC(tls_handshake_timeout,
|
||||
"nvme TLS handshake timeout in seconds (default 10)");
|
||||
#endif
|
||||
@@ -149,6 +166,7 @@ enum nvmet_tcp_queue_state {
|
||||
struct nvmet_tcp_queue {
|
||||
struct socket *sock;
|
||||
struct nvmet_tcp_port *port;
|
||||
struct nvmet_port *nport;
|
||||
struct work_struct io_work;
|
||||
struct nvmet_cq nvme_cq;
|
||||
struct nvmet_sq nvme_sq;
|
||||
@@ -1495,7 +1513,7 @@ static int nvmet_tcp_alloc_cmd(struct nvmet_tcp_queue *queue,
|
||||
u8 hdgst = nvmet_tcp_hdgst_len(queue);
|
||||
|
||||
c->queue = queue;
|
||||
c->req.port = queue->port->nport;
|
||||
c->req.port = queue->nport;
|
||||
|
||||
c->cmd_pdu = page_frag_alloc(&queue->pf_cache,
|
||||
sizeof(*c->cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO);
|
||||
@@ -1904,6 +1922,7 @@ static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue)
|
||||
{
|
||||
int ret = -EOPNOTSUPP;
|
||||
struct tls_handshake_args args;
|
||||
unsigned int timeout = READ_ONCE(tls_handshake_timeout);
|
||||
|
||||
if (queue->state != NVMET_TCP_Q_TLS_HANDSHAKE) {
|
||||
pr_warn("cannot start TLS in state %d\n", queue->state);
|
||||
@@ -1917,7 +1936,7 @@ static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue)
|
||||
args.ta_done = nvmet_tcp_tls_handshake_done;
|
||||
args.ta_data = queue;
|
||||
args.ta_keyring = key_serial(queue->port->nport->keyring);
|
||||
args.ta_timeout_ms = tls_handshake_timeout * 1000;
|
||||
args.ta_timeout_ms = timeout * MSEC_PER_SEC;
|
||||
|
||||
ret = tls_server_hello_psk(&args, GFP_KERNEL);
|
||||
if (ret) {
|
||||
@@ -1925,7 +1944,7 @@ static int nvmet_tcp_tls_handshake(struct nvmet_tcp_queue *queue)
|
||||
pr_err("failed to start TLS, err=%d\n", ret);
|
||||
} else {
|
||||
queue_delayed_work(nvmet_wq, &queue->tls_handshake_tmo_work,
|
||||
tls_handshake_timeout * HZ);
|
||||
(unsigned long)timeout * HZ);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@@ -1951,6 +1970,8 @@ static void nvmet_tcp_alloc_queue(struct nvmet_tcp_port *port,
|
||||
kref_init(&queue->kref);
|
||||
queue->sock = newsock;
|
||||
queue->port = port;
|
||||
/* The port may be gone when the commands are allocated */
|
||||
queue->nport = port->nport;
|
||||
queue->nr_cmds = 0;
|
||||
spin_lock_init(&queue->state_lock);
|
||||
if (queue->port->nport->disc_addr.tsas.tcp.sectype ==
|
||||
@@ -2173,6 +2194,8 @@ static void nvmet_tcp_remove_port(struct nvmet_port *nport)
|
||||
port->sock->sk->sk_data_ready = port->data_ready;
|
||||
port->sock->sk->sk_user_data = NULL;
|
||||
write_unlock_bh(&port->sock->sk->sk_callback_lock);
|
||||
/* The accept work goes on for as long as there is something to accept */
|
||||
kernel_sock_shutdown(port->sock, SHUT_RDWR);
|
||||
cancel_work_sync(&port->accept_work);
|
||||
/*
|
||||
* Destroy the remaining queues, which are not belong to any
|
||||
|
||||
@@ -322,9 +322,17 @@ out:
|
||||
|
||||
void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req)
|
||||
{
|
||||
u32 out_bufsize = (le32_to_cpu(req->cmd->zmr.numd) + 1) << 2;
|
||||
u32 numd = le32_to_cpu(req->cmd->zmr.numd);
|
||||
|
||||
if (!nvmet_check_transfer_len(req, out_bufsize))
|
||||
/* From U32_MAX / 4 up, (numd + 1) << 2 wraps here and in the work. */
|
||||
if (numd >= U32_MAX / 4) {
|
||||
req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, numd);
|
||||
nvmet_req_complete(req,
|
||||
NVME_SC_INVALID_FIELD | NVME_STATUS_DNR);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!nvmet_check_transfer_len(req, (numd + 1) << 2))
|
||||
return;
|
||||
|
||||
INIT_WORK(&req->z.zmgmt_work, nvmet_bdev_zone_zmgmt_recv_work);
|
||||
|
||||
@@ -523,6 +523,10 @@ enum hctx_type {
|
||||
* @tag_list_lock: Serializes tag_list accesses.
|
||||
* @tag_list: List of the request queues that use this tag set. See also
|
||||
* request_queue.tag_set_list.
|
||||
* @quiesce_lock: Serializes blk_mq_[un]quiesce_tagset() against each other
|
||||
* and against a request queue being added to @tag_list.
|
||||
* @quiesce_depth: How often blk_mq_quiesce_tagset() has been called without
|
||||
* blk_mq_unquiesce_tagset().
|
||||
* @srcu: Use as lock when type of the request queue is blocking
|
||||
* (BLK_MQ_F_BLOCKING).
|
||||
* @update_nr_hwq_lock:
|
||||
@@ -548,6 +552,8 @@ struct blk_mq_tag_set {
|
||||
|
||||
struct mutex tag_list_lock;
|
||||
struct list_head tag_list;
|
||||
spinlock_t quiesce_lock;
|
||||
unsigned int quiesce_depth;
|
||||
struct srcu_struct *srcu;
|
||||
|
||||
struct rw_semaphore update_nr_hwq_lock;
|
||||
|
||||
@@ -1821,11 +1821,12 @@ static inline int bio_split_rw_at(struct bio *bio,
|
||||
*/
|
||||
static inline unsigned int max_integrity_io_size(struct queue_limits *lim)
|
||||
{
|
||||
u64 max_intervals;
|
||||
unsigned int buf_size = min_t(unsigned int, lim->max_segment_size,
|
||||
BLK_INTEGRITY_MAX_SIZE);
|
||||
|
||||
max_intervals = BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size;
|
||||
return min_t(u64, lim->max_segment_size,
|
||||
max_intervals << lim->integrity.interval_exp);
|
||||
return min_t(u64, UINT_MAX,
|
||||
(u64)(buf_size / lim->integrity.metadata_size) <<
|
||||
lim->integrity.interval_exp);
|
||||
}
|
||||
|
||||
#define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { }
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
#ifndef _LINUX_CDROM_H
|
||||
#define _LINUX_CDROM_H
|
||||
|
||||
#include <linux/fs.h> /* not really needed, later.. */
|
||||
#include <linux/list.h>
|
||||
#include <linux/blkdev.h>
|
||||
#include <scsi/scsi_common.h>
|
||||
|
||||
@@ -34,6 +34,9 @@ DECLARE_EVENT_CLASS(iocost_iocg_state,
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
u32 hwa, hwi;
|
||||
|
||||
iocg_hweight_cache(iocg, &hwa, &hwi);
|
||||
__assign_str(devname);
|
||||
__assign_str(cgroup);
|
||||
__entry->now = now->now;
|
||||
@@ -44,8 +47,8 @@ DECLARE_EVENT_CLASS(iocost_iocg_state,
|
||||
__entry->vtime = vtime;
|
||||
__entry->weight = iocg->weight;
|
||||
__entry->inuse = iocg->inuse;
|
||||
__entry->hweight_active = iocg->hweight_active;
|
||||
__entry->hweight_inuse = iocg->hweight_inuse;
|
||||
__entry->hweight_active = hwa;
|
||||
__entry->hweight_inuse = hwi;
|
||||
),
|
||||
|
||||
TP_printk("[%s:%s] now=%llu:%llu vrate=%llu "
|
||||
|
||||
@@ -149,8 +149,11 @@ class IocgStat:
|
||||
self.weight = iocg.weight.value_() / WEIGHT_ONE
|
||||
self.active = iocg.active.value_() / WEIGHT_ONE
|
||||
self.inuse = iocg.inuse.value_() / WEIGHT_ONE
|
||||
self.hwa_pct = iocg.hweight_active.value_() * 100 / WEIGHT_ONE
|
||||
self.hwi_pct = iocg.hweight_inuse.value_() * 100 / WEIGHT_ONE
|
||||
hweight_cache = iocg.hweight_cache.counter.value_()
|
||||
hwa = ((hweight_cache >> 16) & (WEIGHT_ONE - 1)) + 1
|
||||
hwi = (hweight_cache & (WEIGHT_ONE - 1)) + 1
|
||||
self.hwa_pct = hwa * 100 / WEIGHT_ONE
|
||||
self.hwi_pct = hwi * 100 / WEIGHT_ONE
|
||||
self.address = iocg.value_()
|
||||
|
||||
vdone = iocg.done_vtime.counter.value_()
|
||||
|
||||
Reference in New Issue
Block a user