Merge tag 'block-7.4-20261106' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull block fixes from Jens Axboe:

 - NVMe fixes via Keith:
    - nvme-tcp use-after-free in nvme_tcp_free_ctrl() on a failed
      connect, and the nvme-rdma counterpart of the NULL queue->ctrl on
      reconnect
    - nvmet-rdma iWARP disconnect and device removal fixes
    - nvmet-tcp: ask for the TLS record type on every receive, and bound
      the connections accepted per work run so a connect flood can't
      hold up port removal; nvmet-auth keep the old keys if the new ones
      fail to decode; nvme-loop delete the controller if a reset fails
    - nvmet-auth reject incomplete DHCHAP messages, and a keep-alive
      timeout overflow
    - nvmet locking and bounds fixes for tbkas reconnects and the
      traddr/trsvcid configfs writes, don't leak stale memory in the ANA
      log page, and cache the passthru reference across queue_response
    - Wait for SRCU readers before splicing the namespace list, with a
      new list_splice_init_srcu() helper, and drop an extra grace period
      in nvmet_ns_disable()
    - Unregister when pr_register() is given a zero new key, and don't
      drop the cdev reference twice on cdev_device_add() failure
    - Quirks for the ADATA XPG GAMMIX S70 BLADE and KIOXIA XG10d
    - Various smaller fixes and cleanups

 - blk-mq: end the request if getting a keyslot fails, which left the
   in-flight count and zone write plug stuck, reset the queue state on
   scheduler teardown so requests don't stall in software queues, and
   fix an out-of-bounds read in blk_mq_free_rqs() after a failed
   nr_hw_queues update

 - Only serialize elevator switches of the same queue

 - dma-buf bio fixes: don't use the bvec fast path for them, and only
   fail the ones that need block layer PI

 - Reject GPT entries with starting_lba > ending_lba

 - Return the soft block size for compat BLKBSZGET, report the correct
   discard alignment in sysfs, and account the flush sequence latency
   only once in iocost

 - virtio_blk fixes: synchronous quiesce in freeze, freeze the virtqueue
   layout at probe time, clamp max_segments to the ring size, and fix
   zone append sector reporting in batched completion

 - null_blk zone fixes and a cache state reset

 - zloop fixes

 - ublk DEL_DEV_ASYNC permission check, and 32-bit selftest build

* tag 'block-7.4-20261106' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (52 commits)
  nvmet: cache passthru reference in __nvmet_req_complete()
  nvmet: drop the redundant auth check in nvmet_parse_io_cmd()
  nvmet: bound the traddr and trsvcid configfs writes
  nvmet: take subsys->lock when forcing reconnects for tbkas
  nvme: wait for SRCU readers before splicing the namespace list
  srcu: Add list_splice_init_srcu()
  nvme-pci: add NVME_QUIRK_DMAPOOL_ALIGN_512 for KIOXIA XG10d
  nvmet: don't return memory contents beyond valid data when reading ANA log page
  nvme-pci: add NO_FUA quirk for ADATA XPG GAMMIX S70 BLADE
  nvmet: drop the extra RCU grace period in nvmet_ns_disable()
  nvme-keyring: Add a missing key_put() call to nvme_tls_key_lookup()
  ublk: add missing DEL_DEV_ASYNC to unprivileged permission check
  selftests: ublk: fix the build and file offsets on 32-bit
  nvmet-rdma: disconnect a queue on device removal
  nvmet-rdma: queue the release work under the mutex
  nvmet-rdma: take a device off the list last
  nvmet-tcp: limit the connections accepted per work run
  nvmet-rdma: disconnect only established queues
  nvmet-auth: keep the old keys if the new ones fail
  nvmet-tcp: ask for the TLS record type on every receive
  ...
This commit is contained in:
Linus Torvalds
2026-11-06 12:36:20 -08:00
38 changed files with 377 additions and 184 deletions
+3
View File
@@ -2936,6 +2936,9 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq)
if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns)
return;
if (unlikely(rq->rq_flags & RQF_FLUSH_SEQ))
return;
switch (req_op(rq)) {
case REQ_OP_READ:
pidx = QOS_RLAT;
+5 -1
View File
@@ -374,7 +374,7 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq,
}
EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
/* called in queue's release handler, tagset has gone away */
/* Clear scheduler tags and reset queue state. */
static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int flags)
{
struct blk_mq_hw_ctx *hctx;
@@ -385,6 +385,10 @@ static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int fla
if (blk_mq_is_shared_tags(flags))
q->sched_shared_tags = NULL;
blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q);
q->nr_requests = q->tag_set->queue_depth;
q->async_depth = q->tag_set->queue_depth;
}
void blk_mq_sched_reg_debugfs(struct request_queue *q)
+17 -9
View File
@@ -3160,8 +3160,15 @@ void blk_mq_submit_bio(struct bio *bio)
goto queue_exit;
}
/*
* The block layer cannot generate or verify PI for a dma-buf bio, as
* that walks the data pages. Only fail the bio if it needs that, not
* if the submitter attached its own integrity payload or if PI
* generation or verification is turned off for the device.
*/
if (unlikely(op_is_dmabuf(bio->bi_opf) &&
bdev_has_integrity_csum(bio->bi_bdev))) {
bdev_has_integrity_csum(bio->bi_bdev) &&
(bio_integrity_action(bio) & BI_ACT_CHECK))) {
bio_endio_status(bio, BLK_STS_NOTSUPP);
goto queue_exit;
}
@@ -3209,16 +3216,15 @@ new_request:
blk_mq_bio_to_request(rq, bio, nr_segs);
ret = blk_crypto_rq_get_keyslot(rq);
if (ret != BLK_STS_OK) {
bio_endio_status(bio, ret);
blk_mq_free_request(rq);
return;
}
if (bio_zone_write_plugging(bio))
blk_zone_write_plug_init_request(rq);
ret = blk_crypto_rq_get_keyslot(rq);
if (ret != BLK_STS_OK) {
blk_mq_end_request(rq, ret);
return;
}
if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
return;
@@ -3501,8 +3507,10 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
if (blk_mq_is_shared_tags(set->flags))
drv_tags = set->shared_tags;
else
else if (hctx_idx < set->nr_hw_queues)
drv_tags = set->tags[hctx_idx];
else
drv_tags = NULL;
if (tags->static_rqs && set->ops->exit_request) {
int i;
+2
View File
@@ -418,6 +418,8 @@ static inline bool bio_may_need_split(struct bio *bio,
if (!bio->bi_io_vec)
return true;
if (op_is_dmabuf(bio->bi_opf))
return true;
bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_offset)
+18 -14
View File
@@ -812,25 +812,29 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf,
* reference during concurrent disk deletion:
* update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del)
* kn->active -> update_nr_hwq_lock (via this sysfs write path)
*
* Use the writer lock instead of the reader lock of update_nr_hwq_lock
* to serialize the two-stage elevator switch steps in
* elevator_change(): the core switch step under the elevator lock and
* the elevator_change_done() step outside the elevator lock.
*/
if (!down_write_trylock(&set->update_nr_hwq_lock)) {
if (!down_read_trylock(&set->update_nr_hwq_lock)) {
ret = -EBUSY;
goto out;
}
if (!blk_queue_no_elv_switch(q)) {
ret = elevator_change(q, &ctx);
if (!ret)
ret = count;
} else {
ret = -ENOENT;
}
up_write(&set->update_nr_hwq_lock);
/*
* The reader lock is shared, so use QUEUE_FLAG_NO_ELV_SWITCH to allow
* only one switch of this queue at a time, and none once the disk is
* going away.
*/
if (test_and_set_bit(QUEUE_FLAG_NO_ELV_SWITCH, &q->queue_flags)) {
ret = -EBUSY;
goto out_unlock;
}
ret = elevator_change(q, &ctx);
if (!ret)
ret = count;
blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q);
out_unlock:
up_read(&set->update_nr_hwq_lock);
out:
if (ctx.type)
elevator_put(ctx.type);
-2
View File
@@ -1018,8 +1018,6 @@ static int blkdev_init_dma_buf_io_ctx(struct file *file,
return -EINVAL;
if (!disk->fops->init_dma_buf_io_ctx)
return -EOPNOTSUPP;
if (bdev_has_integrity_csum(bdev))
return -EOPNOTSUPP;
return disk->fops->init_dma_buf_io_ctx(bdev, ctx);
}
+1 -1
View File
@@ -1146,7 +1146,7 @@ static ssize_t disk_discard_alignment_show(struct device *dev,
{
struct gendisk *disk = dev_to_disk(dev);
return sysfs_emit(buf, "%d\n", bdev_alignment_offset(disk->part0));
return sysfs_emit(buf, "%u\n", bdev_discard_alignment(disk->part0));
}
static ssize_t diskseq_show(struct device *dev,
+2 -2
View File
@@ -835,8 +835,8 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg)
return compat_put_ulong(argp, bdev_nr_sectors(bdev));
/* The data is compatible, but the command number is different */
case BLKBSZGET_32: /* get the logical block size (cf. BLKSSZGET) */
return put_int(argp, bdev_logical_block_size(bdev));
case BLKBSZGET_32: /* get the soft block size (cf. BLKSSZGET) */
return put_int(argp, block_size(bdev));
case BLKBSZSET_32:
return blkdev_bszset(file, mode, argp);
case BLKGETSIZE64_32:
+1
View File
@@ -460,6 +460,7 @@ static inline int
is_pte_valid(const gpt_entry *pte, const u64 lastlba)
{
if ((!efi_guidcmp(pte->partition_type_guid, NULL_GUID)) ||
le64_to_cpu(pte->starting_lba) > le64_to_cpu(pte->ending_lba) ||
le64_to_cpu(pte->starting_lba) > lastlba ||
le64_to_cpu(pte->ending_lba) > lastlba)
return 0;
+1
View File
@@ -1976,6 +1976,7 @@ static int null_add_dev(struct nullb_device *dev)
rv = null_validate_conf(dev);
if (rv)
return rv;
clear_bit(NULLB_DEV_FL_CACHE, &dev->flags);
nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
if (!nullb) {
+46 -17
View File
@@ -311,10 +311,8 @@ static blk_status_t null_check_open(struct nullb_device *dev)
return BLK_STS_OK;
if (dev->nr_zones_imp_open) {
if (null_check_active(dev) == BLK_STS_OK) {
null_close_imp_open_zone(dev);
return BLK_STS_OK;
}
null_close_imp_open_zone(dev);
return BLK_STS_OK;
}
return BLK_STS_ZONE_OPEN_RESOURCE;
@@ -330,8 +328,11 @@ static blk_status_t null_check_open(struct nullb_device *dev)
*
* ZBC states that an implicit open zone shall be closed only if there is not
* room within the open limit. However, with the addition of an active limit,
* it is not certain that closing an implicit open zone will allow a new zone
* to be opened, since we might already be at the active limit capacity.
* it is not certain that closing an implicit open zone will allow an empty
* zone to be opened, since we might already be at the active limit capacity.
* Therefore, the active limit is checked first for an empty zone. A closed
* zone already holds an active zone resource, so only the open limit has to
* be checked for a closed zone.
*/
static blk_status_t null_check_zone_resources(struct nullb_device *dev,
struct nullb_zone *zone)
@@ -743,23 +744,47 @@ blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op,
}
/*
* Set a zone in the read-only or offline condition.
* Release the zone resources held by a zone, before changing the condition of
* the zone to a condition that does not hold any zone resources.
*/
static void null_set_zone_cond(struct nullb_device *dev,
struct nullb_zone *zone, enum blk_zone_cond cond)
static void null_release_zone_resources(struct nullb_device *dev,
struct nullb_zone *zone)
{
if (WARN_ON_ONCE(cond != BLK_ZONE_COND_READONLY &&
cond != BLK_ZONE_COND_OFFLINE))
if (!dev->need_zone_res_mgmt)
return;
spin_lock(&dev->zone_res_lock);
switch (zone->cond) {
case BLK_ZONE_COND_IMP_OPEN:
dev->nr_zones_imp_open--;
break;
case BLK_ZONE_COND_EXP_OPEN:
dev->nr_zones_exp_open--;
break;
case BLK_ZONE_COND_CLOSED:
dev->nr_zones_closed--;
break;
default:
break;
}
spin_unlock(&dev->zone_res_lock);
}
static void null_set_zone_readonly_or_offline(struct nullb_device *dev,
struct nullb_zone *zone,
bool set_offline)
{
enum blk_zone_cond cond =
set_offline ? BLK_ZONE_COND_OFFLINE : BLK_ZONE_COND_READONLY;
null_lock_zone(dev, zone);
/*
* If the read-only condition is requested again to zones already in
* read-only condition, restore back normal empty condition. Do the same
* if the offline condition is requested for offline zones. Otherwise,
* set the specified zone condition to the zones. Finish the zones
* beforehand to free up zone resources.
* set the specified zone condition to the zones. Release the zone
* resources held by the zones beforehand.
*/
if (zone->cond == cond) {
zone->cond = BLK_ZONE_COND_EMPTY;
@@ -767,9 +792,7 @@ static void null_set_zone_cond(struct nullb_device *dev,
if (dev->memory_backed)
null_handle_discard(dev, zone->start, zone->len);
} else {
if (zone->cond != BLK_ZONE_COND_READONLY &&
zone->cond != BLK_ZONE_COND_OFFLINE)
null_finish_zone(dev, zone);
null_release_zone_resources(dev, zone);
zone->cond = cond;
zone->wp = NULL_ZONE_INVALID_WP;
}
@@ -788,6 +811,11 @@ ssize_t zone_cond_store(struct nullb_device *dev, const char *page,
unsigned int zone_no;
int ret;
/* Read-only and offline are the only conditions supported for now */
if (WARN_ON_ONCE(cond != BLK_ZONE_COND_READONLY &&
cond != BLK_ZONE_COND_OFFLINE))
return -EINVAL;
if (!dev->zoned) {
pr_err("null_blk device is not zoned\n");
return -EINVAL;
@@ -813,7 +841,8 @@ ssize_t zone_cond_store(struct nullb_device *dev, const char *page,
return -EINVAL;
}
null_set_zone_cond(dev, &dev->zones[zone_no], cond);
null_set_zone_readonly_or_offline(dev, &dev->zones[zone_no],
cond == BLK_ZONE_COND_OFFLINE);
return count;
}
+1
View File
@@ -5878,6 +5878,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub,
case UBLK_CMD_STOP_DEV:
case UBLK_CMD_ADD_DEV:
case UBLK_CMD_DEL_DEV:
case UBLK_CMD_DEL_DEV_ASYNC:
case UBLK_CMD_SET_PARAMS:
case UBLK_CMD_START_USER_RECOVERY:
case UBLK_CMD_END_USER_RECOVERY:
+41 -16
View File
@@ -331,18 +331,25 @@ static inline u8 virtblk_vbr_status(struct virtblk_req *vbr)
return *((u8 *)&vbr->in_hdr + vbr->in_hdr_len - 1);
}
static inline void virtblk_request_done(struct request *req)
static inline void virtblk_update_zone_append_sector(struct request *req,
struct virtblk_req *vbr)
{
struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
blk_status_t status = virtblk_result(virtblk_vbr_status(vbr));
struct virtio_blk *vblk = req->mq_hctx->queue->queuedata;
virtblk_unmap_data(req, vbr);
virtblk_cleanup_cmd(req);
if (req_op(req) == REQ_OP_ZONE_APPEND)
req->__sector = virtio64_to_cpu(vblk->vdev,
vbr->in_hdr.zone_append.sector);
}
static inline void virtblk_request_done(struct request *req)
{
struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
blk_status_t status = virtblk_result(virtblk_vbr_status(vbr));
virtblk_unmap_data(req, vbr);
virtblk_cleanup_cmd(req);
virtblk_update_zone_append_sector(req, vbr);
blk_mq_end_request(req, status);
}
@@ -958,16 +965,13 @@ static void virtblk_config_changed(struct virtio_device *vdev)
queue_work(virtblk_wq, &vblk->config_work);
}
static int init_vq(struct virtio_blk *vblk)
/* Keep the queue layout unchanged for the lifetime of the tag set. */
static int virtblk_init_queue_layout(struct virtio_blk *vblk)
{
int err;
unsigned short i;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
struct virtio_device *vdev = vblk->vdev;
unsigned short num_vqs;
unsigned short num_poll_vqs;
struct virtio_device *vdev = vblk->vdev;
struct irq_affinity desc = { 0, };
int err;
err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
struct virtio_blk_config, num_queues,
@@ -985,6 +989,7 @@ static int init_vq(struct virtio_blk *vblk)
num_poll_vqs = min_t(unsigned int, poll_queues, num_vqs - 1);
vblk->num_vqs = num_vqs;
vblk->io_queues[HCTX_TYPE_DEFAULT] = num_vqs - num_poll_vqs;
vblk->io_queues[HCTX_TYPE_READ] = 0;
vblk->io_queues[HCTX_TYPE_POLL] = num_poll_vqs;
@@ -994,6 +999,20 @@ static int init_vq(struct virtio_blk *vblk)
vblk->io_queues[HCTX_TYPE_READ],
vblk->io_queues[HCTX_TYPE_POLL]);
return 0;
}
static int init_vq(struct virtio_blk *vblk)
{
int err;
unsigned short i;
struct virtqueue_info *vqs_info;
struct virtqueue **vqs;
unsigned short num_vqs = vblk->num_vqs;
unsigned short num_poll_vqs = vblk->io_queues[HCTX_TYPE_POLL];
struct virtio_device *vdev = vblk->vdev;
struct irq_affinity desc = { 0, };
vblk->vqs = kmalloc_objs(*vblk->vqs, num_vqs);
if (!vblk->vqs)
return -ENOMEM;
@@ -1049,7 +1068,6 @@ static int init_vq(struct virtio_blk *vblk)
spin_lock_init(&vblk->vqs[i].lock);
vblk->vqs[i].vq = vqs[i];
}
vblk->num_vqs = num_vqs;
out:
kfree(vqs);
@@ -1220,8 +1238,11 @@ static void virtblk_complete_batch(struct io_comp_batch *iob)
struct request *req;
rq_list_for_each(&iob->req_list, req) {
virtblk_unmap_data(req, blk_mq_rq_to_pdu(req));
struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
virtblk_unmap_data(req, vbr);
virtblk_cleanup_cmd(req);
virtblk_update_zone_append_sector(req, vbr);
}
blk_mq_end_request_batch(iob);
}
@@ -1512,6 +1533,10 @@ static int virtblk_probe(struct virtio_device *vdev)
INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
err = virtblk_init_queue_layout(vblk);
if (err)
goto out_free_vblk;
err = init_vq(vblk);
if (err)
goto out_free_vblk;
@@ -1639,7 +1664,7 @@ static int virtblk_freeze_priv(struct virtio_device *vdev)
/* Ensure no requests in virtqueues before deleting vqs. */
memflags = blk_mq_freeze_queue(q);
blk_mq_quiesce_queue_nowait(q);
blk_mq_quiesce_queue(q);
blk_mq_unfreeze_queue(q, memflags);
/* Ensure we don't receive any more interrupts */
+4 -3
View File
@@ -1137,7 +1137,7 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,
goto out_fput;
}
if (zlo->block_size) {
if (!zlo->block_size) {
ret = zloop_get_block_size(zlo, zone);
if (ret)
goto out_fput;
@@ -1161,7 +1161,8 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts,
if (restore && file_sectors != zlo->zone_size) {
pr_err("Invalid conventional zone %u file size (%llu sectors != %llu)\n",
zone_no, file_sectors, zlo->zone_capacity);
zone_no, file_sectors, zlo->zone_size);
ret = -EINVAL;
goto out_fput;
}
@@ -1339,8 +1340,8 @@ static int zloop_ctl_add(struct zloop_options *opts)
zlo->disk = blk_mq_alloc_disk(&zlo->tag_set, &lim, zlo);
if (IS_ERR(zlo->disk)) {
pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret);
ret = PTR_ERR(zlo->disk);
pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret);
goto out_cleanup_tags;
}
zlo->disk->flags = GENHD_FL_NO_PART;
+1
View File
@@ -35,6 +35,7 @@ struct key *nvme_tls_key_lookup(key_serial_t key_id)
}
if (nvme_tls_psk_revoked(key)) {
pr_err("key id %08x revoked\n", key_id);
key_put(key);
return ERR_PTR(-EKEYREVOKED);
}
return key;
+18 -5
View File
@@ -2482,6 +2482,8 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns,
lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;
else
lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA);
if (ns->ctrl->quirks & NVME_QUIRK_NO_FUA)
lim.features &= ~BLK_FEAT_FUA;
if (info->is_rotational)
lim.features |= BLK_FEAT_ROTATIONAL;
@@ -3953,10 +3955,16 @@ static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
return 0;
}
/* The char device of a namespace rather than of a namespace head */
static bool nvme_is_ns_cdev(struct device *dev)
{
return dev->parent->class == &nvme_class;
}
static void nvme_cdev_rel(struct device *dev)
{
ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
if (dev->parent->class == &nvme_class)
if (nvme_is_ns_cdev(dev))
nvme_put_ns(container_of(dev, struct nvme_ns, cdev_device));
else
nvme_put_ns_head(container_of(dev, struct nvme_ns_head,
@@ -3990,6 +3998,14 @@ int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
device_initialize(cdev_device);
cdev_init(cdev, fops);
cdev->owner = owner;
/* Undone in nvme_cdev_rel() */
if (nvme_is_ns_cdev(cdev_device))
nvme_get_ns(container_of(cdev_device, struct nvme_ns,
cdev_device));
else
nvme_get_ns_head(container_of(cdev_device,
struct nvme_ns_head,
cdev_device));
ret = cdev_device_add(cdev, cdev_device);
if (ret)
put_device(cdev_device);
@@ -4022,13 +4038,11 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns)
{
ns->cdev_device.parent = ns->ctrl->device;
nvme_get_ns(ns); /* Undone in nvme_cdev_rel() */
if (nvme_cdev_add(&ns->cdev, &ns->cdev_device,
&nvme_ns_chr_fops, ns->ctrl->ops->module,
ns->ctrl->instance, ns->head->instance)) {
dev_err(ns->ctrl->device, "Unable to create the ng%dn%d device\n",
ns->ctrl->instance, ns->head->instance);
nvme_put_ns(ns);
return;
}
set_bit(NVME_NS_CDEV_LIVE, &ns->flags);
@@ -4755,9 +4769,8 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
mutex_lock(&ctrl->namespaces_lock);
list_splice_init_rcu(&ctrl->namespaces, &ns_list, synchronize_rcu);
list_splice_init_srcu(&ctrl->namespaces, &ns_list, &ctrl->srcu);
mutex_unlock(&ctrl->namespaces_lock);
synchronize_srcu(&ctrl->srcu);
list_for_each_entry_safe(ns, next, &ns_list, list)
nvme_ns_remove(ns);
-2
View File
@@ -681,14 +681,12 @@ static void nvme_add_ns_head_cdev(struct nvme_ns_head *head)
{
head->cdev_device.parent = &head->subsys->dev;
nvme_get_ns_head(head); /* Undone in nvme_cdev_rel() */
if (nvme_cdev_add(&head->cdev, &head->cdev_device,
&nvme_ns_head_chr_fops, THIS_MODULE,
head->subsys->instance, head->instance)) {
dev_err(disk_to_dev(head->disk),
"Unable to create the ng%dn%d device\n",
head->subsys->instance, head->instance);
nvme_put_ns_head(head);
return;
}
set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags);
+8
View File
@@ -183,6 +183,12 @@ enum nvme_quirks {
* Admin queue DMA buffers must be page aligned
*/
NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23),
/*
* The device is extremely slow on FUA writes but fast on Flush;
* do not advertise FUA so the block layer emulates it.
*/
NVME_QUIRK_NO_FUA = (1 << 24),
};
static inline char *nvme_quirk_name(enum nvme_quirks q)
@@ -236,6 +242,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q)
return "dmapool_align_512";
case NVME_QUIRK_ADMIN_PAGE_ALIGN:
return "admin_page_align";
case NVME_QUIRK_NO_FUA:
return "no_fua";
}
return "unknown";
+4
View File
@@ -4430,6 +4430,8 @@ static const struct pci_device_id nvme_id_table[] = {
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1344, 0x6004), /* Micron 4100AT */
.driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, },
{ PCI_DEVICE(0x1e0f, 0x0032), /* KIOXIA XG10d */
.driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, },
{ PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */
.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
{ PCI_DEVICE(0x1c5c, 0x174a), /* SK Hynix P31 SSD */
@@ -4483,6 +4485,8 @@ static const struct pci_device_id nvme_id_table[] = {
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1cc1, 0x5350), /* ADATA XPG GAMMIX S50 */
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1cc1, 0x5370), /* ADATA XPG GAMMIX S70 BLADE */
.driver_data = NVME_QUIRK_NO_FUA, },
{ PCI_DEVICE(0x1dbe, 0x5216), /* Acer/INNOGRIT FA100/5216 NVMe SSD */
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */
+6 -2
View File
@@ -128,8 +128,12 @@ static int nvme_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
data.crkey = cpu_to_le64(old_key);
data.nrkey = cpu_to_le64(new_key);
cdw10 = old_key ? NVME_PR_REGISTER_ACT_REPLACE :
NVME_PR_REGISTER_ACT_REG;
if (!new_key)
cdw10 = NVME_PR_REGISTER_ACT_UNREG;
else if (old_key)
cdw10 = NVME_PR_REGISTER_ACT_REPLACE;
else
cdw10 = NVME_PR_REGISTER_ACT_REG;
cdw10 |= (flags & PR_FL_IGNORE_KEY) ? NVME_PR_IGNORE_KEY : 0;
cdw10 |= NVME_PR_CPTPL_PERSIST;
+4 -3
View File
@@ -735,7 +735,6 @@ static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl,
for (i = 0; i < nr_queues; i++) {
struct nvme_rdma_queue *queue = &ctrl->queues[first + i];
queue->ctrl = ctrl;
queue->queue_size = queue_size;
ctxs[i].queue = queue;
@@ -798,7 +797,6 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
bool pi_capable = false;
int error;
ctrl->queues[0].ctrl = ctrl;
ctrl->queues[0].queue_size = NVME_AQ_DEPTH;
error = nvme_rdma_alloc_queue(&ctrl->queues[0]);
if (error)
@@ -2284,7 +2282,7 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev,
struct nvmf_ctrl_options *opts)
{
struct nvme_rdma_ctrl *ctrl;
int ret;
int ret, i;
ctrl = kzalloc_obj(*ctrl);
if (!ctrl)
@@ -2343,6 +2341,9 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev,
if (!ctrl->queues)
goto out_free_ctrl;
for (i = 0; i < ctrl->ctrl.queue_count; i++)
ctrl->queues[i].ctrl = ctrl;
ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops,
0 /* no quirks, we're perfect! */);
if (ret)
+5 -5
View File
@@ -199,6 +199,7 @@ struct nvme_tcp_ctrl {
struct delayed_work connect_work;
struct nvme_tcp_request async_req;
u32 io_queues[HCTX_MAX_TYPES];
unsigned int nr_allocated_queues;
struct net *net;
};
@@ -2848,8 +2849,6 @@ static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
{
struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
/* the same count used to size and initialize the queue array */
unsigned int nr_queues = NVME_TCP_OPT_NR_QUEUES(nctrl->opts);
unsigned int i;
if (list_empty_careful(&ctrl->list))
@@ -2862,7 +2861,7 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
nvmf_free_options(nctrl->opts);
free_ctrl:
put_net(ctrl->net);
for (i = 0; i < nr_queues; i++)
for (i = 0; i < ctrl->nr_allocated_queues; i++)
nvme_tcp_exit_queue(ctrl, i);
kfree(ctrl->queues);
kfree(ctrl);
@@ -3256,13 +3255,14 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,
goto out_free_ctrl;
}
ctrl->queues = kzalloc_objs(*ctrl->queues, ctrl->ctrl.queue_count);
ctrl->nr_allocated_queues = ctrl->ctrl.queue_count;
ctrl->queues = kzalloc_objs(*ctrl->queues, ctrl->nr_allocated_queues);
if (!ctrl->queues) {
ret = -ENOMEM;
goto out_free_ctrl;
}
for (i = 0; i < ctrl->ctrl.queue_count; i++)
for (i = 0; i < ctrl->nr_allocated_queues; i++)
nvme_tcp_init_queue(ctrl, i);
ctrl->net = get_net(current->nsproxy->net_ns);
+17 -3
View File
@@ -543,6 +543,15 @@ static void nvmet_execute_get_log_page_ana(struct nvmet_req *req)
u16 ngrps = 0;
u16 status;
/*
* The header and one descriptor per enabled group are the only bytes
* written below; zero the whole transfer first so the unwritten tail
* does not leak stale kernel memory to the host.
*/
status = nvmet_zero_sgl(req, 0, req->transfer_len);
if (status)
goto out;
status = NVME_SC_INTERNAL;
desc = kmalloc_flex(*desc, nsids, NVMET_MAX_NAMESPACES);
if (!desc)
@@ -1227,8 +1236,13 @@ u16 nvmet_set_feat_kato(struct nvmet_req *req)
{
u32 val32 = le32_to_cpu(req->cmd->common.cdw11);
if (val32 > NVMET_MAX_KATO * MSEC_PER_SEC) {
req->error_loc = offsetof(struct nvme_common_command, cdw11);
return NVME_SC_KA_TIMEOUT_INVALID | NVME_STATUS_DNR;
}
nvmet_stop_keep_alive_timer(req->sq->ctrl);
req->sq->ctrl->kato = DIV_ROUND_UP(val32, 1000);
req->sq->ctrl->kato = DIV_ROUND_UP_ULL(val32, MSEC_PER_SEC);
nvmet_start_keep_alive_timer(req->sq->ctrl);
nvmet_set_result(req, req->sq->ctrl->kato);
@@ -1326,7 +1340,7 @@ static u16 nvmet_set_feat_arbitration(struct nvmet_req *req)
.hpw = (cdw11 >> 24) & 0xff,
.mpw = (cdw11 >> 16) & 0xff,
.lpw = (cdw11 >> 8) & 0xff,
.ab = cdw11 & 0x3,
.ab = cdw11 & 0x7,
};
if (!ctrl->ops->set_feature) {
@@ -1481,7 +1495,7 @@ static u16 nvmet_get_feat_arbitration(struct nvmet_req *req)
((u32)arb.hpw << 24) |
((u32)arb.mpw << 16) |
((u32)arb.lpw << 8) |
(arb.ab & 0x3));
(arb.ab & 0x7));
return NVME_SC_SUCCESS;
}
+28 -41
View File
@@ -135,6 +135,7 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
int ret = 0;
struct nvmet_host_link *p;
struct nvmet_host *host = NULL;
struct nvme_dhchap_key *host_key, *ctrl_key = NULL;
down_read(&nvmet_config_sem);
if (nvmet_is_disc_subsys(ctrl->subsys))
@@ -172,55 +173,41 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
goto out_unlock;
}
if (host->dhchap_hash_id == ctrl->shash_id) {
pr_debug("Re-use existing hash ID %d\n",
ctrl->shash_id);
} else {
ctrl->shash_id = host->dhchap_hash_id;
}
/* Skip the 'DHHC-1:XX:' prefix */
nvme_auth_free_key(ctrl->host_key);
ctrl->host_key = nvme_auth_extract_key(host->dhchap_secret + 10,
host->dhchap_key_hash);
if (IS_ERR(ctrl->host_key)) {
/*
* Skip the 'DHHC-1:XX:' prefix. Keep the old keys until the new ones
* are there: a controller without a host key needs no authentication.
*/
host_key = nvme_auth_extract_key(host->dhchap_secret + 10,
host->dhchap_key_hash);
if (IS_ERR(host_key)) {
ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE;
ctrl->host_key = NULL;
goto out_free_hash;
goto out_unlock;
}
if (host->dhchap_ctrl_secret) {
ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret + 10,
host->dhchap_ctrl_key_hash);
if (IS_ERR(ctrl_key)) {
nvme_auth_free_key(host_key);
ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE;
goto out_unlock;
}
}
nvme_auth_free_key(ctrl->host_key);
ctrl->host_key = host_key;
nvme_auth_free_key(ctrl->ctrl_key);
ctrl->ctrl_key = ctrl_key;
ctrl->shash_id = host->dhchap_hash_id;
#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG
pr_debug("%s: using hash %s key %*ph\n", __func__,
ctrl->host_key->hash > 0 ?
nvme_auth_hmac_name(ctrl->host_key->hash) : "none",
(int)ctrl->host_key->len, ctrl->host_key->key);
if (ctrl->ctrl_key)
pr_debug("%s: using ctrl hash %s key %*ph\n", __func__,
ctrl->ctrl_key->hash > 0 ?
nvme_auth_hmac_name(ctrl->ctrl_key->hash) : "none",
(int)ctrl->ctrl_key->len, ctrl->ctrl_key->key);
#endif
nvme_auth_free_key(ctrl->ctrl_key);
if (!host->dhchap_ctrl_secret) {
ctrl->ctrl_key = NULL;
goto out_unlock;
}
ctrl->ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret + 10,
host->dhchap_ctrl_key_hash);
if (IS_ERR(ctrl->ctrl_key)) {
ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE;
ctrl->ctrl_key = NULL;
goto out_free_hash;
}
#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG
pr_debug("%s: using ctrl hash %s key %*ph\n", __func__,
ctrl->ctrl_key->hash > 0 ?
nvme_auth_hmac_name(ctrl->ctrl_key->hash) : "none",
(int)ctrl->ctrl_key->len, ctrl->ctrl_key->key);
#endif
out_free_hash:
if (ret) {
if (ctrl->host_key) {
nvme_auth_free_key(ctrl->host_key);
ctrl->host_key = NULL;
}
ctrl->shash_id = 0;
}
out_unlock:
up_read(&nvmet_config_sem);
+4 -2
View File
@@ -150,7 +150,7 @@ static ssize_t nvmet_addr_traddr_store(struct config_item *item,
if (nvmet_is_port_enabled(port, __func__))
return -EACCES;
if (sscanf(page, "%s\n", port->disc_addr.traddr) != 1)
if (sscanf(page, "%255s", port->disc_addr.traddr) != 1)
return -EINVAL;
return count;
}
@@ -242,7 +242,7 @@ static ssize_t nvmet_addr_trsvcid_store(struct config_item *item,
if (nvmet_is_port_enabled(port, __func__))
return -EACCES;
if (sscanf(page, "%s\n", port->disc_addr.trsvcid) != 1)
if (sscanf(page, "%31s", port->disc_addr.trsvcid) != 1)
return -EINVAL;
return count;
}
@@ -1466,8 +1466,10 @@ static ssize_t nvmet_subsys_attr_tbkas_store(struct config_item *item,
subsys->tbkas = tbkas;
/* Force reconnect */
mutex_lock(&subsys->lock);
list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry)
ctrl->ops->delete_ctrl(ctrl);
mutex_unlock(&subsys->lock);
up_write(&nvmet_config_sem);
return count;
+17 -10
View File
@@ -669,13 +669,15 @@ void nvmet_ns_disable(struct nvmet_ns *ns)
/*
* Now that we removed the namespaces from the lookup list, we
* can kill the per_cpu ref and wait for any remaining references
* to be dropped, as well as a RCU grace period for anyone only
* using the namespace under rcu_read_lock(). Note that we can't
* use call_rcu here as we need to ensure the namespaces have
* been fully destroyed before unloading the module.
* to be dropped. Rely on that wait for anyone only using the
* namespace under rcu_read_lock() too: the ref can't drop to zero
* before an RCU grace period has passed since the kill. This only
* holds as long as nothing switches the ref to atomic mode before
* the kill; bring back synchronize_rcu() here if that changes.
* Note that we can't use call_rcu here as we need to ensure the
* namespaces have been fully destroyed before unloading the module.
*/
percpu_ref_kill(&ns->ref);
synchronize_rcu();
wait_for_completion(&ns->disable_done);
percpu_ref_exit(&ns->ref);
@@ -804,6 +806,7 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
{
struct nvmet_ns *ns = req->ns;
struct nvmet_pr_per_ctrl_ref *pc_ref = req->pc_ref;
struct nvmet_passthru *passthru = nvmet_passthru_ref_held(req);
if (!req->sq->sqhd_disabled)
nvmet_update_sq_head(req);
@@ -822,7 +825,8 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status)
if (ns)
nvmet_put_namespace(ns);
nvmet_put_passthru_ref(req);
if (passthru)
nvmet_put_passthru_ref(passthru);
}
void nvmet_req_complete(struct nvmet_req *req, u16 status)
@@ -1120,9 +1124,6 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req)
if (nvme_is_fabrics(cmd))
return nvmet_parse_fabrics_io_cmd(req);
if (unlikely(!nvmet_check_auth_status(req)))
return NVME_SC_AUTH_REQUIRED | NVME_STATUS_DNR;
ret = nvmet_check_ctrl_status(req);
if (unlikely(ret))
return ret;
@@ -1620,6 +1621,12 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
u8 dhchap_status;
int ret;
if (kato > NVMET_MAX_KATO * MSEC_PER_SEC) {
args->status = NVME_SC_KA_TIMEOUT_INVALID | NVME_STATUS_DNR;
args->error_loc = offsetof(struct nvmf_connect_command, kato);
return NULL;
}
args->status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR;
subsys = nvmet_find_get_subsys(args->port, args->subsysnqn);
if (!subsys) {
@@ -1694,7 +1701,7 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args)
kato = NVMET_DISC_KATO_MS;
/* keep-alive timeout in seconds */
ctrl->kato = DIV_ROUND_UP(kato, 1000);
ctrl->kato = DIV_ROUND_UP_ULL(kato, MSEC_PER_SEC);
ctrl->err_counter = 0;
spin_lock_init(&ctrl->error_lock);
+5 -5
View File
@@ -301,7 +301,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req)
nvmet_check_transfer_len(req, tl);
return;
}
if (tl < offsetof(struct nvmf_auth_dhchap_success2_data, rsvd1)) {
if (tl < sizeof(struct nvmf_auth_dhchap_failure_data)) {
status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
req->error_loc =
offsetof(struct nvmf_auth_send_command, tl);
@@ -448,18 +448,18 @@ complete:
nvmet_req_complete(req, status);
}
static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
static int nvmet_auth_challenge(struct nvmet_req *req, void *d, u32 al)
{
struct nvmf_auth_dhchap_challenge_data *data = d;
struct nvmet_ctrl *ctrl = req->sq->ctrl;
int ret = 0;
int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
int data_size = sizeof(*data) + hash_len;
size_t data_size = sizeof(*data) + hash_len;
if (ctrl->dh_tfm)
data_size += ctrl->dh_keysize;
if (al < data_size) {
pr_debug("%s: buffer too small (al %d need %d)\n", __func__,
pr_debug("%s: buffer too small (al %u need %zu)\n", __func__,
al, data_size);
return -EINVAL;
}
@@ -488,7 +488,7 @@ static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
return ret;
}
static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al)
static int nvmet_auth_success1(struct nvmet_req *req, void *d, u32 al)
{
struct nvmf_auth_dhchap_success1_data *data = d;
struct nvmet_ctrl *ctrl = req->sq->ctrl;
+3 -2
View File
@@ -420,6 +420,8 @@ static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl)
{
nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl));
/* Left quiesced if a failed reset has left no admin queue behind */
nvme_unquiesce_admin_queue(ctrl);
nvme_remove_admin_tag_set(ctrl);
}
@@ -483,9 +485,8 @@ out_destroy_admin:
nvme_cancel_admin_tagset(&ctrl->ctrl);
nvme_loop_destroy_admin_queue(ctrl);
out_disable:
nvme_remove_admin_tag_set(&ctrl->ctrl);
dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
nvme_uninit_ctrl(&ctrl->ctrl);
nvme_delete_ctrl(&ctrl->ctrl);
}
static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
+25 -7
View File
@@ -7,6 +7,7 @@
#define _NVMET_H
#include <linux/dma-mapping.h>
#include <linux/jiffies.h>
#include <linux/types.h>
#include <linux/device.h>
#include <linux/kref.h>
@@ -714,6 +715,13 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type,
#define NVMET_KAS 10
#define NVMET_DISC_KATO_MS 120000
/*
* Largest keep-alive timeout in seconds that can be reported back in
* milliseconds and converted to jiffies without overflowing.
*/
#define NVMET_MAX_KATO \
min3(U32_MAX / MSEC_PER_SEC, U32_MAX / HZ, MAX_JIFFY_OFFSET / HZ)
int __init nvmet_init_configfs(void);
void __exit nvmet_exit_configfs(void);
@@ -824,6 +832,15 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru(
return subsys->passthru;
}
static inline struct nvmet_passthru *nvmet_passthru_ref_held(
struct nvmet_req *req)
{
if (req->p.ref_held)
return nvmet_req_subsys(req)->passthru;
return NULL;
}
static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
{
if (!percpu_ref_tryget_live(&nvmet_req_subsys(req)->passthru->ref))
@@ -833,13 +850,9 @@ static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
return true;
}
static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
static inline void nvmet_put_passthru_ref(struct nvmet_passthru *passthru)
{
if (!req->p.ref_held)
return;
req->p.ref_held = false;
percpu_ref_put(&nvmet_req_subsys(req)->passthru->ref);
percpu_ref_put(&passthru->ref);
}
#else /* CONFIG_NVME_TARGET_PASSTHRU */
static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys)
@@ -865,11 +878,16 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru(
{
return NULL;
}
static inline struct nvmet_passthru *nvmet_passthru_ref_held(
struct nvmet_req *req)
{
return NULL;
}
static inline bool nvmet_get_passthru_ref(struct nvmet_req *req)
{
return false;
}
static inline void nvmet_put_passthru_ref(struct nvmet_req *req)
static inline void nvmet_put_passthru_ref(struct nvmet_passthru *passthru)
{
}
#endif /* CONFIG_NVME_TARGET_PASSTHRU */
+1 -1
View File
@@ -2611,7 +2611,7 @@ static const struct pci_epf_device_id nvmet_pci_epf_ids[] = {
{},
};
static struct pci_epf_ops nvmet_pci_epf_ops = {
static const struct pci_epf_ops nvmet_pci_epf_ops = {
.bind = nvmet_pci_epf_bind,
.unbind = nvmet_pci_epf_unbind,
.add_cfs = nvmet_pci_epf_add_cfs,
+35 -21
View File
@@ -71,7 +71,6 @@ struct nvmet_rdma_rsp {
struct nvmet_req req;
bool allocated;
u8 n_rdma;
u32 flags;
u32 invalidate_rkey;
@@ -1197,13 +1196,18 @@ static void nvmet_rdma_free_dev(struct kref *ref)
struct nvmet_rdma_device *ndev =
container_of(ref, struct nvmet_rdma_device, ref);
nvmet_rdma_destroy_srqs(ndev);
ib_dealloc_pd(ndev->pd);
/*
* Only delete from the device list once all RDMA resources are torn
* down because nvmet_rdma_remove_one waits for the release work only
* if the device still is on the list.
*/
mutex_lock(&device_list_mutex);
list_del(&ndev->entry);
mutex_unlock(&device_list_mutex);
nvmet_rdma_destroy_srqs(ndev);
ib_dealloc_pd(ndev->pd);
kfree(ndev);
}
@@ -1694,7 +1698,7 @@ out_unlock:
static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
{
bool disconnect = false;
bool disconnect = false, live = false;
unsigned long flags;
pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state);
@@ -1713,6 +1717,7 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
}
fallthrough;
case NVMET_RDMA_Q_LIVE:
live = queue->state == NVMET_RDMA_Q_LIVE;
queue->state = NVMET_RDMA_Q_DISCONNECTING;
disconnect = true;
break;
@@ -1722,24 +1727,34 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
spin_unlock_irqrestore(&queue->state_lock, flags);
if (disconnect) {
rdma_disconnect(queue->cm_id);
/*
* Over iWARP rdma_disconnect() sleeps until the CM has handled
* the ESTABLISHED event of this connection. The CM handles the
* events of all connections one at a time, and our handler
* takes nvmet_rdma_queue_mutex, which callers of this function
* hold. So a queue that has not seen the event is left to the
* release work: ib_drain_qp() moves the queue pair to the error
* state, which ends the connection.
*/
if (live)
rdma_disconnect(queue->cm_id);
queue_work(nvmet_wq, &queue->release_work);
}
}
static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue)
{
bool disconnect = false;
/*
* Queue the release work with &nvmet_rdma_queue_mutex held, so that
* nvmet_rdma_remove_one() either finds the queue on the list or the
* work has already been queued.
*/
mutex_lock(&nvmet_rdma_queue_mutex);
if (!list_empty(&queue->queue_list)) {
list_del_init(&queue->queue_list);
disconnect = true;
__nvmet_rdma_queue_disconnect(queue);
}
mutex_unlock(&nvmet_rdma_queue_mutex);
if (disconnect)
__nvmet_rdma_queue_disconnect(queue);
}
static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
@@ -1766,10 +1781,13 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id,
* queue cm_id and/or a device bound listener cm_id (where in this
* case queue will be null).
*
* We registered an ib_client to handle device removal for queues,
* so we only need to handle the listening port cm_ids. In this case
* we nullify the priv to prevent double cm_id destruction and destroying
* the cm_id implicitly by returning a non-zero rc to the callout.
* The RDMA CM waits until each cm_id of the device is destroyed, so a queue
* has to go now. Our ib_client also removes the queues, but it cannot be
* relied on here: built in, it is called after the RDMA CM's client.
*
* For a listening port cm_id we nullify the priv to prevent double cm_id
* destruction and destroying the cm_id implicitly by returning a non-zero
* rc to the callout.
*/
static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id,
struct nvmet_rdma_queue *queue)
@@ -1777,11 +1795,7 @@ static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id,
struct nvmet_rdma_port *port;
if (queue) {
/*
* This is a queue cm_id. we have registered
* an ib_client to handle queues removal
* so don't interfere and just return.
*/
nvmet_rdma_queue_disconnect(queue);
return 0;
}
+11 -2
View File
@@ -102,6 +102,7 @@ MODULE_PARM_DESC(tls_handshake_timeout,
#define NVMET_TCP_RECV_BUDGET 8
#define NVMET_TCP_SEND_BUDGET 8
#define NVMET_TCP_IO_WORK_BUDGET 64
#define NVMET_TCP_ACCEPT_BUDGET 16
enum nvmet_tcp_send_state {
NVMET_TCP_SEND_DATA_PDU,
@@ -1302,6 +1303,11 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue)
int len, ret;
while (msg_data_left(&cmd->recv_msg)) {
if (queue->tls_pskid) {
/* The last receive has used it up */
cmd->recv_msg.msg_control = cmd->recv_cbuf;
cmd->recv_msg.msg_controllen = sizeof(cmd->recv_cbuf);
}
len = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg,
cmd->recv_msg.msg_flags);
if (len <= 0)
@@ -2068,9 +2074,9 @@ static void nvmet_tcp_accept_work(struct work_struct *w)
struct nvmet_tcp_port *port =
container_of(w, struct nvmet_tcp_port, accept_work);
struct socket *newsock;
int ret;
int ret, i;
while (true) {
for (i = 0; i < NVMET_TCP_ACCEPT_BUDGET; i++) {
ret = kernel_accept(port->sock, &newsock, O_NONBLOCK);
if (ret < 0) {
if (ret != -EAGAIN)
@@ -2079,6 +2085,9 @@ static void nvmet_tcp_accept_work(struct work_struct *w)
}
nvmet_tcp_alloc_queue(port, newsock);
}
/* There may be more. Come to an end for those who flush nvmet_wq. */
queue_work(nvmet_wq, &port->accept_work);
}
static void nvmet_tcp_listen_data_ready(struct sock *sk)
+1 -3
View File
@@ -678,7 +678,7 @@ enum {
QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */
QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */
QUEUE_FLAG_DISABLE_WBT_DEF, /* for sched to disable/enable wbt */
QUEUE_FLAG_NO_ELV_SWITCH, /* can't switch elevator any more */
QUEUE_FLAG_NO_ELV_SWITCH, /* a switch is running or disk is going away */
QUEUE_FLAG_QOS_ENABLED, /* qos is enabled */
QUEUE_FLAG_BIO_ISSUE_TIME, /* record bio->issue_time_ns */
QUEUE_FLAG_ZONED_QD1_WRITES, /* Limit zoned devices writes to QD=1 */
@@ -719,8 +719,6 @@ void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE)
#define blk_queue_disable_wbt(q) \
test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags)
#define blk_queue_no_elv_switch(q) \
test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags)
#define blk_queue_zoned_qd1_writes(q) \
test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags)
+36
View File
@@ -18,6 +18,7 @@
#include <linux/mutex.h>
#include <linux/rcupdate.h>
#include <linux/rculist.h>
#include <linux/workqueue.h>
#include <linux/rcu_segcblist.h>
@@ -118,6 +119,41 @@ void cleanup_srcu_struct(struct srcu_struct *ssp);
void synchronize_srcu(struct srcu_struct *ssp);
void synchronize_srcu_atomic(struct srcu_struct *ssp);
/**
* list_splice_init_srcu - splice an SRCU-protected list into an existing list
* @list: the SRCU-protected list to splice
* @head: the place in the existing list to splice the first list into
* @ssp: the srcu_struct that readers of @list use
*
* Like list_splice_init_rcu(), but wait for the SRCU readers of @list
* instead of RCU ones, so that none of them can follow a next pointer
* onto @head.
*
* Note that this function blocks, and that the caller must prevent any
* other updates to @list.
*/
static inline void list_splice_init_srcu(struct list_head *list,
struct list_head *head,
struct srcu_struct *ssp)
{
struct list_head *first = list->next;
struct list_head *last = list->prev;
struct list_head *next = head->next;
if (list_empty(list))
return;
INIT_LIST_HEAD_RCU(list);
synchronize_srcu(ssp);
ASSERT_EXCLUSIVE_ACCESS(*first);
ASSERT_EXCLUSIVE_ACCESS(*last);
last->next = next;
rcu_assign_pointer(list_next_rcu(head), first);
first->prev = head;
next->prev = last;
}
#define SRCU_GET_STATE_COMPLETED 0x1
/**
+1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
CFLAGS += -O3 -Wl,-no-as-needed -Wall -I $(top_srcdir)/usr/include
CFLAGS += -D_FILE_OFFSET_BITS=64
ifneq ($(WERROR),0)
CFLAGS += -Werror
endif
+3 -3
View File
@@ -376,10 +376,10 @@ static int __ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_qu
if (ublk_queue_use_auto_zc(q))
elem->buf_index = ublk_batch_io_buf_idx(t, q, i);
else if (!ublk_queue_no_buf(q))
elem->buf_addr = (__u64)io->buf_addr;
elem->buf_addr = (__u64)(uintptr_t)io->buf_addr;
}
sqe->addr = (__u64)buf;
sqe->addr = (__u64)(uintptr_t)buf;
sqe->len = t->commit_buf_elem_size * nr_elem;
ublk_init_batch_cmd(t, q->q_id, sqe, UBLK_U_IO_PREP_IO_CMDS,
@@ -473,7 +473,7 @@ static void __ublk_batch_commit_io_cmds(struct ublk_thread *t,
ublk_io_alloc_sqes(t, &sqe, 1);
buf_idx = cb->buf_idx;
sqe->addr = (__u64)cb->elem;
sqe->addr = (__u64)(uintptr_t)cb->elem;
sqe->len = nr_elem * t->commit_buf_elem_size;
/* commit isn't per-queue command */
+1 -1
View File
@@ -204,7 +204,7 @@ int ublk_ctrl_get_params(struct ublk_dev *dev,
struct ublk_ctrl_cmd_data data = {
.cmd_op = UBLK_U_CMD_GET_PARAMS,
.flags = CTRL_CMD_HAS_BUF,
.addr = (__u64)params,
.addr = (__u64)(uintptr_t)params,
.len = sizeof(*params),
};
+1 -1
View File
@@ -148,7 +148,7 @@ static int ublk_ctrl_get_affinity(struct ublk_dev *ctrl_dev, cpu_set_t **ptr_buf
for (i = 0; i < ctrl_dev->dev_info.nr_hw_queues; i++) {
data.data[0] = i;
data.len = sizeof(cpu_set_t);
data.addr = (__u64)&buf[i];
data.addr = (__u64)(uintptr_t)&buf[i];
ret = __ublk_ctrl_cmd(ctrl_dev, &data);
if (ret < 0) {