diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 1c1fe07a96ea..1c1025157d8e 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -2936,6 +2936,9 @@ static void ioc_rqos_done(struct rq_qos *rqos, struct request *rq) if (!ioc->enabled || !rq->alloc_time_ns || !rq->start_time_ns) return; + if (unlikely(rq->rq_flags & RQF_FLUSH_SEQ)) + return; + switch (req_op(rq)) { case REQ_OP_READ: pidx = QOS_RLAT; diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index 0a00f5a76f5a..d7b45778732b 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -374,7 +374,7 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq, } EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge); -/* called in queue's release handler, tagset has gone away */ +/* Clear scheduler tags and reset queue state. */ static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int flags) { struct blk_mq_hw_ctx *hctx; @@ -385,6 +385,10 @@ static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int fla if (blk_mq_is_shared_tags(flags)) q->sched_shared_tags = NULL; + + blk_queue_flag_clear(QUEUE_FLAG_SQ_SCHED, q); + q->nr_requests = q->tag_set->queue_depth; + q->async_depth = q->tag_set->queue_depth; } void blk_mq_sched_reg_debugfs(struct request_queue *q) diff --git a/block/blk-mq.c b/block/blk-mq.c index d9ab295be4d9..5ae22d556540 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -3160,8 +3160,15 @@ void blk_mq_submit_bio(struct bio *bio) goto queue_exit; } + /* + * The block layer cannot generate or verify PI for a dma-buf bio, as + * that walks the data pages. Only fail the bio if it needs that, not + * if the submitter attached its own integrity payload or if PI + * generation or verification is turned off for the device. + */ if (unlikely(op_is_dmabuf(bio->bi_opf) && - bdev_has_integrity_csum(bio->bi_bdev))) { + bdev_has_integrity_csum(bio->bi_bdev) && + (bio_integrity_action(bio) & BI_ACT_CHECK))) { bio_endio_status(bio, BLK_STS_NOTSUPP); goto queue_exit; } @@ -3209,16 +3216,15 @@ new_request: blk_mq_bio_to_request(rq, bio, nr_segs); - ret = blk_crypto_rq_get_keyslot(rq); - if (ret != BLK_STS_OK) { - bio_endio_status(bio, ret); - blk_mq_free_request(rq); - return; - } - if (bio_zone_write_plugging(bio)) blk_zone_write_plug_init_request(rq); + ret = blk_crypto_rq_get_keyslot(rq); + if (ret != BLK_STS_OK) { + blk_mq_end_request(rq, ret); + return; + } + if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq)) return; @@ -3501,8 +3507,10 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, if (blk_mq_is_shared_tags(set->flags)) drv_tags = set->shared_tags; - else + else if (hctx_idx < set->nr_hw_queues) drv_tags = set->tags[hctx_idx]; + else + drv_tags = NULL; if (tags->static_rqs && set->ops->exit_request) { int i; diff --git a/block/blk.h b/block/blk.h index ed2172472622..0e38e4b88cf5 100644 --- a/block/blk.h +++ b/block/blk.h @@ -418,6 +418,8 @@ static inline bool bio_may_need_split(struct bio *bio, if (!bio->bi_io_vec) return true; + if (op_is_dmabuf(bio->bi_opf)) + return true; bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_offset) diff --git a/block/elevator.c b/block/elevator.c index 2161b6eea680..66edf1a5768a 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -812,25 +812,29 @@ ssize_t elv_iosched_store(struct gendisk *disk, const char *buf, * reference during concurrent disk deletion: * update_nr_hwq_lock -> kn->active (via del_gendisk -> kobject_del) * kn->active -> update_nr_hwq_lock (via this sysfs write path) - * - * Use the writer lock instead of the reader lock of update_nr_hwq_lock - * to serialize the two-stage elevator switch steps in - * elevator_change(): the core switch step under the elevator lock and - * the elevator_change_done() step outside the elevator lock. */ - if (!down_write_trylock(&set->update_nr_hwq_lock)) { + if (!down_read_trylock(&set->update_nr_hwq_lock)) { ret = -EBUSY; goto out; } - if (!blk_queue_no_elv_switch(q)) { - ret = elevator_change(q, &ctx); - if (!ret) - ret = count; - } else { - ret = -ENOENT; - } - up_write(&set->update_nr_hwq_lock); + /* + * The reader lock is shared, so use QUEUE_FLAG_NO_ELV_SWITCH to allow + * only one switch of this queue at a time, and none once the disk is + * going away. + */ + if (test_and_set_bit(QUEUE_FLAG_NO_ELV_SWITCH, &q->queue_flags)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = elevator_change(q, &ctx); + if (!ret) + ret = count; + + blk_queue_flag_clear(QUEUE_FLAG_NO_ELV_SWITCH, q); +out_unlock: + up_read(&set->update_nr_hwq_lock); out: if (ctx.type) elevator_put(ctx.type); diff --git a/block/fops.c b/block/fops.c index e5c27cc35ca1..a0ccadaa9c79 100644 --- a/block/fops.c +++ b/block/fops.c @@ -1018,8 +1018,6 @@ static int blkdev_init_dma_buf_io_ctx(struct file *file, return -EINVAL; if (!disk->fops->init_dma_buf_io_ctx) return -EOPNOTSUPP; - if (bdev_has_integrity_csum(bdev)) - return -EOPNOTSUPP; return disk->fops->init_dma_buf_io_ctx(bdev, ctx); } diff --git a/block/genhd.c b/block/genhd.c index 10ca8b4d6eea..69204d44adae 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1146,7 +1146,7 @@ static ssize_t disk_discard_alignment_show(struct device *dev, { struct gendisk *disk = dev_to_disk(dev); - return sysfs_emit(buf, "%d\n", bdev_alignment_offset(disk->part0)); + return sysfs_emit(buf, "%u\n", bdev_discard_alignment(disk->part0)); } static ssize_t diskseq_show(struct device *dev, diff --git a/block/ioctl.c b/block/ioctl.c index 64b4e6c0f696..fb2f4b4d2505 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -835,8 +835,8 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) return compat_put_ulong(argp, bdev_nr_sectors(bdev)); /* The data is compatible, but the command number is different */ - case BLKBSZGET_32: /* get the logical block size (cf. BLKSSZGET) */ - return put_int(argp, bdev_logical_block_size(bdev)); + case BLKBSZGET_32: /* get the soft block size (cf. BLKSSZGET) */ + return put_int(argp, block_size(bdev)); case BLKBSZSET_32: return blkdev_bszset(file, mode, argp); case BLKGETSIZE64_32: diff --git a/block/partitions/efi.c b/block/partitions/efi.c index 9865d59093fa..b31227a13db0 100644 --- a/block/partitions/efi.c +++ b/block/partitions/efi.c @@ -460,6 +460,7 @@ static inline int is_pte_valid(const gpt_entry *pte, const u64 lastlba) { if ((!efi_guidcmp(pte->partition_type_guid, NULL_GUID)) || + le64_to_cpu(pte->starting_lba) > le64_to_cpu(pte->ending_lba) || le64_to_cpu(pte->starting_lba) > lastlba || le64_to_cpu(pte->ending_lba) > lastlba) return 0; diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 6beb1f5b71ac..ec0d7ba4d938 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1976,6 +1976,7 @@ static int null_add_dev(struct nullb_device *dev) rv = null_validate_conf(dev); if (rv) return rv; + clear_bit(NULLB_DEV_FL_CACHE, &dev->flags); nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node); if (!nullb) { diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 7e9b5ffb9e84..b271d0a88ff2 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -311,10 +311,8 @@ static blk_status_t null_check_open(struct nullb_device *dev) return BLK_STS_OK; if (dev->nr_zones_imp_open) { - if (null_check_active(dev) == BLK_STS_OK) { - null_close_imp_open_zone(dev); - return BLK_STS_OK; - } + null_close_imp_open_zone(dev); + return BLK_STS_OK; } return BLK_STS_ZONE_OPEN_RESOURCE; @@ -330,8 +328,11 @@ static blk_status_t null_check_open(struct nullb_device *dev) * * ZBC states that an implicit open zone shall be closed only if there is not * room within the open limit. However, with the addition of an active limit, - * it is not certain that closing an implicit open zone will allow a new zone - * to be opened, since we might already be at the active limit capacity. + * it is not certain that closing an implicit open zone will allow an empty + * zone to be opened, since we might already be at the active limit capacity. + * Therefore, the active limit is checked first for an empty zone. A closed + * zone already holds an active zone resource, so only the open limit has to + * be checked for a closed zone. */ static blk_status_t null_check_zone_resources(struct nullb_device *dev, struct nullb_zone *zone) @@ -743,23 +744,47 @@ blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op, } /* - * Set a zone in the read-only or offline condition. + * Release the zone resources held by a zone, before changing the condition of + * the zone to a condition that does not hold any zone resources. */ -static void null_set_zone_cond(struct nullb_device *dev, - struct nullb_zone *zone, enum blk_zone_cond cond) +static void null_release_zone_resources(struct nullb_device *dev, + struct nullb_zone *zone) { - if (WARN_ON_ONCE(cond != BLK_ZONE_COND_READONLY && - cond != BLK_ZONE_COND_OFFLINE)) + if (!dev->need_zone_res_mgmt) return; + spin_lock(&dev->zone_res_lock); + switch (zone->cond) { + case BLK_ZONE_COND_IMP_OPEN: + dev->nr_zones_imp_open--; + break; + case BLK_ZONE_COND_EXP_OPEN: + dev->nr_zones_exp_open--; + break; + case BLK_ZONE_COND_CLOSED: + dev->nr_zones_closed--; + break; + default: + break; + } + spin_unlock(&dev->zone_res_lock); +} + +static void null_set_zone_readonly_or_offline(struct nullb_device *dev, + struct nullb_zone *zone, + bool set_offline) +{ + enum blk_zone_cond cond = + set_offline ? BLK_ZONE_COND_OFFLINE : BLK_ZONE_COND_READONLY; + null_lock_zone(dev, zone); /* * If the read-only condition is requested again to zones already in * read-only condition, restore back normal empty condition. Do the same * if the offline condition is requested for offline zones. Otherwise, - * set the specified zone condition to the zones. Finish the zones - * beforehand to free up zone resources. + * set the specified zone condition to the zones. Release the zone + * resources held by the zones beforehand. */ if (zone->cond == cond) { zone->cond = BLK_ZONE_COND_EMPTY; @@ -767,9 +792,7 @@ static void null_set_zone_cond(struct nullb_device *dev, if (dev->memory_backed) null_handle_discard(dev, zone->start, zone->len); } else { - if (zone->cond != BLK_ZONE_COND_READONLY && - zone->cond != BLK_ZONE_COND_OFFLINE) - null_finish_zone(dev, zone); + null_release_zone_resources(dev, zone); zone->cond = cond; zone->wp = NULL_ZONE_INVALID_WP; } @@ -788,6 +811,11 @@ ssize_t zone_cond_store(struct nullb_device *dev, const char *page, unsigned int zone_no; int ret; + /* Read-only and offline are the only conditions supported for now */ + if (WARN_ON_ONCE(cond != BLK_ZONE_COND_READONLY && + cond != BLK_ZONE_COND_OFFLINE)) + return -EINVAL; + if (!dev->zoned) { pr_err("null_blk device is not zoned\n"); return -EINVAL; @@ -813,7 +841,8 @@ ssize_t zone_cond_store(struct nullb_device *dev, const char *page, return -EINVAL; } - null_set_zone_cond(dev, &dev->zones[zone_no], cond); + null_set_zone_readonly_or_offline(dev, &dev->zones[zone_no], + cond == BLK_ZONE_COND_OFFLINE); return count; } diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 72e9c435208c..3d322fa1bc83 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -5878,6 +5878,7 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, case UBLK_CMD_STOP_DEV: case UBLK_CMD_ADD_DEV: case UBLK_CMD_DEL_DEV: + case UBLK_CMD_DEL_DEV_ASYNC: case UBLK_CMD_SET_PARAMS: case UBLK_CMD_START_USER_RECOVERY: case UBLK_CMD_END_USER_RECOVERY: diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index ddbb05d3e98c..2b462005cdb6 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -331,18 +331,25 @@ static inline u8 virtblk_vbr_status(struct virtblk_req *vbr) return *((u8 *)&vbr->in_hdr + vbr->in_hdr_len - 1); } -static inline void virtblk_request_done(struct request *req) +static inline void virtblk_update_zone_append_sector(struct request *req, + struct virtblk_req *vbr) { - struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); - blk_status_t status = virtblk_result(virtblk_vbr_status(vbr)); struct virtio_blk *vblk = req->mq_hctx->queue->queuedata; - virtblk_unmap_data(req, vbr); - virtblk_cleanup_cmd(req); - if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = virtio64_to_cpu(vblk->vdev, vbr->in_hdr.zone_append.sector); +} + +static inline void virtblk_request_done(struct request *req) +{ + struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); + blk_status_t status = virtblk_result(virtblk_vbr_status(vbr)); + + virtblk_unmap_data(req, vbr); + virtblk_cleanup_cmd(req); + + virtblk_update_zone_append_sector(req, vbr); blk_mq_end_request(req, status); } @@ -958,16 +965,13 @@ static void virtblk_config_changed(struct virtio_device *vdev) queue_work(virtblk_wq, &vblk->config_work); } -static int init_vq(struct virtio_blk *vblk) +/* Keep the queue layout unchanged for the lifetime of the tag set. */ +static int virtblk_init_queue_layout(struct virtio_blk *vblk) { - int err; - unsigned short i; - struct virtqueue_info *vqs_info; - struct virtqueue **vqs; + struct virtio_device *vdev = vblk->vdev; unsigned short num_vqs; unsigned short num_poll_vqs; - struct virtio_device *vdev = vblk->vdev; - struct irq_affinity desc = { 0, }; + int err; err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ, struct virtio_blk_config, num_queues, @@ -985,6 +989,7 @@ static int init_vq(struct virtio_blk *vblk) num_poll_vqs = min_t(unsigned int, poll_queues, num_vqs - 1); + vblk->num_vqs = num_vqs; vblk->io_queues[HCTX_TYPE_DEFAULT] = num_vqs - num_poll_vqs; vblk->io_queues[HCTX_TYPE_READ] = 0; vblk->io_queues[HCTX_TYPE_POLL] = num_poll_vqs; @@ -994,6 +999,20 @@ static int init_vq(struct virtio_blk *vblk) vblk->io_queues[HCTX_TYPE_READ], vblk->io_queues[HCTX_TYPE_POLL]); + return 0; +} + +static int init_vq(struct virtio_blk *vblk) +{ + int err; + unsigned short i; + struct virtqueue_info *vqs_info; + struct virtqueue **vqs; + unsigned short num_vqs = vblk->num_vqs; + unsigned short num_poll_vqs = vblk->io_queues[HCTX_TYPE_POLL]; + struct virtio_device *vdev = vblk->vdev; + struct irq_affinity desc = { 0, }; + vblk->vqs = kmalloc_objs(*vblk->vqs, num_vqs); if (!vblk->vqs) return -ENOMEM; @@ -1049,7 +1068,6 @@ static int init_vq(struct virtio_blk *vblk) spin_lock_init(&vblk->vqs[i].lock); vblk->vqs[i].vq = vqs[i]; } - vblk->num_vqs = num_vqs; out: kfree(vqs); @@ -1220,8 +1238,11 @@ static void virtblk_complete_batch(struct io_comp_batch *iob) struct request *req; rq_list_for_each(&iob->req_list, req) { - virtblk_unmap_data(req, blk_mq_rq_to_pdu(req)); + struct virtblk_req *vbr = blk_mq_rq_to_pdu(req); + + virtblk_unmap_data(req, vbr); virtblk_cleanup_cmd(req); + virtblk_update_zone_append_sector(req, vbr); } blk_mq_end_request_batch(iob); } @@ -1512,6 +1533,10 @@ static int virtblk_probe(struct virtio_device *vdev) INIT_WORK(&vblk->config_work, virtblk_config_changed_work); + err = virtblk_init_queue_layout(vblk); + if (err) + goto out_free_vblk; + err = init_vq(vblk); if (err) goto out_free_vblk; @@ -1639,7 +1664,7 @@ static int virtblk_freeze_priv(struct virtio_device *vdev) /* Ensure no requests in virtqueues before deleting vqs. */ memflags = blk_mq_freeze_queue(q); - blk_mq_quiesce_queue_nowait(q); + blk_mq_quiesce_queue(q); blk_mq_unfreeze_queue(q, memflags); /* Ensure we don't receive any more interrupts */ diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 65634a39b0bf..96666742523b 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -1137,7 +1137,7 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts, goto out_fput; } - if (zlo->block_size) { + if (!zlo->block_size) { ret = zloop_get_block_size(zlo, zone); if (ret) goto out_fput; @@ -1161,7 +1161,8 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts, if (restore && file_sectors != zlo->zone_size) { pr_err("Invalid conventional zone %u file size (%llu sectors != %llu)\n", - zone_no, file_sectors, zlo->zone_capacity); + zone_no, file_sectors, zlo->zone_size); + ret = -EINVAL; goto out_fput; } @@ -1339,8 +1340,8 @@ static int zloop_ctl_add(struct zloop_options *opts) zlo->disk = blk_mq_alloc_disk(&zlo->tag_set, &lim, zlo); if (IS_ERR(zlo->disk)) { - pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret); ret = PTR_ERR(zlo->disk); + pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret); goto out_cleanup_tags; } zlo->disk->flags = GENHD_FL_NO_PART; diff --git a/drivers/nvme/common/keyring.c b/drivers/nvme/common/keyring.c index 32d16c53133b..87cb51e1f5a7 100644 --- a/drivers/nvme/common/keyring.c +++ b/drivers/nvme/common/keyring.c @@ -35,6 +35,7 @@ struct key *nvme_tls_key_lookup(key_serial_t key_id) } if (nvme_tls_psk_revoked(key)) { pr_err("key id %08x revoked\n", key_id); + key_put(key); return ERR_PTR(-EKEYREVOKED); } return key; diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a7a4d09644c1..f4aa02869b79 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2482,6 +2482,8 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA; else lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA); + if (ns->ctrl->quirks & NVME_QUIRK_NO_FUA) + lim.features &= ~BLK_FEAT_FUA; if (info->is_rotational) lim.features |= BLK_FEAT_ROTATIONAL; @@ -3953,10 +3955,16 @@ static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys, return 0; } +/* The char device of a namespace rather than of a namespace head */ +static bool nvme_is_ns_cdev(struct device *dev) +{ + return dev->parent->class == &nvme_class; +} + static void nvme_cdev_rel(struct device *dev) { ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt)); - if (dev->parent->class == &nvme_class) + if (nvme_is_ns_cdev(dev)) nvme_put_ns(container_of(dev, struct nvme_ns, cdev_device)); else nvme_put_ns_head(container_of(dev, struct nvme_ns_head, @@ -3990,6 +3998,14 @@ int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, device_initialize(cdev_device); cdev_init(cdev, fops); cdev->owner = owner; + /* Undone in nvme_cdev_rel() */ + if (nvme_is_ns_cdev(cdev_device)) + nvme_get_ns(container_of(cdev_device, struct nvme_ns, + cdev_device)); + else + nvme_get_ns_head(container_of(cdev_device, + struct nvme_ns_head, + cdev_device)); ret = cdev_device_add(cdev, cdev_device); if (ret) put_device(cdev_device); @@ -4022,13 +4038,11 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns) { ns->cdev_device.parent = ns->ctrl->device; - nvme_get_ns(ns); /* Undone in nvme_cdev_rel() */ if (nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops, ns->ctrl->ops->module, ns->ctrl->instance, ns->head->instance)) { dev_err(ns->ctrl->device, "Unable to create the ng%dn%d device\n", ns->ctrl->instance, ns->head->instance); - nvme_put_ns(ns); return; } set_bit(NVME_NS_CDEV_LIVE, &ns->flags); @@ -4755,9 +4769,8 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl) nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO); mutex_lock(&ctrl->namespaces_lock); - list_splice_init_rcu(&ctrl->namespaces, &ns_list, synchronize_rcu); + list_splice_init_srcu(&ctrl->namespaces, &ns_list, &ctrl->srcu); mutex_unlock(&ctrl->namespaces_lock); - synchronize_srcu(&ctrl->srcu); list_for_each_entry_safe(ns, next, &ns_list, list) nvme_ns_remove(ns); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9e0a0fd77c56..ae8059366b57 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -681,14 +681,12 @@ static void nvme_add_ns_head_cdev(struct nvme_ns_head *head) { head->cdev_device.parent = &head->subsys->dev; - nvme_get_ns_head(head); /* Undone in nvme_cdev_rel() */ if (nvme_cdev_add(&head->cdev, &head->cdev_device, &nvme_ns_head_chr_fops, THIS_MODULE, head->subsys->instance, head->instance)) { dev_err(disk_to_dev(head->disk), "Unable to create the ng%dn%d device\n", head->subsys->instance, head->instance); - nvme_put_ns_head(head); return; } set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 2d8f681079e8..8833a463f1c8 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -183,6 +183,12 @@ enum nvme_quirks { * Admin queue DMA buffers must be page aligned */ NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), + + /* + * The device is extremely slow on FUA writes but fast on Flush; + * do not advertise FUA so the block layer emulates it. + */ + NVME_QUIRK_NO_FUA = (1 << 24), }; static inline char *nvme_quirk_name(enum nvme_quirks q) @@ -236,6 +242,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q) return "dmapool_align_512"; case NVME_QUIRK_ADMIN_PAGE_ALIGN: return "admin_page_align"; + case NVME_QUIRK_NO_FUA: + return "no_fua"; } return "unknown"; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 019a1e3cc0a2..f8088d5aa724 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -4430,6 +4430,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1344, 0x6004), /* Micron 4100AT */ .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, + { PCI_DEVICE(0x1e0f, 0x0032), /* KIOXIA XG10d */ + .driver_data = NVME_QUIRK_DMAPOOL_ALIGN_512, }, { PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */ .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x1c5c, 0x174a), /* SK Hynix P31 SSD */ @@ -4483,6 +4485,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1cc1, 0x5350), /* ADATA XPG GAMMIX S50 */ .driver_data = NVME_QUIRK_BOGUS_NID, }, + { PCI_DEVICE(0x1cc1, 0x5370), /* ADATA XPG GAMMIX S70 BLADE */ + .driver_data = NVME_QUIRK_NO_FUA, }, { PCI_DEVICE(0x1dbe, 0x5216), /* Acer/INNOGRIT FA100/5216 NVMe SSD */ .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1dbe, 0x5236), /* ADATA XPG GAMMIX S70 */ diff --git a/drivers/nvme/host/pr.c b/drivers/nvme/host/pr.c index fe7dbe264815..d8b6ac4d83bb 100644 --- a/drivers/nvme/host/pr.c +++ b/drivers/nvme/host/pr.c @@ -128,8 +128,12 @@ static int nvme_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, data.crkey = cpu_to_le64(old_key); data.nrkey = cpu_to_le64(new_key); - cdw10 = old_key ? NVME_PR_REGISTER_ACT_REPLACE : - NVME_PR_REGISTER_ACT_REG; + if (!new_key) + cdw10 = NVME_PR_REGISTER_ACT_UNREG; + else if (old_key) + cdw10 = NVME_PR_REGISTER_ACT_REPLACE; + else + cdw10 = NVME_PR_REGISTER_ACT_REG; cdw10 |= (flags & PR_FL_IGNORE_KEY) ? NVME_PR_IGNORE_KEY : 0; cdw10 |= NVME_PR_CPTPL_PERSIST; diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 07216890f75c..32189fdfc0f5 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -735,7 +735,6 @@ static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, for (i = 0; i < nr_queues; i++) { struct nvme_rdma_queue *queue = &ctrl->queues[first + i]; - queue->ctrl = ctrl; queue->queue_size = queue_size; ctxs[i].queue = queue; @@ -798,7 +797,6 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, bool pi_capable = false; int error; - ctrl->queues[0].ctrl = ctrl; ctrl->queues[0].queue_size = NVME_AQ_DEPTH; error = nvme_rdma_alloc_queue(&ctrl->queues[0]); if (error) @@ -2284,7 +2282,7 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts) { struct nvme_rdma_ctrl *ctrl; - int ret; + int ret, i; ctrl = kzalloc_obj(*ctrl); if (!ctrl) @@ -2343,6 +2341,9 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev, if (!ctrl->queues) goto out_free_ctrl; + for (i = 0; i < ctrl->ctrl.queue_count; i++) + ctrl->queues[i].ctrl = ctrl; + ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_rdma_ctrl_ops, 0 /* no quirks, we're perfect! */); if (ret) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index bc46e5f3eaa7..7ecc6c007f8d 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -199,6 +199,7 @@ struct nvme_tcp_ctrl { struct delayed_work connect_work; struct nvme_tcp_request async_req; u32 io_queues[HCTX_MAX_TYPES]; + unsigned int nr_allocated_queues; struct net *net; }; @@ -2848,8 +2849,6 @@ static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); - /* the same count used to size and initialize the queue array */ - unsigned int nr_queues = NVME_TCP_OPT_NR_QUEUES(nctrl->opts); unsigned int i; if (list_empty_careful(&ctrl->list)) @@ -2862,7 +2861,7 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) nvmf_free_options(nctrl->opts); free_ctrl: put_net(ctrl->net); - for (i = 0; i < nr_queues; i++) + for (i = 0; i < ctrl->nr_allocated_queues; i++) nvme_tcp_exit_queue(ctrl, i); kfree(ctrl->queues); kfree(ctrl); @@ -3256,13 +3255,14 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, goto out_free_ctrl; } - ctrl->queues = kzalloc_objs(*ctrl->queues, ctrl->ctrl.queue_count); + ctrl->nr_allocated_queues = ctrl->ctrl.queue_count; + ctrl->queues = kzalloc_objs(*ctrl->queues, ctrl->nr_allocated_queues); if (!ctrl->queues) { ret = -ENOMEM; goto out_free_ctrl; } - for (i = 0; i < ctrl->ctrl.queue_count; i++) + for (i = 0; i < ctrl->nr_allocated_queues; i++) nvme_tcp_init_queue(ctrl, i); ctrl->net = get_net(current->nsproxy->net_ns); diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 26028be8fe4d..f12d83098644 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -543,6 +543,15 @@ static void nvmet_execute_get_log_page_ana(struct nvmet_req *req) u16 ngrps = 0; u16 status; + /* + * The header and one descriptor per enabled group are the only bytes + * written below; zero the whole transfer first so the unwritten tail + * does not leak stale kernel memory to the host. + */ + status = nvmet_zero_sgl(req, 0, req->transfer_len); + if (status) + goto out; + status = NVME_SC_INTERNAL; desc = kmalloc_flex(*desc, nsids, NVMET_MAX_NAMESPACES); if (!desc) @@ -1227,8 +1236,13 @@ u16 nvmet_set_feat_kato(struct nvmet_req *req) { u32 val32 = le32_to_cpu(req->cmd->common.cdw11); + if (val32 > NVMET_MAX_KATO * MSEC_PER_SEC) { + req->error_loc = offsetof(struct nvme_common_command, cdw11); + return NVME_SC_KA_TIMEOUT_INVALID | NVME_STATUS_DNR; + } + nvmet_stop_keep_alive_timer(req->sq->ctrl); - req->sq->ctrl->kato = DIV_ROUND_UP(val32, 1000); + req->sq->ctrl->kato = DIV_ROUND_UP_ULL(val32, MSEC_PER_SEC); nvmet_start_keep_alive_timer(req->sq->ctrl); nvmet_set_result(req, req->sq->ctrl->kato); @@ -1326,7 +1340,7 @@ static u16 nvmet_set_feat_arbitration(struct nvmet_req *req) .hpw = (cdw11 >> 24) & 0xff, .mpw = (cdw11 >> 16) & 0xff, .lpw = (cdw11 >> 8) & 0xff, - .ab = cdw11 & 0x3, + .ab = cdw11 & 0x7, }; if (!ctrl->ops->set_feature) { @@ -1481,7 +1495,7 @@ static u16 nvmet_get_feat_arbitration(struct nvmet_req *req) ((u32)arb.hpw << 24) | ((u32)arb.mpw << 16) | ((u32)arb.lpw << 8) | - (arb.ab & 0x3)); + (arb.ab & 0x7)); return NVME_SC_SUCCESS; } diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c index 574025730834..ccba84b1344e 100644 --- a/drivers/nvme/target/auth.c +++ b/drivers/nvme/target/auth.c @@ -135,6 +135,7 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset) int ret = 0; struct nvmet_host_link *p; struct nvmet_host *host = NULL; + struct nvme_dhchap_key *host_key, *ctrl_key = NULL; down_read(&nvmet_config_sem); if (nvmet_is_disc_subsys(ctrl->subsys)) @@ -172,55 +173,41 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset) goto out_unlock; } - if (host->dhchap_hash_id == ctrl->shash_id) { - pr_debug("Re-use existing hash ID %d\n", - ctrl->shash_id); - } else { - ctrl->shash_id = host->dhchap_hash_id; - } - - /* Skip the 'DHHC-1:XX:' prefix */ - nvme_auth_free_key(ctrl->host_key); - ctrl->host_key = nvme_auth_extract_key(host->dhchap_secret + 10, - host->dhchap_key_hash); - if (IS_ERR(ctrl->host_key)) { + /* + * Skip the 'DHHC-1:XX:' prefix. Keep the old keys until the new ones + * are there: a controller without a host key needs no authentication. + */ + host_key = nvme_auth_extract_key(host->dhchap_secret + 10, + host->dhchap_key_hash); + if (IS_ERR(host_key)) { ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE; - ctrl->host_key = NULL; - goto out_free_hash; + goto out_unlock; } + if (host->dhchap_ctrl_secret) { + ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret + 10, + host->dhchap_ctrl_key_hash); + if (IS_ERR(ctrl_key)) { + nvme_auth_free_key(host_key); + ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE; + goto out_unlock; + } + } + nvme_auth_free_key(ctrl->host_key); + ctrl->host_key = host_key; + nvme_auth_free_key(ctrl->ctrl_key); + ctrl->ctrl_key = ctrl_key; + ctrl->shash_id = host->dhchap_hash_id; #ifdef CONFIG_NVME_TARGET_AUTH_DEBUG pr_debug("%s: using hash %s key %*ph\n", __func__, ctrl->host_key->hash > 0 ? nvme_auth_hmac_name(ctrl->host_key->hash) : "none", (int)ctrl->host_key->len, ctrl->host_key->key); + if (ctrl->ctrl_key) + pr_debug("%s: using ctrl hash %s key %*ph\n", __func__, + ctrl->ctrl_key->hash > 0 ? + nvme_auth_hmac_name(ctrl->ctrl_key->hash) : "none", + (int)ctrl->ctrl_key->len, ctrl->ctrl_key->key); #endif - nvme_auth_free_key(ctrl->ctrl_key); - if (!host->dhchap_ctrl_secret) { - ctrl->ctrl_key = NULL; - goto out_unlock; - } - - ctrl->ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret + 10, - host->dhchap_ctrl_key_hash); - if (IS_ERR(ctrl->ctrl_key)) { - ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE; - ctrl->ctrl_key = NULL; - goto out_free_hash; - } -#ifdef CONFIG_NVME_TARGET_AUTH_DEBUG - pr_debug("%s: using ctrl hash %s key %*ph\n", __func__, - ctrl->ctrl_key->hash > 0 ? - nvme_auth_hmac_name(ctrl->ctrl_key->hash) : "none", - (int)ctrl->ctrl_key->len, ctrl->ctrl_key->key); -#endif -out_free_hash: - if (ret) { - if (ctrl->host_key) { - nvme_auth_free_key(ctrl->host_key); - ctrl->host_key = NULL; - } - ctrl->shash_id = 0; - } out_unlock: up_read(&nvmet_config_sem); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index b3a5242d06f9..bde94656e4b7 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -150,7 +150,7 @@ static ssize_t nvmet_addr_traddr_store(struct config_item *item, if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - if (sscanf(page, "%s\n", port->disc_addr.traddr) != 1) + if (sscanf(page, "%255s", port->disc_addr.traddr) != 1) return -EINVAL; return count; } @@ -242,7 +242,7 @@ static ssize_t nvmet_addr_trsvcid_store(struct config_item *item, if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - if (sscanf(page, "%s\n", port->disc_addr.trsvcid) != 1) + if (sscanf(page, "%31s", port->disc_addr.trsvcid) != 1) return -EINVAL; return count; } @@ -1466,8 +1466,10 @@ static ssize_t nvmet_subsys_attr_tbkas_store(struct config_item *item, subsys->tbkas = tbkas; /* Force reconnect */ + mutex_lock(&subsys->lock); list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) ctrl->ops->delete_ctrl(ctrl); + mutex_unlock(&subsys->lock); up_write(&nvmet_config_sem); return count; diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a8171b88035c..cc1955bc8c84 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -669,13 +669,15 @@ void nvmet_ns_disable(struct nvmet_ns *ns) /* * Now that we removed the namespaces from the lookup list, we * can kill the per_cpu ref and wait for any remaining references - * to be dropped, as well as a RCU grace period for anyone only - * using the namespace under rcu_read_lock(). Note that we can't - * use call_rcu here as we need to ensure the namespaces have - * been fully destroyed before unloading the module. + * to be dropped. Rely on that wait for anyone only using the + * namespace under rcu_read_lock() too: the ref can't drop to zero + * before an RCU grace period has passed since the kill. This only + * holds as long as nothing switches the ref to atomic mode before + * the kill; bring back synchronize_rcu() here if that changes. + * Note that we can't use call_rcu here as we need to ensure the + * namespaces have been fully destroyed before unloading the module. */ percpu_ref_kill(&ns->ref); - synchronize_rcu(); wait_for_completion(&ns->disable_done); percpu_ref_exit(&ns->ref); @@ -804,6 +806,7 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status) { struct nvmet_ns *ns = req->ns; struct nvmet_pr_per_ctrl_ref *pc_ref = req->pc_ref; + struct nvmet_passthru *passthru = nvmet_passthru_ref_held(req); if (!req->sq->sqhd_disabled) nvmet_update_sq_head(req); @@ -822,7 +825,8 @@ static void __nvmet_req_complete(struct nvmet_req *req, u16 status) if (ns) nvmet_put_namespace(ns); - nvmet_put_passthru_ref(req); + if (passthru) + nvmet_put_passthru_ref(passthru); } void nvmet_req_complete(struct nvmet_req *req, u16 status) @@ -1120,9 +1124,6 @@ static u16 nvmet_parse_io_cmd(struct nvmet_req *req) if (nvme_is_fabrics(cmd)) return nvmet_parse_fabrics_io_cmd(req); - if (unlikely(!nvmet_check_auth_status(req))) - return NVME_SC_AUTH_REQUIRED | NVME_STATUS_DNR; - ret = nvmet_check_ctrl_status(req); if (unlikely(ret)) return ret; @@ -1620,6 +1621,12 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) u8 dhchap_status; int ret; + if (kato > NVMET_MAX_KATO * MSEC_PER_SEC) { + args->status = NVME_SC_KA_TIMEOUT_INVALID | NVME_STATUS_DNR; + args->error_loc = offsetof(struct nvmf_connect_command, kato); + return NULL; + } + args->status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR; subsys = nvmet_find_get_subsys(args->port, args->subsysnqn); if (!subsys) { @@ -1694,7 +1701,7 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) kato = NVMET_DISC_KATO_MS; /* keep-alive timeout in seconds */ - ctrl->kato = DIV_ROUND_UP(kato, 1000); + ctrl->kato = DIV_ROUND_UP_ULL(kato, MSEC_PER_SEC); ctrl->err_counter = 0; spin_lock_init(&ctrl->error_lock); diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 70136f6c22e8..34c2e14d0bd9 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -301,7 +301,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) nvmet_check_transfer_len(req, tl); return; } - if (tl < offsetof(struct nvmf_auth_dhchap_success2_data, rsvd1)) { + if (tl < sizeof(struct nvmf_auth_dhchap_failure_data)) { status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; req->error_loc = offsetof(struct nvmf_auth_send_command, tl); @@ -448,18 +448,18 @@ complete: nvmet_req_complete(req, status); } -static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al) +static int nvmet_auth_challenge(struct nvmet_req *req, void *d, u32 al) { struct nvmf_auth_dhchap_challenge_data *data = d; struct nvmet_ctrl *ctrl = req->sq->ctrl; int ret = 0; int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id); - int data_size = sizeof(*data) + hash_len; + size_t data_size = sizeof(*data) + hash_len; if (ctrl->dh_tfm) data_size += ctrl->dh_keysize; if (al < data_size) { - pr_debug("%s: buffer too small (al %d need %d)\n", __func__, + pr_debug("%s: buffer too small (al %u need %zu)\n", __func__, al, data_size); return -EINVAL; } @@ -488,7 +488,7 @@ static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al) return ret; } -static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al) +static int nvmet_auth_success1(struct nvmet_req *req, void *d, u32 al) { struct nvmf_auth_dhchap_success1_data *data = d; struct nvmet_ctrl *ctrl = req->sq->ctrl; diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index e579a307798b..80b67608eb15 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -420,6 +420,8 @@ static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) static void nvme_loop_delete_ctrl_host(struct nvme_ctrl *ctrl) { nvme_loop_shutdown_ctrl(to_loop_ctrl(ctrl)); + /* Left quiesced if a failed reset has left no admin queue behind */ + nvme_unquiesce_admin_queue(ctrl); nvme_remove_admin_tag_set(ctrl); } @@ -483,9 +485,8 @@ out_destroy_admin: nvme_cancel_admin_tagset(&ctrl->ctrl); nvme_loop_destroy_admin_queue(ctrl); out_disable: - nvme_remove_admin_tag_set(&ctrl->ctrl); dev_warn(ctrl->ctrl.device, "Removing after reset failure\n"); - nvme_uninit_ctrl(&ctrl->ctrl); + nvme_delete_ctrl(&ctrl->ctrl); } static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = { diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index d7804bf91aaa..2c2460ef5e2b 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -7,6 +7,7 @@ #define _NVMET_H #include +#include #include #include #include @@ -714,6 +715,13 @@ void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, #define NVMET_KAS 10 #define NVMET_DISC_KATO_MS 120000 +/* + * Largest keep-alive timeout in seconds that can be reported back in + * milliseconds and converted to jiffies without overflowing. + */ +#define NVMET_MAX_KATO \ + min3(U32_MAX / MSEC_PER_SEC, U32_MAX / HZ, MAX_JIFFY_OFFSET / HZ) + int __init nvmet_init_configfs(void); void __exit nvmet_exit_configfs(void); @@ -824,6 +832,15 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru( return subsys->passthru; } +static inline struct nvmet_passthru *nvmet_passthru_ref_held( + struct nvmet_req *req) +{ + if (req->p.ref_held) + return nvmet_req_subsys(req)->passthru; + + return NULL; +} + static inline bool nvmet_get_passthru_ref(struct nvmet_req *req) { if (!percpu_ref_tryget_live(&nvmet_req_subsys(req)->passthru->ref)) @@ -833,13 +850,9 @@ static inline bool nvmet_get_passthru_ref(struct nvmet_req *req) return true; } -static inline void nvmet_put_passthru_ref(struct nvmet_req *req) +static inline void nvmet_put_passthru_ref(struct nvmet_passthru *passthru) { - if (!req->p.ref_held) - return; - - req->p.ref_held = false; - percpu_ref_put(&nvmet_req_subsys(req)->passthru->ref); + percpu_ref_put(&passthru->ref); } #else /* CONFIG_NVME_TARGET_PASSTHRU */ static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys) @@ -865,11 +878,16 @@ static inline struct nvmet_passthru *nvmet_subsys_passthru( { return NULL; } +static inline struct nvmet_passthru *nvmet_passthru_ref_held( + struct nvmet_req *req) +{ + return NULL; +} static inline bool nvmet_get_passthru_ref(struct nvmet_req *req) { return false; } -static inline void nvmet_put_passthru_ref(struct nvmet_req *req) +static inline void nvmet_put_passthru_ref(struct nvmet_passthru *passthru) { } #endif /* CONFIG_NVME_TARGET_PASSTHRU */ diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index a8df176c1a57..cfc52454325b 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -2611,7 +2611,7 @@ static const struct pci_epf_device_id nvmet_pci_epf_ids[] = { {}, }; -static struct pci_epf_ops nvmet_pci_epf_ops = { +static const struct pci_epf_ops nvmet_pci_epf_ops = { .bind = nvmet_pci_epf_bind, .unbind = nvmet_pci_epf_unbind, .add_cfs = nvmet_pci_epf_add_cfs, diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index b9e39d1657d2..f386c388da18 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -71,7 +71,6 @@ struct nvmet_rdma_rsp { struct nvmet_req req; - bool allocated; u8 n_rdma; u32 flags; u32 invalidate_rkey; @@ -1197,13 +1196,18 @@ static void nvmet_rdma_free_dev(struct kref *ref) struct nvmet_rdma_device *ndev = container_of(ref, struct nvmet_rdma_device, ref); + nvmet_rdma_destroy_srqs(ndev); + ib_dealloc_pd(ndev->pd); + + /* + * Only delete from the device list once all RDMA resources are torn + * down because nvmet_rdma_remove_one waits for the release work only + * if the device still is on the list. + */ mutex_lock(&device_list_mutex); list_del(&ndev->entry); mutex_unlock(&device_list_mutex); - nvmet_rdma_destroy_srqs(ndev); - ib_dealloc_pd(ndev->pd); - kfree(ndev); } @@ -1694,7 +1698,7 @@ out_unlock: static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) { - bool disconnect = false; + bool disconnect = false, live = false; unsigned long flags; pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state); @@ -1713,6 +1717,7 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) } fallthrough; case NVMET_RDMA_Q_LIVE: + live = queue->state == NVMET_RDMA_Q_LIVE; queue->state = NVMET_RDMA_Q_DISCONNECTING; disconnect = true; break; @@ -1722,24 +1727,34 @@ static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) spin_unlock_irqrestore(&queue->state_lock, flags); if (disconnect) { - rdma_disconnect(queue->cm_id); + /* + * Over iWARP rdma_disconnect() sleeps until the CM has handled + * the ESTABLISHED event of this connection. The CM handles the + * events of all connections one at a time, and our handler + * takes nvmet_rdma_queue_mutex, which callers of this function + * hold. So a queue that has not seen the event is left to the + * release work: ib_drain_qp() moves the queue pair to the error + * state, which ends the connection. + */ + if (live) + rdma_disconnect(queue->cm_id); queue_work(nvmet_wq, &queue->release_work); } } static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) { - bool disconnect = false; - + /* + * Queue the release work with &nvmet_rdma_queue_mutex held, so that + * nvmet_rdma_remove_one() either finds the queue on the list or the + * work has already been queued. + */ mutex_lock(&nvmet_rdma_queue_mutex); if (!list_empty(&queue->queue_list)) { list_del_init(&queue->queue_list); - disconnect = true; + __nvmet_rdma_queue_disconnect(queue); } mutex_unlock(&nvmet_rdma_queue_mutex); - - if (disconnect) - __nvmet_rdma_queue_disconnect(queue); } static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, @@ -1766,10 +1781,13 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, * queue cm_id and/or a device bound listener cm_id (where in this * case queue will be null). * - * We registered an ib_client to handle device removal for queues, - * so we only need to handle the listening port cm_ids. In this case - * we nullify the priv to prevent double cm_id destruction and destroying - * the cm_id implicitly by returning a non-zero rc to the callout. + * The RDMA CM waits until each cm_id of the device is destroyed, so a queue + * has to go now. Our ib_client also removes the queues, but it cannot be + * relied on here: built in, it is called after the RDMA CM's client. + * + * For a listening port cm_id we nullify the priv to prevent double cm_id + * destruction and destroying the cm_id implicitly by returning a non-zero + * rc to the callout. */ static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, struct nvmet_rdma_queue *queue) @@ -1777,11 +1795,7 @@ static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, struct nvmet_rdma_port *port; if (queue) { - /* - * This is a queue cm_id. we have registered - * an ib_client to handle queues removal - * so don't interfere and just return. - */ + nvmet_rdma_queue_disconnect(queue); return 0; } diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 8092ce4914b6..5105317cc61e 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -102,6 +102,7 @@ MODULE_PARM_DESC(tls_handshake_timeout, #define NVMET_TCP_RECV_BUDGET 8 #define NVMET_TCP_SEND_BUDGET 8 #define NVMET_TCP_IO_WORK_BUDGET 64 +#define NVMET_TCP_ACCEPT_BUDGET 16 enum nvmet_tcp_send_state { NVMET_TCP_SEND_DATA_PDU, @@ -1302,6 +1303,11 @@ static int nvmet_tcp_try_recv_data(struct nvmet_tcp_queue *queue) int len, ret; while (msg_data_left(&cmd->recv_msg)) { + if (queue->tls_pskid) { + /* The last receive has used it up */ + cmd->recv_msg.msg_control = cmd->recv_cbuf; + cmd->recv_msg.msg_controllen = sizeof(cmd->recv_cbuf); + } len = sock_recvmsg(cmd->queue->sock, &cmd->recv_msg, cmd->recv_msg.msg_flags); if (len <= 0) @@ -2068,9 +2074,9 @@ static void nvmet_tcp_accept_work(struct work_struct *w) struct nvmet_tcp_port *port = container_of(w, struct nvmet_tcp_port, accept_work); struct socket *newsock; - int ret; + int ret, i; - while (true) { + for (i = 0; i < NVMET_TCP_ACCEPT_BUDGET; i++) { ret = kernel_accept(port->sock, &newsock, O_NONBLOCK); if (ret < 0) { if (ret != -EAGAIN) @@ -2079,6 +2085,9 @@ static void nvmet_tcp_accept_work(struct work_struct *w) } nvmet_tcp_alloc_queue(port, newsock); } + + /* There may be more. Come to an end for those who flush nvmet_wq. */ + queue_work(nvmet_wq, &port->accept_work); } static void nvmet_tcp_listen_data_ready(struct sock *sk) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 0c2010230632..215753171cef 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -678,7 +678,7 @@ enum { QUEUE_FLAG_HCTX_ACTIVE, /* at least one blk-mq hctx is active */ QUEUE_FLAG_SQ_SCHED, /* single queue style io dispatch */ QUEUE_FLAG_DISABLE_WBT_DEF, /* for sched to disable/enable wbt */ - QUEUE_FLAG_NO_ELV_SWITCH, /* can't switch elevator any more */ + QUEUE_FLAG_NO_ELV_SWITCH, /* a switch is running or disk is going away */ QUEUE_FLAG_QOS_ENABLED, /* qos is enabled */ QUEUE_FLAG_BIO_ISSUE_TIME, /* record bio->issue_time_ns */ QUEUE_FLAG_ZONED_QD1_WRITES, /* Limit zoned devices writes to QD=1 */ @@ -719,8 +719,6 @@ void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); ((q)->limits.features & BLK_FEAT_SKIP_TAGSET_QUIESCE) #define blk_queue_disable_wbt(q) \ test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags) -#define blk_queue_no_elv_switch(q) \ - test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags) #define blk_queue_zoned_qd1_writes(q) \ test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 1a8a465a5650..d4dcaeb83e2b 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -118,6 +119,41 @@ void cleanup_srcu_struct(struct srcu_struct *ssp); void synchronize_srcu(struct srcu_struct *ssp); void synchronize_srcu_atomic(struct srcu_struct *ssp); +/** + * list_splice_init_srcu - splice an SRCU-protected list into an existing list + * @list: the SRCU-protected list to splice + * @head: the place in the existing list to splice the first list into + * @ssp: the srcu_struct that readers of @list use + * + * Like list_splice_init_rcu(), but wait for the SRCU readers of @list + * instead of RCU ones, so that none of them can follow a next pointer + * onto @head. + * + * Note that this function blocks, and that the caller must prevent any + * other updates to @list. + */ +static inline void list_splice_init_srcu(struct list_head *list, + struct list_head *head, + struct srcu_struct *ssp) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *next = head->next; + + if (list_empty(list)) + return; + + INIT_LIST_HEAD_RCU(list); + synchronize_srcu(ssp); + ASSERT_EXCLUSIVE_ACCESS(*first); + ASSERT_EXCLUSIVE_ACCESS(*last); + + last->next = next; + rcu_assign_pointer(list_next_rcu(head), first); + first->prev = head; + next->prev = last; +} + #define SRCU_GET_STATE_COMPLETED 0x1 /** diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile index fc64b8f02833..1a50d4695c9f 100644 --- a/tools/testing/selftests/ublk/Makefile +++ b/tools/testing/selftests/ublk/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 CFLAGS += -O3 -Wl,-no-as-needed -Wall -I $(top_srcdir)/usr/include +CFLAGS += -D_FILE_OFFSET_BITS=64 ifneq ($(WERROR),0) CFLAGS += -Werror endif diff --git a/tools/testing/selftests/ublk/batch.c b/tools/testing/selftests/ublk/batch.c index dfc81ee17d07..625a9c32a80b 100644 --- a/tools/testing/selftests/ublk/batch.c +++ b/tools/testing/selftests/ublk/batch.c @@ -376,10 +376,10 @@ static int __ublk_batch_queue_prep_io_cmds(struct ublk_thread *t, struct ublk_qu if (ublk_queue_use_auto_zc(q)) elem->buf_index = ublk_batch_io_buf_idx(t, q, i); else if (!ublk_queue_no_buf(q)) - elem->buf_addr = (__u64)io->buf_addr; + elem->buf_addr = (__u64)(uintptr_t)io->buf_addr; } - sqe->addr = (__u64)buf; + sqe->addr = (__u64)(uintptr_t)buf; sqe->len = t->commit_buf_elem_size * nr_elem; ublk_init_batch_cmd(t, q->q_id, sqe, UBLK_U_IO_PREP_IO_CMDS, @@ -473,7 +473,7 @@ static void __ublk_batch_commit_io_cmds(struct ublk_thread *t, ublk_io_alloc_sqes(t, &sqe, 1); buf_idx = cb->buf_idx; - sqe->addr = (__u64)cb->elem; + sqe->addr = (__u64)(uintptr_t)cb->elem; sqe->len = nr_elem * t->commit_buf_elem_size; /* commit isn't per-queue command */ diff --git a/tools/testing/selftests/ublk/ctrl.c b/tools/testing/selftests/ublk/ctrl.c index 53d54799f909..5e0762cfc4ea 100644 --- a/tools/testing/selftests/ublk/ctrl.c +++ b/tools/testing/selftests/ublk/ctrl.c @@ -204,7 +204,7 @@ int ublk_ctrl_get_params(struct ublk_dev *dev, struct ublk_ctrl_cmd_data data = { .cmd_op = UBLK_U_CMD_GET_PARAMS, .flags = CTRL_CMD_HAS_BUF, - .addr = (__u64)params, + .addr = (__u64)(uintptr_t)params, .len = sizeof(*params), }; diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 5a4ff689c1f7..115f25351d30 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -148,7 +148,7 @@ static int ublk_ctrl_get_affinity(struct ublk_dev *ctrl_dev, cpu_set_t **ptr_buf for (i = 0; i < ctrl_dev->dev_info.nr_hw_queues; i++) { data.data[0] = i; data.len = sizeof(cpu_set_t); - data.addr = (__u64)&buf[i]; + data.addr = (__u64)(uintptr_t)&buf[i]; ret = __ublk_ctrl_cmd(ctrl_dev, &data); if (ret < 0) {