diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c index 179b76173255..461044dca367 100644 --- a/drivers/md/dm-cache-metadata.c +++ b/drivers/md/dm-cache-metadata.c @@ -766,6 +766,7 @@ static struct dm_cache_metadata *metadata_open(struct block_device *bdev, return ERR_PTR(-ENOMEM); } + INIT_LIST_HEAD(&cmd->list); cmd->version = metadata_version; refcount_set(&cmd->ref_count, 1); init_rwsem(&cmd->root_lock); @@ -1432,7 +1433,8 @@ static int __load_mappings(struct dm_cache_metadata *cmd, from_cblock(cmd->cache_blocks), &cmd->dirty_cursor); if (r) { - dm_array_cursor_end(&cmd->hint_cursor); + if (hints_valid) + dm_array_cursor_end(&cmd->hint_cursor); dm_array_cursor_end(&cmd->mapping_cursor); return r; } diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index d292a44c921e..850c4b651f09 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -2383,9 +2384,7 @@ static int create_cache_policy(struct cache *cache, struct cache_args *ca, static bool too_many_discard_blocks(sector_t discard_block_size, sector_t origin_size) { - (void) sector_div(origin_size, discard_block_size); - - return origin_size > MAX_DISCARD_BLOCKS; + return div64_u64(origin_size, discard_block_size) > MAX_DISCARD_BLOCKS; } static sector_t calculate_discard_block_size(sector_t cache_block_size, @@ -2422,6 +2421,7 @@ static int cache_create(struct cache_args *ca, struct cache **result) struct cache *cache; struct dm_target *ti = ca->ti; dm_block_t origin_blocks; + sector_t discard_block_size; struct dm_cache_metadata *cmd; bool may_format = ca->features.mode == CM_WRITE; @@ -2521,9 +2521,15 @@ static int cache_create(struct cache_args *ca, struct cache **result) } clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size)); - cache->discard_block_size = + discard_block_size = calculate_discard_block_size(cache->sectors_per_block, cache->origin_sectors); + if (discard_block_size > U32_MAX) { + *error = "Discard block size exceeds 32 bits"; + r = -EOVERFLOW; + goto bad; + } + cache->discard_block_size = discard_block_size; cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors, cache->discard_block_size)); cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks)); @@ -2544,6 +2550,7 @@ static int cache_create(struct cache_args *ca, struct cache **result) if (IS_ERR(cache->copier)) { *error = "could not create kcopyd client"; r = PTR_ERR(cache->copier); + cache->copier = NULL; goto bad; } diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index a7655bba7ce3..02038a9cd21d 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1982,8 +1982,9 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) /* crypt_convert should have filled the clone bio */ BUG_ON(io->ctx.iter_out.bi_size); - if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) || - test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) { + if (likely(!async) && + (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) || + test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) { dm_submit_bio_remap(io->base_bio, clone); return; } @@ -2847,8 +2848,11 @@ static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api) char *start, *end, *mac_alg = NULL; struct crypto_ahash *mac; - if (!strstarts(cipher_api, "authenc(")) + if (!strstarts(cipher_api, "authenc(")) { + if (test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags)) + return -EINVAL; return 0; + } start = strchr(cipher_api, '('); end = strchr(cipher_api, ','); @@ -3229,6 +3233,14 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) ti->error = "Cannot allocate encryption context"; return -ENOMEM; } + + ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL); + if (ret < 0) { + ti->error = "Cannot initialize page counter"; + kfree_sensitive(cc); + return ret; + } + cc->key_size = key_size; cc->sector_size = (1 << SECTOR_SHIFT); cc->sector_shift = 0; @@ -3240,10 +3252,6 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) crypt_calculate_pages_per_client(); spin_unlock(&dm_crypt_clients_lock); - ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL); - if (ret < 0) - goto bad; - /* Optional parameters need to be read before cipher constructor */ if (argc > 5) { ret = crypt_ctr_optional(ti, argc - 5, &argv[5]); diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 802b377613b0..6968e38888db 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2908,6 +2908,11 @@ static void integrity_commit(struct work_struct *w) spin_lock_irq(&ic->endio_wait.lock); flushes = bio_list_get(&ic->flush_bio_list); + if (unlikely(dm_integrity_failed(ic))) { + wake_up_locked(&ic->endio_wait); + spin_unlock_irq(&ic->endio_wait.lock); + goto release_flush_bios; + } if (unlikely(ic->mode != 'J')) { spin_unlock_irq(&ic->endio_wait.lock); dm_integrity_flush_buffers(ic, true); @@ -3888,6 +3893,24 @@ static bool superblock_changed(struct dm_integrity_c *ic) memcmp(ic->sb->salt, ic->sb_copy->salt, SALT_SIZE) != 0; } +/* + * The superblock is authenticated only if fix_hmac and journal_mac are used, + * so SB_FLAG_DIRTY_BITMAP may have been set by someone who doesn't know the + * key. Apply the policy that the constructor applies to SB_FLAG_RECALCULATING. + */ +static bool dm_integrity_may_recalc_dirty_bitmap(struct dm_integrity_c *ic) +{ + if (!ic->internal_hash) { + DMERR("Dirty bitmap found, but internal_hash is not used"); + return false; + } + if (dm_integrity_disable_recalculate(ic)) { + DMERR("Dirty bitmap found, but HMAC recalculation requires \"legacy_recalculate\""); + return false; + } + return true; +} + static void dm_integrity_resume(struct dm_target *ti) { struct dm_integrity_c *ic = ti->private; @@ -3977,8 +4000,10 @@ static void dm_integrity_resume(struct dm_target *ti) if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) || ic->reset_recalculate_flag) { - ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); - ic->sb->recalc_sector = cpu_to_le64(0); + if (dm_integrity_may_recalc_dirty_bitmap(ic)) { + ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); + ic->sb->recalc_sector = cpu_to_le64(0); + } } init_journal(ic, 0, ic->journal_sections, 0); replay_journal(ic); @@ -5409,7 +5434,6 @@ try_smaller_buffer: r = create_journal(ic, &ti->error); if (r) goto bad; - } if (ic->mode == 'B') { diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 8f945f0aeda5..4a7cb0926c17 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -1794,7 +1794,7 @@ static void retrieve_deps(struct dm_table *table, deps->count = count; count = 0; list_for_each_entry(dd, dm_table_get_devices(table), list) - deps->dev[count++] = huge_encode_dev(dd->dm_dev->bdev->bd_dev); + deps->dev[count++] = huge_encode_dev(dd->bdev->bd_dev); param->data_size = param->data_start + needed; } diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c index 3ca35aa4e057..48daa41728d6 100644 --- a/drivers/md/dm-log-userspace-base.c +++ b/drivers/md/dm-log-userspace-base.c @@ -294,9 +294,11 @@ static int userspace_ctr(struct dm_dirty_log *log, struct dm_target *ti, } r = dm_get_device(ti, devices_rdata, dm_table_get_mode(ti->table), &lc->log_dev); - if (r) + if (r) { DMERR("Failed to register %s with device-mapper", devices_rdata); + goto out; + } } if (lc->integrated_flush) { @@ -315,6 +317,8 @@ static int userspace_ctr(struct dm_dirty_log *log, struct dm_target *ti, out: kfree(devices_rdata); if (r) { + if (lc->log_dev) + dm_put_device(lc->ti, lc->log_dev); mempool_exit(&lc->flush_entry_pool); kfree(lc); kfree(ctr_str); diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index ae9318f5f21b..d2e94b96d2e4 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -797,7 +797,8 @@ static void queue_if_no_path_timeout_work(struct timer_list *t) static void enable_nopath_timeout(struct multipath *m) { unsigned long queue_if_no_path_timeout = - READ_ONCE(queue_if_no_path_timeout_secs) * HZ; + min_t(unsigned long, READ_ONCE(queue_if_no_path_timeout_secs), + MAX_JIFFY_OFFSET / HZ) * HZ; lockdep_assert_held(&m->lock); @@ -1085,8 +1086,9 @@ static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) if (hw_argc > 1) { char *p; - int i, j, len = 4; + int i, j, len; + len = snprintf(NULL, 0, "%d", hw_argc - 1) + 1; for (i = 0; i <= hw_argc - 2; i++) len += strlen(as->argv[i]) + 1; p = m->hw_handler_params = kzalloc(len, GFP_KERNEL); diff --git a/drivers/md/dm-pcache/backing_dev.c b/drivers/md/dm-pcache/backing_dev.c index 23002beb192a..84b4e76367fa 100644 --- a/drivers/md/dm-pcache/backing_dev.c +++ b/drivers/md/dm-pcache/backing_dev.c @@ -336,9 +336,9 @@ struct pcache_backing_dev_req *backing_dev_req_create(struct pcache_backing_dev return backing_req; } -void backing_dev_flush(struct pcache_backing_dev *backing_dev) +int backing_dev_flush(struct pcache_backing_dev *backing_dev) { - blkdev_issue_flush(backing_dev->dm_dev->bdev); + return blkdev_issue_flush(backing_dev->dm_dev->bdev); } int pcache_backing_init(void) diff --git a/drivers/md/dm-pcache/backing_dev.h b/drivers/md/dm-pcache/backing_dev.h index b371cba483b9..8422e5e76e29 100644 --- a/drivers/md/dm-pcache/backing_dev.h +++ b/drivers/md/dm-pcache/backing_dev.h @@ -120,7 +120,7 @@ struct pcache_backing_dev_req *backing_dev_req_alloc(struct pcache_backing_dev * struct pcache_backing_dev_req_opts *opts); void backing_dev_req_init(struct pcache_backing_dev_req *backing_req, struct pcache_backing_dev_req_opts *opts); -void backing_dev_flush(struct pcache_backing_dev *backing_dev); +int backing_dev_flush(struct pcache_backing_dev *backing_dev); int pcache_backing_init(void); void pcache_backing_exit(void); diff --git a/drivers/md/dm-pcache/cache_writeback.c b/drivers/md/dm-pcache/cache_writeback.c index c8a4c8110a58..e09560f172dd 100644 --- a/drivers/md/dm-pcache/cache_writeback.c +++ b/drivers/md/dm-pcache/cache_writeback.c @@ -18,12 +18,18 @@ static void writeback_ctx_end(struct pcache_cache *cache, int ret) return; if (!cache->writeback_ctx.ret) { - backing_dev_flush(cache->backing_dev); - - mutex_lock(&cache->dirty_tail_lock); - cache_pos_advance(&cache->dirty_tail, cache->writeback_ctx.advance); - cache_encode_dirty_tail(cache); - mutex_unlock(&cache->dirty_tail_lock); + ret = backing_dev_flush(cache->backing_dev); + if (ret) { + pcache_dev_err(CACHE_TO_PCACHE(cache), + "failed to flush backing device: %d", ret); + cache->writeback_ctx.ret = ret; + } else { + mutex_lock(&cache->dirty_tail_lock); + cache_pos_advance(&cache->dirty_tail, + cache->writeback_ctx.advance); + cache_encode_dirty_tail(cache); + mutex_unlock(&cache->dirty_tail_lock); + } } queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0); } @@ -76,8 +82,16 @@ static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cach void cache_writeback_exit(struct pcache_cache *cache) { + int ret; + cancel_delayed_work_sync(&cache->writeback_work); - backing_dev_flush(cache->backing_dev); + + ret = backing_dev_flush(cache->backing_dev); + if (ret) + pcache_dev_err(CACHE_TO_PCACHE(cache), + "failed to flush backing device during exit: %d", + ret); + cache_tree_exit(&cache->writeback_key_tree); } diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index ac019a0f5a92..e44d92462aff 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -2420,7 +2420,8 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) */ if (!test_bit(FirstUse, &r->flags) && (r->raid_disk >= 0)) { role = le32_to_cpu(sb2->array_position); - if (role < 0) + /* Bound @role before it indexes the failed_devices stack bitmap. */ + if (role < 0 || role >= rs->raid_disks) continue; if (role != r->raid_disk) { @@ -3333,7 +3334,7 @@ static void raid_dtr(struct dm_target *ti) rs->md.dm_gendisk = NULL; mddev_unlock(&rs->md); - if (work_pending(&rs->md.event_work)) + if (rs->md.event_work.func) flush_work(&rs->md.event_work); raid_set_free(rs); } diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 797c432594d8..6968e806eb84 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1358,8 +1358,10 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) ti->private = s; ti->num_flush_bios = num_flush_bios; - if (s->discard_zeroes_cow) + if (s->discard_zeroes_cow) { ti->num_discard_bios = (s->discard_passdown_origin ? 2 : 1); + ti->max_discard_granularity = true; + } ti->per_io_data_size = sizeof(struct dm_snap_tracked_chunk); /* Add snapshot to the list of snapshots for this origin */ @@ -1946,8 +1948,8 @@ static void zero_exception(struct dm_snapshot *s, struct dm_exception *e, static bool io_overlaps_chunk(struct dm_snapshot *s, struct bio *bio) { - return bio->bi_iter.bi_size == - (s->store->chunk_size << SECTOR_SHIFT); + return bio->bi_iter.bi_size == (s->store->chunk_size << SECTOR_SHIFT) && + !(bio->bi_iter.bi_sector & s->store->chunk_mask); } static int snapshot_map(struct dm_target *ti, struct bio *bio) @@ -2007,11 +2009,13 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio) e = dm_lookup_exception(&s->complete, chunk); if (e) { remap_exception(s, e, bio, chunk); - if (unlikely(bio_op(bio) == REQ_OP_DISCARD) && - io_overlaps_chunk(s, bio)) { + if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { dm_exception_table_unlock(&lock); up_read(&s->lock); - zero_exception(s, e, bio, chunk); + if (io_overlaps_chunk(s, bio)) + zero_exception(s, e, bio, chunk); + else + bio_endio(bio); r = DM_MAPIO_SUBMITTED; /* discard is not issued */ goto out; } @@ -2414,7 +2418,7 @@ static void snapshot_io_hints(struct dm_target *ti, struct queue_limits *limits) snap = snap_src; /* All discards are split on chunk_size boundary */ - limits->discard_granularity = snap->store->chunk_size; + limits->discard_granularity = snap->store->chunk_size << SECTOR_SHIFT; limits->max_hw_discard_sectors = snap->store->chunk_size; up_read(&_origins_lock); diff --git a/drivers/md/dm-sysfs.c b/drivers/md/dm-sysfs.c index 22bc70923a83..c9f81231df7b 100644 --- a/drivers/md/dm-sysfs.c +++ b/drivers/md/dm-sysfs.c @@ -128,9 +128,20 @@ static const struct kobj_type dm_ktype = { */ int dm_sysfs_init(struct mapped_device *md) { - return kobject_init_and_add(dm_kobject(md), &dm_ktype, - &disk_to_dev(dm_disk(md))->kobj, - "%s", "dm"); + struct kobject *kobj = dm_kobject(md); + int r; + + /* The kobject must be zeroed before calling kobject_init */ + memset(kobj, 0, sizeof(struct kobject)); + r = kobject_init_and_add(kobj, &dm_ktype, + &disk_to_dev(dm_disk(md))->kobj, + "%s", "dm"); + if (unlikely(r)) { + /* Weird semantics - we must call kobject_put on failure */ + dm_sysfs_exit(md); + return r; + } + return 0; } /* diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 2e8d0436386f..48f25ab6b7fa 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -165,14 +165,16 @@ int dm_table_create(struct dm_table **result, blk_mode_t mode, static void free_devices(struct list_head *devices, struct mapped_device *md) { - struct list_head *tmp, *next; + struct dm_dev_internal *dd, *dd_next; + struct dm_dev_ref *ref, *ref_next; - list_for_each_safe(tmp, next, devices) { - struct dm_dev_internal *dd = - list_entry(tmp, struct dm_dev_internal, list); - DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s", - dm_device_name(md), dd->dm_dev->name); - dm_put_table_device(md, dd->dm_dev); + list_for_each_entry_safe(dd, dd_next, devices, list) { + list_for_each_entry_safe(ref, ref_next, &dd->refs, list) { + DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s", + dm_device_name(md), ref->dm_dev->name); + dm_put_table_device(md, ref->dm_dev); + kfree(ref); + } kfree(dd); } } @@ -218,12 +220,27 @@ static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev) struct dm_dev_internal *dd; list_for_each_entry(dd, l, list) - if (dd->dm_dev->bdev->bd_dev == dev) + if (dd->bdev->bd_dev == dev) return dd; return NULL; } +/* + * See if the device is already open with a mode that covers the + * requested one. + */ +static struct dm_dev_ref *find_ref(struct dm_dev_internal *dd, blk_mode_t mode) +{ + struct dm_dev_ref *ref; + + list_for_each_entry(ref, &dd->refs, list) + if ((ref->dm_dev->mode & mode) == mode) + return ref; + + return NULL; +} + /* * If possible, this checks an area of a destination device is invalid. */ @@ -303,31 +320,6 @@ static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev, return 0; } -/* - * This upgrades the mode on an already open dm_dev, being - * careful to leave things as they were if we fail to reopen the - * device and not to touch the existing bdev field in case - * it is accessed concurrently. - */ -static int upgrade_mode(struct dm_dev_internal *dd, blk_mode_t new_mode, - struct mapped_device *md) -{ - int r; - struct dm_dev *old_dev, *new_dev; - - old_dev = dd->dm_dev; - - r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev, - dd->dm_dev->mode | new_mode, &new_dev); - if (r) - return r; - - dd->dm_dev = new_dev; - dm_put_table_device(md, old_dev); - - return 0; -} - /* * Note: the __ref annotation is because this function can call the __init * marked early_lookup_bdev when called during early boot code from dm-init.c. @@ -360,7 +352,9 @@ EXPORT_SYMBOL(dm_devt_from_path); /* * Add a device to the list, or just increment the usage count if - * it's already present. + * it's already present with a mode that covers the requested one. + * Otherwise open the device again with the requested mode, so the + * dm_devs returned earlier stay valid. */ int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode, struct dm_dev **result) @@ -368,6 +362,7 @@ int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode, int r; dev_t dev; struct dm_dev_internal *dd; + struct dm_dev_ref *ref; struct dm_table *t = ti->table; BUG_ON(!t); @@ -380,30 +375,46 @@ int dm_get_device(struct dm_target *ti, const char *path, blk_mode_t mode, return -EINVAL; dd = find_device(&t->devices, dev); - if (!dd) { + if (dd) { + ref = find_ref(dd, mode); + if (ref) { + refcount_inc(&ref->count); + goto out; + } + } else { dd = kmalloc_obj(*dd); if (!dd) return -ENOMEM; - - r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev); - if (r) { - kfree(dd); - return r; - } - - refcount_set(&dd->count, 1); - list_add(&dd->list, &t->devices); - goto out; - - } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) { - r = upgrade_mode(dd, mode, t->md); - if (r) - return r; + INIT_LIST_HEAD(&dd->refs); } - refcount_inc(&dd->count); + + ref = kmalloc_obj(*ref); + if (!ref) { + r = -ENOMEM; + goto bad; + } + + r = dm_get_table_device(t->md, dev, mode, &ref->dm_dev); + if (r) { + kfree(ref); + goto bad; + } + refcount_set(&ref->count, 1); + + /* A device in the list always has at least one ref */ + if (list_empty(&dd->refs)) { + dd->bdev = ref->dm_dev->bdev; + list_add(&dd->list, &t->devices); + } + list_add_tail(&ref->list, &dd->refs); out: - *result = dd->dm_dev; + *result = ref->dm_dev; return 0; + +bad: + if (list_empty(&dd->refs)) + kfree(dd); + return r; } EXPORT_SYMBOL(dm_get_device); @@ -453,23 +464,28 @@ static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev, */ void dm_put_device(struct dm_target *ti, struct dm_dev *d) { - int found = 0; struct list_head *devices = &ti->table->devices; struct dm_dev_internal *dd; + struct dm_dev_ref *ref; - list_for_each_entry(dd, devices, list) { - if (dd->dm_dev == d) { - found = 1; - break; - } - } - if (!found) { - DMERR("%s: device %s not in table devices list", - dm_device_name(ti->table->md), d->name); + list_for_each_entry(dd, devices, list) + list_for_each_entry(ref, &dd->refs, list) + if (ref->dm_dev == d) + goto found; + + DMERR("%s: device %s not in table devices list", + dm_device_name(ti->table->md), d->name); + return; + +found: + if (!refcount_dec_and_test(&ref->count)) return; - } - if (refcount_dec_and_test(&dd->count)) { - dm_put_table_device(ti->table->md, d); + + dm_put_table_device(ti->table->md, d); + list_del(&ref->list); + kfree(ref); + + if (list_empty(&dd->refs)) { list_del(&dd->list); kfree(dd); } diff --git a/drivers/md/dm-thin-metadata.c b/drivers/md/dm-thin-metadata.c index e60e1326376a..03df0b1aab78 100644 --- a/drivers/md/dm-thin-metadata.c +++ b/drivers/md/dm-thin-metadata.c @@ -1225,10 +1225,13 @@ static int __create_snap(struct dm_pool_metadata *pmd, __close_device(td); if (r) - goto bad; + goto bad2; return 0; +bad2: + list_del(&td->list); + kfree(td); bad: dm_btree_remove(&pmd->tl_info, pmd->root, &key, &pmd->root); dm_btree_remove(&pmd->details_info, pmd->details_root, diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 358ed77ffb2b..0b65c1b09f47 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -1703,6 +1703,7 @@ static void break_up_discard_bio(struct thin_c *tc, dm_block_t begin, dm_block_t (void) build_key(tc->td, PHYSICAL, data_begin, data_begin + len, &data_key); if (bio_detain(tc->pool, &data_key, NULL, &data_cell)) { /* contention, we'll give up with this range */ + virt_begin += len; data_begin += len; continue; } @@ -2542,7 +2543,9 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) struct pool_c *pt = pool->ti->private; bool needs_check = dm_pool_metadata_needs_check(pool->pmd); enum pool_mode old_mode = get_pool_mode(pool); - unsigned long no_space_timeout = READ_ONCE(no_space_timeout_secs) * HZ; + unsigned long no_space_timeout = + min_t(unsigned long, READ_ONCE(no_space_timeout_secs), + MAX_JIFFY_OFFSET / HZ) * HZ; /* * Never allow the pool to transition to PM_WRITE mode if user @@ -3146,7 +3149,7 @@ static int parse_pool_features(struct dm_arg_set *as, struct pool_features *pf, const char *arg_name; static const struct dm_arg _args[] = { - {0, 4, "Invalid number of pool feature arguments"}, + {0, 5, "Invalid number of pool feature arguments"}, }; /* diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 60cf906c0361..306ed1f6c118 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -170,6 +170,15 @@ static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); } +/* + * Skip verity work in response to I/O error when system is shutting down. + */ +static inline bool verity_is_system_shutting_down(void) +{ + return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF + || system_state == SYSTEM_RESTART; +} + /* * Handle verification errors. */ @@ -219,6 +228,15 @@ out: if (v->mode == DM_VERITY_MODE_LOGGING) return 0; + /* + * Devices may already be quiesced (e.g. UFS set offline by its + * shutdown hook), so a failure here does not imply corruption. Fail + * the I/O but do not restart or panic, which would also override the + * reboot reason of the shutdown already in progress. + */ + if (verity_is_system_shutting_down()) + return 1; + if (v->mode == DM_VERITY_MODE_RESTART) kernel_restart("dm-verity device corrupted"); @@ -593,15 +611,6 @@ error: return r; } -/* - * Skip verity work in response to I/O error when system is shutting down. - */ -static inline bool verity_is_system_shutting_down(void) -{ - return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF - || system_state == SYSTEM_RESTART; -} - static void restart_io_error(struct work_struct *w) { kernel_restart("dm-verity device has I/O error"); diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 35916c6e490e..9ddbb67e97e5 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -33,6 +33,8 @@ static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t, nr_zones - args->zone_idx); if (ret < 0) return ret; + if (!ret) + return -EIO; } while (args->zone_idx < nr_zones && args->next_sector < get_capacity(md->disk)); diff --git a/drivers/md/dm-zoned-reclaim.c b/drivers/md/dm-zoned-reclaim.c index c041413c729e..152cb8788114 100644 --- a/drivers/md/dm-zoned-reclaim.c +++ b/drivers/md/dm-zoned-reclaim.c @@ -288,7 +288,13 @@ static int dmz_reclaim_rnd_data(struct dmz_reclaim *zrc, struct dm_zone *dzone) again: szone = dmz_alloc_zone(zmd, zrc->dev_idx, alloc_flags | DMZ_ALLOC_RECLAIM); - if (!szone && alloc_flags == DMZ_ALLOC_SEQ && dmz_nr_cache_zones(zmd)) { + /* + * If no free sequential zone is available, a cache zone can still be + * moved into a free random zone of the zoned device, which frees cache + * space. Do not do this for a random zone: moving it into another + * random zone frees nothing and, on an idle target, repeats forever. + */ + if (!szone && alloc_flags == DMZ_ALLOC_SEQ && dmz_is_cache(dzone)) { alloc_flags = DMZ_ALLOC_RND; goto again; } @@ -537,6 +543,16 @@ static void dmz_reclaim_work(struct work_struct *work) dmz_nr_rnd_zones(zmd, zrc->dev_idx)); ret = dmz_do_reclaim(zrc); + if (ret == -ENOSPC || ret == -EBUSY) { + /* + * No zone to copy into (-ENOSPC) or no zone to reclaim + * (-EBUSY): retrying at once cannot make progress and would + * spin. Poll again after the idle period instead, without + * postponing work that an allocation has already queued. + */ + queue_delayed_work(zrc->wq, &zrc->work, DMZ_IDLE_PERIOD); + return; + } if (ret && ret != -EINTR) { if (!dmz_check_dev(zmd)) return; diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 1881d86111fb..38eaae8ac9b4 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -983,7 +983,7 @@ static void __dm_io_complete(struct dm_io *io, bool first_stage) if (requeued) return; - if (unlikely(requeue_flush_with_data)) { + if (unlikely(requeue_flush_with_data) && !io_error) { /* * Preflush done for flush with data, reissue * without REQ_PREFLUSH. @@ -991,7 +991,7 @@ static void __dm_io_complete(struct dm_io *io, bool first_stage) bio->bi_opf &= ~REQ_PREFLUSH; queue_io(md, bio); } else { - /* done with normal IO or empty flush */ + /* done with normal IO, empty flush or failed preflush */ if (io_error) bio->bi_status = io_error; bio_endio(bio); @@ -1588,7 +1588,7 @@ static void __send_empty_flush(struct clone_info *ci) */ clone = alloc_tio(ci, NULL, 0, &len, GFP_NOIO); atomic_add(1, &ci->io->io_count); - bio_set_dev(clone, dd->dm_dev->bdev); + bio_set_dev(clone, dd->bdev); clone->bi_end_io = clone_endio; dm_submit_bio_remap(clone, NULL); } @@ -1929,8 +1929,13 @@ static blk_status_t __send_zone_reset_all(struct clone_info *ci) break; } - /* Release the reference that alloc_io() took for submission. */ - atomic_sub(1, &ci->io->io_count); + /* + * Release the reference that alloc_io() took for submission. On error, + * that reference was not consumed: dm_split_and_process_bio() releases + * it on the error path, so do not release it here a second time. + */ + if (sts == BLK_STS_OK) + atomic_dec(&ci->io->io_count); return sts; } @@ -2606,13 +2611,17 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) WARN_ON_ONCE(type == DM_TYPE_NONE); + /* A failed setup may have left the queue initialized or the disk dead. */ + if (md->tag_set || test_bit(GD_DEAD, &md->disk->state)) + return -ENODEV; + if (type == DM_TYPE_REQUEST_BASED) { - md->disk->fops = &dm_rq_blk_dops; r = dm_mq_init_request_queue(md, t); if (r) { DMERR("Cannot initialize queue for request-based dm mapped device"); return r; } + md->disk->fops = &dm_rq_blk_dops; } r = dm_calculate_queue_limits(t, &limits); diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 7a795979ec72..34f58060243f 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -37,10 +37,26 @@ /* * List of devices that a metadevice uses and should open/close. + * There is one dm_dev_internal per device. It holds one dm_dev_ref + * for each mode that the targets opened the device with. */ struct dm_dev_internal { struct list_head list; - refcount_t count; + struct block_device *bdev; + struct list_head refs; +}; + +/* + * One open of a device with one mode, shared by the targets of a table. + * dm_get_device() reuses a dm_dev_ref whose mode covers the requested + * mode, and adds a new one only when none does. So the dm_dev_refs of + * a device have different modes, and a table holds each table_device + * at most once. table_device->count is still the number of tables that + * use the table_device. + */ +struct dm_dev_ref { + struct list_head list; + refcount_t count; /* dm_get_device() calls that returned dm_dev */ struct dm_dev *dm_dev; }; diff --git a/drivers/md/persistent-data/dm-btree-remove.c b/drivers/md/persistent-data/dm-btree-remove.c index 2357ff9b53f3..18f53c672202 100644 --- a/drivers/md/persistent-data/dm-btree-remove.c +++ b/drivers/md/persistent-data/dm-btree-remove.c @@ -253,7 +253,9 @@ static int __rebalance2(struct dm_btree_info *info, struct btree_node *parent, /* * Merge */ - node_copy(left, right, -nr_right); + ret = node_copy(left, right, -nr_right); + if (unlikely(ret)) + return ret; left->header.nr_entries = cpu_to_le32(nr_left + nr_right); delete_at(parent, r->index); @@ -315,13 +317,16 @@ static int delete_center_node(struct dm_btree_info *info, struct btree_node *par { uint32_t max_entries = le32_to_cpu(left->header.max_entries); unsigned int shift = min(max_entries - nr_left, nr_center); + int ret; if (nr_left + shift > max_entries) { DMERR("node shift out of bounds"); return -EINVAL; } - node_copy(left, center, -shift); + ret = node_copy(left, center, -shift); + if (unlikely(ret)) + return ret; left->header.nr_entries = cpu_to_le32(nr_left + shift); if (shift != nr_center) { @@ -333,7 +338,9 @@ static int delete_center_node(struct dm_btree_info *info, struct btree_node *par } node_shift(right, shift); - node_copy(center, right, shift); + ret = node_copy(center, right, shift); + if (unlikely(ret)) + return ret; right->header.nr_entries = cpu_to_le32(nr_right + shift); } *key_ptr(parent, r->index) = right->keys[0]; diff --git a/drivers/md/persistent-data/dm-btree.c b/drivers/md/persistent-data/dm-btree.c index dd02eee4a23c..abfbbab8c7c6 100644 --- a/drivers/md/persistent-data/dm-btree.c +++ b/drivers/md/persistent-data/dm-btree.c @@ -1582,7 +1582,13 @@ int dm_btree_cursor_begin(struct dm_btree_info *info, dm_block_t root, if (r) return r; - return find_leaf(c); + r = find_leaf(c); + if (unlikely(r)) { + dm_btree_cursor_end(c); + return r; + } + + return 0; } EXPORT_SYMBOL_GPL(dm_btree_cursor_begin); diff --git a/drivers/md/persistent-data/dm-space-map-common.c b/drivers/md/persistent-data/dm-space-map-common.c index 22a551c407da..833aefb0ab99 100644 --- a/drivers/md/persistent-data/dm-space-map-common.c +++ b/drivers/md/persistent-data/dm-space-map-common.c @@ -1190,17 +1190,20 @@ static dm_block_t disk_ll_max_entries(struct ll_disk *ll) static int disk_ll_commit(struct ll_disk *ll) { - int r = 0; + int r; unsigned int i; for (i = 0; i < IE_CACHE_SIZE; i++) { struct ie_cache *iec = ll->ie_cache + i; - if (iec->valid && iec->dirty) + if (iec->valid && iec->dirty) { r = ie_cache_writeback(ll, iec); + if (unlikely(r)) + return r; + } } - return r; + return 0; } int sm_ll_new_disk(struct ll_disk *ll, struct dm_transaction_manager *tm)