1792 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2012 Red Hat, Inc.4 *5 * Author: Mikulas Patocka <mpatocka@redhat.com>6 *7 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors8 *9 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set10 * default prefetch value. Data are read in "prefetch_cluster" chunks from the11 * hash device. Setting this greatly improves performance when data and hash12 * are on the same disk on different partitions on devices with poor random13 * access behavior.14 */15 16#include "dm-verity.h"17#include "dm-verity-fec.h"18#include "dm-verity-verify-sig.h"19#include "dm-audit.h"20#include <linux/module.h>21#include <linux/reboot.h>22#include <linux/scatterlist.h>23#include <linux/string.h>24#include <linux/jump_label.h>25#include <linux/security.h>26 27#define DM_MSG_PREFIX "verity"28 29#define DM_VERITY_ENV_LENGTH 4230#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"31 32#define DM_VERITY_DEFAULT_PREFETCH_SIZE 26214433 34#define DM_VERITY_MAX_CORRUPTED_ERRS 10035 36#define DM_VERITY_OPT_LOGGING "ignore_corruption"37#define DM_VERITY_OPT_RESTART "restart_on_corruption"38#define DM_VERITY_OPT_PANIC "panic_on_corruption"39#define DM_VERITY_OPT_ERROR_RESTART "restart_on_error"40#define DM_VERITY_OPT_ERROR_PANIC "panic_on_error"41#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"42#define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"43#define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet"44 45#define DM_VERITY_OPTS_MAX (5 + DM_VERITY_OPTS_FEC + \46 DM_VERITY_ROOT_HASH_VERIFICATION_OPTS)47 48static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;49 50module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);51 52static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);53 54/* Is at least one dm-verity instance using ahash_tfm instead of shash_tfm? */55static DEFINE_STATIC_KEY_FALSE(ahash_enabled);56 57struct dm_verity_prefetch_work {58 struct work_struct work;59 struct dm_verity *v;60 unsigned short ioprio;61 sector_t block;62 unsigned int n_blocks;63};64 65/*66 * Auxiliary structure appended to each dm-bufio buffer. If the value67 * hash_verified is nonzero, hash of the block has been verified.68 *69 * The variable hash_verified is set to 0 when allocating the buffer, then70 * it can be changed to 1 and it is never reset to 0 again.71 *72 * There is no lock around this value, a race condition can at worst cause73 * that multiple processes verify the hash of the same buffer simultaneously74 * and write 1 to hash_verified simultaneously.75 * This condition is harmless, so we don't need locking.76 */77struct buffer_aux {78 int hash_verified;79};80 81/*82 * Initialize struct buffer_aux for a freshly created buffer.83 */84static void dm_bufio_alloc_callback(struct dm_buffer *buf)85{86 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);87 88 aux->hash_verified = 0;89}90 91/*92 * Translate input sector number to the sector number on the target device.93 */94static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)95{96 return v->data_start + dm_target_offset(v->ti, bi_sector);97}98 99/*100 * Return hash position of a specified block at a specified tree level101 * (0 is the lowest level).102 * The lowest "hash_per_block_bits"-bits of the result denote hash position103 * inside a hash block. The remaining bits denote location of the hash block.104 */105static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,106 int level)107{108 return block >> (level * v->hash_per_block_bits);109}110 111static int verity_ahash_update(struct dm_verity *v, struct ahash_request *req,112 const u8 *data, size_t len,113 struct crypto_wait *wait)114{115 struct scatterlist sg;116 117 if (likely(!is_vmalloc_addr(data))) {118 sg_init_one(&sg, data, len);119 ahash_request_set_crypt(req, &sg, NULL, len);120 return crypto_wait_req(crypto_ahash_update(req), wait);121 }122 123 do {124 int r;125 size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data));126 127 flush_kernel_vmap_range((void *)data, this_step);128 sg_init_table(&sg, 1);129 sg_set_page(&sg, vmalloc_to_page(data), this_step, offset_in_page(data));130 ahash_request_set_crypt(req, &sg, NULL, this_step);131 r = crypto_wait_req(crypto_ahash_update(req), wait);132 if (unlikely(r))133 return r;134 data += this_step;135 len -= this_step;136 } while (len);137 138 return 0;139}140 141/*142 * Wrapper for crypto_ahash_init, which handles verity salting.143 */144static int verity_ahash_init(struct dm_verity *v, struct ahash_request *req,145 struct crypto_wait *wait, bool may_sleep)146{147 int r;148 149 ahash_request_set_tfm(req, v->ahash_tfm);150 ahash_request_set_callback(req,151 may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0,152 crypto_req_done, (void *)wait);153 crypto_init_wait(wait);154 155 r = crypto_wait_req(crypto_ahash_init(req), wait);156 157 if (unlikely(r < 0)) {158 if (r != -ENOMEM)159 DMERR("crypto_ahash_init failed: %d", r);160 return r;161 }162 163 if (likely(v->salt_size && (v->version >= 1)))164 r = verity_ahash_update(v, req, v->salt, v->salt_size, wait);165 166 return r;167}168 169static int verity_ahash_final(struct dm_verity *v, struct ahash_request *req,170 u8 *digest, struct crypto_wait *wait)171{172 int r;173 174 if (unlikely(v->salt_size && (!v->version))) {175 r = verity_ahash_update(v, req, v->salt, v->salt_size, wait);176 177 if (r < 0) {178 DMERR("%s failed updating salt: %d", __func__, r);179 goto out;180 }181 }182 183 ahash_request_set_crypt(req, NULL, digest, 0);184 r = crypto_wait_req(crypto_ahash_final(req), wait);185out:186 return r;187}188 189int verity_hash(struct dm_verity *v, struct dm_verity_io *io,190 const u8 *data, size_t len, u8 *digest, bool may_sleep)191{192 int r;193 194 if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) {195 struct ahash_request *req = verity_io_hash_req(v, io);196 struct crypto_wait wait;197 198 r = verity_ahash_init(v, req, &wait, may_sleep) ?:199 verity_ahash_update(v, req, data, len, &wait) ?:200 verity_ahash_final(v, req, digest, &wait);201 } else {202 struct shash_desc *desc = verity_io_hash_req(v, io);203 204 desc->tfm = v->shash_tfm;205 r = crypto_shash_import(desc, v->initial_hashstate) ?:206 crypto_shash_finup(desc, data, len, digest);207 }208 if (unlikely(r))209 DMERR("Error hashing block: %d", r);210 return r;211}212 213static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,214 sector_t *hash_block, unsigned int *offset)215{216 sector_t position = verity_position_at_level(v, block, level);217 unsigned int idx;218 219 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);220 221 if (!offset)222 return;223 224 idx = position & ((1 << v->hash_per_block_bits) - 1);225 if (!v->version)226 *offset = idx * v->digest_size;227 else228 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);229}230 231/*232 * Handle verification errors.233 */234static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,235 unsigned long long block)236{237 char verity_env[DM_VERITY_ENV_LENGTH];238 char *envp[] = { verity_env, NULL };239 const char *type_str = "";240 struct mapped_device *md = dm_table_get_md(v->ti->table);241 242 /* Corruption should be visible in device status in all modes */243 v->hash_failed = true;244 245 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)246 goto out;247 248 v->corrupted_errs++;249 250 switch (type) {251 case DM_VERITY_BLOCK_TYPE_DATA:252 type_str = "data";253 break;254 case DM_VERITY_BLOCK_TYPE_METADATA:255 type_str = "metadata";256 break;257 default:258 BUG();259 }260 261 DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name,262 type_str, block);263 264 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) {265 DMERR("%s: reached maximum errors", v->data_dev->name);266 dm_audit_log_target(DM_MSG_PREFIX, "max-corrupted-errors", v->ti, 0);267 }268 269 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu",270 DM_VERITY_ENV_VAR_NAME, type, block);271 272 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp);273 274out:275 if (v->mode == DM_VERITY_MODE_LOGGING)276 return 0;277 278 if (v->mode == DM_VERITY_MODE_RESTART)279 kernel_restart("dm-verity device corrupted");280 281 if (v->mode == DM_VERITY_MODE_PANIC)282 panic("dm-verity device corrupted");283 284 return 1;285}286 287/*288 * Verify hash of a metadata block pertaining to the specified data block289 * ("block" argument) at a specified level ("level" argument).290 *291 * On successful return, verity_io_want_digest(v, io) contains the hash value292 * for a lower tree level or for the data block (if we're at the lowest level).293 *294 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.295 * If "skip_unverified" is false, unverified buffer is hashed and verified296 * against current value of verity_io_want_digest(v, io).297 */298static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,299 sector_t block, int level, bool skip_unverified,300 u8 *want_digest)301{302 struct dm_buffer *buf;303 struct buffer_aux *aux;304 u8 *data;305 int r;306 sector_t hash_block;307 unsigned int offset;308 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);309 310 verity_hash_at_level(v, block, level, &hash_block, &offset);311 312 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {313 data = dm_bufio_get(v->bufio, hash_block, &buf);314 if (data == NULL) {315 /*316 * In tasklet and the hash was not in the bufio cache.317 * Return early and resume execution from a work-queue318 * to read the hash from disk.319 */320 return -EAGAIN;321 }322 } else {323 data = dm_bufio_read_with_ioprio(v->bufio, hash_block,324 &buf, bio_prio(bio));325 }326 327 if (IS_ERR(data))328 return PTR_ERR(data);329 330 aux = dm_bufio_get_aux_data(buf);331 332 if (!aux->hash_verified) {333 if (skip_unverified) {334 r = 1;335 goto release_ret_r;336 }337 338 r = verity_hash(v, io, data, 1 << v->hash_dev_block_bits,339 verity_io_real_digest(v, io), !io->in_bh);340 if (unlikely(r < 0))341 goto release_ret_r;342 343 if (likely(memcmp(verity_io_real_digest(v, io), want_digest,344 v->digest_size) == 0))345 aux->hash_verified = 1;346 else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {347 /*348 * Error handling code (FEC included) cannot be run in a349 * tasklet since it may sleep, so fallback to work-queue.350 */351 r = -EAGAIN;352 goto release_ret_r;353 } else if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_METADATA,354 hash_block, data) == 0)355 aux->hash_verified = 1;356 else if (verity_handle_err(v,357 DM_VERITY_BLOCK_TYPE_METADATA,358 hash_block)) {359 struct bio *bio;360 io->had_mismatch = true;361 bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);362 dm_audit_log_bio(DM_MSG_PREFIX, "verify-metadata", bio,363 block, 0);364 r = -EIO;365 goto release_ret_r;366 }367 }368 369 data += offset;370 memcpy(want_digest, data, v->digest_size);371 r = 0;372 373release_ret_r:374 dm_bufio_release(buf);375 return r;376}377 378/*379 * Find a hash for a given block, write it to digest and verify the integrity380 * of the hash tree if necessary.381 */382int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,383 sector_t block, u8 *digest, bool *is_zero)384{385 int r = 0, i;386 387 if (likely(v->levels)) {388 /*389 * First, we try to get the requested hash for390 * the current block. If the hash block itself is391 * verified, zero is returned. If it isn't, this392 * function returns 1 and we fall back to whole393 * chain verification.394 */395 r = verity_verify_level(v, io, block, 0, true, digest);396 if (likely(r <= 0))397 goto out;398 }399 400 memcpy(digest, v->root_digest, v->digest_size);401 402 for (i = v->levels - 1; i >= 0; i--) {403 r = verity_verify_level(v, io, block, i, false, digest);404 if (unlikely(r))405 goto out;406 }407out:408 if (!r && v->zero_digest)409 *is_zero = !memcmp(v->zero_digest, digest, v->digest_size);410 else411 *is_zero = false;412 413 return r;414}415 416static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,417 sector_t cur_block, u8 *dest)418{419 struct page *page;420 void *buffer;421 int r;422 struct dm_io_request io_req;423 struct dm_io_region io_loc;424 425 page = mempool_alloc(&v->recheck_pool, GFP_NOIO);426 buffer = page_to_virt(page);427 428 io_req.bi_opf = REQ_OP_READ;429 io_req.mem.type = DM_IO_KMEM;430 io_req.mem.ptr.addr = buffer;431 io_req.notify.fn = NULL;432 io_req.client = v->io;433 io_loc.bdev = v->data_dev->bdev;434 io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);435 io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);436 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);437 if (unlikely(r))438 goto free_ret;439 440 r = verity_hash(v, io, buffer, 1 << v->data_dev_block_bits,441 verity_io_real_digest(v, io), true);442 if (unlikely(r))443 goto free_ret;444 445 if (memcmp(verity_io_real_digest(v, io),446 verity_io_want_digest(v, io), v->digest_size)) {447 r = -EIO;448 goto free_ret;449 }450 451 memcpy(dest, buffer, 1 << v->data_dev_block_bits);452 r = 0;453free_ret:454 mempool_free(page, &v->recheck_pool);455 456 return r;457}458 459static int verity_handle_data_hash_mismatch(struct dm_verity *v,460 struct dm_verity_io *io,461 struct bio *bio, sector_t blkno,462 u8 *data)463{464 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {465 /*466 * Error handling code (FEC included) cannot be run in the467 * BH workqueue, so fallback to a standard workqueue.468 */469 return -EAGAIN;470 }471 if (verity_recheck(v, io, blkno, data) == 0) {472 if (v->validated_blocks)473 set_bit(blkno, v->validated_blocks);474 return 0;475 }476#if defined(CONFIG_DM_VERITY_FEC)477 if (verity_fec_decode(v, io, DM_VERITY_BLOCK_TYPE_DATA, blkno,478 data) == 0)479 return 0;480#endif481 if (bio->bi_status)482 return -EIO; /* Error correction failed; Just return error */483 484 if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA, blkno)) {485 io->had_mismatch = true;486 dm_audit_log_bio(DM_MSG_PREFIX, "verify-data", bio, blkno, 0);487 return -EIO;488 }489 return 0;490}491 492/*493 * Verify one "dm_verity_io" structure.494 */495static int verity_verify_io(struct dm_verity_io *io)496{497 struct dm_verity *v = io->v;498 const unsigned int block_size = 1 << v->data_dev_block_bits;499 struct bvec_iter iter_copy;500 struct bvec_iter *iter;501 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);502 unsigned int b;503 504 if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {505 /*506 * Copy the iterator in case we need to restart507 * verification in a work-queue.508 */509 iter_copy = io->iter;510 iter = &iter_copy;511 } else512 iter = &io->iter;513 514 for (b = 0; b < io->n_blocks;515 b++, bio_advance_iter(bio, iter, block_size)) {516 int r;517 sector_t cur_block = io->block + b;518 bool is_zero;519 struct bio_vec bv;520 void *data;521 522 if (v->validated_blocks && bio->bi_status == BLK_STS_OK &&523 likely(test_bit(cur_block, v->validated_blocks)))524 continue;525 526 r = verity_hash_for_block(v, io, cur_block,527 verity_io_want_digest(v, io),528 &is_zero);529 if (unlikely(r < 0))530 return r;531 532 bv = bio_iter_iovec(bio, *iter);533 if (unlikely(bv.bv_len < block_size)) {534 /*535 * Data block spans pages. This should not happen,536 * since dm-verity sets dma_alignment to the data block537 * size minus 1, and dm-verity also doesn't allow the538 * data block size to be greater than PAGE_SIZE.539 */540 DMERR_LIMIT("unaligned io (data block spans pages)");541 return -EIO;542 }543 544 data = bvec_kmap_local(&bv);545 546 if (is_zero) {547 /*548 * If we expect a zero block, don't validate, just549 * return zeros.550 */551 memset(data, 0, block_size);552 kunmap_local(data);553 continue;554 }555 556 r = verity_hash(v, io, data, block_size,557 verity_io_real_digest(v, io), !io->in_bh);558 if (unlikely(r < 0)) {559 kunmap_local(data);560 return r;561 }562 563 if (likely(memcmp(verity_io_real_digest(v, io),564 verity_io_want_digest(v, io), v->digest_size) == 0)) {565 if (v->validated_blocks)566 set_bit(cur_block, v->validated_blocks);567 kunmap_local(data);568 continue;569 }570 r = verity_handle_data_hash_mismatch(v, io, bio, cur_block,571 data);572 kunmap_local(data);573 if (unlikely(r))574 return r;575 }576 577 return 0;578}579 580/*581 * Skip verity work in response to I/O error when system is shutting down.582 */583static inline bool verity_is_system_shutting_down(void)584{585 return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF586 || system_state == SYSTEM_RESTART;587}588 589static void restart_io_error(struct work_struct *w)590{591 kernel_restart("dm-verity device has I/O error");592}593 594/*595 * End one "io" structure with a given error.596 */597static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)598{599 struct dm_verity *v = io->v;600 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);601 602 bio->bi_end_io = io->orig_bi_end_io;603 bio->bi_status = status;604 605 if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)606 verity_fec_finish_io(io);607 608 if (unlikely(status != BLK_STS_OK) &&609 unlikely(!(bio->bi_opf & REQ_RAHEAD)) &&610 !io->had_mismatch &&611 !verity_is_system_shutting_down()) {612 if (v->error_mode == DM_VERITY_MODE_PANIC) {613 panic("dm-verity device has I/O error");614 }615 if (v->error_mode == DM_VERITY_MODE_RESTART) {616 static DECLARE_WORK(restart_work, restart_io_error);617 queue_work(v->verify_wq, &restart_work);618 /*619 * We deliberately don't call bio_endio here, because620 * the machine will be restarted anyway.621 */622 return;623 }624 }625 626 bio_endio(bio);627}628 629static void verity_work(struct work_struct *w)630{631 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);632 633 io->in_bh = false;634 635 verity_finish_io(io, errno_to_blk_status(verity_verify_io(io)));636}637 638static void verity_bh_work(struct work_struct *w)639{640 struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);641 int err;642 643 io->in_bh = true;644 err = verity_verify_io(io);645 if (err == -EAGAIN || err == -ENOMEM) {646 /* fallback to retrying with work-queue */647 INIT_WORK(&io->work, verity_work);648 queue_work(io->v->verify_wq, &io->work);649 return;650 }651 652 verity_finish_io(io, errno_to_blk_status(err));653}654 655static void verity_end_io(struct bio *bio)656{657 struct dm_verity_io *io = bio->bi_private;658 659 if (bio->bi_status &&660 (!verity_fec_is_enabled(io->v) ||661 verity_is_system_shutting_down() ||662 (bio->bi_opf & REQ_RAHEAD))) {663 verity_finish_io(io, bio->bi_status);664 return;665 }666 667 if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq) {668 INIT_WORK(&io->bh_work, verity_bh_work);669 queue_work(system_bh_wq, &io->bh_work);670 } else {671 INIT_WORK(&io->work, verity_work);672 queue_work(io->v->verify_wq, &io->work);673 }674}675 676/*677 * Prefetch buffers for the specified io.678 * The root buffer is not prefetched, it is assumed that it will be cached679 * all the time.680 */681static void verity_prefetch_io(struct work_struct *work)682{683 struct dm_verity_prefetch_work *pw =684 container_of(work, struct dm_verity_prefetch_work, work);685 struct dm_verity *v = pw->v;686 int i;687 688 for (i = v->levels - 2; i >= 0; i--) {689 sector_t hash_block_start;690 sector_t hash_block_end;691 692 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);693 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);694 695 if (!i) {696 unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster);697 698 cluster >>= v->data_dev_block_bits;699 if (unlikely(!cluster))700 goto no_prefetch_cluster;701 702 if (unlikely(cluster & (cluster - 1)))703 cluster = 1 << __fls(cluster);704 705 hash_block_start &= ~(sector_t)(cluster - 1);706 hash_block_end |= cluster - 1;707 if (unlikely(hash_block_end >= v->hash_blocks))708 hash_block_end = v->hash_blocks - 1;709 }710no_prefetch_cluster:711 dm_bufio_prefetch_with_ioprio(v->bufio, hash_block_start,712 hash_block_end - hash_block_start + 1,713 pw->ioprio);714 }715 716 kfree(pw);717}718 719static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io,720 unsigned short ioprio)721{722 sector_t block = io->block;723 unsigned int n_blocks = io->n_blocks;724 struct dm_verity_prefetch_work *pw;725 726 if (v->validated_blocks) {727 while (n_blocks && test_bit(block, v->validated_blocks)) {728 block++;729 n_blocks--;730 }731 while (n_blocks && test_bit(block + n_blocks - 1,732 v->validated_blocks))733 n_blocks--;734 if (!n_blocks)735 return;736 }737 738 pw = kmalloc(sizeof(struct dm_verity_prefetch_work),739 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);740 741 if (!pw)742 return;743 744 INIT_WORK(&pw->work, verity_prefetch_io);745 pw->v = v;746 pw->block = block;747 pw->n_blocks = n_blocks;748 pw->ioprio = ioprio;749 queue_work(v->verify_wq, &pw->work);750}751 752/*753 * Bio map function. It allocates dm_verity_io structure and bio vector and754 * fills them. Then it issues prefetches and the I/O.755 */756static int verity_map(struct dm_target *ti, struct bio *bio)757{758 struct dm_verity *v = ti->private;759 struct dm_verity_io *io;760 761 bio_set_dev(bio, v->data_dev->bdev);762 bio->bi_iter.bi_sector = verity_map_sector(v, bio->bi_iter.bi_sector);763 764 if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) &765 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {766 DMERR_LIMIT("unaligned io");767 return DM_MAPIO_KILL;768 }769 770 if (bio_end_sector(bio) >>771 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {772 DMERR_LIMIT("io out of range");773 return DM_MAPIO_KILL;774 }775 776 if (bio_data_dir(bio) == WRITE)777 return DM_MAPIO_KILL;778 779 io = dm_per_bio_data(bio, ti->per_io_data_size);780 io->v = v;781 io->orig_bi_end_io = bio->bi_end_io;782 io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);783 io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;784 io->had_mismatch = false;785 786 bio->bi_end_io = verity_end_io;787 bio->bi_private = io;788 io->iter = bio->bi_iter;789 790 verity_fec_init_io(io);791 792 verity_submit_prefetch(v, io, bio_prio(bio));793 794 submit_bio_noacct(bio);795 796 return DM_MAPIO_SUBMITTED;797}798 799/*800 * Status: V (valid) or C (corruption found)801 */802static void verity_status(struct dm_target *ti, status_type_t type,803 unsigned int status_flags, char *result, unsigned int maxlen)804{805 struct dm_verity *v = ti->private;806 unsigned int args = 0;807 unsigned int sz = 0;808 unsigned int x;809 810 switch (type) {811 case STATUSTYPE_INFO:812 DMEMIT("%c", v->hash_failed ? 'C' : 'V');813 break;814 case STATUSTYPE_TABLE:815 DMEMIT("%u %s %s %u %u %llu %llu %s ",816 v->version,817 v->data_dev->name,818 v->hash_dev->name,819 1 << v->data_dev_block_bits,820 1 << v->hash_dev_block_bits,821 (unsigned long long)v->data_blocks,822 (unsigned long long)v->hash_start,823 v->alg_name824 );825 for (x = 0; x < v->digest_size; x++)826 DMEMIT("%02x", v->root_digest[x]);827 DMEMIT(" ");828 if (!v->salt_size)829 DMEMIT("-");830 else831 for (x = 0; x < v->salt_size; x++)832 DMEMIT("%02x", v->salt[x]);833 if (v->mode != DM_VERITY_MODE_EIO)834 args++;835 if (v->error_mode != DM_VERITY_MODE_EIO)836 args++;837 if (verity_fec_is_enabled(v))838 args += DM_VERITY_OPTS_FEC;839 if (v->zero_digest)840 args++;841 if (v->validated_blocks)842 args++;843 if (v->use_bh_wq)844 args++;845 if (v->signature_key_desc)846 args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;847 if (!args)848 return;849 DMEMIT(" %u", args);850 if (v->mode != DM_VERITY_MODE_EIO) {851 DMEMIT(" ");852 switch (v->mode) {853 case DM_VERITY_MODE_LOGGING:854 DMEMIT(DM_VERITY_OPT_LOGGING);855 break;856 case DM_VERITY_MODE_RESTART:857 DMEMIT(DM_VERITY_OPT_RESTART);858 break;859 case DM_VERITY_MODE_PANIC:860 DMEMIT(DM_VERITY_OPT_PANIC);861 break;862 default:863 BUG();864 }865 }866 if (v->error_mode != DM_VERITY_MODE_EIO) {867 DMEMIT(" ");868 switch (v->error_mode) {869 case DM_VERITY_MODE_RESTART:870 DMEMIT(DM_VERITY_OPT_ERROR_RESTART);871 break;872 case DM_VERITY_MODE_PANIC:873 DMEMIT(DM_VERITY_OPT_ERROR_PANIC);874 break;875 default:876 BUG();877 }878 }879 if (v->zero_digest)880 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);881 if (v->validated_blocks)882 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);883 if (v->use_bh_wq)884 DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);885 sz = verity_fec_status_table(v, sz, result, maxlen);886 if (v->signature_key_desc)887 DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY888 " %s", v->signature_key_desc);889 break;890 891 case STATUSTYPE_IMA:892 DMEMIT_TARGET_NAME_VERSION(ti->type);893 DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V');894 DMEMIT(",verity_version=%u", v->version);895 DMEMIT(",data_device_name=%s", v->data_dev->name);896 DMEMIT(",hash_device_name=%s", v->hash_dev->name);897 DMEMIT(",verity_algorithm=%s", v->alg_name);898 899 DMEMIT(",root_digest=");900 for (x = 0; x < v->digest_size; x++)901 DMEMIT("%02x", v->root_digest[x]);902 903 DMEMIT(",salt=");904 if (!v->salt_size)905 DMEMIT("-");906 else907 for (x = 0; x < v->salt_size; x++)908 DMEMIT("%02x", v->salt[x]);909 910 DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n');911 DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n');912 if (v->signature_key_desc)913 DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc);914 915 if (v->mode != DM_VERITY_MODE_EIO) {916 DMEMIT(",verity_mode=");917 switch (v->mode) {918 case DM_VERITY_MODE_LOGGING:919 DMEMIT(DM_VERITY_OPT_LOGGING);920 break;921 case DM_VERITY_MODE_RESTART:922 DMEMIT(DM_VERITY_OPT_RESTART);923 break;924 case DM_VERITY_MODE_PANIC:925 DMEMIT(DM_VERITY_OPT_PANIC);926 break;927 default:928 DMEMIT("invalid");929 }930 }931 if (v->error_mode != DM_VERITY_MODE_EIO) {932 DMEMIT(",verity_error_mode=");933 switch (v->error_mode) {934 case DM_VERITY_MODE_RESTART:935 DMEMIT(DM_VERITY_OPT_ERROR_RESTART);936 break;937 case DM_VERITY_MODE_PANIC:938 DMEMIT(DM_VERITY_OPT_ERROR_PANIC);939 break;940 default:941 DMEMIT("invalid");942 }943 }944 DMEMIT(";");945 break;946 }947}948 949static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)950{951 struct dm_verity *v = ti->private;952 953 *bdev = v->data_dev->bdev;954 955 if (v->data_start || ti->len != bdev_nr_sectors(v->data_dev->bdev))956 return 1;957 return 0;958}959 960static int verity_iterate_devices(struct dm_target *ti,961 iterate_devices_callout_fn fn, void *data)962{963 struct dm_verity *v = ti->private;964 965 return fn(ti, v->data_dev, v->data_start, ti->len, data);966}967 968static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)969{970 struct dm_verity *v = ti->private;971 972 if (limits->logical_block_size < 1 << v->data_dev_block_bits)973 limits->logical_block_size = 1 << v->data_dev_block_bits;974 975 if (limits->physical_block_size < 1 << v->data_dev_block_bits)976 limits->physical_block_size = 1 << v->data_dev_block_bits;977 978 limits->io_min = limits->logical_block_size;979 980 /*981 * Similar to what dm-crypt does, opt dm-verity out of support for982 * direct I/O that is aligned to less than the traditional direct I/O983 * alignment requirement of logical_block_size. This prevents dm-verity984 * data blocks from crossing pages, eliminating various edge cases.985 */986 limits->dma_alignment = limits->logical_block_size - 1;987}988 989#ifdef CONFIG_SECURITY990 991static int verity_init_sig(struct dm_verity *v, const void *sig,992 size_t sig_size)993{994 v->sig_size = sig_size;995 996 if (sig) {997 v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL);998 if (!v->root_digest_sig)999 return -ENOMEM;1000 }1001 1002 return 0;1003}1004 1005static void verity_free_sig(struct dm_verity *v)1006{1007 kfree(v->root_digest_sig);1008}1009 1010#else1011 1012static inline int verity_init_sig(struct dm_verity *v, const void *sig,1013 size_t sig_size)1014{1015 return 0;1016}1017 1018static inline void verity_free_sig(struct dm_verity *v)1019{1020}1021 1022#endif /* CONFIG_SECURITY */1023 1024static void verity_dtr(struct dm_target *ti)1025{1026 struct dm_verity *v = ti->private;1027 1028 if (v->verify_wq)1029 destroy_workqueue(v->verify_wq);1030 1031 mempool_exit(&v->recheck_pool);1032 if (v->io)1033 dm_io_client_destroy(v->io);1034 1035 if (v->bufio)1036 dm_bufio_client_destroy(v->bufio);1037 1038 kvfree(v->validated_blocks);1039 kfree(v->salt);1040 kfree(v->initial_hashstate);1041 kfree(v->root_digest);1042 kfree(v->zero_digest);1043 verity_free_sig(v);1044 1045 if (v->ahash_tfm) {1046 static_branch_dec(&ahash_enabled);1047 crypto_free_ahash(v->ahash_tfm);1048 } else {1049 crypto_free_shash(v->shash_tfm);1050 }1051 1052 kfree(v->alg_name);1053 1054 if (v->hash_dev)1055 dm_put_device(ti, v->hash_dev);1056 1057 if (v->data_dev)1058 dm_put_device(ti, v->data_dev);1059 1060 verity_fec_dtr(v);1061 1062 kfree(v->signature_key_desc);1063 1064 if (v->use_bh_wq)1065 static_branch_dec(&use_bh_wq_enabled);1066 1067 kfree(v);1068 1069 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);1070}1071 1072static int verity_alloc_most_once(struct dm_verity *v)1073{1074 struct dm_target *ti = v->ti;1075 1076 /* the bitset can only handle INT_MAX blocks */1077 if (v->data_blocks > INT_MAX) {1078 ti->error = "device too large to use check_at_most_once";1079 return -E2BIG;1080 }1081 1082 v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks),1083 sizeof(unsigned long),1084 GFP_KERNEL);1085 if (!v->validated_blocks) {1086 ti->error = "failed to allocate bitset for check_at_most_once";1087 return -ENOMEM;1088 }1089 1090 return 0;1091}1092 1093static int verity_alloc_zero_digest(struct dm_verity *v)1094{1095 int r = -ENOMEM;1096 struct dm_verity_io *io;1097 u8 *zero_data;1098 1099 v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);1100 1101 if (!v->zero_digest)1102 return r;1103 1104 io = kmalloc(sizeof(*io) + v->hash_reqsize, GFP_KERNEL);1105 1106 if (!io)1107 return r; /* verity_dtr will free zero_digest */1108 1109 zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);1110 1111 if (!zero_data)1112 goto out;1113 1114 r = verity_hash(v, io, zero_data, 1 << v->data_dev_block_bits,1115 v->zero_digest, true);1116 1117out:1118 kfree(io);1119 kfree(zero_data);1120 1121 return r;1122}1123 1124static inline bool verity_is_verity_mode(const char *arg_name)1125{1126 return (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING) ||1127 !strcasecmp(arg_name, DM_VERITY_OPT_RESTART) ||1128 !strcasecmp(arg_name, DM_VERITY_OPT_PANIC));1129}1130 1131static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name)1132{1133 if (v->mode)1134 return -EINVAL;1135 1136 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING))1137 v->mode = DM_VERITY_MODE_LOGGING;1138 else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART))1139 v->mode = DM_VERITY_MODE_RESTART;1140 else if (!strcasecmp(arg_name, DM_VERITY_OPT_PANIC))1141 v->mode = DM_VERITY_MODE_PANIC;1142 1143 return 0;1144}1145 1146static inline bool verity_is_verity_error_mode(const char *arg_name)1147{1148 return (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_RESTART) ||1149 !strcasecmp(arg_name, DM_VERITY_OPT_ERROR_PANIC));1150}1151 1152static int verity_parse_verity_error_mode(struct dm_verity *v, const char *arg_name)1153{1154 if (v->error_mode)1155 return -EINVAL;1156 1157 if (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_RESTART))1158 v->error_mode = DM_VERITY_MODE_RESTART;1159 else if (!strcasecmp(arg_name, DM_VERITY_OPT_ERROR_PANIC))1160 v->error_mode = DM_VERITY_MODE_PANIC;1161 1162 return 0;1163}1164 1165static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,1166 struct dm_verity_sig_opts *verify_args,1167 bool only_modifier_opts)1168{1169 int r = 0;1170 unsigned int argc;1171 struct dm_target *ti = v->ti;1172 const char *arg_name;1173 1174 static const struct dm_arg _args[] = {1175 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},1176 };1177 1178 r = dm_read_arg_group(_args, as, &argc, &ti->error);1179 if (r)1180 return -EINVAL;1181 1182 if (!argc)1183 return 0;1184 1185 do {1186 arg_name = dm_shift_arg(as);1187 argc--;1188 1189 if (verity_is_verity_mode(arg_name)) {1190 if (only_modifier_opts)1191 continue;1192 r = verity_parse_verity_mode(v, arg_name);1193 if (r) {1194 ti->error = "Conflicting error handling parameters";1195 return r;1196 }1197 continue;1198 1199 } else if (verity_is_verity_error_mode(arg_name)) {1200 if (only_modifier_opts)1201 continue;1202 r = verity_parse_verity_error_mode(v, arg_name);1203 if (r) {1204 ti->error = "Conflicting error handling parameters";1205 return r;1206 }1207 continue;1208 1209 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_IGN_ZEROES)) {1210 if (only_modifier_opts)1211 continue;1212 r = verity_alloc_zero_digest(v);1213 if (r) {1214 ti->error = "Cannot allocate zero digest";1215 return r;1216 }1217 continue;1218 1219 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) {1220 if (only_modifier_opts)1221 continue;1222 r = verity_alloc_most_once(v);1223 if (r)1224 return r;1225 continue;1226 1227 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {1228 v->use_bh_wq = true;1229 static_branch_inc(&use_bh_wq_enabled);1230 continue;1231 1232 } else if (verity_is_fec_opt_arg(arg_name)) {1233 if (only_modifier_opts)1234 continue;1235 r = verity_fec_parse_opt_args(as, v, &argc, arg_name);1236 if (r)1237 return r;1238 continue;1239 1240 } else if (verity_verify_is_sig_opt_arg(arg_name)) {1241 if (only_modifier_opts)1242 continue;1243 r = verity_verify_sig_parse_opt_args(as, v,1244 verify_args,1245 &argc, arg_name);1246 if (r)1247 return r;1248 continue;1249 1250 } else if (only_modifier_opts) {1251 /*1252 * Ignore unrecognized opt, could easily be an extra1253 * argument to an option whose parsing was skipped.1254 * Normal parsing (@only_modifier_opts=false) will1255 * properly parse all options (and their extra args).1256 */1257 continue;1258 }1259 1260 DMERR("Unrecognized verity feature request: %s", arg_name);1261 ti->error = "Unrecognized verity feature request";1262 return -EINVAL;1263 } while (argc && !r);1264 1265 return r;1266}1267 1268static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)1269{1270 struct dm_target *ti = v->ti;1271 struct crypto_ahash *ahash;1272 struct crypto_shash *shash = NULL;1273 const char *driver_name;1274 1275 v->alg_name = kstrdup(alg_name, GFP_KERNEL);1276 if (!v->alg_name) {1277 ti->error = "Cannot allocate algorithm name";1278 return -ENOMEM;1279 }1280 1281 /*1282 * Allocate the hash transformation object that this dm-verity instance1283 * will use. The vast majority of dm-verity users use CPU-based1284 * hashing, so when possible use the shash API to minimize the crypto1285 * API overhead. If the ahash API resolves to a different driver1286 * (likely an off-CPU hardware offload), use ahash instead. Also use1287 * ahash if the obsolete dm-verity format with the appended salt is1288 * being used, so that quirk only needs to be handled in one place.1289 */1290 ahash = crypto_alloc_ahash(alg_name, 0,1291 v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0);1292 if (IS_ERR(ahash)) {1293 ti->error = "Cannot initialize hash function";1294 return PTR_ERR(ahash);1295 }1296 driver_name = crypto_ahash_driver_name(ahash);1297 if (v->version >= 1 /* salt prepended, not appended? */) {1298 shash = crypto_alloc_shash(alg_name, 0, 0);1299 if (!IS_ERR(shash) &&1300 strcmp(crypto_shash_driver_name(shash), driver_name) != 0) {1301 /*1302 * ahash gave a different driver than shash, so probably1303 * this is a case of real hardware offload. Use ahash.1304 */1305 crypto_free_shash(shash);1306 shash = NULL;1307 }1308 }1309 if (!IS_ERR_OR_NULL(shash)) {1310 crypto_free_ahash(ahash);1311 ahash = NULL;1312 v->shash_tfm = shash;1313 v->digest_size = crypto_shash_digestsize(shash);1314 v->hash_reqsize = sizeof(struct shash_desc) +1315 crypto_shash_descsize(shash);1316 DMINFO("%s using shash \"%s\"", alg_name, driver_name);1317 } else {1318 v->ahash_tfm = ahash;1319 static_branch_inc(&ahash_enabled);1320 v->digest_size = crypto_ahash_digestsize(ahash);1321 v->hash_reqsize = sizeof(struct ahash_request) +1322 crypto_ahash_reqsize(ahash);1323 DMINFO("%s using ahash \"%s\"", alg_name, driver_name);1324 }1325 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {1326 ti->error = "Digest size too big";1327 return -EINVAL;1328 }1329 return 0;1330}1331 1332static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg)1333{1334 struct dm_target *ti = v->ti;1335 1336 if (strcmp(arg, "-") != 0) {1337 v->salt_size = strlen(arg) / 2;1338 v->salt = kmalloc(v->salt_size, GFP_KERNEL);1339 if (!v->salt) {1340 ti->error = "Cannot allocate salt";1341 return -ENOMEM;1342 }1343 if (strlen(arg) != v->salt_size * 2 ||1344 hex2bin(v->salt, arg, v->salt_size)) {1345 ti->error = "Invalid salt";1346 return -EINVAL;1347 }1348 }1349 if (v->shash_tfm) {1350 SHASH_DESC_ON_STACK(desc, v->shash_tfm);1351 int r;1352 1353 /*1354 * Compute the pre-salted hash state that can be passed to1355 * crypto_shash_import() for each block later.1356 */1357 v->initial_hashstate = kmalloc(1358 crypto_shash_statesize(v->shash_tfm), GFP_KERNEL);1359 if (!v->initial_hashstate) {1360 ti->error = "Cannot allocate initial hash state";1361 return -ENOMEM;1362 }1363 desc->tfm = v->shash_tfm;1364 r = crypto_shash_init(desc) ?:1365 crypto_shash_update(desc, v->salt, v->salt_size) ?:1366 crypto_shash_export(desc, v->initial_hashstate);1367 if (r) {1368 ti->error = "Cannot set up initial hash state";1369 return r;1370 }1371 }1372 return 0;1373}1374 1375/*1376 * Target parameters:1377 * <version> The current format is version 1.1378 * Vsn 0 is compatible with original Chromium OS releases.1379 * <data device>1380 * <hash device>1381 * <data block size>1382 * <hash block size>1383 * <the number of data blocks>1384 * <hash start block>1385 * <algorithm>1386 * <digest>1387 * <salt> Hex string or "-" if no salt.1388 */1389static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)1390{1391 struct dm_verity *v;1392 struct dm_verity_sig_opts verify_args = {0};1393 struct dm_arg_set as;1394 unsigned int num;1395 unsigned long long num_ll;1396 int r;1397 int i;1398 sector_t hash_position;1399 char dummy;1400 char *root_hash_digest_to_validate;1401 1402 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);1403 if (!v) {1404 ti->error = "Cannot allocate verity structure";1405 return -ENOMEM;1406 }1407 ti->private = v;1408 v->ti = ti;1409 1410 r = verity_fec_ctr_alloc(v);1411 if (r)1412 goto bad;1413 1414 if ((dm_table_get_mode(ti->table) & ~BLK_OPEN_READ)) {1415 ti->error = "Device must be readonly";1416 r = -EINVAL;1417 goto bad;1418 }1419 1420 if (argc < 10) {1421 ti->error = "Not enough arguments";1422 r = -EINVAL;1423 goto bad;1424 }1425 1426 /* Parse optional parameters that modify primary args */1427 if (argc > 10) {1428 as.argc = argc - 10;1429 as.argv = argv + 10;1430 r = verity_parse_opt_args(&as, v, &verify_args, true);1431 if (r < 0)1432 goto bad;1433 }1434 1435 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||1436 num > 1) {1437 ti->error = "Invalid version";1438 r = -EINVAL;1439 goto bad;1440 }1441 v->version = num;1442 1443 r = dm_get_device(ti, argv[1], BLK_OPEN_READ, &v->data_dev);1444 if (r) {1445 ti->error = "Data device lookup failed";1446 goto bad;1447 }1448 1449 r = dm_get_device(ti, argv[2], BLK_OPEN_READ, &v->hash_dev);1450 if (r) {1451 ti->error = "Hash device lookup failed";1452 goto bad;1453 }1454 1455 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||1456 !num || (num & (num - 1)) ||1457 num < bdev_logical_block_size(v->data_dev->bdev) ||1458 num > PAGE_SIZE) {1459 ti->error = "Invalid data device block size";1460 r = -EINVAL;1461 goto bad;1462 }1463 v->data_dev_block_bits = __ffs(num);1464 1465 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||1466 !num || (num & (num - 1)) ||1467 num < bdev_logical_block_size(v->hash_dev->bdev) ||1468 num > INT_MAX) {1469 ti->error = "Invalid hash device block size";1470 r = -EINVAL;1471 goto bad;1472 }1473 v->hash_dev_block_bits = __ffs(num);1474 1475 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||1476 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))1477 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {1478 ti->error = "Invalid data blocks";1479 r = -EINVAL;1480 goto bad;1481 }1482 v->data_blocks = num_ll;1483 1484 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {1485 ti->error = "Data device is too small";1486 r = -EINVAL;1487 goto bad;1488 }1489 1490 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||1491 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))1492 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {1493 ti->error = "Invalid hash start";1494 r = -EINVAL;1495 goto bad;1496 }1497 v->hash_start = num_ll;1498 1499 r = verity_setup_hash_alg(v, argv[7]);1500 if (r)1501 goto bad;1502 1503 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);1504 if (!v->root_digest) {1505 ti->error = "Cannot allocate root digest";1506 r = -ENOMEM;1507 goto bad;1508 }1509 if (strlen(argv[8]) != v->digest_size * 2 ||1510 hex2bin(v->root_digest, argv[8], v->digest_size)) {1511 ti->error = "Invalid root digest";1512 r = -EINVAL;1513 goto bad;1514 }1515 root_hash_digest_to_validate = argv[8];1516 1517 r = verity_setup_salt_and_hashstate(v, argv[9]);1518 if (r)1519 goto bad;1520 1521 argv += 10;1522 argc -= 10;1523 1524 /* Optional parameters */1525 if (argc) {1526 as.argc = argc;1527 as.argv = argv;1528 r = verity_parse_opt_args(&as, v, &verify_args, false);1529 if (r < 0)1530 goto bad;1531 }1532 1533 /* Root hash signature is a optional parameter*/1534 r = verity_verify_root_hash(root_hash_digest_to_validate,1535 strlen(root_hash_digest_to_validate),1536 verify_args.sig,1537 verify_args.sig_size);1538 if (r < 0) {1539 ti->error = "Root hash verification failed";1540 goto bad;1541 }1542 1543 r = verity_init_sig(v, verify_args.sig, verify_args.sig_size);1544 if (r < 0) {1545 ti->error = "Cannot allocate root digest signature";1546 goto bad;1547 }1548 1549 v->hash_per_block_bits =1550 __fls((1 << v->hash_dev_block_bits) / v->digest_size);1551 1552 v->levels = 0;1553 if (v->data_blocks)1554 while (v->hash_per_block_bits * v->levels < 64 &&1555 (unsigned long long)(v->data_blocks - 1) >>1556 (v->hash_per_block_bits * v->levels))1557 v->levels++;1558 1559 if (v->levels > DM_VERITY_MAX_LEVELS) {1560 ti->error = "Too many tree levels";1561 r = -E2BIG;1562 goto bad;1563 }1564 1565 hash_position = v->hash_start;1566 for (i = v->levels - 1; i >= 0; i--) {1567 sector_t s;1568 1569 v->hash_level_block[i] = hash_position;1570 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)1571 >> ((i + 1) * v->hash_per_block_bits);1572 if (hash_position + s < hash_position) {1573 ti->error = "Hash device offset overflow";1574 r = -E2BIG;1575 goto bad;1576 }1577 hash_position += s;1578 }1579 v->hash_blocks = hash_position;1580 1581 r = mempool_init_page_pool(&v->recheck_pool, 1, 0);1582 if (unlikely(r)) {1583 ti->error = "Cannot allocate mempool";1584 goto bad;1585 }1586 1587 v->io = dm_io_client_create();1588 if (IS_ERR(v->io)) {1589 r = PTR_ERR(v->io);1590 v->io = NULL;1591 ti->error = "Cannot allocate dm io";1592 goto bad;1593 }1594 1595 v->bufio = dm_bufio_client_create(v->hash_dev->bdev,1596 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),1597 dm_bufio_alloc_callback, NULL,1598 v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);1599 if (IS_ERR(v->bufio)) {1600 ti->error = "Cannot initialize dm-bufio";1601 r = PTR_ERR(v->bufio);1602 v->bufio = NULL;1603 goto bad;1604 }1605 1606 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {1607 ti->error = "Hash device is too small";1608 r = -E2BIG;1609 goto bad;1610 }1611 1612 /*1613 * Using WQ_HIGHPRI improves throughput and completion latency by1614 * reducing wait times when reading from a dm-verity device.1615 *1616 * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI1617 * allows verify_wq to preempt softirq since verification in BH workqueue1618 * will fall-back to using it for error handling (or if the bufio cache1619 * doesn't have required hashes).1620 */1621 v->verify_wq = alloc_workqueue("kverityd", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);1622 if (!v->verify_wq) {1623 ti->error = "Cannot allocate workqueue";1624 r = -ENOMEM;1625 goto bad;1626 }1627 1628 ti->per_io_data_size = sizeof(struct dm_verity_io) + v->hash_reqsize;1629 1630 r = verity_fec_ctr(v);1631 if (r)1632 goto bad;1633 1634 ti->per_io_data_size = roundup(ti->per_io_data_size,1635 __alignof__(struct dm_verity_io));1636 1637 verity_verify_sig_opts_cleanup(&verify_args);1638 1639 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);1640 1641 return 0;1642 1643bad:1644 1645 verity_verify_sig_opts_cleanup(&verify_args);1646 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);1647 verity_dtr(ti);1648 1649 return r;1650}1651 1652/*1653 * Get the verity mode (error behavior) of a verity target.1654 *1655 * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity1656 * target.1657 */1658int dm_verity_get_mode(struct dm_target *ti)1659{1660 struct dm_verity *v = ti->private;1661 1662 if (!dm_is_verity_target(ti))1663 return -EINVAL;1664 1665 return v->mode;1666}1667 1668/*1669 * Get the root digest of a verity target.1670 *1671 * Returns a copy of the root digest, the caller is responsible for1672 * freeing the memory of the digest.1673 */1674int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size)1675{1676 struct dm_verity *v = ti->private;1677 1678 if (!dm_is_verity_target(ti))1679 return -EINVAL;1680 1681 *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL);1682 if (*root_digest == NULL)1683 return -ENOMEM;1684 1685 *digest_size = v->digest_size;1686 1687 return 0;1688}1689 1690#ifdef CONFIG_SECURITY1691 1692#ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG1693 1694static int verity_security_set_signature(struct block_device *bdev,1695 struct dm_verity *v)1696{1697 /*1698 * if the dm-verity target is unsigned, v->root_digest_sig will1699 * be NULL, and the hook call is still required to let LSMs mark1700 * the device as unsigned. This information is crucial for LSMs to1701 * block operations such as execution on unsigned files1702 */1703 return security_bdev_setintegrity(bdev,1704 LSM_INT_DMVERITY_SIG_VALID,1705 v->root_digest_sig,1706 v->sig_size);1707}1708 1709#else1710 1711static inline int verity_security_set_signature(struct block_device *bdev,1712 struct dm_verity *v)1713{1714 return 0;1715}1716 1717#endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */1718 1719/*1720 * Expose verity target's root hash and signature data to LSMs before resume.1721 *1722 * Returns 0 on success, or -ENOMEM if the system is out of memory.1723 */1724static int verity_preresume(struct dm_target *ti)1725{1726 struct block_device *bdev;1727 struct dm_verity_digest root_digest;1728 struct dm_verity *v;1729 int r;1730 1731 v = ti->private;1732 bdev = dm_disk(dm_table_get_md(ti->table))->part0;1733 root_digest.digest = v->root_digest;1734 root_digest.digest_len = v->digest_size;1735 if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm)1736 root_digest.alg = crypto_ahash_alg_name(v->ahash_tfm);1737 else1738 root_digest.alg = crypto_shash_alg_name(v->shash_tfm);1739 1740 r = security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, &root_digest,1741 sizeof(root_digest));1742 if (r)1743 return r;1744 1745 r = verity_security_set_signature(bdev, v);1746 if (r)1747 goto bad;1748 1749 return 0;1750 1751bad:1752 1753 security_bdev_setintegrity(bdev, LSM_INT_DMVERITY_ROOTHASH, NULL, 0);1754 1755 return r;1756}1757 1758#endif /* CONFIG_SECURITY */1759 1760static struct target_type verity_target = {1761 .name = "verity",1762/* Note: the LSMs depend on the singleton and immutable features */1763 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,1764 .version = {1, 10, 0},1765 .module = THIS_MODULE,1766 .ctr = verity_ctr,1767 .dtr = verity_dtr,1768 .map = verity_map,1769 .status = verity_status,1770 .prepare_ioctl = verity_prepare_ioctl,1771 .iterate_devices = verity_iterate_devices,1772 .io_hints = verity_io_hints,1773#ifdef CONFIG_SECURITY1774 .preresume = verity_preresume,1775#endif /* CONFIG_SECURITY */1776};1777module_dm(verity);1778 1779/*1780 * Check whether a DM target is a verity target.1781 */1782bool dm_is_verity_target(struct dm_target *ti)1783{1784 return ti->type == &verity_target;1785}1786 1787MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");1788MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");1789MODULE_AUTHOR("Will Drewry <wad@chromium.org>");1790MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");1791MODULE_LICENSE("GPL");1792