474 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * t10_pi.c - Functions for generating and verifying T10 Protection4 * Information.5 */6 7#include <linux/t10-pi.h>8#include <linux/blk-integrity.h>9#include <linux/crc-t10dif.h>10#include <linux/crc64.h>11#include <net/checksum.h>12#include <linux/unaligned.h>13#include "blk.h"14 15struct blk_integrity_iter {16 void *prot_buf;17 void *data_buf;18 sector_t seed;19 unsigned int data_size;20 unsigned short interval;21 const char *disk_name;22};23 24static __be16 t10_pi_csum(__be16 csum, void *data, unsigned int len,25 unsigned char csum_type)26{27 if (csum_type == BLK_INTEGRITY_CSUM_IP)28 return (__force __be16)ip_compute_csum(data, len);29 return cpu_to_be16(crc_t10dif_update(be16_to_cpu(csum), data, len));30}31 32/*33 * Type 1 and Type 2 protection use the same format: 16 bit guard tag,34 * 16 bit app tag, 32 bit reference tag. Type 3 does not define the ref35 * tag.36 */37static void t10_pi_generate(struct blk_integrity_iter *iter,38 struct blk_integrity *bi)39{40 u8 offset = bi->pi_offset;41 unsigned int i;42 43 for (i = 0 ; i < iter->data_size ; i += iter->interval) {44 struct t10_pi_tuple *pi = iter->prot_buf + offset;45 46 pi->guard_tag = t10_pi_csum(0, iter->data_buf, iter->interval,47 bi->csum_type);48 if (offset)49 pi->guard_tag = t10_pi_csum(pi->guard_tag,50 iter->prot_buf, offset, bi->csum_type);51 pi->app_tag = 0;52 53 if (bi->flags & BLK_INTEGRITY_REF_TAG)54 pi->ref_tag = cpu_to_be32(lower_32_bits(iter->seed));55 else56 pi->ref_tag = 0;57 58 iter->data_buf += iter->interval;59 iter->prot_buf += bi->tuple_size;60 iter->seed++;61 }62}63 64static blk_status_t t10_pi_verify(struct blk_integrity_iter *iter,65 struct blk_integrity *bi)66{67 u8 offset = bi->pi_offset;68 unsigned int i;69 70 for (i = 0 ; i < iter->data_size ; i += iter->interval) {71 struct t10_pi_tuple *pi = iter->prot_buf + offset;72 __be16 csum;73 74 if (bi->flags & BLK_INTEGRITY_REF_TAG) {75 if (pi->app_tag == T10_PI_APP_ESCAPE)76 goto next;77 78 if (be32_to_cpu(pi->ref_tag) !=79 lower_32_bits(iter->seed)) {80 pr_err("%s: ref tag error at location %llu " \81 "(rcvd %u)\n", iter->disk_name,82 (unsigned long long)83 iter->seed, be32_to_cpu(pi->ref_tag));84 return BLK_STS_PROTECTION;85 }86 } else {87 if (pi->app_tag == T10_PI_APP_ESCAPE &&88 pi->ref_tag == T10_PI_REF_ESCAPE)89 goto next;90 }91 92 csum = t10_pi_csum(0, iter->data_buf, iter->interval,93 bi->csum_type);94 if (offset)95 csum = t10_pi_csum(csum, iter->prot_buf, offset,96 bi->csum_type);97 98 if (pi->guard_tag != csum) {99 pr_err("%s: guard tag error at sector %llu " \100 "(rcvd %04x, want %04x)\n", iter->disk_name,101 (unsigned long long)iter->seed,102 be16_to_cpu(pi->guard_tag), be16_to_cpu(csum));103 return BLK_STS_PROTECTION;104 }105 106next:107 iter->data_buf += iter->interval;108 iter->prot_buf += bi->tuple_size;109 iter->seed++;110 }111 112 return BLK_STS_OK;113}114 115/**116 * t10_pi_type1_prepare - prepare PI prior submitting request to device117 * @rq: request with PI that should be prepared118 *119 * For Type 1/Type 2, the virtual start sector is the one that was120 * originally submitted by the block layer for the ref_tag usage. Due to121 * partitioning, MD/DM cloning, etc. the actual physical start sector is122 * likely to be different. Remap protection information to match the123 * physical LBA.124 */125static void t10_pi_type1_prepare(struct request *rq)126{127 struct blk_integrity *bi = &rq->q->limits.integrity;128 const int tuple_sz = bi->tuple_size;129 u32 ref_tag = t10_pi_ref_tag(rq);130 u8 offset = bi->pi_offset;131 struct bio *bio;132 133 __rq_for_each_bio(bio, rq) {134 struct bio_integrity_payload *bip = bio_integrity(bio);135 u32 virt = bip_get_seed(bip) & 0xffffffff;136 struct bio_vec iv;137 struct bvec_iter iter;138 139 /* Already remapped? */140 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)141 break;142 143 bip_for_each_vec(iv, bip, iter) {144 unsigned int j;145 void *p;146 147 p = bvec_kmap_local(&iv);148 for (j = 0; j < iv.bv_len; j += tuple_sz) {149 struct t10_pi_tuple *pi = p + offset;150 151 if (be32_to_cpu(pi->ref_tag) == virt)152 pi->ref_tag = cpu_to_be32(ref_tag);153 virt++;154 ref_tag++;155 p += tuple_sz;156 }157 kunmap_local(p);158 }159 160 bip->bip_flags |= BIP_MAPPED_INTEGRITY;161 }162}163 164/**165 * t10_pi_type1_complete - prepare PI prior returning request to the blk layer166 * @rq: request with PI that should be prepared167 * @nr_bytes: total bytes to prepare168 *169 * For Type 1/Type 2, the virtual start sector is the one that was170 * originally submitted by the block layer for the ref_tag usage. Due to171 * partitioning, MD/DM cloning, etc. the actual physical start sector is172 * likely to be different. Since the physical start sector was submitted173 * to the device, we should remap it back to virtual values expected by the174 * block layer.175 */176static void t10_pi_type1_complete(struct request *rq, unsigned int nr_bytes)177{178 struct blk_integrity *bi = &rq->q->limits.integrity;179 unsigned intervals = nr_bytes >> bi->interval_exp;180 const int tuple_sz = bi->tuple_size;181 u32 ref_tag = t10_pi_ref_tag(rq);182 u8 offset = bi->pi_offset;183 struct bio *bio;184 185 __rq_for_each_bio(bio, rq) {186 struct bio_integrity_payload *bip = bio_integrity(bio);187 u32 virt = bip_get_seed(bip) & 0xffffffff;188 struct bio_vec iv;189 struct bvec_iter iter;190 191 bip_for_each_vec(iv, bip, iter) {192 unsigned int j;193 void *p;194 195 p = bvec_kmap_local(&iv);196 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {197 struct t10_pi_tuple *pi = p + offset;198 199 if (be32_to_cpu(pi->ref_tag) == ref_tag)200 pi->ref_tag = cpu_to_be32(virt);201 virt++;202 ref_tag++;203 intervals--;204 p += tuple_sz;205 }206 kunmap_local(p);207 }208 }209}210 211static __be64 ext_pi_crc64(u64 crc, void *data, unsigned int len)212{213 return cpu_to_be64(crc64_rocksoft_update(crc, data, len));214}215 216static void ext_pi_crc64_generate(struct blk_integrity_iter *iter,217 struct blk_integrity *bi)218{219 u8 offset = bi->pi_offset;220 unsigned int i;221 222 for (i = 0 ; i < iter->data_size ; i += iter->interval) {223 struct crc64_pi_tuple *pi = iter->prot_buf + offset;224 225 pi->guard_tag = ext_pi_crc64(0, iter->data_buf, iter->interval);226 if (offset)227 pi->guard_tag = ext_pi_crc64(be64_to_cpu(pi->guard_tag),228 iter->prot_buf, offset);229 pi->app_tag = 0;230 231 if (bi->flags & BLK_INTEGRITY_REF_TAG)232 put_unaligned_be48(iter->seed, pi->ref_tag);233 else234 put_unaligned_be48(0ULL, pi->ref_tag);235 236 iter->data_buf += iter->interval;237 iter->prot_buf += bi->tuple_size;238 iter->seed++;239 }240}241 242static bool ext_pi_ref_escape(const u8 ref_tag[6])243{244 static const u8 ref_escape[6] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };245 246 return memcmp(ref_tag, ref_escape, sizeof(ref_escape)) == 0;247}248 249static blk_status_t ext_pi_crc64_verify(struct blk_integrity_iter *iter,250 struct blk_integrity *bi)251{252 u8 offset = bi->pi_offset;253 unsigned int i;254 255 for (i = 0; i < iter->data_size; i += iter->interval) {256 struct crc64_pi_tuple *pi = iter->prot_buf + offset;257 u64 ref, seed;258 __be64 csum;259 260 if (bi->flags & BLK_INTEGRITY_REF_TAG) {261 if (pi->app_tag == T10_PI_APP_ESCAPE)262 goto next;263 264 ref = get_unaligned_be48(pi->ref_tag);265 seed = lower_48_bits(iter->seed);266 if (ref != seed) {267 pr_err("%s: ref tag error at location %llu (rcvd %llu)\n",268 iter->disk_name, seed, ref);269 return BLK_STS_PROTECTION;270 }271 } else {272 if (pi->app_tag == T10_PI_APP_ESCAPE &&273 ext_pi_ref_escape(pi->ref_tag))274 goto next;275 }276 277 csum = ext_pi_crc64(0, iter->data_buf, iter->interval);278 if (offset)279 csum = ext_pi_crc64(be64_to_cpu(csum), iter->prot_buf,280 offset);281 282 if (pi->guard_tag != csum) {283 pr_err("%s: guard tag error at sector %llu " \284 "(rcvd %016llx, want %016llx)\n",285 iter->disk_name, (unsigned long long)iter->seed,286 be64_to_cpu(pi->guard_tag), be64_to_cpu(csum));287 return BLK_STS_PROTECTION;288 }289 290next:291 iter->data_buf += iter->interval;292 iter->prot_buf += bi->tuple_size;293 iter->seed++;294 }295 296 return BLK_STS_OK;297}298 299static void ext_pi_type1_prepare(struct request *rq)300{301 struct blk_integrity *bi = &rq->q->limits.integrity;302 const int tuple_sz = bi->tuple_size;303 u64 ref_tag = ext_pi_ref_tag(rq);304 u8 offset = bi->pi_offset;305 struct bio *bio;306 307 __rq_for_each_bio(bio, rq) {308 struct bio_integrity_payload *bip = bio_integrity(bio);309 u64 virt = lower_48_bits(bip_get_seed(bip));310 struct bio_vec iv;311 struct bvec_iter iter;312 313 /* Already remapped? */314 if (bip->bip_flags & BIP_MAPPED_INTEGRITY)315 break;316 317 bip_for_each_vec(iv, bip, iter) {318 unsigned int j;319 void *p;320 321 p = bvec_kmap_local(&iv);322 for (j = 0; j < iv.bv_len; j += tuple_sz) {323 struct crc64_pi_tuple *pi = p + offset;324 u64 ref = get_unaligned_be48(pi->ref_tag);325 326 if (ref == virt)327 put_unaligned_be48(ref_tag, pi->ref_tag);328 virt++;329 ref_tag++;330 p += tuple_sz;331 }332 kunmap_local(p);333 }334 335 bip->bip_flags |= BIP_MAPPED_INTEGRITY;336 }337}338 339static void ext_pi_type1_complete(struct request *rq, unsigned int nr_bytes)340{341 struct blk_integrity *bi = &rq->q->limits.integrity;342 unsigned intervals = nr_bytes >> bi->interval_exp;343 const int tuple_sz = bi->tuple_size;344 u64 ref_tag = ext_pi_ref_tag(rq);345 u8 offset = bi->pi_offset;346 struct bio *bio;347 348 __rq_for_each_bio(bio, rq) {349 struct bio_integrity_payload *bip = bio_integrity(bio);350 u64 virt = lower_48_bits(bip_get_seed(bip));351 struct bio_vec iv;352 struct bvec_iter iter;353 354 bip_for_each_vec(iv, bip, iter) {355 unsigned int j;356 void *p;357 358 p = bvec_kmap_local(&iv);359 for (j = 0; j < iv.bv_len && intervals; j += tuple_sz) {360 struct crc64_pi_tuple *pi = p + offset;361 u64 ref = get_unaligned_be48(pi->ref_tag);362 363 if (ref == ref_tag)364 put_unaligned_be48(virt, pi->ref_tag);365 virt++;366 ref_tag++;367 intervals--;368 p += tuple_sz;369 }370 kunmap_local(p);371 }372 }373}374 375void blk_integrity_generate(struct bio *bio)376{377 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);378 struct bio_integrity_payload *bip = bio_integrity(bio);379 struct blk_integrity_iter iter;380 struct bvec_iter bviter;381 struct bio_vec bv;382 383 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;384 iter.interval = 1 << bi->interval_exp;385 iter.seed = bio->bi_iter.bi_sector;386 iter.prot_buf = bvec_virt(bip->bip_vec);387 bio_for_each_segment(bv, bio, bviter) {388 void *kaddr = bvec_kmap_local(&bv);389 390 iter.data_buf = kaddr;391 iter.data_size = bv.bv_len;392 switch (bi->csum_type) {393 case BLK_INTEGRITY_CSUM_CRC64:394 ext_pi_crc64_generate(&iter, bi);395 break;396 case BLK_INTEGRITY_CSUM_CRC:397 case BLK_INTEGRITY_CSUM_IP:398 t10_pi_generate(&iter, bi);399 break;400 default:401 break;402 }403 kunmap_local(kaddr);404 }405}406 407void blk_integrity_verify(struct bio *bio)408{409 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);410 struct bio_integrity_payload *bip = bio_integrity(bio);411 struct blk_integrity_iter iter;412 struct bvec_iter bviter;413 struct bio_vec bv;414 415 /*416 * At the moment verify is called bi_iter has been advanced during split417 * and completion, so use the copy created during submission here.418 */419 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;420 iter.interval = 1 << bi->interval_exp;421 iter.seed = bip->bio_iter.bi_sector;422 iter.prot_buf = bvec_virt(bip->bip_vec);423 __bio_for_each_segment(bv, bio, bviter, bip->bio_iter) {424 void *kaddr = bvec_kmap_local(&bv);425 blk_status_t ret = BLK_STS_OK;426 427 iter.data_buf = kaddr;428 iter.data_size = bv.bv_len;429 switch (bi->csum_type) {430 case BLK_INTEGRITY_CSUM_CRC64:431 ret = ext_pi_crc64_verify(&iter, bi);432 break;433 case BLK_INTEGRITY_CSUM_CRC:434 case BLK_INTEGRITY_CSUM_IP:435 ret = t10_pi_verify(&iter, bi);436 break;437 default:438 break;439 }440 kunmap_local(kaddr);441 442 if (ret) {443 bio->bi_status = ret;444 return;445 }446 }447}448 449void blk_integrity_prepare(struct request *rq)450{451 struct blk_integrity *bi = &rq->q->limits.integrity;452 453 if (!(bi->flags & BLK_INTEGRITY_REF_TAG))454 return;455 456 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)457 ext_pi_type1_prepare(rq);458 else459 t10_pi_type1_prepare(rq);460}461 462void blk_integrity_complete(struct request *rq, unsigned int nr_bytes)463{464 struct blk_integrity *bi = &rq->q->limits.integrity;465 466 if (!(bi->flags & BLK_INTEGRITY_REF_TAG))467 return;468 469 if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC64)470 ext_pi_type1_complete(rq, nr_bytes);471 else472 t10_pi_type1_complete(rq, nr_bytes);473}474