3506 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * raid1.c : Multiple Devices driver for Linux4 *5 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat6 *7 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman8 *9 * RAID-1 management functions.10 *11 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 200012 *13 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>14 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>15 *16 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support17 * bitmapped intelligence in resync:18 *19 * - bitmap marked during normal i/o20 * - bitmap used to skip nondirty blocks during sync21 *22 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:23 * - persistent bitmap code24 */25 26#include <linux/slab.h>27#include <linux/delay.h>28#include <linux/blkdev.h>29#include <linux/module.h>30#include <linux/seq_file.h>31#include <linux/ratelimit.h>32#include <linux/interval_tree_generic.h>33 34#include <trace/events/block.h>35 36#include "md.h"37#include "raid1.h"38#include "md-bitmap.h"39 40#define UNSUPPORTED_MDDEV_FLAGS \41 ((1L << MD_HAS_JOURNAL) | \42 (1L << MD_JOURNAL_CLEAN) | \43 (1L << MD_HAS_PPL) | \44 (1L << MD_HAS_MULTIPLE_PPLS))45 46static void allow_barrier(struct r1conf *conf, sector_t sector_nr);47static void lower_barrier(struct r1conf *conf, sector_t sector_nr);48 49#define RAID_1_10_NAME "raid1"50#include "raid1-10.c"51 52#define START(node) ((node)->start)53#define LAST(node) ((node)->last)54INTERVAL_TREE_DEFINE(struct serial_info, node, sector_t, _subtree_last,55 START, LAST, static inline, raid1_rb);56 57static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio,58 struct serial_info *si, int idx)59{60 unsigned long flags;61 int ret = 0;62 sector_t lo = r1_bio->sector;63 sector_t hi = lo + r1_bio->sectors;64 struct serial_in_rdev *serial = &rdev->serial[idx];65 66 spin_lock_irqsave(&serial->serial_lock, flags);67 /* collision happened */68 if (raid1_rb_iter_first(&serial->serial_rb, lo, hi))69 ret = -EBUSY;70 else {71 si->start = lo;72 si->last = hi;73 raid1_rb_insert(si, &serial->serial_rb);74 }75 spin_unlock_irqrestore(&serial->serial_lock, flags);76 77 return ret;78}79 80static void wait_for_serialization(struct md_rdev *rdev, struct r1bio *r1_bio)81{82 struct mddev *mddev = rdev->mddev;83 struct serial_info *si;84 int idx = sector_to_idx(r1_bio->sector);85 struct serial_in_rdev *serial = &rdev->serial[idx];86 87 if (WARN_ON(!mddev->serial_info_pool))88 return;89 si = mempool_alloc(mddev->serial_info_pool, GFP_NOIO);90 wait_event(serial->serial_io_wait,91 check_and_add_serial(rdev, r1_bio, si, idx) == 0);92}93 94static void remove_serial(struct md_rdev *rdev, sector_t lo, sector_t hi)95{96 struct serial_info *si;97 unsigned long flags;98 int found = 0;99 struct mddev *mddev = rdev->mddev;100 int idx = sector_to_idx(lo);101 struct serial_in_rdev *serial = &rdev->serial[idx];102 103 spin_lock_irqsave(&serial->serial_lock, flags);104 for (si = raid1_rb_iter_first(&serial->serial_rb, lo, hi);105 si; si = raid1_rb_iter_next(si, lo, hi)) {106 if (si->start == lo && si->last == hi) {107 raid1_rb_remove(si, &serial->serial_rb);108 mempool_free(si, mddev->serial_info_pool);109 found = 1;110 break;111 }112 }113 if (!found)114 WARN(1, "The write IO is not recorded for serialization\n");115 spin_unlock_irqrestore(&serial->serial_lock, flags);116 wake_up(&serial->serial_io_wait);117}118 119/*120 * for resync bio, r1bio pointer can be retrieved from the per-bio121 * 'struct resync_pages'.122 */123static inline struct r1bio *get_resync_r1bio(struct bio *bio)124{125 return get_resync_pages(bio)->raid_bio;126}127 128static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)129{130 struct pool_info *pi = data;131 int size = offsetof(struct r1bio, bios[pi->raid_disks]);132 133 /* allocate a r1bio with room for raid_disks entries in the bios array */134 return kzalloc(size, gfp_flags);135}136 137#define RESYNC_DEPTH 32138#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)139#define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH)140#define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9)141#define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW)142#define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9)143 144static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)145{146 struct pool_info *pi = data;147 struct r1bio *r1_bio;148 struct bio *bio;149 int need_pages;150 int j;151 struct resync_pages *rps;152 153 r1_bio = r1bio_pool_alloc(gfp_flags, pi);154 if (!r1_bio)155 return NULL;156 157 rps = kmalloc_array(pi->raid_disks, sizeof(struct resync_pages),158 gfp_flags);159 if (!rps)160 goto out_free_r1bio;161 162 /*163 * Allocate bios : 1 for reading, n-1 for writing164 */165 for (j = pi->raid_disks ; j-- ; ) {166 bio = bio_kmalloc(RESYNC_PAGES, gfp_flags);167 if (!bio)168 goto out_free_bio;169 bio_init(bio, NULL, bio->bi_inline_vecs, RESYNC_PAGES, 0);170 r1_bio->bios[j] = bio;171 }172 /*173 * Allocate RESYNC_PAGES data pages and attach them to174 * the first bio.175 * If this is a user-requested check/repair, allocate176 * RESYNC_PAGES for each bio.177 */178 if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery))179 need_pages = pi->raid_disks;180 else181 need_pages = 1;182 for (j = 0; j < pi->raid_disks; j++) {183 struct resync_pages *rp = &rps[j];184 185 bio = r1_bio->bios[j];186 187 if (j < need_pages) {188 if (resync_alloc_pages(rp, gfp_flags))189 goto out_free_pages;190 } else {191 memcpy(rp, &rps[0], sizeof(*rp));192 resync_get_all_pages(rp);193 }194 195 rp->raid_bio = r1_bio;196 bio->bi_private = rp;197 }198 199 r1_bio->master_bio = NULL;200 201 return r1_bio;202 203out_free_pages:204 while (--j >= 0)205 resync_free_pages(&rps[j]);206 207out_free_bio:208 while (++j < pi->raid_disks) {209 bio_uninit(r1_bio->bios[j]);210 kfree(r1_bio->bios[j]);211 }212 kfree(rps);213 214out_free_r1bio:215 rbio_pool_free(r1_bio, data);216 return NULL;217}218 219static void r1buf_pool_free(void *__r1_bio, void *data)220{221 struct pool_info *pi = data;222 int i;223 struct r1bio *r1bio = __r1_bio;224 struct resync_pages *rp = NULL;225 226 for (i = pi->raid_disks; i--; ) {227 rp = get_resync_pages(r1bio->bios[i]);228 resync_free_pages(rp);229 bio_uninit(r1bio->bios[i]);230 kfree(r1bio->bios[i]);231 }232 233 /* resync pages array stored in the 1st bio's .bi_private */234 kfree(rp);235 236 rbio_pool_free(r1bio, data);237}238 239static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio)240{241 int i;242 243 for (i = 0; i < conf->raid_disks * 2; i++) {244 struct bio **bio = r1_bio->bios + i;245 if (!BIO_SPECIAL(*bio))246 bio_put(*bio);247 *bio = NULL;248 }249}250 251static void free_r1bio(struct r1bio *r1_bio)252{253 struct r1conf *conf = r1_bio->mddev->private;254 255 put_all_bios(conf, r1_bio);256 mempool_free(r1_bio, &conf->r1bio_pool);257}258 259static void put_buf(struct r1bio *r1_bio)260{261 struct r1conf *conf = r1_bio->mddev->private;262 sector_t sect = r1_bio->sector;263 int i;264 265 for (i = 0; i < conf->raid_disks * 2; i++) {266 struct bio *bio = r1_bio->bios[i];267 if (bio->bi_end_io)268 rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);269 }270 271 mempool_free(r1_bio, &conf->r1buf_pool);272 273 lower_barrier(conf, sect);274}275 276static void reschedule_retry(struct r1bio *r1_bio)277{278 unsigned long flags;279 struct mddev *mddev = r1_bio->mddev;280 struct r1conf *conf = mddev->private;281 int idx;282 283 idx = sector_to_idx(r1_bio->sector);284 spin_lock_irqsave(&conf->device_lock, flags);285 list_add(&r1_bio->retry_list, &conf->retry_list);286 atomic_inc(&conf->nr_queued[idx]);287 spin_unlock_irqrestore(&conf->device_lock, flags);288 289 wake_up(&conf->wait_barrier);290 md_wakeup_thread(mddev->thread);291}292 293/*294 * raid_end_bio_io() is called when we have finished servicing a mirrored295 * operation and are ready to return a success/failure code to the buffer296 * cache layer.297 */298static void call_bio_endio(struct r1bio *r1_bio)299{300 struct bio *bio = r1_bio->master_bio;301 302 if (!test_bit(R1BIO_Uptodate, &r1_bio->state))303 bio->bi_status = BLK_STS_IOERR;304 305 bio_endio(bio);306}307 308static void raid_end_bio_io(struct r1bio *r1_bio)309{310 struct bio *bio = r1_bio->master_bio;311 struct r1conf *conf = r1_bio->mddev->private;312 sector_t sector = r1_bio->sector;313 314 /* if nobody has done the final endio yet, do it now */315 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {316 pr_debug("raid1: sync end %s on sectors %llu-%llu\n",317 (bio_data_dir(bio) == WRITE) ? "write" : "read",318 (unsigned long long) bio->bi_iter.bi_sector,319 (unsigned long long) bio_end_sector(bio) - 1);320 321 call_bio_endio(r1_bio);322 }323 324 free_r1bio(r1_bio);325 /*326 * Wake up any possible resync thread that waits for the device327 * to go idle. All I/Os, even write-behind writes, are done.328 */329 allow_barrier(conf, sector);330}331 332/*333 * Update disk head position estimator based on IRQ completion info.334 */335static inline void update_head_pos(int disk, struct r1bio *r1_bio)336{337 struct r1conf *conf = r1_bio->mddev->private;338 339 conf->mirrors[disk].head_position =340 r1_bio->sector + (r1_bio->sectors);341}342 343/*344 * Find the disk number which triggered given bio345 */346static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio)347{348 int mirror;349 struct r1conf *conf = r1_bio->mddev->private;350 int raid_disks = conf->raid_disks;351 352 for (mirror = 0; mirror < raid_disks * 2; mirror++)353 if (r1_bio->bios[mirror] == bio)354 break;355 356 BUG_ON(mirror == raid_disks * 2);357 update_head_pos(mirror, r1_bio);358 359 return mirror;360}361 362static void raid1_end_read_request(struct bio *bio)363{364 int uptodate = !bio->bi_status;365 struct r1bio *r1_bio = bio->bi_private;366 struct r1conf *conf = r1_bio->mddev->private;367 struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev;368 369 /*370 * this branch is our 'one mirror IO has finished' event handler:371 */372 update_head_pos(r1_bio->read_disk, r1_bio);373 374 if (uptodate)375 set_bit(R1BIO_Uptodate, &r1_bio->state);376 else if (test_bit(FailFast, &rdev->flags) &&377 test_bit(R1BIO_FailFast, &r1_bio->state))378 /* This was a fail-fast read so we definitely379 * want to retry */380 ;381 else {382 /* If all other devices have failed, we want to return383 * the error upwards rather than fail the last device.384 * Here we redefine "uptodate" to mean "Don't want to retry"385 */386 unsigned long flags;387 spin_lock_irqsave(&conf->device_lock, flags);388 if (r1_bio->mddev->degraded == conf->raid_disks ||389 (r1_bio->mddev->degraded == conf->raid_disks-1 &&390 test_bit(In_sync, &rdev->flags)))391 uptodate = 1;392 spin_unlock_irqrestore(&conf->device_lock, flags);393 }394 395 if (uptodate) {396 raid_end_bio_io(r1_bio);397 rdev_dec_pending(rdev, conf->mddev);398 } else {399 /*400 * oops, read error:401 */402 pr_err_ratelimited("md/raid1:%s: %pg: rescheduling sector %llu\n",403 mdname(conf->mddev),404 rdev->bdev,405 (unsigned long long)r1_bio->sector);406 set_bit(R1BIO_ReadError, &r1_bio->state);407 reschedule_retry(r1_bio);408 /* don't drop the reference on read_disk yet */409 }410}411 412static void close_write(struct r1bio *r1_bio)413{414 struct mddev *mddev = r1_bio->mddev;415 416 /* it really is the end of this request */417 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {418 bio_free_pages(r1_bio->behind_master_bio);419 bio_put(r1_bio->behind_master_bio);420 r1_bio->behind_master_bio = NULL;421 }422 423 /* clear the bitmap if all writes complete successfully */424 mddev->bitmap_ops->endwrite(mddev, r1_bio->sector, r1_bio->sectors,425 !test_bit(R1BIO_Degraded, &r1_bio->state),426 test_bit(R1BIO_BehindIO, &r1_bio->state));427 md_write_end(mddev);428}429 430static void r1_bio_write_done(struct r1bio *r1_bio)431{432 if (!atomic_dec_and_test(&r1_bio->remaining))433 return;434 435 if (test_bit(R1BIO_WriteError, &r1_bio->state))436 reschedule_retry(r1_bio);437 else {438 close_write(r1_bio);439 if (test_bit(R1BIO_MadeGood, &r1_bio->state))440 reschedule_retry(r1_bio);441 else442 raid_end_bio_io(r1_bio);443 }444}445 446static void raid1_end_write_request(struct bio *bio)447{448 struct r1bio *r1_bio = bio->bi_private;449 int behind = test_bit(R1BIO_BehindIO, &r1_bio->state);450 struct r1conf *conf = r1_bio->mddev->private;451 struct bio *to_put = NULL;452 int mirror = find_bio_disk(r1_bio, bio);453 struct md_rdev *rdev = conf->mirrors[mirror].rdev;454 bool discard_error;455 sector_t lo = r1_bio->sector;456 sector_t hi = r1_bio->sector + r1_bio->sectors;457 458 discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD;459 460 /*461 * 'one mirror IO has finished' event handler:462 */463 if (bio->bi_status && !discard_error) {464 set_bit(WriteErrorSeen, &rdev->flags);465 if (!test_and_set_bit(WantReplacement, &rdev->flags))466 set_bit(MD_RECOVERY_NEEDED, &467 conf->mddev->recovery);468 469 if (test_bit(FailFast, &rdev->flags) &&470 (bio->bi_opf & MD_FAILFAST) &&471 /* We never try FailFast to WriteMostly devices */472 !test_bit(WriteMostly, &rdev->flags)) {473 md_error(r1_bio->mddev, rdev);474 }475 476 /*477 * When the device is faulty, it is not necessary to478 * handle write error.479 */480 if (!test_bit(Faulty, &rdev->flags))481 set_bit(R1BIO_WriteError, &r1_bio->state);482 else {483 /* Fail the request */484 set_bit(R1BIO_Degraded, &r1_bio->state);485 /* Finished with this branch */486 r1_bio->bios[mirror] = NULL;487 to_put = bio;488 }489 } else {490 /*491 * Set R1BIO_Uptodate in our master bio, so that we492 * will return a good error code for to the higher493 * levels even if IO on some other mirrored buffer494 * fails.495 *496 * The 'master' represents the composite IO operation497 * to user-side. So if something waits for IO, then it498 * will wait for the 'master' bio.499 */500 r1_bio->bios[mirror] = NULL;501 to_put = bio;502 /*503 * Do not set R1BIO_Uptodate if the current device is504 * rebuilding or Faulty. This is because we cannot use505 * such device for properly reading the data back (we could506 * potentially use it, if the current write would have felt507 * before rdev->recovery_offset, but for simplicity we don't508 * check this here.509 */510 if (test_bit(In_sync, &rdev->flags) &&511 !test_bit(Faulty, &rdev->flags))512 set_bit(R1BIO_Uptodate, &r1_bio->state);513 514 /* Maybe we can clear some bad blocks. */515 if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) &&516 !discard_error) {517 r1_bio->bios[mirror] = IO_MADE_GOOD;518 set_bit(R1BIO_MadeGood, &r1_bio->state);519 }520 }521 522 if (behind) {523 if (test_bit(CollisionCheck, &rdev->flags))524 remove_serial(rdev, lo, hi);525 if (test_bit(WriteMostly, &rdev->flags))526 atomic_dec(&r1_bio->behind_remaining);527 528 /*529 * In behind mode, we ACK the master bio once the I/O530 * has safely reached all non-writemostly531 * disks. Setting the Returned bit ensures that this532 * gets done only once -- we don't ever want to return533 * -EIO here, instead we'll wait534 */535 if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&536 test_bit(R1BIO_Uptodate, &r1_bio->state)) {537 /* Maybe we can return now */538 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {539 struct bio *mbio = r1_bio->master_bio;540 pr_debug("raid1: behind end write sectors"541 " %llu-%llu\n",542 (unsigned long long) mbio->bi_iter.bi_sector,543 (unsigned long long) bio_end_sector(mbio) - 1);544 call_bio_endio(r1_bio);545 }546 }547 } else if (rdev->mddev->serialize_policy)548 remove_serial(rdev, lo, hi);549 if (r1_bio->bios[mirror] == NULL)550 rdev_dec_pending(rdev, conf->mddev);551 552 /*553 * Let's see if all mirrored write operations have finished554 * already.555 */556 r1_bio_write_done(r1_bio);557 558 if (to_put)559 bio_put(to_put);560}561 562static sector_t align_to_barrier_unit_end(sector_t start_sector,563 sector_t sectors)564{565 sector_t len;566 567 WARN_ON(sectors == 0);568 /*569 * len is the number of sectors from start_sector to end of the570 * barrier unit which start_sector belongs to.571 */572 len = round_up(start_sector + 1, BARRIER_UNIT_SECTOR_SIZE) -573 start_sector;574 575 if (len > sectors)576 len = sectors;577 578 return len;579}580 581static void update_read_sectors(struct r1conf *conf, int disk,582 sector_t this_sector, int len)583{584 struct raid1_info *info = &conf->mirrors[disk];585 586 atomic_inc(&info->rdev->nr_pending);587 if (info->next_seq_sect != this_sector)588 info->seq_start = this_sector;589 info->next_seq_sect = this_sector + len;590}591 592static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio,593 int *max_sectors)594{595 sector_t this_sector = r1_bio->sector;596 int len = r1_bio->sectors;597 int disk;598 599 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {600 struct md_rdev *rdev;601 int read_len;602 603 if (r1_bio->bios[disk] == IO_BLOCKED)604 continue;605 606 rdev = conf->mirrors[disk].rdev;607 if (!rdev || test_bit(Faulty, &rdev->flags))608 continue;609 610 /* choose the first disk even if it has some bad blocks. */611 read_len = raid1_check_read_range(rdev, this_sector, &len);612 if (read_len > 0) {613 update_read_sectors(conf, disk, this_sector, read_len);614 *max_sectors = read_len;615 return disk;616 }617 }618 619 return -1;620}621 622static bool rdev_in_recovery(struct md_rdev *rdev, struct r1bio *r1_bio)623{624 return !test_bit(In_sync, &rdev->flags) &&625 rdev->recovery_offset < r1_bio->sector + r1_bio->sectors;626}627 628static int choose_bb_rdev(struct r1conf *conf, struct r1bio *r1_bio,629 int *max_sectors)630{631 sector_t this_sector = r1_bio->sector;632 int best_disk = -1;633 int best_len = 0;634 int disk;635 636 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {637 struct md_rdev *rdev;638 int len;639 int read_len;640 641 if (r1_bio->bios[disk] == IO_BLOCKED)642 continue;643 644 rdev = conf->mirrors[disk].rdev;645 if (!rdev || test_bit(Faulty, &rdev->flags) ||646 rdev_in_recovery(rdev, r1_bio) ||647 test_bit(WriteMostly, &rdev->flags))648 continue;649 650 /* keep track of the disk with the most readable sectors. */651 len = r1_bio->sectors;652 read_len = raid1_check_read_range(rdev, this_sector, &len);653 if (read_len > best_len) {654 best_disk = disk;655 best_len = read_len;656 }657 }658 659 if (best_disk != -1) {660 *max_sectors = best_len;661 update_read_sectors(conf, best_disk, this_sector, best_len);662 }663 664 return best_disk;665}666 667static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio,668 int *max_sectors)669{670 sector_t this_sector = r1_bio->sector;671 int bb_disk = -1;672 int bb_read_len = 0;673 int disk;674 675 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {676 struct md_rdev *rdev;677 int len;678 int read_len;679 680 if (r1_bio->bios[disk] == IO_BLOCKED)681 continue;682 683 rdev = conf->mirrors[disk].rdev;684 if (!rdev || test_bit(Faulty, &rdev->flags) ||685 !test_bit(WriteMostly, &rdev->flags) ||686 rdev_in_recovery(rdev, r1_bio))687 continue;688 689 /* there are no bad blocks, we can use this disk */690 len = r1_bio->sectors;691 read_len = raid1_check_read_range(rdev, this_sector, &len);692 if (read_len == r1_bio->sectors) {693 *max_sectors = read_len;694 update_read_sectors(conf, disk, this_sector, read_len);695 return disk;696 }697 698 /*699 * there are partial bad blocks, choose the rdev with largest700 * read length.701 */702 if (read_len > bb_read_len) {703 bb_disk = disk;704 bb_read_len = read_len;705 }706 }707 708 if (bb_disk != -1) {709 *max_sectors = bb_read_len;710 update_read_sectors(conf, bb_disk, this_sector, bb_read_len);711 }712 713 return bb_disk;714}715 716static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio)717{718 /* TODO: address issues with this check and concurrency. */719 return conf->mirrors[disk].next_seq_sect == r1_bio->sector ||720 conf->mirrors[disk].head_position == r1_bio->sector;721}722 723/*724 * If buffered sequential IO size exceeds optimal iosize, check if there is idle725 * disk. If yes, choose the idle disk.726 */727static bool should_choose_next(struct r1conf *conf, int disk)728{729 struct raid1_info *mirror = &conf->mirrors[disk];730 int opt_iosize;731 732 if (!test_bit(Nonrot, &mirror->rdev->flags))733 return false;734 735 opt_iosize = bdev_io_opt(mirror->rdev->bdev) >> 9;736 return opt_iosize > 0 && mirror->seq_start != MaxSector &&737 mirror->next_seq_sect > opt_iosize &&738 mirror->next_seq_sect - opt_iosize >= mirror->seq_start;739}740 741static bool rdev_readable(struct md_rdev *rdev, struct r1bio *r1_bio)742{743 if (!rdev || test_bit(Faulty, &rdev->flags))744 return false;745 746 if (rdev_in_recovery(rdev, r1_bio))747 return false;748 749 /* don't read from slow disk unless have to */750 if (test_bit(WriteMostly, &rdev->flags))751 return false;752 753 /* don't split IO for bad blocks unless have to */754 if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors))755 return false;756 757 return true;758}759 760struct read_balance_ctl {761 sector_t closest_dist;762 int closest_dist_disk;763 int min_pending;764 int min_pending_disk;765 int sequential_disk;766 int readable_disks;767};768 769static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio)770{771 int disk;772 struct read_balance_ctl ctl = {773 .closest_dist_disk = -1,774 .closest_dist = MaxSector,775 .min_pending_disk = -1,776 .min_pending = UINT_MAX,777 .sequential_disk = -1,778 };779 780 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {781 struct md_rdev *rdev;782 sector_t dist;783 unsigned int pending;784 785 if (r1_bio->bios[disk] == IO_BLOCKED)786 continue;787 788 rdev = conf->mirrors[disk].rdev;789 if (!rdev_readable(rdev, r1_bio))790 continue;791 792 /* At least two disks to choose from so failfast is OK */793 if (ctl.readable_disks++ == 1)794 set_bit(R1BIO_FailFast, &r1_bio->state);795 796 pending = atomic_read(&rdev->nr_pending);797 dist = abs(r1_bio->sector - conf->mirrors[disk].head_position);798 799 /* Don't change to another disk for sequential reads */800 if (is_sequential(conf, disk, r1_bio)) {801 if (!should_choose_next(conf, disk))802 return disk;803 804 /*805 * Add 'pending' to avoid choosing this disk if806 * there is other idle disk.807 */808 pending++;809 /*810 * If there is no other idle disk, this disk811 * will be chosen.812 */813 ctl.sequential_disk = disk;814 }815 816 if (ctl.min_pending > pending) {817 ctl.min_pending = pending;818 ctl.min_pending_disk = disk;819 }820 821 if (ctl.closest_dist > dist) {822 ctl.closest_dist = dist;823 ctl.closest_dist_disk = disk;824 }825 }826 827 /*828 * sequential IO size exceeds optimal iosize, however, there is no other829 * idle disk, so choose the sequential disk.830 */831 if (ctl.sequential_disk != -1 && ctl.min_pending != 0)832 return ctl.sequential_disk;833 834 /*835 * If all disks are rotational, choose the closest disk. If any disk is836 * non-rotational, choose the disk with less pending request even the837 * disk is rotational, which might/might not be optimal for raids with838 * mixed ratation/non-rotational disks depending on workload.839 */840 if (ctl.min_pending_disk != -1 &&841 (READ_ONCE(conf->nonrot_disks) || ctl.min_pending == 0))842 return ctl.min_pending_disk;843 else844 return ctl.closest_dist_disk;845}846 847/*848 * This routine returns the disk from which the requested read should be done.849 *850 * 1) If resync is in progress, find the first usable disk and use it even if it851 * has some bad blocks.852 *853 * 2) Now that there is no resync, loop through all disks and skipping slow854 * disks and disks with bad blocks for now. Only pay attention to key disk855 * choice.856 *857 * 3) If we've made it this far, now look for disks with bad blocks and choose858 * the one with most number of sectors.859 *860 * 4) If we are all the way at the end, we have no choice but to use a disk even861 * if it is write mostly.862 *863 * The rdev for the device selected will have nr_pending incremented.864 */865static int read_balance(struct r1conf *conf, struct r1bio *r1_bio,866 int *max_sectors)867{868 int disk;869 870 clear_bit(R1BIO_FailFast, &r1_bio->state);871 872 if (raid1_should_read_first(conf->mddev, r1_bio->sector,873 r1_bio->sectors))874 return choose_first_rdev(conf, r1_bio, max_sectors);875 876 disk = choose_best_rdev(conf, r1_bio);877 if (disk >= 0) {878 *max_sectors = r1_bio->sectors;879 update_read_sectors(conf, disk, r1_bio->sector,880 r1_bio->sectors);881 return disk;882 }883 884 /*885 * If we are here it means we didn't find a perfectly good disk so886 * now spend a bit more time trying to find one with the most good887 * sectors.888 */889 disk = choose_bb_rdev(conf, r1_bio, max_sectors);890 if (disk >= 0)891 return disk;892 893 return choose_slow_rdev(conf, r1_bio, max_sectors);894}895 896static void wake_up_barrier(struct r1conf *conf)897{898 if (wq_has_sleeper(&conf->wait_barrier))899 wake_up(&conf->wait_barrier);900}901 902static void flush_bio_list(struct r1conf *conf, struct bio *bio)903{904 /* flush any pending bitmap writes to disk before proceeding w/ I/O */905 raid1_prepare_flush_writes(conf->mddev);906 wake_up_barrier(conf);907 908 while (bio) { /* submit pending writes */909 struct bio *next = bio->bi_next;910 911 raid1_submit_write(bio);912 bio = next;913 cond_resched();914 }915}916 917static void flush_pending_writes(struct r1conf *conf)918{919 /* Any writes that have been queued but are awaiting920 * bitmap updates get flushed here.921 */922 spin_lock_irq(&conf->device_lock);923 924 if (conf->pending_bio_list.head) {925 struct blk_plug plug;926 struct bio *bio;927 928 bio = bio_list_get(&conf->pending_bio_list);929 spin_unlock_irq(&conf->device_lock);930 931 /*932 * As this is called in a wait_event() loop (see freeze_array),933 * current->state might be TASK_UNINTERRUPTIBLE which will934 * cause a warning when we prepare to wait again. As it is935 * rare that this path is taken, it is perfectly safe to force936 * us to go around the wait_event() loop again, so the warning937 * is a false-positive. Silence the warning by resetting938 * thread state939 */940 __set_current_state(TASK_RUNNING);941 blk_start_plug(&plug);942 flush_bio_list(conf, bio);943 blk_finish_plug(&plug);944 } else945 spin_unlock_irq(&conf->device_lock);946}947 948/* Barriers....949 * Sometimes we need to suspend IO while we do something else,950 * either some resync/recovery, or reconfigure the array.951 * To do this we raise a 'barrier'.952 * The 'barrier' is a counter that can be raised multiple times953 * to count how many activities are happening which preclude954 * normal IO.955 * We can only raise the barrier if there is no pending IO.956 * i.e. if nr_pending == 0.957 * We choose only to raise the barrier if no-one is waiting for the958 * barrier to go down. This means that as soon as an IO request959 * is ready, no other operations which require a barrier will start960 * until the IO request has had a chance.961 *962 * So: regular IO calls 'wait_barrier'. When that returns there963 * is no backgroup IO happening, It must arrange to call964 * allow_barrier when it has finished its IO.965 * backgroup IO calls must call raise_barrier. Once that returns966 * there is no normal IO happeing. It must arrange to call967 * lower_barrier when the particular background IO completes.968 *969 * If resync/recovery is interrupted, returns -EINTR;970 * Otherwise, returns 0.971 */972static int raise_barrier(struct r1conf *conf, sector_t sector_nr)973{974 int idx = sector_to_idx(sector_nr);975 976 spin_lock_irq(&conf->resync_lock);977 978 /* Wait until no block IO is waiting */979 wait_event_lock_irq(conf->wait_barrier,980 !atomic_read(&conf->nr_waiting[idx]),981 conf->resync_lock);982 983 /* block any new IO from starting */984 atomic_inc(&conf->barrier[idx]);985 /*986 * In raise_barrier() we firstly increase conf->barrier[idx] then987 * check conf->nr_pending[idx]. In _wait_barrier() we firstly988 * increase conf->nr_pending[idx] then check conf->barrier[idx].989 * A memory barrier here to make sure conf->nr_pending[idx] won't990 * be fetched before conf->barrier[idx] is increased. Otherwise991 * there will be a race between raise_barrier() and _wait_barrier().992 */993 smp_mb__after_atomic();994 995 /* For these conditions we must wait:996 * A: while the array is in frozen state997 * B: while conf->nr_pending[idx] is not 0, meaning regular I/O998 * existing in corresponding I/O barrier bucket.999 * C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches1000 * max resync count which allowed on current I/O barrier bucket.1001 */1002 wait_event_lock_irq(conf->wait_barrier,1003 (!conf->array_frozen &&1004 !atomic_read(&conf->nr_pending[idx]) &&1005 atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) ||1006 test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery),1007 conf->resync_lock);1008 1009 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {1010 atomic_dec(&conf->barrier[idx]);1011 spin_unlock_irq(&conf->resync_lock);1012 wake_up(&conf->wait_barrier);1013 return -EINTR;1014 }1015 1016 atomic_inc(&conf->nr_sync_pending);1017 spin_unlock_irq(&conf->resync_lock);1018 1019 return 0;1020}1021 1022static void lower_barrier(struct r1conf *conf, sector_t sector_nr)1023{1024 int idx = sector_to_idx(sector_nr);1025 1026 BUG_ON(atomic_read(&conf->barrier[idx]) <= 0);1027 1028 atomic_dec(&conf->barrier[idx]);1029 atomic_dec(&conf->nr_sync_pending);1030 wake_up(&conf->wait_barrier);1031}1032 1033static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait)1034{1035 bool ret = true;1036 1037 /*1038 * We need to increase conf->nr_pending[idx] very early here,1039 * then raise_barrier() can be blocked when it waits for1040 * conf->nr_pending[idx] to be 0. Then we can avoid holding1041 * conf->resync_lock when there is no barrier raised in same1042 * barrier unit bucket. Also if the array is frozen, I/O1043 * should be blocked until array is unfrozen.1044 */1045 atomic_inc(&conf->nr_pending[idx]);1046 /*1047 * In _wait_barrier() we firstly increase conf->nr_pending[idx], then1048 * check conf->barrier[idx]. In raise_barrier() we firstly increase1049 * conf->barrier[idx], then check conf->nr_pending[idx]. A memory1050 * barrier is necessary here to make sure conf->barrier[idx] won't be1051 * fetched before conf->nr_pending[idx] is increased. Otherwise there1052 * will be a race between _wait_barrier() and raise_barrier().1053 */1054 smp_mb__after_atomic();1055 1056 /*1057 * Don't worry about checking two atomic_t variables at same time1058 * here. If during we check conf->barrier[idx], the array is1059 * frozen (conf->array_frozen is 1), and chonf->barrier[idx] is1060 * 0, it is safe to return and make the I/O continue. Because the1061 * array is frozen, all I/O returned here will eventually complete1062 * or be queued, no race will happen. See code comment in1063 * frozen_array().1064 */1065 if (!READ_ONCE(conf->array_frozen) &&1066 !atomic_read(&conf->barrier[idx]))1067 return ret;1068 1069 /*1070 * After holding conf->resync_lock, conf->nr_pending[idx]1071 * should be decreased before waiting for barrier to drop.1072 * Otherwise, we may encounter a race condition because1073 * raise_barrer() might be waiting for conf->nr_pending[idx]1074 * to be 0 at same time.1075 */1076 spin_lock_irq(&conf->resync_lock);1077 atomic_inc(&conf->nr_waiting[idx]);1078 atomic_dec(&conf->nr_pending[idx]);1079 /*1080 * In case freeze_array() is waiting for1081 * get_unqueued_pending() == extra1082 */1083 wake_up_barrier(conf);1084 /* Wait for the barrier in same barrier unit bucket to drop. */1085 1086 /* Return false when nowait flag is set */1087 if (nowait) {1088 ret = false;1089 } else {1090 wait_event_lock_irq(conf->wait_barrier,1091 !conf->array_frozen &&1092 !atomic_read(&conf->barrier[idx]),1093 conf->resync_lock);1094 atomic_inc(&conf->nr_pending[idx]);1095 }1096 1097 atomic_dec(&conf->nr_waiting[idx]);1098 spin_unlock_irq(&conf->resync_lock);1099 return ret;1100}1101 1102static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)1103{1104 int idx = sector_to_idx(sector_nr);1105 bool ret = true;1106 1107 /*1108 * Very similar to _wait_barrier(). The difference is, for read1109 * I/O we don't need wait for sync I/O, but if the whole array1110 * is frozen, the read I/O still has to wait until the array is1111 * unfrozen. Since there is no ordering requirement with1112 * conf->barrier[idx] here, memory barrier is unnecessary as well.1113 */1114 atomic_inc(&conf->nr_pending[idx]);1115 1116 if (!READ_ONCE(conf->array_frozen))1117 return ret;1118 1119 spin_lock_irq(&conf->resync_lock);1120 atomic_inc(&conf->nr_waiting[idx]);1121 atomic_dec(&conf->nr_pending[idx]);1122 /*1123 * In case freeze_array() is waiting for1124 * get_unqueued_pending() == extra1125 */1126 wake_up_barrier(conf);1127 /* Wait for array to be unfrozen */1128 1129 /* Return false when nowait flag is set */1130 if (nowait) {1131 /* Return false when nowait flag is set */1132 ret = false;1133 } else {1134 wait_event_lock_irq(conf->wait_barrier,1135 !conf->array_frozen,1136 conf->resync_lock);1137 atomic_inc(&conf->nr_pending[idx]);1138 }1139 1140 atomic_dec(&conf->nr_waiting[idx]);1141 spin_unlock_irq(&conf->resync_lock);1142 return ret;1143}1144 1145static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait)1146{1147 int idx = sector_to_idx(sector_nr);1148 1149 return _wait_barrier(conf, idx, nowait);1150}1151 1152static void _allow_barrier(struct r1conf *conf, int idx)1153{1154 atomic_dec(&conf->nr_pending[idx]);1155 wake_up_barrier(conf);1156}1157 1158static void allow_barrier(struct r1conf *conf, sector_t sector_nr)1159{1160 int idx = sector_to_idx(sector_nr);1161 1162 _allow_barrier(conf, idx);1163}1164 1165/* conf->resync_lock should be held */1166static int get_unqueued_pending(struct r1conf *conf)1167{1168 int idx, ret;1169 1170 ret = atomic_read(&conf->nr_sync_pending);1171 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++)1172 ret += atomic_read(&conf->nr_pending[idx]) -1173 atomic_read(&conf->nr_queued[idx]);1174 1175 return ret;1176}1177 1178static void freeze_array(struct r1conf *conf, int extra)1179{1180 /* Stop sync I/O and normal I/O and wait for everything to1181 * go quiet.1182 * This is called in two situations:1183 * 1) management command handlers (reshape, remove disk, quiesce).1184 * 2) one normal I/O request failed.1185 1186 * After array_frozen is set to 1, new sync IO will be blocked at1187 * raise_barrier(), and new normal I/O will blocked at _wait_barrier()1188 * or wait_read_barrier(). The flying I/Os will either complete or be1189 * queued. When everything goes quite, there are only queued I/Os left.1190 1191 * Every flying I/O contributes to a conf->nr_pending[idx], idx is the1192 * barrier bucket index which this I/O request hits. When all sync and1193 * normal I/O are queued, sum of all conf->nr_pending[] will match sum1194 * of all conf->nr_queued[]. But normal I/O failure is an exception,1195 * in handle_read_error(), we may call freeze_array() before trying to1196 * fix the read error. In this case, the error read I/O is not queued,1197 * so get_unqueued_pending() == 1.1198 *1199 * Therefore before this function returns, we need to wait until1200 * get_unqueued_pendings(conf) gets equal to extra. For1201 * normal I/O context, extra is 1, in rested situations extra is 0.1202 */1203 spin_lock_irq(&conf->resync_lock);1204 conf->array_frozen = 1;1205 mddev_add_trace_msg(conf->mddev, "raid1 wait freeze");1206 wait_event_lock_irq_cmd(1207 conf->wait_barrier,1208 get_unqueued_pending(conf) == extra,1209 conf->resync_lock,1210 flush_pending_writes(conf));1211 spin_unlock_irq(&conf->resync_lock);1212}1213static void unfreeze_array(struct r1conf *conf)1214{1215 /* reverse the effect of the freeze */1216 spin_lock_irq(&conf->resync_lock);1217 conf->array_frozen = 0;1218 spin_unlock_irq(&conf->resync_lock);1219 wake_up(&conf->wait_barrier);1220}1221 1222static void alloc_behind_master_bio(struct r1bio *r1_bio,1223 struct bio *bio)1224{1225 int size = bio->bi_iter.bi_size;1226 unsigned vcnt = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;1227 int i = 0;1228 struct bio *behind_bio = NULL;1229 1230 behind_bio = bio_alloc_bioset(NULL, vcnt, 0, GFP_NOIO,1231 &r1_bio->mddev->bio_set);1232 1233 /* discard op, we don't support writezero/writesame yet */1234 if (!bio_has_data(bio)) {1235 behind_bio->bi_iter.bi_size = size;1236 goto skip_copy;1237 }1238 1239 while (i < vcnt && size) {1240 struct page *page;1241 int len = min_t(int, PAGE_SIZE, size);1242 1243 page = alloc_page(GFP_NOIO);1244 if (unlikely(!page))1245 goto free_pages;1246 1247 if (!bio_add_page(behind_bio, page, len, 0)) {1248 put_page(page);1249 goto free_pages;1250 }1251 1252 size -= len;1253 i++;1254 }1255 1256 bio_copy_data(behind_bio, bio);1257skip_copy:1258 r1_bio->behind_master_bio = behind_bio;1259 set_bit(R1BIO_BehindIO, &r1_bio->state);1260 1261 return;1262 1263free_pages:1264 pr_debug("%dB behind alloc failed, doing sync I/O\n",1265 bio->bi_iter.bi_size);1266 bio_free_pages(behind_bio);1267 bio_put(behind_bio);1268}1269 1270static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule)1271{1272 struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb,1273 cb);1274 struct mddev *mddev = plug->cb.data;1275 struct r1conf *conf = mddev->private;1276 struct bio *bio;1277 1278 if (from_schedule) {1279 spin_lock_irq(&conf->device_lock);1280 bio_list_merge(&conf->pending_bio_list, &plug->pending);1281 spin_unlock_irq(&conf->device_lock);1282 wake_up_barrier(conf);1283 md_wakeup_thread(mddev->thread);1284 kfree(plug);1285 return;1286 }1287 1288 /* we aren't scheduling, so we can do the write-out directly. */1289 bio = bio_list_get(&plug->pending);1290 flush_bio_list(conf, bio);1291 kfree(plug);1292}1293 1294static void init_r1bio(struct r1bio *r1_bio, struct mddev *mddev, struct bio *bio)1295{1296 r1_bio->master_bio = bio;1297 r1_bio->sectors = bio_sectors(bio);1298 r1_bio->state = 0;1299 r1_bio->mddev = mddev;1300 r1_bio->sector = bio->bi_iter.bi_sector;1301}1302 1303static inline struct r1bio *1304alloc_r1bio(struct mddev *mddev, struct bio *bio)1305{1306 struct r1conf *conf = mddev->private;1307 struct r1bio *r1_bio;1308 1309 r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO);1310 /* Ensure no bio records IO_BLOCKED */1311 memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0]));1312 init_r1bio(r1_bio, mddev, bio);1313 return r1_bio;1314}1315 1316static void raid1_read_request(struct mddev *mddev, struct bio *bio,1317 int max_read_sectors, struct r1bio *r1_bio)1318{1319 struct r1conf *conf = mddev->private;1320 struct raid1_info *mirror;1321 struct bio *read_bio;1322 const enum req_op op = bio_op(bio);1323 const blk_opf_t do_sync = bio->bi_opf & REQ_SYNC;1324 int max_sectors;1325 int rdisk;1326 bool r1bio_existed = !!r1_bio;1327 1328 /*1329 * If r1_bio is set, we are blocking the raid1d thread1330 * so there is a tiny risk of deadlock. So ask for1331 * emergency memory if needed.1332 */1333 gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO;1334 1335 /*1336 * Still need barrier for READ in case that whole1337 * array is frozen.1338 */1339 if (!wait_read_barrier(conf, bio->bi_iter.bi_sector,1340 bio->bi_opf & REQ_NOWAIT)) {1341 bio_wouldblock_error(bio);1342 return;1343 }1344 1345 if (!r1_bio)1346 r1_bio = alloc_r1bio(mddev, bio);1347 else1348 init_r1bio(r1_bio, mddev, bio);1349 r1_bio->sectors = max_read_sectors;1350 1351 /*1352 * make_request() can abort the operation when read-ahead is being1353 * used and no empty request is available.1354 */1355 rdisk = read_balance(conf, r1_bio, &max_sectors);1356 if (rdisk < 0) {1357 /* couldn't find anywhere to read from */1358 if (r1bio_existed)1359 pr_crit_ratelimited("md/raid1:%s: %pg: unrecoverable I/O read error for block %llu\n",1360 mdname(mddev),1361 conf->mirrors[r1_bio->read_disk].rdev->bdev,1362 r1_bio->sector);1363 raid_end_bio_io(r1_bio);1364 return;1365 }1366 mirror = conf->mirrors + rdisk;1367 1368 if (r1bio_existed)1369 pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %pg\n",1370 mdname(mddev),1371 (unsigned long long)r1_bio->sector,1372 mirror->rdev->bdev);1373 1374 if (test_bit(WriteMostly, &mirror->rdev->flags)) {1375 /*1376 * Reading from a write-mostly device must take care not to1377 * over-take any writes that are 'behind'1378 */1379 mddev_add_trace_msg(mddev, "raid1 wait behind writes");1380 mddev->bitmap_ops->wait_behind_writes(mddev);1381 }1382 1383 if (max_sectors < bio_sectors(bio)) {1384 struct bio *split = bio_split(bio, max_sectors,1385 gfp, &conf->bio_split);1386 bio_chain(split, bio);1387 submit_bio_noacct(bio);1388 bio = split;1389 r1_bio->master_bio = bio;1390 r1_bio->sectors = max_sectors;1391 }1392 1393 r1_bio->read_disk = rdisk;1394 if (!r1bio_existed) {1395 md_account_bio(mddev, &bio);1396 r1_bio->master_bio = bio;1397 }1398 read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp,1399 &mddev->bio_set);1400 1401 r1_bio->bios[rdisk] = read_bio;1402 1403 read_bio->bi_iter.bi_sector = r1_bio->sector +1404 mirror->rdev->data_offset;1405 read_bio->bi_end_io = raid1_end_read_request;1406 read_bio->bi_opf = op | do_sync;1407 if (test_bit(FailFast, &mirror->rdev->flags) &&1408 test_bit(R1BIO_FailFast, &r1_bio->state))1409 read_bio->bi_opf |= MD_FAILFAST;1410 read_bio->bi_private = r1_bio;1411 mddev_trace_remap(mddev, read_bio, r1_bio->sector);1412 submit_bio_noacct(read_bio);1413}1414 1415static void raid1_write_request(struct mddev *mddev, struct bio *bio,1416 int max_write_sectors)1417{1418 struct r1conf *conf = mddev->private;1419 struct r1bio *r1_bio;1420 int i, disks;1421 unsigned long flags;1422 struct md_rdev *blocked_rdev;1423 int first_clone;1424 int max_sectors;1425 bool write_behind = false;1426 bool is_discard = (bio_op(bio) == REQ_OP_DISCARD);1427 1428 if (mddev_is_clustered(mddev) &&1429 md_cluster_ops->area_resyncing(mddev, WRITE,1430 bio->bi_iter.bi_sector, bio_end_sector(bio))) {1431 1432 DEFINE_WAIT(w);1433 if (bio->bi_opf & REQ_NOWAIT) {1434 bio_wouldblock_error(bio);1435 return;1436 }1437 for (;;) {1438 prepare_to_wait(&conf->wait_barrier,1439 &w, TASK_IDLE);1440 if (!md_cluster_ops->area_resyncing(mddev, WRITE,1441 bio->bi_iter.bi_sector,1442 bio_end_sector(bio)))1443 break;1444 schedule();1445 }1446 finish_wait(&conf->wait_barrier, &w);1447 }1448 1449 /*1450 * Register the new request and wait if the reconstruction1451 * thread has put up a bar for new requests.1452 * Continue immediately if no resync is active currently.1453 */1454 if (!wait_barrier(conf, bio->bi_iter.bi_sector,1455 bio->bi_opf & REQ_NOWAIT)) {1456 bio_wouldblock_error(bio);1457 return;1458 }1459 1460 retry_write:1461 r1_bio = alloc_r1bio(mddev, bio);1462 r1_bio->sectors = max_write_sectors;1463 1464 /* first select target devices under rcu_lock and1465 * inc refcount on their rdev. Record them by setting1466 * bios[x] to bio1467 * If there are known/acknowledged bad blocks on any device on1468 * which we have seen a write error, we want to avoid writing those1469 * blocks.1470 * This potentially requires several writes to write around1471 * the bad blocks. Each set of writes gets it's own r1bio1472 * with a set of bios attached.1473 */1474 1475 disks = conf->raid_disks * 2;1476 blocked_rdev = NULL;1477 max_sectors = r1_bio->sectors;1478 for (i = 0; i < disks; i++) {1479 struct md_rdev *rdev = conf->mirrors[i].rdev;1480 1481 /*1482 * The write-behind io is only attempted on drives marked as1483 * write-mostly, which means we could allocate write behind1484 * bio later.1485 */1486 if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags))1487 write_behind = true;1488 1489 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {1490 atomic_inc(&rdev->nr_pending);1491 blocked_rdev = rdev;1492 break;1493 }1494 r1_bio->bios[i] = NULL;1495 if (!rdev || test_bit(Faulty, &rdev->flags)) {1496 if (i < conf->raid_disks)1497 set_bit(R1BIO_Degraded, &r1_bio->state);1498 continue;1499 }1500 1501 atomic_inc(&rdev->nr_pending);1502 if (test_bit(WriteErrorSeen, &rdev->flags)) {1503 sector_t first_bad;1504 int bad_sectors;1505 int is_bad;1506 1507 is_bad = is_badblock(rdev, r1_bio->sector, max_sectors,1508 &first_bad, &bad_sectors);1509 if (is_bad < 0) {1510 /* mustn't write here until the bad block is1511 * acknowledged*/1512 set_bit(BlockedBadBlocks, &rdev->flags);1513 blocked_rdev = rdev;1514 break;1515 }1516 if (is_bad && first_bad <= r1_bio->sector) {1517 /* Cannot write here at all */1518 bad_sectors -= (r1_bio->sector - first_bad);1519 if (bad_sectors < max_sectors)1520 /* mustn't write more than bad_sectors1521 * to other devices yet1522 */1523 max_sectors = bad_sectors;1524 rdev_dec_pending(rdev, mddev);1525 /* We don't set R1BIO_Degraded as that1526 * only applies if the disk is1527 * missing, so it might be re-added,1528 * and we want to know to recover this1529 * chunk.1530 * In this case the device is here,1531 * and the fact that this chunk is not1532 * in-sync is recorded in the bad1533 * block log1534 */1535 continue;1536 }1537 if (is_bad) {1538 int good_sectors = first_bad - r1_bio->sector;1539 if (good_sectors < max_sectors)1540 max_sectors = good_sectors;1541 }1542 }1543 r1_bio->bios[i] = bio;1544 }1545 1546 if (unlikely(blocked_rdev)) {1547 /* Wait for this device to become unblocked */1548 int j;1549 1550 for (j = 0; j < i; j++)1551 if (r1_bio->bios[j])1552 rdev_dec_pending(conf->mirrors[j].rdev, mddev);1553 mempool_free(r1_bio, &conf->r1bio_pool);1554 allow_barrier(conf, bio->bi_iter.bi_sector);1555 1556 if (bio->bi_opf & REQ_NOWAIT) {1557 bio_wouldblock_error(bio);1558 return;1559 }1560 mddev_add_trace_msg(mddev, "raid1 wait rdev %d blocked",1561 blocked_rdev->raid_disk);1562 md_wait_for_blocked_rdev(blocked_rdev, mddev);1563 wait_barrier(conf, bio->bi_iter.bi_sector, false);1564 goto retry_write;1565 }1566 1567 /*1568 * When using a bitmap, we may call alloc_behind_master_bio below.1569 * alloc_behind_master_bio allocates a copy of the data payload a page1570 * at a time and thus needs a new bio that can fit the whole payload1571 * this bio in page sized chunks.1572 */1573 if (write_behind && mddev->bitmap)1574 max_sectors = min_t(int, max_sectors,1575 BIO_MAX_VECS * (PAGE_SIZE >> 9));1576 if (max_sectors < bio_sectors(bio)) {1577 struct bio *split = bio_split(bio, max_sectors,1578 GFP_NOIO, &conf->bio_split);1579 bio_chain(split, bio);1580 submit_bio_noacct(bio);1581 bio = split;1582 r1_bio->master_bio = bio;1583 r1_bio->sectors = max_sectors;1584 }1585 1586 md_account_bio(mddev, &bio);1587 r1_bio->master_bio = bio;1588 atomic_set(&r1_bio->remaining, 1);1589 atomic_set(&r1_bio->behind_remaining, 0);1590 1591 first_clone = 1;1592 1593 for (i = 0; i < disks; i++) {1594 struct bio *mbio = NULL;1595 struct md_rdev *rdev = conf->mirrors[i].rdev;1596 if (!r1_bio->bios[i])1597 continue;1598 1599 if (first_clone) {1600 unsigned long max_write_behind =1601 mddev->bitmap_info.max_write_behind;1602 struct md_bitmap_stats stats;1603 int err;1604 1605 /* do behind I/O ?1606 * Not if there are too many, or cannot1607 * allocate memory, or a reader on WriteMostly1608 * is waiting for behind writes to flush */1609 err = mddev->bitmap_ops->get_stats(mddev->bitmap, &stats);1610 if (!err && write_behind && !stats.behind_wait &&1611 stats.behind_writes < max_write_behind)1612 alloc_behind_master_bio(r1_bio, bio);1613 1614 mddev->bitmap_ops->startwrite(1615 mddev, r1_bio->sector, r1_bio->sectors,1616 test_bit(R1BIO_BehindIO, &r1_bio->state));1617 first_clone = 0;1618 }1619 1620 if (r1_bio->behind_master_bio) {1621 mbio = bio_alloc_clone(rdev->bdev,1622 r1_bio->behind_master_bio,1623 GFP_NOIO, &mddev->bio_set);1624 if (test_bit(CollisionCheck, &rdev->flags))1625 wait_for_serialization(rdev, r1_bio);1626 if (test_bit(WriteMostly, &rdev->flags))1627 atomic_inc(&r1_bio->behind_remaining);1628 } else {1629 mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO,1630 &mddev->bio_set);1631 1632 if (mddev->serialize_policy)1633 wait_for_serialization(rdev, r1_bio);1634 }1635 1636 r1_bio->bios[i] = mbio;1637 1638 mbio->bi_iter.bi_sector = (r1_bio->sector + rdev->data_offset);1639 mbio->bi_end_io = raid1_end_write_request;1640 mbio->bi_opf = bio_op(bio) | (bio->bi_opf & (REQ_SYNC | REQ_FUA));1641 if (test_bit(FailFast, &rdev->flags) &&1642 !test_bit(WriteMostly, &rdev->flags) &&1643 conf->raid_disks - mddev->degraded > 1)1644 mbio->bi_opf |= MD_FAILFAST;1645 mbio->bi_private = r1_bio;1646 1647 atomic_inc(&r1_bio->remaining);1648 mddev_trace_remap(mddev, mbio, r1_bio->sector);1649 /* flush_pending_writes() needs access to the rdev so...*/1650 mbio->bi_bdev = (void *)rdev;1651 if (!raid1_add_bio_to_plug(mddev, mbio, raid1_unplug, disks)) {1652 spin_lock_irqsave(&conf->device_lock, flags);1653 bio_list_add(&conf->pending_bio_list, mbio);1654 spin_unlock_irqrestore(&conf->device_lock, flags);1655 md_wakeup_thread(mddev->thread);1656 }1657 }1658 1659 r1_bio_write_done(r1_bio);1660 1661 /* In case raid1d snuck in to freeze_array */1662 wake_up_barrier(conf);1663}1664 1665static bool raid1_make_request(struct mddev *mddev, struct bio *bio)1666{1667 sector_t sectors;1668 1669 if (unlikely(bio->bi_opf & REQ_PREFLUSH)1670 && md_flush_request(mddev, bio))1671 return true;1672 1673 /*1674 * There is a limit to the maximum size, but1675 * the read/write handler might find a lower limit1676 * due to bad blocks. To avoid multiple splits,1677 * we pass the maximum number of sectors down1678 * and let the lower level perform the split.1679 */1680 sectors = align_to_barrier_unit_end(1681 bio->bi_iter.bi_sector, bio_sectors(bio));1682 1683 if (bio_data_dir(bio) == READ)1684 raid1_read_request(mddev, bio, sectors, NULL);1685 else {1686 md_write_start(mddev,bio);1687 raid1_write_request(mddev, bio, sectors);1688 }1689 return true;1690}1691 1692static void raid1_status(struct seq_file *seq, struct mddev *mddev)1693{1694 struct r1conf *conf = mddev->private;1695 int i;1696 1697 lockdep_assert_held(&mddev->lock);1698 1699 seq_printf(seq, " [%d/%d] [", conf->raid_disks,1700 conf->raid_disks - mddev->degraded);1701 for (i = 0; i < conf->raid_disks; i++) {1702 struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev);1703 1704 seq_printf(seq, "%s",1705 rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");1706 }1707 seq_printf(seq, "]");1708}1709 1710/**1711 * raid1_error() - RAID1 error handler.1712 * @mddev: affected md device.1713 * @rdev: member device to fail.1714 *1715 * The routine acknowledges &rdev failure and determines new @mddev state.1716 * If it failed, then:1717 * - &MD_BROKEN flag is set in &mddev->flags.1718 * - recovery is disabled.1719 * Otherwise, it must be degraded:1720 * - recovery is interrupted.1721 * - &mddev->degraded is bumped.1722 *1723 * @rdev is marked as &Faulty excluding case when array is failed and1724 * &mddev->fail_last_dev is off.1725 */1726static void raid1_error(struct mddev *mddev, struct md_rdev *rdev)1727{1728 struct r1conf *conf = mddev->private;1729 unsigned long flags;1730 1731 spin_lock_irqsave(&conf->device_lock, flags);1732 1733 if (test_bit(In_sync, &rdev->flags) &&1734 (conf->raid_disks - mddev->degraded) == 1) {1735 set_bit(MD_BROKEN, &mddev->flags);1736 1737 if (!mddev->fail_last_dev) {1738 conf->recovery_disabled = mddev->recovery_disabled;1739 spin_unlock_irqrestore(&conf->device_lock, flags);1740 return;1741 }1742 }1743 set_bit(Blocked, &rdev->flags);1744 if (test_and_clear_bit(In_sync, &rdev->flags))1745 mddev->degraded++;1746 set_bit(Faulty, &rdev->flags);1747 spin_unlock_irqrestore(&conf->device_lock, flags);1748 /*1749 * if recovery is running, make sure it aborts.1750 */1751 set_bit(MD_RECOVERY_INTR, &mddev->recovery);1752 set_mask_bits(&mddev->sb_flags, 0,1753 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));1754 pr_crit("md/raid1:%s: Disk failure on %pg, disabling device.\n"1755 "md/raid1:%s: Operation continuing on %d devices.\n",1756 mdname(mddev), rdev->bdev,1757 mdname(mddev), conf->raid_disks - mddev->degraded);1758}1759 1760static void print_conf(struct r1conf *conf)1761{1762 int i;1763 1764 pr_debug("RAID1 conf printout:\n");1765 if (!conf) {1766 pr_debug("(!conf)\n");1767 return;1768 }1769 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,1770 conf->raid_disks);1771 1772 lockdep_assert_held(&conf->mddev->reconfig_mutex);1773 for (i = 0; i < conf->raid_disks; i++) {1774 struct md_rdev *rdev = conf->mirrors[i].rdev;1775 if (rdev)1776 pr_debug(" disk %d, wo:%d, o:%d, dev:%pg\n",1777 i, !test_bit(In_sync, &rdev->flags),1778 !test_bit(Faulty, &rdev->flags),1779 rdev->bdev);1780 }1781}1782 1783static void close_sync(struct r1conf *conf)1784{1785 int idx;1786 1787 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) {1788 _wait_barrier(conf, idx, false);1789 _allow_barrier(conf, idx);1790 }1791 1792 mempool_exit(&conf->r1buf_pool);1793}1794 1795static int raid1_spare_active(struct mddev *mddev)1796{1797 int i;1798 struct r1conf *conf = mddev->private;1799 int count = 0;1800 unsigned long flags;1801 1802 /*1803 * Find all failed disks within the RAID1 configuration1804 * and mark them readable.1805 * Called under mddev lock, so rcu protection not needed.1806 * device_lock used to avoid races with raid1_end_read_request1807 * which expects 'In_sync' flags and ->degraded to be consistent.1808 */1809 spin_lock_irqsave(&conf->device_lock, flags);1810 for (i = 0; i < conf->raid_disks; i++) {1811 struct md_rdev *rdev = conf->mirrors[i].rdev;1812 struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev;1813 if (repl1814 && !test_bit(Candidate, &repl->flags)1815 && repl->recovery_offset == MaxSector1816 && !test_bit(Faulty, &repl->flags)1817 && !test_and_set_bit(In_sync, &repl->flags)) {1818 /* replacement has just become active */1819 if (!rdev ||1820 !test_and_clear_bit(In_sync, &rdev->flags))1821 count++;1822 if (rdev) {1823 /* Replaced device not technically1824 * faulty, but we need to be sure1825 * it gets removed and never re-added1826 */1827 set_bit(Faulty, &rdev->flags);1828 sysfs_notify_dirent_safe(1829 rdev->sysfs_state);1830 }1831 }1832 if (rdev1833 && rdev->recovery_offset == MaxSector1834 && !test_bit(Faulty, &rdev->flags)1835 && !test_and_set_bit(In_sync, &rdev->flags)) {1836 count++;1837 sysfs_notify_dirent_safe(rdev->sysfs_state);1838 }1839 }1840 mddev->degraded -= count;1841 spin_unlock_irqrestore(&conf->device_lock, flags);1842 1843 print_conf(conf);1844 return count;1845}1846 1847static bool raid1_add_conf(struct r1conf *conf, struct md_rdev *rdev, int disk,1848 bool replacement)1849{1850 struct raid1_info *info = conf->mirrors + disk;1851 1852 if (replacement)1853 info += conf->raid_disks;1854 1855 if (info->rdev)1856 return false;1857 1858 if (bdev_nonrot(rdev->bdev)) {1859 set_bit(Nonrot, &rdev->flags);1860 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks + 1);1861 }1862 1863 rdev->raid_disk = disk;1864 info->head_position = 0;1865 info->seq_start = MaxSector;1866 WRITE_ONCE(info->rdev, rdev);1867 1868 return true;1869}1870 1871static bool raid1_remove_conf(struct r1conf *conf, int disk)1872{1873 struct raid1_info *info = conf->mirrors + disk;1874 struct md_rdev *rdev = info->rdev;1875 1876 if (!rdev || test_bit(In_sync, &rdev->flags) ||1877 atomic_read(&rdev->nr_pending))1878 return false;1879 1880 /* Only remove non-faulty devices if recovery is not possible. */1881 if (!test_bit(Faulty, &rdev->flags) &&1882 rdev->mddev->recovery_disabled != conf->recovery_disabled &&1883 rdev->mddev->degraded < conf->raid_disks)1884 return false;1885 1886 if (test_and_clear_bit(Nonrot, &rdev->flags))1887 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks - 1);1888 1889 WRITE_ONCE(info->rdev, NULL);1890 return true;1891}1892 1893static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)1894{1895 struct r1conf *conf = mddev->private;1896 int err = -EEXIST;1897 int mirror = 0, repl_slot = -1;1898 struct raid1_info *p;1899 int first = 0;1900 int last = conf->raid_disks - 1;1901 1902 if (mddev->recovery_disabled == conf->recovery_disabled)1903 return -EBUSY;1904 1905 if (rdev->raid_disk >= 0)1906 first = last = rdev->raid_disk;1907 1908 /*1909 * find the disk ... but prefer rdev->saved_raid_disk1910 * if possible.1911 */1912 if (rdev->saved_raid_disk >= 0 &&1913 rdev->saved_raid_disk >= first &&1914 rdev->saved_raid_disk < conf->raid_disks &&1915 conf->mirrors[rdev->saved_raid_disk].rdev == NULL)1916 first = last = rdev->saved_raid_disk;1917 1918 for (mirror = first; mirror <= last; mirror++) {1919 p = conf->mirrors + mirror;1920 if (!p->rdev) {1921 err = mddev_stack_new_rdev(mddev, rdev);1922 if (err)1923 return err;1924 1925 raid1_add_conf(conf, rdev, mirror, false);1926 /* As all devices are equivalent, we don't need a full recovery1927 * if this was recently any drive of the array1928 */1929 if (rdev->saved_raid_disk < 0)1930 conf->fullsync = 1;1931 break;1932 }1933 if (test_bit(WantReplacement, &p->rdev->flags) &&1934 p[conf->raid_disks].rdev == NULL && repl_slot < 0)1935 repl_slot = mirror;1936 }1937 1938 if (err && repl_slot >= 0) {1939 /* Add this device as a replacement */1940 clear_bit(In_sync, &rdev->flags);1941 set_bit(Replacement, &rdev->flags);1942 raid1_add_conf(conf, rdev, repl_slot, true);1943 err = 0;1944 conf->fullsync = 1;1945 }1946 1947 print_conf(conf);1948 return err;1949}1950 1951static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)1952{1953 struct r1conf *conf = mddev->private;1954 int err = 0;1955 int number = rdev->raid_disk;1956 struct raid1_info *p = conf->mirrors + number;1957 1958 if (unlikely(number >= conf->raid_disks))1959 goto abort;1960 1961 if (rdev != p->rdev) {1962 number += conf->raid_disks;1963 p = conf->mirrors + number;1964 }1965 1966 print_conf(conf);1967 if (rdev == p->rdev) {1968 if (!raid1_remove_conf(conf, number)) {1969 err = -EBUSY;1970 goto abort;1971 }1972 1973 if (number < conf->raid_disks &&1974 conf->mirrors[conf->raid_disks + number].rdev) {1975 /* We just removed a device that is being replaced.1976 * Move down the replacement. We drain all IO before1977 * doing this to avoid confusion.1978 */1979 struct md_rdev *repl =1980 conf->mirrors[conf->raid_disks + number].rdev;1981 freeze_array(conf, 0);1982 if (atomic_read(&repl->nr_pending)) {1983 /* It means that some queued IO of retry_list1984 * hold repl. Thus, we cannot set replacement1985 * as NULL, avoiding rdev NULL pointer1986 * dereference in sync_request_write and1987 * handle_write_finished.1988 */1989 err = -EBUSY;1990 unfreeze_array(conf);1991 goto abort;1992 }1993 clear_bit(Replacement, &repl->flags);1994 WRITE_ONCE(p->rdev, repl);1995 conf->mirrors[conf->raid_disks + number].rdev = NULL;1996 unfreeze_array(conf);1997 }1998 1999 clear_bit(WantReplacement, &rdev->flags);2000 err = md_integrity_register(mddev);2001 }2002abort:2003 2004 print_conf(conf);2005 return err;2006}2007 2008static void end_sync_read(struct bio *bio)2009{2010 struct r1bio *r1_bio = get_resync_r1bio(bio);2011 2012 update_head_pos(r1_bio->read_disk, r1_bio);2013 2014 /*2015 * we have read a block, now it needs to be re-written,2016 * or re-read if the read failed.2017 * We don't do much here, just schedule handling by raid1d2018 */2019 if (!bio->bi_status)2020 set_bit(R1BIO_Uptodate, &r1_bio->state);2021 2022 if (atomic_dec_and_test(&r1_bio->remaining))2023 reschedule_retry(r1_bio);2024}2025 2026static void abort_sync_write(struct mddev *mddev, struct r1bio *r1_bio)2027{2028 sector_t sync_blocks = 0;2029 sector_t s = r1_bio->sector;2030 long sectors_to_go = r1_bio->sectors;2031 2032 /* make sure these bits don't get cleared. */2033 do {2034 mddev->bitmap_ops->end_sync(mddev, s, &sync_blocks);2035 s += sync_blocks;2036 sectors_to_go -= sync_blocks;2037 } while (sectors_to_go > 0);2038}2039 2040static void put_sync_write_buf(struct r1bio *r1_bio, int uptodate)2041{2042 if (atomic_dec_and_test(&r1_bio->remaining)) {2043 struct mddev *mddev = r1_bio->mddev;2044 int s = r1_bio->sectors;2045 2046 if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||2047 test_bit(R1BIO_WriteError, &r1_bio->state))2048 reschedule_retry(r1_bio);2049 else {2050 put_buf(r1_bio);2051 md_done_sync(mddev, s, uptodate);2052 }2053 }2054}2055 2056static void end_sync_write(struct bio *bio)2057{2058 int uptodate = !bio->bi_status;2059 struct r1bio *r1_bio = get_resync_r1bio(bio);2060 struct mddev *mddev = r1_bio->mddev;2061 struct r1conf *conf = mddev->private;2062 struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev;2063 2064 if (!uptodate) {2065 abort_sync_write(mddev, r1_bio);2066 set_bit(WriteErrorSeen, &rdev->flags);2067 if (!test_and_set_bit(WantReplacement, &rdev->flags))2068 set_bit(MD_RECOVERY_NEEDED, &2069 mddev->recovery);2070 set_bit(R1BIO_WriteError, &r1_bio->state);2071 } else if (rdev_has_badblock(rdev, r1_bio->sector, r1_bio->sectors) &&2072 !rdev_has_badblock(conf->mirrors[r1_bio->read_disk].rdev,2073 r1_bio->sector, r1_bio->sectors)) {2074 set_bit(R1BIO_MadeGood, &r1_bio->state);2075 }2076 2077 put_sync_write_buf(r1_bio, uptodate);2078}2079 2080static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,2081 int sectors, struct page *page, blk_opf_t rw)2082{2083 if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))2084 /* success */2085 return 1;2086 if (rw == REQ_OP_WRITE) {2087 set_bit(WriteErrorSeen, &rdev->flags);2088 if (!test_and_set_bit(WantReplacement,2089 &rdev->flags))2090 set_bit(MD_RECOVERY_NEEDED, &2091 rdev->mddev->recovery);2092 }2093 /* need to record an error - either for the block or the device */2094 if (!rdev_set_badblocks(rdev, sector, sectors, 0))2095 md_error(rdev->mddev, rdev);2096 return 0;2097}2098 2099static int fix_sync_read_error(struct r1bio *r1_bio)2100{2101 /* Try some synchronous reads of other devices to get2102 * good data, much like with normal read errors. Only2103 * read into the pages we already have so we don't2104 * need to re-issue the read request.2105 * We don't need to freeze the array, because being in an2106 * active sync request, there is no normal IO, and2107 * no overlapping syncs.2108 * We don't need to check is_badblock() again as we2109 * made sure that anything with a bad block in range2110 * will have bi_end_io clear.2111 */2112 struct mddev *mddev = r1_bio->mddev;2113 struct r1conf *conf = mddev->private;2114 struct bio *bio = r1_bio->bios[r1_bio->read_disk];2115 struct page **pages = get_resync_pages(bio)->pages;2116 sector_t sect = r1_bio->sector;2117 int sectors = r1_bio->sectors;2118 int idx = 0;2119 struct md_rdev *rdev;2120 2121 rdev = conf->mirrors[r1_bio->read_disk].rdev;2122 if (test_bit(FailFast, &rdev->flags)) {2123 /* Don't try recovering from here - just fail it2124 * ... unless it is the last working device of course */2125 md_error(mddev, rdev);2126 if (test_bit(Faulty, &rdev->flags))2127 /* Don't try to read from here, but make sure2128 * put_buf does it's thing2129 */2130 bio->bi_end_io = end_sync_write;2131 }2132 2133 while(sectors) {2134 int s = sectors;2135 int d = r1_bio->read_disk;2136 int success = 0;2137 int start;2138 2139 if (s > (PAGE_SIZE>>9))2140 s = PAGE_SIZE >> 9;2141 do {2142 if (r1_bio->bios[d]->bi_end_io == end_sync_read) {2143 /* No rcu protection needed here devices2144 * can only be removed when no resync is2145 * active, and resync is currently active2146 */2147 rdev = conf->mirrors[d].rdev;2148 if (sync_page_io(rdev, sect, s<<9,2149 pages[idx],2150 REQ_OP_READ, false)) {2151 success = 1;2152 break;2153 }2154 }2155 d++;2156 if (d == conf->raid_disks * 2)2157 d = 0;2158 } while (!success && d != r1_bio->read_disk);2159 2160 if (!success) {2161 int abort = 0;2162 /* Cannot read from anywhere, this block is lost.2163 * Record a bad block on each device. If that doesn't2164 * work just disable and interrupt the recovery.2165 * Don't fail devices as that won't really help.2166 */2167 pr_crit_ratelimited("md/raid1:%s: %pg: unrecoverable I/O read error for block %llu\n",2168 mdname(mddev), bio->bi_bdev,2169 (unsigned long long)r1_bio->sector);2170 for (d = 0; d < conf->raid_disks * 2; d++) {2171 rdev = conf->mirrors[d].rdev;2172 if (!rdev || test_bit(Faulty, &rdev->flags))2173 continue;2174 if (!rdev_set_badblocks(rdev, sect, s, 0))2175 abort = 1;2176 }2177 if (abort) {2178 conf->recovery_disabled =2179 mddev->recovery_disabled;2180 set_bit(MD_RECOVERY_INTR, &mddev->recovery);2181 md_done_sync(mddev, r1_bio->sectors, 0);2182 put_buf(r1_bio);2183 return 0;2184 }2185 /* Try next page */2186 sectors -= s;2187 sect += s;2188 idx++;2189 continue;2190 }2191 2192 start = d;2193 /* write it back and re-read */2194 while (d != r1_bio->read_disk) {2195 if (d == 0)2196 d = conf->raid_disks * 2;2197 d--;2198 if (r1_bio->bios[d]->bi_end_io != end_sync_read)2199 continue;2200 rdev = conf->mirrors[d].rdev;2201 if (r1_sync_page_io(rdev, sect, s,2202 pages[idx],2203 REQ_OP_WRITE) == 0) {2204 r1_bio->bios[d]->bi_end_io = NULL;2205 rdev_dec_pending(rdev, mddev);2206 }2207 }2208 d = start;2209 while (d != r1_bio->read_disk) {2210 if (d == 0)2211 d = conf->raid_disks * 2;2212 d--;2213 if (r1_bio->bios[d]->bi_end_io != end_sync_read)2214 continue;2215 rdev = conf->mirrors[d].rdev;2216 if (r1_sync_page_io(rdev, sect, s,2217 pages[idx],2218 REQ_OP_READ) != 0)2219 atomic_add(s, &rdev->corrected_errors);2220 }2221 sectors -= s;2222 sect += s;2223 idx ++;2224 }2225 set_bit(R1BIO_Uptodate, &r1_bio->state);2226 bio->bi_status = 0;2227 return 1;2228}2229 2230static void process_checks(struct r1bio *r1_bio)2231{2232 /* We have read all readable devices. If we haven't2233 * got the block, then there is no hope left.2234 * If we have, then we want to do a comparison2235 * and skip the write if everything is the same.2236 * If any blocks failed to read, then we need to2237 * attempt an over-write2238 */2239 struct mddev *mddev = r1_bio->mddev;2240 struct r1conf *conf = mddev->private;2241 int primary;2242 int i;2243 int vcnt;2244 2245 /* Fix variable parts of all bios */2246 vcnt = (r1_bio->sectors + PAGE_SIZE / 512 - 1) >> (PAGE_SHIFT - 9);2247 for (i = 0; i < conf->raid_disks * 2; i++) {2248 blk_status_t status;2249 struct bio *b = r1_bio->bios[i];2250 struct resync_pages *rp = get_resync_pages(b);2251 if (b->bi_end_io != end_sync_read)2252 continue;2253 /* fixup the bio for reuse, but preserve errno */2254 status = b->bi_status;2255 bio_reset(b, conf->mirrors[i].rdev->bdev, REQ_OP_READ);2256 b->bi_status = status;2257 b->bi_iter.bi_sector = r1_bio->sector +2258 conf->mirrors[i].rdev->data_offset;2259 b->bi_end_io = end_sync_read;2260 rp->raid_bio = r1_bio;2261 b->bi_private = rp;2262 2263 /* initialize bvec table again */2264 md_bio_reset_resync_pages(b, rp, r1_bio->sectors << 9);2265 }2266 for (primary = 0; primary < conf->raid_disks * 2; primary++)2267 if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&2268 !r1_bio->bios[primary]->bi_status) {2269 r1_bio->bios[primary]->bi_end_io = NULL;2270 rdev_dec_pending(conf->mirrors[primary].rdev, mddev);2271 break;2272 }2273 r1_bio->read_disk = primary;2274 for (i = 0; i < conf->raid_disks * 2; i++) {2275 int j = 0;2276 struct bio *pbio = r1_bio->bios[primary];2277 struct bio *sbio = r1_bio->bios[i];2278 blk_status_t status = sbio->bi_status;2279 struct page **ppages = get_resync_pages(pbio)->pages;2280 struct page **spages = get_resync_pages(sbio)->pages;2281 struct bio_vec *bi;2282 int page_len[RESYNC_PAGES] = { 0 };2283 struct bvec_iter_all iter_all;2284 2285 if (sbio->bi_end_io != end_sync_read)2286 continue;2287 /* Now we can 'fixup' the error value */2288 sbio->bi_status = 0;2289 2290 bio_for_each_segment_all(bi, sbio, iter_all)2291 page_len[j++] = bi->bv_len;2292 2293 if (!status) {2294 for (j = vcnt; j-- ; ) {2295 if (memcmp(page_address(ppages[j]),2296 page_address(spages[j]),2297 page_len[j]))2298 break;2299 }2300 } else2301 j = 0;2302 if (j >= 0)2303 atomic64_add(r1_bio->sectors, &mddev->resync_mismatches);2304 if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)2305 && !status)) {2306 /* No need to write to this device. */2307 sbio->bi_end_io = NULL;2308 rdev_dec_pending(conf->mirrors[i].rdev, mddev);2309 continue;2310 }2311 2312 bio_copy_data(sbio, pbio);2313 }2314}2315 2316static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio)2317{2318 struct r1conf *conf = mddev->private;2319 int i;2320 int disks = conf->raid_disks * 2;2321 struct bio *wbio;2322 2323 if (!test_bit(R1BIO_Uptodate, &r1_bio->state))2324 /* ouch - failed to read all of that. */2325 if (!fix_sync_read_error(r1_bio))2326 return;2327 2328 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))2329 process_checks(r1_bio);2330 2331 /*2332 * schedule writes2333 */2334 atomic_set(&r1_bio->remaining, 1);2335 for (i = 0; i < disks ; i++) {2336 wbio = r1_bio->bios[i];2337 if (wbio->bi_end_io == NULL ||2338 (wbio->bi_end_io == end_sync_read &&2339 (i == r1_bio->read_disk ||2340 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))2341 continue;2342 if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) {2343 abort_sync_write(mddev, r1_bio);2344 continue;2345 }2346 2347 wbio->bi_opf = REQ_OP_WRITE;2348 if (test_bit(FailFast, &conf->mirrors[i].rdev->flags))2349 wbio->bi_opf |= MD_FAILFAST;2350 2351 wbio->bi_end_io = end_sync_write;2352 atomic_inc(&r1_bio->remaining);2353 md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio));2354 2355 submit_bio_noacct(wbio);2356 }2357 2358 put_sync_write_buf(r1_bio, 1);2359}2360 2361/*2362 * This is a kernel thread which:2363 *2364 * 1. Retries failed read operations on working mirrors.2365 * 2. Updates the raid superblock when problems encounter.2366 * 3. Performs writes following reads for array synchronising.2367 */2368 2369static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio)2370{2371 sector_t sect = r1_bio->sector;2372 int sectors = r1_bio->sectors;2373 int read_disk = r1_bio->read_disk;2374 struct mddev *mddev = conf->mddev;2375 struct md_rdev *rdev = conf->mirrors[read_disk].rdev;2376 2377 if (exceed_read_errors(mddev, rdev)) {2378 r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;2379 return;2380 }2381 2382 while(sectors) {2383 int s = sectors;2384 int d = read_disk;2385 int success = 0;2386 int start;2387 2388 if (s > (PAGE_SIZE>>9))2389 s = PAGE_SIZE >> 9;2390 2391 do {2392 rdev = conf->mirrors[d].rdev;2393 if (rdev &&2394 (test_bit(In_sync, &rdev->flags) ||2395 (!test_bit(Faulty, &rdev->flags) &&2396 rdev->recovery_offset >= sect + s)) &&2397 rdev_has_badblock(rdev, sect, s) == 0) {2398 atomic_inc(&rdev->nr_pending);2399 if (sync_page_io(rdev, sect, s<<9,2400 conf->tmppage, REQ_OP_READ, false))2401 success = 1;2402 rdev_dec_pending(rdev, mddev);2403 if (success)2404 break;2405 }2406 2407 d++;2408 if (d == conf->raid_disks * 2)2409 d = 0;2410 } while (d != read_disk);2411 2412 if (!success) {2413 /* Cannot read from anywhere - mark it bad */2414 struct md_rdev *rdev = conf->mirrors[read_disk].rdev;2415 if (!rdev_set_badblocks(rdev, sect, s, 0))2416 md_error(mddev, rdev);2417 break;2418 }2419 /* write it back and re-read */2420 start = d;2421 while (d != read_disk) {2422 if (d==0)2423 d = conf->raid_disks * 2;2424 d--;2425 rdev = conf->mirrors[d].rdev;2426 if (rdev &&2427 !test_bit(Faulty, &rdev->flags)) {2428 atomic_inc(&rdev->nr_pending);2429 r1_sync_page_io(rdev, sect, s,2430 conf->tmppage, REQ_OP_WRITE);2431 rdev_dec_pending(rdev, mddev);2432 }2433 }2434 d = start;2435 while (d != read_disk) {2436 if (d==0)2437 d = conf->raid_disks * 2;2438 d--;2439 rdev = conf->mirrors[d].rdev;2440 if (rdev &&2441 !test_bit(Faulty, &rdev->flags)) {2442 atomic_inc(&rdev->nr_pending);2443 if (r1_sync_page_io(rdev, sect, s,2444 conf->tmppage, REQ_OP_READ)) {2445 atomic_add(s, &rdev->corrected_errors);2446 pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %pg)\n",2447 mdname(mddev), s,2448 (unsigned long long)(sect +2449 rdev->data_offset),2450 rdev->bdev);2451 }2452 rdev_dec_pending(rdev, mddev);2453 }2454 }2455 sectors -= s;2456 sect += s;2457 }2458}2459 2460static int narrow_write_error(struct r1bio *r1_bio, int i)2461{2462 struct mddev *mddev = r1_bio->mddev;2463 struct r1conf *conf = mddev->private;2464 struct md_rdev *rdev = conf->mirrors[i].rdev;2465 2466 /* bio has the data to be written to device 'i' where2467 * we just recently had a write error.2468 * We repeatedly clone the bio and trim down to one block,2469 * then try the write. Where the write fails we record2470 * a bad block.2471 * It is conceivable that the bio doesn't exactly align with2472 * blocks. We must handle this somehow.2473 *2474 * We currently own a reference on the rdev.2475 */2476 2477 int block_sectors;2478 sector_t sector;2479 int sectors;2480 int sect_to_write = r1_bio->sectors;2481 int ok = 1;2482 2483 if (rdev->badblocks.shift < 0)2484 return 0;2485 2486 block_sectors = roundup(1 << rdev->badblocks.shift,2487 bdev_logical_block_size(rdev->bdev) >> 9);2488 sector = r1_bio->sector;2489 sectors = ((sector + block_sectors)2490 & ~(sector_t)(block_sectors - 1))2491 - sector;2492 2493 while (sect_to_write) {2494 struct bio *wbio;2495 if (sectors > sect_to_write)2496 sectors = sect_to_write;2497 /* Write at 'sector' for 'sectors'*/2498 2499 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {2500 wbio = bio_alloc_clone(rdev->bdev,2501 r1_bio->behind_master_bio,2502 GFP_NOIO, &mddev->bio_set);2503 } else {2504 wbio = bio_alloc_clone(rdev->bdev, r1_bio->master_bio,2505 GFP_NOIO, &mddev->bio_set);2506 }2507 2508 wbio->bi_opf = REQ_OP_WRITE;2509 wbio->bi_iter.bi_sector = r1_bio->sector;2510 wbio->bi_iter.bi_size = r1_bio->sectors << 9;2511 2512 bio_trim(wbio, sector - r1_bio->sector, sectors);2513 wbio->bi_iter.bi_sector += rdev->data_offset;2514 2515 if (submit_bio_wait(wbio) < 0)2516 /* failure! */2517 ok = rdev_set_badblocks(rdev, sector,2518 sectors, 0)2519 && ok;2520 2521 bio_put(wbio);2522 sect_to_write -= sectors;2523 sector += sectors;2524 sectors = block_sectors;2525 }2526 return ok;2527}2528 2529static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio)2530{2531 int m;2532 int s = r1_bio->sectors;2533 for (m = 0; m < conf->raid_disks * 2 ; m++) {2534 struct md_rdev *rdev = conf->mirrors[m].rdev;2535 struct bio *bio = r1_bio->bios[m];2536 if (bio->bi_end_io == NULL)2537 continue;2538 if (!bio->bi_status &&2539 test_bit(R1BIO_MadeGood, &r1_bio->state)) {2540 rdev_clear_badblocks(rdev, r1_bio->sector, s, 0);2541 }2542 if (bio->bi_status &&2543 test_bit(R1BIO_WriteError, &r1_bio->state)) {2544 if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0))2545 md_error(conf->mddev, rdev);2546 }2547 }2548 put_buf(r1_bio);2549 md_done_sync(conf->mddev, s, 1);2550}2551 2552static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio)2553{2554 int m, idx;2555 bool fail = false;2556 2557 for (m = 0; m < conf->raid_disks * 2 ; m++)2558 if (r1_bio->bios[m] == IO_MADE_GOOD) {2559 struct md_rdev *rdev = conf->mirrors[m].rdev;2560 rdev_clear_badblocks(rdev,2561 r1_bio->sector,2562 r1_bio->sectors, 0);2563 rdev_dec_pending(rdev, conf->mddev);2564 } else if (r1_bio->bios[m] != NULL) {2565 /* This drive got a write error. We need to2566 * narrow down and record precise write2567 * errors.2568 */2569 fail = true;2570 if (!narrow_write_error(r1_bio, m)) {2571 md_error(conf->mddev,2572 conf->mirrors[m].rdev);2573 /* an I/O failed, we can't clear the bitmap */2574 set_bit(R1BIO_Degraded, &r1_bio->state);2575 }2576 rdev_dec_pending(conf->mirrors[m].rdev,2577 conf->mddev);2578 }2579 if (fail) {2580 spin_lock_irq(&conf->device_lock);2581 list_add(&r1_bio->retry_list, &conf->bio_end_io_list);2582 idx = sector_to_idx(r1_bio->sector);2583 atomic_inc(&conf->nr_queued[idx]);2584 spin_unlock_irq(&conf->device_lock);2585 /*2586 * In case freeze_array() is waiting for condition2587 * get_unqueued_pending() == extra to be true.2588 */2589 wake_up(&conf->wait_barrier);2590 md_wakeup_thread(conf->mddev->thread);2591 } else {2592 if (test_bit(R1BIO_WriteError, &r1_bio->state))2593 close_write(r1_bio);2594 raid_end_bio_io(r1_bio);2595 }2596}2597 2598static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)2599{2600 struct mddev *mddev = conf->mddev;2601 struct bio *bio;2602 struct md_rdev *rdev;2603 sector_t sector;2604 2605 clear_bit(R1BIO_ReadError, &r1_bio->state);2606 /* we got a read error. Maybe the drive is bad. Maybe just2607 * the block and we can fix it.2608 * We freeze all other IO, and try reading the block from2609 * other devices. When we find one, we re-write2610 * and check it that fixes the read error.2611 * This is all done synchronously while the array is2612 * frozen2613 */2614 2615 bio = r1_bio->bios[r1_bio->read_disk];2616 bio_put(bio);2617 r1_bio->bios[r1_bio->read_disk] = NULL;2618 2619 rdev = conf->mirrors[r1_bio->read_disk].rdev;2620 if (mddev->ro == 02621 && !test_bit(FailFast, &rdev->flags)) {2622 freeze_array(conf, 1);2623 fix_read_error(conf, r1_bio);2624 unfreeze_array(conf);2625 } else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) {2626 md_error(mddev, rdev);2627 } else {2628 r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED;2629 }2630 2631 rdev_dec_pending(rdev, conf->mddev);2632 sector = r1_bio->sector;2633 bio = r1_bio->master_bio;2634 2635 /* Reuse the old r1_bio so that the IO_BLOCKED settings are preserved */2636 r1_bio->state = 0;2637 raid1_read_request(mddev, bio, r1_bio->sectors, r1_bio);2638 allow_barrier(conf, sector);2639}2640 2641static void raid1d(struct md_thread *thread)2642{2643 struct mddev *mddev = thread->mddev;2644 struct r1bio *r1_bio;2645 unsigned long flags;2646 struct r1conf *conf = mddev->private;2647 struct list_head *head = &conf->retry_list;2648 struct blk_plug plug;2649 int idx;2650 2651 md_check_recovery(mddev);2652 2653 if (!list_empty_careful(&conf->bio_end_io_list) &&2654 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {2655 LIST_HEAD(tmp);2656 spin_lock_irqsave(&conf->device_lock, flags);2657 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))2658 list_splice_init(&conf->bio_end_io_list, &tmp);2659 spin_unlock_irqrestore(&conf->device_lock, flags);2660 while (!list_empty(&tmp)) {2661 r1_bio = list_first_entry(&tmp, struct r1bio,2662 retry_list);2663 list_del(&r1_bio->retry_list);2664 idx = sector_to_idx(r1_bio->sector);2665 atomic_dec(&conf->nr_queued[idx]);2666 if (mddev->degraded)2667 set_bit(R1BIO_Degraded, &r1_bio->state);2668 if (test_bit(R1BIO_WriteError, &r1_bio->state))2669 close_write(r1_bio);2670 raid_end_bio_io(r1_bio);2671 }2672 }2673 2674 blk_start_plug(&plug);2675 for (;;) {2676 2677 flush_pending_writes(conf);2678 2679 spin_lock_irqsave(&conf->device_lock, flags);2680 if (list_empty(head)) {2681 spin_unlock_irqrestore(&conf->device_lock, flags);2682 break;2683 }2684 r1_bio = list_entry(head->prev, struct r1bio, retry_list);2685 list_del(head->prev);2686 idx = sector_to_idx(r1_bio->sector);2687 atomic_dec(&conf->nr_queued[idx]);2688 spin_unlock_irqrestore(&conf->device_lock, flags);2689 2690 mddev = r1_bio->mddev;2691 conf = mddev->private;2692 if (test_bit(R1BIO_IsSync, &r1_bio->state)) {2693 if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||2694 test_bit(R1BIO_WriteError, &r1_bio->state))2695 handle_sync_write_finished(conf, r1_bio);2696 else2697 sync_request_write(mddev, r1_bio);2698 } else if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||2699 test_bit(R1BIO_WriteError, &r1_bio->state))2700 handle_write_finished(conf, r1_bio);2701 else if (test_bit(R1BIO_ReadError, &r1_bio->state))2702 handle_read_error(conf, r1_bio);2703 else2704 WARN_ON_ONCE(1);2705 2706 cond_resched();2707 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))2708 md_check_recovery(mddev);2709 }2710 blk_finish_plug(&plug);2711}2712 2713static int init_resync(struct r1conf *conf)2714{2715 int buffs;2716 2717 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;2718 BUG_ON(mempool_initialized(&conf->r1buf_pool));2719 2720 return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc,2721 r1buf_pool_free, conf->poolinfo);2722}2723 2724static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf)2725{2726 struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO);2727 struct resync_pages *rps;2728 struct bio *bio;2729 int i;2730 2731 for (i = conf->poolinfo->raid_disks; i--; ) {2732 bio = r1bio->bios[i];2733 rps = bio->bi_private;2734 bio_reset(bio, NULL, 0);2735 bio->bi_private = rps;2736 }2737 r1bio->master_bio = NULL;2738 return r1bio;2739}2740 2741/*2742 * perform a "sync" on one "block"2743 *2744 * We need to make sure that no normal I/O request - particularly write2745 * requests - conflict with active sync requests.2746 *2747 * This is achieved by tracking pending requests and a 'barrier' concept2748 * that can be installed to exclude normal IO requests.2749 */2750 2751static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,2752 sector_t max_sector, int *skipped)2753{2754 struct r1conf *conf = mddev->private;2755 struct r1bio *r1_bio;2756 struct bio *bio;2757 sector_t nr_sectors;2758 int disk = -1;2759 int i;2760 int wonly = -1;2761 int write_targets = 0, read_targets = 0;2762 sector_t sync_blocks;2763 bool still_degraded = false;2764 int good_sectors = RESYNC_SECTORS;2765 int min_bad = 0; /* number of sectors that are bad in all devices */2766 int idx = sector_to_idx(sector_nr);2767 int page_idx = 0;2768 2769 if (!mempool_initialized(&conf->r1buf_pool))2770 if (init_resync(conf))2771 return 0;2772 2773 if (sector_nr >= max_sector) {2774 /* If we aborted, we need to abort the2775 * sync on the 'current' bitmap chunk (there will2776 * only be one in raid1 resync.2777 * We can find the current addess in mddev->curr_resync2778 */2779 if (mddev->curr_resync < max_sector) /* aborted */2780 mddev->bitmap_ops->end_sync(mddev, mddev->curr_resync,2781 &sync_blocks);2782 else /* completed sync */2783 conf->fullsync = 0;2784 2785 mddev->bitmap_ops->close_sync(mddev);2786 close_sync(conf);2787 2788 if (mddev_is_clustered(mddev)) {2789 conf->cluster_sync_low = 0;2790 conf->cluster_sync_high = 0;2791 }2792 return 0;2793 }2794 2795 if (mddev->bitmap == NULL &&2796 mddev->recovery_cp == MaxSector &&2797 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&2798 conf->fullsync == 0) {2799 *skipped = 1;2800 return max_sector - sector_nr;2801 }2802 /* before building a request, check if we can skip these blocks..2803 * This call the bitmap_start_sync doesn't actually record anything2804 */2805 if (!mddev->bitmap_ops->start_sync(mddev, sector_nr, &sync_blocks, true) &&2806 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {2807 /* We can skip this block, and probably several more */2808 *skipped = 1;2809 return sync_blocks;2810 }2811 2812 /*2813 * If there is non-resync activity waiting for a turn, then let it2814 * though before starting on this new sync request.2815 */2816 if (atomic_read(&conf->nr_waiting[idx]))2817 schedule_timeout_uninterruptible(1);2818 2819 /* we are incrementing sector_nr below. To be safe, we check against2820 * sector_nr + two times RESYNC_SECTORS2821 */2822 2823 mddev->bitmap_ops->cond_end_sync(mddev, sector_nr,2824 mddev_is_clustered(mddev) &&2825 (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high));2826 2827 if (raise_barrier(conf, sector_nr))2828 return 0;2829 2830 r1_bio = raid1_alloc_init_r1buf(conf);2831 2832 /*2833 * If we get a correctably read error during resync or recovery,2834 * we might want to read from a different device. So we2835 * flag all drives that could conceivably be read from for READ,2836 * and any others (which will be non-In_sync devices) for WRITE.2837 * If a read fails, we try reading from something else for which READ2838 * is OK.2839 */2840 2841 r1_bio->mddev = mddev;2842 r1_bio->sector = sector_nr;2843 r1_bio->state = 0;2844 set_bit(R1BIO_IsSync, &r1_bio->state);2845 /* make sure good_sectors won't go across barrier unit boundary */2846 good_sectors = align_to_barrier_unit_end(sector_nr, good_sectors);2847 2848 for (i = 0; i < conf->raid_disks * 2; i++) {2849 struct md_rdev *rdev;2850 bio = r1_bio->bios[i];2851 2852 rdev = conf->mirrors[i].rdev;2853 if (rdev == NULL ||2854 test_bit(Faulty, &rdev->flags)) {2855 if (i < conf->raid_disks)2856 still_degraded = true;2857 } else if (!test_bit(In_sync, &rdev->flags)) {2858 bio->bi_opf = REQ_OP_WRITE;2859 bio->bi_end_io = end_sync_write;2860 write_targets ++;2861 } else {2862 /* may need to read from here */2863 sector_t first_bad = MaxSector;2864 int bad_sectors;2865 2866 if (is_badblock(rdev, sector_nr, good_sectors,2867 &first_bad, &bad_sectors)) {2868 if (first_bad > sector_nr)2869 good_sectors = first_bad - sector_nr;2870 else {2871 bad_sectors -= (sector_nr - first_bad);2872 if (min_bad == 0 ||2873 min_bad > bad_sectors)2874 min_bad = bad_sectors;2875 }2876 }2877 if (sector_nr < first_bad) {2878 if (test_bit(WriteMostly, &rdev->flags)) {2879 if (wonly < 0)2880 wonly = i;2881 } else {2882 if (disk < 0)2883 disk = i;2884 }2885 bio->bi_opf = REQ_OP_READ;2886 bio->bi_end_io = end_sync_read;2887 read_targets++;2888 } else if (!test_bit(WriteErrorSeen, &rdev->flags) &&2889 test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&2890 !test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) {2891 /*2892 * The device is suitable for reading (InSync),2893 * but has bad block(s) here. Let's try to correct them,2894 * if we are doing resync or repair. Otherwise, leave2895 * this device alone for this sync request.2896 */2897 bio->bi_opf = REQ_OP_WRITE;2898 bio->bi_end_io = end_sync_write;2899 write_targets++;2900 }2901 }2902 if (rdev && bio->bi_end_io) {2903 atomic_inc(&rdev->nr_pending);2904 bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;2905 bio_set_dev(bio, rdev->bdev);2906 if (test_bit(FailFast, &rdev->flags))2907 bio->bi_opf |= MD_FAILFAST;2908 }2909 }2910 if (disk < 0)2911 disk = wonly;2912 r1_bio->read_disk = disk;2913 2914 if (read_targets == 0 && min_bad > 0) {2915 /* These sectors are bad on all InSync devices, so we2916 * need to mark them bad on all write targets2917 */2918 int ok = 1;2919 for (i = 0 ; i < conf->raid_disks * 2 ; i++)2920 if (r1_bio->bios[i]->bi_end_io == end_sync_write) {2921 struct md_rdev *rdev = conf->mirrors[i].rdev;2922 ok = rdev_set_badblocks(rdev, sector_nr,2923 min_bad, 02924 ) && ok;2925 }2926 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);2927 *skipped = 1;2928 put_buf(r1_bio);2929 2930 if (!ok) {2931 /* Cannot record the badblocks, so need to2932 * abort the resync.2933 * If there are multiple read targets, could just2934 * fail the really bad ones ???2935 */2936 conf->recovery_disabled = mddev->recovery_disabled;2937 set_bit(MD_RECOVERY_INTR, &mddev->recovery);2938 return 0;2939 } else2940 return min_bad;2941 2942 }2943 if (min_bad > 0 && min_bad < good_sectors) {2944 /* only resync enough to reach the next bad->good2945 * transition */2946 good_sectors = min_bad;2947 }2948 2949 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0)2950 /* extra read targets are also write targets */2951 write_targets += read_targets-1;2952 2953 if (write_targets == 0 || read_targets == 0) {2954 /* There is nowhere to write, so all non-sync2955 * drives must be failed - so we are finished2956 */2957 sector_t rv;2958 if (min_bad > 0)2959 max_sector = sector_nr + min_bad;2960 rv = max_sector - sector_nr;2961 *skipped = 1;2962 put_buf(r1_bio);2963 return rv;2964 }2965 2966 if (max_sector > mddev->resync_max)2967 max_sector = mddev->resync_max; /* Don't do IO beyond here */2968 if (max_sector > sector_nr + good_sectors)2969 max_sector = sector_nr + good_sectors;2970 nr_sectors = 0;2971 sync_blocks = 0;2972 do {2973 struct page *page;2974 int len = PAGE_SIZE;2975 if (sector_nr + (len>>9) > max_sector)2976 len = (max_sector - sector_nr) << 9;2977 if (len == 0)2978 break;2979 if (sync_blocks == 0) {2980 if (!mddev->bitmap_ops->start_sync(mddev, sector_nr,2981 &sync_blocks, still_degraded) &&2982 !conf->fullsync &&2983 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))2984 break;2985 if ((len >> 9) > sync_blocks)2986 len = sync_blocks<<9;2987 }2988 2989 for (i = 0 ; i < conf->raid_disks * 2; i++) {2990 struct resync_pages *rp;2991 2992 bio = r1_bio->bios[i];2993 rp = get_resync_pages(bio);2994 if (bio->bi_end_io) {2995 page = resync_fetch_page(rp, page_idx);2996 2997 /*2998 * won't fail because the vec table is big2999 * enough to hold all these pages3000 */3001 __bio_add_page(bio, page, len, 0);3002 }3003 }3004 nr_sectors += len>>9;3005 sector_nr += len>>9;3006 sync_blocks -= (len>>9);3007 } while (++page_idx < RESYNC_PAGES);3008 3009 r1_bio->sectors = nr_sectors;3010 3011 if (mddev_is_clustered(mddev) &&3012 conf->cluster_sync_high < sector_nr + nr_sectors) {3013 conf->cluster_sync_low = mddev->curr_resync_completed;3014 conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS;3015 /* Send resync message */3016 md_cluster_ops->resync_info_update(mddev,3017 conf->cluster_sync_low,3018 conf->cluster_sync_high);3019 }3020 3021 /* For a user-requested sync, we read all readable devices and do a3022 * compare3023 */3024 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {3025 atomic_set(&r1_bio->remaining, read_targets);3026 for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) {3027 bio = r1_bio->bios[i];3028 if (bio->bi_end_io == end_sync_read) {3029 read_targets--;3030 md_sync_acct_bio(bio, nr_sectors);3031 if (read_targets == 1)3032 bio->bi_opf &= ~MD_FAILFAST;3033 submit_bio_noacct(bio);3034 }3035 }3036 } else {3037 atomic_set(&r1_bio->remaining, 1);3038 bio = r1_bio->bios[r1_bio->read_disk];3039 md_sync_acct_bio(bio, nr_sectors);3040 if (read_targets == 1)3041 bio->bi_opf &= ~MD_FAILFAST;3042 submit_bio_noacct(bio);3043 }3044 return nr_sectors;3045}3046 3047static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks)3048{3049 if (sectors)3050 return sectors;3051 3052 return mddev->dev_sectors;3053}3054 3055static struct r1conf *setup_conf(struct mddev *mddev)3056{3057 struct r1conf *conf;3058 int i;3059 struct raid1_info *disk;3060 struct md_rdev *rdev;3061 int err = -ENOMEM;3062 3063 conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL);3064 if (!conf)3065 goto abort;3066 3067 conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR,3068 sizeof(atomic_t), GFP_KERNEL);3069 if (!conf->nr_pending)3070 goto abort;3071 3072 conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR,3073 sizeof(atomic_t), GFP_KERNEL);3074 if (!conf->nr_waiting)3075 goto abort;3076 3077 conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR,3078 sizeof(atomic_t), GFP_KERNEL);3079 if (!conf->nr_queued)3080 goto abort;3081 3082 conf->barrier = kcalloc(BARRIER_BUCKETS_NR,3083 sizeof(atomic_t), GFP_KERNEL);3084 if (!conf->barrier)3085 goto abort;3086 3087 conf->mirrors = kzalloc(array3_size(sizeof(struct raid1_info),3088 mddev->raid_disks, 2),3089 GFP_KERNEL);3090 if (!conf->mirrors)3091 goto abort;3092 3093 conf->tmppage = alloc_page(GFP_KERNEL);3094 if (!conf->tmppage)3095 goto abort;3096 3097 conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL);3098 if (!conf->poolinfo)3099 goto abort;3100 conf->poolinfo->raid_disks = mddev->raid_disks * 2;3101 err = mempool_init(&conf->r1bio_pool, NR_RAID_BIOS, r1bio_pool_alloc,3102 rbio_pool_free, conf->poolinfo);3103 if (err)3104 goto abort;3105 3106 err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);3107 if (err)3108 goto abort;3109 3110 conf->poolinfo->mddev = mddev;3111 3112 err = -EINVAL;3113 spin_lock_init(&conf->device_lock);3114 conf->raid_disks = mddev->raid_disks;3115 rdev_for_each(rdev, mddev) {3116 int disk_idx = rdev->raid_disk;3117 3118 if (disk_idx >= conf->raid_disks || disk_idx < 0)3119 continue;3120 3121 if (!raid1_add_conf(conf, rdev, disk_idx,3122 test_bit(Replacement, &rdev->flags)))3123 goto abort;3124 }3125 conf->mddev = mddev;3126 INIT_LIST_HEAD(&conf->retry_list);3127 INIT_LIST_HEAD(&conf->bio_end_io_list);3128 3129 spin_lock_init(&conf->resync_lock);3130 init_waitqueue_head(&conf->wait_barrier);3131 3132 bio_list_init(&conf->pending_bio_list);3133 conf->recovery_disabled = mddev->recovery_disabled - 1;3134 3135 err = -EIO;3136 for (i = 0; i < conf->raid_disks * 2; i++) {3137 3138 disk = conf->mirrors + i;3139 3140 if (i < conf->raid_disks &&3141 disk[conf->raid_disks].rdev) {3142 /* This slot has a replacement. */3143 if (!disk->rdev) {3144 /* No original, just make the replacement3145 * a recovering spare3146 */3147 disk->rdev =3148 disk[conf->raid_disks].rdev;3149 disk[conf->raid_disks].rdev = NULL;3150 } else if (!test_bit(In_sync, &disk->rdev->flags))3151 /* Original is not in_sync - bad */3152 goto abort;3153 }3154 3155 if (!disk->rdev ||3156 !test_bit(In_sync, &disk->rdev->flags)) {3157 disk->head_position = 0;3158 if (disk->rdev &&3159 (disk->rdev->saved_raid_disk < 0))3160 conf->fullsync = 1;3161 }3162 }3163 3164 err = -ENOMEM;3165 rcu_assign_pointer(conf->thread,3166 md_register_thread(raid1d, mddev, "raid1"));3167 if (!conf->thread)3168 goto abort;3169 3170 return conf;3171 3172 abort:3173 if (conf) {3174 mempool_exit(&conf->r1bio_pool);3175 kfree(conf->mirrors);3176 safe_put_page(conf->tmppage);3177 kfree(conf->poolinfo);3178 kfree(conf->nr_pending);3179 kfree(conf->nr_waiting);3180 kfree(conf->nr_queued);3181 kfree(conf->barrier);3182 bioset_exit(&conf->bio_split);3183 kfree(conf);3184 }3185 return ERR_PTR(err);3186}3187 3188static int raid1_set_limits(struct mddev *mddev)3189{3190 struct queue_limits lim;3191 int err;3192 3193 md_init_stacking_limits(&lim);3194 lim.max_write_zeroes_sectors = 0;3195 err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY);3196 if (err) {3197 queue_limits_cancel_update(mddev->gendisk->queue);3198 return err;3199 }3200 return queue_limits_set(mddev->gendisk->queue, &lim);3201}3202 3203static int raid1_run(struct mddev *mddev)3204{3205 struct r1conf *conf;3206 int i;3207 int ret;3208 3209 if (mddev->level != 1) {3210 pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n",3211 mdname(mddev), mddev->level);3212 return -EIO;3213 }3214 if (mddev->reshape_position != MaxSector) {3215 pr_warn("md/raid1:%s: reshape_position set but not supported\n",3216 mdname(mddev));3217 return -EIO;3218 }3219 3220 /*3221 * copy the already verified devices into our private RAID13222 * bookkeeping area. [whatever we allocate in run(),3223 * should be freed in raid1_free()]3224 */3225 if (mddev->private == NULL)3226 conf = setup_conf(mddev);3227 else3228 conf = mddev->private;3229 3230 if (IS_ERR(conf))3231 return PTR_ERR(conf);3232 3233 if (!mddev_is_dm(mddev)) {3234 ret = raid1_set_limits(mddev);3235 if (ret)3236 return ret;3237 }3238 3239 mddev->degraded = 0;3240 for (i = 0; i < conf->raid_disks; i++)3241 if (conf->mirrors[i].rdev == NULL ||3242 !test_bit(In_sync, &conf->mirrors[i].rdev->flags) ||3243 test_bit(Faulty, &conf->mirrors[i].rdev->flags))3244 mddev->degraded++;3245 /*3246 * RAID1 needs at least one disk in active3247 */3248 if (conf->raid_disks - mddev->degraded < 1) {3249 md_unregister_thread(mddev, &conf->thread);3250 return -EINVAL;3251 }3252 3253 if (conf->raid_disks - mddev->degraded == 1)3254 mddev->recovery_cp = MaxSector;3255 3256 if (mddev->recovery_cp != MaxSector)3257 pr_info("md/raid1:%s: not clean -- starting background reconstruction\n",3258 mdname(mddev));3259 pr_info("md/raid1:%s: active with %d out of %d mirrors\n",3260 mdname(mddev), mddev->raid_disks - mddev->degraded,3261 mddev->raid_disks);3262 3263 /*3264 * Ok, everything is just fine now3265 */3266 rcu_assign_pointer(mddev->thread, conf->thread);3267 rcu_assign_pointer(conf->thread, NULL);3268 mddev->private = conf;3269 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);3270 3271 md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));3272 3273 ret = md_integrity_register(mddev);3274 if (ret)3275 md_unregister_thread(mddev, &mddev->thread);3276 return ret;3277}3278 3279static void raid1_free(struct mddev *mddev, void *priv)3280{3281 struct r1conf *conf = priv;3282 3283 mempool_exit(&conf->r1bio_pool);3284 kfree(conf->mirrors);3285 safe_put_page(conf->tmppage);3286 kfree(conf->poolinfo);3287 kfree(conf->nr_pending);3288 kfree(conf->nr_waiting);3289 kfree(conf->nr_queued);3290 kfree(conf->barrier);3291 bioset_exit(&conf->bio_split);3292 kfree(conf);3293}3294 3295static int raid1_resize(struct mddev *mddev, sector_t sectors)3296{3297 /* no resync is happening, and there is enough space3298 * on all devices, so we can resize.3299 * We need to make sure resync covers any new space.3300 * If the array is shrinking we should possibly wait until3301 * any io in the removed space completes, but it hardly seems3302 * worth it.3303 */3304 sector_t newsize = raid1_size(mddev, sectors, 0);3305 int ret;3306 3307 if (mddev->external_size &&3308 mddev->array_sectors > newsize)3309 return -EINVAL;3310 3311 ret = mddev->bitmap_ops->resize(mddev, newsize, 0, false);3312 if (ret)3313 return ret;3314 3315 md_set_array_sectors(mddev, newsize);3316 if (sectors > mddev->dev_sectors &&3317 mddev->recovery_cp > mddev->dev_sectors) {3318 mddev->recovery_cp = mddev->dev_sectors;3319 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);3320 }3321 mddev->dev_sectors = sectors;3322 mddev->resync_max_sectors = sectors;3323 return 0;3324}3325 3326static int raid1_reshape(struct mddev *mddev)3327{3328 /* We need to:3329 * 1/ resize the r1bio_pool3330 * 2/ resize conf->mirrors3331 *3332 * We allocate a new r1bio_pool if we can.3333 * Then raise a device barrier and wait until all IO stops.3334 * Then resize conf->mirrors and swap in the new r1bio pool.3335 *3336 * At the same time, we "pack" the devices so that all the missing3337 * devices have the higher raid_disk numbers.3338 */3339 mempool_t newpool, oldpool;3340 struct pool_info *newpoolinfo;3341 struct raid1_info *newmirrors;3342 struct r1conf *conf = mddev->private;3343 int cnt, raid_disks;3344 unsigned long flags;3345 int d, d2;3346 int ret;3347 3348 memset(&newpool, 0, sizeof(newpool));3349 memset(&oldpool, 0, sizeof(oldpool));3350 3351 /* Cannot change chunk_size, layout, or level */3352 if (mddev->chunk_sectors != mddev->new_chunk_sectors ||3353 mddev->layout != mddev->new_layout ||3354 mddev->level != mddev->new_level) {3355 mddev->new_chunk_sectors = mddev->chunk_sectors;3356 mddev->new_layout = mddev->layout;3357 mddev->new_level = mddev->level;3358 return -EINVAL;3359 }3360 3361 if (!mddev_is_clustered(mddev))3362 md_allow_write(mddev);3363 3364 raid_disks = mddev->raid_disks + mddev->delta_disks;3365 3366 if (raid_disks < conf->raid_disks) {3367 cnt=0;3368 for (d= 0; d < conf->raid_disks; d++)3369 if (conf->mirrors[d].rdev)3370 cnt++;3371 if (cnt > raid_disks)3372 return -EBUSY;3373 }3374 3375 newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);3376 if (!newpoolinfo)3377 return -ENOMEM;3378 newpoolinfo->mddev = mddev;3379 newpoolinfo->raid_disks = raid_disks * 2;3380 3381 ret = mempool_init(&newpool, NR_RAID_BIOS, r1bio_pool_alloc,3382 rbio_pool_free, newpoolinfo);3383 if (ret) {3384 kfree(newpoolinfo);3385 return ret;3386 }3387 newmirrors = kzalloc(array3_size(sizeof(struct raid1_info),3388 raid_disks, 2),3389 GFP_KERNEL);3390 if (!newmirrors) {3391 kfree(newpoolinfo);3392 mempool_exit(&newpool);3393 return -ENOMEM;3394 }3395 3396 freeze_array(conf, 0);3397 3398 /* ok, everything is stopped */3399 oldpool = conf->r1bio_pool;3400 conf->r1bio_pool = newpool;3401 3402 for (d = d2 = 0; d < conf->raid_disks; d++) {3403 struct md_rdev *rdev = conf->mirrors[d].rdev;3404 if (rdev && rdev->raid_disk != d2) {3405 sysfs_unlink_rdev(mddev, rdev);3406 rdev->raid_disk = d2;3407 sysfs_unlink_rdev(mddev, rdev);3408 if (sysfs_link_rdev(mddev, rdev))3409 pr_warn("md/raid1:%s: cannot register rd%d\n",3410 mdname(mddev), rdev->raid_disk);3411 }3412 if (rdev)3413 newmirrors[d2++].rdev = rdev;3414 }3415 kfree(conf->mirrors);3416 conf->mirrors = newmirrors;3417 kfree(conf->poolinfo);3418 conf->poolinfo = newpoolinfo;3419 3420 spin_lock_irqsave(&conf->device_lock, flags);3421 mddev->degraded += (raid_disks - conf->raid_disks);3422 spin_unlock_irqrestore(&conf->device_lock, flags);3423 conf->raid_disks = mddev->raid_disks = raid_disks;3424 mddev->delta_disks = 0;3425 3426 unfreeze_array(conf);3427 3428 set_bit(MD_RECOVERY_RECOVER, &mddev->recovery);3429 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);3430 md_wakeup_thread(mddev->thread);3431 3432 mempool_exit(&oldpool);3433 return 0;3434}3435 3436static void raid1_quiesce(struct mddev *mddev, int quiesce)3437{3438 struct r1conf *conf = mddev->private;3439 3440 if (quiesce)3441 freeze_array(conf, 0);3442 else3443 unfreeze_array(conf);3444}3445 3446static void *raid1_takeover(struct mddev *mddev)3447{3448 /* raid1 can take over:3449 * raid5 with 2 devices, any layout or chunk size3450 */3451 if (mddev->level == 5 && mddev->raid_disks == 2) {3452 struct r1conf *conf;3453 mddev->new_level = 1;3454 mddev->new_layout = 0;3455 mddev->new_chunk_sectors = 0;3456 conf = setup_conf(mddev);3457 if (!IS_ERR(conf)) {3458 /* Array must appear to be quiesced */3459 conf->array_frozen = 1;3460 mddev_clear_unsupported_flags(mddev,3461 UNSUPPORTED_MDDEV_FLAGS);3462 }3463 return conf;3464 }3465 return ERR_PTR(-EINVAL);3466}3467 3468static struct md_personality raid1_personality =3469{3470 .name = "raid1",3471 .level = 1,3472 .owner = THIS_MODULE,3473 .make_request = raid1_make_request,3474 .run = raid1_run,3475 .free = raid1_free,3476 .status = raid1_status,3477 .error_handler = raid1_error,3478 .hot_add_disk = raid1_add_disk,3479 .hot_remove_disk= raid1_remove_disk,3480 .spare_active = raid1_spare_active,3481 .sync_request = raid1_sync_request,3482 .resize = raid1_resize,3483 .size = raid1_size,3484 .check_reshape = raid1_reshape,3485 .quiesce = raid1_quiesce,3486 .takeover = raid1_takeover,3487};3488 3489static int __init raid_init(void)3490{3491 return register_md_personality(&raid1_personality);3492}3493 3494static void raid_exit(void)3495{3496 unregister_md_personality(&raid1_personality);3497}3498 3499module_init(raid_init);3500module_exit(raid_exit);3501MODULE_LICENSE("GPL");3502MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD");3503MODULE_ALIAS("md-personality-3"); /* RAID1 */3504MODULE_ALIAS("md-raid1");3505MODULE_ALIAS("md-level-1");3506