9077 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * raid5.c : Multiple Devices driver for Linux4 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman5 * Copyright (C) 1999, 2000 Ingo Molnar6 * Copyright (C) 2002, 2003 H. Peter Anvin7 *8 * RAID-4/5/6 management functions.9 * Thanks to Penguin Computing for making the RAID-6 development possible10 * by donating a test server!11 */12 13/*14 * BITMAP UNPLUGGING:15 *16 * The sequencing for updating the bitmap reliably is a little17 * subtle (and I got it wrong the first time) so it deserves some18 * explanation.19 *20 * We group bitmap updates into batches. Each batch has a number.21 * We may write out several batches at once, but that isn't very important.22 * conf->seq_write is the number of the last batch successfully written.23 * conf->seq_flush is the number of the last batch that was closed to24 * new additions.25 * When we discover that we will need to write to any block in a stripe26 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq27 * the number of the batch it will be in. This is seq_flush+1.28 * When we are ready to do a write, if that batch hasn't been written yet,29 * we plug the array and queue the stripe for later.30 * When an unplug happens, we increment bm_flush, thus closing the current31 * batch.32 * When we notice that bm_flush > bm_write, we write out all pending updates33 * to the bitmap, and advance bm_write to where bm_flush was.34 * This may occasionally write a bit out twice, but is sure never to35 * miss any bits.36 */37 38#include <linux/blkdev.h>39#include <linux/kthread.h>40#include <linux/raid/pq.h>41#include <linux/async_tx.h>42#include <linux/module.h>43#include <linux/async.h>44#include <linux/seq_file.h>45#include <linux/cpu.h>46#include <linux/slab.h>47#include <linux/ratelimit.h>48#include <linux/nodemask.h>49 50#include <trace/events/block.h>51#include <linux/list_sort.h>52 53#include "md.h"54#include "raid5.h"55#include "raid0.h"56#include "md-bitmap.h"57#include "raid5-log.h"58 59#define UNSUPPORTED_MDDEV_FLAGS (1L << MD_FAILFAST_SUPPORTED)60 61#define cpu_to_group(cpu) cpu_to_node(cpu)62#define ANY_GROUP NUMA_NO_NODE63 64#define RAID5_MAX_REQ_STRIPES 25665 66static bool devices_handle_discard_safely = false;67module_param(devices_handle_discard_safely, bool, 0644);68MODULE_PARM_DESC(devices_handle_discard_safely,69 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");70static struct workqueue_struct *raid5_wq;71 72static void raid5_quiesce(struct mddev *mddev, int quiesce);73 74static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)75{76 int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;77 return &conf->stripe_hashtbl[hash];78}79 80static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)81{82 return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;83}84 85static inline void lock_device_hash_lock(struct r5conf *conf, int hash)86 __acquires(&conf->device_lock)87{88 spin_lock_irq(conf->hash_locks + hash);89 spin_lock(&conf->device_lock);90}91 92static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)93 __releases(&conf->device_lock)94{95 spin_unlock(&conf->device_lock);96 spin_unlock_irq(conf->hash_locks + hash);97}98 99static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)100 __acquires(&conf->device_lock)101{102 int i;103 spin_lock_irq(conf->hash_locks);104 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)105 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);106 spin_lock(&conf->device_lock);107}108 109static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)110 __releases(&conf->device_lock)111{112 int i;113 spin_unlock(&conf->device_lock);114 for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--)115 spin_unlock(conf->hash_locks + i);116 spin_unlock_irq(conf->hash_locks);117}118 119/* Find first data disk in a raid6 stripe */120static inline int raid6_d0(struct stripe_head *sh)121{122 if (sh->ddf_layout)123 /* ddf always start from first device */124 return 0;125 /* md starts just after Q block */126 if (sh->qd_idx == sh->disks - 1)127 return 0;128 else129 return sh->qd_idx + 1;130}131static inline int raid6_next_disk(int disk, int raid_disks)132{133 disk++;134 return (disk < raid_disks) ? disk : 0;135}136 137/* When walking through the disks in a raid5, starting at raid6_d0,138 * We need to map each disk to a 'slot', where the data disks are slot139 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk140 * is raid_disks-1. This help does that mapping.141 */142static int raid6_idx_to_slot(int idx, struct stripe_head *sh,143 int *count, int syndrome_disks)144{145 int slot = *count;146 147 if (sh->ddf_layout)148 (*count)++;149 if (idx == sh->pd_idx)150 return syndrome_disks;151 if (idx == sh->qd_idx)152 return syndrome_disks + 1;153 if (!sh->ddf_layout)154 (*count)++;155 return slot;156}157 158static void print_raid5_conf(struct r5conf *conf);159 160static int stripe_operations_active(struct stripe_head *sh)161{162 return sh->check_state || sh->reconstruct_state ||163 test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||164 test_bit(STRIPE_COMPUTE_RUN, &sh->state);165}166 167static bool stripe_is_lowprio(struct stripe_head *sh)168{169 return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||170 test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&171 !test_bit(STRIPE_R5C_CACHING, &sh->state);172}173 174static void raid5_wakeup_stripe_thread(struct stripe_head *sh)175 __must_hold(&sh->raid_conf->device_lock)176{177 struct r5conf *conf = sh->raid_conf;178 struct r5worker_group *group;179 int thread_cnt;180 int i, cpu = sh->cpu;181 182 if (!cpu_online(cpu)) {183 cpu = cpumask_any(cpu_online_mask);184 sh->cpu = cpu;185 }186 187 if (list_empty(&sh->lru)) {188 struct r5worker_group *group;189 group = conf->worker_groups + cpu_to_group(cpu);190 if (stripe_is_lowprio(sh))191 list_add_tail(&sh->lru, &group->loprio_list);192 else193 list_add_tail(&sh->lru, &group->handle_list);194 group->stripes_cnt++;195 sh->group = group;196 }197 198 if (conf->worker_cnt_per_group == 0) {199 md_wakeup_thread(conf->mddev->thread);200 return;201 }202 203 group = conf->worker_groups + cpu_to_group(sh->cpu);204 205 group->workers[0].working = true;206 /* at least one worker should run to avoid race */207 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);208 209 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;210 /* wakeup more workers */211 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {212 if (group->workers[i].working == false) {213 group->workers[i].working = true;214 queue_work_on(sh->cpu, raid5_wq,215 &group->workers[i].work);216 thread_cnt--;217 }218 }219}220 221static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,222 struct list_head *temp_inactive_list)223 __must_hold(&conf->device_lock)224{225 int i;226 int injournal = 0; /* number of date pages with R5_InJournal */227 228 BUG_ON(!list_empty(&sh->lru));229 BUG_ON(atomic_read(&conf->active_stripes)==0);230 231 if (r5c_is_writeback(conf->log))232 for (i = sh->disks; i--; )233 if (test_bit(R5_InJournal, &sh->dev[i].flags))234 injournal++;235 /*236 * In the following cases, the stripe cannot be released to cached237 * lists. Therefore, we make the stripe write out and set238 * STRIPE_HANDLE:239 * 1. when quiesce in r5c write back;240 * 2. when resync is requested fot the stripe.241 */242 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||243 (conf->quiesce && r5c_is_writeback(conf->log) &&244 !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {245 if (test_bit(STRIPE_R5C_CACHING, &sh->state))246 r5c_make_stripe_write_out(sh);247 set_bit(STRIPE_HANDLE, &sh->state);248 }249 250 if (test_bit(STRIPE_HANDLE, &sh->state)) {251 if (test_bit(STRIPE_DELAYED, &sh->state) &&252 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))253 list_add_tail(&sh->lru, &conf->delayed_list);254 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&255 sh->bm_seq - conf->seq_write > 0)256 list_add_tail(&sh->lru, &conf->bitmap_list);257 else {258 clear_bit(STRIPE_DELAYED, &sh->state);259 clear_bit(STRIPE_BIT_DELAY, &sh->state);260 if (conf->worker_cnt_per_group == 0) {261 if (stripe_is_lowprio(sh))262 list_add_tail(&sh->lru,263 &conf->loprio_list);264 else265 list_add_tail(&sh->lru,266 &conf->handle_list);267 } else {268 raid5_wakeup_stripe_thread(sh);269 return;270 }271 }272 md_wakeup_thread(conf->mddev->thread);273 } else {274 BUG_ON(stripe_operations_active(sh));275 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))276 if (atomic_dec_return(&conf->preread_active_stripes)277 < IO_THRESHOLD)278 md_wakeup_thread(conf->mddev->thread);279 atomic_dec(&conf->active_stripes);280 if (!test_bit(STRIPE_EXPANDING, &sh->state)) {281 if (!r5c_is_writeback(conf->log))282 list_add_tail(&sh->lru, temp_inactive_list);283 else {284 WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));285 if (injournal == 0)286 list_add_tail(&sh->lru, temp_inactive_list);287 else if (injournal == conf->raid_disks - conf->max_degraded) {288 /* full stripe */289 if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))290 atomic_inc(&conf->r5c_cached_full_stripes);291 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))292 atomic_dec(&conf->r5c_cached_partial_stripes);293 list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);294 r5c_check_cached_full_stripe(conf);295 } else296 /*297 * STRIPE_R5C_PARTIAL_STRIPE is set in298 * r5c_try_caching_write(). No need to299 * set it again.300 */301 list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);302 }303 }304 }305}306 307static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,308 struct list_head *temp_inactive_list)309 __must_hold(&conf->device_lock)310{311 if (atomic_dec_and_test(&sh->count))312 do_release_stripe(conf, sh, temp_inactive_list);313}314 315/*316 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list317 *318 * Be careful: Only one task can add/delete stripes from temp_inactive_list at319 * given time. Adding stripes only takes device lock, while deleting stripes320 * only takes hash lock.321 */322static void release_inactive_stripe_list(struct r5conf *conf,323 struct list_head *temp_inactive_list,324 int hash)325{326 int size;327 bool do_wakeup = false;328 unsigned long flags;329 330 if (hash == NR_STRIPE_HASH_LOCKS) {331 size = NR_STRIPE_HASH_LOCKS;332 hash = NR_STRIPE_HASH_LOCKS - 1;333 } else334 size = 1;335 while (size) {336 struct list_head *list = &temp_inactive_list[size - 1];337 338 /*339 * We don't hold any lock here yet, raid5_get_active_stripe() might340 * remove stripes from the list341 */342 if (!list_empty_careful(list)) {343 spin_lock_irqsave(conf->hash_locks + hash, flags);344 if (list_empty(conf->inactive_list + hash) &&345 !list_empty(list))346 atomic_dec(&conf->empty_inactive_list_nr);347 list_splice_tail_init(list, conf->inactive_list + hash);348 do_wakeup = true;349 spin_unlock_irqrestore(conf->hash_locks + hash, flags);350 }351 size--;352 hash--;353 }354 355 if (do_wakeup) {356 wake_up(&conf->wait_for_stripe);357 if (atomic_read(&conf->active_stripes) == 0)358 wake_up(&conf->wait_for_quiescent);359 if (conf->retry_read_aligned)360 md_wakeup_thread(conf->mddev->thread);361 }362}363 364static int release_stripe_list(struct r5conf *conf,365 struct list_head *temp_inactive_list)366 __must_hold(&conf->device_lock)367{368 struct stripe_head *sh, *t;369 int count = 0;370 struct llist_node *head;371 372 head = llist_del_all(&conf->released_stripes);373 head = llist_reverse_order(head);374 llist_for_each_entry_safe(sh, t, head, release_list) {375 int hash;376 377 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */378 smp_mb();379 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);380 /*381 * Don't worry the bit is set here, because if the bit is set382 * again, the count is always > 1. This is true for383 * STRIPE_ON_UNPLUG_LIST bit too.384 */385 hash = sh->hash_lock_index;386 __release_stripe(conf, sh, &temp_inactive_list[hash]);387 count++;388 }389 390 return count;391}392 393void raid5_release_stripe(struct stripe_head *sh)394{395 struct r5conf *conf = sh->raid_conf;396 unsigned long flags;397 struct list_head list;398 int hash;399 bool wakeup;400 401 /* Avoid release_list until the last reference.402 */403 if (atomic_add_unless(&sh->count, -1, 1))404 return;405 406 if (unlikely(!conf->mddev->thread) ||407 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))408 goto slow_path;409 wakeup = llist_add(&sh->release_list, &conf->released_stripes);410 if (wakeup)411 md_wakeup_thread(conf->mddev->thread);412 return;413slow_path:414 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */415 if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {416 INIT_LIST_HEAD(&list);417 hash = sh->hash_lock_index;418 do_release_stripe(conf, sh, &list);419 spin_unlock_irqrestore(&conf->device_lock, flags);420 release_inactive_stripe_list(conf, &list, hash);421 }422}423 424static inline void remove_hash(struct stripe_head *sh)425{426 pr_debug("remove_hash(), stripe %llu\n",427 (unsigned long long)sh->sector);428 429 hlist_del_init(&sh->hash);430}431 432static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)433{434 struct hlist_head *hp = stripe_hash(conf, sh->sector);435 436 pr_debug("insert_hash(), stripe %llu\n",437 (unsigned long long)sh->sector);438 439 hlist_add_head(&sh->hash, hp);440}441 442/* find an idle stripe, make sure it is unhashed, and return it. */443static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)444{445 struct stripe_head *sh = NULL;446 struct list_head *first;447 448 if (list_empty(conf->inactive_list + hash))449 goto out;450 first = (conf->inactive_list + hash)->next;451 sh = list_entry(first, struct stripe_head, lru);452 list_del_init(first);453 remove_hash(sh);454 atomic_inc(&conf->active_stripes);455 BUG_ON(hash != sh->hash_lock_index);456 if (list_empty(conf->inactive_list + hash))457 atomic_inc(&conf->empty_inactive_list_nr);458out:459 return sh;460}461 462#if PAGE_SIZE != DEFAULT_STRIPE_SIZE463static void free_stripe_pages(struct stripe_head *sh)464{465 int i;466 struct page *p;467 468 /* Have not allocate page pool */469 if (!sh->pages)470 return;471 472 for (i = 0; i < sh->nr_pages; i++) {473 p = sh->pages[i];474 if (p)475 put_page(p);476 sh->pages[i] = NULL;477 }478}479 480static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)481{482 int i;483 struct page *p;484 485 for (i = 0; i < sh->nr_pages; i++) {486 /* The page have allocated. */487 if (sh->pages[i])488 continue;489 490 p = alloc_page(gfp);491 if (!p) {492 free_stripe_pages(sh);493 return -ENOMEM;494 }495 sh->pages[i] = p;496 }497 return 0;498}499 500static int501init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)502{503 int nr_pages, cnt;504 505 if (sh->pages)506 return 0;507 508 /* Each of the sh->dev[i] need one conf->stripe_size */509 cnt = PAGE_SIZE / conf->stripe_size;510 nr_pages = (disks + cnt - 1) / cnt;511 512 sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);513 if (!sh->pages)514 return -ENOMEM;515 sh->nr_pages = nr_pages;516 sh->stripes_per_page = cnt;517 return 0;518}519#endif520 521static void shrink_buffers(struct stripe_head *sh)522{523 int i;524 int num = sh->raid_conf->pool_size;525 526#if PAGE_SIZE == DEFAULT_STRIPE_SIZE527 for (i = 0; i < num ; i++) {528 struct page *p;529 530 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);531 p = sh->dev[i].page;532 if (!p)533 continue;534 sh->dev[i].page = NULL;535 put_page(p);536 }537#else538 for (i = 0; i < num; i++)539 sh->dev[i].page = NULL;540 free_stripe_pages(sh); /* Free pages */541#endif542}543 544static int grow_buffers(struct stripe_head *sh, gfp_t gfp)545{546 int i;547 int num = sh->raid_conf->pool_size;548 549#if PAGE_SIZE == DEFAULT_STRIPE_SIZE550 for (i = 0; i < num; i++) {551 struct page *page;552 553 if (!(page = alloc_page(gfp))) {554 return 1;555 }556 sh->dev[i].page = page;557 sh->dev[i].orig_page = page;558 sh->dev[i].offset = 0;559 }560#else561 if (alloc_stripe_pages(sh, gfp))562 return -ENOMEM;563 564 for (i = 0; i < num; i++) {565 sh->dev[i].page = raid5_get_dev_page(sh, i);566 sh->dev[i].orig_page = sh->dev[i].page;567 sh->dev[i].offset = raid5_get_page_offset(sh, i);568 }569#endif570 return 0;571}572 573static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,574 struct stripe_head *sh);575 576static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)577{578 struct r5conf *conf = sh->raid_conf;579 int i, seq;580 581 BUG_ON(atomic_read(&sh->count) != 0);582 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));583 BUG_ON(stripe_operations_active(sh));584 BUG_ON(sh->batch_head);585 586 pr_debug("init_stripe called, stripe %llu\n",587 (unsigned long long)sector);588retry:589 seq = read_seqcount_begin(&conf->gen_lock);590 sh->generation = conf->generation - previous;591 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;592 sh->sector = sector;593 stripe_set_idx(sector, conf, previous, sh);594 sh->state = 0;595 596 for (i = sh->disks; i--; ) {597 struct r5dev *dev = &sh->dev[i];598 599 if (dev->toread || dev->read || dev->towrite || dev->written ||600 test_bit(R5_LOCKED, &dev->flags)) {601 pr_err("sector=%llx i=%d %p %p %p %p %d\n",602 (unsigned long long)sh->sector, i, dev->toread,603 dev->read, dev->towrite, dev->written,604 test_bit(R5_LOCKED, &dev->flags));605 WARN_ON(1);606 }607 dev->flags = 0;608 dev->sector = raid5_compute_blocknr(sh, i, previous);609 }610 if (read_seqcount_retry(&conf->gen_lock, seq))611 goto retry;612 sh->overwrite_disks = 0;613 insert_hash(conf, sh);614 sh->cpu = smp_processor_id();615 set_bit(STRIPE_BATCH_READY, &sh->state);616}617 618static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,619 short generation)620{621 struct stripe_head *sh;622 623 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);624 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)625 if (sh->sector == sector && sh->generation == generation)626 return sh;627 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);628 return NULL;629}630 631static struct stripe_head *find_get_stripe(struct r5conf *conf,632 sector_t sector, short generation, int hash)633{634 int inc_empty_inactive_list_flag;635 struct stripe_head *sh;636 637 sh = __find_stripe(conf, sector, generation);638 if (!sh)639 return NULL;640 641 if (atomic_inc_not_zero(&sh->count))642 return sh;643 644 /*645 * Slow path. The reference count is zero which means the stripe must646 * be on a list (sh->lru). Must remove the stripe from the list that647 * references it with the device_lock held.648 */649 650 spin_lock(&conf->device_lock);651 if (!atomic_read(&sh->count)) {652 if (!test_bit(STRIPE_HANDLE, &sh->state))653 atomic_inc(&conf->active_stripes);654 BUG_ON(list_empty(&sh->lru) &&655 !test_bit(STRIPE_EXPANDING, &sh->state));656 inc_empty_inactive_list_flag = 0;657 if (!list_empty(conf->inactive_list + hash))658 inc_empty_inactive_list_flag = 1;659 list_del_init(&sh->lru);660 if (list_empty(conf->inactive_list + hash) &&661 inc_empty_inactive_list_flag)662 atomic_inc(&conf->empty_inactive_list_nr);663 if (sh->group) {664 sh->group->stripes_cnt--;665 sh->group = NULL;666 }667 }668 atomic_inc(&sh->count);669 spin_unlock(&conf->device_lock);670 671 return sh;672}673 674/*675 * Need to check if array has failed when deciding whether to:676 * - start an array677 * - remove non-faulty devices678 * - add a spare679 * - allow a reshape680 * This determination is simple when no reshape is happening.681 * However if there is a reshape, we need to carefully check682 * both the before and after sections.683 * This is because some failed devices may only affect one684 * of the two sections, and some non-in_sync devices may685 * be insync in the section most affected by failed devices.686 *687 * Most calls to this function hold &conf->device_lock. Calls688 * in raid5_run() do not require the lock as no other threads689 * have been started yet.690 */691int raid5_calc_degraded(struct r5conf *conf)692{693 int degraded, degraded2;694 int i;695 696 degraded = 0;697 for (i = 0; i < conf->previous_raid_disks; i++) {698 struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);699 700 if (rdev && test_bit(Faulty, &rdev->flags))701 rdev = READ_ONCE(conf->disks[i].replacement);702 if (!rdev || test_bit(Faulty, &rdev->flags))703 degraded++;704 else if (test_bit(In_sync, &rdev->flags))705 ;706 else707 /* not in-sync or faulty.708 * If the reshape increases the number of devices,709 * this is being recovered by the reshape, so710 * this 'previous' section is not in_sync.711 * If the number of devices is being reduced however,712 * the device can only be part of the array if713 * we are reverting a reshape, so this section will714 * be in-sync.715 */716 if (conf->raid_disks >= conf->previous_raid_disks)717 degraded++;718 }719 if (conf->raid_disks == conf->previous_raid_disks)720 return degraded;721 degraded2 = 0;722 for (i = 0; i < conf->raid_disks; i++) {723 struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);724 725 if (rdev && test_bit(Faulty, &rdev->flags))726 rdev = READ_ONCE(conf->disks[i].replacement);727 if (!rdev || test_bit(Faulty, &rdev->flags))728 degraded2++;729 else if (test_bit(In_sync, &rdev->flags))730 ;731 else732 /* not in-sync or faulty.733 * If reshape increases the number of devices, this734 * section has already been recovered, else it735 * almost certainly hasn't.736 */737 if (conf->raid_disks <= conf->previous_raid_disks)738 degraded2++;739 }740 if (degraded2 > degraded)741 return degraded2;742 return degraded;743}744 745static bool has_failed(struct r5conf *conf)746{747 int degraded = conf->mddev->degraded;748 749 if (test_bit(MD_BROKEN, &conf->mddev->flags))750 return true;751 752 if (conf->mddev->reshape_position != MaxSector)753 degraded = raid5_calc_degraded(conf);754 755 return degraded > conf->max_degraded;756}757 758enum stripe_result {759 STRIPE_SUCCESS = 0,760 STRIPE_RETRY,761 STRIPE_SCHEDULE_AND_RETRY,762 STRIPE_FAIL,763 STRIPE_WAIT_RESHAPE,764};765 766struct stripe_request_ctx {767 /* a reference to the last stripe_head for batching */768 struct stripe_head *batch_last;769 770 /* first sector in the request */771 sector_t first_sector;772 773 /* last sector in the request */774 sector_t last_sector;775 776 /*777 * bitmap to track stripe sectors that have been added to stripes778 * add one to account for unaligned requests779 */780 DECLARE_BITMAP(sectors_to_do, RAID5_MAX_REQ_STRIPES + 1);781 782 /* the request had REQ_PREFLUSH, cleared after the first stripe_head */783 bool do_flush;784};785 786/*787 * Block until another thread clears R5_INACTIVE_BLOCKED or788 * there are fewer than 3/4 the maximum number of active stripes789 * and there is an inactive stripe available.790 */791static bool is_inactive_blocked(struct r5conf *conf, int hash)792{793 if (list_empty(conf->inactive_list + hash))794 return false;795 796 if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))797 return true;798 799 return (atomic_read(&conf->active_stripes) <800 (conf->max_nr_stripes * 3 / 4));801}802 803struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,804 struct stripe_request_ctx *ctx, sector_t sector,805 unsigned int flags)806{807 struct stripe_head *sh;808 int hash = stripe_hash_locks_hash(conf, sector);809 int previous = !!(flags & R5_GAS_PREVIOUS);810 811 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);812 813 spin_lock_irq(conf->hash_locks + hash);814 815 for (;;) {816 if (!(flags & R5_GAS_NOQUIESCE) && conf->quiesce) {817 /*818 * Must release the reference to batch_last before819 * waiting, on quiesce, otherwise the batch_last will820 * hold a reference to a stripe and raid5_quiesce()821 * will deadlock waiting for active_stripes to go to822 * zero.823 */824 if (ctx && ctx->batch_last) {825 raid5_release_stripe(ctx->batch_last);826 ctx->batch_last = NULL;827 }828 829 wait_event_lock_irq(conf->wait_for_quiescent,830 !conf->quiesce,831 *(conf->hash_locks + hash));832 }833 834 sh = find_get_stripe(conf, sector, conf->generation - previous,835 hash);836 if (sh)837 break;838 839 if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {840 sh = get_free_stripe(conf, hash);841 if (sh) {842 r5c_check_stripe_cache_usage(conf);843 init_stripe(sh, sector, previous);844 atomic_inc(&sh->count);845 break;846 }847 848 if (!test_bit(R5_DID_ALLOC, &conf->cache_state))849 set_bit(R5_ALLOC_MORE, &conf->cache_state);850 }851 852 if (flags & R5_GAS_NOBLOCK)853 break;854 855 set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);856 r5l_wake_reclaim(conf->log, 0);857 858 /* release batch_last before wait to avoid risk of deadlock */859 if (ctx && ctx->batch_last) {860 raid5_release_stripe(ctx->batch_last);861 ctx->batch_last = NULL;862 }863 864 wait_event_lock_irq(conf->wait_for_stripe,865 is_inactive_blocked(conf, hash),866 *(conf->hash_locks + hash));867 clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);868 }869 870 spin_unlock_irq(conf->hash_locks + hash);871 return sh;872}873 874static bool is_full_stripe_write(struct stripe_head *sh)875{876 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));877 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);878}879 880static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)881 __acquires(&sh1->stripe_lock)882 __acquires(&sh2->stripe_lock)883{884 if (sh1 > sh2) {885 spin_lock_irq(&sh2->stripe_lock);886 spin_lock_nested(&sh1->stripe_lock, 1);887 } else {888 spin_lock_irq(&sh1->stripe_lock);889 spin_lock_nested(&sh2->stripe_lock, 1);890 }891}892 893static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)894 __releases(&sh1->stripe_lock)895 __releases(&sh2->stripe_lock)896{897 spin_unlock(&sh1->stripe_lock);898 spin_unlock_irq(&sh2->stripe_lock);899}900 901/* Only freshly new full stripe normal write stripe can be added to a batch list */902static bool stripe_can_batch(struct stripe_head *sh)903{904 struct r5conf *conf = sh->raid_conf;905 906 if (raid5_has_log(conf) || raid5_has_ppl(conf))907 return false;908 return test_bit(STRIPE_BATCH_READY, &sh->state) &&909 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&910 is_full_stripe_write(sh);911}912 913/* we only do back search */914static void stripe_add_to_batch_list(struct r5conf *conf,915 struct stripe_head *sh, struct stripe_head *last_sh)916{917 struct stripe_head *head;918 sector_t head_sector, tmp_sec;919 int hash;920 int dd_idx;921 922 /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */923 tmp_sec = sh->sector;924 if (!sector_div(tmp_sec, conf->chunk_sectors))925 return;926 head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);927 928 if (last_sh && head_sector == last_sh->sector) {929 head = last_sh;930 atomic_inc(&head->count);931 } else {932 hash = stripe_hash_locks_hash(conf, head_sector);933 spin_lock_irq(conf->hash_locks + hash);934 head = find_get_stripe(conf, head_sector, conf->generation,935 hash);936 spin_unlock_irq(conf->hash_locks + hash);937 if (!head)938 return;939 if (!stripe_can_batch(head))940 goto out;941 }942 943 lock_two_stripes(head, sh);944 /* clear_batch_ready clear the flag */945 if (!stripe_can_batch(head) || !stripe_can_batch(sh))946 goto unlock_out;947 948 if (sh->batch_head)949 goto unlock_out;950 951 dd_idx = 0;952 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)953 dd_idx++;954 if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||955 bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))956 goto unlock_out;957 958 if (head->batch_head) {959 spin_lock(&head->batch_head->batch_lock);960 /* This batch list is already running */961 if (!stripe_can_batch(head)) {962 spin_unlock(&head->batch_head->batch_lock);963 goto unlock_out;964 }965 /*966 * We must assign batch_head of this stripe within the967 * batch_lock, otherwise clear_batch_ready of batch head968 * stripe could clear BATCH_READY bit of this stripe and969 * this stripe->batch_head doesn't get assigned, which970 * could confuse clear_batch_ready for this stripe971 */972 sh->batch_head = head->batch_head;973 974 /*975 * at this point, head's BATCH_READY could be cleared, but we976 * can still add the stripe to batch list977 */978 list_add(&sh->batch_list, &head->batch_list);979 spin_unlock(&head->batch_head->batch_lock);980 } else {981 head->batch_head = head;982 sh->batch_head = head->batch_head;983 spin_lock(&head->batch_lock);984 list_add_tail(&sh->batch_list, &head->batch_list);985 spin_unlock(&head->batch_lock);986 }987 988 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))989 if (atomic_dec_return(&conf->preread_active_stripes)990 < IO_THRESHOLD)991 md_wakeup_thread(conf->mddev->thread);992 993 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {994 int seq = sh->bm_seq;995 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&996 sh->batch_head->bm_seq > seq)997 seq = sh->batch_head->bm_seq;998 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);999 sh->batch_head->bm_seq = seq;1000 }1001 1002 atomic_inc(&sh->count);1003unlock_out:1004 unlock_two_stripes(head, sh);1005out:1006 raid5_release_stripe(head);1007}1008 1009/* Determine if 'data_offset' or 'new_data_offset' should be used1010 * in this stripe_head.1011 */1012static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)1013{1014 sector_t progress = conf->reshape_progress;1015 /* Need a memory barrier to make sure we see the value1016 * of conf->generation, or ->data_offset that was set before1017 * reshape_progress was updated.1018 */1019 smp_rmb();1020 if (progress == MaxSector)1021 return 0;1022 if (sh->generation == conf->generation - 1)1023 return 0;1024 /* We are in a reshape, and this is a new-generation stripe,1025 * so use new_data_offset.1026 */1027 return 1;1028}1029 1030static void dispatch_bio_list(struct bio_list *tmp)1031{1032 struct bio *bio;1033 1034 while ((bio = bio_list_pop(tmp)))1035 submit_bio_noacct(bio);1036}1037 1038static int cmp_stripe(void *priv, const struct list_head *a,1039 const struct list_head *b)1040{1041 const struct r5pending_data *da = list_entry(a,1042 struct r5pending_data, sibling);1043 const struct r5pending_data *db = list_entry(b,1044 struct r5pending_data, sibling);1045 if (da->sector > db->sector)1046 return 1;1047 if (da->sector < db->sector)1048 return -1;1049 return 0;1050}1051 1052static void dispatch_defer_bios(struct r5conf *conf, int target,1053 struct bio_list *list)1054{1055 struct r5pending_data *data;1056 struct list_head *first, *next = NULL;1057 int cnt = 0;1058 1059 if (conf->pending_data_cnt == 0)1060 return;1061 1062 list_sort(NULL, &conf->pending_list, cmp_stripe);1063 1064 first = conf->pending_list.next;1065 1066 /* temporarily move the head */1067 if (conf->next_pending_data)1068 list_move_tail(&conf->pending_list,1069 &conf->next_pending_data->sibling);1070 1071 while (!list_empty(&conf->pending_list)) {1072 data = list_first_entry(&conf->pending_list,1073 struct r5pending_data, sibling);1074 if (&data->sibling == first)1075 first = data->sibling.next;1076 next = data->sibling.next;1077 1078 bio_list_merge(list, &data->bios);1079 list_move(&data->sibling, &conf->free_list);1080 cnt++;1081 if (cnt >= target)1082 break;1083 }1084 conf->pending_data_cnt -= cnt;1085 BUG_ON(conf->pending_data_cnt < 0 || cnt < target);1086 1087 if (next != &conf->pending_list)1088 conf->next_pending_data = list_entry(next,1089 struct r5pending_data, sibling);1090 else1091 conf->next_pending_data = NULL;1092 /* list isn't empty */1093 if (first != &conf->pending_list)1094 list_move_tail(&conf->pending_list, first);1095}1096 1097static void flush_deferred_bios(struct r5conf *conf)1098{1099 struct bio_list tmp = BIO_EMPTY_LIST;1100 1101 if (conf->pending_data_cnt == 0)1102 return;1103 1104 spin_lock(&conf->pending_bios_lock);1105 dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);1106 BUG_ON(conf->pending_data_cnt != 0);1107 spin_unlock(&conf->pending_bios_lock);1108 1109 dispatch_bio_list(&tmp);1110}1111 1112static void defer_issue_bios(struct r5conf *conf, sector_t sector,1113 struct bio_list *bios)1114{1115 struct bio_list tmp = BIO_EMPTY_LIST;1116 struct r5pending_data *ent;1117 1118 spin_lock(&conf->pending_bios_lock);1119 ent = list_first_entry(&conf->free_list, struct r5pending_data,1120 sibling);1121 list_move_tail(&ent->sibling, &conf->pending_list);1122 ent->sector = sector;1123 bio_list_init(&ent->bios);1124 bio_list_merge(&ent->bios, bios);1125 conf->pending_data_cnt++;1126 if (conf->pending_data_cnt >= PENDING_IO_MAX)1127 dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);1128 1129 spin_unlock(&conf->pending_bios_lock);1130 1131 dispatch_bio_list(&tmp);1132}1133 1134static void1135raid5_end_read_request(struct bio *bi);1136static void1137raid5_end_write_request(struct bio *bi);1138 1139static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)1140{1141 struct r5conf *conf = sh->raid_conf;1142 int i, disks = sh->disks;1143 struct stripe_head *head_sh = sh;1144 struct bio_list pending_bios = BIO_EMPTY_LIST;1145 struct r5dev *dev;1146 bool should_defer;1147 1148 might_sleep();1149 1150 if (log_stripe(sh, s) == 0)1151 return;1152 1153 should_defer = conf->batch_bio_dispatch && conf->group_cnt;1154 1155 for (i = disks; i--; ) {1156 enum req_op op;1157 blk_opf_t op_flags = 0;1158 int replace_only = 0;1159 struct bio *bi, *rbi;1160 struct md_rdev *rdev, *rrdev = NULL;1161 1162 sh = head_sh;1163 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {1164 op = REQ_OP_WRITE;1165 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))1166 op_flags = REQ_FUA;1167 if (test_bit(R5_Discard, &sh->dev[i].flags))1168 op = REQ_OP_DISCARD;1169 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))1170 op = REQ_OP_READ;1171 else if (test_and_clear_bit(R5_WantReplace,1172 &sh->dev[i].flags)) {1173 op = REQ_OP_WRITE;1174 replace_only = 1;1175 } else1176 continue;1177 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))1178 op_flags |= REQ_SYNC;1179 1180again:1181 dev = &sh->dev[i];1182 bi = &dev->req;1183 rbi = &dev->rreq; /* For writing to replacement */1184 1185 rdev = conf->disks[i].rdev;1186 rrdev = conf->disks[i].replacement;1187 if (op_is_write(op)) {1188 if (replace_only)1189 rdev = NULL;1190 if (rdev == rrdev)1191 /* We raced and saw duplicates */1192 rrdev = NULL;1193 } else {1194 if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)1195 rdev = rrdev;1196 rrdev = NULL;1197 }1198 1199 if (rdev && test_bit(Faulty, &rdev->flags))1200 rdev = NULL;1201 if (rdev)1202 atomic_inc(&rdev->nr_pending);1203 if (rrdev && test_bit(Faulty, &rrdev->flags))1204 rrdev = NULL;1205 if (rrdev)1206 atomic_inc(&rrdev->nr_pending);1207 1208 /* We have already checked bad blocks for reads. Now1209 * need to check for writes. We never accept write errors1210 * on the replacement, so we don't to check rrdev.1211 */1212 while (op_is_write(op) && rdev &&1213 test_bit(WriteErrorSeen, &rdev->flags)) {1214 int bad = rdev_has_badblock(rdev, sh->sector,1215 RAID5_STRIPE_SECTORS(conf));1216 if (!bad)1217 break;1218 1219 if (bad < 0) {1220 set_bit(BlockedBadBlocks, &rdev->flags);1221 if (!conf->mddev->external &&1222 conf->mddev->sb_flags) {1223 /* It is very unlikely, but we might1224 * still need to write out the1225 * bad block log - better give it1226 * a chance*/1227 md_check_recovery(conf->mddev);1228 }1229 /*1230 * Because md_wait_for_blocked_rdev1231 * will dec nr_pending, we must1232 * increment it first.1233 */1234 atomic_inc(&rdev->nr_pending);1235 md_wait_for_blocked_rdev(rdev, conf->mddev);1236 } else {1237 /* Acknowledged bad block - skip the write */1238 rdev_dec_pending(rdev, conf->mddev);1239 rdev = NULL;1240 }1241 }1242 1243 if (rdev) {1244 if (s->syncing || s->expanding || s->expanded1245 || s->replacing)1246 md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));1247 1248 set_bit(STRIPE_IO_STARTED, &sh->state);1249 1250 bio_init(bi, rdev->bdev, &dev->vec, 1, op | op_flags);1251 bi->bi_end_io = op_is_write(op)1252 ? raid5_end_write_request1253 : raid5_end_read_request;1254 bi->bi_private = sh;1255 1256 pr_debug("%s: for %llu schedule op %d on disc %d\n",1257 __func__, (unsigned long long)sh->sector,1258 bi->bi_opf, i);1259 atomic_inc(&sh->count);1260 if (sh != head_sh)1261 atomic_inc(&head_sh->count);1262 if (use_new_offset(conf, sh))1263 bi->bi_iter.bi_sector = (sh->sector1264 + rdev->new_data_offset);1265 else1266 bi->bi_iter.bi_sector = (sh->sector1267 + rdev->data_offset);1268 if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))1269 bi->bi_opf |= REQ_NOMERGE;1270 1271 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))1272 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));1273 1274 if (!op_is_write(op) &&1275 test_bit(R5_InJournal, &sh->dev[i].flags))1276 /*1277 * issuing read for a page in journal, this1278 * must be preparing for prexor in rmw; read1279 * the data into orig_page1280 */1281 sh->dev[i].vec.bv_page = sh->dev[i].orig_page;1282 else1283 sh->dev[i].vec.bv_page = sh->dev[i].page;1284 bi->bi_vcnt = 1;1285 bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);1286 bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;1287 bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);1288 /*1289 * If this is discard request, set bi_vcnt 0. We don't1290 * want to confuse SCSI because SCSI will replace payload1291 */1292 if (op == REQ_OP_DISCARD)1293 bi->bi_vcnt = 0;1294 if (rrdev)1295 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);1296 1297 mddev_trace_remap(conf->mddev, bi, sh->dev[i].sector);1298 if (should_defer && op_is_write(op))1299 bio_list_add(&pending_bios, bi);1300 else1301 submit_bio_noacct(bi);1302 }1303 if (rrdev) {1304 if (s->syncing || s->expanding || s->expanded1305 || s->replacing)1306 md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));1307 1308 set_bit(STRIPE_IO_STARTED, &sh->state);1309 1310 bio_init(rbi, rrdev->bdev, &dev->rvec, 1, op | op_flags);1311 BUG_ON(!op_is_write(op));1312 rbi->bi_end_io = raid5_end_write_request;1313 rbi->bi_private = sh;1314 1315 pr_debug("%s: for %llu schedule op %d on "1316 "replacement disc %d\n",1317 __func__, (unsigned long long)sh->sector,1318 rbi->bi_opf, i);1319 atomic_inc(&sh->count);1320 if (sh != head_sh)1321 atomic_inc(&head_sh->count);1322 if (use_new_offset(conf, sh))1323 rbi->bi_iter.bi_sector = (sh->sector1324 + rrdev->new_data_offset);1325 else1326 rbi->bi_iter.bi_sector = (sh->sector1327 + rrdev->data_offset);1328 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))1329 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));1330 sh->dev[i].rvec.bv_page = sh->dev[i].page;1331 rbi->bi_vcnt = 1;1332 rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);1333 rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;1334 rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);1335 /*1336 * If this is discard request, set bi_vcnt 0. We don't1337 * want to confuse SCSI because SCSI will replace payload1338 */1339 if (op == REQ_OP_DISCARD)1340 rbi->bi_vcnt = 0;1341 mddev_trace_remap(conf->mddev, rbi, sh->dev[i].sector);1342 if (should_defer && op_is_write(op))1343 bio_list_add(&pending_bios, rbi);1344 else1345 submit_bio_noacct(rbi);1346 }1347 if (!rdev && !rrdev) {1348 if (op_is_write(op))1349 set_bit(STRIPE_DEGRADED, &sh->state);1350 pr_debug("skip op %d on disc %d for sector %llu\n",1351 bi->bi_opf, i, (unsigned long long)sh->sector);1352 clear_bit(R5_LOCKED, &sh->dev[i].flags);1353 set_bit(STRIPE_HANDLE, &sh->state);1354 }1355 1356 if (!head_sh->batch_head)1357 continue;1358 sh = list_first_entry(&sh->batch_list, struct stripe_head,1359 batch_list);1360 if (sh != head_sh)1361 goto again;1362 }1363 1364 if (should_defer && !bio_list_empty(&pending_bios))1365 defer_issue_bios(conf, head_sh->sector, &pending_bios);1366}1367 1368static struct dma_async_tx_descriptor *1369async_copy_data(int frombio, struct bio *bio, struct page **page,1370 unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,1371 struct stripe_head *sh, int no_skipcopy)1372{1373 struct bio_vec bvl;1374 struct bvec_iter iter;1375 struct page *bio_page;1376 int page_offset;1377 struct async_submit_ctl submit;1378 enum async_tx_flags flags = 0;1379 struct r5conf *conf = sh->raid_conf;1380 1381 if (bio->bi_iter.bi_sector >= sector)1382 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;1383 else1384 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;1385 1386 if (frombio)1387 flags |= ASYNC_TX_FENCE;1388 init_async_submit(&submit, flags, tx, NULL, NULL, NULL);1389 1390 bio_for_each_segment(bvl, bio, iter) {1391 int len = bvl.bv_len;1392 int clen;1393 int b_offset = 0;1394 1395 if (page_offset < 0) {1396 b_offset = -page_offset;1397 page_offset += b_offset;1398 len -= b_offset;1399 }1400 1401 if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))1402 clen = RAID5_STRIPE_SIZE(conf) - page_offset;1403 else1404 clen = len;1405 1406 if (clen > 0) {1407 b_offset += bvl.bv_offset;1408 bio_page = bvl.bv_page;1409 if (frombio) {1410 if (conf->skip_copy &&1411 b_offset == 0 && page_offset == 0 &&1412 clen == RAID5_STRIPE_SIZE(conf) &&1413 !no_skipcopy)1414 *page = bio_page;1415 else1416 tx = async_memcpy(*page, bio_page, page_offset + poff,1417 b_offset, clen, &submit);1418 } else1419 tx = async_memcpy(bio_page, *page, b_offset,1420 page_offset + poff, clen, &submit);1421 }1422 /* chain the operations */1423 submit.depend_tx = tx;1424 1425 if (clen < len) /* hit end of page */1426 break;1427 page_offset += len;1428 }1429 1430 return tx;1431}1432 1433static void ops_complete_biofill(void *stripe_head_ref)1434{1435 struct stripe_head *sh = stripe_head_ref;1436 int i;1437 struct r5conf *conf = sh->raid_conf;1438 1439 pr_debug("%s: stripe %llu\n", __func__,1440 (unsigned long long)sh->sector);1441 1442 /* clear completed biofills */1443 for (i = sh->disks; i--; ) {1444 struct r5dev *dev = &sh->dev[i];1445 1446 /* acknowledge completion of a biofill operation */1447 /* and check if we need to reply to a read request,1448 * new R5_Wantfill requests are held off until1449 * !STRIPE_BIOFILL_RUN1450 */1451 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {1452 struct bio *rbi, *rbi2;1453 1454 BUG_ON(!dev->read);1455 rbi = dev->read;1456 dev->read = NULL;1457 while (rbi && rbi->bi_iter.bi_sector <1458 dev->sector + RAID5_STRIPE_SECTORS(conf)) {1459 rbi2 = r5_next_bio(conf, rbi, dev->sector);1460 bio_endio(rbi);1461 rbi = rbi2;1462 }1463 }1464 }1465 clear_bit(STRIPE_BIOFILL_RUN, &sh->state);1466 1467 set_bit(STRIPE_HANDLE, &sh->state);1468 raid5_release_stripe(sh);1469}1470 1471static void ops_run_biofill(struct stripe_head *sh)1472{1473 struct dma_async_tx_descriptor *tx = NULL;1474 struct async_submit_ctl submit;1475 int i;1476 struct r5conf *conf = sh->raid_conf;1477 1478 BUG_ON(sh->batch_head);1479 pr_debug("%s: stripe %llu\n", __func__,1480 (unsigned long long)sh->sector);1481 1482 for (i = sh->disks; i--; ) {1483 struct r5dev *dev = &sh->dev[i];1484 if (test_bit(R5_Wantfill, &dev->flags)) {1485 struct bio *rbi;1486 spin_lock_irq(&sh->stripe_lock);1487 dev->read = rbi = dev->toread;1488 dev->toread = NULL;1489 spin_unlock_irq(&sh->stripe_lock);1490 while (rbi && rbi->bi_iter.bi_sector <1491 dev->sector + RAID5_STRIPE_SECTORS(conf)) {1492 tx = async_copy_data(0, rbi, &dev->page,1493 dev->offset,1494 dev->sector, tx, sh, 0);1495 rbi = r5_next_bio(conf, rbi, dev->sector);1496 }1497 }1498 }1499 1500 atomic_inc(&sh->count);1501 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);1502 async_trigger_callback(&submit);1503}1504 1505static void mark_target_uptodate(struct stripe_head *sh, int target)1506{1507 struct r5dev *tgt;1508 1509 if (target < 0)1510 return;1511 1512 tgt = &sh->dev[target];1513 set_bit(R5_UPTODATE, &tgt->flags);1514 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));1515 clear_bit(R5_Wantcompute, &tgt->flags);1516}1517 1518static void ops_complete_compute(void *stripe_head_ref)1519{1520 struct stripe_head *sh = stripe_head_ref;1521 1522 pr_debug("%s: stripe %llu\n", __func__,1523 (unsigned long long)sh->sector);1524 1525 /* mark the computed target(s) as uptodate */1526 mark_target_uptodate(sh, sh->ops.target);1527 mark_target_uptodate(sh, sh->ops.target2);1528 1529 clear_bit(STRIPE_COMPUTE_RUN, &sh->state);1530 if (sh->check_state == check_state_compute_run)1531 sh->check_state = check_state_compute_result;1532 set_bit(STRIPE_HANDLE, &sh->state);1533 raid5_release_stripe(sh);1534}1535 1536/* return a pointer to the address conversion region of the scribble buffer */1537static struct page **to_addr_page(struct raid5_percpu *percpu, int i)1538{1539 return percpu->scribble + i * percpu->scribble_obj_size;1540}1541 1542/* return a pointer to the address conversion region of the scribble buffer */1543static addr_conv_t *to_addr_conv(struct stripe_head *sh,1544 struct raid5_percpu *percpu, int i)1545{1546 return (void *) (to_addr_page(percpu, i) + sh->disks + 2);1547}1548 1549/*1550 * Return a pointer to record offset address.1551 */1552static unsigned int *1553to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)1554{1555 return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);1556}1557 1558static struct dma_async_tx_descriptor *1559ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)1560{1561 int disks = sh->disks;1562 struct page **xor_srcs = to_addr_page(percpu, 0);1563 unsigned int *off_srcs = to_addr_offs(sh, percpu);1564 int target = sh->ops.target;1565 struct r5dev *tgt = &sh->dev[target];1566 struct page *xor_dest = tgt->page;1567 unsigned int off_dest = tgt->offset;1568 int count = 0;1569 struct dma_async_tx_descriptor *tx;1570 struct async_submit_ctl submit;1571 int i;1572 1573 BUG_ON(sh->batch_head);1574 1575 pr_debug("%s: stripe %llu block: %d\n",1576 __func__, (unsigned long long)sh->sector, target);1577 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));1578 1579 for (i = disks; i--; ) {1580 if (i != target) {1581 off_srcs[count] = sh->dev[i].offset;1582 xor_srcs[count++] = sh->dev[i].page;1583 }1584 }1585 1586 atomic_inc(&sh->count);1587 1588 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,1589 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));1590 if (unlikely(count == 1))1591 tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],1592 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);1593 else1594 tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,1595 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);1596 1597 return tx;1598}1599 1600/* set_syndrome_sources - populate source buffers for gen_syndrome1601 * @srcs - (struct page *) array of size sh->disks1602 * @offs - (unsigned int) array of offset for each page1603 * @sh - stripe_head to parse1604 *1605 * Populates srcs in proper layout order for the stripe and returns the1606 * 'count' of sources to be used in a call to async_gen_syndrome. The P1607 * destination buffer is recorded in srcs[count] and the Q destination1608 * is recorded in srcs[count+1]].1609 */1610static int set_syndrome_sources(struct page **srcs,1611 unsigned int *offs,1612 struct stripe_head *sh,1613 int srctype)1614{1615 int disks = sh->disks;1616 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);1617 int d0_idx = raid6_d0(sh);1618 int count;1619 int i;1620 1621 for (i = 0; i < disks; i++)1622 srcs[i] = NULL;1623 1624 count = 0;1625 i = d0_idx;1626 do {1627 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);1628 struct r5dev *dev = &sh->dev[i];1629 1630 if (i == sh->qd_idx || i == sh->pd_idx ||1631 (srctype == SYNDROME_SRC_ALL) ||1632 (srctype == SYNDROME_SRC_WANT_DRAIN &&1633 (test_bit(R5_Wantdrain, &dev->flags) ||1634 test_bit(R5_InJournal, &dev->flags))) ||1635 (srctype == SYNDROME_SRC_WRITTEN &&1636 (dev->written ||1637 test_bit(R5_InJournal, &dev->flags)))) {1638 if (test_bit(R5_InJournal, &dev->flags))1639 srcs[slot] = sh->dev[i].orig_page;1640 else1641 srcs[slot] = sh->dev[i].page;1642 /*1643 * For R5_InJournal, PAGE_SIZE must be 4KB and will1644 * not shared page. In that case, dev[i].offset1645 * is 0.1646 */1647 offs[slot] = sh->dev[i].offset;1648 }1649 i = raid6_next_disk(i, disks);1650 } while (i != d0_idx);1651 1652 return syndrome_disks;1653}1654 1655static struct dma_async_tx_descriptor *1656ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)1657{1658 int disks = sh->disks;1659 struct page **blocks = to_addr_page(percpu, 0);1660 unsigned int *offs = to_addr_offs(sh, percpu);1661 int target;1662 int qd_idx = sh->qd_idx;1663 struct dma_async_tx_descriptor *tx;1664 struct async_submit_ctl submit;1665 struct r5dev *tgt;1666 struct page *dest;1667 unsigned int dest_off;1668 int i;1669 int count;1670 1671 BUG_ON(sh->batch_head);1672 if (sh->ops.target < 0)1673 target = sh->ops.target2;1674 else if (sh->ops.target2 < 0)1675 target = sh->ops.target;1676 else1677 /* we should only have one valid target */1678 BUG();1679 BUG_ON(target < 0);1680 pr_debug("%s: stripe %llu block: %d\n",1681 __func__, (unsigned long long)sh->sector, target);1682 1683 tgt = &sh->dev[target];1684 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));1685 dest = tgt->page;1686 dest_off = tgt->offset;1687 1688 atomic_inc(&sh->count);1689 1690 if (target == qd_idx) {1691 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);1692 blocks[count] = NULL; /* regenerating p is not necessary */1693 BUG_ON(blocks[count+1] != dest); /* q should already be set */1694 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,1695 ops_complete_compute, sh,1696 to_addr_conv(sh, percpu, 0));1697 tx = async_gen_syndrome(blocks, offs, count+2,1698 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);1699 } else {1700 /* Compute any data- or p-drive using XOR */1701 count = 0;1702 for (i = disks; i-- ; ) {1703 if (i == target || i == qd_idx)1704 continue;1705 offs[count] = sh->dev[i].offset;1706 blocks[count++] = sh->dev[i].page;1707 }1708 1709 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,1710 NULL, ops_complete_compute, sh,1711 to_addr_conv(sh, percpu, 0));1712 tx = async_xor_offs(dest, dest_off, blocks, offs, count,1713 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);1714 }1715 1716 return tx;1717}1718 1719static struct dma_async_tx_descriptor *1720ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)1721{1722 int i, count, disks = sh->disks;1723 int syndrome_disks = sh->ddf_layout ? disks : disks-2;1724 int d0_idx = raid6_d0(sh);1725 int faila = -1, failb = -1;1726 int target = sh->ops.target;1727 int target2 = sh->ops.target2;1728 struct r5dev *tgt = &sh->dev[target];1729 struct r5dev *tgt2 = &sh->dev[target2];1730 struct dma_async_tx_descriptor *tx;1731 struct page **blocks = to_addr_page(percpu, 0);1732 unsigned int *offs = to_addr_offs(sh, percpu);1733 struct async_submit_ctl submit;1734 1735 BUG_ON(sh->batch_head);1736 pr_debug("%s: stripe %llu block1: %d block2: %d\n",1737 __func__, (unsigned long long)sh->sector, target, target2);1738 BUG_ON(target < 0 || target2 < 0);1739 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));1740 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));1741 1742 /* we need to open-code set_syndrome_sources to handle the1743 * slot number conversion for 'faila' and 'failb'1744 */1745 for (i = 0; i < disks ; i++) {1746 offs[i] = 0;1747 blocks[i] = NULL;1748 }1749 count = 0;1750 i = d0_idx;1751 do {1752 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);1753 1754 offs[slot] = sh->dev[i].offset;1755 blocks[slot] = sh->dev[i].page;1756 1757 if (i == target)1758 faila = slot;1759 if (i == target2)1760 failb = slot;1761 i = raid6_next_disk(i, disks);1762 } while (i != d0_idx);1763 1764 BUG_ON(faila == failb);1765 if (failb < faila)1766 swap(faila, failb);1767 pr_debug("%s: stripe: %llu faila: %d failb: %d\n",1768 __func__, (unsigned long long)sh->sector, faila, failb);1769 1770 atomic_inc(&sh->count);1771 1772 if (failb == syndrome_disks+1) {1773 /* Q disk is one of the missing disks */1774 if (faila == syndrome_disks) {1775 /* Missing P+Q, just recompute */1776 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,1777 ops_complete_compute, sh,1778 to_addr_conv(sh, percpu, 0));1779 return async_gen_syndrome(blocks, offs, syndrome_disks+2,1780 RAID5_STRIPE_SIZE(sh->raid_conf),1781 &submit);1782 } else {1783 struct page *dest;1784 unsigned int dest_off;1785 int data_target;1786 int qd_idx = sh->qd_idx;1787 1788 /* Missing D+Q: recompute D from P, then recompute Q */1789 if (target == qd_idx)1790 data_target = target2;1791 else1792 data_target = target;1793 1794 count = 0;1795 for (i = disks; i-- ; ) {1796 if (i == data_target || i == qd_idx)1797 continue;1798 offs[count] = sh->dev[i].offset;1799 blocks[count++] = sh->dev[i].page;1800 }1801 dest = sh->dev[data_target].page;1802 dest_off = sh->dev[data_target].offset;1803 init_async_submit(&submit,1804 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,1805 NULL, NULL, NULL,1806 to_addr_conv(sh, percpu, 0));1807 tx = async_xor_offs(dest, dest_off, blocks, offs, count,1808 RAID5_STRIPE_SIZE(sh->raid_conf),1809 &submit);1810 1811 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);1812 init_async_submit(&submit, ASYNC_TX_FENCE, tx,1813 ops_complete_compute, sh,1814 to_addr_conv(sh, percpu, 0));1815 return async_gen_syndrome(blocks, offs, count+2,1816 RAID5_STRIPE_SIZE(sh->raid_conf),1817 &submit);1818 }1819 } else {1820 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,1821 ops_complete_compute, sh,1822 to_addr_conv(sh, percpu, 0));1823 if (failb == syndrome_disks) {1824 /* We're missing D+P. */1825 return async_raid6_datap_recov(syndrome_disks+2,1826 RAID5_STRIPE_SIZE(sh->raid_conf),1827 faila,1828 blocks, offs, &submit);1829 } else {1830 /* We're missing D+D. */1831 return async_raid6_2data_recov(syndrome_disks+2,1832 RAID5_STRIPE_SIZE(sh->raid_conf),1833 faila, failb,1834 blocks, offs, &submit);1835 }1836 }1837}1838 1839static void ops_complete_prexor(void *stripe_head_ref)1840{1841 struct stripe_head *sh = stripe_head_ref;1842 1843 pr_debug("%s: stripe %llu\n", __func__,1844 (unsigned long long)sh->sector);1845 1846 if (r5c_is_writeback(sh->raid_conf->log))1847 /*1848 * raid5-cache write back uses orig_page during prexor.1849 * After prexor, it is time to free orig_page1850 */1851 r5c_release_extra_page(sh);1852}1853 1854static struct dma_async_tx_descriptor *1855ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,1856 struct dma_async_tx_descriptor *tx)1857{1858 int disks = sh->disks;1859 struct page **xor_srcs = to_addr_page(percpu, 0);1860 unsigned int *off_srcs = to_addr_offs(sh, percpu);1861 int count = 0, pd_idx = sh->pd_idx, i;1862 struct async_submit_ctl submit;1863 1864 /* existing parity data subtracted */1865 unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;1866 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;1867 1868 BUG_ON(sh->batch_head);1869 pr_debug("%s: stripe %llu\n", __func__,1870 (unsigned long long)sh->sector);1871 1872 for (i = disks; i--; ) {1873 struct r5dev *dev = &sh->dev[i];1874 /* Only process blocks that are known to be uptodate */1875 if (test_bit(R5_InJournal, &dev->flags)) {1876 /*1877 * For this case, PAGE_SIZE must be equal to 4KB and1878 * page offset is zero.1879 */1880 off_srcs[count] = dev->offset;1881 xor_srcs[count++] = dev->orig_page;1882 } else if (test_bit(R5_Wantdrain, &dev->flags)) {1883 off_srcs[count] = dev->offset;1884 xor_srcs[count++] = dev->page;1885 }1886 }1887 1888 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,1889 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));1890 tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,1891 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);1892 1893 return tx;1894}1895 1896static struct dma_async_tx_descriptor *1897ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,1898 struct dma_async_tx_descriptor *tx)1899{1900 struct page **blocks = to_addr_page(percpu, 0);1901 unsigned int *offs = to_addr_offs(sh, percpu);1902 int count;1903 struct async_submit_ctl submit;1904 1905 pr_debug("%s: stripe %llu\n", __func__,1906 (unsigned long long)sh->sector);1907 1908 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);1909 1910 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,1911 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));1912 tx = async_gen_syndrome(blocks, offs, count+2,1913 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);1914 1915 return tx;1916}1917 1918static struct dma_async_tx_descriptor *1919ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)1920{1921 struct r5conf *conf = sh->raid_conf;1922 int disks = sh->disks;1923 int i;1924 struct stripe_head *head_sh = sh;1925 1926 pr_debug("%s: stripe %llu\n", __func__,1927 (unsigned long long)sh->sector);1928 1929 for (i = disks; i--; ) {1930 struct r5dev *dev;1931 struct bio *chosen;1932 1933 sh = head_sh;1934 if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {1935 struct bio *wbi;1936 1937again:1938 dev = &sh->dev[i];1939 /*1940 * clear R5_InJournal, so when rewriting a page in1941 * journal, it is not skipped by r5l_log_stripe()1942 */1943 clear_bit(R5_InJournal, &dev->flags);1944 spin_lock_irq(&sh->stripe_lock);1945 chosen = dev->towrite;1946 dev->towrite = NULL;1947 sh->overwrite_disks = 0;1948 BUG_ON(dev->written);1949 wbi = dev->written = chosen;1950 spin_unlock_irq(&sh->stripe_lock);1951 WARN_ON(dev->page != dev->orig_page);1952 1953 while (wbi && wbi->bi_iter.bi_sector <1954 dev->sector + RAID5_STRIPE_SECTORS(conf)) {1955 if (wbi->bi_opf & REQ_FUA)1956 set_bit(R5_WantFUA, &dev->flags);1957 if (wbi->bi_opf & REQ_SYNC)1958 set_bit(R5_SyncIO, &dev->flags);1959 if (bio_op(wbi) == REQ_OP_DISCARD)1960 set_bit(R5_Discard, &dev->flags);1961 else {1962 tx = async_copy_data(1, wbi, &dev->page,1963 dev->offset,1964 dev->sector, tx, sh,1965 r5c_is_writeback(conf->log));1966 if (dev->page != dev->orig_page &&1967 !r5c_is_writeback(conf->log)) {1968 set_bit(R5_SkipCopy, &dev->flags);1969 clear_bit(R5_UPTODATE, &dev->flags);1970 clear_bit(R5_OVERWRITE, &dev->flags);1971 }1972 }1973 wbi = r5_next_bio(conf, wbi, dev->sector);1974 }1975 1976 if (head_sh->batch_head) {1977 sh = list_first_entry(&sh->batch_list,1978 struct stripe_head,1979 batch_list);1980 if (sh == head_sh)1981 continue;1982 goto again;1983 }1984 }1985 }1986 1987 return tx;1988}1989 1990static void ops_complete_reconstruct(void *stripe_head_ref)1991{1992 struct stripe_head *sh = stripe_head_ref;1993 int disks = sh->disks;1994 int pd_idx = sh->pd_idx;1995 int qd_idx = sh->qd_idx;1996 int i;1997 bool fua = false, sync = false, discard = false;1998 1999 pr_debug("%s: stripe %llu\n", __func__,2000 (unsigned long long)sh->sector);2001 2002 for (i = disks; i--; ) {2003 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);2004 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);2005 discard |= test_bit(R5_Discard, &sh->dev[i].flags);2006 }2007 2008 for (i = disks; i--; ) {2009 struct r5dev *dev = &sh->dev[i];2010 2011 if (dev->written || i == pd_idx || i == qd_idx) {2012 if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) {2013 set_bit(R5_UPTODATE, &dev->flags);2014 if (test_bit(STRIPE_EXPAND_READY, &sh->state))2015 set_bit(R5_Expanded, &dev->flags);2016 }2017 if (fua)2018 set_bit(R5_WantFUA, &dev->flags);2019 if (sync)2020 set_bit(R5_SyncIO, &dev->flags);2021 }2022 }2023 2024 if (sh->reconstruct_state == reconstruct_state_drain_run)2025 sh->reconstruct_state = reconstruct_state_drain_result;2026 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)2027 sh->reconstruct_state = reconstruct_state_prexor_drain_result;2028 else {2029 BUG_ON(sh->reconstruct_state != reconstruct_state_run);2030 sh->reconstruct_state = reconstruct_state_result;2031 }2032 2033 set_bit(STRIPE_HANDLE, &sh->state);2034 raid5_release_stripe(sh);2035}2036 2037static void2038ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,2039 struct dma_async_tx_descriptor *tx)2040{2041 int disks = sh->disks;2042 struct page **xor_srcs;2043 unsigned int *off_srcs;2044 struct async_submit_ctl submit;2045 int count, pd_idx = sh->pd_idx, i;2046 struct page *xor_dest;2047 unsigned int off_dest;2048 int prexor = 0;2049 unsigned long flags;2050 int j = 0;2051 struct stripe_head *head_sh = sh;2052 int last_stripe;2053 2054 pr_debug("%s: stripe %llu\n", __func__,2055 (unsigned long long)sh->sector);2056 2057 for (i = 0; i < sh->disks; i++) {2058 if (pd_idx == i)2059 continue;2060 if (!test_bit(R5_Discard, &sh->dev[i].flags))2061 break;2062 }2063 if (i >= sh->disks) {2064 atomic_inc(&sh->count);2065 set_bit(R5_Discard, &sh->dev[pd_idx].flags);2066 ops_complete_reconstruct(sh);2067 return;2068 }2069again:2070 count = 0;2071 xor_srcs = to_addr_page(percpu, j);2072 off_srcs = to_addr_offs(sh, percpu);2073 /* check if prexor is active which means only process blocks2074 * that are part of a read-modify-write (written)2075 */2076 if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {2077 prexor = 1;2078 off_dest = off_srcs[count] = sh->dev[pd_idx].offset;2079 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;2080 for (i = disks; i--; ) {2081 struct r5dev *dev = &sh->dev[i];2082 if (head_sh->dev[i].written ||2083 test_bit(R5_InJournal, &head_sh->dev[i].flags)) {2084 off_srcs[count] = dev->offset;2085 xor_srcs[count++] = dev->page;2086 }2087 }2088 } else {2089 xor_dest = sh->dev[pd_idx].page;2090 off_dest = sh->dev[pd_idx].offset;2091 for (i = disks; i--; ) {2092 struct r5dev *dev = &sh->dev[i];2093 if (i != pd_idx) {2094 off_srcs[count] = dev->offset;2095 xor_srcs[count++] = dev->page;2096 }2097 }2098 }2099 2100 /* 1/ if we prexor'd then the dest is reused as a source2101 * 2/ if we did not prexor then we are redoing the parity2102 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST2103 * for the synchronous xor case2104 */2105 last_stripe = !head_sh->batch_head ||2106 list_first_entry(&sh->batch_list,2107 struct stripe_head, batch_list) == head_sh;2108 if (last_stripe) {2109 flags = ASYNC_TX_ACK |2110 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);2111 2112 atomic_inc(&head_sh->count);2113 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,2114 to_addr_conv(sh, percpu, j));2115 } else {2116 flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;2117 init_async_submit(&submit, flags, tx, NULL, NULL,2118 to_addr_conv(sh, percpu, j));2119 }2120 2121 if (unlikely(count == 1))2122 tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],2123 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);2124 else2125 tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,2126 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);2127 if (!last_stripe) {2128 j++;2129 sh = list_first_entry(&sh->batch_list, struct stripe_head,2130 batch_list);2131 goto again;2132 }2133}2134 2135static void2136ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,2137 struct dma_async_tx_descriptor *tx)2138{2139 struct async_submit_ctl submit;2140 struct page **blocks;2141 unsigned int *offs;2142 int count, i, j = 0;2143 struct stripe_head *head_sh = sh;2144 int last_stripe;2145 int synflags;2146 unsigned long txflags;2147 2148 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);2149 2150 for (i = 0; i < sh->disks; i++) {2151 if (sh->pd_idx == i || sh->qd_idx == i)2152 continue;2153 if (!test_bit(R5_Discard, &sh->dev[i].flags))2154 break;2155 }2156 if (i >= sh->disks) {2157 atomic_inc(&sh->count);2158 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);2159 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);2160 ops_complete_reconstruct(sh);2161 return;2162 }2163 2164again:2165 blocks = to_addr_page(percpu, j);2166 offs = to_addr_offs(sh, percpu);2167 2168 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {2169 synflags = SYNDROME_SRC_WRITTEN;2170 txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;2171 } else {2172 synflags = SYNDROME_SRC_ALL;2173 txflags = ASYNC_TX_ACK;2174 }2175 2176 count = set_syndrome_sources(blocks, offs, sh, synflags);2177 last_stripe = !head_sh->batch_head ||2178 list_first_entry(&sh->batch_list,2179 struct stripe_head, batch_list) == head_sh;2180 2181 if (last_stripe) {2182 atomic_inc(&head_sh->count);2183 init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,2184 head_sh, to_addr_conv(sh, percpu, j));2185 } else2186 init_async_submit(&submit, 0, tx, NULL, NULL,2187 to_addr_conv(sh, percpu, j));2188 tx = async_gen_syndrome(blocks, offs, count+2,2189 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);2190 if (!last_stripe) {2191 j++;2192 sh = list_first_entry(&sh->batch_list, struct stripe_head,2193 batch_list);2194 goto again;2195 }2196}2197 2198static void ops_complete_check(void *stripe_head_ref)2199{2200 struct stripe_head *sh = stripe_head_ref;2201 2202 pr_debug("%s: stripe %llu\n", __func__,2203 (unsigned long long)sh->sector);2204 2205 sh->check_state = check_state_check_result;2206 set_bit(STRIPE_HANDLE, &sh->state);2207 raid5_release_stripe(sh);2208}2209 2210static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)2211{2212 int disks = sh->disks;2213 int pd_idx = sh->pd_idx;2214 int qd_idx = sh->qd_idx;2215 struct page *xor_dest;2216 unsigned int off_dest;2217 struct page **xor_srcs = to_addr_page(percpu, 0);2218 unsigned int *off_srcs = to_addr_offs(sh, percpu);2219 struct dma_async_tx_descriptor *tx;2220 struct async_submit_ctl submit;2221 int count;2222 int i;2223 2224 pr_debug("%s: stripe %llu\n", __func__,2225 (unsigned long long)sh->sector);2226 2227 BUG_ON(sh->batch_head);2228 count = 0;2229 xor_dest = sh->dev[pd_idx].page;2230 off_dest = sh->dev[pd_idx].offset;2231 off_srcs[count] = off_dest;2232 xor_srcs[count++] = xor_dest;2233 for (i = disks; i--; ) {2234 if (i == pd_idx || i == qd_idx)2235 continue;2236 off_srcs[count] = sh->dev[i].offset;2237 xor_srcs[count++] = sh->dev[i].page;2238 }2239 2240 init_async_submit(&submit, 0, NULL, NULL, NULL,2241 to_addr_conv(sh, percpu, 0));2242 tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,2243 RAID5_STRIPE_SIZE(sh->raid_conf),2244 &sh->ops.zero_sum_result, &submit);2245 2246 atomic_inc(&sh->count);2247 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);2248 tx = async_trigger_callback(&submit);2249}2250 2251static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)2252{2253 struct page **srcs = to_addr_page(percpu, 0);2254 unsigned int *offs = to_addr_offs(sh, percpu);2255 struct async_submit_ctl submit;2256 int count;2257 2258 pr_debug("%s: stripe %llu checkp: %d\n", __func__,2259 (unsigned long long)sh->sector, checkp);2260 2261 BUG_ON(sh->batch_head);2262 count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);2263 if (!checkp)2264 srcs[count] = NULL;2265 2266 atomic_inc(&sh->count);2267 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,2268 sh, to_addr_conv(sh, percpu, 0));2269 async_syndrome_val(srcs, offs, count+2,2270 RAID5_STRIPE_SIZE(sh->raid_conf),2271 &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);2272}2273 2274static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)2275{2276 int overlap_clear = 0, i, disks = sh->disks;2277 struct dma_async_tx_descriptor *tx = NULL;2278 struct r5conf *conf = sh->raid_conf;2279 int level = conf->level;2280 struct raid5_percpu *percpu;2281 2282 local_lock(&conf->percpu->lock);2283 percpu = this_cpu_ptr(conf->percpu);2284 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {2285 ops_run_biofill(sh);2286 overlap_clear++;2287 }2288 2289 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {2290 if (level < 6)2291 tx = ops_run_compute5(sh, percpu);2292 else {2293 if (sh->ops.target2 < 0 || sh->ops.target < 0)2294 tx = ops_run_compute6_1(sh, percpu);2295 else2296 tx = ops_run_compute6_2(sh, percpu);2297 }2298 /* terminate the chain if reconstruct is not set to be run */2299 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))2300 async_tx_ack(tx);2301 }2302 2303 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {2304 if (level < 6)2305 tx = ops_run_prexor5(sh, percpu, tx);2306 else2307 tx = ops_run_prexor6(sh, percpu, tx);2308 }2309 2310 if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))2311 tx = ops_run_partial_parity(sh, percpu, tx);2312 2313 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {2314 tx = ops_run_biodrain(sh, tx);2315 overlap_clear++;2316 }2317 2318 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {2319 if (level < 6)2320 ops_run_reconstruct5(sh, percpu, tx);2321 else2322 ops_run_reconstruct6(sh, percpu, tx);2323 }2324 2325 if (test_bit(STRIPE_OP_CHECK, &ops_request)) {2326 if (sh->check_state == check_state_run)2327 ops_run_check_p(sh, percpu);2328 else if (sh->check_state == check_state_run_q)2329 ops_run_check_pq(sh, percpu, 0);2330 else if (sh->check_state == check_state_run_pq)2331 ops_run_check_pq(sh, percpu, 1);2332 else2333 BUG();2334 }2335 2336 if (overlap_clear && !sh->batch_head) {2337 for (i = disks; i--; ) {2338 struct r5dev *dev = &sh->dev[i];2339 if (test_and_clear_bit(R5_Overlap, &dev->flags))2340 wake_up_bit(&dev->flags, R5_Overlap);2341 }2342 }2343 local_unlock(&conf->percpu->lock);2344}2345 2346static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)2347{2348#if PAGE_SIZE != DEFAULT_STRIPE_SIZE2349 kfree(sh->pages);2350#endif2351 if (sh->ppl_page)2352 __free_page(sh->ppl_page);2353 kmem_cache_free(sc, sh);2354}2355 2356static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,2357 int disks, struct r5conf *conf)2358{2359 struct stripe_head *sh;2360 2361 sh = kmem_cache_zalloc(sc, gfp);2362 if (sh) {2363 spin_lock_init(&sh->stripe_lock);2364 spin_lock_init(&sh->batch_lock);2365 INIT_LIST_HEAD(&sh->batch_list);2366 INIT_LIST_HEAD(&sh->lru);2367 INIT_LIST_HEAD(&sh->r5c);2368 INIT_LIST_HEAD(&sh->log_list);2369 atomic_set(&sh->count, 1);2370 sh->raid_conf = conf;2371 sh->log_start = MaxSector;2372 2373 if (raid5_has_ppl(conf)) {2374 sh->ppl_page = alloc_page(gfp);2375 if (!sh->ppl_page) {2376 free_stripe(sc, sh);2377 return NULL;2378 }2379 }2380#if PAGE_SIZE != DEFAULT_STRIPE_SIZE2381 if (init_stripe_shared_pages(sh, conf, disks)) {2382 free_stripe(sc, sh);2383 return NULL;2384 }2385#endif2386 }2387 return sh;2388}2389static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)2390{2391 struct stripe_head *sh;2392 2393 sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);2394 if (!sh)2395 return 0;2396 2397 if (grow_buffers(sh, gfp)) {2398 shrink_buffers(sh);2399 free_stripe(conf->slab_cache, sh);2400 return 0;2401 }2402 sh->hash_lock_index =2403 conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;2404 /* we just created an active stripe so... */2405 atomic_inc(&conf->active_stripes);2406 2407 raid5_release_stripe(sh);2408 WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes + 1);2409 return 1;2410}2411 2412static int grow_stripes(struct r5conf *conf, int num)2413{2414 struct kmem_cache *sc;2415 size_t namelen = sizeof(conf->cache_name[0]);2416 int devs = max(conf->raid_disks, conf->previous_raid_disks);2417 2418 if (mddev_is_dm(conf->mddev))2419 snprintf(conf->cache_name[0], namelen,2420 "raid%d-%p", conf->level, conf->mddev);2421 else2422 snprintf(conf->cache_name[0], namelen,2423 "raid%d-%s", conf->level, mdname(conf->mddev));2424 snprintf(conf->cache_name[1], namelen, "%.27s-alt", conf->cache_name[0]);2425 2426 conf->active_name = 0;2427 sc = kmem_cache_create(conf->cache_name[conf->active_name],2428 struct_size_t(struct stripe_head, dev, devs),2429 0, 0, NULL);2430 if (!sc)2431 return 1;2432 conf->slab_cache = sc;2433 conf->pool_size = devs;2434 while (num--)2435 if (!grow_one_stripe(conf, GFP_KERNEL))2436 return 1;2437 2438 return 0;2439}2440 2441/**2442 * scribble_alloc - allocate percpu scribble buffer for required size2443 * of the scribble region2444 * @percpu: from for_each_present_cpu() of the caller2445 * @num: total number of disks in the array2446 * @cnt: scribble objs count for required size of the scribble region2447 *2448 * The scribble buffer size must be enough to contain:2449 * 1/ a struct page pointer for each device in the array +22450 * 2/ room to convert each entry in (1) to its corresponding dma2451 * (dma_map_page()) or page (page_address()) address.2452 *2453 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we2454 * calculate over all devices (not just the data blocks), using zeros in place2455 * of the P and Q blocks.2456 */2457static int scribble_alloc(struct raid5_percpu *percpu,2458 int num, int cnt)2459{2460 size_t obj_size =2461 sizeof(struct page *) * (num + 2) +2462 sizeof(addr_conv_t) * (num + 2) +2463 sizeof(unsigned int) * (num + 2);2464 void *scribble;2465 2466 /*2467 * If here is in raid array suspend context, it is in memalloc noio2468 * context as well, there is no potential recursive memory reclaim2469 * I/Os with the GFP_KERNEL flag.2470 */2471 scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);2472 if (!scribble)2473 return -ENOMEM;2474 2475 kvfree(percpu->scribble);2476 2477 percpu->scribble = scribble;2478 percpu->scribble_obj_size = obj_size;2479 return 0;2480}2481 2482static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)2483{2484 unsigned long cpu;2485 int err = 0;2486 2487 /* Never shrink. */2488 if (conf->scribble_disks >= new_disks &&2489 conf->scribble_sectors >= new_sectors)2490 return 0;2491 2492 raid5_quiesce(conf->mddev, true);2493 cpus_read_lock();2494 2495 for_each_present_cpu(cpu) {2496 struct raid5_percpu *percpu;2497 2498 percpu = per_cpu_ptr(conf->percpu, cpu);2499 err = scribble_alloc(percpu, new_disks,2500 new_sectors / RAID5_STRIPE_SECTORS(conf));2501 if (err)2502 break;2503 }2504 2505 cpus_read_unlock();2506 raid5_quiesce(conf->mddev, false);2507 2508 if (!err) {2509 conf->scribble_disks = new_disks;2510 conf->scribble_sectors = new_sectors;2511 }2512 return err;2513}2514 2515static int resize_stripes(struct r5conf *conf, int newsize)2516{2517 /* Make all the stripes able to hold 'newsize' devices.2518 * New slots in each stripe get 'page' set to a new page.2519 *2520 * This happens in stages:2521 * 1/ create a new kmem_cache and allocate the required number of2522 * stripe_heads.2523 * 2/ gather all the old stripe_heads and transfer the pages across2524 * to the new stripe_heads. This will have the side effect of2525 * freezing the array as once all stripe_heads have been collected,2526 * no IO will be possible. Old stripe heads are freed once their2527 * pages have been transferred over, and the old kmem_cache is2528 * freed when all stripes are done.2529 * 3/ reallocate conf->disks to be suitable bigger. If this fails,2530 * we simple return a failure status - no need to clean anything up.2531 * 4/ allocate new pages for the new slots in the new stripe_heads.2532 * If this fails, we don't bother trying the shrink the2533 * stripe_heads down again, we just leave them as they are.2534 * As each stripe_head is processed the new one is released into2535 * active service.2536 *2537 * Once step2 is started, we cannot afford to wait for a write,2538 * so we use GFP_NOIO allocations.2539 */2540 struct stripe_head *osh, *nsh;2541 LIST_HEAD(newstripes);2542 struct disk_info *ndisks;2543 int err = 0;2544 struct kmem_cache *sc;2545 int i;2546 int hash, cnt;2547 2548 md_allow_write(conf->mddev);2549 2550 /* Step 1 */2551 sc = kmem_cache_create(conf->cache_name[1-conf->active_name],2552 struct_size_t(struct stripe_head, dev, newsize),2553 0, 0, NULL);2554 if (!sc)2555 return -ENOMEM;2556 2557 /* Need to ensure auto-resizing doesn't interfere */2558 mutex_lock(&conf->cache_size_mutex);2559 2560 for (i = conf->max_nr_stripes; i; i--) {2561 nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);2562 if (!nsh)2563 break;2564 2565 list_add(&nsh->lru, &newstripes);2566 }2567 if (i) {2568 /* didn't get enough, give up */2569 while (!list_empty(&newstripes)) {2570 nsh = list_entry(newstripes.next, struct stripe_head, lru);2571 list_del(&nsh->lru);2572 free_stripe(sc, nsh);2573 }2574 kmem_cache_destroy(sc);2575 mutex_unlock(&conf->cache_size_mutex);2576 return -ENOMEM;2577 }2578 /* Step 2 - Must use GFP_NOIO now.2579 * OK, we have enough stripes, start collecting inactive2580 * stripes and copying them over2581 */2582 hash = 0;2583 cnt = 0;2584 list_for_each_entry(nsh, &newstripes, lru) {2585 lock_device_hash_lock(conf, hash);2586 wait_event_cmd(conf->wait_for_stripe,2587 !list_empty(conf->inactive_list + hash),2588 unlock_device_hash_lock(conf, hash),2589 lock_device_hash_lock(conf, hash));2590 osh = get_free_stripe(conf, hash);2591 unlock_device_hash_lock(conf, hash);2592 2593#if PAGE_SIZE != DEFAULT_STRIPE_SIZE2594 for (i = 0; i < osh->nr_pages; i++) {2595 nsh->pages[i] = osh->pages[i];2596 osh->pages[i] = NULL;2597 }2598#endif2599 for(i=0; i<conf->pool_size; i++) {2600 nsh->dev[i].page = osh->dev[i].page;2601 nsh->dev[i].orig_page = osh->dev[i].page;2602 nsh->dev[i].offset = osh->dev[i].offset;2603 }2604 nsh->hash_lock_index = hash;2605 free_stripe(conf->slab_cache, osh);2606 cnt++;2607 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +2608 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {2609 hash++;2610 cnt = 0;2611 }2612 }2613 kmem_cache_destroy(conf->slab_cache);2614 2615 /* Step 3.2616 * At this point, we are holding all the stripes so the array2617 * is completely stalled, so now is a good time to resize2618 * conf->disks and the scribble region2619 */2620 ndisks = kcalloc(newsize, sizeof(struct disk_info), GFP_NOIO);2621 if (ndisks) {2622 for (i = 0; i < conf->pool_size; i++)2623 ndisks[i] = conf->disks[i];2624 2625 for (i = conf->pool_size; i < newsize; i++) {2626 ndisks[i].extra_page = alloc_page(GFP_NOIO);2627 if (!ndisks[i].extra_page)2628 err = -ENOMEM;2629 }2630 2631 if (err) {2632 for (i = conf->pool_size; i < newsize; i++)2633 if (ndisks[i].extra_page)2634 put_page(ndisks[i].extra_page);2635 kfree(ndisks);2636 } else {2637 kfree(conf->disks);2638 conf->disks = ndisks;2639 }2640 } else2641 err = -ENOMEM;2642 2643 conf->slab_cache = sc;2644 conf->active_name = 1-conf->active_name;2645 2646 /* Step 4, return new stripes to service */2647 while(!list_empty(&newstripes)) {2648 nsh = list_entry(newstripes.next, struct stripe_head, lru);2649 list_del_init(&nsh->lru);2650 2651#if PAGE_SIZE != DEFAULT_STRIPE_SIZE2652 for (i = 0; i < nsh->nr_pages; i++) {2653 if (nsh->pages[i])2654 continue;2655 nsh->pages[i] = alloc_page(GFP_NOIO);2656 if (!nsh->pages[i])2657 err = -ENOMEM;2658 }2659 2660 for (i = conf->raid_disks; i < newsize; i++) {2661 if (nsh->dev[i].page)2662 continue;2663 nsh->dev[i].page = raid5_get_dev_page(nsh, i);2664 nsh->dev[i].orig_page = nsh->dev[i].page;2665 nsh->dev[i].offset = raid5_get_page_offset(nsh, i);2666 }2667#else2668 for (i=conf->raid_disks; i < newsize; i++)2669 if (nsh->dev[i].page == NULL) {2670 struct page *p = alloc_page(GFP_NOIO);2671 nsh->dev[i].page = p;2672 nsh->dev[i].orig_page = p;2673 nsh->dev[i].offset = 0;2674 if (!p)2675 err = -ENOMEM;2676 }2677#endif2678 raid5_release_stripe(nsh);2679 }2680 /* critical section pass, GFP_NOIO no longer needed */2681 2682 if (!err)2683 conf->pool_size = newsize;2684 mutex_unlock(&conf->cache_size_mutex);2685 2686 return err;2687}2688 2689static int drop_one_stripe(struct r5conf *conf)2690{2691 struct stripe_head *sh;2692 int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;2693 2694 spin_lock_irq(conf->hash_locks + hash);2695 sh = get_free_stripe(conf, hash);2696 spin_unlock_irq(conf->hash_locks + hash);2697 if (!sh)2698 return 0;2699 BUG_ON(atomic_read(&sh->count));2700 shrink_buffers(sh);2701 free_stripe(conf->slab_cache, sh);2702 atomic_dec(&conf->active_stripes);2703 WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes - 1);2704 return 1;2705}2706 2707static void shrink_stripes(struct r5conf *conf)2708{2709 while (conf->max_nr_stripes &&2710 drop_one_stripe(conf))2711 ;2712 2713 kmem_cache_destroy(conf->slab_cache);2714 conf->slab_cache = NULL;2715}2716 2717static void raid5_end_read_request(struct bio * bi)2718{2719 struct stripe_head *sh = bi->bi_private;2720 struct r5conf *conf = sh->raid_conf;2721 int disks = sh->disks, i;2722 struct md_rdev *rdev = NULL;2723 sector_t s;2724 2725 for (i=0 ; i<disks; i++)2726 if (bi == &sh->dev[i].req)2727 break;2728 2729 pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",2730 (unsigned long long)sh->sector, i, atomic_read(&sh->count),2731 bi->bi_status);2732 if (i == disks) {2733 BUG();2734 return;2735 }2736 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))2737 /* If replacement finished while this request was outstanding,2738 * 'replacement' might be NULL already.2739 * In that case it moved down to 'rdev'.2740 * rdev is not removed until all requests are finished.2741 */2742 rdev = conf->disks[i].replacement;2743 if (!rdev)2744 rdev = conf->disks[i].rdev;2745 2746 if (use_new_offset(conf, sh))2747 s = sh->sector + rdev->new_data_offset;2748 else2749 s = sh->sector + rdev->data_offset;2750 if (!bi->bi_status) {2751 set_bit(R5_UPTODATE, &sh->dev[i].flags);2752 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {2753 /* Note that this cannot happen on a2754 * replacement device. We just fail those on2755 * any error2756 */2757 pr_info_ratelimited(2758 "md/raid:%s: read error corrected (%lu sectors at %llu on %pg)\n",2759 mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),2760 (unsigned long long)s,2761 rdev->bdev);2762 atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);2763 clear_bit(R5_ReadError, &sh->dev[i].flags);2764 clear_bit(R5_ReWrite, &sh->dev[i].flags);2765 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))2766 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);2767 2768 if (test_bit(R5_InJournal, &sh->dev[i].flags))2769 /*2770 * end read for a page in journal, this2771 * must be preparing for prexor in rmw2772 */2773 set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);2774 2775 if (atomic_read(&rdev->read_errors))2776 atomic_set(&rdev->read_errors, 0);2777 } else {2778 int retry = 0;2779 int set_bad = 0;2780 2781 clear_bit(R5_UPTODATE, &sh->dev[i].flags);2782 if (!(bi->bi_status == BLK_STS_PROTECTION))2783 atomic_inc(&rdev->read_errors);2784 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))2785 pr_warn_ratelimited(2786 "md/raid:%s: read error on replacement device (sector %llu on %pg).\n",2787 mdname(conf->mddev),2788 (unsigned long long)s,2789 rdev->bdev);2790 else if (conf->mddev->degraded >= conf->max_degraded) {2791 set_bad = 1;2792 pr_warn_ratelimited(2793 "md/raid:%s: read error not correctable (sector %llu on %pg).\n",2794 mdname(conf->mddev),2795 (unsigned long long)s,2796 rdev->bdev);2797 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {2798 /* Oh, no!!! */2799 set_bad = 1;2800 pr_warn_ratelimited(2801 "md/raid:%s: read error NOT corrected!! (sector %llu on %pg).\n",2802 mdname(conf->mddev),2803 (unsigned long long)s,2804 rdev->bdev);2805 } else if (atomic_read(&rdev->read_errors)2806 > conf->max_nr_stripes) {2807 if (!test_bit(Faulty, &rdev->flags)) {2808 pr_warn("md/raid:%s: %d read_errors > %d stripes\n",2809 mdname(conf->mddev),2810 atomic_read(&rdev->read_errors),2811 conf->max_nr_stripes);2812 pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n",2813 mdname(conf->mddev), rdev->bdev);2814 }2815 } else2816 retry = 1;2817 if (set_bad && test_bit(In_sync, &rdev->flags)2818 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))2819 retry = 1;2820 if (retry)2821 if (sh->qd_idx >= 0 && sh->pd_idx == i)2822 set_bit(R5_ReadError, &sh->dev[i].flags);2823 else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {2824 set_bit(R5_ReadError, &sh->dev[i].flags);2825 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);2826 } else2827 set_bit(R5_ReadNoMerge, &sh->dev[i].flags);2828 else {2829 clear_bit(R5_ReadError, &sh->dev[i].flags);2830 clear_bit(R5_ReWrite, &sh->dev[i].flags);2831 if (!(set_bad2832 && test_bit(In_sync, &rdev->flags)2833 && rdev_set_badblocks(2834 rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))2835 md_error(conf->mddev, rdev);2836 }2837 }2838 rdev_dec_pending(rdev, conf->mddev);2839 bio_uninit(bi);2840 clear_bit(R5_LOCKED, &sh->dev[i].flags);2841 set_bit(STRIPE_HANDLE, &sh->state);2842 raid5_release_stripe(sh);2843}2844 2845static void raid5_end_write_request(struct bio *bi)2846{2847 struct stripe_head *sh = bi->bi_private;2848 struct r5conf *conf = sh->raid_conf;2849 int disks = sh->disks, i;2850 struct md_rdev *rdev;2851 int replacement = 0;2852 2853 for (i = 0 ; i < disks; i++) {2854 if (bi == &sh->dev[i].req) {2855 rdev = conf->disks[i].rdev;2856 break;2857 }2858 if (bi == &sh->dev[i].rreq) {2859 rdev = conf->disks[i].replacement;2860 if (rdev)2861 replacement = 1;2862 else2863 /* rdev was removed and 'replacement'2864 * replaced it. rdev is not removed2865 * until all requests are finished.2866 */2867 rdev = conf->disks[i].rdev;2868 break;2869 }2870 }2871 pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",2872 (unsigned long long)sh->sector, i, atomic_read(&sh->count),2873 bi->bi_status);2874 if (i == disks) {2875 BUG();2876 return;2877 }2878 2879 if (replacement) {2880 if (bi->bi_status)2881 md_error(conf->mddev, rdev);2882 else if (rdev_has_badblock(rdev, sh->sector,2883 RAID5_STRIPE_SECTORS(conf)))2884 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);2885 } else {2886 if (bi->bi_status) {2887 set_bit(STRIPE_DEGRADED, &sh->state);2888 set_bit(WriteErrorSeen, &rdev->flags);2889 set_bit(R5_WriteError, &sh->dev[i].flags);2890 if (!test_and_set_bit(WantReplacement, &rdev->flags))2891 set_bit(MD_RECOVERY_NEEDED,2892 &rdev->mddev->recovery);2893 } else if (rdev_has_badblock(rdev, sh->sector,2894 RAID5_STRIPE_SECTORS(conf))) {2895 set_bit(R5_MadeGood, &sh->dev[i].flags);2896 if (test_bit(R5_ReadError, &sh->dev[i].flags))2897 /* That was a successful write so make2898 * sure it looks like we already did2899 * a re-write.2900 */2901 set_bit(R5_ReWrite, &sh->dev[i].flags);2902 }2903 }2904 rdev_dec_pending(rdev, conf->mddev);2905 2906 if (sh->batch_head && bi->bi_status && !replacement)2907 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);2908 2909 bio_uninit(bi);2910 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))2911 clear_bit(R5_LOCKED, &sh->dev[i].flags);2912 set_bit(STRIPE_HANDLE, &sh->state);2913 2914 if (sh->batch_head && sh != sh->batch_head)2915 raid5_release_stripe(sh->batch_head);2916 raid5_release_stripe(sh);2917}2918 2919static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)2920{2921 struct r5conf *conf = mddev->private;2922 unsigned long flags;2923 pr_debug("raid456: error called\n");2924 2925 pr_crit("md/raid:%s: Disk failure on %pg, disabling device.\n",2926 mdname(mddev), rdev->bdev);2927 2928 spin_lock_irqsave(&conf->device_lock, flags);2929 set_bit(Faulty, &rdev->flags);2930 clear_bit(In_sync, &rdev->flags);2931 mddev->degraded = raid5_calc_degraded(conf);2932 2933 if (has_failed(conf)) {2934 set_bit(MD_BROKEN, &conf->mddev->flags);2935 conf->recovery_disabled = mddev->recovery_disabled;2936 2937 pr_crit("md/raid:%s: Cannot continue operation (%d/%d failed).\n",2938 mdname(mddev), mddev->degraded, conf->raid_disks);2939 } else {2940 pr_crit("md/raid:%s: Operation continuing on %d devices.\n",2941 mdname(mddev), conf->raid_disks - mddev->degraded);2942 }2943 2944 spin_unlock_irqrestore(&conf->device_lock, flags);2945 set_bit(MD_RECOVERY_INTR, &mddev->recovery);2946 2947 set_bit(Blocked, &rdev->flags);2948 set_mask_bits(&mddev->sb_flags, 0,2949 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));2950 r5c_update_on_rdev_error(mddev, rdev);2951}2952 2953/*2954 * Input: a 'big' sector number,2955 * Output: index of the data and parity disk, and the sector # in them.2956 */2957sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,2958 int previous, int *dd_idx,2959 struct stripe_head *sh)2960{2961 sector_t stripe, stripe2;2962 sector_t chunk_number;2963 unsigned int chunk_offset;2964 int pd_idx, qd_idx;2965 int ddf_layout = 0;2966 sector_t new_sector;2967 int algorithm = previous ? conf->prev_algo2968 : conf->algorithm;2969 int sectors_per_chunk = previous ? conf->prev_chunk_sectors2970 : conf->chunk_sectors;2971 int raid_disks = previous ? conf->previous_raid_disks2972 : conf->raid_disks;2973 int data_disks = raid_disks - conf->max_degraded;2974 2975 /* First compute the information on this sector */2976 2977 /*2978 * Compute the chunk number and the sector offset inside the chunk2979 */2980 chunk_offset = sector_div(r_sector, sectors_per_chunk);2981 chunk_number = r_sector;2982 2983 /*2984 * Compute the stripe number2985 */2986 stripe = chunk_number;2987 *dd_idx = sector_div(stripe, data_disks);2988 stripe2 = stripe;2989 /*2990 * Select the parity disk based on the user selected algorithm.2991 */2992 pd_idx = qd_idx = -1;2993 switch(conf->level) {2994 case 4:2995 pd_idx = data_disks;2996 break;2997 case 5:2998 switch (algorithm) {2999 case ALGORITHM_LEFT_ASYMMETRIC:3000 pd_idx = data_disks - sector_div(stripe2, raid_disks);3001 if (*dd_idx >= pd_idx)3002 (*dd_idx)++;3003 break;3004 case ALGORITHM_RIGHT_ASYMMETRIC:3005 pd_idx = sector_div(stripe2, raid_disks);3006 if (*dd_idx >= pd_idx)3007 (*dd_idx)++;3008 break;3009 case ALGORITHM_LEFT_SYMMETRIC:3010 pd_idx = data_disks - sector_div(stripe2, raid_disks);3011 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;3012 break;3013 case ALGORITHM_RIGHT_SYMMETRIC:3014 pd_idx = sector_div(stripe2, raid_disks);3015 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;3016 break;3017 case ALGORITHM_PARITY_0:3018 pd_idx = 0;3019 (*dd_idx)++;3020 break;3021 case ALGORITHM_PARITY_N:3022 pd_idx = data_disks;3023 break;3024 default:3025 BUG();3026 }3027 break;3028 case 6:3029 3030 switch (algorithm) {3031 case ALGORITHM_LEFT_ASYMMETRIC:3032 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);3033 qd_idx = pd_idx + 1;3034 if (pd_idx == raid_disks-1) {3035 (*dd_idx)++; /* Q D D D P */3036 qd_idx = 0;3037 } else if (*dd_idx >= pd_idx)3038 (*dd_idx) += 2; /* D D P Q D */3039 break;3040 case ALGORITHM_RIGHT_ASYMMETRIC:3041 pd_idx = sector_div(stripe2, raid_disks);3042 qd_idx = pd_idx + 1;3043 if (pd_idx == raid_disks-1) {3044 (*dd_idx)++; /* Q D D D P */3045 qd_idx = 0;3046 } else if (*dd_idx >= pd_idx)3047 (*dd_idx) += 2; /* D D P Q D */3048 break;3049 case ALGORITHM_LEFT_SYMMETRIC:3050 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);3051 qd_idx = (pd_idx + 1) % raid_disks;3052 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;3053 break;3054 case ALGORITHM_RIGHT_SYMMETRIC:3055 pd_idx = sector_div(stripe2, raid_disks);3056 qd_idx = (pd_idx + 1) % raid_disks;3057 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;3058 break;3059 3060 case ALGORITHM_PARITY_0:3061 pd_idx = 0;3062 qd_idx = 1;3063 (*dd_idx) += 2;3064 break;3065 case ALGORITHM_PARITY_N:3066 pd_idx = data_disks;3067 qd_idx = data_disks + 1;3068 break;3069 3070 case ALGORITHM_ROTATING_ZERO_RESTART:3071 /* Exactly the same as RIGHT_ASYMMETRIC, but or3072 * of blocks for computing Q is different.3073 */3074 pd_idx = sector_div(stripe2, raid_disks);3075 qd_idx = pd_idx + 1;3076 if (pd_idx == raid_disks-1) {3077 (*dd_idx)++; /* Q D D D P */3078 qd_idx = 0;3079 } else if (*dd_idx >= pd_idx)3080 (*dd_idx) += 2; /* D D P Q D */3081 ddf_layout = 1;3082 break;3083 3084 case ALGORITHM_ROTATING_N_RESTART:3085 /* Same a left_asymmetric, by first stripe is3086 * D D D P Q rather than3087 * Q D D D P3088 */3089 stripe2 += 1;3090 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);3091 qd_idx = pd_idx + 1;3092 if (pd_idx == raid_disks-1) {3093 (*dd_idx)++; /* Q D D D P */3094 qd_idx = 0;3095 } else if (*dd_idx >= pd_idx)3096 (*dd_idx) += 2; /* D D P Q D */3097 ddf_layout = 1;3098 break;3099 3100 case ALGORITHM_ROTATING_N_CONTINUE:3101 /* Same as left_symmetric but Q is before P */3102 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);3103 qd_idx = (pd_idx + raid_disks - 1) % raid_disks;3104 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;3105 ddf_layout = 1;3106 break;3107 3108 case ALGORITHM_LEFT_ASYMMETRIC_6:3109 /* RAID5 left_asymmetric, with Q on last device */3110 pd_idx = data_disks - sector_div(stripe2, raid_disks-1);3111 if (*dd_idx >= pd_idx)3112 (*dd_idx)++;3113 qd_idx = raid_disks - 1;3114 break;3115 3116 case ALGORITHM_RIGHT_ASYMMETRIC_6:3117 pd_idx = sector_div(stripe2, raid_disks-1);3118 if (*dd_idx >= pd_idx)3119 (*dd_idx)++;3120 qd_idx = raid_disks - 1;3121 break;3122 3123 case ALGORITHM_LEFT_SYMMETRIC_6:3124 pd_idx = data_disks - sector_div(stripe2, raid_disks-1);3125 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);3126 qd_idx = raid_disks - 1;3127 break;3128 3129 case ALGORITHM_RIGHT_SYMMETRIC_6:3130 pd_idx = sector_div(stripe2, raid_disks-1);3131 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);3132 qd_idx = raid_disks - 1;3133 break;3134 3135 case ALGORITHM_PARITY_0_6:3136 pd_idx = 0;3137 (*dd_idx)++;3138 qd_idx = raid_disks - 1;3139 break;3140 3141 default:3142 BUG();3143 }3144 break;3145 }3146 3147 if (sh) {3148 sh->pd_idx = pd_idx;3149 sh->qd_idx = qd_idx;3150 sh->ddf_layout = ddf_layout;3151 }3152 /*3153 * Finally, compute the new sector number3154 */3155 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;3156 return new_sector;3157}3158 3159sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)3160{3161 struct r5conf *conf = sh->raid_conf;3162 int raid_disks = sh->disks;3163 int data_disks = raid_disks - conf->max_degraded;3164 sector_t new_sector = sh->sector, check;3165 int sectors_per_chunk = previous ? conf->prev_chunk_sectors3166 : conf->chunk_sectors;3167 int algorithm = previous ? conf->prev_algo3168 : conf->algorithm;3169 sector_t stripe;3170 int chunk_offset;3171 sector_t chunk_number;3172 int dummy1, dd_idx = i;3173 sector_t r_sector;3174 struct stripe_head sh2;3175 3176 chunk_offset = sector_div(new_sector, sectors_per_chunk);3177 stripe = new_sector;3178 3179 if (i == sh->pd_idx)3180 return 0;3181 switch(conf->level) {3182 case 4: break;3183 case 5:3184 switch (algorithm) {3185 case ALGORITHM_LEFT_ASYMMETRIC:3186 case ALGORITHM_RIGHT_ASYMMETRIC:3187 if (i > sh->pd_idx)3188 i--;3189 break;3190 case ALGORITHM_LEFT_SYMMETRIC:3191 case ALGORITHM_RIGHT_SYMMETRIC:3192 if (i < sh->pd_idx)3193 i += raid_disks;3194 i -= (sh->pd_idx + 1);3195 break;3196 case ALGORITHM_PARITY_0:3197 i -= 1;3198 break;3199 case ALGORITHM_PARITY_N:3200 break;3201 default:3202 BUG();3203 }3204 break;3205 case 6:3206 if (i == sh->qd_idx)3207 return 0; /* It is the Q disk */3208 switch (algorithm) {3209 case ALGORITHM_LEFT_ASYMMETRIC:3210 case ALGORITHM_RIGHT_ASYMMETRIC:3211 case ALGORITHM_ROTATING_ZERO_RESTART:3212 case ALGORITHM_ROTATING_N_RESTART:3213 if (sh->pd_idx == raid_disks-1)3214 i--; /* Q D D D P */3215 else if (i > sh->pd_idx)3216 i -= 2; /* D D P Q D */3217 break;3218 case ALGORITHM_LEFT_SYMMETRIC:3219 case ALGORITHM_RIGHT_SYMMETRIC:3220 if (sh->pd_idx == raid_disks-1)3221 i--; /* Q D D D P */3222 else {3223 /* D D P Q D */3224 if (i < sh->pd_idx)3225 i += raid_disks;3226 i -= (sh->pd_idx + 2);3227 }3228 break;3229 case ALGORITHM_PARITY_0:3230 i -= 2;3231 break;3232 case ALGORITHM_PARITY_N:3233 break;3234 case ALGORITHM_ROTATING_N_CONTINUE:3235 /* Like left_symmetric, but P is before Q */3236 if (sh->pd_idx == 0)3237 i--; /* P D D D Q */3238 else {3239 /* D D Q P D */3240 if (i < sh->pd_idx)3241 i += raid_disks;3242 i -= (sh->pd_idx + 1);3243 }3244 break;3245 case ALGORITHM_LEFT_ASYMMETRIC_6:3246 case ALGORITHM_RIGHT_ASYMMETRIC_6:3247 if (i > sh->pd_idx)3248 i--;3249 break;3250 case ALGORITHM_LEFT_SYMMETRIC_6:3251 case ALGORITHM_RIGHT_SYMMETRIC_6:3252 if (i < sh->pd_idx)3253 i += data_disks + 1;3254 i -= (sh->pd_idx + 1);3255 break;3256 case ALGORITHM_PARITY_0_6:3257 i -= 1;3258 break;3259 default:3260 BUG();3261 }3262 break;3263 }3264 3265 chunk_number = stripe * data_disks + i;3266 r_sector = chunk_number * sectors_per_chunk + chunk_offset;3267 3268 check = raid5_compute_sector(conf, r_sector,3269 previous, &dummy1, &sh2);3270 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx3271 || sh2.qd_idx != sh->qd_idx) {3272 pr_warn("md/raid:%s: compute_blocknr: map not correct\n",3273 mdname(conf->mddev));3274 return 0;3275 }3276 return r_sector;3277}3278 3279/*3280 * There are cases where we want handle_stripe_dirtying() and3281 * schedule_reconstruction() to delay towrite to some dev of a stripe.3282 *3283 * This function checks whether we want to delay the towrite. Specifically,3284 * we delay the towrite when:3285 *3286 * 1. degraded stripe has a non-overwrite to the missing dev, AND this3287 * stripe has data in journal (for other devices).3288 *3289 * In this case, when reading data for the non-overwrite dev, it is3290 * necessary to handle complex rmw of write back cache (prexor with3291 * orig_page, and xor with page). To keep read path simple, we would3292 * like to flush data in journal to RAID disks first, so complex rmw3293 * is handled in the write patch (handle_stripe_dirtying).3294 *3295 * 2. when journal space is critical (R5C_LOG_CRITICAL=1)3296 *3297 * It is important to be able to flush all stripes in raid5-cache.3298 * Therefore, we need reserve some space on the journal device for3299 * these flushes. If flush operation includes pending writes to the3300 * stripe, we need to reserve (conf->raid_disk + 1) pages per stripe3301 * for the flush out. If we exclude these pending writes from flush3302 * operation, we only need (conf->max_degraded + 1) pages per stripe.3303 * Therefore, excluding pending writes in these cases enables more3304 * efficient use of the journal device.3305 *3306 * Note: To make sure the stripe makes progress, we only delay3307 * towrite for stripes with data already in journal (injournal > 0).3308 * When LOG_CRITICAL, stripes with injournal == 0 will be sent to3309 * no_space_stripes list.3310 *3311 * 3. during journal failure3312 * In journal failure, we try to flush all cached data to raid disks3313 * based on data in stripe cache. The array is read-only to upper3314 * layers, so we would skip all pending writes.3315 *3316 */3317static inline bool delay_towrite(struct r5conf *conf,3318 struct r5dev *dev,3319 struct stripe_head_state *s)3320{3321 /* case 1 above */3322 if (!test_bit(R5_OVERWRITE, &dev->flags) &&3323 !test_bit(R5_Insync, &dev->flags) && s->injournal)3324 return true;3325 /* case 2 above */3326 if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&3327 s->injournal > 0)3328 return true;3329 /* case 3 above */3330 if (s->log_failed && s->injournal)3331 return true;3332 return false;3333}3334 3335static void3336schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,3337 int rcw, int expand)3338{3339 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;3340 struct r5conf *conf = sh->raid_conf;3341 int level = conf->level;3342 3343 if (rcw) {3344 /*3345 * In some cases, handle_stripe_dirtying initially decided to3346 * run rmw and allocates extra page for prexor. However, rcw is3347 * cheaper later on. We need to free the extra page now,3348 * because we won't be able to do that in ops_complete_prexor().3349 */3350 r5c_release_extra_page(sh);3351 3352 for (i = disks; i--; ) {3353 struct r5dev *dev = &sh->dev[i];3354 3355 if (dev->towrite && !delay_towrite(conf, dev, s)) {3356 set_bit(R5_LOCKED, &dev->flags);3357 set_bit(R5_Wantdrain, &dev->flags);3358 if (!expand)3359 clear_bit(R5_UPTODATE, &dev->flags);3360 s->locked++;3361 } else if (test_bit(R5_InJournal, &dev->flags)) {3362 set_bit(R5_LOCKED, &dev->flags);3363 s->locked++;3364 }3365 }3366 /* if we are not expanding this is a proper write request, and3367 * there will be bios with new data to be drained into the3368 * stripe cache3369 */3370 if (!expand) {3371 if (!s->locked)3372 /* False alarm, nothing to do */3373 return;3374 sh->reconstruct_state = reconstruct_state_drain_run;3375 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);3376 } else3377 sh->reconstruct_state = reconstruct_state_run;3378 3379 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);3380 3381 if (s->locked + conf->max_degraded == disks)3382 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))3383 atomic_inc(&conf->pending_full_writes);3384 } else {3385 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||3386 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));3387 BUG_ON(level == 6 &&3388 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||3389 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));3390 3391 for (i = disks; i--; ) {3392 struct r5dev *dev = &sh->dev[i];3393 if (i == pd_idx || i == qd_idx)3394 continue;3395 3396 if (dev->towrite &&3397 (test_bit(R5_UPTODATE, &dev->flags) ||3398 test_bit(R5_Wantcompute, &dev->flags))) {3399 set_bit(R5_Wantdrain, &dev->flags);3400 set_bit(R5_LOCKED, &dev->flags);3401 clear_bit(R5_UPTODATE, &dev->flags);3402 s->locked++;3403 } else if (test_bit(R5_InJournal, &dev->flags)) {3404 set_bit(R5_LOCKED, &dev->flags);3405 s->locked++;3406 }3407 }3408 if (!s->locked)3409 /* False alarm - nothing to do */3410 return;3411 sh->reconstruct_state = reconstruct_state_prexor_drain_run;3412 set_bit(STRIPE_OP_PREXOR, &s->ops_request);3413 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);3414 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);3415 }3416 3417 /* keep the parity disk(s) locked while asynchronous operations3418 * are in flight3419 */3420 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);3421 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);3422 s->locked++;3423 3424 if (level == 6) {3425 int qd_idx = sh->qd_idx;3426 struct r5dev *dev = &sh->dev[qd_idx];3427 3428 set_bit(R5_LOCKED, &dev->flags);3429 clear_bit(R5_UPTODATE, &dev->flags);3430 s->locked++;3431 }3432 3433 if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&3434 test_bit(STRIPE_OP_BIODRAIN, &s->ops_request) &&3435 !test_bit(STRIPE_FULL_WRITE, &sh->state) &&3436 test_bit(R5_Insync, &sh->dev[pd_idx].flags))3437 set_bit(STRIPE_OP_PARTIAL_PARITY, &s->ops_request);3438 3439 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",3440 __func__, (unsigned long long)sh->sector,3441 s->locked, s->ops_request);3442}3443 3444static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi,3445 int dd_idx, int forwrite)3446{3447 struct r5conf *conf = sh->raid_conf;3448 struct bio **bip;3449 3450 pr_debug("checking bi b#%llu to stripe s#%llu\n",3451 bi->bi_iter.bi_sector, sh->sector);3452 3453 /* Don't allow new IO added to stripes in batch list */3454 if (sh->batch_head)3455 return true;3456 3457 if (forwrite)3458 bip = &sh->dev[dd_idx].towrite;3459 else3460 bip = &sh->dev[dd_idx].toread;3461 3462 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {3463 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)3464 return true;3465 bip = &(*bip)->bi_next;3466 }3467 3468 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))3469 return true;3470 3471 if (forwrite && raid5_has_ppl(conf)) {3472 /*3473 * With PPL only writes to consecutive data chunks within a3474 * stripe are allowed because for a single stripe_head we can3475 * only have one PPL entry at a time, which describes one data3476 * range. Not really an overlap, but R5_Overlap can be3477 * used to handle this.3478 */3479 sector_t sector;3480 sector_t first = 0;3481 sector_t last = 0;3482 int count = 0;3483 int i;3484 3485 for (i = 0; i < sh->disks; i++) {3486 if (i != sh->pd_idx &&3487 (i == dd_idx || sh->dev[i].towrite)) {3488 sector = sh->dev[i].sector;3489 if (count == 0 || sector < first)3490 first = sector;3491 if (sector > last)3492 last = sector;3493 count++;3494 }3495 }3496 3497 if (first + conf->chunk_sectors * (count - 1) != last)3498 return true;3499 }3500 3501 return false;3502}3503 3504static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,3505 int dd_idx, int forwrite, int previous)3506{3507 struct r5conf *conf = sh->raid_conf;3508 struct bio **bip;3509 int firstwrite = 0;3510 3511 if (forwrite) {3512 bip = &sh->dev[dd_idx].towrite;3513 if (!*bip)3514 firstwrite = 1;3515 } else {3516 bip = &sh->dev[dd_idx].toread;3517 }3518 3519 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector)3520 bip = &(*bip)->bi_next;3521 3522 if (!forwrite || previous)3523 clear_bit(STRIPE_BATCH_READY, &sh->state);3524 3525 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);3526 if (*bip)3527 bi->bi_next = *bip;3528 *bip = bi;3529 bio_inc_remaining(bi);3530 md_write_inc(conf->mddev, bi);3531 3532 if (forwrite) {3533 /* check if page is covered */3534 sector_t sector = sh->dev[dd_idx].sector;3535 for (bi=sh->dev[dd_idx].towrite;3536 sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&3537 bi && bi->bi_iter.bi_sector <= sector;3538 bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {3539 if (bio_end_sector(bi) >= sector)3540 sector = bio_end_sector(bi);3541 }3542 if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))3543 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))3544 sh->overwrite_disks++;3545 }3546 3547 pr_debug("added bi b#%llu to stripe s#%llu, disk %d, logical %llu\n",3548 (*bip)->bi_iter.bi_sector, sh->sector, dd_idx,3549 sh->dev[dd_idx].sector);3550 3551 if (conf->mddev->bitmap && firstwrite) {3552 /* Cannot hold spinlock over bitmap_startwrite,3553 * but must ensure this isn't added to a batch until3554 * we have added to the bitmap and set bm_seq.3555 * So set STRIPE_BITMAP_PENDING to prevent3556 * batching.3557 * If multiple __add_stripe_bio() calls race here they3558 * much all set STRIPE_BITMAP_PENDING. So only the first one3559 * to complete "bitmap_startwrite" gets to set3560 * STRIPE_BIT_DELAY. This is important as once a stripe3561 * is added to a batch, STRIPE_BIT_DELAY cannot be changed3562 * any more.3563 */3564 set_bit(STRIPE_BITMAP_PENDING, &sh->state);3565 spin_unlock_irq(&sh->stripe_lock);3566 conf->mddev->bitmap_ops->startwrite(conf->mddev, sh->sector,3567 RAID5_STRIPE_SECTORS(conf), false);3568 spin_lock_irq(&sh->stripe_lock);3569 clear_bit(STRIPE_BITMAP_PENDING, &sh->state);3570 if (!sh->batch_head) {3571 sh->bm_seq = conf->seq_flush+1;3572 set_bit(STRIPE_BIT_DELAY, &sh->state);3573 }3574 }3575}3576 3577/*3578 * Each stripe/dev can have one or more bios attached.3579 * toread/towrite point to the first in a chain.3580 * The bi_next chain must be in order.3581 */3582static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi,3583 int dd_idx, int forwrite, int previous)3584{3585 spin_lock_irq(&sh->stripe_lock);3586 3587 if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {3588 set_bit(R5_Overlap, &sh->dev[dd_idx].flags);3589 spin_unlock_irq(&sh->stripe_lock);3590 return false;3591 }3592 3593 __add_stripe_bio(sh, bi, dd_idx, forwrite, previous);3594 spin_unlock_irq(&sh->stripe_lock);3595 return true;3596}3597 3598static void end_reshape(struct r5conf *conf);3599 3600static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,3601 struct stripe_head *sh)3602{3603 int sectors_per_chunk =3604 previous ? conf->prev_chunk_sectors : conf->chunk_sectors;3605 int dd_idx;3606 int chunk_offset = sector_div(stripe, sectors_per_chunk);3607 int disks = previous ? conf->previous_raid_disks : conf->raid_disks;3608 3609 raid5_compute_sector(conf,3610 stripe * (disks - conf->max_degraded)3611 *sectors_per_chunk + chunk_offset,3612 previous,3613 &dd_idx, sh);3614}3615 3616static void3617handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,3618 struct stripe_head_state *s, int disks)3619{3620 int i;3621 BUG_ON(sh->batch_head);3622 for (i = disks; i--; ) {3623 struct bio *bi;3624 int bitmap_end = 0;3625 3626 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {3627 struct md_rdev *rdev = conf->disks[i].rdev;3628 3629 if (rdev && test_bit(In_sync, &rdev->flags) &&3630 !test_bit(Faulty, &rdev->flags))3631 atomic_inc(&rdev->nr_pending);3632 else3633 rdev = NULL;3634 if (rdev) {3635 if (!rdev_set_badblocks(3636 rdev,3637 sh->sector,3638 RAID5_STRIPE_SECTORS(conf), 0))3639 md_error(conf->mddev, rdev);3640 rdev_dec_pending(rdev, conf->mddev);3641 }3642 }3643 spin_lock_irq(&sh->stripe_lock);3644 /* fail all writes first */3645 bi = sh->dev[i].towrite;3646 sh->dev[i].towrite = NULL;3647 sh->overwrite_disks = 0;3648 spin_unlock_irq(&sh->stripe_lock);3649 if (bi)3650 bitmap_end = 1;3651 3652 log_stripe_write_finished(sh);3653 3654 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))3655 wake_up_bit(&sh->dev[i].flags, R5_Overlap);3656 3657 while (bi && bi->bi_iter.bi_sector <3658 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {3659 struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);3660 3661 md_write_end(conf->mddev);3662 bio_io_error(bi);3663 bi = nextbi;3664 }3665 if (bitmap_end)3666 conf->mddev->bitmap_ops->endwrite(conf->mddev,3667 sh->sector, RAID5_STRIPE_SECTORS(conf),3668 false, false);3669 bitmap_end = 0;3670 /* and fail all 'written' */3671 bi = sh->dev[i].written;3672 sh->dev[i].written = NULL;3673 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {3674 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));3675 sh->dev[i].page = sh->dev[i].orig_page;3676 }3677 3678 if (bi) bitmap_end = 1;3679 while (bi && bi->bi_iter.bi_sector <3680 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {3681 struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);3682 3683 md_write_end(conf->mddev);3684 bio_io_error(bi);3685 bi = bi2;3686 }3687 3688 /* fail any reads if this device is non-operational and3689 * the data has not reached the cache yet.3690 */3691 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&3692 s->failed > conf->max_degraded &&3693 (!test_bit(R5_Insync, &sh->dev[i].flags) ||3694 test_bit(R5_ReadError, &sh->dev[i].flags))) {3695 spin_lock_irq(&sh->stripe_lock);3696 bi = sh->dev[i].toread;3697 sh->dev[i].toread = NULL;3698 spin_unlock_irq(&sh->stripe_lock);3699 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))3700 wake_up_bit(&sh->dev[i].flags, R5_Overlap);3701 if (bi)3702 s->to_read--;3703 while (bi && bi->bi_iter.bi_sector <3704 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {3705 struct bio *nextbi =3706 r5_next_bio(conf, bi, sh->dev[i].sector);3707 3708 bio_io_error(bi);3709 bi = nextbi;3710 }3711 }3712 if (bitmap_end)3713 conf->mddev->bitmap_ops->endwrite(conf->mddev,3714 sh->sector, RAID5_STRIPE_SECTORS(conf),3715 false, false);3716 /* If we were in the middle of a write the parity block might3717 * still be locked - so just clear all R5_LOCKED flags3718 */3719 clear_bit(R5_LOCKED, &sh->dev[i].flags);3720 }3721 s->to_write = 0;3722 s->written = 0;3723 3724 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))3725 if (atomic_dec_and_test(&conf->pending_full_writes))3726 md_wakeup_thread(conf->mddev->thread);3727}3728 3729static void3730handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,3731 struct stripe_head_state *s)3732{3733 int abort = 0;3734 int i;3735 3736 BUG_ON(sh->batch_head);3737 clear_bit(STRIPE_SYNCING, &sh->state);3738 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))3739 wake_up_bit(&sh->dev[sh->pd_idx].flags, R5_Overlap);3740 s->syncing = 0;3741 s->replacing = 0;3742 /* There is nothing more to do for sync/check/repair.3743 * Don't even need to abort as that is handled elsewhere3744 * if needed, and not always wanted e.g. if there is a known3745 * bad block here.3746 * For recover/replace we need to record a bad block on all3747 * non-sync devices, or abort the recovery3748 */3749 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {3750 /* During recovery devices cannot be removed, so3751 * locking and refcounting of rdevs is not needed3752 */3753 for (i = 0; i < conf->raid_disks; i++) {3754 struct md_rdev *rdev = conf->disks[i].rdev;3755 3756 if (rdev3757 && !test_bit(Faulty, &rdev->flags)3758 && !test_bit(In_sync, &rdev->flags)3759 && !rdev_set_badblocks(rdev, sh->sector,3760 RAID5_STRIPE_SECTORS(conf), 0))3761 abort = 1;3762 rdev = conf->disks[i].replacement;3763 3764 if (rdev3765 && !test_bit(Faulty, &rdev->flags)3766 && !test_bit(In_sync, &rdev->flags)3767 && !rdev_set_badblocks(rdev, sh->sector,3768 RAID5_STRIPE_SECTORS(conf), 0))3769 abort = 1;3770 }3771 if (abort)3772 conf->recovery_disabled =3773 conf->mddev->recovery_disabled;3774 }3775 md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);3776}3777 3778static int want_replace(struct stripe_head *sh, int disk_idx)3779{3780 struct md_rdev *rdev;3781 int rv = 0;3782 3783 rdev = sh->raid_conf->disks[disk_idx].replacement;3784 if (rdev3785 && !test_bit(Faulty, &rdev->flags)3786 && !test_bit(In_sync, &rdev->flags)3787 && (rdev->recovery_offset <= sh->sector3788 || rdev->mddev->recovery_cp <= sh->sector))3789 rv = 1;3790 return rv;3791}3792 3793static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,3794 int disk_idx, int disks)3795{3796 struct r5dev *dev = &sh->dev[disk_idx];3797 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],3798 &sh->dev[s->failed_num[1]] };3799 int i;3800 bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);3801 3802 3803 if (test_bit(R5_LOCKED, &dev->flags) ||3804 test_bit(R5_UPTODATE, &dev->flags))3805 /* No point reading this as we already have it or have3806 * decided to get it.3807 */3808 return 0;3809 3810 if (dev->toread ||3811 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))3812 /* We need this block to directly satisfy a request */3813 return 1;3814 3815 if (s->syncing || s->expanding ||3816 (s->replacing && want_replace(sh, disk_idx)))3817 /* When syncing, or expanding we read everything.3818 * When replacing, we need the replaced block.3819 */3820 return 1;3821 3822 if ((s->failed >= 1 && fdev[0]->toread) ||3823 (s->failed >= 2 && fdev[1]->toread))3824 /* If we want to read from a failed device, then3825 * we need to actually read every other device.3826 */3827 return 1;3828 3829 /* Sometimes neither read-modify-write nor reconstruct-write3830 * cycles can work. In those cases we read every block we3831 * can. Then the parity-update is certain to have enough to3832 * work with.3833 * This can only be a problem when we need to write something,3834 * and some device has failed. If either of those tests3835 * fail we need look no further.3836 */3837 if (!s->failed || !s->to_write)3838 return 0;3839 3840 if (test_bit(R5_Insync, &dev->flags) &&3841 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))3842 /* Pre-reads at not permitted until after short delay3843 * to gather multiple requests. However if this3844 * device is no Insync, the block could only be computed3845 * and there is no need to delay that.3846 */3847 return 0;3848 3849 for (i = 0; i < s->failed && i < 2; i++) {3850 if (fdev[i]->towrite &&3851 !test_bit(R5_UPTODATE, &fdev[i]->flags) &&3852 !test_bit(R5_OVERWRITE, &fdev[i]->flags))3853 /* If we have a partial write to a failed3854 * device, then we will need to reconstruct3855 * the content of that device, so all other3856 * devices must be read.3857 */3858 return 1;3859 3860 if (s->failed >= 2 &&3861 (fdev[i]->towrite ||3862 s->failed_num[i] == sh->pd_idx ||3863 s->failed_num[i] == sh->qd_idx) &&3864 !test_bit(R5_UPTODATE, &fdev[i]->flags))3865 /* In max degraded raid6, If the failed disk is P, Q,3866 * or we want to read the failed disk, we need to do3867 * reconstruct-write.3868 */3869 force_rcw = true;3870 }3871 3872 /* If we are forced to do a reconstruct-write, because parity3873 * cannot be trusted and we are currently recovering it, there3874 * is extra need to be careful.3875 * If one of the devices that we would need to read, because3876 * it is not being overwritten (and maybe not written at all)3877 * is missing/faulty, then we need to read everything we can.3878 */3879 if (!force_rcw &&3880 sh->sector < sh->raid_conf->mddev->recovery_cp)3881 /* reconstruct-write isn't being forced */3882 return 0;3883 for (i = 0; i < s->failed && i < 2; i++) {3884 if (s->failed_num[i] != sh->pd_idx &&3885 s->failed_num[i] != sh->qd_idx &&3886 !test_bit(R5_UPTODATE, &fdev[i]->flags) &&3887 !test_bit(R5_OVERWRITE, &fdev[i]->flags))3888 return 1;3889 }3890 3891 return 0;3892}3893 3894/* fetch_block - checks the given member device to see if its data needs3895 * to be read or computed to satisfy a request.3896 *3897 * Returns 1 when no more member devices need to be checked, otherwise returns3898 * 0 to tell the loop in handle_stripe_fill to continue3899 */3900static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,3901 int disk_idx, int disks)3902{3903 struct r5dev *dev = &sh->dev[disk_idx];3904 3905 /* is the data in this block needed, and can we get it? */3906 if (need_this_block(sh, s, disk_idx, disks)) {3907 /* we would like to get this block, possibly by computing it,3908 * otherwise read it if the backing disk is insync3909 */3910 BUG_ON(test_bit(R5_Wantcompute, &dev->flags));3911 BUG_ON(test_bit(R5_Wantread, &dev->flags));3912 BUG_ON(sh->batch_head);3913 3914 /*3915 * In the raid6 case if the only non-uptodate disk is P3916 * then we already trusted P to compute the other failed3917 * drives. It is safe to compute rather than re-read P.3918 * In other cases we only compute blocks from failed3919 * devices, otherwise check/repair might fail to detect3920 * a real inconsistency.3921 */3922 3923 if ((s->uptodate == disks - 1) &&3924 ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||3925 (s->failed && (disk_idx == s->failed_num[0] ||3926 disk_idx == s->failed_num[1])))) {3927 /* have disk failed, and we're requested to fetch it;3928 * do compute it3929 */3930 pr_debug("Computing stripe %llu block %d\n",3931 (unsigned long long)sh->sector, disk_idx);3932 set_bit(STRIPE_COMPUTE_RUN, &sh->state);3933 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);3934 set_bit(R5_Wantcompute, &dev->flags);3935 sh->ops.target = disk_idx;3936 sh->ops.target2 = -1; /* no 2nd target */3937 s->req_compute = 1;3938 /* Careful: from this point on 'uptodate' is in the eye3939 * of raid_run_ops which services 'compute' operations3940 * before writes. R5_Wantcompute flags a block that will3941 * be R5_UPTODATE by the time it is needed for a3942 * subsequent operation.3943 */3944 s->uptodate++;3945 return 1;3946 } else if (s->uptodate == disks-2 && s->failed >= 2) {3947 /* Computing 2-failure is *very* expensive; only3948 * do it if failed >= 23949 */3950 int other;3951 for (other = disks; other--; ) {3952 if (other == disk_idx)3953 continue;3954 if (!test_bit(R5_UPTODATE,3955 &sh->dev[other].flags))3956 break;3957 }3958 BUG_ON(other < 0);3959 pr_debug("Computing stripe %llu blocks %d,%d\n",3960 (unsigned long long)sh->sector,3961 disk_idx, other);3962 set_bit(STRIPE_COMPUTE_RUN, &sh->state);3963 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);3964 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);3965 set_bit(R5_Wantcompute, &sh->dev[other].flags);3966 sh->ops.target = disk_idx;3967 sh->ops.target2 = other;3968 s->uptodate += 2;3969 s->req_compute = 1;3970 return 1;3971 } else if (test_bit(R5_Insync, &dev->flags)) {3972 set_bit(R5_LOCKED, &dev->flags);3973 set_bit(R5_Wantread, &dev->flags);3974 s->locked++;3975 pr_debug("Reading block %d (sync=%d)\n",3976 disk_idx, s->syncing);3977 }3978 }3979 3980 return 0;3981}3982 3983/*3984 * handle_stripe_fill - read or compute data to satisfy pending requests.3985 */3986static void handle_stripe_fill(struct stripe_head *sh,3987 struct stripe_head_state *s,3988 int disks)3989{3990 int i;3991 3992 /* look for blocks to read/compute, skip this if a compute3993 * is already in flight, or if the stripe contents are in the3994 * midst of changing due to a write3995 */3996 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&3997 !sh->reconstruct_state) {3998 3999 /*4000 * For degraded stripe with data in journal, do not handle4001 * read requests yet, instead, flush the stripe to raid4002 * disks first, this avoids handling complex rmw of write4003 * back cache (prexor with orig_page, and then xor with4004 * page) in the read path4005 */4006 if (s->to_read && s->injournal && s->failed) {4007 if (test_bit(STRIPE_R5C_CACHING, &sh->state))4008 r5c_make_stripe_write_out(sh);4009 goto out;4010 }4011 4012 for (i = disks; i--; )4013 if (fetch_block(sh, s, i, disks))4014 break;4015 }4016out:4017 set_bit(STRIPE_HANDLE, &sh->state);4018}4019 4020static void break_stripe_batch_list(struct stripe_head *head_sh,4021 unsigned long handle_flags);4022/* handle_stripe_clean_event4023 * any written block on an uptodate or failed drive can be returned.4024 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but4025 * never LOCKED, so we don't need to test 'failed' directly.4026 */4027static void handle_stripe_clean_event(struct r5conf *conf,4028 struct stripe_head *sh, int disks)4029{4030 int i;4031 struct r5dev *dev;4032 int discard_pending = 0;4033 struct stripe_head *head_sh = sh;4034 bool do_endio = false;4035 4036 for (i = disks; i--; )4037 if (sh->dev[i].written) {4038 dev = &sh->dev[i];4039 if (!test_bit(R5_LOCKED, &dev->flags) &&4040 (test_bit(R5_UPTODATE, &dev->flags) ||4041 test_bit(R5_Discard, &dev->flags) ||4042 test_bit(R5_SkipCopy, &dev->flags))) {4043 /* We can return any write requests */4044 struct bio *wbi, *wbi2;4045 pr_debug("Return write for disc %d\n", i);4046 if (test_and_clear_bit(R5_Discard, &dev->flags))4047 clear_bit(R5_UPTODATE, &dev->flags);4048 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {4049 WARN_ON(test_bit(R5_UPTODATE, &dev->flags));4050 }4051 do_endio = true;4052 4053returnbi:4054 dev->page = dev->orig_page;4055 wbi = dev->written;4056 dev->written = NULL;4057 while (wbi && wbi->bi_iter.bi_sector <4058 dev->sector + RAID5_STRIPE_SECTORS(conf)) {4059 wbi2 = r5_next_bio(conf, wbi, dev->sector);4060 md_write_end(conf->mddev);4061 bio_endio(wbi);4062 wbi = wbi2;4063 }4064 conf->mddev->bitmap_ops->endwrite(conf->mddev,4065 sh->sector, RAID5_STRIPE_SECTORS(conf),4066 !test_bit(STRIPE_DEGRADED, &sh->state),4067 false);4068 if (head_sh->batch_head) {4069 sh = list_first_entry(&sh->batch_list,4070 struct stripe_head,4071 batch_list);4072 if (sh != head_sh) {4073 dev = &sh->dev[i];4074 goto returnbi;4075 }4076 }4077 sh = head_sh;4078 dev = &sh->dev[i];4079 } else if (test_bit(R5_Discard, &dev->flags))4080 discard_pending = 1;4081 }4082 4083 log_stripe_write_finished(sh);4084 4085 if (!discard_pending &&4086 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {4087 int hash;4088 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);4089 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);4090 if (sh->qd_idx >= 0) {4091 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);4092 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);4093 }4094 /* now that discard is done we can proceed with any sync */4095 clear_bit(STRIPE_DISCARD, &sh->state);4096 /*4097 * SCSI discard will change some bio fields and the stripe has4098 * no updated data, so remove it from hash list and the stripe4099 * will be reinitialized4100 */4101unhash:4102 hash = sh->hash_lock_index;4103 spin_lock_irq(conf->hash_locks + hash);4104 remove_hash(sh);4105 spin_unlock_irq(conf->hash_locks + hash);4106 if (head_sh->batch_head) {4107 sh = list_first_entry(&sh->batch_list,4108 struct stripe_head, batch_list);4109 if (sh != head_sh)4110 goto unhash;4111 }4112 sh = head_sh;4113 4114 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))4115 set_bit(STRIPE_HANDLE, &sh->state);4116 4117 }4118 4119 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))4120 if (atomic_dec_and_test(&conf->pending_full_writes))4121 md_wakeup_thread(conf->mddev->thread);4122 4123 if (head_sh->batch_head && do_endio)4124 break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);4125}4126 4127/*4128 * For RMW in write back cache, we need extra page in prexor to store the4129 * old data. This page is stored in dev->orig_page.4130 *4131 * This function checks whether we have data for prexor. The exact logic4132 * is:4133 * R5_UPTODATE && (!R5_InJournal || R5_OrigPageUPTDODATE)4134 */4135static inline bool uptodate_for_rmw(struct r5dev *dev)4136{4137 return (test_bit(R5_UPTODATE, &dev->flags)) &&4138 (!test_bit(R5_InJournal, &dev->flags) ||4139 test_bit(R5_OrigPageUPTDODATE, &dev->flags));4140}4141 4142static int handle_stripe_dirtying(struct r5conf *conf,4143 struct stripe_head *sh,4144 struct stripe_head_state *s,4145 int disks)4146{4147 int rmw = 0, rcw = 0, i;4148 sector_t recovery_cp = conf->mddev->recovery_cp;4149 4150 /* Check whether resync is now happening or should start.4151 * If yes, then the array is dirty (after unclean shutdown or4152 * initial creation), so parity in some stripes might be inconsistent.4153 * In this case, we need to always do reconstruct-write, to ensure4154 * that in case of drive failure or read-error correction, we4155 * generate correct data from the parity.4156 */4157 if (conf->rmw_level == PARITY_DISABLE_RMW ||4158 (recovery_cp < MaxSector && sh->sector >= recovery_cp &&4159 s->failed == 0)) {4160 /* Calculate the real rcw later - for now make it4161 * look like rcw is cheaper4162 */4163 rcw = 1; rmw = 2;4164 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",4165 conf->rmw_level, (unsigned long long)recovery_cp,4166 (unsigned long long)sh->sector);4167 } else for (i = disks; i--; ) {4168 /* would I have to read this buffer for read_modify_write */4169 struct r5dev *dev = &sh->dev[i];4170 if (((dev->towrite && !delay_towrite(conf, dev, s)) ||4171 i == sh->pd_idx || i == sh->qd_idx ||4172 test_bit(R5_InJournal, &dev->flags)) &&4173 !test_bit(R5_LOCKED, &dev->flags) &&4174 !(uptodate_for_rmw(dev) ||4175 test_bit(R5_Wantcompute, &dev->flags))) {4176 if (test_bit(R5_Insync, &dev->flags))4177 rmw++;4178 else4179 rmw += 2*disks; /* cannot read it */4180 }4181 /* Would I have to read this buffer for reconstruct_write */4182 if (!test_bit(R5_OVERWRITE, &dev->flags) &&4183 i != sh->pd_idx && i != sh->qd_idx &&4184 !test_bit(R5_LOCKED, &dev->flags) &&4185 !(test_bit(R5_UPTODATE, &dev->flags) ||4186 test_bit(R5_Wantcompute, &dev->flags))) {4187 if (test_bit(R5_Insync, &dev->flags))4188 rcw++;4189 else4190 rcw += 2*disks;4191 }4192 }4193 4194 pr_debug("for sector %llu state 0x%lx, rmw=%d rcw=%d\n",4195 (unsigned long long)sh->sector, sh->state, rmw, rcw);4196 set_bit(STRIPE_HANDLE, &sh->state);4197 if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_PREFER_RMW)) && rmw > 0) {4198 /* prefer read-modify-write, but need to get some data */4199 mddev_add_trace_msg(conf->mddev, "raid5 rmw %llu %d",4200 sh->sector, rmw);4201 4202 for (i = disks; i--; ) {4203 struct r5dev *dev = &sh->dev[i];4204 if (test_bit(R5_InJournal, &dev->flags) &&4205 dev->page == dev->orig_page &&4206 !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {4207 /* alloc page for prexor */4208 struct page *p = alloc_page(GFP_NOIO);4209 4210 if (p) {4211 dev->orig_page = p;4212 continue;4213 }4214 4215 /*4216 * alloc_page() failed, try use4217 * disk_info->extra_page4218 */4219 if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE,4220 &conf->cache_state)) {4221 r5c_use_extra_page(sh);4222 break;4223 }4224 4225 /* extra_page in use, add to delayed_list */4226 set_bit(STRIPE_DELAYED, &sh->state);4227 s->waiting_extra_page = 1;4228 return -EAGAIN;4229 }4230 }4231 4232 for (i = disks; i--; ) {4233 struct r5dev *dev = &sh->dev[i];4234 if (((dev->towrite && !delay_towrite(conf, dev, s)) ||4235 i == sh->pd_idx || i == sh->qd_idx ||4236 test_bit(R5_InJournal, &dev->flags)) &&4237 !test_bit(R5_LOCKED, &dev->flags) &&4238 !(uptodate_for_rmw(dev) ||4239 test_bit(R5_Wantcompute, &dev->flags)) &&4240 test_bit(R5_Insync, &dev->flags)) {4241 if (test_bit(STRIPE_PREREAD_ACTIVE,4242 &sh->state)) {4243 pr_debug("Read_old block %d for r-m-w\n",4244 i);4245 set_bit(R5_LOCKED, &dev->flags);4246 set_bit(R5_Wantread, &dev->flags);4247 s->locked++;4248 } else4249 set_bit(STRIPE_DELAYED, &sh->state);4250 }4251 }4252 }4253 if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_PREFER_RMW)) && rcw > 0) {4254 /* want reconstruct write, but need to get some data */4255 int qread =0;4256 rcw = 0;4257 for (i = disks; i--; ) {4258 struct r5dev *dev = &sh->dev[i];4259 if (!test_bit(R5_OVERWRITE, &dev->flags) &&4260 i != sh->pd_idx && i != sh->qd_idx &&4261 !test_bit(R5_LOCKED, &dev->flags) &&4262 !(test_bit(R5_UPTODATE, &dev->flags) ||4263 test_bit(R5_Wantcompute, &dev->flags))) {4264 rcw++;4265 if (test_bit(R5_Insync, &dev->flags) &&4266 test_bit(STRIPE_PREREAD_ACTIVE,4267 &sh->state)) {4268 pr_debug("Read_old block "4269 "%d for Reconstruct\n", i);4270 set_bit(R5_LOCKED, &dev->flags);4271 set_bit(R5_Wantread, &dev->flags);4272 s->locked++;4273 qread++;4274 } else4275 set_bit(STRIPE_DELAYED, &sh->state);4276 }4277 }4278 if (rcw && !mddev_is_dm(conf->mddev))4279 blk_add_trace_msg(conf->mddev->gendisk->queue,4280 "raid5 rcw %llu %d %d %d",4281 (unsigned long long)sh->sector, rcw, qread,4282 test_bit(STRIPE_DELAYED, &sh->state));4283 }4284 4285 if (rcw > disks && rmw > disks &&4286 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))4287 set_bit(STRIPE_DELAYED, &sh->state);4288 4289 /* now if nothing is locked, and if we have enough data,4290 * we can start a write request4291 */4292 /* since handle_stripe can be called at any time we need to handle the4293 * case where a compute block operation has been submitted and then a4294 * subsequent call wants to start a write request. raid_run_ops only4295 * handles the case where compute block and reconstruct are requested4296 * simultaneously. If this is not the case then new writes need to be4297 * held off until the compute completes.4298 */4299 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&4300 (s->locked == 0 && (rcw == 0 || rmw == 0) &&4301 !test_bit(STRIPE_BIT_DELAY, &sh->state)))4302 schedule_reconstruction(sh, s, rcw == 0, 0);4303 return 0;4304}4305 4306static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,4307 struct stripe_head_state *s, int disks)4308{4309 struct r5dev *dev = NULL;4310 4311 BUG_ON(sh->batch_head);4312 set_bit(STRIPE_HANDLE, &sh->state);4313 4314 switch (sh->check_state) {4315 case check_state_idle:4316 /* start a new check operation if there are no failures */4317 if (s->failed == 0) {4318 BUG_ON(s->uptodate != disks);4319 sh->check_state = check_state_run;4320 set_bit(STRIPE_OP_CHECK, &s->ops_request);4321 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);4322 s->uptodate--;4323 break;4324 }4325 dev = &sh->dev[s->failed_num[0]];4326 fallthrough;4327 case check_state_compute_result:4328 sh->check_state = check_state_idle;4329 if (!dev)4330 dev = &sh->dev[sh->pd_idx];4331 4332 /* check that a write has not made the stripe insync */4333 if (test_bit(STRIPE_INSYNC, &sh->state))4334 break;4335 4336 /* either failed parity check, or recovery is happening */4337 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));4338 BUG_ON(s->uptodate != disks);4339 4340 set_bit(R5_LOCKED, &dev->flags);4341 s->locked++;4342 set_bit(R5_Wantwrite, &dev->flags);4343 4344 clear_bit(STRIPE_DEGRADED, &sh->state);4345 set_bit(STRIPE_INSYNC, &sh->state);4346 break;4347 case check_state_run:4348 break; /* we will be called again upon completion */4349 case check_state_check_result:4350 sh->check_state = check_state_idle;4351 4352 /* if a failure occurred during the check operation, leave4353 * STRIPE_INSYNC not set and let the stripe be handled again4354 */4355 if (s->failed)4356 break;4357 4358 /* handle a successful check operation, if parity is correct4359 * we are done. Otherwise update the mismatch count and repair4360 * parity if !MD_RECOVERY_CHECK4361 */4362 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)4363 /* parity is correct (on disc,4364 * not in buffer any more)4365 */4366 set_bit(STRIPE_INSYNC, &sh->state);4367 else {4368 atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);4369 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {4370 /* don't try to repair!! */4371 set_bit(STRIPE_INSYNC, &sh->state);4372 pr_warn_ratelimited("%s: mismatch sector in range "4373 "%llu-%llu\n", mdname(conf->mddev),4374 (unsigned long long) sh->sector,4375 (unsigned long long) sh->sector +4376 RAID5_STRIPE_SECTORS(conf));4377 } else {4378 sh->check_state = check_state_compute_run;4379 set_bit(STRIPE_COMPUTE_RUN, &sh->state);4380 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);4381 set_bit(R5_Wantcompute,4382 &sh->dev[sh->pd_idx].flags);4383 sh->ops.target = sh->pd_idx;4384 sh->ops.target2 = -1;4385 s->uptodate++;4386 }4387 }4388 break;4389 case check_state_compute_run:4390 break;4391 default:4392 pr_err("%s: unknown check_state: %d sector: %llu\n",4393 __func__, sh->check_state,4394 (unsigned long long) sh->sector);4395 BUG();4396 }4397}4398 4399static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,4400 struct stripe_head_state *s,4401 int disks)4402{4403 int pd_idx = sh->pd_idx;4404 int qd_idx = sh->qd_idx;4405 struct r5dev *dev;4406 4407 BUG_ON(sh->batch_head);4408 set_bit(STRIPE_HANDLE, &sh->state);4409 4410 BUG_ON(s->failed > 2);4411 4412 /* Want to check and possibly repair P and Q.4413 * However there could be one 'failed' device, in which4414 * case we can only check one of them, possibly using the4415 * other to generate missing data4416 */4417 4418 switch (sh->check_state) {4419 case check_state_idle:4420 /* start a new check operation if there are < 2 failures */4421 if (s->failed == s->q_failed) {4422 /* The only possible failed device holds Q, so it4423 * makes sense to check P (If anything else were failed,4424 * we would have used P to recreate it).4425 */4426 sh->check_state = check_state_run;4427 }4428 if (!s->q_failed && s->failed < 2) {4429 /* Q is not failed, and we didn't use it to generate4430 * anything, so it makes sense to check it4431 */4432 if (sh->check_state == check_state_run)4433 sh->check_state = check_state_run_pq;4434 else4435 sh->check_state = check_state_run_q;4436 }4437 4438 /* discard potentially stale zero_sum_result */4439 sh->ops.zero_sum_result = 0;4440 4441 if (sh->check_state == check_state_run) {4442 /* async_xor_zero_sum destroys the contents of P */4443 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);4444 s->uptodate--;4445 }4446 if (sh->check_state >= check_state_run &&4447 sh->check_state <= check_state_run_pq) {4448 /* async_syndrome_zero_sum preserves P and Q, so4449 * no need to mark them !uptodate here4450 */4451 set_bit(STRIPE_OP_CHECK, &s->ops_request);4452 break;4453 }4454 4455 /* we have 2-disk failure */4456 BUG_ON(s->failed != 2);4457 fallthrough;4458 case check_state_compute_result:4459 sh->check_state = check_state_idle;4460 4461 /* check that a write has not made the stripe insync */4462 if (test_bit(STRIPE_INSYNC, &sh->state))4463 break;4464 4465 /* now write out any block on a failed drive,4466 * or P or Q if they were recomputed4467 */4468 dev = NULL;4469 if (s->failed == 2) {4470 dev = &sh->dev[s->failed_num[1]];4471 s->locked++;4472 set_bit(R5_LOCKED, &dev->flags);4473 set_bit(R5_Wantwrite, &dev->flags);4474 }4475 if (s->failed >= 1) {4476 dev = &sh->dev[s->failed_num[0]];4477 s->locked++;4478 set_bit(R5_LOCKED, &dev->flags);4479 set_bit(R5_Wantwrite, &dev->flags);4480 }4481 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {4482 dev = &sh->dev[pd_idx];4483 s->locked++;4484 set_bit(R5_LOCKED, &dev->flags);4485 set_bit(R5_Wantwrite, &dev->flags);4486 }4487 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {4488 dev = &sh->dev[qd_idx];4489 s->locked++;4490 set_bit(R5_LOCKED, &dev->flags);4491 set_bit(R5_Wantwrite, &dev->flags);4492 }4493 if (WARN_ONCE(dev && !test_bit(R5_UPTODATE, &dev->flags),4494 "%s: disk%td not up to date\n",4495 mdname(conf->mddev),4496 dev - (struct r5dev *) &sh->dev)) {4497 clear_bit(R5_LOCKED, &dev->flags);4498 clear_bit(R5_Wantwrite, &dev->flags);4499 s->locked--;4500 }4501 clear_bit(STRIPE_DEGRADED, &sh->state);4502 4503 set_bit(STRIPE_INSYNC, &sh->state);4504 break;4505 case check_state_run:4506 case check_state_run_q:4507 case check_state_run_pq:4508 break; /* we will be called again upon completion */4509 case check_state_check_result:4510 sh->check_state = check_state_idle;4511 4512 /* handle a successful check operation, if parity is correct4513 * we are done. Otherwise update the mismatch count and repair4514 * parity if !MD_RECOVERY_CHECK4515 */4516 if (sh->ops.zero_sum_result == 0) {4517 /* both parities are correct */4518 if (!s->failed)4519 set_bit(STRIPE_INSYNC, &sh->state);4520 else {4521 /* in contrast to the raid5 case we can validate4522 * parity, but still have a failure to write4523 * back4524 */4525 sh->check_state = check_state_compute_result;4526 /* Returning at this point means that we may go4527 * off and bring p and/or q uptodate again so4528 * we make sure to check zero_sum_result again4529 * to verify if p or q need writeback4530 */4531 }4532 } else {4533 atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);4534 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {4535 /* don't try to repair!! */4536 set_bit(STRIPE_INSYNC, &sh->state);4537 pr_warn_ratelimited("%s: mismatch sector in range "4538 "%llu-%llu\n", mdname(conf->mddev),4539 (unsigned long long) sh->sector,4540 (unsigned long long) sh->sector +4541 RAID5_STRIPE_SECTORS(conf));4542 } else {4543 int *target = &sh->ops.target;4544 4545 sh->ops.target = -1;4546 sh->ops.target2 = -1;4547 sh->check_state = check_state_compute_run;4548 set_bit(STRIPE_COMPUTE_RUN, &sh->state);4549 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);4550 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {4551 set_bit(R5_Wantcompute,4552 &sh->dev[pd_idx].flags);4553 *target = pd_idx;4554 target = &sh->ops.target2;4555 s->uptodate++;4556 }4557 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {4558 set_bit(R5_Wantcompute,4559 &sh->dev[qd_idx].flags);4560 *target = qd_idx;4561 s->uptodate++;4562 }4563 }4564 }4565 break;4566 case check_state_compute_run:4567 break;4568 default:4569 pr_warn("%s: unknown check_state: %d sector: %llu\n",4570 __func__, sh->check_state,4571 (unsigned long long) sh->sector);4572 BUG();4573 }4574}4575 4576static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)4577{4578 int i;4579 4580 /* We have read all the blocks in this stripe and now we need to4581 * copy some of them into a target stripe for expand.4582 */4583 struct dma_async_tx_descriptor *tx = NULL;4584 BUG_ON(sh->batch_head);4585 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);4586 for (i = 0; i < sh->disks; i++)4587 if (i != sh->pd_idx && i != sh->qd_idx) {4588 int dd_idx, j;4589 struct stripe_head *sh2;4590 struct async_submit_ctl submit;4591 4592 sector_t bn = raid5_compute_blocknr(sh, i, 1);4593 sector_t s = raid5_compute_sector(conf, bn, 0,4594 &dd_idx, NULL);4595 sh2 = raid5_get_active_stripe(conf, NULL, s,4596 R5_GAS_NOBLOCK | R5_GAS_NOQUIESCE);4597 if (sh2 == NULL)4598 /* so far only the early blocks of this stripe4599 * have been requested. When later blocks4600 * get requested, we will try again4601 */4602 continue;4603 if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||4604 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {4605 /* must have already done this block */4606 raid5_release_stripe(sh2);4607 continue;4608 }4609 4610 /* place all the copies on one channel */4611 init_async_submit(&submit, 0, tx, NULL, NULL, NULL);4612 tx = async_memcpy(sh2->dev[dd_idx].page,4613 sh->dev[i].page, sh2->dev[dd_idx].offset,4614 sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),4615 &submit);4616 4617 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);4618 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);4619 for (j = 0; j < conf->raid_disks; j++)4620 if (j != sh2->pd_idx &&4621 j != sh2->qd_idx &&4622 !test_bit(R5_Expanded, &sh2->dev[j].flags))4623 break;4624 if (j == conf->raid_disks) {4625 set_bit(STRIPE_EXPAND_READY, &sh2->state);4626 set_bit(STRIPE_HANDLE, &sh2->state);4627 }4628 raid5_release_stripe(sh2);4629 4630 }4631 /* done submitting copies, wait for them to complete */4632 async_tx_quiesce(&tx);4633}4634 4635/*4636 * handle_stripe - do things to a stripe.4637 *4638 * We lock the stripe by setting STRIPE_ACTIVE and then examine the4639 * state of various bits to see what needs to be done.4640 * Possible results:4641 * return some read requests which now have data4642 * return some write requests which are safely on storage4643 * schedule a read on some buffers4644 * schedule a write of some buffers4645 * return confirmation of parity correctness4646 *4647 */4648 4649static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)4650{4651 struct r5conf *conf = sh->raid_conf;4652 int disks = sh->disks;4653 struct r5dev *dev;4654 int i;4655 int do_recovery = 0;4656 4657 memset(s, 0, sizeof(*s));4658 4659 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;4660 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;4661 s->failed_num[0] = -1;4662 s->failed_num[1] = -1;4663 s->log_failed = r5l_log_disk_error(conf);4664 4665 /* Now to look around and see what can be done */4666 for (i=disks; i--; ) {4667 struct md_rdev *rdev;4668 int is_bad = 0;4669 4670 dev = &sh->dev[i];4671 4672 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",4673 i, dev->flags,4674 dev->toread, dev->towrite, dev->written);4675 /* maybe we can reply to a read4676 *4677 * new wantfill requests are only permitted while4678 * ops_complete_biofill is guaranteed to be inactive4679 */4680 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&4681 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))4682 set_bit(R5_Wantfill, &dev->flags);4683 4684 /* now count some things */4685 if (test_bit(R5_LOCKED, &dev->flags))4686 s->locked++;4687 if (test_bit(R5_UPTODATE, &dev->flags))4688 s->uptodate++;4689 if (test_bit(R5_Wantcompute, &dev->flags)) {4690 s->compute++;4691 BUG_ON(s->compute > 2);4692 }4693 4694 if (test_bit(R5_Wantfill, &dev->flags))4695 s->to_fill++;4696 else if (dev->toread)4697 s->to_read++;4698 if (dev->towrite) {4699 s->to_write++;4700 if (!test_bit(R5_OVERWRITE, &dev->flags))4701 s->non_overwrite++;4702 }4703 if (dev->written)4704 s->written++;4705 /* Prefer to use the replacement for reads, but only4706 * if it is recovered enough and has no bad blocks.4707 */4708 rdev = conf->disks[i].replacement;4709 if (rdev && !test_bit(Faulty, &rdev->flags) &&4710 rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&4711 !rdev_has_badblock(rdev, sh->sector,4712 RAID5_STRIPE_SECTORS(conf)))4713 set_bit(R5_ReadRepl, &dev->flags);4714 else {4715 if (rdev && !test_bit(Faulty, &rdev->flags))4716 set_bit(R5_NeedReplace, &dev->flags);4717 else4718 clear_bit(R5_NeedReplace, &dev->flags);4719 rdev = conf->disks[i].rdev;4720 clear_bit(R5_ReadRepl, &dev->flags);4721 }4722 if (rdev && test_bit(Faulty, &rdev->flags))4723 rdev = NULL;4724 if (rdev) {4725 is_bad = rdev_has_badblock(rdev, sh->sector,4726 RAID5_STRIPE_SECTORS(conf));4727 if (s->blocked_rdev == NULL4728 && (test_bit(Blocked, &rdev->flags)4729 || is_bad < 0)) {4730 if (is_bad < 0)4731 set_bit(BlockedBadBlocks,4732 &rdev->flags);4733 s->blocked_rdev = rdev;4734 atomic_inc(&rdev->nr_pending);4735 }4736 }4737 clear_bit(R5_Insync, &dev->flags);4738 if (!rdev)4739 /* Not in-sync */;4740 else if (is_bad) {4741 /* also not in-sync */4742 if (!test_bit(WriteErrorSeen, &rdev->flags) &&4743 test_bit(R5_UPTODATE, &dev->flags)) {4744 /* treat as in-sync, but with a read error4745 * which we can now try to correct4746 */4747 set_bit(R5_Insync, &dev->flags);4748 set_bit(R5_ReadError, &dev->flags);4749 }4750 } else if (test_bit(In_sync, &rdev->flags))4751 set_bit(R5_Insync, &dev->flags);4752 else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)4753 /* in sync if before recovery_offset */4754 set_bit(R5_Insync, &dev->flags);4755 else if (test_bit(R5_UPTODATE, &dev->flags) &&4756 test_bit(R5_Expanded, &dev->flags))4757 /* If we've reshaped into here, we assume it is Insync.4758 * We will shortly update recovery_offset to make4759 * it official.4760 */4761 set_bit(R5_Insync, &dev->flags);4762 4763 if (test_bit(R5_WriteError, &dev->flags)) {4764 /* This flag does not apply to '.replacement'4765 * only to .rdev, so make sure to check that*/4766 struct md_rdev *rdev2 = conf->disks[i].rdev;4767 4768 if (rdev2 == rdev)4769 clear_bit(R5_Insync, &dev->flags);4770 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {4771 s->handle_bad_blocks = 1;4772 atomic_inc(&rdev2->nr_pending);4773 } else4774 clear_bit(R5_WriteError, &dev->flags);4775 }4776 if (test_bit(R5_MadeGood, &dev->flags)) {4777 /* This flag does not apply to '.replacement'4778 * only to .rdev, so make sure to check that*/4779 struct md_rdev *rdev2 = conf->disks[i].rdev;4780 4781 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {4782 s->handle_bad_blocks = 1;4783 atomic_inc(&rdev2->nr_pending);4784 } else4785 clear_bit(R5_MadeGood, &dev->flags);4786 }4787 if (test_bit(R5_MadeGoodRepl, &dev->flags)) {4788 struct md_rdev *rdev2 = conf->disks[i].replacement;4789 4790 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {4791 s->handle_bad_blocks = 1;4792 atomic_inc(&rdev2->nr_pending);4793 } else4794 clear_bit(R5_MadeGoodRepl, &dev->flags);4795 }4796 if (!test_bit(R5_Insync, &dev->flags)) {4797 /* The ReadError flag will just be confusing now */4798 clear_bit(R5_ReadError, &dev->flags);4799 clear_bit(R5_ReWrite, &dev->flags);4800 }4801 if (test_bit(R5_ReadError, &dev->flags))4802 clear_bit(R5_Insync, &dev->flags);4803 if (!test_bit(R5_Insync, &dev->flags)) {4804 if (s->failed < 2)4805 s->failed_num[s->failed] = i;4806 s->failed++;4807 if (rdev && !test_bit(Faulty, &rdev->flags))4808 do_recovery = 1;4809 else if (!rdev) {4810 rdev = conf->disks[i].replacement;4811 if (rdev && !test_bit(Faulty, &rdev->flags))4812 do_recovery = 1;4813 }4814 }4815 4816 if (test_bit(R5_InJournal, &dev->flags))4817 s->injournal++;4818 if (test_bit(R5_InJournal, &dev->flags) && dev->written)4819 s->just_cached++;4820 }4821 if (test_bit(STRIPE_SYNCING, &sh->state)) {4822 /* If there is a failed device being replaced,4823 * we must be recovering.4824 * else if we are after recovery_cp, we must be syncing4825 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.4826 * else we can only be replacing4827 * sync and recovery both need to read all devices, and so4828 * use the same flag.4829 */4830 if (do_recovery ||4831 sh->sector >= conf->mddev->recovery_cp ||4832 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))4833 s->syncing = 1;4834 else4835 s->replacing = 1;4836 }4837}4838 4839/*4840 * Return '1' if this is a member of batch, or '0' if it is a lone stripe or4841 * a head which can now be handled.4842 */4843static int clear_batch_ready(struct stripe_head *sh)4844{4845 struct stripe_head *tmp;4846 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))4847 return (sh->batch_head && sh->batch_head != sh);4848 spin_lock(&sh->stripe_lock);4849 if (!sh->batch_head) {4850 spin_unlock(&sh->stripe_lock);4851 return 0;4852 }4853 4854 /*4855 * this stripe could be added to a batch list before we check4856 * BATCH_READY, skips it4857 */4858 if (sh->batch_head != sh) {4859 spin_unlock(&sh->stripe_lock);4860 return 1;4861 }4862 spin_lock(&sh->batch_lock);4863 list_for_each_entry(tmp, &sh->batch_list, batch_list)4864 clear_bit(STRIPE_BATCH_READY, &tmp->state);4865 spin_unlock(&sh->batch_lock);4866 spin_unlock(&sh->stripe_lock);4867 4868 /*4869 * BATCH_READY is cleared, no new stripes can be added.4870 * batch_list can be accessed without lock4871 */4872 return 0;4873}4874 4875static void break_stripe_batch_list(struct stripe_head *head_sh,4876 unsigned long handle_flags)4877{4878 struct stripe_head *sh, *next;4879 int i;4880 4881 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {4882 4883 list_del_init(&sh->batch_list);4884 4885 WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |4886 (1 << STRIPE_SYNCING) |4887 (1 << STRIPE_REPLACED) |4888 (1 << STRIPE_DELAYED) |4889 (1 << STRIPE_BIT_DELAY) |4890 (1 << STRIPE_FULL_WRITE) |4891 (1 << STRIPE_BIOFILL_RUN) |4892 (1 << STRIPE_COMPUTE_RUN) |4893 (1 << STRIPE_DISCARD) |4894 (1 << STRIPE_BATCH_READY) |4895 (1 << STRIPE_BATCH_ERR) |4896 (1 << STRIPE_BITMAP_PENDING)),4897 "stripe state: %lx\n", sh->state);4898 WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) |4899 (1 << STRIPE_REPLACED)),4900 "head stripe state: %lx\n", head_sh->state);4901 4902 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |4903 (1 << STRIPE_PREREAD_ACTIVE) |4904 (1 << STRIPE_DEGRADED) |4905 (1 << STRIPE_ON_UNPLUG_LIST)),4906 head_sh->state & (1 << STRIPE_INSYNC));4907 4908 sh->check_state = head_sh->check_state;4909 sh->reconstruct_state = head_sh->reconstruct_state;4910 spin_lock_irq(&sh->stripe_lock);4911 sh->batch_head = NULL;4912 spin_unlock_irq(&sh->stripe_lock);4913 for (i = 0; i < sh->disks; i++) {4914 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))4915 wake_up_bit(&sh->dev[i].flags, R5_Overlap);4916 sh->dev[i].flags = head_sh->dev[i].flags &4917 (~((1 << R5_WriteError) | (1 << R5_Overlap)));4918 }4919 if (handle_flags == 0 ||4920 sh->state & handle_flags)4921 set_bit(STRIPE_HANDLE, &sh->state);4922 raid5_release_stripe(sh);4923 }4924 spin_lock_irq(&head_sh->stripe_lock);4925 head_sh->batch_head = NULL;4926 spin_unlock_irq(&head_sh->stripe_lock);4927 for (i = 0; i < head_sh->disks; i++)4928 if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))4929 wake_up_bit(&head_sh->dev[i].flags, R5_Overlap);4930 if (head_sh->state & handle_flags)4931 set_bit(STRIPE_HANDLE, &head_sh->state);4932}4933 4934static void handle_stripe(struct stripe_head *sh)4935{4936 struct stripe_head_state s;4937 struct r5conf *conf = sh->raid_conf;4938 int i;4939 int prexor;4940 int disks = sh->disks;4941 struct r5dev *pdev, *qdev;4942 4943 clear_bit(STRIPE_HANDLE, &sh->state);4944 4945 /*4946 * handle_stripe should not continue handle the batched stripe, only4947 * the head of batch list or lone stripe can continue. Otherwise we4948 * could see break_stripe_batch_list warns about the STRIPE_ACTIVE4949 * is set for the batched stripe.4950 */4951 if (clear_batch_ready(sh))4952 return;4953 4954 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {4955 /* already being handled, ensure it gets handled4956 * again when current action finishes */4957 set_bit(STRIPE_HANDLE, &sh->state);4958 return;4959 }4960 4961 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))4962 break_stripe_batch_list(sh, 0);4963 4964 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {4965 spin_lock(&sh->stripe_lock);4966 /*4967 * Cannot process 'sync' concurrently with 'discard'.4968 * Flush data in r5cache before 'sync'.4969 */4970 if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&4971 !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&4972 !test_bit(STRIPE_DISCARD, &sh->state) &&4973 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {4974 set_bit(STRIPE_SYNCING, &sh->state);4975 clear_bit(STRIPE_INSYNC, &sh->state);4976 clear_bit(STRIPE_REPLACED, &sh->state);4977 }4978 spin_unlock(&sh->stripe_lock);4979 }4980 clear_bit(STRIPE_DELAYED, &sh->state);4981 4982 pr_debug("handling stripe %llu, state=%#lx cnt=%d, "4983 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",4984 (unsigned long long)sh->sector, sh->state,4985 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,4986 sh->check_state, sh->reconstruct_state);4987 4988 analyse_stripe(sh, &s);4989 4990 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))4991 goto finish;4992 4993 if (s.handle_bad_blocks ||4994 test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {4995 set_bit(STRIPE_HANDLE, &sh->state);4996 goto finish;4997 }4998 4999 if (unlikely(s.blocked_rdev)) {5000 if (s.syncing || s.expanding || s.expanded ||5001 s.replacing || s.to_write || s.written) {5002 set_bit(STRIPE_HANDLE, &sh->state);5003 goto finish;5004 }5005 /* There is nothing for the blocked_rdev to block */5006 rdev_dec_pending(s.blocked_rdev, conf->mddev);5007 s.blocked_rdev = NULL;5008 }5009 5010 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {5011 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);5012 set_bit(STRIPE_BIOFILL_RUN, &sh->state);5013 }5014 5015 pr_debug("locked=%d uptodate=%d to_read=%d"5016 " to_write=%d failed=%d failed_num=%d,%d\n",5017 s.locked, s.uptodate, s.to_read, s.to_write, s.failed,5018 s.failed_num[0], s.failed_num[1]);5019 /*5020 * check if the array has lost more than max_degraded devices and,5021 * if so, some requests might need to be failed.5022 *5023 * When journal device failed (log_failed), we will only process5024 * the stripe if there is data need write to raid disks5025 */5026 if (s.failed > conf->max_degraded ||5027 (s.log_failed && s.injournal == 0)) {5028 sh->check_state = 0;5029 sh->reconstruct_state = 0;5030 break_stripe_batch_list(sh, 0);5031 if (s.to_read+s.to_write+s.written)5032 handle_failed_stripe(conf, sh, &s, disks);5033 if (s.syncing + s.replacing)5034 handle_failed_sync(conf, sh, &s);5035 }5036 5037 /* Now we check to see if any write operations have recently5038 * completed5039 */5040 prexor = 0;5041 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)5042 prexor = 1;5043 if (sh->reconstruct_state == reconstruct_state_drain_result ||5044 sh->reconstruct_state == reconstruct_state_prexor_drain_result) {5045 sh->reconstruct_state = reconstruct_state_idle;5046 5047 /* All the 'written' buffers and the parity block are ready to5048 * be written back to disk5049 */5050 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&5051 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));5052 BUG_ON(sh->qd_idx >= 0 &&5053 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&5054 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));5055 for (i = disks; i--; ) {5056 struct r5dev *dev = &sh->dev[i];5057 if (test_bit(R5_LOCKED, &dev->flags) &&5058 (i == sh->pd_idx || i == sh->qd_idx ||5059 dev->written || test_bit(R5_InJournal,5060 &dev->flags))) {5061 pr_debug("Writing block %d\n", i);5062 set_bit(R5_Wantwrite, &dev->flags);5063 if (prexor)5064 continue;5065 if (s.failed > 1)5066 continue;5067 if (!test_bit(R5_Insync, &dev->flags) ||5068 ((i == sh->pd_idx || i == sh->qd_idx) &&5069 s.failed == 0))5070 set_bit(STRIPE_INSYNC, &sh->state);5071 }5072 }5073 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))5074 s.dec_preread_active = 1;5075 }5076 5077 /*5078 * might be able to return some write requests if the parity blocks5079 * are safe, or on a failed drive5080 */5081 pdev = &sh->dev[sh->pd_idx];5082 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)5083 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);5084 qdev = &sh->dev[sh->qd_idx];5085 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)5086 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)5087 || conf->level < 6;5088 5089 if (s.written &&5090 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)5091 && !test_bit(R5_LOCKED, &pdev->flags)5092 && (test_bit(R5_UPTODATE, &pdev->flags) ||5093 test_bit(R5_Discard, &pdev->flags))))) &&5094 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)5095 && !test_bit(R5_LOCKED, &qdev->flags)5096 && (test_bit(R5_UPTODATE, &qdev->flags) ||5097 test_bit(R5_Discard, &qdev->flags))))))5098 handle_stripe_clean_event(conf, sh, disks);5099 5100 if (s.just_cached)5101 r5c_handle_cached_data_endio(conf, sh, disks);5102 log_stripe_write_finished(sh);5103 5104 /* Now we might consider reading some blocks, either to check/generate5105 * parity, or to satisfy requests5106 * or to load a block that is being partially written.5107 */5108 if (s.to_read || s.non_overwrite5109 || (s.to_write && s.failed)5110 || (s.syncing && (s.uptodate + s.compute < disks))5111 || s.replacing5112 || s.expanding)5113 handle_stripe_fill(sh, &s, disks);5114 5115 /*5116 * When the stripe finishes full journal write cycle (write to journal5117 * and raid disk), this is the clean up procedure so it is ready for5118 * next operation.5119 */5120 r5c_finish_stripe_write_out(conf, sh, &s);5121 5122 /*5123 * Now to consider new write requests, cache write back and what else,5124 * if anything should be read. We do not handle new writes when:5125 * 1/ A 'write' operation (copy+xor) is already in flight.5126 * 2/ A 'check' operation is in flight, as it may clobber the parity5127 * block.5128 * 3/ A r5c cache log write is in flight.5129 */5130 5131 if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {5132 if (!r5c_is_writeback(conf->log)) {5133 if (s.to_write)5134 handle_stripe_dirtying(conf, sh, &s, disks);5135 } else { /* write back cache */5136 int ret = 0;5137 5138 /* First, try handle writes in caching phase */5139 if (s.to_write)5140 ret = r5c_try_caching_write(conf, sh, &s,5141 disks);5142 /*5143 * If caching phase failed: ret == -EAGAIN5144 * OR5145 * stripe under reclaim: !caching && injournal5146 *5147 * fall back to handle_stripe_dirtying()5148 */5149 if (ret == -EAGAIN ||5150 /* stripe under reclaim: !caching && injournal */5151 (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&5152 s.injournal > 0)) {5153 ret = handle_stripe_dirtying(conf, sh, &s,5154 disks);5155 if (ret == -EAGAIN)5156 goto finish;5157 }5158 }5159 }5160 5161 /* maybe we need to check and possibly fix the parity for this stripe5162 * Any reads will already have been scheduled, so we just see if enough5163 * data is available. The parity check is held off while parity5164 * dependent operations are in flight.5165 */5166 if (sh->check_state ||5167 (s.syncing && s.locked == 0 &&5168 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&5169 !test_bit(STRIPE_INSYNC, &sh->state))) {5170 if (conf->level == 6)5171 handle_parity_checks6(conf, sh, &s, disks);5172 else5173 handle_parity_checks5(conf, sh, &s, disks);5174 }5175 5176 if ((s.replacing || s.syncing) && s.locked == 05177 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)5178 && !test_bit(STRIPE_REPLACED, &sh->state)) {5179 /* Write out to replacement devices where possible */5180 for (i = 0; i < conf->raid_disks; i++)5181 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {5182 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));5183 set_bit(R5_WantReplace, &sh->dev[i].flags);5184 set_bit(R5_LOCKED, &sh->dev[i].flags);5185 s.locked++;5186 }5187 if (s.replacing)5188 set_bit(STRIPE_INSYNC, &sh->state);5189 set_bit(STRIPE_REPLACED, &sh->state);5190 }5191 if ((s.syncing || s.replacing) && s.locked == 0 &&5192 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&5193 test_bit(STRIPE_INSYNC, &sh->state)) {5194 md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);5195 clear_bit(STRIPE_SYNCING, &sh->state);5196 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))5197 wake_up_bit(&sh->dev[sh->pd_idx].flags, R5_Overlap);5198 }5199 5200 /* If the failed drives are just a ReadError, then we might need5201 * to progress the repair/check process5202 */5203 if (s.failed <= conf->max_degraded && !conf->mddev->ro)5204 for (i = 0; i < s.failed; i++) {5205 struct r5dev *dev = &sh->dev[s.failed_num[i]];5206 if (test_bit(R5_ReadError, &dev->flags)5207 && !test_bit(R5_LOCKED, &dev->flags)5208 && test_bit(R5_UPTODATE, &dev->flags)5209 ) {5210 if (!test_bit(R5_ReWrite, &dev->flags)) {5211 set_bit(R5_Wantwrite, &dev->flags);5212 set_bit(R5_ReWrite, &dev->flags);5213 } else5214 /* let's read it back */5215 set_bit(R5_Wantread, &dev->flags);5216 set_bit(R5_LOCKED, &dev->flags);5217 s.locked++;5218 }5219 }5220 5221 /* Finish reconstruct operations initiated by the expansion process */5222 if (sh->reconstruct_state == reconstruct_state_result) {5223 struct stripe_head *sh_src5224 = raid5_get_active_stripe(conf, NULL, sh->sector,5225 R5_GAS_PREVIOUS | R5_GAS_NOBLOCK |5226 R5_GAS_NOQUIESCE);5227 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {5228 /* sh cannot be written until sh_src has been read.5229 * so arrange for sh to be delayed a little5230 */5231 set_bit(STRIPE_DELAYED, &sh->state);5232 set_bit(STRIPE_HANDLE, &sh->state);5233 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,5234 &sh_src->state))5235 atomic_inc(&conf->preread_active_stripes);5236 raid5_release_stripe(sh_src);5237 goto finish;5238 }5239 if (sh_src)5240 raid5_release_stripe(sh_src);5241 5242 sh->reconstruct_state = reconstruct_state_idle;5243 clear_bit(STRIPE_EXPANDING, &sh->state);5244 for (i = conf->raid_disks; i--; ) {5245 set_bit(R5_Wantwrite, &sh->dev[i].flags);5246 set_bit(R5_LOCKED, &sh->dev[i].flags);5247 s.locked++;5248 }5249 }5250 5251 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&5252 !sh->reconstruct_state) {5253 /* Need to write out all blocks after computing parity */5254 sh->disks = conf->raid_disks;5255 stripe_set_idx(sh->sector, conf, 0, sh);5256 schedule_reconstruction(sh, &s, 1, 1);5257 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {5258 clear_bit(STRIPE_EXPAND_READY, &sh->state);5259 atomic_dec(&conf->reshape_stripes);5260 wake_up(&conf->wait_for_reshape);5261 md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);5262 }5263 5264 if (s.expanding && s.locked == 0 &&5265 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))5266 handle_stripe_expansion(conf, sh);5267 5268finish:5269 /* wait for this device to become unblocked */5270 if (unlikely(s.blocked_rdev)) {5271 if (conf->mddev->external)5272 md_wait_for_blocked_rdev(s.blocked_rdev,5273 conf->mddev);5274 else5275 /* Internal metadata will immediately5276 * be written by raid5d, so we don't5277 * need to wait here.5278 */5279 rdev_dec_pending(s.blocked_rdev,5280 conf->mddev);5281 }5282 5283 if (s.handle_bad_blocks)5284 for (i = disks; i--; ) {5285 struct md_rdev *rdev;5286 struct r5dev *dev = &sh->dev[i];5287 if (test_and_clear_bit(R5_WriteError, &dev->flags)) {5288 /* We own a safe reference to the rdev */5289 rdev = conf->disks[i].rdev;5290 if (!rdev_set_badblocks(rdev, sh->sector,5291 RAID5_STRIPE_SECTORS(conf), 0))5292 md_error(conf->mddev, rdev);5293 rdev_dec_pending(rdev, conf->mddev);5294 }5295 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {5296 rdev = conf->disks[i].rdev;5297 rdev_clear_badblocks(rdev, sh->sector,5298 RAID5_STRIPE_SECTORS(conf), 0);5299 rdev_dec_pending(rdev, conf->mddev);5300 }5301 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {5302 rdev = conf->disks[i].replacement;5303 if (!rdev)5304 /* rdev have been moved down */5305 rdev = conf->disks[i].rdev;5306 rdev_clear_badblocks(rdev, sh->sector,5307 RAID5_STRIPE_SECTORS(conf), 0);5308 rdev_dec_pending(rdev, conf->mddev);5309 }5310 }5311 5312 if (s.ops_request)5313 raid_run_ops(sh, s.ops_request);5314 5315 ops_run_io(sh, &s);5316 5317 if (s.dec_preread_active) {5318 /* We delay this until after ops_run_io so that if make_request5319 * is waiting on a flush, it won't continue until the writes5320 * have actually been submitted.5321 */5322 atomic_dec(&conf->preread_active_stripes);5323 if (atomic_read(&conf->preread_active_stripes) <5324 IO_THRESHOLD)5325 md_wakeup_thread(conf->mddev->thread);5326 }5327 5328 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);5329}5330 5331static void raid5_activate_delayed(struct r5conf *conf)5332 __must_hold(&conf->device_lock)5333{5334 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {5335 while (!list_empty(&conf->delayed_list)) {5336 struct list_head *l = conf->delayed_list.next;5337 struct stripe_head *sh;5338 sh = list_entry(l, struct stripe_head, lru);5339 list_del_init(l);5340 clear_bit(STRIPE_DELAYED, &sh->state);5341 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))5342 atomic_inc(&conf->preread_active_stripes);5343 list_add_tail(&sh->lru, &conf->hold_list);5344 raid5_wakeup_stripe_thread(sh);5345 }5346 }5347}5348 5349static void activate_bit_delay(struct r5conf *conf,5350 struct list_head *temp_inactive_list)5351 __must_hold(&conf->device_lock)5352{5353 struct list_head head;5354 list_add(&head, &conf->bitmap_list);5355 list_del_init(&conf->bitmap_list);5356 while (!list_empty(&head)) {5357 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);5358 int hash;5359 list_del_init(&sh->lru);5360 atomic_inc(&sh->count);5361 hash = sh->hash_lock_index;5362 __release_stripe(conf, sh, &temp_inactive_list[hash]);5363 }5364}5365 5366static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)5367{5368 struct r5conf *conf = mddev->private;5369 sector_t sector = bio->bi_iter.bi_sector;5370 unsigned int chunk_sectors;5371 unsigned int bio_sectors = bio_sectors(bio);5372 5373 chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);5374 return chunk_sectors >=5375 ((sector & (chunk_sectors - 1)) + bio_sectors);5376}5377 5378/*5379 * add bio to the retry LIFO ( in O(1) ... we are in interrupt )5380 * later sampled by raid5d.5381 */5382static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)5383{5384 unsigned long flags;5385 5386 spin_lock_irqsave(&conf->device_lock, flags);5387 5388 bi->bi_next = conf->retry_read_aligned_list;5389 conf->retry_read_aligned_list = bi;5390 5391 spin_unlock_irqrestore(&conf->device_lock, flags);5392 md_wakeup_thread(conf->mddev->thread);5393}5394 5395static struct bio *remove_bio_from_retry(struct r5conf *conf,5396 unsigned int *offset)5397{5398 struct bio *bi;5399 5400 bi = conf->retry_read_aligned;5401 if (bi) {5402 *offset = conf->retry_read_offset;5403 conf->retry_read_aligned = NULL;5404 return bi;5405 }5406 bi = conf->retry_read_aligned_list;5407 if(bi) {5408 conf->retry_read_aligned_list = bi->bi_next;5409 bi->bi_next = NULL;5410 *offset = 0;5411 }5412 5413 return bi;5414}5415 5416/*5417 * The "raid5_align_endio" should check if the read succeeded and if it5418 * did, call bio_endio on the original bio (having bio_put the new bio5419 * first).5420 * If the read failed..5421 */5422static void raid5_align_endio(struct bio *bi)5423{5424 struct bio *raid_bi = bi->bi_private;5425 struct md_rdev *rdev = (void *)raid_bi->bi_next;5426 struct mddev *mddev = rdev->mddev;5427 struct r5conf *conf = mddev->private;5428 blk_status_t error = bi->bi_status;5429 5430 bio_put(bi);5431 raid_bi->bi_next = NULL;5432 rdev_dec_pending(rdev, conf->mddev);5433 5434 if (!error) {5435 bio_endio(raid_bi);5436 if (atomic_dec_and_test(&conf->active_aligned_reads))5437 wake_up(&conf->wait_for_quiescent);5438 return;5439 }5440 5441 pr_debug("raid5_align_endio : io error...handing IO for a retry\n");5442 5443 add_bio_to_retry(raid_bi, conf);5444}5445 5446static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)5447{5448 struct r5conf *conf = mddev->private;5449 struct bio *align_bio;5450 struct md_rdev *rdev;5451 sector_t sector, end_sector;5452 int dd_idx;5453 bool did_inc;5454 5455 if (!in_chunk_boundary(mddev, raid_bio)) {5456 pr_debug("%s: non aligned\n", __func__);5457 return 0;5458 }5459 5460 sector = raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 0,5461 &dd_idx, NULL);5462 end_sector = sector + bio_sectors(raid_bio);5463 5464 if (r5c_big_stripe_cached(conf, sector))5465 return 0;5466 5467 rdev = conf->disks[dd_idx].replacement;5468 if (!rdev || test_bit(Faulty, &rdev->flags) ||5469 rdev->recovery_offset < end_sector) {5470 rdev = conf->disks[dd_idx].rdev;5471 if (!rdev)5472 return 0;5473 if (test_bit(Faulty, &rdev->flags) ||5474 !(test_bit(In_sync, &rdev->flags) ||5475 rdev->recovery_offset >= end_sector))5476 return 0;5477 }5478 5479 atomic_inc(&rdev->nr_pending);5480 5481 if (rdev_has_badblock(rdev, sector, bio_sectors(raid_bio))) {5482 rdev_dec_pending(rdev, mddev);5483 return 0;5484 }5485 5486 md_account_bio(mddev, &raid_bio);5487 raid_bio->bi_next = (void *)rdev;5488 5489 align_bio = bio_alloc_clone(rdev->bdev, raid_bio, GFP_NOIO,5490 &mddev->bio_set);5491 align_bio->bi_end_io = raid5_align_endio;5492 align_bio->bi_private = raid_bio;5493 align_bio->bi_iter.bi_sector = sector;5494 5495 /* No reshape active, so we can trust rdev->data_offset */5496 align_bio->bi_iter.bi_sector += rdev->data_offset;5497 5498 did_inc = false;5499 if (conf->quiesce == 0) {5500 atomic_inc(&conf->active_aligned_reads);5501 did_inc = true;5502 }5503 /* need a memory barrier to detect the race with raid5_quiesce() */5504 if (!did_inc || smp_load_acquire(&conf->quiesce) != 0) {5505 /* quiesce is in progress, so we need to undo io activation and wait5506 * for it to finish5507 */5508 if (did_inc && atomic_dec_and_test(&conf->active_aligned_reads))5509 wake_up(&conf->wait_for_quiescent);5510 spin_lock_irq(&conf->device_lock);5511 wait_event_lock_irq(conf->wait_for_quiescent, conf->quiesce == 0,5512 conf->device_lock);5513 atomic_inc(&conf->active_aligned_reads);5514 spin_unlock_irq(&conf->device_lock);5515 }5516 5517 mddev_trace_remap(mddev, align_bio, raid_bio->bi_iter.bi_sector);5518 submit_bio_noacct(align_bio);5519 return 1;5520}5521 5522static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)5523{5524 struct bio *split;5525 sector_t sector = raid_bio->bi_iter.bi_sector;5526 unsigned chunk_sects = mddev->chunk_sectors;5527 unsigned sectors = chunk_sects - (sector & (chunk_sects-1));5528 5529 if (sectors < bio_sectors(raid_bio)) {5530 struct r5conf *conf = mddev->private;5531 split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);5532 bio_chain(split, raid_bio);5533 submit_bio_noacct(raid_bio);5534 raid_bio = split;5535 }5536 5537 if (!raid5_read_one_chunk(mddev, raid_bio))5538 return raid_bio;5539 5540 return NULL;5541}5542 5543/* __get_priority_stripe - get the next stripe to process5544 *5545 * Full stripe writes are allowed to pass preread active stripes up until5546 * the bypass_threshold is exceeded. In general the bypass_count5547 * increments when the handle_list is handled before the hold_list; however, it5548 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a5549 * stripe with in flight i/o. The bypass_count will be reset when the5550 * head of the hold_list has changed, i.e. the head was promoted to the5551 * handle_list.5552 */5553static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)5554 __must_hold(&conf->device_lock)5555{5556 struct stripe_head *sh, *tmp;5557 struct list_head *handle_list = NULL;5558 struct r5worker_group *wg;5559 bool second_try = !r5c_is_writeback(conf->log) &&5560 !r5l_log_disk_error(conf);5561 bool try_loprio = test_bit(R5C_LOG_TIGHT, &conf->cache_state) ||5562 r5l_log_disk_error(conf);5563 5564again:5565 wg = NULL;5566 sh = NULL;5567 if (conf->worker_cnt_per_group == 0) {5568 handle_list = try_loprio ? &conf->loprio_list :5569 &conf->handle_list;5570 } else if (group != ANY_GROUP) {5571 handle_list = try_loprio ? &conf->worker_groups[group].loprio_list :5572 &conf->worker_groups[group].handle_list;5573 wg = &conf->worker_groups[group];5574 } else {5575 int i;5576 for (i = 0; i < conf->group_cnt; i++) {5577 handle_list = try_loprio ? &conf->worker_groups[i].loprio_list :5578 &conf->worker_groups[i].handle_list;5579 wg = &conf->worker_groups[i];5580 if (!list_empty(handle_list))5581 break;5582 }5583 }5584 5585 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",5586 __func__,5587 list_empty(handle_list) ? "empty" : "busy",5588 list_empty(&conf->hold_list) ? "empty" : "busy",5589 atomic_read(&conf->pending_full_writes), conf->bypass_count);5590 5591 if (!list_empty(handle_list)) {5592 sh = list_entry(handle_list->next, typeof(*sh), lru);5593 5594 if (list_empty(&conf->hold_list))5595 conf->bypass_count = 0;5596 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {5597 if (conf->hold_list.next == conf->last_hold)5598 conf->bypass_count++;5599 else {5600 conf->last_hold = conf->hold_list.next;5601 conf->bypass_count -= conf->bypass_threshold;5602 if (conf->bypass_count < 0)5603 conf->bypass_count = 0;5604 }5605 }5606 } else if (!list_empty(&conf->hold_list) &&5607 ((conf->bypass_threshold &&5608 conf->bypass_count > conf->bypass_threshold) ||5609 atomic_read(&conf->pending_full_writes) == 0)) {5610 5611 list_for_each_entry(tmp, &conf->hold_list, lru) {5612 if (conf->worker_cnt_per_group == 0 ||5613 group == ANY_GROUP ||5614 !cpu_online(tmp->cpu) ||5615 cpu_to_group(tmp->cpu) == group) {5616 sh = tmp;5617 break;5618 }5619 }5620 5621 if (sh) {5622 conf->bypass_count -= conf->bypass_threshold;5623 if (conf->bypass_count < 0)5624 conf->bypass_count = 0;5625 }5626 wg = NULL;5627 }5628 5629 if (!sh) {5630 if (second_try)5631 return NULL;5632 second_try = true;5633 try_loprio = !try_loprio;5634 goto again;5635 }5636 5637 if (wg) {5638 wg->stripes_cnt--;5639 sh->group = NULL;5640 }5641 list_del_init(&sh->lru);5642 BUG_ON(atomic_inc_return(&sh->count) != 1);5643 return sh;5644}5645 5646struct raid5_plug_cb {5647 struct blk_plug_cb cb;5648 struct list_head list;5649 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];5650};5651 5652static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)5653{5654 struct raid5_plug_cb *cb = container_of(5655 blk_cb, struct raid5_plug_cb, cb);5656 struct stripe_head *sh;5657 struct mddev *mddev = cb->cb.data;5658 struct r5conf *conf = mddev->private;5659 int cnt = 0;5660 int hash;5661 5662 if (cb->list.next && !list_empty(&cb->list)) {5663 spin_lock_irq(&conf->device_lock);5664 while (!list_empty(&cb->list)) {5665 sh = list_first_entry(&cb->list, struct stripe_head, lru);5666 list_del_init(&sh->lru);5667 /*5668 * avoid race release_stripe_plug() sees5669 * STRIPE_ON_UNPLUG_LIST clear but the stripe5670 * is still in our list5671 */5672 smp_mb__before_atomic();5673 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);5674 /*5675 * STRIPE_ON_RELEASE_LIST could be set here. In that5676 * case, the count is always > 1 here5677 */5678 hash = sh->hash_lock_index;5679 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);5680 cnt++;5681 }5682 spin_unlock_irq(&conf->device_lock);5683 }5684 release_inactive_stripe_list(conf, cb->temp_inactive_list,5685 NR_STRIPE_HASH_LOCKS);5686 if (!mddev_is_dm(mddev))5687 trace_block_unplug(mddev->gendisk->queue, cnt, !from_schedule);5688 kfree(cb);5689}5690 5691static void release_stripe_plug(struct mddev *mddev,5692 struct stripe_head *sh)5693{5694 struct blk_plug_cb *blk_cb = blk_check_plugged(5695 raid5_unplug, mddev,5696 sizeof(struct raid5_plug_cb));5697 struct raid5_plug_cb *cb;5698 5699 if (!blk_cb) {5700 raid5_release_stripe(sh);5701 return;5702 }5703 5704 cb = container_of(blk_cb, struct raid5_plug_cb, cb);5705 5706 if (cb->list.next == NULL) {5707 int i;5708 INIT_LIST_HEAD(&cb->list);5709 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)5710 INIT_LIST_HEAD(cb->temp_inactive_list + i);5711 }5712 5713 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))5714 list_add_tail(&sh->lru, &cb->list);5715 else5716 raid5_release_stripe(sh);5717}5718 5719static void make_discard_request(struct mddev *mddev, struct bio *bi)5720{5721 struct r5conf *conf = mddev->private;5722 sector_t logical_sector, last_sector;5723 struct stripe_head *sh;5724 int stripe_sectors;5725 5726 /* We need to handle this when io_uring supports discard/trim */5727 if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT))5728 return;5729 5730 if (mddev->reshape_position != MaxSector)5731 /* Skip discard while reshape is happening */5732 return;5733 5734 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);5735 last_sector = bio_end_sector(bi);5736 5737 bi->bi_next = NULL;5738 5739 stripe_sectors = conf->chunk_sectors *5740 (conf->raid_disks - conf->max_degraded);5741 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,5742 stripe_sectors);5743 sector_div(last_sector, stripe_sectors);5744 5745 logical_sector *= conf->chunk_sectors;5746 last_sector *= conf->chunk_sectors;5747 5748 for (; logical_sector < last_sector;5749 logical_sector += RAID5_STRIPE_SECTORS(conf)) {5750 DEFINE_WAIT(w);5751 int d;5752 again:5753 sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0);5754 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);5755 if (test_bit(STRIPE_SYNCING, &sh->state)) {5756 raid5_release_stripe(sh);5757 wait_on_bit(&sh->dev[sh->pd_idx].flags, R5_Overlap,5758 TASK_UNINTERRUPTIBLE);5759 goto again;5760 }5761 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);5762 spin_lock_irq(&sh->stripe_lock);5763 for (d = 0; d < conf->raid_disks; d++) {5764 if (d == sh->pd_idx || d == sh->qd_idx)5765 continue;5766 if (sh->dev[d].towrite || sh->dev[d].toread) {5767 set_bit(R5_Overlap, &sh->dev[d].flags);5768 spin_unlock_irq(&sh->stripe_lock);5769 raid5_release_stripe(sh);5770 wait_on_bit(&sh->dev[d].flags, R5_Overlap,5771 TASK_UNINTERRUPTIBLE);5772 goto again;5773 }5774 }5775 set_bit(STRIPE_DISCARD, &sh->state);5776 sh->overwrite_disks = 0;5777 for (d = 0; d < conf->raid_disks; d++) {5778 if (d == sh->pd_idx || d == sh->qd_idx)5779 continue;5780 sh->dev[d].towrite = bi;5781 set_bit(R5_OVERWRITE, &sh->dev[d].flags);5782 bio_inc_remaining(bi);5783 md_write_inc(mddev, bi);5784 sh->overwrite_disks++;5785 }5786 spin_unlock_irq(&sh->stripe_lock);5787 if (conf->mddev->bitmap) {5788 for (d = 0; d < conf->raid_disks - conf->max_degraded;5789 d++)5790 mddev->bitmap_ops->startwrite(mddev, sh->sector,5791 RAID5_STRIPE_SECTORS(conf), false);5792 sh->bm_seq = conf->seq_flush + 1;5793 set_bit(STRIPE_BIT_DELAY, &sh->state);5794 }5795 5796 set_bit(STRIPE_HANDLE, &sh->state);5797 clear_bit(STRIPE_DELAYED, &sh->state);5798 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))5799 atomic_inc(&conf->preread_active_stripes);5800 release_stripe_plug(mddev, sh);5801 }5802 5803 bio_endio(bi);5804}5805 5806static bool ahead_of_reshape(struct mddev *mddev, sector_t sector,5807 sector_t reshape_sector)5808{5809 return mddev->reshape_backwards ? sector < reshape_sector :5810 sector >= reshape_sector;5811}5812 5813static bool range_ahead_of_reshape(struct mddev *mddev, sector_t min,5814 sector_t max, sector_t reshape_sector)5815{5816 return mddev->reshape_backwards ? max < reshape_sector :5817 min >= reshape_sector;5818}5819 5820static bool stripe_ahead_of_reshape(struct mddev *mddev, struct r5conf *conf,5821 struct stripe_head *sh)5822{5823 sector_t max_sector = 0, min_sector = MaxSector;5824 bool ret = false;5825 int dd_idx;5826 5827 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {5828 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)5829 continue;5830 5831 min_sector = min(min_sector, sh->dev[dd_idx].sector);5832 max_sector = max(max_sector, sh->dev[dd_idx].sector);5833 }5834 5835 spin_lock_irq(&conf->device_lock);5836 5837 if (!range_ahead_of_reshape(mddev, min_sector, max_sector,5838 conf->reshape_progress))5839 /* mismatch, need to try again */5840 ret = true;5841 5842 spin_unlock_irq(&conf->device_lock);5843 5844 return ret;5845}5846 5847static int add_all_stripe_bios(struct r5conf *conf,5848 struct stripe_request_ctx *ctx, struct stripe_head *sh,5849 struct bio *bi, int forwrite, int previous)5850{5851 int dd_idx;5852 5853 spin_lock_irq(&sh->stripe_lock);5854 5855 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {5856 struct r5dev *dev = &sh->dev[dd_idx];5857 5858 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)5859 continue;5860 5861 if (dev->sector < ctx->first_sector ||5862 dev->sector >= ctx->last_sector)5863 continue;5864 5865 if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {5866 set_bit(R5_Overlap, &dev->flags);5867 spin_unlock_irq(&sh->stripe_lock);5868 raid5_release_stripe(sh);5869 /* release batch_last before wait to avoid risk of deadlock */5870 if (ctx->batch_last) {5871 raid5_release_stripe(ctx->batch_last);5872 ctx->batch_last = NULL;5873 }5874 md_wakeup_thread(conf->mddev->thread);5875 wait_on_bit(&dev->flags, R5_Overlap, TASK_UNINTERRUPTIBLE);5876 return 0;5877 }5878 }5879 5880 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {5881 struct r5dev *dev = &sh->dev[dd_idx];5882 5883 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)5884 continue;5885 5886 if (dev->sector < ctx->first_sector ||5887 dev->sector >= ctx->last_sector)5888 continue;5889 5890 __add_stripe_bio(sh, bi, dd_idx, forwrite, previous);5891 clear_bit((dev->sector - ctx->first_sector) >>5892 RAID5_STRIPE_SHIFT(conf), ctx->sectors_to_do);5893 }5894 5895 spin_unlock_irq(&sh->stripe_lock);5896 return 1;5897}5898 5899enum reshape_loc {5900 LOC_NO_RESHAPE,5901 LOC_AHEAD_OF_RESHAPE,5902 LOC_INSIDE_RESHAPE,5903 LOC_BEHIND_RESHAPE,5904};5905 5906static enum reshape_loc get_reshape_loc(struct mddev *mddev,5907 struct r5conf *conf, sector_t logical_sector)5908{5909 sector_t reshape_progress, reshape_safe;5910 /*5911 * Spinlock is needed as reshape_progress may be5912 * 64bit on a 32bit platform, and so it might be5913 * possible to see a half-updated value5914 * Of course reshape_progress could change after5915 * the lock is dropped, so once we get a reference5916 * to the stripe that we think it is, we will have5917 * to check again.5918 */5919 spin_lock_irq(&conf->device_lock);5920 reshape_progress = conf->reshape_progress;5921 reshape_safe = conf->reshape_safe;5922 spin_unlock_irq(&conf->device_lock);5923 if (reshape_progress == MaxSector)5924 return LOC_NO_RESHAPE;5925 if (ahead_of_reshape(mddev, logical_sector, reshape_progress))5926 return LOC_AHEAD_OF_RESHAPE;5927 if (ahead_of_reshape(mddev, logical_sector, reshape_safe))5928 return LOC_INSIDE_RESHAPE;5929 return LOC_BEHIND_RESHAPE;5930}5931 5932static enum stripe_result make_stripe_request(struct mddev *mddev,5933 struct r5conf *conf, struct stripe_request_ctx *ctx,5934 sector_t logical_sector, struct bio *bi)5935{5936 const int rw = bio_data_dir(bi);5937 enum stripe_result ret;5938 struct stripe_head *sh;5939 sector_t new_sector;5940 int previous = 0, flags = 0;5941 int seq, dd_idx;5942 5943 seq = read_seqcount_begin(&conf->gen_lock);5944 5945 if (unlikely(conf->reshape_progress != MaxSector)) {5946 enum reshape_loc loc = get_reshape_loc(mddev, conf,5947 logical_sector);5948 if (loc == LOC_INSIDE_RESHAPE) {5949 ret = STRIPE_SCHEDULE_AND_RETRY;5950 goto out;5951 }5952 if (loc == LOC_AHEAD_OF_RESHAPE)5953 previous = 1;5954 }5955 5956 new_sector = raid5_compute_sector(conf, logical_sector, previous,5957 &dd_idx, NULL);5958 pr_debug("raid456: %s, sector %llu logical %llu\n", __func__,5959 new_sector, logical_sector);5960 5961 if (previous)5962 flags |= R5_GAS_PREVIOUS;5963 if (bi->bi_opf & REQ_RAHEAD)5964 flags |= R5_GAS_NOBLOCK;5965 sh = raid5_get_active_stripe(conf, ctx, new_sector, flags);5966 if (unlikely(!sh)) {5967 /* cannot get stripe, just give-up */5968 bi->bi_status = BLK_STS_IOERR;5969 return STRIPE_FAIL;5970 }5971 5972 if (unlikely(previous) &&5973 stripe_ahead_of_reshape(mddev, conf, sh)) {5974 /*5975 * Expansion moved on while waiting for a stripe.5976 * Expansion could still move past after this5977 * test, but as we are holding a reference to5978 * 'sh', we know that if that happens,5979 * STRIPE_EXPANDING will get set and the expansion5980 * won't proceed until we finish with the stripe.5981 */5982 ret = STRIPE_SCHEDULE_AND_RETRY;5983 goto out_release;5984 }5985 5986 if (read_seqcount_retry(&conf->gen_lock, seq)) {5987 /* Might have got the wrong stripe_head by accident */5988 ret = STRIPE_RETRY;5989 goto out_release;5990 }5991 5992 if (test_bit(STRIPE_EXPANDING, &sh->state)) {5993 md_wakeup_thread(mddev->thread);5994 ret = STRIPE_SCHEDULE_AND_RETRY;5995 goto out_release;5996 }5997 5998 if (!add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {5999 ret = STRIPE_RETRY;6000 goto out;6001 }6002 6003 if (stripe_can_batch(sh)) {6004 stripe_add_to_batch_list(conf, sh, ctx->batch_last);6005 if (ctx->batch_last)6006 raid5_release_stripe(ctx->batch_last);6007 atomic_inc(&sh->count);6008 ctx->batch_last = sh;6009 }6010 6011 if (ctx->do_flush) {6012 set_bit(STRIPE_R5C_PREFLUSH, &sh->state);6013 /* we only need flush for one stripe */6014 ctx->do_flush = false;6015 }6016 6017 set_bit(STRIPE_HANDLE, &sh->state);6018 clear_bit(STRIPE_DELAYED, &sh->state);6019 if ((!sh->batch_head || sh == sh->batch_head) &&6020 (bi->bi_opf & REQ_SYNC) &&6021 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))6022 atomic_inc(&conf->preread_active_stripes);6023 6024 release_stripe_plug(mddev, sh);6025 return STRIPE_SUCCESS;6026 6027out_release:6028 raid5_release_stripe(sh);6029out:6030 if (ret == STRIPE_SCHEDULE_AND_RETRY && reshape_interrupted(mddev)) {6031 bi->bi_status = BLK_STS_RESOURCE;6032 ret = STRIPE_WAIT_RESHAPE;6033 pr_err_ratelimited("dm-raid456: io across reshape position while reshape can't make progress");6034 }6035 return ret;6036}6037 6038/*6039 * If the bio covers multiple data disks, find sector within the bio that has6040 * the lowest chunk offset in the first chunk.6041 */6042static sector_t raid5_bio_lowest_chunk_sector(struct r5conf *conf,6043 struct bio *bi)6044{6045 int sectors_per_chunk = conf->chunk_sectors;6046 int raid_disks = conf->raid_disks;6047 int dd_idx;6048 struct stripe_head sh;6049 unsigned int chunk_offset;6050 sector_t r_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);6051 sector_t sector;6052 6053 /* We pass in fake stripe_head to get back parity disk numbers */6054 sector = raid5_compute_sector(conf, r_sector, 0, &dd_idx, &sh);6055 chunk_offset = sector_div(sector, sectors_per_chunk);6056 if (sectors_per_chunk - chunk_offset >= bio_sectors(bi))6057 return r_sector;6058 /*6059 * Bio crosses to the next data disk. Check whether it's in the same6060 * chunk.6061 */6062 dd_idx++;6063 while (dd_idx == sh.pd_idx || dd_idx == sh.qd_idx)6064 dd_idx++;6065 if (dd_idx >= raid_disks)6066 return r_sector;6067 return r_sector + sectors_per_chunk - chunk_offset;6068}6069 6070static bool raid5_make_request(struct mddev *mddev, struct bio * bi)6071{6072 DEFINE_WAIT_FUNC(wait, woken_wake_function);6073 bool on_wq;6074 struct r5conf *conf = mddev->private;6075 sector_t logical_sector;6076 struct stripe_request_ctx ctx = {};6077 const int rw = bio_data_dir(bi);6078 enum stripe_result res;6079 int s, stripe_cnt;6080 6081 if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {6082 int ret = log_handle_flush_request(conf, bi);6083 6084 if (ret == 0)6085 return true;6086 if (ret == -ENODEV) {6087 if (md_flush_request(mddev, bi))6088 return true;6089 }6090 /* ret == -EAGAIN, fallback */6091 /*6092 * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH,6093 * we need to flush journal device6094 */6095 ctx.do_flush = bi->bi_opf & REQ_PREFLUSH;6096 }6097 6098 md_write_start(mddev, bi);6099 /*6100 * If array is degraded, better not do chunk aligned read because6101 * later we might have to read it again in order to reconstruct6102 * data on failed drives.6103 */6104 if (rw == READ && mddev->degraded == 0 &&6105 mddev->reshape_position == MaxSector) {6106 bi = chunk_aligned_read(mddev, bi);6107 if (!bi)6108 return true;6109 }6110 6111 if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) {6112 make_discard_request(mddev, bi);6113 md_write_end(mddev);6114 return true;6115 }6116 6117 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);6118 ctx.first_sector = logical_sector;6119 ctx.last_sector = bio_end_sector(bi);6120 bi->bi_next = NULL;6121 6122 stripe_cnt = DIV_ROUND_UP_SECTOR_T(ctx.last_sector - logical_sector,6123 RAID5_STRIPE_SECTORS(conf));6124 bitmap_set(ctx.sectors_to_do, 0, stripe_cnt);6125 6126 pr_debug("raid456: %s, logical %llu to %llu\n", __func__,6127 bi->bi_iter.bi_sector, ctx.last_sector);6128 6129 /* Bail out if conflicts with reshape and REQ_NOWAIT is set */6130 if ((bi->bi_opf & REQ_NOWAIT) &&6131 (conf->reshape_progress != MaxSector) &&6132 get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) {6133 bio_wouldblock_error(bi);6134 if (rw == WRITE)6135 md_write_end(mddev);6136 return true;6137 }6138 md_account_bio(mddev, &bi);6139 6140 /*6141 * Lets start with the stripe with the lowest chunk offset in the first6142 * chunk. That has the best chances of creating IOs adjacent to6143 * previous IOs in case of sequential IO and thus creates the most6144 * sequential IO pattern. We don't bother with the optimization when6145 * reshaping as the performance benefit is not worth the complexity.6146 */6147 if (likely(conf->reshape_progress == MaxSector)) {6148 logical_sector = raid5_bio_lowest_chunk_sector(conf, bi);6149 on_wq = false;6150 } else {6151 add_wait_queue(&conf->wait_for_reshape, &wait);6152 on_wq = true;6153 }6154 s = (logical_sector - ctx.first_sector) >> RAID5_STRIPE_SHIFT(conf);6155 6156 while (1) {6157 res = make_stripe_request(mddev, conf, &ctx, logical_sector,6158 bi);6159 if (res == STRIPE_FAIL || res == STRIPE_WAIT_RESHAPE)6160 break;6161 6162 if (res == STRIPE_RETRY)6163 continue;6164 6165 if (res == STRIPE_SCHEDULE_AND_RETRY) {6166 WARN_ON_ONCE(!on_wq);6167 /*6168 * Must release the reference to batch_last before6169 * scheduling and waiting for work to be done,6170 * otherwise the batch_last stripe head could prevent6171 * raid5_activate_delayed() from making progress6172 * and thus deadlocking.6173 */6174 if (ctx.batch_last) {6175 raid5_release_stripe(ctx.batch_last);6176 ctx.batch_last = NULL;6177 }6178 6179 wait_woken(&wait, TASK_UNINTERRUPTIBLE,6180 MAX_SCHEDULE_TIMEOUT);6181 continue;6182 }6183 6184 s = find_next_bit_wrap(ctx.sectors_to_do, stripe_cnt, s);6185 if (s == stripe_cnt)6186 break;6187 6188 logical_sector = ctx.first_sector +6189 (s << RAID5_STRIPE_SHIFT(conf));6190 }6191 if (unlikely(on_wq))6192 remove_wait_queue(&conf->wait_for_reshape, &wait);6193 6194 if (ctx.batch_last)6195 raid5_release_stripe(ctx.batch_last);6196 6197 if (rw == WRITE)6198 md_write_end(mddev);6199 if (res == STRIPE_WAIT_RESHAPE) {6200 md_free_cloned_bio(bi);6201 return false;6202 }6203 6204 bio_endio(bi);6205 return true;6206}6207 6208static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);6209 6210static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)6211{6212 /* reshaping is quite different to recovery/resync so it is6213 * handled quite separately ... here.6214 *6215 * On each call to sync_request, we gather one chunk worth of6216 * destination stripes and flag them as expanding.6217 * Then we find all the source stripes and request reads.6218 * As the reads complete, handle_stripe will copy the data6219 * into the destination stripe and release that stripe.6220 */6221 struct r5conf *conf = mddev->private;6222 struct stripe_head *sh;6223 struct md_rdev *rdev;6224 sector_t first_sector, last_sector;6225 int raid_disks = conf->previous_raid_disks;6226 int data_disks = raid_disks - conf->max_degraded;6227 int new_data_disks = conf->raid_disks - conf->max_degraded;6228 int i;6229 int dd_idx;6230 sector_t writepos, readpos, safepos;6231 sector_t stripe_addr;6232 int reshape_sectors;6233 struct list_head stripes;6234 sector_t retn;6235 6236 if (sector_nr == 0) {6237 /* If restarting in the middle, skip the initial sectors */6238 if (mddev->reshape_backwards &&6239 conf->reshape_progress < raid5_size(mddev, 0, 0)) {6240 sector_nr = raid5_size(mddev, 0, 0)6241 - conf->reshape_progress;6242 } else if (mddev->reshape_backwards &&6243 conf->reshape_progress == MaxSector) {6244 /* shouldn't happen, but just in case, finish up.*/6245 sector_nr = MaxSector;6246 } else if (!mddev->reshape_backwards &&6247 conf->reshape_progress > 0)6248 sector_nr = conf->reshape_progress;6249 sector_div(sector_nr, new_data_disks);6250 if (sector_nr) {6251 mddev->curr_resync_completed = sector_nr;6252 sysfs_notify_dirent_safe(mddev->sysfs_completed);6253 *skipped = 1;6254 retn = sector_nr;6255 goto finish;6256 }6257 }6258 6259 /* We need to process a full chunk at a time.6260 * If old and new chunk sizes differ, we need to process the6261 * largest of these6262 */6263 6264 reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors);6265 6266 /* We update the metadata at least every 10 seconds, or when6267 * the data about to be copied would over-write the source of6268 * the data at the front of the range. i.e. one new_stripe6269 * along from reshape_progress new_maps to after where6270 * reshape_safe old_maps to6271 */6272 writepos = conf->reshape_progress;6273 sector_div(writepos, new_data_disks);6274 readpos = conf->reshape_progress;6275 sector_div(readpos, data_disks);6276 safepos = conf->reshape_safe;6277 sector_div(safepos, data_disks);6278 if (mddev->reshape_backwards) {6279 if (WARN_ON(writepos < reshape_sectors))6280 return MaxSector;6281 6282 writepos -= reshape_sectors;6283 readpos += reshape_sectors;6284 safepos += reshape_sectors;6285 } else {6286 writepos += reshape_sectors;6287 /* readpos and safepos are worst-case calculations.6288 * A negative number is overly pessimistic, and causes6289 * obvious problems for unsigned storage. So clip to 0.6290 */6291 readpos -= min_t(sector_t, reshape_sectors, readpos);6292 safepos -= min_t(sector_t, reshape_sectors, safepos);6293 }6294 6295 /* Having calculated the 'writepos' possibly use it6296 * to set 'stripe_addr' which is where we will write to.6297 */6298 if (mddev->reshape_backwards) {6299 if (WARN_ON(conf->reshape_progress == 0))6300 return MaxSector;6301 6302 stripe_addr = writepos;6303 if (WARN_ON((mddev->dev_sectors &6304 ~((sector_t)reshape_sectors - 1)) -6305 reshape_sectors - stripe_addr != sector_nr))6306 return MaxSector;6307 } else {6308 if (WARN_ON(writepos != sector_nr + reshape_sectors))6309 return MaxSector;6310 6311 stripe_addr = sector_nr;6312 }6313 6314 /* 'writepos' is the most advanced device address we might write.6315 * 'readpos' is the least advanced device address we might read.6316 * 'safepos' is the least address recorded in the metadata as having6317 * been reshaped.6318 * If there is a min_offset_diff, these are adjusted either by6319 * increasing the safepos/readpos if diff is negative, or6320 * increasing writepos if diff is positive.6321 * If 'readpos' is then behind 'writepos', there is no way that we can6322 * ensure safety in the face of a crash - that must be done by userspace6323 * making a backup of the data. So in that case there is no particular6324 * rush to update metadata.6325 * Otherwise if 'safepos' is behind 'writepos', then we really need to6326 * update the metadata to advance 'safepos' to match 'readpos' so that6327 * we can be safe in the event of a crash.6328 * So we insist on updating metadata if safepos is behind writepos and6329 * readpos is beyond writepos.6330 * In any case, update the metadata every 10 seconds.6331 * Maybe that number should be configurable, but I'm not sure it is6332 * worth it.... maybe it could be a multiple of safemode_delay???6333 */6334 if (conf->min_offset_diff < 0) {6335 safepos += -conf->min_offset_diff;6336 readpos += -conf->min_offset_diff;6337 } else6338 writepos += conf->min_offset_diff;6339 6340 if ((mddev->reshape_backwards6341 ? (safepos > writepos && readpos < writepos)6342 : (safepos < writepos && readpos > writepos)) ||6343 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {6344 /* Cannot proceed until we've updated the superblock... */6345 wait_event(conf->wait_for_reshape,6346 atomic_read(&conf->reshape_stripes)==06347 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));6348 if (atomic_read(&conf->reshape_stripes) != 0)6349 return 0;6350 mddev->reshape_position = conf->reshape_progress;6351 mddev->curr_resync_completed = sector_nr;6352 if (!mddev->reshape_backwards)6353 /* Can update recovery_offset */6354 rdev_for_each(rdev, mddev)6355 if (rdev->raid_disk >= 0 &&6356 !test_bit(Journal, &rdev->flags) &&6357 !test_bit(In_sync, &rdev->flags) &&6358 rdev->recovery_offset < sector_nr)6359 rdev->recovery_offset = sector_nr;6360 6361 conf->reshape_checkpoint = jiffies;6362 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);6363 md_wakeup_thread(mddev->thread);6364 wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||6365 test_bit(MD_RECOVERY_INTR, &mddev->recovery));6366 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))6367 return 0;6368 spin_lock_irq(&conf->device_lock);6369 conf->reshape_safe = mddev->reshape_position;6370 spin_unlock_irq(&conf->device_lock);6371 wake_up(&conf->wait_for_reshape);6372 sysfs_notify_dirent_safe(mddev->sysfs_completed);6373 }6374 6375 INIT_LIST_HEAD(&stripes);6376 for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {6377 int j;6378 int skipped_disk = 0;6379 sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i,6380 R5_GAS_NOQUIESCE);6381 set_bit(STRIPE_EXPANDING, &sh->state);6382 atomic_inc(&conf->reshape_stripes);6383 /* If any of this stripe is beyond the end of the old6384 * array, then we need to zero those blocks6385 */6386 for (j=sh->disks; j--;) {6387 sector_t s;6388 if (j == sh->pd_idx)6389 continue;6390 if (conf->level == 6 &&6391 j == sh->qd_idx)6392 continue;6393 s = raid5_compute_blocknr(sh, j, 0);6394 if (s < raid5_size(mddev, 0, 0)) {6395 skipped_disk = 1;6396 continue;6397 }6398 memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));6399 set_bit(R5_Expanded, &sh->dev[j].flags);6400 set_bit(R5_UPTODATE, &sh->dev[j].flags);6401 }6402 if (!skipped_disk) {6403 set_bit(STRIPE_EXPAND_READY, &sh->state);6404 set_bit(STRIPE_HANDLE, &sh->state);6405 }6406 list_add(&sh->lru, &stripes);6407 }6408 spin_lock_irq(&conf->device_lock);6409 if (mddev->reshape_backwards)6410 conf->reshape_progress -= reshape_sectors * new_data_disks;6411 else6412 conf->reshape_progress += reshape_sectors * new_data_disks;6413 spin_unlock_irq(&conf->device_lock);6414 /* Ok, those stripe are ready. We can start scheduling6415 * reads on the source stripes.6416 * The source stripes are determined by mapping the first and last6417 * block on the destination stripes.6418 */6419 first_sector =6420 raid5_compute_sector(conf, stripe_addr*(new_data_disks),6421 1, &dd_idx, NULL);6422 last_sector =6423 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)6424 * new_data_disks - 1),6425 1, &dd_idx, NULL);6426 if (last_sector >= mddev->dev_sectors)6427 last_sector = mddev->dev_sectors - 1;6428 while (first_sector <= last_sector) {6429 sh = raid5_get_active_stripe(conf, NULL, first_sector,6430 R5_GAS_PREVIOUS | R5_GAS_NOQUIESCE);6431 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);6432 set_bit(STRIPE_HANDLE, &sh->state);6433 raid5_release_stripe(sh);6434 first_sector += RAID5_STRIPE_SECTORS(conf);6435 }6436 /* Now that the sources are clearly marked, we can release6437 * the destination stripes6438 */6439 while (!list_empty(&stripes)) {6440 sh = list_entry(stripes.next, struct stripe_head, lru);6441 list_del_init(&sh->lru);6442 raid5_release_stripe(sh);6443 }6444 /* If this takes us to the resync_max point where we have to pause,6445 * then we need to write out the superblock.6446 */6447 sector_nr += reshape_sectors;6448 retn = reshape_sectors;6449finish:6450 if (mddev->curr_resync_completed > mddev->resync_max ||6451 (sector_nr - mddev->curr_resync_completed) * 26452 >= mddev->resync_max - mddev->curr_resync_completed) {6453 /* Cannot proceed until we've updated the superblock... */6454 wait_event(conf->wait_for_reshape,6455 atomic_read(&conf->reshape_stripes) == 06456 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));6457 if (atomic_read(&conf->reshape_stripes) != 0)6458 goto ret;6459 mddev->reshape_position = conf->reshape_progress;6460 mddev->curr_resync_completed = sector_nr;6461 if (!mddev->reshape_backwards)6462 /* Can update recovery_offset */6463 rdev_for_each(rdev, mddev)6464 if (rdev->raid_disk >= 0 &&6465 !test_bit(Journal, &rdev->flags) &&6466 !test_bit(In_sync, &rdev->flags) &&6467 rdev->recovery_offset < sector_nr)6468 rdev->recovery_offset = sector_nr;6469 conf->reshape_checkpoint = jiffies;6470 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);6471 md_wakeup_thread(mddev->thread);6472 wait_event(mddev->sb_wait,6473 !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)6474 || test_bit(MD_RECOVERY_INTR, &mddev->recovery));6475 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))6476 goto ret;6477 spin_lock_irq(&conf->device_lock);6478 conf->reshape_safe = mddev->reshape_position;6479 spin_unlock_irq(&conf->device_lock);6480 wake_up(&conf->wait_for_reshape);6481 sysfs_notify_dirent_safe(mddev->sysfs_completed);6482 }6483ret:6484 return retn;6485}6486 6487static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_nr,6488 sector_t max_sector, int *skipped)6489{6490 struct r5conf *conf = mddev->private;6491 struct stripe_head *sh;6492 sector_t sync_blocks;6493 bool still_degraded = false;6494 int i;6495 6496 if (sector_nr >= max_sector) {6497 /* just being told to finish up .. nothing much to do */6498 6499 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {6500 end_reshape(conf);6501 return 0;6502 }6503 6504 if (mddev->curr_resync < max_sector) /* aborted */6505 mddev->bitmap_ops->end_sync(mddev, mddev->curr_resync,6506 &sync_blocks);6507 else /* completed sync */6508 conf->fullsync = 0;6509 mddev->bitmap_ops->close_sync(mddev);6510 6511 return 0;6512 }6513 6514 /* Allow raid5_quiesce to complete */6515 wait_event(conf->wait_for_reshape, conf->quiesce != 2);6516 6517 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))6518 return reshape_request(mddev, sector_nr, skipped);6519 6520 /* No need to check resync_max as we never do more than one6521 * stripe, and as resync_max will always be on a chunk boundary,6522 * if the check in md_do_sync didn't fire, there is no chance6523 * of overstepping resync_max here6524 */6525 6526 /* if there is too many failed drives and we are trying6527 * to resync, then assert that we are finished, because there is6528 * nothing we can do.6529 */6530 if (mddev->degraded >= conf->max_degraded &&6531 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {6532 sector_t rv = mddev->dev_sectors - sector_nr;6533 *skipped = 1;6534 return rv;6535 }6536 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&6537 !conf->fullsync &&6538 !mddev->bitmap_ops->start_sync(mddev, sector_nr, &sync_blocks,6539 true) &&6540 sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {6541 /* we can skip this block, and probably more */6542 do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));6543 *skipped = 1;6544 /* keep things rounded to whole stripes */6545 return sync_blocks * RAID5_STRIPE_SECTORS(conf);6546 }6547 6548 mddev->bitmap_ops->cond_end_sync(mddev, sector_nr, false);6549 6550 sh = raid5_get_active_stripe(conf, NULL, sector_nr,6551 R5_GAS_NOBLOCK);6552 if (sh == NULL) {6553 sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0);6554 /* make sure we don't swamp the stripe cache if someone else6555 * is trying to get access6556 */6557 schedule_timeout_uninterruptible(1);6558 }6559 /* Need to check if array will still be degraded after recovery/resync6560 * Note in case of > 1 drive failures it's possible we're rebuilding6561 * one drive while leaving another faulty drive in array.6562 */6563 for (i = 0; i < conf->raid_disks; i++) {6564 struct md_rdev *rdev = conf->disks[i].rdev;6565 6566 if (rdev == NULL || test_bit(Faulty, &rdev->flags))6567 still_degraded = true;6568 }6569 6570 mddev->bitmap_ops->start_sync(mddev, sector_nr, &sync_blocks,6571 still_degraded);6572 6573 set_bit(STRIPE_SYNC_REQUESTED, &sh->state);6574 set_bit(STRIPE_HANDLE, &sh->state);6575 6576 raid5_release_stripe(sh);6577 6578 return RAID5_STRIPE_SECTORS(conf);6579}6580 6581static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio,6582 unsigned int offset)6583{6584 /* We may not be able to submit a whole bio at once as there6585 * may not be enough stripe_heads available.6586 * We cannot pre-allocate enough stripe_heads as we may need6587 * more than exist in the cache (if we allow ever large chunks).6588 * So we do one stripe head at a time and record in6589 * ->bi_hw_segments how many have been done.6590 *6591 * We *know* that this entire raid_bio is in one chunk, so6592 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.6593 */6594 struct stripe_head *sh;6595 int dd_idx;6596 sector_t sector, logical_sector, last_sector;6597 int scnt = 0;6598 int handled = 0;6599 6600 logical_sector = raid_bio->bi_iter.bi_sector &6601 ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);6602 sector = raid5_compute_sector(conf, logical_sector,6603 0, &dd_idx, NULL);6604 last_sector = bio_end_sector(raid_bio);6605 6606 for (; logical_sector < last_sector;6607 logical_sector += RAID5_STRIPE_SECTORS(conf),6608 sector += RAID5_STRIPE_SECTORS(conf),6609 scnt++) {6610 6611 if (scnt < offset)6612 /* already done this stripe */6613 continue;6614 6615 sh = raid5_get_active_stripe(conf, NULL, sector,6616 R5_GAS_NOBLOCK | R5_GAS_NOQUIESCE);6617 if (!sh) {6618 /* failed to get a stripe - must wait */6619 conf->retry_read_aligned = raid_bio;6620 conf->retry_read_offset = scnt;6621 return handled;6622 }6623 6624 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {6625 raid5_release_stripe(sh);6626 conf->retry_read_aligned = raid_bio;6627 conf->retry_read_offset = scnt;6628 return handled;6629 }6630 6631 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);6632 handle_stripe(sh);6633 raid5_release_stripe(sh);6634 handled++;6635 }6636 6637 bio_endio(raid_bio);6638 6639 if (atomic_dec_and_test(&conf->active_aligned_reads))6640 wake_up(&conf->wait_for_quiescent);6641 return handled;6642}6643 6644static int handle_active_stripes(struct r5conf *conf, int group,6645 struct r5worker *worker,6646 struct list_head *temp_inactive_list)6647 __must_hold(&conf->device_lock)6648{6649 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;6650 int i, batch_size = 0, hash;6651 bool release_inactive = false;6652 6653 while (batch_size < MAX_STRIPE_BATCH &&6654 (sh = __get_priority_stripe(conf, group)) != NULL)6655 batch[batch_size++] = sh;6656 6657 if (batch_size == 0) {6658 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)6659 if (!list_empty(temp_inactive_list + i))6660 break;6661 if (i == NR_STRIPE_HASH_LOCKS) {6662 spin_unlock_irq(&conf->device_lock);6663 log_flush_stripe_to_raid(conf);6664 spin_lock_irq(&conf->device_lock);6665 return batch_size;6666 }6667 release_inactive = true;6668 }6669 spin_unlock_irq(&conf->device_lock);6670 6671 release_inactive_stripe_list(conf, temp_inactive_list,6672 NR_STRIPE_HASH_LOCKS);6673 6674 r5l_flush_stripe_to_raid(conf->log);6675 if (release_inactive) {6676 spin_lock_irq(&conf->device_lock);6677 return 0;6678 }6679 6680 for (i = 0; i < batch_size; i++)6681 handle_stripe(batch[i]);6682 log_write_stripe_run(conf);6683 6684 cond_resched();6685 6686 spin_lock_irq(&conf->device_lock);6687 for (i = 0; i < batch_size; i++) {6688 hash = batch[i]->hash_lock_index;6689 __release_stripe(conf, batch[i], &temp_inactive_list[hash]);6690 }6691 return batch_size;6692}6693 6694static void raid5_do_work(struct work_struct *work)6695{6696 struct r5worker *worker = container_of(work, struct r5worker, work);6697 struct r5worker_group *group = worker->group;6698 struct r5conf *conf = group->conf;6699 struct mddev *mddev = conf->mddev;6700 int group_id = group - conf->worker_groups;6701 int handled;6702 struct blk_plug plug;6703 6704 pr_debug("+++ raid5worker active\n");6705 6706 blk_start_plug(&plug);6707 handled = 0;6708 spin_lock_irq(&conf->device_lock);6709 while (1) {6710 int batch_size, released;6711 6712 released = release_stripe_list(conf, worker->temp_inactive_list);6713 6714 batch_size = handle_active_stripes(conf, group_id, worker,6715 worker->temp_inactive_list);6716 worker->working = false;6717 if (!batch_size && !released)6718 break;6719 handled += batch_size;6720 wait_event_lock_irq(mddev->sb_wait,6721 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),6722 conf->device_lock);6723 }6724 pr_debug("%d stripes handled\n", handled);6725 6726 spin_unlock_irq(&conf->device_lock);6727 6728 flush_deferred_bios(conf);6729 6730 r5l_flush_stripe_to_raid(conf->log);6731 6732 async_tx_issue_pending_all();6733 blk_finish_plug(&plug);6734 6735 pr_debug("--- raid5worker inactive\n");6736}6737 6738/*6739 * This is our raid5 kernel thread.6740 *6741 * We scan the hash table for stripes which can be handled now.6742 * During the scan, completed stripes are saved for us by the interrupt6743 * handler, so that they will not have to wait for our next wakeup.6744 */6745static void raid5d(struct md_thread *thread)6746{6747 struct mddev *mddev = thread->mddev;6748 struct r5conf *conf = mddev->private;6749 int handled;6750 struct blk_plug plug;6751 6752 pr_debug("+++ raid5d active\n");6753 6754 md_check_recovery(mddev);6755 6756 blk_start_plug(&plug);6757 handled = 0;6758 spin_lock_irq(&conf->device_lock);6759 while (1) {6760 struct bio *bio;6761 int batch_size, released;6762 unsigned int offset;6763 6764 if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags))6765 break;6766 6767 released = release_stripe_list(conf, conf->temp_inactive_list);6768 if (released)6769 clear_bit(R5_DID_ALLOC, &conf->cache_state);6770 6771 if (6772 !list_empty(&conf->bitmap_list)) {6773 /* Now is a good time to flush some bitmap updates */6774 conf->seq_flush++;6775 spin_unlock_irq(&conf->device_lock);6776 mddev->bitmap_ops->unplug(mddev, true);6777 spin_lock_irq(&conf->device_lock);6778 conf->seq_write = conf->seq_flush;6779 activate_bit_delay(conf, conf->temp_inactive_list);6780 }6781 raid5_activate_delayed(conf);6782 6783 while ((bio = remove_bio_from_retry(conf, &offset))) {6784 int ok;6785 spin_unlock_irq(&conf->device_lock);6786 ok = retry_aligned_read(conf, bio, offset);6787 spin_lock_irq(&conf->device_lock);6788 if (!ok)6789 break;6790 handled++;6791 }6792 6793 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,6794 conf->temp_inactive_list);6795 if (!batch_size && !released)6796 break;6797 handled += batch_size;6798 6799 if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) {6800 spin_unlock_irq(&conf->device_lock);6801 md_check_recovery(mddev);6802 spin_lock_irq(&conf->device_lock);6803 }6804 }6805 pr_debug("%d stripes handled\n", handled);6806 6807 spin_unlock_irq(&conf->device_lock);6808 if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&6809 mutex_trylock(&conf->cache_size_mutex)) {6810 grow_one_stripe(conf, __GFP_NOWARN);6811 /* Set flag even if allocation failed. This helps6812 * slow down allocation requests when mem is short6813 */6814 set_bit(R5_DID_ALLOC, &conf->cache_state);6815 mutex_unlock(&conf->cache_size_mutex);6816 }6817 6818 flush_deferred_bios(conf);6819 6820 r5l_flush_stripe_to_raid(conf->log);6821 6822 async_tx_issue_pending_all();6823 blk_finish_plug(&plug);6824 6825 pr_debug("--- raid5d inactive\n");6826}6827 6828static ssize_t6829raid5_show_stripe_cache_size(struct mddev *mddev, char *page)6830{6831 struct r5conf *conf;6832 int ret = 0;6833 spin_lock(&mddev->lock);6834 conf = mddev->private;6835 if (conf)6836 ret = sprintf(page, "%d\n", conf->min_nr_stripes);6837 spin_unlock(&mddev->lock);6838 return ret;6839}6840 6841int6842raid5_set_cache_size(struct mddev *mddev, int size)6843{6844 int result = 0;6845 struct r5conf *conf = mddev->private;6846 6847 if (size <= 16 || size > 32768)6848 return -EINVAL;6849 6850 WRITE_ONCE(conf->min_nr_stripes, size);6851 mutex_lock(&conf->cache_size_mutex);6852 while (size < conf->max_nr_stripes &&6853 drop_one_stripe(conf))6854 ;6855 mutex_unlock(&conf->cache_size_mutex);6856 6857 md_allow_write(mddev);6858 6859 mutex_lock(&conf->cache_size_mutex);6860 while (size > conf->max_nr_stripes)6861 if (!grow_one_stripe(conf, GFP_KERNEL)) {6862 WRITE_ONCE(conf->min_nr_stripes, conf->max_nr_stripes);6863 result = -ENOMEM;6864 break;6865 }6866 mutex_unlock(&conf->cache_size_mutex);6867 6868 return result;6869}6870EXPORT_SYMBOL(raid5_set_cache_size);6871 6872static ssize_t6873raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)6874{6875 struct r5conf *conf;6876 unsigned long new;6877 int err;6878 6879 if (len >= PAGE_SIZE)6880 return -EINVAL;6881 if (kstrtoul(page, 10, &new))6882 return -EINVAL;6883 err = mddev_lock(mddev);6884 if (err)6885 return err;6886 conf = mddev->private;6887 if (!conf)6888 err = -ENODEV;6889 else6890 err = raid5_set_cache_size(mddev, new);6891 mddev_unlock(mddev);6892 6893 return err ?: len;6894}6895 6896static struct md_sysfs_entry6897raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,6898 raid5_show_stripe_cache_size,6899 raid5_store_stripe_cache_size);6900 6901static ssize_t6902raid5_show_rmw_level(struct mddev *mddev, char *page)6903{6904 struct r5conf *conf = mddev->private;6905 if (conf)6906 return sprintf(page, "%d\n", conf->rmw_level);6907 else6908 return 0;6909}6910 6911static ssize_t6912raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)6913{6914 struct r5conf *conf = mddev->private;6915 unsigned long new;6916 6917 if (!conf)6918 return -ENODEV;6919 6920 if (len >= PAGE_SIZE)6921 return -EINVAL;6922 6923 if (kstrtoul(page, 10, &new))6924 return -EINVAL;6925 6926 if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)6927 return -EINVAL;6928 6929 if (new != PARITY_DISABLE_RMW &&6930 new != PARITY_ENABLE_RMW &&6931 new != PARITY_PREFER_RMW)6932 return -EINVAL;6933 6934 conf->rmw_level = new;6935 return len;6936}6937 6938static struct md_sysfs_entry6939raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,6940 raid5_show_rmw_level,6941 raid5_store_rmw_level);6942 6943static ssize_t6944raid5_show_stripe_size(struct mddev *mddev, char *page)6945{6946 struct r5conf *conf;6947 int ret = 0;6948 6949 spin_lock(&mddev->lock);6950 conf = mddev->private;6951 if (conf)6952 ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));6953 spin_unlock(&mddev->lock);6954 return ret;6955}6956 6957#if PAGE_SIZE != DEFAULT_STRIPE_SIZE6958static ssize_t6959raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)6960{6961 struct r5conf *conf;6962 unsigned long new;6963 int err;6964 int size;6965 6966 if (len >= PAGE_SIZE)6967 return -EINVAL;6968 if (kstrtoul(page, 10, &new))6969 return -EINVAL;6970 6971 /*6972 * The value should not be bigger than PAGE_SIZE. It requires to6973 * be multiple of DEFAULT_STRIPE_SIZE and the value should be power6974 * of two.6975 */6976 if (new % DEFAULT_STRIPE_SIZE != 0 ||6977 new > PAGE_SIZE || new == 0 ||6978 new != roundup_pow_of_two(new))6979 return -EINVAL;6980 6981 err = mddev_suspend_and_lock(mddev);6982 if (err)6983 return err;6984 6985 conf = mddev->private;6986 if (!conf) {6987 err = -ENODEV;6988 goto out_unlock;6989 }6990 6991 if (new == conf->stripe_size)6992 goto out_unlock;6993 6994 pr_debug("md/raid: change stripe_size from %lu to %lu\n",6995 conf->stripe_size, new);6996 6997 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||6998 mddev->reshape_position != MaxSector || mddev->sysfs_active) {6999 err = -EBUSY;7000 goto out_unlock;7001 }7002 7003 mutex_lock(&conf->cache_size_mutex);7004 size = conf->max_nr_stripes;7005 7006 shrink_stripes(conf);7007 7008 conf->stripe_size = new;7009 conf->stripe_shift = ilog2(new) - 9;7010 conf->stripe_sectors = new >> 9;7011 if (grow_stripes(conf, size)) {7012 pr_warn("md/raid:%s: couldn't allocate buffers\n",7013 mdname(mddev));7014 err = -ENOMEM;7015 }7016 mutex_unlock(&conf->cache_size_mutex);7017 7018out_unlock:7019 mddev_unlock_and_resume(mddev);7020 return err ?: len;7021}7022 7023static struct md_sysfs_entry7024raid5_stripe_size = __ATTR(stripe_size, 0644,7025 raid5_show_stripe_size,7026 raid5_store_stripe_size);7027#else7028static struct md_sysfs_entry7029raid5_stripe_size = __ATTR(stripe_size, 0444,7030 raid5_show_stripe_size,7031 NULL);7032#endif7033 7034static ssize_t7035raid5_show_preread_threshold(struct mddev *mddev, char *page)7036{7037 struct r5conf *conf;7038 int ret = 0;7039 spin_lock(&mddev->lock);7040 conf = mddev->private;7041 if (conf)7042 ret = sprintf(page, "%d\n", conf->bypass_threshold);7043 spin_unlock(&mddev->lock);7044 return ret;7045}7046 7047static ssize_t7048raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)7049{7050 struct r5conf *conf;7051 unsigned long new;7052 int err;7053 7054 if (len >= PAGE_SIZE)7055 return -EINVAL;7056 if (kstrtoul(page, 10, &new))7057 return -EINVAL;7058 7059 err = mddev_lock(mddev);7060 if (err)7061 return err;7062 conf = mddev->private;7063 if (!conf)7064 err = -ENODEV;7065 else if (new > conf->min_nr_stripes)7066 err = -EINVAL;7067 else7068 conf->bypass_threshold = new;7069 mddev_unlock(mddev);7070 return err ?: len;7071}7072 7073static struct md_sysfs_entry7074raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,7075 S_IRUGO | S_IWUSR,7076 raid5_show_preread_threshold,7077 raid5_store_preread_threshold);7078 7079static ssize_t7080raid5_show_skip_copy(struct mddev *mddev, char *page)7081{7082 struct r5conf *conf;7083 int ret = 0;7084 spin_lock(&mddev->lock);7085 conf = mddev->private;7086 if (conf)7087 ret = sprintf(page, "%d\n", conf->skip_copy);7088 spin_unlock(&mddev->lock);7089 return ret;7090}7091 7092static ssize_t7093raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)7094{7095 struct r5conf *conf;7096 unsigned long new;7097 int err;7098 7099 if (len >= PAGE_SIZE)7100 return -EINVAL;7101 if (kstrtoul(page, 10, &new))7102 return -EINVAL;7103 new = !!new;7104 7105 err = mddev_suspend_and_lock(mddev);7106 if (err)7107 return err;7108 conf = mddev->private;7109 if (!conf)7110 err = -ENODEV;7111 else if (new != conf->skip_copy) {7112 struct request_queue *q = mddev->gendisk->queue;7113 struct queue_limits lim = queue_limits_start_update(q);7114 7115 conf->skip_copy = new;7116 if (new)7117 lim.features |= BLK_FEAT_STABLE_WRITES;7118 else7119 lim.features &= ~BLK_FEAT_STABLE_WRITES;7120 err = queue_limits_commit_update(q, &lim);7121 }7122 mddev_unlock_and_resume(mddev);7123 return err ?: len;7124}7125 7126static struct md_sysfs_entry7127raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,7128 raid5_show_skip_copy,7129 raid5_store_skip_copy);7130 7131static ssize_t7132stripe_cache_active_show(struct mddev *mddev, char *page)7133{7134 struct r5conf *conf = mddev->private;7135 if (conf)7136 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));7137 else7138 return 0;7139}7140 7141static struct md_sysfs_entry7142raid5_stripecache_active = __ATTR_RO(stripe_cache_active);7143 7144static ssize_t7145raid5_show_group_thread_cnt(struct mddev *mddev, char *page)7146{7147 struct r5conf *conf;7148 int ret = 0;7149 spin_lock(&mddev->lock);7150 conf = mddev->private;7151 if (conf)7152 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);7153 spin_unlock(&mddev->lock);7154 return ret;7155}7156 7157static int alloc_thread_groups(struct r5conf *conf, int cnt,7158 int *group_cnt,7159 struct r5worker_group **worker_groups);7160static ssize_t7161raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)7162{7163 struct r5conf *conf;7164 unsigned int new;7165 int err;7166 struct r5worker_group *new_groups, *old_groups;7167 int group_cnt;7168 7169 if (len >= PAGE_SIZE)7170 return -EINVAL;7171 if (kstrtouint(page, 10, &new))7172 return -EINVAL;7173 /* 8192 should be big enough */7174 if (new > 8192)7175 return -EINVAL;7176 7177 err = mddev_suspend_and_lock(mddev);7178 if (err)7179 return err;7180 conf = mddev->private;7181 if (!conf)7182 err = -ENODEV;7183 else if (new != conf->worker_cnt_per_group) {7184 old_groups = conf->worker_groups;7185 if (old_groups)7186 flush_workqueue(raid5_wq);7187 7188 err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);7189 if (!err) {7190 spin_lock_irq(&conf->device_lock);7191 conf->group_cnt = group_cnt;7192 conf->worker_cnt_per_group = new;7193 conf->worker_groups = new_groups;7194 spin_unlock_irq(&conf->device_lock);7195 7196 if (old_groups)7197 kfree(old_groups[0].workers);7198 kfree(old_groups);7199 }7200 }7201 mddev_unlock_and_resume(mddev);7202 7203 return err ?: len;7204}7205 7206static struct md_sysfs_entry7207raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,7208 raid5_show_group_thread_cnt,7209 raid5_store_group_thread_cnt);7210 7211static struct attribute *raid5_attrs[] = {7212 &raid5_stripecache_size.attr,7213 &raid5_stripecache_active.attr,7214 &raid5_preread_bypass_threshold.attr,7215 &raid5_group_thread_cnt.attr,7216 &raid5_skip_copy.attr,7217 &raid5_rmw_level.attr,7218 &raid5_stripe_size.attr,7219 &r5c_journal_mode.attr,7220 &ppl_write_hint.attr,7221 NULL,7222};7223static const struct attribute_group raid5_attrs_group = {7224 .name = NULL,7225 .attrs = raid5_attrs,7226};7227 7228static int alloc_thread_groups(struct r5conf *conf, int cnt, int *group_cnt,7229 struct r5worker_group **worker_groups)7230{7231 int i, j, k;7232 ssize_t size;7233 struct r5worker *workers;7234 7235 if (cnt == 0) {7236 *group_cnt = 0;7237 *worker_groups = NULL;7238 return 0;7239 }7240 *group_cnt = num_possible_nodes();7241 size = sizeof(struct r5worker) * cnt;7242 workers = kcalloc(size, *group_cnt, GFP_NOIO);7243 *worker_groups = kcalloc(*group_cnt, sizeof(struct r5worker_group),7244 GFP_NOIO);7245 if (!*worker_groups || !workers) {7246 kfree(workers);7247 kfree(*worker_groups);7248 return -ENOMEM;7249 }7250 7251 for (i = 0; i < *group_cnt; i++) {7252 struct r5worker_group *group;7253 7254 group = &(*worker_groups)[i];7255 INIT_LIST_HEAD(&group->handle_list);7256 INIT_LIST_HEAD(&group->loprio_list);7257 group->conf = conf;7258 group->workers = workers + i * cnt;7259 7260 for (j = 0; j < cnt; j++) {7261 struct r5worker *worker = group->workers + j;7262 worker->group = group;7263 INIT_WORK(&worker->work, raid5_do_work);7264 7265 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)7266 INIT_LIST_HEAD(worker->temp_inactive_list + k);7267 }7268 }7269 7270 return 0;7271}7272 7273static void free_thread_groups(struct r5conf *conf)7274{7275 if (conf->worker_groups)7276 kfree(conf->worker_groups[0].workers);7277 kfree(conf->worker_groups);7278 conf->worker_groups = NULL;7279}7280 7281static sector_t7282raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)7283{7284 struct r5conf *conf = mddev->private;7285 7286 if (!sectors)7287 sectors = mddev->dev_sectors;7288 if (!raid_disks)7289 /* size is defined by the smallest of previous and new size */7290 raid_disks = min(conf->raid_disks, conf->previous_raid_disks);7291 7292 sectors &= ~((sector_t)conf->chunk_sectors - 1);7293 sectors &= ~((sector_t)conf->prev_chunk_sectors - 1);7294 return sectors * (raid_disks - conf->max_degraded);7295}7296 7297static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)7298{7299 safe_put_page(percpu->spare_page);7300 percpu->spare_page = NULL;7301 kvfree(percpu->scribble);7302 percpu->scribble = NULL;7303}7304 7305static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)7306{7307 if (conf->level == 6 && !percpu->spare_page) {7308 percpu->spare_page = alloc_page(GFP_KERNEL);7309 if (!percpu->spare_page)7310 return -ENOMEM;7311 }7312 7313 if (scribble_alloc(percpu,7314 max(conf->raid_disks,7315 conf->previous_raid_disks),7316 max(conf->chunk_sectors,7317 conf->prev_chunk_sectors)7318 / RAID5_STRIPE_SECTORS(conf))) {7319 free_scratch_buffer(conf, percpu);7320 return -ENOMEM;7321 }7322 7323 local_lock_init(&percpu->lock);7324 return 0;7325}7326 7327static int raid456_cpu_dead(unsigned int cpu, struct hlist_node *node)7328{7329 struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);7330 7331 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));7332 return 0;7333}7334 7335static void raid5_free_percpu(struct r5conf *conf)7336{7337 if (!conf->percpu)7338 return;7339 7340 cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);7341 free_percpu(conf->percpu);7342}7343 7344static void free_conf(struct r5conf *conf)7345{7346 int i;7347 7348 log_exit(conf);7349 7350 shrinker_free(conf->shrinker);7351 free_thread_groups(conf);7352 shrink_stripes(conf);7353 raid5_free_percpu(conf);7354 for (i = 0; i < conf->pool_size; i++)7355 if (conf->disks[i].extra_page)7356 put_page(conf->disks[i].extra_page);7357 kfree(conf->disks);7358 bioset_exit(&conf->bio_split);7359 kfree(conf->stripe_hashtbl);7360 kfree(conf->pending_data);7361 kfree(conf);7362}7363 7364static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)7365{7366 struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);7367 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);7368 7369 if (alloc_scratch_buffer(conf, percpu)) {7370 pr_warn("%s: failed memory allocation for cpu%u\n",7371 __func__, cpu);7372 return -ENOMEM;7373 }7374 return 0;7375}7376 7377static int raid5_alloc_percpu(struct r5conf *conf)7378{7379 int err = 0;7380 7381 conf->percpu = alloc_percpu(struct raid5_percpu);7382 if (!conf->percpu)7383 return -ENOMEM;7384 7385 err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);7386 if (!err) {7387 conf->scribble_disks = max(conf->raid_disks,7388 conf->previous_raid_disks);7389 conf->scribble_sectors = max(conf->chunk_sectors,7390 conf->prev_chunk_sectors);7391 }7392 return err;7393}7394 7395static unsigned long raid5_cache_scan(struct shrinker *shrink,7396 struct shrink_control *sc)7397{7398 struct r5conf *conf = shrink->private_data;7399 unsigned long ret = SHRINK_STOP;7400 7401 if (mutex_trylock(&conf->cache_size_mutex)) {7402 ret= 0;7403 while (ret < sc->nr_to_scan &&7404 conf->max_nr_stripes > conf->min_nr_stripes) {7405 if (drop_one_stripe(conf) == 0) {7406 ret = SHRINK_STOP;7407 break;7408 }7409 ret++;7410 }7411 mutex_unlock(&conf->cache_size_mutex);7412 }7413 return ret;7414}7415 7416static unsigned long raid5_cache_count(struct shrinker *shrink,7417 struct shrink_control *sc)7418{7419 struct r5conf *conf = shrink->private_data;7420 int max_stripes = READ_ONCE(conf->max_nr_stripes);7421 int min_stripes = READ_ONCE(conf->min_nr_stripes);7422 7423 if (max_stripes < min_stripes)7424 /* unlikely, but not impossible */7425 return 0;7426 return max_stripes - min_stripes;7427}7428 7429static struct r5conf *setup_conf(struct mddev *mddev)7430{7431 struct r5conf *conf;7432 int raid_disk, memory, max_disks;7433 struct md_rdev *rdev;7434 struct disk_info *disk;7435 char pers_name[6];7436 int i;7437 int group_cnt;7438 struct r5worker_group *new_group;7439 int ret = -ENOMEM;7440 7441 if (mddev->new_level != 57442 && mddev->new_level != 47443 && mddev->new_level != 6) {7444 pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n",7445 mdname(mddev), mddev->new_level);7446 return ERR_PTR(-EIO);7447 }7448 if ((mddev->new_level == 57449 && !algorithm_valid_raid5(mddev->new_layout)) ||7450 (mddev->new_level == 67451 && !algorithm_valid_raid6(mddev->new_layout))) {7452 pr_warn("md/raid:%s: layout %d not supported\n",7453 mdname(mddev), mddev->new_layout);7454 return ERR_PTR(-EIO);7455 }7456 if (mddev->new_level == 6 && mddev->raid_disks < 4) {7457 pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",7458 mdname(mddev), mddev->raid_disks);7459 return ERR_PTR(-EINVAL);7460 }7461 7462 if (!mddev->new_chunk_sectors ||7463 (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||7464 !is_power_of_2(mddev->new_chunk_sectors)) {7465 pr_warn("md/raid:%s: invalid chunk size %d\n",7466 mdname(mddev), mddev->new_chunk_sectors << 9);7467 return ERR_PTR(-EINVAL);7468 }7469 7470 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);7471 if (conf == NULL)7472 goto abort;7473 7474#if PAGE_SIZE != DEFAULT_STRIPE_SIZE7475 conf->stripe_size = DEFAULT_STRIPE_SIZE;7476 conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;7477 conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;7478#endif7479 INIT_LIST_HEAD(&conf->free_list);7480 INIT_LIST_HEAD(&conf->pending_list);7481 conf->pending_data = kcalloc(PENDING_IO_MAX,7482 sizeof(struct r5pending_data),7483 GFP_KERNEL);7484 if (!conf->pending_data)7485 goto abort;7486 for (i = 0; i < PENDING_IO_MAX; i++)7487 list_add(&conf->pending_data[i].sibling, &conf->free_list);7488 /* Don't enable multi-threading by default*/7489 if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {7490 conf->group_cnt = group_cnt;7491 conf->worker_cnt_per_group = 0;7492 conf->worker_groups = new_group;7493 } else7494 goto abort;7495 spin_lock_init(&conf->device_lock);7496 seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);7497 mutex_init(&conf->cache_size_mutex);7498 7499 init_waitqueue_head(&conf->wait_for_quiescent);7500 init_waitqueue_head(&conf->wait_for_stripe);7501 init_waitqueue_head(&conf->wait_for_reshape);7502 INIT_LIST_HEAD(&conf->handle_list);7503 INIT_LIST_HEAD(&conf->loprio_list);7504 INIT_LIST_HEAD(&conf->hold_list);7505 INIT_LIST_HEAD(&conf->delayed_list);7506 INIT_LIST_HEAD(&conf->bitmap_list);7507 init_llist_head(&conf->released_stripes);7508 atomic_set(&conf->active_stripes, 0);7509 atomic_set(&conf->preread_active_stripes, 0);7510 atomic_set(&conf->active_aligned_reads, 0);7511 spin_lock_init(&conf->pending_bios_lock);7512 conf->batch_bio_dispatch = true;7513 rdev_for_each(rdev, mddev) {7514 if (test_bit(Journal, &rdev->flags))7515 continue;7516 if (bdev_nonrot(rdev->bdev)) {7517 conf->batch_bio_dispatch = false;7518 break;7519 }7520 }7521 7522 conf->bypass_threshold = BYPASS_THRESHOLD;7523 conf->recovery_disabled = mddev->recovery_disabled - 1;7524 7525 conf->raid_disks = mddev->raid_disks;7526 if (mddev->reshape_position == MaxSector)7527 conf->previous_raid_disks = mddev->raid_disks;7528 else7529 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;7530 max_disks = max(conf->raid_disks, conf->previous_raid_disks);7531 7532 conf->disks = kcalloc(max_disks, sizeof(struct disk_info),7533 GFP_KERNEL);7534 7535 if (!conf->disks)7536 goto abort;7537 7538 for (i = 0; i < max_disks; i++) {7539 conf->disks[i].extra_page = alloc_page(GFP_KERNEL);7540 if (!conf->disks[i].extra_page)7541 goto abort;7542 }7543 7544 ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);7545 if (ret)7546 goto abort;7547 conf->mddev = mddev;7548 7549 ret = -ENOMEM;7550 conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);7551 if (!conf->stripe_hashtbl)7552 goto abort;7553 7554 /* We init hash_locks[0] separately to that it can be used7555 * as the reference lock in the spin_lock_nest_lock() call7556 * in lock_all_device_hash_locks_irq in order to convince7557 * lockdep that we know what we are doing.7558 */7559 spin_lock_init(conf->hash_locks);7560 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)7561 spin_lock_init(conf->hash_locks + i);7562 7563 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)7564 INIT_LIST_HEAD(conf->inactive_list + i);7565 7566 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)7567 INIT_LIST_HEAD(conf->temp_inactive_list + i);7568 7569 atomic_set(&conf->r5c_cached_full_stripes, 0);7570 INIT_LIST_HEAD(&conf->r5c_full_stripe_list);7571 atomic_set(&conf->r5c_cached_partial_stripes, 0);7572 INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);7573 atomic_set(&conf->r5c_flushing_full_stripes, 0);7574 atomic_set(&conf->r5c_flushing_partial_stripes, 0);7575 7576 conf->level = mddev->new_level;7577 conf->chunk_sectors = mddev->new_chunk_sectors;7578 ret = raid5_alloc_percpu(conf);7579 if (ret)7580 goto abort;7581 7582 pr_debug("raid456: run(%s) called.\n", mdname(mddev));7583 7584 ret = -EIO;7585 rdev_for_each(rdev, mddev) {7586 raid_disk = rdev->raid_disk;7587 if (raid_disk >= max_disks7588 || raid_disk < 0 || test_bit(Journal, &rdev->flags))7589 continue;7590 disk = conf->disks + raid_disk;7591 7592 if (test_bit(Replacement, &rdev->flags)) {7593 if (disk->replacement)7594 goto abort;7595 disk->replacement = rdev;7596 } else {7597 if (disk->rdev)7598 goto abort;7599 disk->rdev = rdev;7600 }7601 7602 if (test_bit(In_sync, &rdev->flags)) {7603 pr_info("md/raid:%s: device %pg operational as raid disk %d\n",7604 mdname(mddev), rdev->bdev, raid_disk);7605 } else if (rdev->saved_raid_disk != raid_disk)7606 /* Cannot rely on bitmap to complete recovery */7607 conf->fullsync = 1;7608 }7609 7610 conf->level = mddev->new_level;7611 if (conf->level == 6) {7612 conf->max_degraded = 2;7613 if (raid6_call.xor_syndrome)7614 conf->rmw_level = PARITY_ENABLE_RMW;7615 else7616 conf->rmw_level = PARITY_DISABLE_RMW;7617 } else {7618 conf->max_degraded = 1;7619 conf->rmw_level = PARITY_ENABLE_RMW;7620 }7621 conf->algorithm = mddev->new_layout;7622 conf->reshape_progress = mddev->reshape_position;7623 if (conf->reshape_progress != MaxSector) {7624 conf->prev_chunk_sectors = mddev->chunk_sectors;7625 conf->prev_algo = mddev->layout;7626 } else {7627 conf->prev_chunk_sectors = conf->chunk_sectors;7628 conf->prev_algo = conf->algorithm;7629 }7630 7631 conf->min_nr_stripes = NR_STRIPES;7632 if (mddev->reshape_position != MaxSector) {7633 int stripes = max_t(int,7634 ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,7635 ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);7636 conf->min_nr_stripes = max(NR_STRIPES, stripes);7637 if (conf->min_nr_stripes != NR_STRIPES)7638 pr_info("md/raid:%s: force stripe size %d for reshape\n",7639 mdname(mddev), conf->min_nr_stripes);7640 }7641 memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +7642 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;7643 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);7644 if (grow_stripes(conf, conf->min_nr_stripes)) {7645 pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n",7646 mdname(mddev), memory);7647 ret = -ENOMEM;7648 goto abort;7649 } else7650 pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory);7651 /*7652 * Losing a stripe head costs more than the time to refill it,7653 * it reduces the queue depth and so can hurt throughput.7654 * So set it rather large, scaled by number of devices.7655 */7656 conf->shrinker = shrinker_alloc(0, "md-raid5:%s", mdname(mddev));7657 if (!conf->shrinker) {7658 ret = -ENOMEM;7659 pr_warn("md/raid:%s: couldn't allocate shrinker.\n",7660 mdname(mddev));7661 goto abort;7662 }7663 7664 conf->shrinker->seeks = DEFAULT_SEEKS * conf->raid_disks * 4;7665 conf->shrinker->scan_objects = raid5_cache_scan;7666 conf->shrinker->count_objects = raid5_cache_count;7667 conf->shrinker->batch = 128;7668 conf->shrinker->private_data = conf;7669 7670 shrinker_register(conf->shrinker);7671 7672 sprintf(pers_name, "raid%d", mddev->new_level);7673 rcu_assign_pointer(conf->thread,7674 md_register_thread(raid5d, mddev, pers_name));7675 if (!conf->thread) {7676 pr_warn("md/raid:%s: couldn't allocate thread.\n",7677 mdname(mddev));7678 ret = -ENOMEM;7679 goto abort;7680 }7681 7682 return conf;7683 7684 abort:7685 if (conf)7686 free_conf(conf);7687 return ERR_PTR(ret);7688}7689 7690static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)7691{7692 switch (algo) {7693 case ALGORITHM_PARITY_0:7694 if (raid_disk < max_degraded)7695 return 1;7696 break;7697 case ALGORITHM_PARITY_N:7698 if (raid_disk >= raid_disks - max_degraded)7699 return 1;7700 break;7701 case ALGORITHM_PARITY_0_6:7702 if (raid_disk == 0 ||7703 raid_disk == raid_disks - 1)7704 return 1;7705 break;7706 case ALGORITHM_LEFT_ASYMMETRIC_6:7707 case ALGORITHM_RIGHT_ASYMMETRIC_6:7708 case ALGORITHM_LEFT_SYMMETRIC_6:7709 case ALGORITHM_RIGHT_SYMMETRIC_6:7710 if (raid_disk == raid_disks - 1)7711 return 1;7712 }7713 return 0;7714}7715 7716static int raid5_set_limits(struct mddev *mddev)7717{7718 struct r5conf *conf = mddev->private;7719 struct queue_limits lim;7720 int data_disks, stripe;7721 struct md_rdev *rdev;7722 7723 /*7724 * The read-ahead size must cover two whole stripes, which is7725 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices.7726 */7727 data_disks = conf->previous_raid_disks - conf->max_degraded;7728 7729 /*7730 * We can only discard a whole stripe. It doesn't make sense to7731 * discard data disk but write parity disk7732 */7733 stripe = roundup_pow_of_two(data_disks * (mddev->chunk_sectors << 9));7734 7735 md_init_stacking_limits(&lim);7736 lim.io_min = mddev->chunk_sectors << 9;7737 lim.io_opt = lim.io_min * (conf->raid_disks - conf->max_degraded);7738 lim.features |= BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE;7739 lim.discard_granularity = stripe;7740 lim.max_write_zeroes_sectors = 0;7741 mddev_stack_rdev_limits(mddev, &lim, 0);7742 rdev_for_each(rdev, mddev)7743 queue_limits_stack_bdev(&lim, rdev->bdev, rdev->new_data_offset,7744 mddev->gendisk->disk_name);7745 7746 /*7747 * Zeroing is required for discard, otherwise data could be lost.7748 *7749 * Consider a scenario: discard a stripe (the stripe could be7750 * inconsistent if discard_zeroes_data is 0); write one disk of the7751 * stripe (the stripe could be inconsistent again depending on which7752 * disks are used to calculate parity); the disk is broken; The stripe7753 * data of this disk is lost.7754 *7755 * We only allow DISCARD if the sysadmin has confirmed that only safe7756 * devices are in use by setting a module parameter. A better idea7757 * might be to turn DISCARD into WRITE_ZEROES requests, as that is7758 * required to be safe.7759 */7760 if (!devices_handle_discard_safely ||7761 lim.max_discard_sectors < (stripe >> 9) ||7762 lim.discard_granularity < stripe)7763 lim.max_hw_discard_sectors = 0;7764 7765 /*7766 * Requests require having a bitmap for each stripe.7767 * Limit the max sectors based on this.7768 */7769 lim.max_hw_sectors = RAID5_MAX_REQ_STRIPES << RAID5_STRIPE_SHIFT(conf);7770 7771 /* No restrictions on the number of segments in the request */7772 lim.max_segments = USHRT_MAX;7773 7774 return queue_limits_set(mddev->gendisk->queue, &lim);7775}7776 7777static int raid5_run(struct mddev *mddev)7778{7779 struct r5conf *conf;7780 int dirty_parity_disks = 0;7781 struct md_rdev *rdev;7782 struct md_rdev *journal_dev = NULL;7783 sector_t reshape_offset = 0;7784 int i;7785 long long min_offset_diff = 0;7786 int first = 1;7787 int ret = -EIO;7788 7789 if (mddev->recovery_cp != MaxSector)7790 pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",7791 mdname(mddev));7792 7793 rdev_for_each(rdev, mddev) {7794 long long diff;7795 7796 if (test_bit(Journal, &rdev->flags)) {7797 journal_dev = rdev;7798 continue;7799 }7800 if (rdev->raid_disk < 0)7801 continue;7802 diff = (rdev->new_data_offset - rdev->data_offset);7803 if (first) {7804 min_offset_diff = diff;7805 first = 0;7806 } else if (mddev->reshape_backwards &&7807 diff < min_offset_diff)7808 min_offset_diff = diff;7809 else if (!mddev->reshape_backwards &&7810 diff > min_offset_diff)7811 min_offset_diff = diff;7812 }7813 7814 if ((test_bit(MD_HAS_JOURNAL, &mddev->flags) || journal_dev) &&7815 (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {7816 pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",7817 mdname(mddev));7818 return -EINVAL;7819 }7820 7821 if (mddev->reshape_position != MaxSector) {7822 /* Check that we can continue the reshape.7823 * Difficulties arise if the stripe we would write to7824 * next is at or after the stripe we would read from next.7825 * For a reshape that changes the number of devices, this7826 * is only possible for a very short time, and mdadm makes7827 * sure that time appears to have past before assembling7828 * the array. So we fail if that time hasn't passed.7829 * For a reshape that keeps the number of devices the same7830 * mdadm must be monitoring the reshape can keeping the7831 * critical areas read-only and backed up. It will start7832 * the array in read-only mode, so we check for that.7833 */7834 sector_t here_new, here_old;7835 int old_disks;7836 int max_degraded = (mddev->level == 6 ? 2 : 1);7837 int chunk_sectors;7838 int new_data_disks;7839 7840 if (journal_dev) {7841 pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",7842 mdname(mddev));7843 return -EINVAL;7844 }7845 7846 if (mddev->new_level != mddev->level) {7847 pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",7848 mdname(mddev));7849 return -EINVAL;7850 }7851 old_disks = mddev->raid_disks - mddev->delta_disks;7852 /* reshape_position must be on a new-stripe boundary, and one7853 * further up in new geometry must map after here in old7854 * geometry.7855 * If the chunk sizes are different, then as we perform reshape7856 * in units of the largest of the two, reshape_position needs7857 * be a multiple of the largest chunk size times new data disks.7858 */7859 here_new = mddev->reshape_position;7860 chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);7861 new_data_disks = mddev->raid_disks - max_degraded;7862 if (sector_div(here_new, chunk_sectors * new_data_disks)) {7863 pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",7864 mdname(mddev));7865 return -EINVAL;7866 }7867 reshape_offset = here_new * chunk_sectors;7868 /* here_new is the stripe we will write to */7869 here_old = mddev->reshape_position;7870 sector_div(here_old, chunk_sectors * (old_disks-max_degraded));7871 /* here_old is the first stripe that we might need to read7872 * from */7873 if (mddev->delta_disks == 0) {7874 /* We cannot be sure it is safe to start an in-place7875 * reshape. It is only safe if user-space is monitoring7876 * and taking constant backups.7877 * mdadm always starts a situation like this in7878 * readonly mode so it can take control before7879 * allowing any writes. So just check for that.7880 */7881 if (abs(min_offset_diff) >= mddev->chunk_sectors &&7882 abs(min_offset_diff) >= mddev->new_chunk_sectors)7883 /* not really in-place - so OK */;7884 else if (mddev->ro == 0) {7885 pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",7886 mdname(mddev));7887 return -EINVAL;7888 }7889 } else if (mddev->reshape_backwards7890 ? (here_new * chunk_sectors + min_offset_diff <=7891 here_old * chunk_sectors)7892 : (here_new * chunk_sectors >=7893 here_old * chunk_sectors + (-min_offset_diff))) {7894 /* Reading from the same stripe as writing to - bad */7895 pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",7896 mdname(mddev));7897 return -EINVAL;7898 }7899 pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));7900 /* OK, we should be able to continue; */7901 } else {7902 BUG_ON(mddev->level != mddev->new_level);7903 BUG_ON(mddev->layout != mddev->new_layout);7904 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);7905 BUG_ON(mddev->delta_disks != 0);7906 }7907 7908 if (test_bit(MD_HAS_JOURNAL, &mddev->flags) &&7909 test_bit(MD_HAS_PPL, &mddev->flags)) {7910 pr_warn("md/raid:%s: using journal device and PPL not allowed - disabling PPL\n",7911 mdname(mddev));7912 clear_bit(MD_HAS_PPL, &mddev->flags);7913 clear_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags);7914 }7915 7916 if (mddev->private == NULL)7917 conf = setup_conf(mddev);7918 else7919 conf = mddev->private;7920 7921 if (IS_ERR(conf))7922 return PTR_ERR(conf);7923 7924 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {7925 if (!journal_dev) {7926 pr_warn("md/raid:%s: journal disk is missing, force array readonly\n",7927 mdname(mddev));7928 mddev->ro = 1;7929 set_disk_ro(mddev->gendisk, 1);7930 } else if (mddev->recovery_cp == MaxSector)7931 set_bit(MD_JOURNAL_CLEAN, &mddev->flags);7932 }7933 7934 conf->min_offset_diff = min_offset_diff;7935 rcu_assign_pointer(mddev->thread, conf->thread);7936 rcu_assign_pointer(conf->thread, NULL);7937 mddev->private = conf;7938 7939 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;7940 i++) {7941 rdev = conf->disks[i].rdev;7942 if (!rdev)7943 continue;7944 if (conf->disks[i].replacement &&7945 conf->reshape_progress != MaxSector) {7946 /* replacements and reshape simply do not mix. */7947 pr_warn("md: cannot handle concurrent replacement and reshape.\n");7948 goto abort;7949 }7950 if (test_bit(In_sync, &rdev->flags))7951 continue;7952 /* This disc is not fully in-sync. However if it7953 * just stored parity (beyond the recovery_offset),7954 * when we don't need to be concerned about the7955 * array being dirty.7956 * When reshape goes 'backwards', we never have7957 * partially completed devices, so we only need7958 * to worry about reshape going forwards.7959 */7960 /* Hack because v0.91 doesn't store recovery_offset properly. */7961 if (mddev->major_version == 0 &&7962 mddev->minor_version > 90)7963 rdev->recovery_offset = reshape_offset;7964 7965 if (rdev->recovery_offset < reshape_offset) {7966 /* We need to check old and new layout */7967 if (!only_parity(rdev->raid_disk,7968 conf->algorithm,7969 conf->raid_disks,7970 conf->max_degraded))7971 continue;7972 }7973 if (!only_parity(rdev->raid_disk,7974 conf->prev_algo,7975 conf->previous_raid_disks,7976 conf->max_degraded))7977 continue;7978 dirty_parity_disks++;7979 }7980 7981 /*7982 * 0 for a fully functional array, 1 or 2 for a degraded array.7983 */7984 mddev->degraded = raid5_calc_degraded(conf);7985 7986 if (has_failed(conf)) {7987 pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n",7988 mdname(mddev), mddev->degraded, conf->raid_disks);7989 goto abort;7990 }7991 7992 /* device size must be a multiple of chunk size */7993 mddev->dev_sectors &= ~((sector_t)mddev->chunk_sectors - 1);7994 mddev->resync_max_sectors = mddev->dev_sectors;7995 7996 if (mddev->degraded > dirty_parity_disks &&7997 mddev->recovery_cp != MaxSector) {7998 if (test_bit(MD_HAS_PPL, &mddev->flags))7999 pr_crit("md/raid:%s: starting dirty degraded array with PPL.\n",8000 mdname(mddev));8001 else if (mddev->ok_start_degraded)8002 pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",8003 mdname(mddev));8004 else {8005 pr_crit("md/raid:%s: cannot start dirty degraded array.\n",8006 mdname(mddev));8007 goto abort;8008 }8009 }8010 8011 pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n",8012 mdname(mddev), conf->level,8013 mddev->raid_disks-mddev->degraded, mddev->raid_disks,8014 mddev->new_layout);8015 8016 print_raid5_conf(conf);8017 8018 if (conf->reshape_progress != MaxSector) {8019 conf->reshape_safe = conf->reshape_progress;8020 atomic_set(&conf->reshape_stripes, 0);8021 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);8022 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);8023 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);8024 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);8025 }8026 8027 /* Ok, everything is just fine now */8028 if (mddev->to_remove == &raid5_attrs_group)8029 mddev->to_remove = NULL;8030 else if (mddev->kobj.sd &&8031 sysfs_create_group(&mddev->kobj, &raid5_attrs_group))8032 pr_warn("raid5: failed to create sysfs attributes for %s\n",8033 mdname(mddev));8034 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));8035 8036 if (!mddev_is_dm(mddev)) {8037 ret = raid5_set_limits(mddev);8038 if (ret)8039 goto abort;8040 }8041 8042 if (log_init(conf, journal_dev, raid5_has_ppl(conf)))8043 goto abort;8044 8045 return 0;8046abort:8047 md_unregister_thread(mddev, &mddev->thread);8048 print_raid5_conf(conf);8049 free_conf(conf);8050 mddev->private = NULL;8051 pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));8052 return ret;8053}8054 8055static void raid5_free(struct mddev *mddev, void *priv)8056{8057 struct r5conf *conf = priv;8058 8059 free_conf(conf);8060 mddev->to_remove = &raid5_attrs_group;8061}8062 8063static void raid5_status(struct seq_file *seq, struct mddev *mddev)8064{8065 struct r5conf *conf = mddev->private;8066 int i;8067 8068 lockdep_assert_held(&mddev->lock);8069 8070 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,8071 conf->chunk_sectors / 2, mddev->layout);8072 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);8073 for (i = 0; i < conf->raid_disks; i++) {8074 struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);8075 8076 seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");8077 }8078 seq_printf (seq, "]");8079}8080 8081static void print_raid5_conf(struct r5conf *conf)8082{8083 struct md_rdev *rdev;8084 int i;8085 8086 pr_debug("RAID conf printout:\n");8087 if (!conf) {8088 pr_debug("(conf==NULL)\n");8089 return;8090 }8091 pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,8092 conf->raid_disks,8093 conf->raid_disks - conf->mddev->degraded);8094 8095 for (i = 0; i < conf->raid_disks; i++) {8096 rdev = conf->disks[i].rdev;8097 if (rdev)8098 pr_debug(" disk %d, o:%d, dev:%pg\n",8099 i, !test_bit(Faulty, &rdev->flags),8100 rdev->bdev);8101 }8102}8103 8104static int raid5_spare_active(struct mddev *mddev)8105{8106 int i;8107 struct r5conf *conf = mddev->private;8108 struct md_rdev *rdev, *replacement;8109 int count = 0;8110 unsigned long flags;8111 8112 for (i = 0; i < conf->raid_disks; i++) {8113 rdev = conf->disks[i].rdev;8114 replacement = conf->disks[i].replacement;8115 if (replacement8116 && replacement->recovery_offset == MaxSector8117 && !test_bit(Faulty, &replacement->flags)8118 && !test_and_set_bit(In_sync, &replacement->flags)) {8119 /* Replacement has just become active. */8120 if (!rdev8121 || !test_and_clear_bit(In_sync, &rdev->flags))8122 count++;8123 if (rdev) {8124 /* Replaced device not technically faulty,8125 * but we need to be sure it gets removed8126 * and never re-added.8127 */8128 set_bit(Faulty, &rdev->flags);8129 sysfs_notify_dirent_safe(8130 rdev->sysfs_state);8131 }8132 sysfs_notify_dirent_safe(replacement->sysfs_state);8133 } else if (rdev8134 && rdev->recovery_offset == MaxSector8135 && !test_bit(Faulty, &rdev->flags)8136 && !test_and_set_bit(In_sync, &rdev->flags)) {8137 count++;8138 sysfs_notify_dirent_safe(rdev->sysfs_state);8139 }8140 }8141 spin_lock_irqsave(&conf->device_lock, flags);8142 mddev->degraded = raid5_calc_degraded(conf);8143 spin_unlock_irqrestore(&conf->device_lock, flags);8144 print_raid5_conf(conf);8145 return count;8146}8147 8148static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)8149{8150 struct r5conf *conf = mddev->private;8151 int err = 0;8152 int number = rdev->raid_disk;8153 struct md_rdev **rdevp;8154 struct disk_info *p;8155 struct md_rdev *tmp;8156 8157 print_raid5_conf(conf);8158 if (test_bit(Journal, &rdev->flags) && conf->log) {8159 /*8160 * we can't wait pending write here, as this is called in8161 * raid5d, wait will deadlock.8162 * neilb: there is no locking about new writes here,8163 * so this cannot be safe.8164 */8165 if (atomic_read(&conf->active_stripes) ||8166 atomic_read(&conf->r5c_cached_full_stripes) ||8167 atomic_read(&conf->r5c_cached_partial_stripes)) {8168 return -EBUSY;8169 }8170 log_exit(conf);8171 return 0;8172 }8173 if (unlikely(number >= conf->pool_size))8174 return 0;8175 p = conf->disks + number;8176 if (rdev == p->rdev)8177 rdevp = &p->rdev;8178 else if (rdev == p->replacement)8179 rdevp = &p->replacement;8180 else8181 return 0;8182 8183 if (number >= conf->raid_disks &&8184 conf->reshape_progress == MaxSector)8185 clear_bit(In_sync, &rdev->flags);8186 8187 if (test_bit(In_sync, &rdev->flags) ||8188 atomic_read(&rdev->nr_pending)) {8189 err = -EBUSY;8190 goto abort;8191 }8192 /* Only remove non-faulty devices if recovery8193 * isn't possible.8194 */8195 if (!test_bit(Faulty, &rdev->flags) &&8196 mddev->recovery_disabled != conf->recovery_disabled &&8197 !has_failed(conf) &&8198 (!p->replacement || p->replacement == rdev) &&8199 number < conf->raid_disks) {8200 err = -EBUSY;8201 goto abort;8202 }8203 WRITE_ONCE(*rdevp, NULL);8204 if (!err) {8205 err = log_modify(conf, rdev, false);8206 if (err)8207 goto abort;8208 }8209 8210 tmp = p->replacement;8211 if (tmp) {8212 /* We must have just cleared 'rdev' */8213 WRITE_ONCE(p->rdev, tmp);8214 clear_bit(Replacement, &tmp->flags);8215 WRITE_ONCE(p->replacement, NULL);8216 8217 if (!err)8218 err = log_modify(conf, tmp, true);8219 }8220 8221 clear_bit(WantReplacement, &rdev->flags);8222abort:8223 8224 print_raid5_conf(conf);8225 return err;8226}8227 8228static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)8229{8230 struct r5conf *conf = mddev->private;8231 int ret, err = -EEXIST;8232 int disk;8233 struct disk_info *p;8234 struct md_rdev *tmp;8235 int first = 0;8236 int last = conf->raid_disks - 1;8237 8238 if (test_bit(Journal, &rdev->flags)) {8239 if (conf->log)8240 return -EBUSY;8241 8242 rdev->raid_disk = 0;8243 /*8244 * The array is in readonly mode if journal is missing, so no8245 * write requests running. We should be safe8246 */8247 ret = log_init(conf, rdev, false);8248 if (ret)8249 return ret;8250 8251 ret = r5l_start(conf->log);8252 if (ret)8253 return ret;8254 8255 return 0;8256 }8257 if (mddev->recovery_disabled == conf->recovery_disabled)8258 return -EBUSY;8259 8260 if (rdev->saved_raid_disk < 0 && has_failed(conf))8261 /* no point adding a device */8262 return -EINVAL;8263 8264 if (rdev->raid_disk >= 0)8265 first = last = rdev->raid_disk;8266 8267 /*8268 * find the disk ... but prefer rdev->saved_raid_disk8269 * if possible.8270 */8271 if (rdev->saved_raid_disk >= first &&8272 rdev->saved_raid_disk <= last &&8273 conf->disks[rdev->saved_raid_disk].rdev == NULL)8274 first = rdev->saved_raid_disk;8275 8276 for (disk = first; disk <= last; disk++) {8277 p = conf->disks + disk;8278 if (p->rdev == NULL) {8279 clear_bit(In_sync, &rdev->flags);8280 rdev->raid_disk = disk;8281 if (rdev->saved_raid_disk != disk)8282 conf->fullsync = 1;8283 WRITE_ONCE(p->rdev, rdev);8284 8285 err = log_modify(conf, rdev, true);8286 8287 goto out;8288 }8289 }8290 for (disk = first; disk <= last; disk++) {8291 p = conf->disks + disk;8292 tmp = p->rdev;8293 if (test_bit(WantReplacement, &tmp->flags) &&8294 mddev->reshape_position == MaxSector &&8295 p->replacement == NULL) {8296 clear_bit(In_sync, &rdev->flags);8297 set_bit(Replacement, &rdev->flags);8298 rdev->raid_disk = disk;8299 err = 0;8300 conf->fullsync = 1;8301 WRITE_ONCE(p->replacement, rdev);8302 break;8303 }8304 }8305out:8306 print_raid5_conf(conf);8307 return err;8308}8309 8310static int raid5_resize(struct mddev *mddev, sector_t sectors)8311{8312 /* no resync is happening, and there is enough space8313 * on all devices, so we can resize.8314 * We need to make sure resync covers any new space.8315 * If the array is shrinking we should possibly wait until8316 * any io in the removed space completes, but it hardly seems8317 * worth it.8318 */8319 sector_t newsize;8320 struct r5conf *conf = mddev->private;8321 int ret;8322 8323 if (raid5_has_log(conf) || raid5_has_ppl(conf))8324 return -EINVAL;8325 sectors &= ~((sector_t)conf->chunk_sectors - 1);8326 newsize = raid5_size(mddev, sectors, mddev->raid_disks);8327 if (mddev->external_size &&8328 mddev->array_sectors > newsize)8329 return -EINVAL;8330 8331 ret = mddev->bitmap_ops->resize(mddev, sectors, 0, false);8332 if (ret)8333 return ret;8334 8335 md_set_array_sectors(mddev, newsize);8336 if (sectors > mddev->dev_sectors &&8337 mddev->recovery_cp > mddev->dev_sectors) {8338 mddev->recovery_cp = mddev->dev_sectors;8339 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);8340 }8341 mddev->dev_sectors = sectors;8342 mddev->resync_max_sectors = sectors;8343 return 0;8344}8345 8346static int check_stripe_cache(struct mddev *mddev)8347{8348 /* Can only proceed if there are plenty of stripe_heads.8349 * We need a minimum of one full stripe,, and for sensible progress8350 * it is best to have about 4 times that.8351 * If we require 4 times, then the default 256 4K stripe_heads will8352 * allow for chunk sizes up to 256K, which is probably OK.8353 * If the chunk size is greater, user-space should request more8354 * stripe_heads first.8355 */8356 struct r5conf *conf = mddev->private;8357 if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 48358 > conf->min_nr_stripes ||8359 ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 48360 > conf->min_nr_stripes) {8361 pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",8362 mdname(mddev),8363 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)8364 / RAID5_STRIPE_SIZE(conf))*4);8365 return 0;8366 }8367 return 1;8368}8369 8370static int check_reshape(struct mddev *mddev)8371{8372 struct r5conf *conf = mddev->private;8373 8374 if (raid5_has_log(conf) || raid5_has_ppl(conf))8375 return -EINVAL;8376 if (mddev->delta_disks == 0 &&8377 mddev->new_layout == mddev->layout &&8378 mddev->new_chunk_sectors == mddev->chunk_sectors)8379 return 0; /* nothing to do */8380 if (has_failed(conf))8381 return -EINVAL;8382 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {8383 /* We might be able to shrink, but the devices must8384 * be made bigger first.8385 * For raid6, 4 is the minimum size.8386 * Otherwise 2 is the minimum8387 */8388 int min = 2;8389 if (mddev->level == 6)8390 min = 4;8391 if (mddev->raid_disks + mddev->delta_disks < min)8392 return -EINVAL;8393 }8394 8395 if (!check_stripe_cache(mddev))8396 return -ENOSPC;8397 8398 if (mddev->new_chunk_sectors > mddev->chunk_sectors ||8399 mddev->delta_disks > 0)8400 if (resize_chunks(conf,8401 conf->previous_raid_disks8402 + max(0, mddev->delta_disks),8403 max(mddev->new_chunk_sectors,8404 mddev->chunk_sectors)8405 ) < 0)8406 return -ENOMEM;8407 8408 if (conf->previous_raid_disks + mddev->delta_disks <= conf->pool_size)8409 return 0; /* never bother to shrink */8410 return resize_stripes(conf, (conf->previous_raid_disks8411 + mddev->delta_disks));8412}8413 8414static int raid5_start_reshape(struct mddev *mddev)8415{8416 struct r5conf *conf = mddev->private;8417 struct md_rdev *rdev;8418 int spares = 0;8419 int i;8420 unsigned long flags;8421 8422 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))8423 return -EBUSY;8424 8425 if (!check_stripe_cache(mddev))8426 return -ENOSPC;8427 8428 if (has_failed(conf))8429 return -EINVAL;8430 8431 /* raid5 can't handle concurrent reshape and recovery */8432 if (mddev->recovery_cp < MaxSector)8433 return -EBUSY;8434 for (i = 0; i < conf->raid_disks; i++)8435 if (conf->disks[i].replacement)8436 return -EBUSY;8437 8438 rdev_for_each(rdev, mddev) {8439 if (!test_bit(In_sync, &rdev->flags)8440 && !test_bit(Faulty, &rdev->flags))8441 spares++;8442 }8443 8444 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)8445 /* Not enough devices even to make a degraded array8446 * of that size8447 */8448 return -EINVAL;8449 8450 /* Refuse to reduce size of the array. Any reductions in8451 * array size must be through explicit setting of array_size8452 * attribute.8453 */8454 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)8455 < mddev->array_sectors) {8456 pr_warn("md/raid:%s: array size must be reduced before number of disks\n",8457 mdname(mddev));8458 return -EINVAL;8459 }8460 8461 atomic_set(&conf->reshape_stripes, 0);8462 spin_lock_irq(&conf->device_lock);8463 write_seqcount_begin(&conf->gen_lock);8464 conf->previous_raid_disks = conf->raid_disks;8465 conf->raid_disks += mddev->delta_disks;8466 conf->prev_chunk_sectors = conf->chunk_sectors;8467 conf->chunk_sectors = mddev->new_chunk_sectors;8468 conf->prev_algo = conf->algorithm;8469 conf->algorithm = mddev->new_layout;8470 conf->generation++;8471 /* Code that selects data_offset needs to see the generation update8472 * if reshape_progress has been set - so a memory barrier needed.8473 */8474 smp_mb();8475 if (mddev->reshape_backwards)8476 conf->reshape_progress = raid5_size(mddev, 0, 0);8477 else8478 conf->reshape_progress = 0;8479 conf->reshape_safe = conf->reshape_progress;8480 write_seqcount_end(&conf->gen_lock);8481 spin_unlock_irq(&conf->device_lock);8482 8483 /* Now make sure any requests that proceeded on the assumption8484 * the reshape wasn't running - like Discard or Read - have8485 * completed.8486 */8487 raid5_quiesce(mddev, true);8488 raid5_quiesce(mddev, false);8489 8490 /* Add some new drives, as many as will fit.8491 * We know there are enough to make the newly sized array work.8492 * Don't add devices if we are reducing the number of8493 * devices in the array. This is because it is not possible8494 * to correctly record the "partially reconstructed" state of8495 * such devices during the reshape and confusion could result.8496 */8497 if (mddev->delta_disks >= 0) {8498 rdev_for_each(rdev, mddev)8499 if (rdev->raid_disk < 0 &&8500 !test_bit(Faulty, &rdev->flags)) {8501 if (raid5_add_disk(mddev, rdev) == 0) {8502 if (rdev->raid_disk8503 >= conf->previous_raid_disks)8504 set_bit(In_sync, &rdev->flags);8505 else8506 rdev->recovery_offset = 0;8507 8508 /* Failure here is OK */8509 sysfs_link_rdev(mddev, rdev);8510 }8511 } else if (rdev->raid_disk >= conf->previous_raid_disks8512 && !test_bit(Faulty, &rdev->flags)) {8513 /* This is a spare that was manually added */8514 set_bit(In_sync, &rdev->flags);8515 }8516 8517 /* When a reshape changes the number of devices,8518 * ->degraded is measured against the larger of the8519 * pre and post number of devices.8520 */8521 spin_lock_irqsave(&conf->device_lock, flags);8522 mddev->degraded = raid5_calc_degraded(conf);8523 spin_unlock_irqrestore(&conf->device_lock, flags);8524 }8525 mddev->raid_disks = conf->raid_disks;8526 mddev->reshape_position = conf->reshape_progress;8527 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);8528 8529 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);8530 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);8531 clear_bit(MD_RECOVERY_DONE, &mddev->recovery);8532 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);8533 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);8534 conf->reshape_checkpoint = jiffies;8535 md_new_event();8536 return 0;8537}8538 8539/* This is called from the reshape thread and should make any8540 * changes needed in 'conf'8541 */8542static void end_reshape(struct r5conf *conf)8543{8544 8545 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {8546 struct md_rdev *rdev;8547 8548 spin_lock_irq(&conf->device_lock);8549 conf->previous_raid_disks = conf->raid_disks;8550 md_finish_reshape(conf->mddev);8551 smp_wmb();8552 conf->reshape_progress = MaxSector;8553 conf->mddev->reshape_position = MaxSector;8554 rdev_for_each(rdev, conf->mddev)8555 if (rdev->raid_disk >= 0 &&8556 !test_bit(Journal, &rdev->flags) &&8557 !test_bit(In_sync, &rdev->flags))8558 rdev->recovery_offset = MaxSector;8559 spin_unlock_irq(&conf->device_lock);8560 wake_up(&conf->wait_for_reshape);8561 8562 mddev_update_io_opt(conf->mddev,8563 conf->raid_disks - conf->max_degraded);8564 }8565}8566 8567/* This is called from the raid5d thread with mddev_lock held.8568 * It makes config changes to the device.8569 */8570static void raid5_finish_reshape(struct mddev *mddev)8571{8572 struct r5conf *conf = mddev->private;8573 struct md_rdev *rdev;8574 8575 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {8576 8577 if (mddev->delta_disks <= 0) {8578 int d;8579 spin_lock_irq(&conf->device_lock);8580 mddev->degraded = raid5_calc_degraded(conf);8581 spin_unlock_irq(&conf->device_lock);8582 for (d = conf->raid_disks ;8583 d < conf->raid_disks - mddev->delta_disks;8584 d++) {8585 rdev = conf->disks[d].rdev;8586 if (rdev)8587 clear_bit(In_sync, &rdev->flags);8588 rdev = conf->disks[d].replacement;8589 if (rdev)8590 clear_bit(In_sync, &rdev->flags);8591 }8592 }8593 mddev->layout = conf->algorithm;8594 mddev->chunk_sectors = conf->chunk_sectors;8595 mddev->reshape_position = MaxSector;8596 mddev->delta_disks = 0;8597 mddev->reshape_backwards = 0;8598 }8599}8600 8601static void raid5_quiesce(struct mddev *mddev, int quiesce)8602{8603 struct r5conf *conf = mddev->private;8604 8605 if (quiesce) {8606 /* stop all writes */8607 lock_all_device_hash_locks_irq(conf);8608 /* '2' tells resync/reshape to pause so that all8609 * active stripes can drain8610 */8611 r5c_flush_cache(conf, INT_MAX);8612 /* need a memory barrier to make sure read_one_chunk() sees8613 * quiesce started and reverts to slow (locked) path.8614 */8615 smp_store_release(&conf->quiesce, 2);8616 wait_event_cmd(conf->wait_for_quiescent,8617 atomic_read(&conf->active_stripes) == 0 &&8618 atomic_read(&conf->active_aligned_reads) == 0,8619 unlock_all_device_hash_locks_irq(conf),8620 lock_all_device_hash_locks_irq(conf));8621 conf->quiesce = 1;8622 unlock_all_device_hash_locks_irq(conf);8623 /* allow reshape to continue */8624 wake_up(&conf->wait_for_reshape);8625 } else {8626 /* re-enable writes */8627 lock_all_device_hash_locks_irq(conf);8628 conf->quiesce = 0;8629 wake_up(&conf->wait_for_quiescent);8630 wake_up(&conf->wait_for_reshape);8631 unlock_all_device_hash_locks_irq(conf);8632 }8633 log_quiesce(conf, quiesce);8634}8635 8636static void *raid45_takeover_raid0(struct mddev *mddev, int level)8637{8638 struct r0conf *raid0_conf = mddev->private;8639 sector_t sectors;8640 8641 /* for raid0 takeover only one zone is supported */8642 if (raid0_conf->nr_strip_zones > 1) {8643 pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n",8644 mdname(mddev));8645 return ERR_PTR(-EINVAL);8646 }8647 8648 sectors = raid0_conf->strip_zone[0].zone_end;8649 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);8650 mddev->dev_sectors = sectors;8651 mddev->new_level = level;8652 mddev->new_layout = ALGORITHM_PARITY_N;8653 mddev->new_chunk_sectors = mddev->chunk_sectors;8654 mddev->raid_disks += 1;8655 mddev->delta_disks = 1;8656 /* make sure it will be not marked as dirty */8657 mddev->recovery_cp = MaxSector;8658 8659 return setup_conf(mddev);8660}8661 8662static void *raid5_takeover_raid1(struct mddev *mddev)8663{8664 int chunksect;8665 void *ret;8666 8667 if (mddev->raid_disks != 2 ||8668 mddev->degraded > 1)8669 return ERR_PTR(-EINVAL);8670 8671 /* Should check if there are write-behind devices? */8672 8673 chunksect = 64*2; /* 64K by default */8674 8675 /* The array must be an exact multiple of chunksize */8676 while (chunksect && (mddev->array_sectors & (chunksect-1)))8677 chunksect >>= 1;8678 8679 if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))8680 /* array size does not allow a suitable chunk size */8681 return ERR_PTR(-EINVAL);8682 8683 mddev->new_level = 5;8684 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;8685 mddev->new_chunk_sectors = chunksect;8686 8687 ret = setup_conf(mddev);8688 if (!IS_ERR(ret))8689 mddev_clear_unsupported_flags(mddev,8690 UNSUPPORTED_MDDEV_FLAGS);8691 return ret;8692}8693 8694static void *raid5_takeover_raid6(struct mddev *mddev)8695{8696 int new_layout;8697 8698 switch (mddev->layout) {8699 case ALGORITHM_LEFT_ASYMMETRIC_6:8700 new_layout = ALGORITHM_LEFT_ASYMMETRIC;8701 break;8702 case ALGORITHM_RIGHT_ASYMMETRIC_6:8703 new_layout = ALGORITHM_RIGHT_ASYMMETRIC;8704 break;8705 case ALGORITHM_LEFT_SYMMETRIC_6:8706 new_layout = ALGORITHM_LEFT_SYMMETRIC;8707 break;8708 case ALGORITHM_RIGHT_SYMMETRIC_6:8709 new_layout = ALGORITHM_RIGHT_SYMMETRIC;8710 break;8711 case ALGORITHM_PARITY_0_6:8712 new_layout = ALGORITHM_PARITY_0;8713 break;8714 case ALGORITHM_PARITY_N:8715 new_layout = ALGORITHM_PARITY_N;8716 break;8717 default:8718 return ERR_PTR(-EINVAL);8719 }8720 mddev->new_level = 5;8721 mddev->new_layout = new_layout;8722 mddev->delta_disks = -1;8723 mddev->raid_disks -= 1;8724 return setup_conf(mddev);8725}8726 8727static int raid5_check_reshape(struct mddev *mddev)8728{8729 /* For a 2-drive array, the layout and chunk size can be changed8730 * immediately as not restriping is needed.8731 * For larger arrays we record the new value - after validation8732 * to be used by a reshape pass.8733 */8734 struct r5conf *conf = mddev->private;8735 int new_chunk = mddev->new_chunk_sectors;8736 8737 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))8738 return -EINVAL;8739 if (new_chunk > 0) {8740 if (!is_power_of_2(new_chunk))8741 return -EINVAL;8742 if (new_chunk < (PAGE_SIZE>>9))8743 return -EINVAL;8744 if (mddev->array_sectors & (new_chunk-1))8745 /* not factor of array size */8746 return -EINVAL;8747 }8748 8749 /* They look valid */8750 8751 if (mddev->raid_disks == 2) {8752 /* can make the change immediately */8753 if (mddev->new_layout >= 0) {8754 conf->algorithm = mddev->new_layout;8755 mddev->layout = mddev->new_layout;8756 }8757 if (new_chunk > 0) {8758 conf->chunk_sectors = new_chunk ;8759 mddev->chunk_sectors = new_chunk;8760 }8761 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);8762 md_wakeup_thread(mddev->thread);8763 }8764 return check_reshape(mddev);8765}8766 8767static int raid6_check_reshape(struct mddev *mddev)8768{8769 int new_chunk = mddev->new_chunk_sectors;8770 8771 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))8772 return -EINVAL;8773 if (new_chunk > 0) {8774 if (!is_power_of_2(new_chunk))8775 return -EINVAL;8776 if (new_chunk < (PAGE_SIZE >> 9))8777 return -EINVAL;8778 if (mddev->array_sectors & (new_chunk-1))8779 /* not factor of array size */8780 return -EINVAL;8781 }8782 8783 /* They look valid */8784 return check_reshape(mddev);8785}8786 8787static void *raid5_takeover(struct mddev *mddev)8788{8789 /* raid5 can take over:8790 * raid0 - if there is only one strip zone - make it a raid4 layout8791 * raid1 - if there are two drives. We need to know the chunk size8792 * raid4 - trivial - just use a raid4 layout.8793 * raid6 - Providing it is a *_6 layout8794 */8795 if (mddev->level == 0)8796 return raid45_takeover_raid0(mddev, 5);8797 if (mddev->level == 1)8798 return raid5_takeover_raid1(mddev);8799 if (mddev->level == 4) {8800 mddev->new_layout = ALGORITHM_PARITY_N;8801 mddev->new_level = 5;8802 return setup_conf(mddev);8803 }8804 if (mddev->level == 6)8805 return raid5_takeover_raid6(mddev);8806 8807 return ERR_PTR(-EINVAL);8808}8809 8810static void *raid4_takeover(struct mddev *mddev)8811{8812 /* raid4 can take over:8813 * raid0 - if there is only one strip zone8814 * raid5 - if layout is right8815 */8816 if (mddev->level == 0)8817 return raid45_takeover_raid0(mddev, 4);8818 if (mddev->level == 5 &&8819 mddev->layout == ALGORITHM_PARITY_N) {8820 mddev->new_layout = 0;8821 mddev->new_level = 4;8822 return setup_conf(mddev);8823 }8824 return ERR_PTR(-EINVAL);8825}8826 8827static struct md_personality raid5_personality;8828 8829static void *raid6_takeover(struct mddev *mddev)8830{8831 /* Currently can only take over a raid5. We map the8832 * personality to an equivalent raid6 personality8833 * with the Q block at the end.8834 */8835 int new_layout;8836 8837 if (mddev->pers != &raid5_personality)8838 return ERR_PTR(-EINVAL);8839 if (mddev->degraded > 1)8840 return ERR_PTR(-EINVAL);8841 if (mddev->raid_disks > 253)8842 return ERR_PTR(-EINVAL);8843 if (mddev->raid_disks < 3)8844 return ERR_PTR(-EINVAL);8845 8846 switch (mddev->layout) {8847 case ALGORITHM_LEFT_ASYMMETRIC:8848 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;8849 break;8850 case ALGORITHM_RIGHT_ASYMMETRIC:8851 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;8852 break;8853 case ALGORITHM_LEFT_SYMMETRIC:8854 new_layout = ALGORITHM_LEFT_SYMMETRIC_6;8855 break;8856 case ALGORITHM_RIGHT_SYMMETRIC:8857 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;8858 break;8859 case ALGORITHM_PARITY_0:8860 new_layout = ALGORITHM_PARITY_0_6;8861 break;8862 case ALGORITHM_PARITY_N:8863 new_layout = ALGORITHM_PARITY_N;8864 break;8865 default:8866 return ERR_PTR(-EINVAL);8867 }8868 mddev->new_level = 6;8869 mddev->new_layout = new_layout;8870 mddev->delta_disks = 1;8871 mddev->raid_disks += 1;8872 return setup_conf(mddev);8873}8874 8875static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)8876{8877 struct r5conf *conf;8878 int err;8879 8880 err = mddev_suspend_and_lock(mddev);8881 if (err)8882 return err;8883 conf = mddev->private;8884 if (!conf) {8885 mddev_unlock_and_resume(mddev);8886 return -ENODEV;8887 }8888 8889 if (strncmp(buf, "ppl", 3) == 0) {8890 /* ppl only works with RAID 5 */8891 if (!raid5_has_ppl(conf) && conf->level == 5) {8892 err = log_init(conf, NULL, true);8893 if (!err) {8894 err = resize_stripes(conf, conf->pool_size);8895 if (err)8896 log_exit(conf);8897 }8898 } else8899 err = -EINVAL;8900 } else if (strncmp(buf, "resync", 6) == 0) {8901 if (raid5_has_ppl(conf)) {8902 log_exit(conf);8903 err = resize_stripes(conf, conf->pool_size);8904 } else if (test_bit(MD_HAS_JOURNAL, &conf->mddev->flags) &&8905 r5l_log_disk_error(conf)) {8906 bool journal_dev_exists = false;8907 struct md_rdev *rdev;8908 8909 rdev_for_each(rdev, mddev)8910 if (test_bit(Journal, &rdev->flags)) {8911 journal_dev_exists = true;8912 break;8913 }8914 8915 if (!journal_dev_exists)8916 clear_bit(MD_HAS_JOURNAL, &mddev->flags);8917 else /* need remove journal device first */8918 err = -EBUSY;8919 } else8920 err = -EINVAL;8921 } else {8922 err = -EINVAL;8923 }8924 8925 if (!err)8926 md_update_sb(mddev, 1);8927 8928 mddev_unlock_and_resume(mddev);8929 8930 return err;8931}8932 8933static int raid5_start(struct mddev *mddev)8934{8935 struct r5conf *conf = mddev->private;8936 8937 return r5l_start(conf->log);8938}8939 8940/*8941 * This is only used for dm-raid456, caller already frozen sync_thread, hence8942 * if rehsape is still in progress, io that is waiting for reshape can never be8943 * done now, hence wake up and handle those IO.8944 */8945static void raid5_prepare_suspend(struct mddev *mddev)8946{8947 struct r5conf *conf = mddev->private;8948 8949 wake_up(&conf->wait_for_reshape);8950}8951 8952static struct md_personality raid6_personality =8953{8954 .name = "raid6",8955 .level = 6,8956 .owner = THIS_MODULE,8957 .make_request = raid5_make_request,8958 .run = raid5_run,8959 .start = raid5_start,8960 .free = raid5_free,8961 .status = raid5_status,8962 .error_handler = raid5_error,8963 .hot_add_disk = raid5_add_disk,8964 .hot_remove_disk= raid5_remove_disk,8965 .spare_active = raid5_spare_active,8966 .sync_request = raid5_sync_request,8967 .resize = raid5_resize,8968 .size = raid5_size,8969 .check_reshape = raid6_check_reshape,8970 .start_reshape = raid5_start_reshape,8971 .finish_reshape = raid5_finish_reshape,8972 .quiesce = raid5_quiesce,8973 .takeover = raid6_takeover,8974 .change_consistency_policy = raid5_change_consistency_policy,8975 .prepare_suspend = raid5_prepare_suspend,8976};8977static struct md_personality raid5_personality =8978{8979 .name = "raid5",8980 .level = 5,8981 .owner = THIS_MODULE,8982 .make_request = raid5_make_request,8983 .run = raid5_run,8984 .start = raid5_start,8985 .free = raid5_free,8986 .status = raid5_status,8987 .error_handler = raid5_error,8988 .hot_add_disk = raid5_add_disk,8989 .hot_remove_disk= raid5_remove_disk,8990 .spare_active = raid5_spare_active,8991 .sync_request = raid5_sync_request,8992 .resize = raid5_resize,8993 .size = raid5_size,8994 .check_reshape = raid5_check_reshape,8995 .start_reshape = raid5_start_reshape,8996 .finish_reshape = raid5_finish_reshape,8997 .quiesce = raid5_quiesce,8998 .takeover = raid5_takeover,8999 .change_consistency_policy = raid5_change_consistency_policy,9000 .prepare_suspend = raid5_prepare_suspend,9001};9002 9003static struct md_personality raid4_personality =9004{9005 .name = "raid4",9006 .level = 4,9007 .owner = THIS_MODULE,9008 .make_request = raid5_make_request,9009 .run = raid5_run,9010 .start = raid5_start,9011 .free = raid5_free,9012 .status = raid5_status,9013 .error_handler = raid5_error,9014 .hot_add_disk = raid5_add_disk,9015 .hot_remove_disk= raid5_remove_disk,9016 .spare_active = raid5_spare_active,9017 .sync_request = raid5_sync_request,9018 .resize = raid5_resize,9019 .size = raid5_size,9020 .check_reshape = raid5_check_reshape,9021 .start_reshape = raid5_start_reshape,9022 .finish_reshape = raid5_finish_reshape,9023 .quiesce = raid5_quiesce,9024 .takeover = raid4_takeover,9025 .change_consistency_policy = raid5_change_consistency_policy,9026 .prepare_suspend = raid5_prepare_suspend,9027};9028 9029static int __init raid5_init(void)9030{9031 int ret;9032 9033 raid5_wq = alloc_workqueue("raid5wq",9034 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);9035 if (!raid5_wq)9036 return -ENOMEM;9037 9038 ret = cpuhp_setup_state_multi(CPUHP_MD_RAID5_PREPARE,9039 "md/raid5:prepare",9040 raid456_cpu_up_prepare,9041 raid456_cpu_dead);9042 if (ret) {9043 destroy_workqueue(raid5_wq);9044 return ret;9045 }9046 register_md_personality(&raid6_personality);9047 register_md_personality(&raid5_personality);9048 register_md_personality(&raid4_personality);9049 return 0;9050}9051 9052static void raid5_exit(void)9053{9054 unregister_md_personality(&raid6_personality);9055 unregister_md_personality(&raid5_personality);9056 unregister_md_personality(&raid4_personality);9057 cpuhp_remove_multi_state(CPUHP_MD_RAID5_PREPARE);9058 destroy_workqueue(raid5_wq);9059}9060 9061module_init(raid5_init);9062module_exit(raid5_exit);9063MODULE_LICENSE("GPL");9064MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");9065MODULE_ALIAS("md-personality-4"); /* RAID5 */9066MODULE_ALIAS("md-raid5");9067MODULE_ALIAS("md-raid4");9068MODULE_ALIAS("md-level-5");9069MODULE_ALIAS("md-level-4");9070MODULE_ALIAS("md-personality-8"); /* RAID6 */9071MODULE_ALIAS("md-raid6");9072MODULE_ALIAS("md-level-6");9073 9074/* This used to be two separate modules, they were: */9075MODULE_ALIAS("raid5");9076MODULE_ALIAS("raid6");9077