1346 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.4 * All Rights Reserved.5 */6#include "xfs.h"7#include "xfs_fs.h"8#include "xfs_shared.h"9#include "xfs_format.h"10#include "xfs_log_format.h"11#include "xfs_trans_resv.h"12#include "xfs_bit.h"13#include "xfs_mount.h"14#include "xfs_inode.h"15#include "xfs_bmap.h"16#include "xfs_bmap_btree.h"17#include "xfs_trans_space.h"18#include "xfs_trans.h"19#include "xfs_rtalloc.h"20#include "xfs_error.h"21#include "xfs_rtbitmap.h"22#include "xfs_health.h"23 24/*25 * Realtime allocator bitmap functions shared with userspace.26 */27 28/*29 * Real time buffers need verifiers to avoid runtime warnings during IO.30 * We don't have anything to verify, however, so these are just dummy31 * operations.32 */33static void34xfs_rtbuf_verify_read(35 struct xfs_buf *bp)36{37 return;38}39 40static void41xfs_rtbuf_verify_write(42 struct xfs_buf *bp)43{44 return;45}46 47const struct xfs_buf_ops xfs_rtbuf_ops = {48 .name = "rtbuf",49 .verify_read = xfs_rtbuf_verify_read,50 .verify_write = xfs_rtbuf_verify_write,51};52 53/* Release cached rt bitmap and summary buffers. */54void55xfs_rtbuf_cache_relse(56 struct xfs_rtalloc_args *args)57{58 if (args->rbmbp) {59 xfs_trans_brelse(args->tp, args->rbmbp);60 args->rbmbp = NULL;61 args->rbmoff = NULLFILEOFF;62 }63 if (args->sumbp) {64 xfs_trans_brelse(args->tp, args->sumbp);65 args->sumbp = NULL;66 args->sumoff = NULLFILEOFF;67 }68}69 70/*71 * Get a buffer for the bitmap or summary file block specified.72 * The buffer is returned read and locked.73 */74static int75xfs_rtbuf_get(76 struct xfs_rtalloc_args *args,77 xfs_fileoff_t block, /* block number in bitmap or summary */78 int issum) /* is summary not bitmap */79{80 struct xfs_mount *mp = args->mp;81 struct xfs_buf **cbpp; /* cached block buffer */82 xfs_fileoff_t *coffp; /* cached block number */83 struct xfs_buf *bp; /* block buffer, result */84 struct xfs_inode *ip; /* bitmap or summary inode */85 struct xfs_bmbt_irec map;86 enum xfs_blft type;87 int nmap = 1;88 int error;89 90 if (issum) {91 cbpp = &args->sumbp;92 coffp = &args->sumoff;93 ip = mp->m_rsumip;94 type = XFS_BLFT_RTSUMMARY_BUF;95 } else {96 cbpp = &args->rbmbp;97 coffp = &args->rbmoff;98 ip = mp->m_rbmip;99 type = XFS_BLFT_RTBITMAP_BUF;100 }101 102 /*103 * If we have a cached buffer, and the block number matches, use that.104 */105 if (*cbpp && *coffp == block)106 return 0;107 108 /*109 * Otherwise we have to have to get the buffer. If there was an old110 * one, get rid of it first.111 */112 if (*cbpp) {113 xfs_trans_brelse(args->tp, *cbpp);114 *cbpp = NULL;115 }116 117 error = xfs_bmapi_read(ip, block, 1, &map, &nmap, 0);118 if (error)119 return error;120 121 if (XFS_IS_CORRUPT(mp, nmap == 0 || !xfs_bmap_is_written_extent(&map))) {122 xfs_rt_mark_sick(mp, issum ? XFS_SICK_RT_SUMMARY :123 XFS_SICK_RT_BITMAP);124 return -EFSCORRUPTED;125 }126 127 ASSERT(map.br_startblock != NULLFSBLOCK);128 error = xfs_trans_read_buf(mp, args->tp, mp->m_ddev_targp,129 XFS_FSB_TO_DADDR(mp, map.br_startblock),130 mp->m_bsize, 0, &bp, &xfs_rtbuf_ops);131 if (xfs_metadata_is_sick(error))132 xfs_rt_mark_sick(mp, issum ? XFS_SICK_RT_SUMMARY :133 XFS_SICK_RT_BITMAP);134 if (error)135 return error;136 137 xfs_trans_buf_set_type(args->tp, bp, type);138 *cbpp = bp;139 *coffp = block;140 return 0;141}142 143int144xfs_rtbitmap_read_buf(145 struct xfs_rtalloc_args *args,146 xfs_fileoff_t block)147{148 struct xfs_mount *mp = args->mp;149 150 if (XFS_IS_CORRUPT(mp, block >= mp->m_sb.sb_rbmblocks)) {151 xfs_rt_mark_sick(mp, XFS_SICK_RT_BITMAP);152 return -EFSCORRUPTED;153 }154 155 return xfs_rtbuf_get(args, block, 0);156}157 158int159xfs_rtsummary_read_buf(160 struct xfs_rtalloc_args *args,161 xfs_fileoff_t block)162{163 struct xfs_mount *mp = args->mp;164 165 if (XFS_IS_CORRUPT(mp, block >= mp->m_rsumblocks)) {166 xfs_rt_mark_sick(args->mp, XFS_SICK_RT_SUMMARY);167 return -EFSCORRUPTED;168 }169 return xfs_rtbuf_get(args, block, 1);170}171 172/*173 * Searching backward from start find the first block whose allocated/free state174 * is different from start's.175 */176int177xfs_rtfind_back(178 struct xfs_rtalloc_args *args,179 xfs_rtxnum_t start, /* starting rtext to look at */180 xfs_rtxnum_t *rtx) /* out: start rtext found */181{182 struct xfs_mount *mp = args->mp;183 int bit; /* bit number in the word */184 xfs_fileoff_t block; /* bitmap block number */185 int error; /* error value */186 xfs_rtxnum_t firstbit; /* first useful bit in the word */187 xfs_rtxnum_t i; /* current bit number rel. to start */188 xfs_rtxnum_t len; /* length of inspected area */189 xfs_rtword_t mask; /* mask of relevant bits for value */190 xfs_rtword_t want; /* mask for "good" values */191 xfs_rtword_t wdiff; /* difference from wanted value */192 xfs_rtword_t incore;193 unsigned int word; /* word number in the buffer */194 195 /*196 * Compute and read in starting bitmap block for starting block.197 */198 block = xfs_rtx_to_rbmblock(mp, start);199 error = xfs_rtbitmap_read_buf(args, block);200 if (error)201 return error;202 203 /*204 * Get the first word's index & point to it.205 */206 word = xfs_rtx_to_rbmword(mp, start);207 bit = (int)(start & (XFS_NBWORD - 1));208 len = start + 1;209 /*210 * Compute match value, based on the bit at start: if 1 (free)211 * then all-ones, else all-zeroes.212 */213 incore = xfs_rtbitmap_getword(args, word);214 want = (incore & ((xfs_rtword_t)1 << bit)) ? -1 : 0;215 /*216 * If the starting position is not word-aligned, deal with the217 * partial word.218 */219 if (bit < XFS_NBWORD - 1) {220 /*221 * Calculate first (leftmost) bit number to look at,222 * and mask for all the relevant bits in this word.223 */224 firstbit = max_t(xfs_srtblock_t, bit - len + 1, 0);225 mask = (((xfs_rtword_t)1 << (bit - firstbit + 1)) - 1) <<226 firstbit;227 /*228 * Calculate the difference between the value there229 * and what we're looking for.230 */231 if ((wdiff = (incore ^ want) & mask)) {232 /*233 * Different. Mark where we are and return.234 */235 i = bit - xfs_highbit32(wdiff);236 *rtx = start - i + 1;237 return 0;238 }239 i = bit - firstbit + 1;240 /*241 * Go on to previous block if that's where the previous word is242 * and we need the previous word.243 */244 if (--word == -1 && i < len) {245 /*246 * If done with this block, get the previous one.247 */248 error = xfs_rtbitmap_read_buf(args, --block);249 if (error)250 return error;251 252 word = mp->m_blockwsize - 1;253 }254 } else {255 /*256 * Starting on a word boundary, no partial word.257 */258 i = 0;259 }260 /*261 * Loop over whole words in buffers. When we use up one buffer262 * we move on to the previous one.263 */264 while (len - i >= XFS_NBWORD) {265 /*266 * Compute difference between actual and desired value.267 */268 incore = xfs_rtbitmap_getword(args, word);269 if ((wdiff = incore ^ want)) {270 /*271 * Different, mark where we are and return.272 */273 i += XFS_NBWORD - 1 - xfs_highbit32(wdiff);274 *rtx = start - i + 1;275 return 0;276 }277 i += XFS_NBWORD;278 /*279 * Go on to previous block if that's where the previous word is280 * and we need the previous word.281 */282 if (--word == -1 && i < len) {283 /*284 * If done with this block, get the previous one.285 */286 error = xfs_rtbitmap_read_buf(args, --block);287 if (error)288 return error;289 290 word = mp->m_blockwsize - 1;291 }292 }293 /*294 * If not ending on a word boundary, deal with the last295 * (partial) word.296 */297 if (len - i) {298 /*299 * Calculate first (leftmost) bit number to look at,300 * and mask for all the relevant bits in this word.301 */302 firstbit = XFS_NBWORD - (len - i);303 mask = (((xfs_rtword_t)1 << (len - i)) - 1) << firstbit;304 /*305 * Compute difference between actual and desired value.306 */307 incore = xfs_rtbitmap_getword(args, word);308 if ((wdiff = (incore ^ want) & mask)) {309 /*310 * Different, mark where we are and return.311 */312 i += XFS_NBWORD - 1 - xfs_highbit32(wdiff);313 *rtx = start - i + 1;314 return 0;315 } else316 i = len;317 }318 /*319 * No match, return that we scanned the whole area.320 */321 *rtx = start - i + 1;322 return 0;323}324 325/*326 * Searching forward from start to limit, find the first block whose327 * allocated/free state is different from start's.328 */329int330xfs_rtfind_forw(331 struct xfs_rtalloc_args *args,332 xfs_rtxnum_t start, /* starting rtext to look at */333 xfs_rtxnum_t limit, /* last rtext to look at */334 xfs_rtxnum_t *rtx) /* out: start rtext found */335{336 struct xfs_mount *mp = args->mp;337 int bit; /* bit number in the word */338 xfs_fileoff_t block; /* bitmap block number */339 int error;340 xfs_rtxnum_t i; /* current bit number rel. to start */341 xfs_rtxnum_t lastbit;/* last useful bit in the word */342 xfs_rtxnum_t len; /* length of inspected area */343 xfs_rtword_t mask; /* mask of relevant bits for value */344 xfs_rtword_t want; /* mask for "good" values */345 xfs_rtword_t wdiff; /* difference from wanted value */346 xfs_rtword_t incore;347 unsigned int word; /* word number in the buffer */348 349 ASSERT(start <= limit);350 351 /*352 * Compute and read in starting bitmap block for starting block.353 */354 block = xfs_rtx_to_rbmblock(mp, start);355 error = xfs_rtbitmap_read_buf(args, block);356 if (error)357 return error;358 359 /*360 * Get the first word's index & point to it.361 */362 word = xfs_rtx_to_rbmword(mp, start);363 bit = (int)(start & (XFS_NBWORD - 1));364 len = limit - start + 1;365 /*366 * Compute match value, based on the bit at start: if 1 (free)367 * then all-ones, else all-zeroes.368 */369 incore = xfs_rtbitmap_getword(args, word);370 want = (incore & ((xfs_rtword_t)1 << bit)) ? -1 : 0;371 /*372 * If the starting position is not word-aligned, deal with the373 * partial word.374 */375 if (bit) {376 /*377 * Calculate last (rightmost) bit number to look at,378 * and mask for all the relevant bits in this word.379 */380 lastbit = min(bit + len, XFS_NBWORD);381 mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit;382 /*383 * Calculate the difference between the value there384 * and what we're looking for.385 */386 if ((wdiff = (incore ^ want) & mask)) {387 /*388 * Different. Mark where we are and return.389 */390 i = xfs_lowbit32(wdiff) - bit;391 *rtx = start + i - 1;392 return 0;393 }394 i = lastbit - bit;395 /*396 * Go on to next block if that's where the next word is397 * and we need the next word.398 */399 if (++word == mp->m_blockwsize && i < len) {400 /*401 * If done with this block, get the previous one.402 */403 error = xfs_rtbitmap_read_buf(args, ++block);404 if (error)405 return error;406 407 word = 0;408 }409 } else {410 /*411 * Starting on a word boundary, no partial word.412 */413 i = 0;414 }415 /*416 * Loop over whole words in buffers. When we use up one buffer417 * we move on to the next one.418 */419 while (len - i >= XFS_NBWORD) {420 /*421 * Compute difference between actual and desired value.422 */423 incore = xfs_rtbitmap_getword(args, word);424 if ((wdiff = incore ^ want)) {425 /*426 * Different, mark where we are and return.427 */428 i += xfs_lowbit32(wdiff);429 *rtx = start + i - 1;430 return 0;431 }432 i += XFS_NBWORD;433 /*434 * Go on to next block if that's where the next word is435 * and we need the next word.436 */437 if (++word == mp->m_blockwsize && i < len) {438 /*439 * If done with this block, get the next one.440 */441 error = xfs_rtbitmap_read_buf(args, ++block);442 if (error)443 return error;444 445 word = 0;446 }447 }448 /*449 * If not ending on a word boundary, deal with the last450 * (partial) word.451 */452 if ((lastbit = len - i)) {453 /*454 * Calculate mask for all the relevant bits in this word.455 */456 mask = ((xfs_rtword_t)1 << lastbit) - 1;457 /*458 * Compute difference between actual and desired value.459 */460 incore = xfs_rtbitmap_getword(args, word);461 if ((wdiff = (incore ^ want) & mask)) {462 /*463 * Different, mark where we are and return.464 */465 i += xfs_lowbit32(wdiff);466 *rtx = start + i - 1;467 return 0;468 } else469 i = len;470 }471 /*472 * No match, return that we scanned the whole area.473 */474 *rtx = start + i - 1;475 return 0;476}477 478/* Log rtsummary counter at @infoword. */479static inline void480xfs_trans_log_rtsummary(481 struct xfs_rtalloc_args *args,482 unsigned int infoword)483{484 struct xfs_buf *bp = args->sumbp;485 size_t first, last;486 487 first = (void *)xfs_rsumblock_infoptr(args, infoword) - bp->b_addr;488 last = first + sizeof(xfs_suminfo_t) - 1;489 490 xfs_trans_log_buf(args->tp, bp, first, last);491}492 493/*494 * Modify the summary information for a given extent size, bitmap block495 * combination.496 */497int498xfs_rtmodify_summary(499 struct xfs_rtalloc_args *args,500 int log, /* log2 of extent size */501 xfs_fileoff_t bbno, /* bitmap block number */502 int delta) /* in/out: summary block number */503{504 struct xfs_mount *mp = args->mp;505 xfs_rtsumoff_t so = xfs_rtsumoffs(mp, log, bbno);506 unsigned int infoword;507 xfs_suminfo_t val;508 int error;509 510 error = xfs_rtsummary_read_buf(args, xfs_rtsumoffs_to_block(mp, so));511 if (error)512 return error;513 514 infoword = xfs_rtsumoffs_to_infoword(mp, so);515 val = xfs_suminfo_add(args, infoword, delta);516 517 if (mp->m_rsum_cache) {518 if (val == 0 && log + 1 == mp->m_rsum_cache[bbno])519 mp->m_rsum_cache[bbno] = log;520 if (val != 0 && log >= mp->m_rsum_cache[bbno])521 mp->m_rsum_cache[bbno] = log + 1;522 }523 524 xfs_trans_log_rtsummary(args, infoword);525 return 0;526}527 528/*529 * Read and return the summary information for a given extent size, bitmap block530 * combination.531 */532int533xfs_rtget_summary(534 struct xfs_rtalloc_args *args,535 int log, /* log2 of extent size */536 xfs_fileoff_t bbno, /* bitmap block number */537 xfs_suminfo_t *sum) /* out: summary info for this block */538{539 struct xfs_mount *mp = args->mp;540 xfs_rtsumoff_t so = xfs_rtsumoffs(mp, log, bbno);541 int error;542 543 error = xfs_rtsummary_read_buf(args, xfs_rtsumoffs_to_block(mp, so));544 if (!error)545 *sum = xfs_suminfo_get(args, xfs_rtsumoffs_to_infoword(mp, so));546 return error;547}548 549/* Log rtbitmap block from the word @from to the byte before @next. */550static inline void551xfs_trans_log_rtbitmap(552 struct xfs_rtalloc_args *args,553 unsigned int from,554 unsigned int next)555{556 struct xfs_buf *bp = args->rbmbp;557 size_t first, last;558 559 first = (void *)xfs_rbmblock_wordptr(args, from) - bp->b_addr;560 last = ((void *)xfs_rbmblock_wordptr(args, next) - 1) - bp->b_addr;561 562 xfs_trans_log_buf(args->tp, bp, first, last);563}564 565/*566 * Set the given range of bitmap bits to the given value.567 * Do whatever I/O and logging is required.568 */569int570xfs_rtmodify_range(571 struct xfs_rtalloc_args *args,572 xfs_rtxnum_t start, /* starting rtext to modify */573 xfs_rtxlen_t len, /* length of extent to modify */574 int val) /* 1 for free, 0 for allocated */575{576 struct xfs_mount *mp = args->mp;577 int bit; /* bit number in the word */578 xfs_fileoff_t block; /* bitmap block number */579 int error;580 int i; /* current bit number rel. to start */581 int lastbit; /* last useful bit in word */582 xfs_rtword_t mask; /* mask of relevant bits for value */583 xfs_rtword_t incore;584 unsigned int firstword; /* first word used in the buffer */585 unsigned int word; /* word number in the buffer */586 587 /*588 * Compute starting bitmap block number.589 */590 block = xfs_rtx_to_rbmblock(mp, start);591 /*592 * Read the bitmap block, and point to its data.593 */594 error = xfs_rtbitmap_read_buf(args, block);595 if (error)596 return error;597 598 /*599 * Compute the starting word's address, and starting bit.600 */601 firstword = word = xfs_rtx_to_rbmword(mp, start);602 bit = (int)(start & (XFS_NBWORD - 1));603 /*604 * 0 (allocated) => all zeroes; 1 (free) => all ones.605 */606 val = -val;607 /*608 * If not starting on a word boundary, deal with the first609 * (partial) word.610 */611 if (bit) {612 /*613 * Compute first bit not changed and mask of relevant bits.614 */615 lastbit = min(bit + len, XFS_NBWORD);616 mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit;617 /*618 * Set/clear the active bits.619 */620 incore = xfs_rtbitmap_getword(args, word);621 if (val)622 incore |= mask;623 else624 incore &= ~mask;625 xfs_rtbitmap_setword(args, word, incore);626 i = lastbit - bit;627 /*628 * Go on to the next block if that's where the next word is629 * and we need the next word.630 */631 if (++word == mp->m_blockwsize && i < len) {632 /*633 * Log the changed part of this block.634 * Get the next one.635 */636 xfs_trans_log_rtbitmap(args, firstword, word);637 error = xfs_rtbitmap_read_buf(args, ++block);638 if (error)639 return error;640 641 firstword = word = 0;642 }643 } else {644 /*645 * Starting on a word boundary, no partial word.646 */647 i = 0;648 }649 /*650 * Loop over whole words in buffers. When we use up one buffer651 * we move on to the next one.652 */653 while (len - i >= XFS_NBWORD) {654 /*655 * Set the word value correctly.656 */657 xfs_rtbitmap_setword(args, word, val);658 i += XFS_NBWORD;659 /*660 * Go on to the next block if that's where the next word is661 * and we need the next word.662 */663 if (++word == mp->m_blockwsize && i < len) {664 /*665 * Log the changed part of this block.666 * Get the next one.667 */668 xfs_trans_log_rtbitmap(args, firstword, word);669 error = xfs_rtbitmap_read_buf(args, ++block);670 if (error)671 return error;672 673 firstword = word = 0;674 }675 }676 /*677 * If not ending on a word boundary, deal with the last678 * (partial) word.679 */680 if ((lastbit = len - i)) {681 /*682 * Compute a mask of relevant bits.683 */684 mask = ((xfs_rtword_t)1 << lastbit) - 1;685 /*686 * Set/clear the active bits.687 */688 incore = xfs_rtbitmap_getword(args, word);689 if (val)690 incore |= mask;691 else692 incore &= ~mask;693 xfs_rtbitmap_setword(args, word, incore);694 word++;695 }696 /*697 * Log any remaining changed bytes.698 */699 if (word > firstword)700 xfs_trans_log_rtbitmap(args, firstword, word);701 return 0;702}703 704/*705 * Mark an extent specified by start and len freed.706 * Updates all the summary information as well as the bitmap.707 */708int709xfs_rtfree_range(710 struct xfs_rtalloc_args *args,711 xfs_rtxnum_t start, /* starting rtext to free */712 xfs_rtxlen_t len) /* in/out: summary block number */713{714 struct xfs_mount *mp = args->mp;715 xfs_rtxnum_t end; /* end of the freed extent */716 int error; /* error value */717 xfs_rtxnum_t postblock; /* first rtext freed > end */718 xfs_rtxnum_t preblock; /* first rtext freed < start */719 720 end = start + len - 1;721 /*722 * Modify the bitmap to mark this extent freed.723 */724 error = xfs_rtmodify_range(args, start, len, 1);725 if (error) {726 return error;727 }728 /*729 * Assume we're freeing out of the middle of an allocated extent.730 * We need to find the beginning and end of the extent so we can731 * properly update the summary.732 */733 error = xfs_rtfind_back(args, start, &preblock);734 if (error) {735 return error;736 }737 /*738 * Find the next allocated block (end of allocated extent).739 */740 error = xfs_rtfind_forw(args, end, mp->m_sb.sb_rextents - 1,741 &postblock);742 if (error)743 return error;744 /*745 * If there are blocks not being freed at the front of the746 * old extent, add summary data for them to be allocated.747 */748 if (preblock < start) {749 error = xfs_rtmodify_summary(args,750 xfs_highbit64(start - preblock),751 xfs_rtx_to_rbmblock(mp, preblock), -1);752 if (error) {753 return error;754 }755 }756 /*757 * If there are blocks not being freed at the end of the758 * old extent, add summary data for them to be allocated.759 */760 if (postblock > end) {761 error = xfs_rtmodify_summary(args,762 xfs_highbit64(postblock - end),763 xfs_rtx_to_rbmblock(mp, end + 1), -1);764 if (error) {765 return error;766 }767 }768 /*769 * Increment the summary information corresponding to the entire770 * (new) free extent.771 */772 return xfs_rtmodify_summary(args,773 xfs_highbit64(postblock + 1 - preblock),774 xfs_rtx_to_rbmblock(mp, preblock), 1);775}776 777/*778 * Check that the given range is either all allocated (val = 0) or779 * all free (val = 1).780 */781int782xfs_rtcheck_range(783 struct xfs_rtalloc_args *args,784 xfs_rtxnum_t start, /* starting rtext number of extent */785 xfs_rtxlen_t len, /* length of extent */786 int val, /* 1 for free, 0 for allocated */787 xfs_rtxnum_t *new, /* out: first rtext not matching */788 int *stat) /* out: 1 for matches, 0 for not */789{790 struct xfs_mount *mp = args->mp;791 int bit; /* bit number in the word */792 xfs_fileoff_t block; /* bitmap block number */793 int error;794 xfs_rtxnum_t i; /* current bit number rel. to start */795 xfs_rtxnum_t lastbit; /* last useful bit in word */796 xfs_rtword_t mask; /* mask of relevant bits for value */797 xfs_rtword_t wdiff; /* difference from wanted value */798 xfs_rtword_t incore;799 unsigned int word; /* word number in the buffer */800 801 /*802 * Compute starting bitmap block number803 */804 block = xfs_rtx_to_rbmblock(mp, start);805 /*806 * Read the bitmap block.807 */808 error = xfs_rtbitmap_read_buf(args, block);809 if (error)810 return error;811 812 /*813 * Compute the starting word's address, and starting bit.814 */815 word = xfs_rtx_to_rbmword(mp, start);816 bit = (int)(start & (XFS_NBWORD - 1));817 /*818 * 0 (allocated) => all zero's; 1 (free) => all one's.819 */820 val = -val;821 /*822 * If not starting on a word boundary, deal with the first823 * (partial) word.824 */825 if (bit) {826 /*827 * Compute first bit not examined.828 */829 lastbit = min(bit + len, XFS_NBWORD);830 /*831 * Mask of relevant bits.832 */833 mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit;834 /*835 * Compute difference between actual and desired value.836 */837 incore = xfs_rtbitmap_getword(args, word);838 if ((wdiff = (incore ^ val) & mask)) {839 /*840 * Different, compute first wrong bit and return.841 */842 i = xfs_lowbit32(wdiff) - bit;843 *new = start + i;844 *stat = 0;845 return 0;846 }847 i = lastbit - bit;848 /*849 * Go on to next block if that's where the next word is850 * and we need the next word.851 */852 if (++word == mp->m_blockwsize && i < len) {853 /*854 * If done with this block, get the next one.855 */856 error = xfs_rtbitmap_read_buf(args, ++block);857 if (error)858 return error;859 860 word = 0;861 }862 } else {863 /*864 * Starting on a word boundary, no partial word.865 */866 i = 0;867 }868 /*869 * Loop over whole words in buffers. When we use up one buffer870 * we move on to the next one.871 */872 while (len - i >= XFS_NBWORD) {873 /*874 * Compute difference between actual and desired value.875 */876 incore = xfs_rtbitmap_getword(args, word);877 if ((wdiff = incore ^ val)) {878 /*879 * Different, compute first wrong bit and return.880 */881 i += xfs_lowbit32(wdiff);882 *new = start + i;883 *stat = 0;884 return 0;885 }886 i += XFS_NBWORD;887 /*888 * Go on to next block if that's where the next word is889 * and we need the next word.890 */891 if (++word == mp->m_blockwsize && i < len) {892 /*893 * If done with this block, get the next one.894 */895 error = xfs_rtbitmap_read_buf(args, ++block);896 if (error)897 return error;898 899 word = 0;900 }901 }902 /*903 * If not ending on a word boundary, deal with the last904 * (partial) word.905 */906 if ((lastbit = len - i)) {907 /*908 * Mask of relevant bits.909 */910 mask = ((xfs_rtword_t)1 << lastbit) - 1;911 /*912 * Compute difference between actual and desired value.913 */914 incore = xfs_rtbitmap_getword(args, word);915 if ((wdiff = (incore ^ val) & mask)) {916 /*917 * Different, compute first wrong bit and return.918 */919 i += xfs_lowbit32(wdiff);920 *new = start + i;921 *stat = 0;922 return 0;923 } else924 i = len;925 }926 /*927 * Successful, return.928 */929 *new = start + i;930 *stat = 1;931 return 0;932}933 934#ifdef DEBUG935/*936 * Check that the given extent (block range) is allocated already.937 */938STATIC int939xfs_rtcheck_alloc_range(940 struct xfs_rtalloc_args *args,941 xfs_rtxnum_t start, /* starting rtext number of extent */942 xfs_rtxlen_t len) /* length of extent */943{944 xfs_rtxnum_t new; /* dummy for xfs_rtcheck_range */945 int stat;946 int error;947 948 error = xfs_rtcheck_range(args, start, len, 0, &new, &stat);949 if (error)950 return error;951 ASSERT(stat);952 return 0;953}954#else955#define xfs_rtcheck_alloc_range(a,b,l) (0)956#endif957/*958 * Free an extent in the realtime subvolume. Length is expressed in959 * realtime extents, as is the block number.960 */961int962xfs_rtfree_extent(963 struct xfs_trans *tp, /* transaction pointer */964 xfs_rtxnum_t start, /* starting rtext number to free */965 xfs_rtxlen_t len) /* length of extent freed */966{967 struct xfs_mount *mp = tp->t_mountp;968 struct xfs_rtalloc_args args = {969 .mp = mp,970 .tp = tp,971 };972 int error;973 struct timespec64 atime;974 975 ASSERT(mp->m_rbmip->i_itemp != NULL);976 xfs_assert_ilocked(mp->m_rbmip, XFS_ILOCK_EXCL);977 978 error = xfs_rtcheck_alloc_range(&args, start, len);979 if (error)980 return error;981 982 /*983 * Free the range of realtime blocks.984 */985 error = xfs_rtfree_range(&args, start, len);986 if (error)987 goto out;988 989 /*990 * Mark more blocks free in the superblock.991 */992 xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, (long)len);993 /*994 * If we've now freed all the blocks, reset the file sequence995 * number to 0.996 */997 if (tp->t_frextents_delta + mp->m_sb.sb_frextents ==998 mp->m_sb.sb_rextents) {999 if (!(mp->m_rbmip->i_diflags & XFS_DIFLAG_NEWRTBM))1000 mp->m_rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;1001 1002 atime = inode_get_atime(VFS_I(mp->m_rbmip));1003 atime.tv_sec = 0;1004 inode_set_atime_to_ts(VFS_I(mp->m_rbmip), atime);1005 xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);1006 }1007 error = 0;1008out:1009 xfs_rtbuf_cache_relse(&args);1010 return error;1011}1012 1013/*1014 * Free some blocks in the realtime subvolume. rtbno and rtlen are in units of1015 * rt blocks, not rt extents; must be aligned to the rt extent size; and rtlen1016 * cannot exceed XFS_MAX_BMBT_EXTLEN.1017 */1018int1019xfs_rtfree_blocks(1020 struct xfs_trans *tp,1021 xfs_fsblock_t rtbno,1022 xfs_filblks_t rtlen)1023{1024 struct xfs_mount *mp = tp->t_mountp;1025 xfs_extlen_t mod;1026 1027 ASSERT(rtlen <= XFS_MAX_BMBT_EXTLEN);1028 1029 mod = xfs_rtb_to_rtxoff(mp, rtlen);1030 if (mod) {1031 ASSERT(mod == 0);1032 return -EIO;1033 }1034 1035 mod = xfs_rtb_to_rtxoff(mp, rtbno);1036 if (mod) {1037 ASSERT(mod == 0);1038 return -EIO;1039 }1040 1041 return xfs_rtfree_extent(tp, xfs_rtb_to_rtx(mp, rtbno),1042 xfs_rtb_to_rtx(mp, rtlen));1043}1044 1045/* Find all the free records within a given range. */1046int1047xfs_rtalloc_query_range(1048 struct xfs_mount *mp,1049 struct xfs_trans *tp,1050 xfs_rtxnum_t start,1051 xfs_rtxnum_t end,1052 xfs_rtalloc_query_range_fn fn,1053 void *priv)1054{1055 struct xfs_rtalloc_args args = {1056 .mp = mp,1057 .tp = tp,1058 };1059 int error = 0;1060 1061 if (start > end)1062 return -EINVAL;1063 if (start == end || start >= mp->m_sb.sb_rextents)1064 return 0;1065 1066 end = min(end, mp->m_sb.sb_rextents - 1);1067 1068 /* Iterate the bitmap, looking for discrepancies. */1069 while (start <= end) {1070 struct xfs_rtalloc_rec rec;1071 int is_free;1072 xfs_rtxnum_t rtend;1073 1074 /* Is the first block free? */1075 error = xfs_rtcheck_range(&args, start, 1, 1, &rtend,1076 &is_free);1077 if (error)1078 break;1079 1080 /* How long does the extent go for? */1081 error = xfs_rtfind_forw(&args, start, end, &rtend);1082 if (error)1083 break;1084 1085 if (is_free) {1086 rec.ar_startext = start;1087 rec.ar_extcount = rtend - start + 1;1088 1089 error = fn(mp, tp, &rec, priv);1090 if (error)1091 break;1092 }1093 1094 start = rtend + 1;1095 }1096 1097 xfs_rtbuf_cache_relse(&args);1098 return error;1099}1100 1101/* Find all the free records. */1102int1103xfs_rtalloc_query_all(1104 struct xfs_mount *mp,1105 struct xfs_trans *tp,1106 xfs_rtalloc_query_range_fn fn,1107 void *priv)1108{1109 return xfs_rtalloc_query_range(mp, tp, 0, mp->m_sb.sb_rextents - 1, fn,1110 priv);1111}1112 1113/* Is the given extent all free? */1114int1115xfs_rtalloc_extent_is_free(1116 struct xfs_mount *mp,1117 struct xfs_trans *tp,1118 xfs_rtxnum_t start,1119 xfs_rtxlen_t len,1120 bool *is_free)1121{1122 struct xfs_rtalloc_args args = {1123 .mp = mp,1124 .tp = tp,1125 };1126 xfs_rtxnum_t end;1127 int matches;1128 int error;1129 1130 error = xfs_rtcheck_range(&args, start, len, 1, &end, &matches);1131 xfs_rtbuf_cache_relse(&args);1132 if (error)1133 return error;1134 1135 *is_free = matches;1136 return 0;1137}1138 1139/*1140 * Compute the number of rtbitmap blocks needed to track the given number of rt1141 * extents.1142 */1143xfs_filblks_t1144xfs_rtbitmap_blockcount(1145 struct xfs_mount *mp,1146 xfs_rtbxlen_t rtextents)1147{1148 return howmany_64(rtextents, NBBY * mp->m_sb.sb_blocksize);1149}1150 1151/* Compute the number of rtsummary blocks needed to track the given rt space. */1152xfs_filblks_t1153xfs_rtsummary_blockcount(1154 struct xfs_mount *mp,1155 unsigned int rsumlevels,1156 xfs_extlen_t rbmblocks)1157{1158 unsigned long long rsumwords;1159 1160 rsumwords = (unsigned long long)rsumlevels * rbmblocks;1161 return XFS_B_TO_FSB(mp, rsumwords << XFS_WORDLOG);1162}1163 1164/* Lock both realtime free space metadata inodes for a freespace update. */1165void1166xfs_rtbitmap_lock(1167 struct xfs_mount *mp)1168{1169 xfs_ilock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP);1170 xfs_ilock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM);1171}1172 1173/*1174 * Join both realtime free space metadata inodes to the transaction. The1175 * ILOCKs will be released on transaction commit.1176 */1177void1178xfs_rtbitmap_trans_join(1179 struct xfs_trans *tp)1180{1181 xfs_trans_ijoin(tp, tp->t_mountp->m_rbmip, XFS_ILOCK_EXCL);1182 xfs_trans_ijoin(tp, tp->t_mountp->m_rsumip, XFS_ILOCK_EXCL);1183}1184 1185/* Unlock both realtime free space metadata inodes after a freespace update. */1186void1187xfs_rtbitmap_unlock(1188 struct xfs_mount *mp)1189{1190 xfs_iunlock(mp->m_rsumip, XFS_ILOCK_EXCL | XFS_ILOCK_RTSUM);1191 xfs_iunlock(mp->m_rbmip, XFS_ILOCK_EXCL | XFS_ILOCK_RTBITMAP);1192}1193 1194/*1195 * Lock the realtime free space metadata inodes for a freespace scan. Callers1196 * must walk metadata blocks in order of increasing file offset.1197 */1198void1199xfs_rtbitmap_lock_shared(1200 struct xfs_mount *mp,1201 unsigned int rbmlock_flags)1202{1203 if (rbmlock_flags & XFS_RBMLOCK_BITMAP)1204 xfs_ilock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);1205 1206 if (rbmlock_flags & XFS_RBMLOCK_SUMMARY)1207 xfs_ilock(mp->m_rsumip, XFS_ILOCK_SHARED | XFS_ILOCK_RTSUM);1208}1209 1210/* Unlock the realtime free space metadata inodes after a freespace scan. */1211void1212xfs_rtbitmap_unlock_shared(1213 struct xfs_mount *mp,1214 unsigned int rbmlock_flags)1215{1216 if (rbmlock_flags & XFS_RBMLOCK_SUMMARY)1217 xfs_iunlock(mp->m_rsumip, XFS_ILOCK_SHARED | XFS_ILOCK_RTSUM);1218 1219 if (rbmlock_flags & XFS_RBMLOCK_BITMAP)1220 xfs_iunlock(mp->m_rbmip, XFS_ILOCK_SHARED | XFS_ILOCK_RTBITMAP);1221}1222 1223static int1224xfs_rtfile_alloc_blocks(1225 struct xfs_inode *ip,1226 xfs_fileoff_t offset_fsb,1227 xfs_filblks_t count_fsb,1228 struct xfs_bmbt_irec *map)1229{1230 struct xfs_mount *mp = ip->i_mount;1231 struct xfs_trans *tp;1232 int nmap = 1;1233 int error;1234 1235 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtalloc,1236 XFS_GROWFSRT_SPACE_RES(mp, count_fsb), 0, 0, &tp);1237 if (error)1238 return error;1239 1240 xfs_ilock(ip, XFS_ILOCK_EXCL);1241 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);1242 1243 error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK,1244 XFS_IEXT_ADD_NOSPLIT_CNT);1245 if (error)1246 goto out_trans_cancel;1247 1248 error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb,1249 XFS_BMAPI_METADATA, 0, map, &nmap);1250 if (error)1251 goto out_trans_cancel;1252 1253 return xfs_trans_commit(tp);1254 1255out_trans_cancel:1256 xfs_trans_cancel(tp);1257 return error;1258}1259 1260/* Get a buffer for the block. */1261static int1262xfs_rtfile_initialize_block(1263 struct xfs_inode *ip,1264 xfs_fsblock_t fsbno,1265 void *data)1266{1267 struct xfs_mount *mp = ip->i_mount;1268 struct xfs_trans *tp;1269 struct xfs_buf *bp;1270 const size_t copylen = mp->m_blockwsize << XFS_WORDLOG;1271 enum xfs_blft buf_type;1272 int error;1273 1274 if (ip == mp->m_rsumip)1275 buf_type = XFS_BLFT_RTSUMMARY_BUF;1276 else1277 buf_type = XFS_BLFT_RTBITMAP_BUF;1278 1279 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtzero, 0, 0, 0, &tp);1280 if (error)1281 return error;1282 xfs_ilock(ip, XFS_ILOCK_EXCL);1283 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);1284 1285 error = xfs_trans_get_buf(tp, mp->m_ddev_targp,1286 XFS_FSB_TO_DADDR(mp, fsbno), mp->m_bsize, 0, &bp);1287 if (error) {1288 xfs_trans_cancel(tp);1289 return error;1290 }1291 1292 xfs_trans_buf_set_type(tp, bp, buf_type);1293 bp->b_ops = &xfs_rtbuf_ops;1294 if (data)1295 memcpy(bp->b_addr, data, copylen);1296 else1297 memset(bp->b_addr, 0, copylen);1298 xfs_trans_log_buf(tp, bp, 0, mp->m_sb.sb_blocksize - 1);1299 return xfs_trans_commit(tp);1300}1301 1302/*1303 * Allocate space to the bitmap or summary file, and zero it, for growfs.1304 * @data must be a contiguous buffer large enough to fill all blocks in the1305 * file; or NULL to initialize the contents to zeroes.1306 */1307int1308xfs_rtfile_initialize_blocks(1309 struct xfs_inode *ip, /* inode (bitmap/summary) */1310 xfs_fileoff_t offset_fsb, /* offset to start from */1311 xfs_fileoff_t end_fsb, /* offset to allocate to */1312 void *data) /* data to fill the blocks */1313{1314 struct xfs_mount *mp = ip->i_mount;1315 const size_t copylen = mp->m_blockwsize << XFS_WORDLOG;1316 1317 while (offset_fsb < end_fsb) {1318 struct xfs_bmbt_irec map;1319 xfs_filblks_t i;1320 int error;1321 1322 error = xfs_rtfile_alloc_blocks(ip, offset_fsb,1323 end_fsb - offset_fsb, &map);1324 if (error)1325 return error;1326 1327 /*1328 * Now we need to clear the allocated blocks.1329 *1330 * Do this one block per transaction, to keep it simple.1331 */1332 for (i = 0; i < map.br_blockcount; i++) {1333 error = xfs_rtfile_initialize_block(ip,1334 map.br_startblock + i, data);1335 if (error)1336 return error;1337 if (data)1338 data += copylen;1339 }1340 1341 offset_fsb = map.br_startoff + map.br_blockcount;1342 }1343 1344 return 0;1345}1346