751 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2018-2019 HUAWEI, Inc.4 * https://www.huawei.com/5 */6#include "internal.h"7#include <linux/unaligned.h>8#include <trace/events/erofs.h>9 10struct z_erofs_maprecorder {11 struct inode *inode;12 struct erofs_map_blocks *map;13 unsigned long lcn;14 /* compression extent information gathered */15 u8 type, headtype;16 u16 clusterofs;17 u16 delta[2];18 erofs_blk_t pblk, compressedblks;19 erofs_off_t nextpackoff;20 bool partialref;21};22 23static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m,24 unsigned long lcn)25{26 struct inode *const inode = m->inode;27 struct erofs_inode *const vi = EROFS_I(inode);28 const erofs_off_t pos = Z_EROFS_FULL_INDEX_ALIGN(erofs_iloc(inode) +29 vi->inode_isize + vi->xattr_isize) +30 lcn * sizeof(struct z_erofs_lcluster_index);31 struct z_erofs_lcluster_index *di;32 unsigned int advise;33 34 di = erofs_read_metabuf(&m->map->buf, inode->i_sb, pos, EROFS_KMAP);35 if (IS_ERR(di))36 return PTR_ERR(di);37 m->lcn = lcn;38 m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index);39 40 advise = le16_to_cpu(di->di_advise);41 m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK;42 if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {43 m->clusterofs = 1 << vi->z_logical_clusterbits;44 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);45 if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) {46 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |47 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {48 DBG_BUGON(1);49 return -EFSCORRUPTED;50 }51 m->compressedblks = m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT;52 m->delta[0] = 1;53 }54 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);55 } else {56 m->partialref = !!(advise & Z_EROFS_LI_PARTIAL_REF);57 m->clusterofs = le16_to_cpu(di->di_clusterofs);58 if (m->clusterofs >= 1 << vi->z_logical_clusterbits) {59 DBG_BUGON(1);60 return -EFSCORRUPTED;61 }62 m->pblk = le32_to_cpu(di->di_u.blkaddr);63 }64 return 0;65}66 67static unsigned int decode_compactedbits(unsigned int lobits,68 u8 *in, unsigned int pos, u8 *type)69{70 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);71 const unsigned int lo = v & ((1 << lobits) - 1);72 73 *type = (v >> lobits) & 3;74 return lo;75}76 77static int get_compacted_la_distance(unsigned int lobits,78 unsigned int encodebits,79 unsigned int vcnt, u8 *in, int i)80{81 unsigned int lo, d1 = 0;82 u8 type;83 84 DBG_BUGON(i >= vcnt);85 86 do {87 lo = decode_compactedbits(lobits, in, encodebits * i, &type);88 89 if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)90 return d1;91 ++d1;92 } while (++i < vcnt);93 94 /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */95 if (!(lo & Z_EROFS_LI_D0_CBLKCNT))96 d1 += lo - 1;97 return d1;98}99 100static int unpack_compacted_index(struct z_erofs_maprecorder *m,101 unsigned int amortizedshift,102 erofs_off_t pos, bool lookahead)103{104 struct erofs_inode *const vi = EROFS_I(m->inode);105 const unsigned int lclusterbits = vi->z_logical_clusterbits;106 unsigned int vcnt, lo, lobits, encodebits, nblk, bytes;107 bool big_pcluster;108 u8 *in, type;109 int i;110 111 if (1 << amortizedshift == 4 && lclusterbits <= 14)112 vcnt = 2;113 else if (1 << amortizedshift == 2 && lclusterbits <= 12)114 vcnt = 16;115 else116 return -EOPNOTSUPP;117 118 in = erofs_read_metabuf(&m->map->buf, m->inode->i_sb, pos, EROFS_KMAP);119 if (IS_ERR(in))120 return PTR_ERR(in);121 122 /* it doesn't equal to round_up(..) */123 m->nextpackoff = round_down(pos, vcnt << amortizedshift) +124 (vcnt << amortizedshift);125 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;126 lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U);127 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;128 bytes = pos & ((vcnt << amortizedshift) - 1);129 in -= bytes;130 i = bytes >> amortizedshift;131 132 lo = decode_compactedbits(lobits, in, encodebits * i, &type);133 m->type = type;134 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {135 m->clusterofs = 1 << lclusterbits;136 137 /* figure out lookahead_distance: delta[1] if needed */138 if (lookahead)139 m->delta[1] = get_compacted_la_distance(lobits,140 encodebits, vcnt, in, i);141 if (lo & Z_EROFS_LI_D0_CBLKCNT) {142 if (!big_pcluster) {143 DBG_BUGON(1);144 return -EFSCORRUPTED;145 }146 m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT;147 m->delta[0] = 1;148 return 0;149 } else if (i + 1 != (int)vcnt) {150 m->delta[0] = lo;151 return 0;152 }153 /*154 * since the last lcluster in the pack is special,155 * of which lo saves delta[1] rather than delta[0].156 * Hence, get delta[0] by the previous lcluster indirectly.157 */158 lo = decode_compactedbits(lobits, in,159 encodebits * (i - 1), &type);160 if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)161 lo = 0;162 else if (lo & Z_EROFS_LI_D0_CBLKCNT)163 lo = 1;164 m->delta[0] = lo + 1;165 return 0;166 }167 m->clusterofs = lo;168 m->delta[0] = 0;169 /* figout out blkaddr (pblk) for HEAD lclusters */170 if (!big_pcluster) {171 nblk = 1;172 while (i > 0) {173 --i;174 lo = decode_compactedbits(lobits, in,175 encodebits * i, &type);176 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD)177 i -= lo;178 179 if (i >= 0)180 ++nblk;181 }182 } else {183 nblk = 0;184 while (i > 0) {185 --i;186 lo = decode_compactedbits(lobits, in,187 encodebits * i, &type);188 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {189 if (lo & Z_EROFS_LI_D0_CBLKCNT) {190 --i;191 nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT;192 continue;193 }194 /* bigpcluster shouldn't have plain d0 == 1 */195 if (lo <= 1) {196 DBG_BUGON(1);197 return -EFSCORRUPTED;198 }199 i -= lo - 2;200 continue;201 }202 ++nblk;203 }204 }205 in += (vcnt << amortizedshift) - sizeof(__le32);206 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;207 return 0;208}209 210static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m,211 unsigned long lcn, bool lookahead)212{213 struct inode *const inode = m->inode;214 struct erofs_inode *const vi = EROFS_I(inode);215 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) +216 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);217 unsigned int totalidx = erofs_iblks(inode);218 unsigned int compacted_4b_initial, compacted_2b;219 unsigned int amortizedshift;220 erofs_off_t pos;221 222 if (lcn >= totalidx)223 return -EINVAL;224 225 m->lcn = lcn;226 /* used to align to 32-byte (compacted_2b) alignment */227 compacted_4b_initial = (32 - ebase % 32) / 4;228 if (compacted_4b_initial == 32 / 4)229 compacted_4b_initial = 0;230 231 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) &&232 compacted_4b_initial < totalidx)233 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);234 else235 compacted_2b = 0;236 237 pos = ebase;238 if (lcn < compacted_4b_initial) {239 amortizedshift = 2;240 goto out;241 }242 pos += compacted_4b_initial * 4;243 lcn -= compacted_4b_initial;244 245 if (lcn < compacted_2b) {246 amortizedshift = 1;247 goto out;248 }249 pos += compacted_2b * 2;250 lcn -= compacted_2b;251 amortizedshift = 2;252out:253 pos += lcn * (1 << amortizedshift);254 return unpack_compacted_index(m, amortizedshift, pos, lookahead);255}256 257static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m,258 unsigned int lcn, bool lookahead)259{260 switch (EROFS_I(m->inode)->datalayout) {261 case EROFS_INODE_COMPRESSED_FULL:262 return z_erofs_load_full_lcluster(m, lcn);263 case EROFS_INODE_COMPRESSED_COMPACT:264 return z_erofs_load_compact_lcluster(m, lcn, lookahead);265 default:266 return -EINVAL;267 }268}269 270static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,271 unsigned int lookback_distance)272{273 struct super_block *sb = m->inode->i_sb;274 struct erofs_inode *const vi = EROFS_I(m->inode);275 const unsigned int lclusterbits = vi->z_logical_clusterbits;276 277 while (m->lcn >= lookback_distance) {278 unsigned long lcn = m->lcn - lookback_distance;279 int err;280 281 err = z_erofs_load_lcluster_from_disk(m, lcn, false);282 if (err)283 return err;284 285 switch (m->type) {286 case Z_EROFS_LCLUSTER_TYPE_NONHEAD:287 lookback_distance = m->delta[0];288 if (!lookback_distance)289 goto err_bogus;290 continue;291 case Z_EROFS_LCLUSTER_TYPE_PLAIN:292 case Z_EROFS_LCLUSTER_TYPE_HEAD1:293 case Z_EROFS_LCLUSTER_TYPE_HEAD2:294 m->headtype = m->type;295 m->map->m_la = (lcn << lclusterbits) | m->clusterofs;296 return 0;297 default:298 erofs_err(sb, "unknown type %u @ lcn %lu of nid %llu",299 m->type, lcn, vi->nid);300 DBG_BUGON(1);301 return -EOPNOTSUPP;302 }303 }304err_bogus:305 erofs_err(sb, "bogus lookback distance %u @ lcn %lu of nid %llu",306 lookback_distance, m->lcn, vi->nid);307 DBG_BUGON(1);308 return -EFSCORRUPTED;309}310 311static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,312 unsigned int initial_lcn)313{314 struct super_block *sb = m->inode->i_sb;315 struct erofs_inode *const vi = EROFS_I(m->inode);316 struct erofs_map_blocks *const map = m->map;317 const unsigned int lclusterbits = vi->z_logical_clusterbits;318 unsigned long lcn;319 int err;320 321 DBG_BUGON(m->type != Z_EROFS_LCLUSTER_TYPE_PLAIN &&322 m->type != Z_EROFS_LCLUSTER_TYPE_HEAD1 &&323 m->type != Z_EROFS_LCLUSTER_TYPE_HEAD2);324 DBG_BUGON(m->type != m->headtype);325 326 if (m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN ||327 ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1) &&328 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) ||329 ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) &&330 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {331 map->m_plen = 1ULL << lclusterbits;332 return 0;333 }334 lcn = m->lcn + 1;335 if (m->compressedblks)336 goto out;337 338 err = z_erofs_load_lcluster_from_disk(m, lcn, false);339 if (err)340 return err;341 342 /*343 * If the 1st NONHEAD lcluster has already been handled initially w/o344 * valid compressedblks, which means at least it mustn't be CBLKCNT, or345 * an internal implemenatation error is detected.346 *347 * The following code can also handle it properly anyway, but let's348 * BUG_ON in the debugging mode only for developers to notice that.349 */350 DBG_BUGON(lcn == initial_lcn &&351 m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD);352 353 switch (m->type) {354 case Z_EROFS_LCLUSTER_TYPE_PLAIN:355 case Z_EROFS_LCLUSTER_TYPE_HEAD1:356 case Z_EROFS_LCLUSTER_TYPE_HEAD2:357 /*358 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type359 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.360 */361 m->compressedblks = 1 << (lclusterbits - sb->s_blocksize_bits);362 break;363 case Z_EROFS_LCLUSTER_TYPE_NONHEAD:364 if (m->delta[0] != 1)365 goto err_bonus_cblkcnt;366 if (m->compressedblks)367 break;368 fallthrough;369 default:370 erofs_err(sb, "cannot found CBLKCNT @ lcn %lu of nid %llu", lcn,371 vi->nid);372 DBG_BUGON(1);373 return -EFSCORRUPTED;374 }375out:376 map->m_plen = erofs_pos(sb, m->compressedblks);377 return 0;378err_bonus_cblkcnt:379 erofs_err(sb, "bogus CBLKCNT @ lcn %lu of nid %llu", lcn, vi->nid);380 DBG_BUGON(1);381 return -EFSCORRUPTED;382}383 384static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)385{386 struct inode *inode = m->inode;387 struct erofs_inode *vi = EROFS_I(inode);388 struct erofs_map_blocks *map = m->map;389 unsigned int lclusterbits = vi->z_logical_clusterbits;390 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits;391 int err;392 393 do {394 /* handle the last EOF pcluster (no next HEAD lcluster) */395 if ((lcn << lclusterbits) >= inode->i_size) {396 map->m_llen = inode->i_size - map->m_la;397 return 0;398 }399 400 err = z_erofs_load_lcluster_from_disk(m, lcn, true);401 if (err)402 return err;403 404 if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {405 DBG_BUGON(!m->delta[1] &&406 m->clusterofs != 1 << lclusterbits);407 } else if (m->type == Z_EROFS_LCLUSTER_TYPE_PLAIN ||408 m->type == Z_EROFS_LCLUSTER_TYPE_HEAD1 ||409 m->type == Z_EROFS_LCLUSTER_TYPE_HEAD2) {410 /* go on until the next HEAD lcluster */411 if (lcn != headlcn)412 break;413 m->delta[1] = 1;414 } else {415 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu",416 m->type, lcn, vi->nid);417 DBG_BUGON(1);418 return -EOPNOTSUPP;419 }420 lcn += m->delta[1];421 } while (m->delta[1]);422 423 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;424 return 0;425}426 427static int z_erofs_do_map_blocks(struct inode *inode,428 struct erofs_map_blocks *map, int flags)429{430 struct erofs_inode *const vi = EROFS_I(inode);431 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;432 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;433 struct z_erofs_maprecorder m = {434 .inode = inode,435 .map = map,436 };437 int err = 0;438 unsigned int lclusterbits, endoff, afmt;439 unsigned long initial_lcn;440 unsigned long long ofs, end;441 442 lclusterbits = vi->z_logical_clusterbits;443 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la;444 initial_lcn = ofs >> lclusterbits;445 endoff = ofs & ((1 << lclusterbits) - 1);446 447 err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false);448 if (err)449 goto unmap_out;450 451 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL))452 vi->z_idataoff = m.nextpackoff;453 454 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;455 end = (m.lcn + 1ULL) << lclusterbits;456 457 switch (m.type) {458 case Z_EROFS_LCLUSTER_TYPE_PLAIN:459 case Z_EROFS_LCLUSTER_TYPE_HEAD1:460 case Z_EROFS_LCLUSTER_TYPE_HEAD2:461 if (endoff >= m.clusterofs) {462 m.headtype = m.type;463 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;464 /*465 * For ztailpacking files, in order to inline data more466 * effectively, special EOF lclusters are now supported467 * which can have three parts at most.468 */469 if (ztailpacking && end > inode->i_size)470 end = inode->i_size;471 break;472 }473 /* m.lcn should be >= 1 if endoff < m.clusterofs */474 if (!m.lcn) {475 erofs_err(inode->i_sb,476 "invalid logical cluster 0 at nid %llu",477 vi->nid);478 err = -EFSCORRUPTED;479 goto unmap_out;480 }481 end = (m.lcn << lclusterbits) | m.clusterofs;482 map->m_flags |= EROFS_MAP_FULL_MAPPED;483 m.delta[0] = 1;484 fallthrough;485 case Z_EROFS_LCLUSTER_TYPE_NONHEAD:486 /* get the corresponding first chunk */487 err = z_erofs_extent_lookback(&m, m.delta[0]);488 if (err)489 goto unmap_out;490 break;491 default:492 erofs_err(inode->i_sb,493 "unknown type %u @ offset %llu of nid %llu",494 m.type, ofs, vi->nid);495 err = -EOPNOTSUPP;496 goto unmap_out;497 }498 if (m.partialref)499 map->m_flags |= EROFS_MAP_PARTIAL_REF;500 map->m_llen = end - map->m_la;501 502 if (flags & EROFS_GET_BLOCKS_FINDTAIL) {503 vi->z_tailextent_headlcn = m.lcn;504 /* for non-compact indexes, fragmentoff is 64 bits */505 if (fragment && vi->datalayout == EROFS_INODE_COMPRESSED_FULL)506 vi->z_fragmentoff |= (u64)m.pblk << 32;507 }508 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {509 map->m_flags |= EROFS_MAP_META;510 map->m_pa = vi->z_idataoff;511 map->m_plen = vi->z_idata_size;512 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {513 map->m_flags |= EROFS_MAP_FRAGMENT;514 } else {515 map->m_pa = erofs_pos(inode->i_sb, m.pblk);516 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);517 if (err)518 goto unmap_out;519 }520 521 if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) {522 if (map->m_llen > map->m_plen) {523 DBG_BUGON(1);524 err = -EFSCORRUPTED;525 goto unmap_out;526 }527 afmt = vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER ?528 Z_EROFS_COMPRESSION_INTERLACED :529 Z_EROFS_COMPRESSION_SHIFTED;530 } else {531 afmt = m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2 ?532 vi->z_algorithmtype[1] : vi->z_algorithmtype[0];533 if (!(EROFS_I_SB(inode)->available_compr_algs & (1 << afmt))) {534 erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu",535 afmt, vi->nid);536 err = -EFSCORRUPTED;537 goto unmap_out;538 }539 }540 map->m_algorithmformat = afmt;541 542 if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||543 ((flags & EROFS_GET_BLOCKS_READMORE) &&544 (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA ||545 map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE ||546 map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) &&547 map->m_llen >= i_blocksize(inode))) {548 err = z_erofs_get_extent_decompressedlen(&m);549 if (!err)550 map->m_flags |= EROFS_MAP_FULL_MAPPED;551 }552 553unmap_out:554 erofs_unmap_metabuf(&m.map->buf);555 return err;556}557 558static int z_erofs_fill_inode_lazy(struct inode *inode)559{560 struct erofs_inode *const vi = EROFS_I(inode);561 struct super_block *const sb = inode->i_sb;562 int err, headnr;563 erofs_off_t pos;564 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;565 struct z_erofs_map_header *h;566 567 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {568 /*569 * paired with smp_mb() at the end of the function to ensure570 * fields will only be observed after the bit is set.571 */572 smp_mb();573 return 0;574 }575 576 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))577 return -ERESTARTSYS;578 579 err = 0;580 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))581 goto out_unlock;582 583 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);584 h = erofs_read_metabuf(&buf, sb, pos, EROFS_KMAP);585 if (IS_ERR(h)) {586 err = PTR_ERR(h);587 goto out_unlock;588 }589 590 /*591 * if the highest bit of the 8-byte map header is set, the whole file592 * is stored in the packed inode. The rest bits keeps z_fragmentoff.593 */594 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) {595 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;596 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63);597 vi->z_tailextent_headlcn = 0;598 goto done;599 }600 vi->z_advise = le16_to_cpu(h->h_advise);601 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;602 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;603 604 headnr = 0;605 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||606 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) {607 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel",608 headnr + 1, vi->z_algorithmtype[headnr], vi->nid);609 err = -EOPNOTSUPP;610 goto out_put_metabuf;611 }612 613 vi->z_logical_clusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 7);614 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&615 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |616 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {617 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",618 vi->nid);619 err = -EFSCORRUPTED;620 goto out_put_metabuf;621 }622 if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT &&623 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^624 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {625 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",626 vi->nid);627 err = -EFSCORRUPTED;628 goto out_put_metabuf;629 }630 631 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) {632 struct erofs_map_blocks map = {633 .buf = __EROFS_BUF_INITIALIZER634 };635 636 vi->z_idata_size = le16_to_cpu(h->h_idata_size);637 err = z_erofs_do_map_blocks(inode, &map,638 EROFS_GET_BLOCKS_FINDTAIL);639 erofs_put_metabuf(&map.buf);640 641 if (!map.m_plen ||642 erofs_blkoff(sb, map.m_pa) + map.m_plen > sb->s_blocksize) {643 erofs_err(sb, "invalid tail-packing pclustersize %llu",644 map.m_plen);645 err = -EFSCORRUPTED;646 }647 if (err < 0)648 goto out_put_metabuf;649 }650 651 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER &&652 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) {653 struct erofs_map_blocks map = {654 .buf = __EROFS_BUF_INITIALIZER655 };656 657 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff);658 err = z_erofs_do_map_blocks(inode, &map,659 EROFS_GET_BLOCKS_FINDTAIL);660 erofs_put_metabuf(&map.buf);661 if (err < 0)662 goto out_put_metabuf;663 }664done:665 /* paired with smp_mb() at the beginning of the function */666 smp_mb();667 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);668out_put_metabuf:669 erofs_put_metabuf(&buf);670out_unlock:671 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);672 return err;673}674 675int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,676 int flags)677{678 struct erofs_inode *const vi = EROFS_I(inode);679 int err = 0;680 681 trace_erofs_map_blocks_enter(inode, map, flags);682 if (map->m_la >= inode->i_size) { /* post-EOF unmapped extent */683 map->m_llen = map->m_la + 1 - inode->i_size;684 map->m_la = inode->i_size;685 map->m_flags = 0;686 } else {687 err = z_erofs_fill_inode_lazy(inode);688 if (!err) {689 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) &&690 !vi->z_tailextent_headlcn) {691 map->m_la = 0;692 map->m_llen = inode->i_size;693 map->m_flags = EROFS_MAP_MAPPED |694 EROFS_MAP_FULL_MAPPED | EROFS_MAP_FRAGMENT;695 } else {696 err = z_erofs_do_map_blocks(inode, map, flags);697 }698 }699 if (!err && (map->m_flags & EROFS_MAP_ENCODED) &&700 unlikely(map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE ||701 map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE))702 err = -EOPNOTSUPP;703 if (err)704 map->m_llen = 0;705 }706 trace_erofs_map_blocks_exit(inode, map, flags, err);707 return err;708}709 710static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,711 loff_t length, unsigned int flags,712 struct iomap *iomap, struct iomap *srcmap)713{714 int ret;715 struct erofs_map_blocks map = { .m_la = offset };716 717 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP);718 erofs_put_metabuf(&map.buf);719 if (ret < 0)720 return ret;721 722 iomap->bdev = inode->i_sb->s_bdev;723 iomap->offset = map.m_la;724 iomap->length = map.m_llen;725 if (map.m_flags & EROFS_MAP_MAPPED) {726 iomap->type = IOMAP_MAPPED;727 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ?728 IOMAP_NULL_ADDR : map.m_pa;729 } else {730 iomap->type = IOMAP_HOLE;731 iomap->addr = IOMAP_NULL_ADDR;732 /*733 * No strict rule on how to describe extents for post EOF, yet734 * we need to do like below. Otherwise, iomap itself will get735 * into an endless loop on post EOF.736 *737 * Calculate the effective offset by subtracting extent start738 * (map.m_la) from the requested offset, and add it to length.739 * (NB: offset >= map.m_la always)740 */741 if (iomap->offset >= inode->i_size)742 iomap->length = length + offset - map.m_la;743 }744 iomap->flags = 0;745 return 0;746}747 748const struct iomap_ops z_erofs_iomap_report_ops = {749 .iomap_begin = z_erofs_iomap_begin_report,750};751