469 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2008 Oracle. All rights reserved.4 *5 * Based on jffs2 zlib code:6 * Copyright © 2001-2007 Red Hat, Inc.7 * Created by David Woodhouse <dwmw2@infradead.org>8 */9 10#include <linux/kernel.h>11#include <linux/slab.h>12#include <linux/zlib.h>13#include <linux/zutil.h>14#include <linux/mm.h>15#include <linux/init.h>16#include <linux/err.h>17#include <linux/sched.h>18#include <linux/pagemap.h>19#include <linux/bio.h>20#include <linux/refcount.h>21#include "btrfs_inode.h"22#include "compression.h"23#include "fs.h"24#include "subpage.h"25 26/* workspace buffer size for s390 zlib hardware support */27#define ZLIB_DFLTCC_BUF_SIZE (4 * PAGE_SIZE)28 29struct workspace {30 z_stream strm;31 char *buf;32 unsigned int buf_size;33 struct list_head list;34 int level;35};36 37static struct workspace_manager wsm;38 39struct list_head *zlib_get_workspace(unsigned int level)40{41 struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level);42 struct workspace *workspace = list_entry(ws, struct workspace, list);43 44 workspace->level = level;45 46 return ws;47}48 49void zlib_free_workspace(struct list_head *ws)50{51 struct workspace *workspace = list_entry(ws, struct workspace, list);52 53 kvfree(workspace->strm.workspace);54 kfree(workspace->buf);55 kfree(workspace);56}57 58struct list_head *zlib_alloc_workspace(unsigned int level)59{60 struct workspace *workspace;61 int workspacesize;62 63 workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);64 if (!workspace)65 return ERR_PTR(-ENOMEM);66 67 workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),68 zlib_inflate_workspacesize());69 workspace->strm.workspace = kvzalloc(workspacesize, GFP_KERNEL | __GFP_NOWARN);70 workspace->level = level;71 workspace->buf = NULL;72 /*73 * In case of s390 zlib hardware support, allocate lager workspace74 * buffer. If allocator fails, fall back to a single page buffer.75 */76 if (zlib_deflate_dfltcc_enabled()) {77 workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE,78 __GFP_NOMEMALLOC | __GFP_NORETRY |79 __GFP_NOWARN | GFP_NOIO);80 workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE;81 }82 if (!workspace->buf) {83 workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);84 workspace->buf_size = PAGE_SIZE;85 }86 if (!workspace->strm.workspace || !workspace->buf)87 goto fail;88 89 INIT_LIST_HEAD(&workspace->list);90 91 return &workspace->list;92fail:93 zlib_free_workspace(&workspace->list);94 return ERR_PTR(-ENOMEM);95}96 97int zlib_compress_folios(struct list_head *ws, struct address_space *mapping,98 u64 start, struct folio **folios, unsigned long *out_folios,99 unsigned long *total_in, unsigned long *total_out)100{101 struct workspace *workspace = list_entry(ws, struct workspace, list);102 int ret;103 char *data_in = NULL;104 char *cfolio_out;105 int nr_folios = 0;106 struct folio *in_folio = NULL;107 struct folio *out_folio = NULL;108 unsigned long bytes_left;109 unsigned int in_buf_folios;110 unsigned long len = *total_out;111 unsigned long nr_dest_folios = *out_folios;112 const unsigned long max_out = nr_dest_folios * PAGE_SIZE;113 const u64 orig_end = start + len;114 115 *out_folios = 0;116 *total_out = 0;117 *total_in = 0;118 119 ret = zlib_deflateInit(&workspace->strm, workspace->level);120 if (unlikely(ret != Z_OK)) {121 struct btrfs_inode *inode = BTRFS_I(mapping->host);122 123 btrfs_err(inode->root->fs_info,124 "zlib compression init failed, error %d root %llu inode %llu offset %llu",125 ret, btrfs_root_id(inode->root), btrfs_ino(inode), start);126 ret = -EIO;127 goto out;128 }129 130 workspace->strm.total_in = 0;131 workspace->strm.total_out = 0;132 133 out_folio = btrfs_alloc_compr_folio();134 if (out_folio == NULL) {135 ret = -ENOMEM;136 goto out;137 }138 cfolio_out = folio_address(out_folio);139 folios[0] = out_folio;140 nr_folios = 1;141 142 workspace->strm.next_in = workspace->buf;143 workspace->strm.avail_in = 0;144 workspace->strm.next_out = cfolio_out;145 workspace->strm.avail_out = PAGE_SIZE;146 147 while (workspace->strm.total_in < len) {148 /*149 * Get next input pages and copy the contents to150 * the workspace buffer if required.151 */152 if (workspace->strm.avail_in == 0) {153 bytes_left = len - workspace->strm.total_in;154 in_buf_folios = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE),155 workspace->buf_size / PAGE_SIZE);156 if (in_buf_folios > 1) {157 int i;158 159 /* S390 hardware acceleration path, not subpage. */160 ASSERT(!btrfs_is_subpage(161 inode_to_fs_info(mapping->host),162 mapping));163 for (i = 0; i < in_buf_folios; i++) {164 if (data_in) {165 kunmap_local(data_in);166 folio_put(in_folio);167 data_in = NULL;168 }169 ret = btrfs_compress_filemap_get_folio(mapping,170 start, &in_folio);171 if (ret < 0)172 goto out;173 data_in = kmap_local_folio(in_folio, 0);174 copy_page(workspace->buf + i * PAGE_SIZE,175 data_in);176 start += PAGE_SIZE;177 workspace->strm.avail_in =178 (in_buf_folios << PAGE_SHIFT);179 }180 workspace->strm.next_in = workspace->buf;181 } else {182 unsigned int pg_off;183 unsigned int cur_len;184 185 if (data_in) {186 kunmap_local(data_in);187 folio_put(in_folio);188 data_in = NULL;189 }190 ret = btrfs_compress_filemap_get_folio(mapping,191 start, &in_folio);192 if (ret < 0)193 goto out;194 pg_off = offset_in_page(start);195 cur_len = btrfs_calc_input_length(orig_end, start);196 data_in = kmap_local_folio(in_folio, pg_off);197 start += PAGE_SIZE;198 workspace->strm.next_in = data_in;199 workspace->strm.avail_in = cur_len;200 }201 }202 203 ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH);204 if (unlikely(ret != Z_OK)) {205 struct btrfs_inode *inode = BTRFS_I(mapping->host);206 207 btrfs_warn(inode->root->fs_info,208 "zlib compression failed, error %d root %llu inode %llu offset %llu",209 ret, btrfs_root_id(inode->root), btrfs_ino(inode),210 start);211 zlib_deflateEnd(&workspace->strm);212 ret = -EIO;213 goto out;214 }215 216 /* we're making it bigger, give up */217 if (workspace->strm.total_in > 8192 &&218 workspace->strm.total_in <219 workspace->strm.total_out) {220 ret = -E2BIG;221 goto out;222 }223 /* we need another page for writing out. Test this224 * before the total_in so we will pull in a new page for225 * the stream end if required226 */227 if (workspace->strm.avail_out == 0) {228 if (nr_folios == nr_dest_folios) {229 ret = -E2BIG;230 goto out;231 }232 out_folio = btrfs_alloc_compr_folio();233 if (out_folio == NULL) {234 ret = -ENOMEM;235 goto out;236 }237 cfolio_out = folio_address(out_folio);238 folios[nr_folios] = out_folio;239 nr_folios++;240 workspace->strm.avail_out = PAGE_SIZE;241 workspace->strm.next_out = cfolio_out;242 }243 /* we're all done */244 if (workspace->strm.total_in >= len)245 break;246 if (workspace->strm.total_out > max_out)247 break;248 }249 workspace->strm.avail_in = 0;250 /*251 * Call deflate with Z_FINISH flush parameter providing more output252 * space but no more input data, until it returns with Z_STREAM_END.253 */254 while (ret != Z_STREAM_END) {255 ret = zlib_deflate(&workspace->strm, Z_FINISH);256 if (ret == Z_STREAM_END)257 break;258 if (ret != Z_OK && ret != Z_BUF_ERROR) {259 zlib_deflateEnd(&workspace->strm);260 ret = -EIO;261 goto out;262 } else if (workspace->strm.avail_out == 0) {263 /* Get another folio for the stream end. */264 if (nr_folios == nr_dest_folios) {265 ret = -E2BIG;266 goto out;267 }268 out_folio = btrfs_alloc_compr_folio();269 if (out_folio == NULL) {270 ret = -ENOMEM;271 goto out;272 }273 cfolio_out = folio_address(out_folio);274 folios[nr_folios] = out_folio;275 nr_folios++;276 workspace->strm.avail_out = PAGE_SIZE;277 workspace->strm.next_out = cfolio_out;278 }279 }280 zlib_deflateEnd(&workspace->strm);281 282 if (workspace->strm.total_out >= workspace->strm.total_in) {283 ret = -E2BIG;284 goto out;285 }286 287 ret = 0;288 *total_out = workspace->strm.total_out;289 *total_in = workspace->strm.total_in;290out:291 *out_folios = nr_folios;292 if (data_in) {293 kunmap_local(data_in);294 folio_put(in_folio);295 }296 297 return ret;298}299 300int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)301{302 struct workspace *workspace = list_entry(ws, struct workspace, list);303 int ret = 0, ret2;304 int wbits = MAX_WBITS;305 char *data_in;306 size_t total_out = 0;307 unsigned long folio_in_index = 0;308 size_t srclen = cb->compressed_len;309 unsigned long total_folios_in = DIV_ROUND_UP(srclen, PAGE_SIZE);310 unsigned long buf_start;311 struct folio **folios_in = cb->compressed_folios;312 313 data_in = kmap_local_folio(folios_in[folio_in_index], 0);314 workspace->strm.next_in = data_in;315 workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);316 workspace->strm.total_in = 0;317 318 workspace->strm.total_out = 0;319 workspace->strm.next_out = workspace->buf;320 workspace->strm.avail_out = workspace->buf_size;321 322 /* If it's deflate, and it's got no preset dictionary, then323 we can tell zlib to skip the adler32 check. */324 if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&325 ((data_in[0] & 0x0f) == Z_DEFLATED) &&326 !(((data_in[0]<<8) + data_in[1]) % 31)) {327 328 wbits = -((data_in[0] >> 4) + 8);329 workspace->strm.next_in += 2;330 workspace->strm.avail_in -= 2;331 }332 333 ret = zlib_inflateInit2(&workspace->strm, wbits);334 if (unlikely(ret != Z_OK)) {335 struct btrfs_inode *inode = cb->bbio.inode;336 337 kunmap_local(data_in);338 btrfs_err(inode->root->fs_info,339 "zlib decompression init failed, error %d root %llu inode %llu offset %llu",340 ret, btrfs_root_id(inode->root), btrfs_ino(inode), cb->start);341 return -EIO;342 }343 while (workspace->strm.total_in < srclen) {344 ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);345 if (ret != Z_OK && ret != Z_STREAM_END)346 break;347 348 buf_start = total_out;349 total_out = workspace->strm.total_out;350 351 /* we didn't make progress in this inflate call, we're done */352 if (buf_start == total_out)353 break;354 355 ret2 = btrfs_decompress_buf2page(workspace->buf,356 total_out - buf_start, cb, buf_start);357 if (ret2 == 0) {358 ret = 0;359 goto done;360 }361 362 workspace->strm.next_out = workspace->buf;363 workspace->strm.avail_out = workspace->buf_size;364 365 if (workspace->strm.avail_in == 0) {366 unsigned long tmp;367 kunmap_local(data_in);368 folio_in_index++;369 if (folio_in_index >= total_folios_in) {370 data_in = NULL;371 break;372 }373 data_in = kmap_local_folio(folios_in[folio_in_index], 0);374 workspace->strm.next_in = data_in;375 tmp = srclen - workspace->strm.total_in;376 workspace->strm.avail_in = min(tmp, PAGE_SIZE);377 }378 }379 if (unlikely(ret != Z_STREAM_END)) {380 btrfs_err(cb->bbio.inode->root->fs_info,381 "zlib decompression failed, error %d root %llu inode %llu offset %llu",382 ret, btrfs_root_id(cb->bbio.inode->root),383 btrfs_ino(cb->bbio.inode), cb->start);384 ret = -EIO;385 } else {386 ret = 0;387 }388done:389 zlib_inflateEnd(&workspace->strm);390 if (data_in)391 kunmap_local(data_in);392 return ret;393}394 395int zlib_decompress(struct list_head *ws, const u8 *data_in,396 struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,397 size_t destlen)398{399 struct workspace *workspace = list_entry(ws, struct workspace, list);400 int ret = 0;401 int wbits = MAX_WBITS;402 unsigned long to_copy;403 404 workspace->strm.next_in = data_in;405 workspace->strm.avail_in = srclen;406 workspace->strm.total_in = 0;407 408 workspace->strm.next_out = workspace->buf;409 workspace->strm.avail_out = workspace->buf_size;410 workspace->strm.total_out = 0;411 /* If it's deflate, and it's got no preset dictionary, then412 we can tell zlib to skip the adler32 check. */413 if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&414 ((data_in[0] & 0x0f) == Z_DEFLATED) &&415 !(((data_in[0]<<8) + data_in[1]) % 31)) {416 417 wbits = -((data_in[0] >> 4) + 8);418 workspace->strm.next_in += 2;419 workspace->strm.avail_in -= 2;420 }421 422 ret = zlib_inflateInit2(&workspace->strm, wbits);423 if (unlikely(ret != Z_OK)) {424 struct btrfs_inode *inode = folio_to_inode(dest_folio);425 426 btrfs_err(inode->root->fs_info,427 "zlib decompression init failed, error %d root %llu inode %llu offset %llu",428 ret, btrfs_root_id(inode->root), btrfs_ino(inode),429 folio_pos(dest_folio));430 return -EIO;431 }432 433 /*434 * Everything (in/out buf) should be at most one sector, there should435 * be no need to switch any input/output buffer.436 */437 ret = zlib_inflate(&workspace->strm, Z_FINISH);438 to_copy = min(workspace->strm.total_out, destlen);439 if (ret != Z_STREAM_END)440 goto out;441 442 memcpy_to_folio(dest_folio, dest_pgoff, workspace->buf, to_copy);443 444out:445 if (unlikely(to_copy != destlen)) {446 struct btrfs_inode *inode = folio_to_inode(dest_folio);447 448 btrfs_err(inode->root->fs_info,449"zlib decompression failed, error %d root %llu inode %llu offset %llu decompressed %lu expected %zu",450 ret, btrfs_root_id(inode->root), btrfs_ino(inode),451 folio_pos(dest_folio), to_copy, destlen);452 ret = -EIO;453 } else {454 ret = 0;455 }456 457 zlib_inflateEnd(&workspace->strm);458 459 if (unlikely(to_copy < destlen))460 folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy);461 return ret;462}463 464const struct btrfs_compress_op btrfs_zlib_compress = {465 .workspace_manager = &wsm,466 .max_level = 9,467 .default_level = BTRFS_ZLIB_DEFAULT_LEVEL,468};469