194 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2#include <linux/zlib.h>3#include "compress.h"4 5struct z_erofs_deflate {6 struct z_erofs_deflate *next;7 struct z_stream_s z;8 u8 bounce[PAGE_SIZE];9};10 11static DEFINE_SPINLOCK(z_erofs_deflate_lock);12static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms;13static struct z_erofs_deflate *z_erofs_deflate_head;14static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq);15 16module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444);17 18static void z_erofs_deflate_exit(void)19{20 /* there should be no running fs instance */21 while (z_erofs_deflate_avail_strms) {22 struct z_erofs_deflate *strm;23 24 spin_lock(&z_erofs_deflate_lock);25 strm = z_erofs_deflate_head;26 if (!strm) {27 spin_unlock(&z_erofs_deflate_lock);28 continue;29 }30 z_erofs_deflate_head = NULL;31 spin_unlock(&z_erofs_deflate_lock);32 33 while (strm) {34 struct z_erofs_deflate *n = strm->next;35 36 vfree(strm->z.workspace);37 kfree(strm);38 --z_erofs_deflate_avail_strms;39 strm = n;40 }41 }42}43 44static int __init z_erofs_deflate_init(void)45{46 /* by default, use # of possible CPUs instead */47 if (!z_erofs_deflate_nstrms)48 z_erofs_deflate_nstrms = num_possible_cpus();49 return 0;50}51 52static int z_erofs_load_deflate_config(struct super_block *sb,53 struct erofs_super_block *dsb, void *data, int size)54{55 struct z_erofs_deflate_cfgs *dfl = data;56 static DEFINE_MUTEX(deflate_resize_mutex);57 static bool inited;58 59 if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) {60 erofs_err(sb, "invalid deflate cfgs, size=%u", size);61 return -EINVAL;62 }63 64 if (dfl->windowbits > MAX_WBITS) {65 erofs_err(sb, "unsupported windowbits %u", dfl->windowbits);66 return -EOPNOTSUPP;67 }68 mutex_lock(&deflate_resize_mutex);69 if (!inited) {70 for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms;71 ++z_erofs_deflate_avail_strms) {72 struct z_erofs_deflate *strm;73 74 strm = kzalloc(sizeof(*strm), GFP_KERNEL);75 if (!strm)76 goto failed;77 /* XXX: in-kernel zlib cannot customize windowbits */78 strm->z.workspace = vmalloc(zlib_inflate_workspacesize());79 if (!strm->z.workspace) {80 kfree(strm);81 goto failed;82 }83 84 spin_lock(&z_erofs_deflate_lock);85 strm->next = z_erofs_deflate_head;86 z_erofs_deflate_head = strm;87 spin_unlock(&z_erofs_deflate_lock);88 }89 inited = true;90 }91 mutex_unlock(&deflate_resize_mutex);92 erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!");93 return 0;94failed:95 mutex_unlock(&deflate_resize_mutex);96 z_erofs_deflate_exit();97 return -ENOMEM;98}99 100static int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,101 struct page **pgpl)102{103 struct super_block *sb = rq->sb;104 struct z_erofs_stream_dctx dctx = {105 .rq = rq,106 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,107 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)108 >> PAGE_SHIFT,109 .no = -1, .ni = 0,110 };111 struct z_erofs_deflate *strm;112 int zerr, err;113 114 /* 1. get the exact DEFLATE compressed size */115 dctx.kin = kmap_local_page(*rq->in);116 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,117 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));118 if (err) {119 kunmap_local(dctx.kin);120 return err;121 }122 123 /* 2. get an available DEFLATE context */124again:125 spin_lock(&z_erofs_deflate_lock);126 strm = z_erofs_deflate_head;127 if (!strm) {128 spin_unlock(&z_erofs_deflate_lock);129 wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head));130 goto again;131 }132 z_erofs_deflate_head = strm->next;133 spin_unlock(&z_erofs_deflate_lock);134 135 /* 3. multi-call decompress */136 zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS);137 if (zerr != Z_OK) {138 err = -EIO;139 goto failed_zinit;140 }141 142 rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */143 strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);144 rq->inputsize -= strm->z.avail_in;145 strm->z.next_in = dctx.kin + rq->pageofs_in;146 strm->z.avail_out = 0;147 dctx.bounce = strm->bounce;148 149 while (1) {150 dctx.avail_out = strm->z.avail_out;151 dctx.inbuf_sz = strm->z.avail_in;152 err = z_erofs_stream_switch_bufs(&dctx,153 (void **)&strm->z.next_out,154 (void **)&strm->z.next_in, pgpl);155 if (err)156 break;157 strm->z.avail_out = dctx.avail_out;158 strm->z.avail_in = dctx.inbuf_sz;159 160 zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH);161 if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) {162 if (zerr == Z_OK && rq->partial_decoding)163 break;164 if (zerr == Z_STREAM_END && !rq->outputsize)165 break;166 erofs_err(sb, "failed to decompress %d in[%u] out[%u]",167 zerr, rq->inputsize, rq->outputsize);168 err = -EFSCORRUPTED;169 break;170 }171 }172 if (zlib_inflateEnd(&strm->z) != Z_OK && !err)173 err = -EIO;174 if (dctx.kout)175 kunmap_local(dctx.kout);176failed_zinit:177 kunmap_local(dctx.kin);178 /* 4. push back DEFLATE stream context to the global list */179 spin_lock(&z_erofs_deflate_lock);180 strm->next = z_erofs_deflate_head;181 z_erofs_deflate_head = strm;182 spin_unlock(&z_erofs_deflate_lock);183 wake_up(&z_erofs_deflate_wq);184 return err;185}186 187const struct z_erofs_decompressor z_erofs_deflate_decomp = {188 .config = z_erofs_load_deflate_config,189 .decompress = z_erofs_deflate_decompress,190 .init = z_erofs_deflate_init,191 .exit = z_erofs_deflate_exit,192 .name = "deflate",193};194