226 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2#include <linux/zstd.h>3#include "compress.h"4 5struct z_erofs_zstd {6 struct z_erofs_zstd *next;7 u8 bounce[PAGE_SIZE];8 void *wksp;9 unsigned int wkspsz;10};11 12static DEFINE_SPINLOCK(z_erofs_zstd_lock);13static unsigned int z_erofs_zstd_max_dictsize;14static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;15static struct z_erofs_zstd *z_erofs_zstd_head;16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);17 18module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);19 20static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)21{22 struct z_erofs_zstd *strm;23 24again:25 spin_lock(&z_erofs_zstd_lock);26 strm = z_erofs_zstd_head;27 if (!strm) {28 spin_unlock(&z_erofs_zstd_lock);29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));30 goto again;31 }32 z_erofs_zstd_head = all ? NULL : strm->next;33 spin_unlock(&z_erofs_zstd_lock);34 return strm;35}36 37static void z_erofs_zstd_exit(void)38{39 while (z_erofs_zstd_avail_strms) {40 struct z_erofs_zstd *strm, *n;41 42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {43 n = strm->next;44 45 kvfree(strm->wksp);46 kfree(strm);47 --z_erofs_zstd_avail_strms;48 }49 }50}51 52static int __init z_erofs_zstd_init(void)53{54 /* by default, use # of possible CPUs instead */55 if (!z_erofs_zstd_nstrms)56 z_erofs_zstd_nstrms = num_possible_cpus();57 58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;59 ++z_erofs_zstd_avail_strms) {60 struct z_erofs_zstd *strm;61 62 strm = kzalloc(sizeof(*strm), GFP_KERNEL);63 if (!strm) {64 z_erofs_zstd_exit();65 return -ENOMEM;66 }67 spin_lock(&z_erofs_zstd_lock);68 strm->next = z_erofs_zstd_head;69 z_erofs_zstd_head = strm;70 spin_unlock(&z_erofs_zstd_lock);71 }72 return 0;73}74 75static int z_erofs_load_zstd_config(struct super_block *sb,76 struct erofs_super_block *dsb, void *data, int size)77{78 static DEFINE_MUTEX(zstd_resize_mutex);79 struct z_erofs_zstd_cfgs *zstd = data;80 unsigned int dict_size, wkspsz;81 struct z_erofs_zstd *strm, *head = NULL;82 void *wksp;83 84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {85 erofs_err(sb, "unsupported zstd format, size=%u", size);86 return -EINVAL;87 }88 89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);91 return -EINVAL;92 }93 dict_size = 1U << (zstd->windowlog + 10);94 95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */96 mutex_lock(&zstd_resize_mutex);97 if (z_erofs_zstd_max_dictsize >= dict_size) {98 mutex_unlock(&zstd_resize_mutex);99 return 0;100 }101 102 /* 1. collect/isolate all streams for the following check */103 while (z_erofs_zstd_avail_strms) {104 struct z_erofs_zstd *n;105 106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {107 n = strm->next;108 strm->next = head;109 head = strm;110 --z_erofs_zstd_avail_strms;111 }112 }113 114 /* 2. walk each isolated stream and grow max dict_size if needed */115 wkspsz = zstd_dstream_workspace_bound(dict_size);116 for (strm = head; strm; strm = strm->next) {117 wksp = kvmalloc(wkspsz, GFP_KERNEL);118 if (!wksp)119 break;120 kvfree(strm->wksp);121 strm->wksp = wksp;122 strm->wkspsz = wkspsz;123 }124 125 /* 3. push back all to the global list and update max dict_size */126 spin_lock(&z_erofs_zstd_lock);127 DBG_BUGON(z_erofs_zstd_head);128 z_erofs_zstd_head = head;129 spin_unlock(&z_erofs_zstd_lock);130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;131 wake_up_all(&z_erofs_zstd_wq);132 if (!strm)133 z_erofs_zstd_max_dictsize = dict_size;134 mutex_unlock(&zstd_resize_mutex);135 return strm ? -ENOMEM : 0;136}137 138static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,139 struct page **pgpl)140{141 struct super_block *sb = rq->sb;142 struct z_erofs_stream_dctx dctx = {143 .rq = rq,144 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,145 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)146 >> PAGE_SHIFT,147 .no = -1, .ni = 0,148 };149 zstd_in_buffer in_buf = { NULL, 0, 0 };150 zstd_out_buffer out_buf = { NULL, 0, 0 };151 struct z_erofs_zstd *strm;152 zstd_dstream *stream;153 int zerr, err;154 155 /* 1. get the exact compressed size */156 dctx.kin = kmap_local_page(*rq->in);157 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,158 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));159 if (err) {160 kunmap_local(dctx.kin);161 return err;162 }163 164 /* 2. get an available ZSTD context */165 strm = z_erofs_isolate_strms(false);166 167 /* 3. multi-call decompress */168 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);169 if (!stream) {170 err = -EIO;171 goto failed_zinit;172 }173 174 rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */175 in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);176 rq->inputsize -= in_buf.size;177 in_buf.src = dctx.kin + rq->pageofs_in;178 dctx.bounce = strm->bounce;179 180 do {181 dctx.avail_out = out_buf.size - out_buf.pos;182 dctx.inbuf_sz = in_buf.size;183 dctx.inbuf_pos = in_buf.pos;184 err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,185 (void **)&in_buf.src, pgpl);186 if (err)187 break;188 189 if (out_buf.size == out_buf.pos) {190 out_buf.size = dctx.avail_out;191 out_buf.pos = 0;192 }193 in_buf.size = dctx.inbuf_sz;194 in_buf.pos = dctx.inbuf_pos;195 196 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);197 if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) {198 erofs_err(sb, "failed to decompress in[%u] out[%u]: %s",199 rq->inputsize, rq->outputsize,200 zerr ? zstd_get_error_name(zerr) : "unexpected end of stream");201 err = -EFSCORRUPTED;202 break;203 }204 } while (rq->outputsize || out_buf.pos < out_buf.size);205 206 if (dctx.kout)207 kunmap_local(dctx.kout);208failed_zinit:209 kunmap_local(dctx.kin);210 /* 4. push back ZSTD stream context to the global list */211 spin_lock(&z_erofs_zstd_lock);212 strm->next = z_erofs_zstd_head;213 z_erofs_zstd_head = strm;214 spin_unlock(&z_erofs_zstd_lock);215 wake_up(&z_erofs_zstd_wq);216 return err;217}218 219const struct z_erofs_decompressor z_erofs_zstd_decomp = {220 .config = z_erofs_load_zstd_config,221 .decompress = z_erofs_zstd_decompress,222 .init = z_erofs_zstd_init,223 .exit = z_erofs_zstd_exit,224 .name = "zstd",225};226