242 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2#include <linux/xz.h>3#include "compress.h"4 5struct z_erofs_lzma {6 struct z_erofs_lzma *next;7 struct xz_dec_microlzma *state;8 u8 bounce[PAGE_SIZE];9};10 11/* considering the LZMA performance, no need to use a lockless list for now */12static DEFINE_SPINLOCK(z_erofs_lzma_lock);13static unsigned int z_erofs_lzma_max_dictsize;14static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;15static struct z_erofs_lzma *z_erofs_lzma_head;16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);17 18module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);19 20static void z_erofs_lzma_exit(void)21{22 /* there should be no running fs instance */23 while (z_erofs_lzma_avail_strms) {24 struct z_erofs_lzma *strm;25 26 spin_lock(&z_erofs_lzma_lock);27 strm = z_erofs_lzma_head;28 if (!strm) {29 spin_unlock(&z_erofs_lzma_lock);30 DBG_BUGON(1);31 return;32 }33 z_erofs_lzma_head = NULL;34 spin_unlock(&z_erofs_lzma_lock);35 36 while (strm) {37 struct z_erofs_lzma *n = strm->next;38 39 if (strm->state)40 xz_dec_microlzma_end(strm->state);41 kfree(strm);42 --z_erofs_lzma_avail_strms;43 strm = n;44 }45 }46}47 48static int __init z_erofs_lzma_init(void)49{50 unsigned int i;51 52 /* by default, use # of possible CPUs instead */53 if (!z_erofs_lzma_nstrms)54 z_erofs_lzma_nstrms = num_possible_cpus();55 56 for (i = 0; i < z_erofs_lzma_nstrms; ++i) {57 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);58 59 if (!strm) {60 z_erofs_lzma_exit();61 return -ENOMEM;62 }63 spin_lock(&z_erofs_lzma_lock);64 strm->next = z_erofs_lzma_head;65 z_erofs_lzma_head = strm;66 spin_unlock(&z_erofs_lzma_lock);67 ++z_erofs_lzma_avail_strms;68 }69 return 0;70}71 72static int z_erofs_load_lzma_config(struct super_block *sb,73 struct erofs_super_block *dsb, void *data, int size)74{75 static DEFINE_MUTEX(lzma_resize_mutex);76 struct z_erofs_lzma_cfgs *lzma = data;77 unsigned int dict_size, i;78 struct z_erofs_lzma *strm, *head = NULL;79 int err;80 81 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {82 erofs_err(sb, "invalid lzma cfgs, size=%u", size);83 return -EINVAL;84 }85 if (lzma->format) {86 erofs_err(sb, "unidentified lzma format %x, please check kernel version",87 le16_to_cpu(lzma->format));88 return -EINVAL;89 }90 dict_size = le32_to_cpu(lzma->dict_size);91 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {92 erofs_err(sb, "unsupported lzma dictionary size %u",93 dict_size);94 return -EINVAL;95 }96 97 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */98 mutex_lock(&lzma_resize_mutex);99 100 if (z_erofs_lzma_max_dictsize >= dict_size) {101 mutex_unlock(&lzma_resize_mutex);102 return 0;103 }104 105 /* 1. collect/isolate all streams for the following check */106 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {107 struct z_erofs_lzma *last;108 109again:110 spin_lock(&z_erofs_lzma_lock);111 strm = z_erofs_lzma_head;112 if (!strm) {113 spin_unlock(&z_erofs_lzma_lock);114 wait_event(z_erofs_lzma_wq,115 READ_ONCE(z_erofs_lzma_head));116 goto again;117 }118 z_erofs_lzma_head = NULL;119 spin_unlock(&z_erofs_lzma_lock);120 121 for (last = strm; last->next; last = last->next)122 ++i;123 last->next = head;124 head = strm;125 }126 127 err = 0;128 /* 2. walk each isolated stream and grow max dict_size if needed */129 for (strm = head; strm; strm = strm->next) {130 if (strm->state)131 xz_dec_microlzma_end(strm->state);132 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);133 if (!strm->state)134 err = -ENOMEM;135 }136 137 /* 3. push back all to the global list and update max dict_size */138 spin_lock(&z_erofs_lzma_lock);139 DBG_BUGON(z_erofs_lzma_head);140 z_erofs_lzma_head = head;141 spin_unlock(&z_erofs_lzma_lock);142 wake_up_all(&z_erofs_lzma_wq);143 144 z_erofs_lzma_max_dictsize = dict_size;145 mutex_unlock(&lzma_resize_mutex);146 return err;147}148 149static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,150 struct page **pgpl)151{152 struct super_block *sb = rq->sb;153 struct z_erofs_stream_dctx dctx = {154 .rq = rq,155 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,156 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)157 >> PAGE_SHIFT,158 .no = -1, .ni = 0,159 };160 struct xz_buf buf = {};161 struct z_erofs_lzma *strm;162 enum xz_ret xz_err;163 int err;164 165 /* 1. get the exact LZMA compressed size */166 dctx.kin = kmap_local_page(*rq->in);167 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,168 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));169 if (err) {170 kunmap_local(dctx.kin);171 return err;172 }173 174 /* 2. get an available lzma context */175again:176 spin_lock(&z_erofs_lzma_lock);177 strm = z_erofs_lzma_head;178 if (!strm) {179 spin_unlock(&z_erofs_lzma_lock);180 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));181 goto again;182 }183 z_erofs_lzma_head = strm->next;184 spin_unlock(&z_erofs_lzma_lock);185 186 /* 3. multi-call decompress */187 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,188 !rq->partial_decoding);189 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);190 rq->inputsize -= buf.in_size;191 buf.in = dctx.kin + rq->pageofs_in;192 dctx.bounce = strm->bounce;193 do {194 dctx.avail_out = buf.out_size - buf.out_pos;195 dctx.inbuf_sz = buf.in_size;196 dctx.inbuf_pos = buf.in_pos;197 err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,198 (void **)&buf.in, pgpl);199 if (err)200 break;201 202 if (buf.out_size == buf.out_pos) {203 buf.out_size = dctx.avail_out;204 buf.out_pos = 0;205 }206 buf.in_size = dctx.inbuf_sz;207 buf.in_pos = dctx.inbuf_pos;208 209 xz_err = xz_dec_microlzma_run(strm->state, &buf);210 DBG_BUGON(buf.out_pos > buf.out_size);211 DBG_BUGON(buf.in_pos > buf.in_size);212 213 if (xz_err != XZ_OK) {214 if (xz_err == XZ_STREAM_END && !rq->outputsize)215 break;216 erofs_err(sb, "failed to decompress %d in[%u] out[%u]",217 xz_err, rq->inputsize, rq->outputsize);218 err = -EFSCORRUPTED;219 break;220 }221 } while (1);222 223 if (dctx.kout)224 kunmap_local(dctx.kout);225 kunmap_local(dctx.kin);226 /* 4. push back LZMA stream context to the global list */227 spin_lock(&z_erofs_lzma_lock);228 strm->next = z_erofs_lzma_head;229 z_erofs_lzma_head = strm;230 spin_unlock(&z_erofs_lzma_lock);231 wake_up(&z_erofs_lzma_wq);232 return err;233}234 235const struct z_erofs_decompressor z_erofs_lzma_decomp = {236 .config = z_erofs_load_lzma_config,237 .decompress = z_erofs_lzma_decompress,238 .init = z_erofs_lzma_init,239 .exit = z_erofs_lzma_exit,240 .name = "lzma"241};242