431 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * This contains encryption functions for per-file encryption.4 *5 * Copyright (C) 2015, Google, Inc.6 * Copyright (C) 2015, Motorola Mobility7 *8 * Written by Michael Halcrow, 2014.9 *10 * Filename encryption additions11 * Uday Savagaonkar, 201412 * Encryption policy handling additions13 * Ildar Muslukhov, 201414 * Add fscrypt_pullback_bio_page()15 * Jaegeuk Kim, 2015.16 *17 * This has not yet undergone a rigorous security audit.18 *19 * The usage of AES-XTS should conform to recommendations in NIST20 * Special Publication 800-38E and IEEE P1619/D16.21 */22 23#include <linux/pagemap.h>24#include <linux/mempool.h>25#include <linux/module.h>26#include <linux/scatterlist.h>27#include <linux/ratelimit.h>28#include <crypto/skcipher.h>29#include "fscrypt_private.h"30 31static unsigned int num_prealloc_crypto_pages = 32;32 33module_param(num_prealloc_crypto_pages, uint, 0444);34MODULE_PARM_DESC(num_prealloc_crypto_pages,35 "Number of crypto pages to preallocate");36 37static mempool_t *fscrypt_bounce_page_pool = NULL;38 39static struct workqueue_struct *fscrypt_read_workqueue;40static DEFINE_MUTEX(fscrypt_init_mutex);41 42struct kmem_cache *fscrypt_inode_info_cachep;43 44void fscrypt_enqueue_decrypt_work(struct work_struct *work)45{46 queue_work(fscrypt_read_workqueue, work);47}48EXPORT_SYMBOL(fscrypt_enqueue_decrypt_work);49 50struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags)51{52 if (WARN_ON_ONCE(!fscrypt_bounce_page_pool)) {53 /*54 * Oops, the filesystem called a function that uses the bounce55 * page pool, but it didn't set needs_bounce_pages.56 */57 return NULL;58 }59 return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags);60}61 62/**63 * fscrypt_free_bounce_page() - free a ciphertext bounce page64 * @bounce_page: the bounce page to free, or NULL65 *66 * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(),67 * or by fscrypt_alloc_bounce_page() directly.68 */69void fscrypt_free_bounce_page(struct page *bounce_page)70{71 if (!bounce_page)72 return;73 set_page_private(bounce_page, (unsigned long)NULL);74 ClearPagePrivate(bounce_page);75 mempool_free(bounce_page, fscrypt_bounce_page_pool);76}77EXPORT_SYMBOL(fscrypt_free_bounce_page);78 79/*80 * Generate the IV for the given data unit index within the given file.81 * For filenames encryption, index == 0.82 *83 * Keep this in sync with fscrypt_limit_io_blocks(). fscrypt_limit_io_blocks()84 * needs to know about any IV generation methods where the low bits of IV don't85 * simply contain the data unit index (e.g., IV_INO_LBLK_32).86 */87void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index,88 const struct fscrypt_inode_info *ci)89{90 u8 flags = fscrypt_policy_flags(&ci->ci_policy);91 92 memset(iv, 0, ci->ci_mode->ivsize);93 94 if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) {95 WARN_ON_ONCE(index > U32_MAX);96 WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX);97 index |= (u64)ci->ci_inode->i_ino << 32;98 } else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) {99 WARN_ON_ONCE(index > U32_MAX);100 index = (u32)(ci->ci_hashed_ino + index);101 } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) {102 memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE);103 }104 iv->index = cpu_to_le64(index);105}106 107/* Encrypt or decrypt a single "data unit" of file contents. */108int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci,109 fscrypt_direction_t rw, u64 index,110 struct page *src_page, struct page *dest_page,111 unsigned int len, unsigned int offs,112 gfp_t gfp_flags)113{114 union fscrypt_iv iv;115 struct skcipher_request *req = NULL;116 DECLARE_CRYPTO_WAIT(wait);117 struct scatterlist dst, src;118 struct crypto_skcipher *tfm = ci->ci_enc_key.tfm;119 int res = 0;120 121 if (WARN_ON_ONCE(len <= 0))122 return -EINVAL;123 if (WARN_ON_ONCE(len % FSCRYPT_CONTENTS_ALIGNMENT != 0))124 return -EINVAL;125 126 fscrypt_generate_iv(&iv, index, ci);127 128 req = skcipher_request_alloc(tfm, gfp_flags);129 if (!req)130 return -ENOMEM;131 132 skcipher_request_set_callback(133 req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,134 crypto_req_done, &wait);135 136 sg_init_table(&dst, 1);137 sg_set_page(&dst, dest_page, len, offs);138 sg_init_table(&src, 1);139 sg_set_page(&src, src_page, len, offs);140 skcipher_request_set_crypt(req, &src, &dst, len, &iv);141 if (rw == FS_DECRYPT)142 res = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);143 else144 res = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);145 skcipher_request_free(req);146 if (res) {147 fscrypt_err(ci->ci_inode,148 "%scryption failed for data unit %llu: %d",149 (rw == FS_DECRYPT ? "De" : "En"), index, res);150 return res;151 }152 return 0;153}154 155/**156 * fscrypt_encrypt_pagecache_blocks() - Encrypt data from a pagecache page157 * @page: the locked pagecache page containing the data to encrypt158 * @len: size of the data to encrypt, in bytes159 * @offs: offset within @page of the data to encrypt, in bytes160 * @gfp_flags: memory allocation flags; see details below161 *162 * This allocates a new bounce page and encrypts the given data into it. The163 * length and offset of the data must be aligned to the file's crypto data unit164 * size. Alignment to the filesystem block size fulfills this requirement, as165 * the filesystem block size is always a multiple of the data unit size.166 *167 * In the bounce page, the ciphertext data will be located at the same offset at168 * which the plaintext data was located in the source page. Any other parts of169 * the bounce page will be left uninitialized.170 *171 * This is for use by the filesystem's ->writepages() method.172 *173 * The bounce page allocation is mempool-backed, so it will always succeed when174 * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However,175 * only the first page of each bio can be allocated this way. To prevent176 * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used.177 *178 * Return: the new encrypted bounce page on success; an ERR_PTR() on failure179 */180struct page *fscrypt_encrypt_pagecache_blocks(struct page *page,181 unsigned int len,182 unsigned int offs,183 gfp_t gfp_flags)184 185{186 const struct inode *inode = page->mapping->host;187 const struct fscrypt_inode_info *ci = inode->i_crypt_info;188 const unsigned int du_bits = ci->ci_data_unit_bits;189 const unsigned int du_size = 1U << du_bits;190 struct page *ciphertext_page;191 u64 index = ((u64)page->index << (PAGE_SHIFT - du_bits)) +192 (offs >> du_bits);193 unsigned int i;194 int err;195 196 if (WARN_ON_ONCE(!PageLocked(page)))197 return ERR_PTR(-EINVAL);198 199 if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size)))200 return ERR_PTR(-EINVAL);201 202 ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags);203 if (!ciphertext_page)204 return ERR_PTR(-ENOMEM);205 206 for (i = offs; i < offs + len; i += du_size, index++) {207 err = fscrypt_crypt_data_unit(ci, FS_ENCRYPT, index,208 page, ciphertext_page,209 du_size, i, gfp_flags);210 if (err) {211 fscrypt_free_bounce_page(ciphertext_page);212 return ERR_PTR(err);213 }214 }215 SetPagePrivate(ciphertext_page);216 set_page_private(ciphertext_page, (unsigned long)page);217 return ciphertext_page;218}219EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks);220 221/**222 * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place223 * @inode: The inode to which this block belongs224 * @page: The page containing the block to encrypt225 * @len: Size of block to encrypt. This must be a multiple of226 * FSCRYPT_CONTENTS_ALIGNMENT.227 * @offs: Byte offset within @page at which the block to encrypt begins228 * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based229 * number of the block within the file230 * @gfp_flags: Memory allocation flags231 *232 * Encrypt a possibly-compressed filesystem block that is located in an233 * arbitrary page, not necessarily in the original pagecache page. The @inode234 * and @lblk_num must be specified, as they can't be determined from @page.235 *236 * This is not compatible with fscrypt_operations::supports_subblock_data_units.237 *238 * Return: 0 on success; -errno on failure239 */240int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page,241 unsigned int len, unsigned int offs,242 u64 lblk_num, gfp_t gfp_flags)243{244 if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units))245 return -EOPNOTSUPP;246 return fscrypt_crypt_data_unit(inode->i_crypt_info, FS_ENCRYPT,247 lblk_num, page, page, len, offs,248 gfp_flags);249}250EXPORT_SYMBOL(fscrypt_encrypt_block_inplace);251 252/**253 * fscrypt_decrypt_pagecache_blocks() - Decrypt data from a pagecache folio254 * @folio: the pagecache folio containing the data to decrypt255 * @len: size of the data to decrypt, in bytes256 * @offs: offset within @folio of the data to decrypt, in bytes257 *258 * Decrypt data that has just been read from an encrypted file. The data must259 * be located in a pagecache folio that is still locked and not yet uptodate.260 * The length and offset of the data must be aligned to the file's crypto data261 * unit size. Alignment to the filesystem block size fulfills this requirement,262 * as the filesystem block size is always a multiple of the data unit size.263 *264 * Return: 0 on success; -errno on failure265 */266int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len,267 size_t offs)268{269 const struct inode *inode = folio->mapping->host;270 const struct fscrypt_inode_info *ci = inode->i_crypt_info;271 const unsigned int du_bits = ci->ci_data_unit_bits;272 const unsigned int du_size = 1U << du_bits;273 u64 index = ((u64)folio->index << (PAGE_SHIFT - du_bits)) +274 (offs >> du_bits);275 size_t i;276 int err;277 278 if (WARN_ON_ONCE(!folio_test_locked(folio)))279 return -EINVAL;280 281 if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size)))282 return -EINVAL;283 284 for (i = offs; i < offs + len; i += du_size, index++) {285 struct page *page = folio_page(folio, i >> PAGE_SHIFT);286 287 err = fscrypt_crypt_data_unit(ci, FS_DECRYPT, index, page,288 page, du_size, i & ~PAGE_MASK,289 GFP_NOFS);290 if (err)291 return err;292 }293 return 0;294}295EXPORT_SYMBOL(fscrypt_decrypt_pagecache_blocks);296 297/**298 * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place299 * @inode: The inode to which this block belongs300 * @page: The page containing the block to decrypt301 * @len: Size of block to decrypt. This must be a multiple of302 * FSCRYPT_CONTENTS_ALIGNMENT.303 * @offs: Byte offset within @page at which the block to decrypt begins304 * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based305 * number of the block within the file306 *307 * Decrypt a possibly-compressed filesystem block that is located in an308 * arbitrary page, not necessarily in the original pagecache page. The @inode309 * and @lblk_num must be specified, as they can't be determined from @page.310 *311 * This is not compatible with fscrypt_operations::supports_subblock_data_units.312 *313 * Return: 0 on success; -errno on failure314 */315int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page,316 unsigned int len, unsigned int offs,317 u64 lblk_num)318{319 if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units))320 return -EOPNOTSUPP;321 return fscrypt_crypt_data_unit(inode->i_crypt_info, FS_DECRYPT,322 lblk_num, page, page, len, offs,323 GFP_NOFS);324}325EXPORT_SYMBOL(fscrypt_decrypt_block_inplace);326 327/**328 * fscrypt_initialize() - allocate major buffers for fs encryption.329 * @sb: the filesystem superblock330 *331 * We only call this when we start accessing encrypted files, since it332 * results in memory getting allocated that wouldn't otherwise be used.333 *334 * Return: 0 on success; -errno on failure335 */336int fscrypt_initialize(struct super_block *sb)337{338 int err = 0;339 mempool_t *pool;340 341 /* pairs with smp_store_release() below */342 if (likely(smp_load_acquire(&fscrypt_bounce_page_pool)))343 return 0;344 345 /* No need to allocate a bounce page pool if this FS won't use it. */346 if (!sb->s_cop->needs_bounce_pages)347 return 0;348 349 mutex_lock(&fscrypt_init_mutex);350 if (fscrypt_bounce_page_pool)351 goto out_unlock;352 353 err = -ENOMEM;354 pool = mempool_create_page_pool(num_prealloc_crypto_pages, 0);355 if (!pool)356 goto out_unlock;357 /* pairs with smp_load_acquire() above */358 smp_store_release(&fscrypt_bounce_page_pool, pool);359 err = 0;360out_unlock:361 mutex_unlock(&fscrypt_init_mutex);362 return err;363}364 365void fscrypt_msg(const struct inode *inode, const char *level,366 const char *fmt, ...)367{368 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,369 DEFAULT_RATELIMIT_BURST);370 struct va_format vaf;371 va_list args;372 373 if (!__ratelimit(&rs))374 return;375 376 va_start(args, fmt);377 vaf.fmt = fmt;378 vaf.va = &args;379 if (inode && inode->i_ino)380 printk("%sfscrypt (%s, inode %lu): %pV\n",381 level, inode->i_sb->s_id, inode->i_ino, &vaf);382 else if (inode)383 printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf);384 else385 printk("%sfscrypt: %pV\n", level, &vaf);386 va_end(args);387}388 389/**390 * fscrypt_init() - Set up for fs encryption.391 *392 * Return: 0 on success; -errno on failure393 */394static int __init fscrypt_init(void)395{396 int err = -ENOMEM;397 398 /*399 * Use an unbound workqueue to allow bios to be decrypted in parallel400 * even when they happen to complete on the same CPU. This sacrifices401 * locality, but it's worthwhile since decryption is CPU-intensive.402 *403 * Also use a high-priority workqueue to prioritize decryption work,404 * which blocks reads from completing, over regular application tasks.405 */406 fscrypt_read_workqueue = alloc_workqueue("fscrypt_read_queue",407 WQ_UNBOUND | WQ_HIGHPRI,408 num_online_cpus());409 if (!fscrypt_read_workqueue)410 goto fail;411 412 fscrypt_inode_info_cachep = KMEM_CACHE(fscrypt_inode_info,413 SLAB_RECLAIM_ACCOUNT);414 if (!fscrypt_inode_info_cachep)415 goto fail_free_queue;416 417 err = fscrypt_init_keyring();418 if (err)419 goto fail_free_inode_info;420 421 return 0;422 423fail_free_inode_info:424 kmem_cache_destroy(fscrypt_inode_info_cachep);425fail_free_queue:426 destroy_workqueue(fscrypt_read_workqueue);427fail:428 return err;429}430late_initcall(fscrypt_init)431