613 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright(c) 2017 Intel Corporation. All rights reserved.4 */5#include <linux/pagemap.h>6#include <linux/module.h>7#include <linux/mount.h>8#include <linux/pseudo_fs.h>9#include <linux/magic.h>10#include <linux/pfn_t.h>11#include <linux/cdev.h>12#include <linux/slab.h>13#include <linux/uio.h>14#include <linux/dax.h>15#include <linux/fs.h>16#include <linux/cacheinfo.h>17#include "dax-private.h"18 19/**20 * struct dax_device - anchor object for dax services21 * @inode: core vfs22 * @cdev: optional character interface for "device dax"23 * @private: dax driver private data24 * @flags: state and boolean properties25 * @ops: operations for this device26 * @holder_data: holder of a dax_device: could be filesystem or mapped device27 * @holder_ops: operations for the inner holder28 */29struct dax_device {30 struct inode inode;31 struct cdev cdev;32 void *private;33 unsigned long flags;34 const struct dax_operations *ops;35 void *holder_data;36 const struct dax_holder_operations *holder_ops;37};38 39static dev_t dax_devt;40DEFINE_STATIC_SRCU(dax_srcu);41static struct vfsmount *dax_mnt;42static DEFINE_IDA(dax_minor_ida);43static struct kmem_cache *dax_cache __read_mostly;44static struct super_block *dax_superblock __read_mostly;45 46int dax_read_lock(void)47{48 return srcu_read_lock(&dax_srcu);49}50EXPORT_SYMBOL_GPL(dax_read_lock);51 52void dax_read_unlock(int id)53{54 srcu_read_unlock(&dax_srcu, id);55}56EXPORT_SYMBOL_GPL(dax_read_unlock);57 58#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)59#include <linux/blkdev.h>60 61static DEFINE_XARRAY(dax_hosts);62 63int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)64{65 return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);66}67EXPORT_SYMBOL_GPL(dax_add_host);68 69void dax_remove_host(struct gendisk *disk)70{71 xa_erase(&dax_hosts, (unsigned long)disk);72}73EXPORT_SYMBOL_GPL(dax_remove_host);74 75/**76 * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax77 * @bdev: block device to find a dax_device for78 * @start_off: returns the byte offset into the dax_device that @bdev starts79 * @holder: filesystem or mapped device inside the dax_device80 * @ops: operations for the inner holder81 */82struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,83 void *holder, const struct dax_holder_operations *ops)84{85 struct dax_device *dax_dev;86 u64 part_size;87 int id;88 89 if (!blk_queue_dax(bdev->bd_disk->queue))90 return NULL;91 92 *start_off = get_start_sect(bdev) * SECTOR_SIZE;93 part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE;94 if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) {95 pr_info("%pg: error: unaligned partition for dax\n", bdev);96 return NULL;97 }98 99 id = dax_read_lock();100 dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);101 if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))102 dax_dev = NULL;103 else if (holder) {104 if (!cmpxchg(&dax_dev->holder_data, NULL, holder))105 dax_dev->holder_ops = ops;106 else107 dax_dev = NULL;108 }109 dax_read_unlock(id);110 111 return dax_dev;112}113EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);114 115void fs_put_dax(struct dax_device *dax_dev, void *holder)116{117 if (dax_dev && holder &&118 cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)119 dax_dev->holder_ops = NULL;120 put_dax(dax_dev);121}122EXPORT_SYMBOL_GPL(fs_put_dax);123#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */124 125enum dax_device_flags {126 /* !alive + rcu grace period == no new operations / mappings */127 DAXDEV_ALIVE,128 /* gate whether dax_flush() calls the low level flush routine */129 DAXDEV_WRITE_CACHE,130 /* flag to check if device supports synchronous flush */131 DAXDEV_SYNC,132 /* do not leave the caches dirty after writes */133 DAXDEV_NOCACHE,134 /* handle CPU fetch exceptions during reads */135 DAXDEV_NOMC,136};137 138/**139 * dax_direct_access() - translate a device pgoff to an absolute pfn140 * @dax_dev: a dax_device instance representing the logical memory range141 * @pgoff: offset in pages from the start of the device to translate142 * @nr_pages: number of consecutive pages caller can handle relative to @pfn143 * @mode: indicator on normal access or recovery write144 * @kaddr: output parameter that returns a virtual address mapping of pfn145 * @pfn: output parameter that returns an absolute pfn translation of @pgoff146 *147 * Return: negative errno if an error occurs, otherwise the number of148 * pages accessible at the device relative @pgoff.149 */150long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,151 enum dax_access_mode mode, void **kaddr, pfn_t *pfn)152{153 long avail;154 155 if (!dax_dev)156 return -EOPNOTSUPP;157 158 if (!dax_alive(dax_dev))159 return -ENXIO;160 161 if (nr_pages < 0)162 return -EINVAL;163 164 avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,165 mode, kaddr, pfn);166 if (!avail)167 return -ERANGE;168 return min(avail, nr_pages);169}170EXPORT_SYMBOL_GPL(dax_direct_access);171 172size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,173 size_t bytes, struct iov_iter *i)174{175 if (!dax_alive(dax_dev))176 return 0;177 178 /*179 * The userspace address for the memory copy has already been validated180 * via access_ok() in vfs_write, so use the 'no check' version to bypass181 * the HARDENED_USERCOPY overhead.182 */183 if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))184 return _copy_from_iter_flushcache(addr, bytes, i);185 return _copy_from_iter(addr, bytes, i);186}187 188size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,189 size_t bytes, struct iov_iter *i)190{191 if (!dax_alive(dax_dev))192 return 0;193 194 /*195 * The userspace address for the memory copy has already been validated196 * via access_ok() in vfs_red, so use the 'no check' version to bypass197 * the HARDENED_USERCOPY overhead.198 */199 if (test_bit(DAXDEV_NOMC, &dax_dev->flags))200 return _copy_mc_to_iter(addr, bytes, i);201 return _copy_to_iter(addr, bytes, i);202}203 204int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,205 size_t nr_pages)206{207 int ret;208 209 if (!dax_alive(dax_dev))210 return -ENXIO;211 /*212 * There are no callers that want to zero more than one page as of now.213 * Once users are there, this check can be removed after the214 * device mapper code has been updated to split ranges across targets.215 */216 if (nr_pages != 1)217 return -EIO;218 219 ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);220 return dax_mem2blk_err(ret);221}222EXPORT_SYMBOL_GPL(dax_zero_page_range);223 224size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,225 void *addr, size_t bytes, struct iov_iter *iter)226{227 if (!dax_dev->ops->recovery_write)228 return 0;229 return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);230}231EXPORT_SYMBOL_GPL(dax_recovery_write);232 233int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,234 u64 len, int mf_flags)235{236 int rc, id;237 238 id = dax_read_lock();239 if (!dax_alive(dax_dev)) {240 rc = -ENXIO;241 goto out;242 }243 244 if (!dax_dev->holder_ops) {245 rc = -EOPNOTSUPP;246 goto out;247 }248 249 rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);250out:251 dax_read_unlock(id);252 return rc;253}254EXPORT_SYMBOL_GPL(dax_holder_notify_failure);255 256#ifdef CONFIG_ARCH_HAS_PMEM_API257void arch_wb_cache_pmem(void *addr, size_t size);258void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)259{260 if (unlikely(!dax_write_cache_enabled(dax_dev)))261 return;262 263 arch_wb_cache_pmem(addr, size);264}265#else266void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)267{268}269#endif270EXPORT_SYMBOL_GPL(dax_flush);271 272void dax_write_cache(struct dax_device *dax_dev, bool wc)273{274 if (wc)275 set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);276 else277 clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);278}279EXPORT_SYMBOL_GPL(dax_write_cache);280 281bool dax_write_cache_enabled(struct dax_device *dax_dev)282{283 return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);284}285EXPORT_SYMBOL_GPL(dax_write_cache_enabled);286 287bool dax_synchronous(struct dax_device *dax_dev)288{289 return test_bit(DAXDEV_SYNC, &dax_dev->flags);290}291EXPORT_SYMBOL_GPL(dax_synchronous);292 293void set_dax_synchronous(struct dax_device *dax_dev)294{295 set_bit(DAXDEV_SYNC, &dax_dev->flags);296}297EXPORT_SYMBOL_GPL(set_dax_synchronous);298 299void set_dax_nocache(struct dax_device *dax_dev)300{301 set_bit(DAXDEV_NOCACHE, &dax_dev->flags);302}303EXPORT_SYMBOL_GPL(set_dax_nocache);304 305void set_dax_nomc(struct dax_device *dax_dev)306{307 set_bit(DAXDEV_NOMC, &dax_dev->flags);308}309EXPORT_SYMBOL_GPL(set_dax_nomc);310 311bool dax_alive(struct dax_device *dax_dev)312{313 lockdep_assert_held(&dax_srcu);314 return test_bit(DAXDEV_ALIVE, &dax_dev->flags);315}316EXPORT_SYMBOL_GPL(dax_alive);317 318/*319 * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring320 * that any fault handlers or operations that might have seen321 * dax_alive(), have completed. Any operations that start after322 * synchronize_srcu() has run will abort upon seeing !dax_alive().323 *324 * Note, because alloc_dax() returns an ERR_PTR() on error, callers325 * typically store its result into a local variable in order to check326 * the result. Therefore, care must be taken to populate the struct327 * device dax_dev field make sure the dax_dev is not leaked.328 */329void kill_dax(struct dax_device *dax_dev)330{331 if (!dax_dev)332 return;333 334 if (dax_dev->holder_data != NULL)335 dax_holder_notify_failure(dax_dev, 0, U64_MAX,336 MF_MEM_PRE_REMOVE);337 338 clear_bit(DAXDEV_ALIVE, &dax_dev->flags);339 synchronize_srcu(&dax_srcu);340 341 /* clear holder data */342 dax_dev->holder_ops = NULL;343 dax_dev->holder_data = NULL;344}345EXPORT_SYMBOL_GPL(kill_dax);346 347void run_dax(struct dax_device *dax_dev)348{349 set_bit(DAXDEV_ALIVE, &dax_dev->flags);350}351EXPORT_SYMBOL_GPL(run_dax);352 353static struct inode *dax_alloc_inode(struct super_block *sb)354{355 struct dax_device *dax_dev;356 struct inode *inode;357 358 dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL);359 if (!dax_dev)360 return NULL;361 362 inode = &dax_dev->inode;363 inode->i_rdev = 0;364 return inode;365}366 367static struct dax_device *to_dax_dev(struct inode *inode)368{369 return container_of(inode, struct dax_device, inode);370}371 372static void dax_free_inode(struct inode *inode)373{374 struct dax_device *dax_dev = to_dax_dev(inode);375 if (inode->i_rdev)376 ida_free(&dax_minor_ida, iminor(inode));377 kmem_cache_free(dax_cache, dax_dev);378}379 380static void dax_destroy_inode(struct inode *inode)381{382 struct dax_device *dax_dev = to_dax_dev(inode);383 WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),384 "kill_dax() must be called before final iput()\n");385}386 387static const struct super_operations dax_sops = {388 .statfs = simple_statfs,389 .alloc_inode = dax_alloc_inode,390 .destroy_inode = dax_destroy_inode,391 .free_inode = dax_free_inode,392 .drop_inode = generic_delete_inode,393};394 395static int dax_init_fs_context(struct fs_context *fc)396{397 struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC);398 if (!ctx)399 return -ENOMEM;400 ctx->ops = &dax_sops;401 return 0;402}403 404static struct file_system_type dax_fs_type = {405 .name = "dax",406 .init_fs_context = dax_init_fs_context,407 .kill_sb = kill_anon_super,408};409 410static int dax_test(struct inode *inode, void *data)411{412 dev_t devt = *(dev_t *) data;413 414 return inode->i_rdev == devt;415}416 417static int dax_set(struct inode *inode, void *data)418{419 dev_t devt = *(dev_t *) data;420 421 inode->i_rdev = devt;422 return 0;423}424 425static struct dax_device *dax_dev_get(dev_t devt)426{427 struct dax_device *dax_dev;428 struct inode *inode;429 430 inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),431 dax_test, dax_set, &devt);432 433 if (!inode)434 return NULL;435 436 dax_dev = to_dax_dev(inode);437 if (inode->i_state & I_NEW) {438 set_bit(DAXDEV_ALIVE, &dax_dev->flags);439 inode->i_cdev = &dax_dev->cdev;440 inode->i_mode = S_IFCHR;441 inode->i_flags = S_DAX;442 mapping_set_gfp_mask(&inode->i_data, GFP_USER);443 unlock_new_inode(inode);444 }445 446 return dax_dev;447}448 449struct dax_device *alloc_dax(void *private, const struct dax_operations *ops)450{451 struct dax_device *dax_dev;452 dev_t devt;453 int minor;454 455 /*456 * Unavailable on architectures with virtually aliased data caches,457 * except for device-dax (NULL operations pointer), which does458 * not use aliased mappings from the kernel.459 */460 if (ops && cpu_dcache_is_aliasing())461 return ERR_PTR(-EOPNOTSUPP);462 463 if (WARN_ON_ONCE(ops && !ops->zero_page_range))464 return ERR_PTR(-EINVAL);465 466 minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL);467 if (minor < 0)468 return ERR_PTR(-ENOMEM);469 470 devt = MKDEV(MAJOR(dax_devt), minor);471 dax_dev = dax_dev_get(devt);472 if (!dax_dev)473 goto err_dev;474 475 dax_dev->ops = ops;476 dax_dev->private = private;477 return dax_dev;478 479 err_dev:480 ida_free(&dax_minor_ida, minor);481 return ERR_PTR(-ENOMEM);482}483EXPORT_SYMBOL_GPL(alloc_dax);484 485void put_dax(struct dax_device *dax_dev)486{487 if (!dax_dev)488 return;489 iput(&dax_dev->inode);490}491EXPORT_SYMBOL_GPL(put_dax);492 493/**494 * dax_holder() - obtain the holder of a dax device495 * @dax_dev: a dax_device instance496 *497 * Return: the holder's data which represents the holder if registered,498 * otherwize NULL.499 */500void *dax_holder(struct dax_device *dax_dev)501{502 return dax_dev->holder_data;503}504EXPORT_SYMBOL_GPL(dax_holder);505 506/**507 * inode_dax: convert a public inode into its dax_dev508 * @inode: An inode with i_cdev pointing to a dax_dev509 *510 * Note this is not equivalent to to_dax_dev() which is for private511 * internal use where we know the inode filesystem type == dax_fs_type.512 */513struct dax_device *inode_dax(struct inode *inode)514{515 struct cdev *cdev = inode->i_cdev;516 517 return container_of(cdev, struct dax_device, cdev);518}519EXPORT_SYMBOL_GPL(inode_dax);520 521struct inode *dax_inode(struct dax_device *dax_dev)522{523 return &dax_dev->inode;524}525EXPORT_SYMBOL_GPL(dax_inode);526 527void *dax_get_private(struct dax_device *dax_dev)528{529 if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))530 return NULL;531 return dax_dev->private;532}533EXPORT_SYMBOL_GPL(dax_get_private);534 535static void init_once(void *_dax_dev)536{537 struct dax_device *dax_dev = _dax_dev;538 struct inode *inode = &dax_dev->inode;539 540 memset(dax_dev, 0, sizeof(*dax_dev));541 inode_init_once(inode);542}543 544static int dax_fs_init(void)545{546 int rc;547 548 dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,549 SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,550 init_once);551 if (!dax_cache)552 return -ENOMEM;553 554 dax_mnt = kern_mount(&dax_fs_type);555 if (IS_ERR(dax_mnt)) {556 rc = PTR_ERR(dax_mnt);557 goto err_mount;558 }559 dax_superblock = dax_mnt->mnt_sb;560 561 return 0;562 563 err_mount:564 kmem_cache_destroy(dax_cache);565 566 return rc;567}568 569static void dax_fs_exit(void)570{571 kern_unmount(dax_mnt);572 rcu_barrier();573 kmem_cache_destroy(dax_cache);574}575 576static int __init dax_core_init(void)577{578 int rc;579 580 rc = dax_fs_init();581 if (rc)582 return rc;583 584 rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");585 if (rc)586 goto err_chrdev;587 588 rc = dax_bus_init();589 if (rc)590 goto err_bus;591 return 0;592 593err_bus:594 unregister_chrdev_region(dax_devt, MINORMASK+1);595err_chrdev:596 dax_fs_exit();597 return 0;598}599 600static void __exit dax_core_exit(void)601{602 dax_bus_exit();603 unregister_chrdev_region(dax_devt, MINORMASK+1);604 ida_destroy(&dax_minor_ida);605 dax_fs_exit();606}607 608MODULE_AUTHOR("Intel Corporation");609MODULE_DESCRIPTION("DAX: direct access to differentiated memory");610MODULE_LICENSE("GPL v2");611subsys_initcall(dax_core_init);612module_exit(dax_core_exit);613