275 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2020 Oracle Corporation4 *5 * Module Author: Mike Christie6 */7#include "dm-path-selector.h"8 9#include <linux/device-mapper.h>10#include <linux/module.h>11 12#define DM_MSG_PREFIX "multipath io-affinity"13 14struct path_info {15 struct dm_path *path;16 cpumask_var_t cpumask;17 refcount_t refcount;18 bool failed;19};20 21struct selector {22 struct path_info **path_map;23 cpumask_var_t path_mask;24 atomic_t map_misses;25};26 27static void ioa_free_path(struct selector *s, unsigned int cpu)28{29 struct path_info *pi = s->path_map[cpu];30 31 if (!pi)32 return;33 34 if (refcount_dec_and_test(&pi->refcount)) {35 cpumask_clear_cpu(cpu, s->path_mask);36 free_cpumask_var(pi->cpumask);37 kfree(pi);38 39 s->path_map[cpu] = NULL;40 }41}42 43static int ioa_add_path(struct path_selector *ps, struct dm_path *path,44 int argc, char **argv, char **error)45{46 struct selector *s = ps->context;47 struct path_info *pi = NULL;48 unsigned int cpu;49 int ret;50 51 if (argc != 1) {52 *error = "io-affinity ps: invalid number of arguments";53 return -EINVAL;54 }55 56 pi = kzalloc(sizeof(*pi), GFP_KERNEL);57 if (!pi) {58 *error = "io-affinity ps: Error allocating path context";59 return -ENOMEM;60 }61 62 pi->path = path;63 path->pscontext = pi;64 refcount_set(&pi->refcount, 1);65 66 if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) {67 *error = "io-affinity ps: Error allocating cpumask context";68 ret = -ENOMEM;69 goto free_pi;70 }71 72 ret = cpumask_parse(argv[0], pi->cpumask);73 if (ret) {74 *error = "io-affinity ps: invalid cpumask";75 ret = -EINVAL;76 goto free_mask;77 }78 79 for_each_cpu(cpu, pi->cpumask) {80 if (cpu >= nr_cpu_ids) {81 DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u",82 cpu, nr_cpu_ids);83 break;84 }85 86 if (s->path_map[cpu]) {87 DMWARN("CPU mapping for %u exists. Ignoring.", cpu);88 continue;89 }90 91 cpumask_set_cpu(cpu, s->path_mask);92 s->path_map[cpu] = pi;93 refcount_inc(&pi->refcount);94 }95 96 if (refcount_dec_and_test(&pi->refcount)) {97 *error = "io-affinity ps: No new/valid CPU mapping found";98 ret = -EINVAL;99 goto free_mask;100 }101 102 return 0;103 104free_mask:105 free_cpumask_var(pi->cpumask);106free_pi:107 kfree(pi);108 return ret;109}110 111static int ioa_create(struct path_selector *ps, unsigned int argc, char **argv)112{113 struct selector *s;114 115 s = kmalloc(sizeof(*s), GFP_KERNEL);116 if (!s)117 return -ENOMEM;118 119 s->path_map = kzalloc(nr_cpu_ids * sizeof(struct path_info *),120 GFP_KERNEL);121 if (!s->path_map)122 goto free_selector;123 124 if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL))125 goto free_map;126 127 atomic_set(&s->map_misses, 0);128 ps->context = s;129 return 0;130 131free_map:132 kfree(s->path_map);133free_selector:134 kfree(s);135 return -ENOMEM;136}137 138static void ioa_destroy(struct path_selector *ps)139{140 struct selector *s = ps->context;141 unsigned int cpu;142 143 for_each_cpu(cpu, s->path_mask)144 ioa_free_path(s, cpu);145 146 free_cpumask_var(s->path_mask);147 kfree(s->path_map);148 kfree(s);149 150 ps->context = NULL;151}152 153static int ioa_status(struct path_selector *ps, struct dm_path *path,154 status_type_t type, char *result, unsigned int maxlen)155{156 struct selector *s = ps->context;157 struct path_info *pi;158 int sz = 0;159 160 if (!path) {161 DMEMIT("0 ");162 return sz;163 }164 165 switch (type) {166 case STATUSTYPE_INFO:167 DMEMIT("%d ", atomic_read(&s->map_misses));168 break;169 case STATUSTYPE_TABLE:170 pi = path->pscontext;171 DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask));172 break;173 case STATUSTYPE_IMA:174 *result = '\0';175 break;176 }177 178 return sz;179}180 181static void ioa_fail_path(struct path_selector *ps, struct dm_path *p)182{183 struct path_info *pi = p->pscontext;184 185 pi->failed = true;186}187 188static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p)189{190 struct path_info *pi = p->pscontext;191 192 pi->failed = false;193 return 0;194}195 196static struct dm_path *ioa_select_path(struct path_selector *ps,197 size_t nr_bytes)198{199 unsigned int cpu, node;200 struct selector *s = ps->context;201 const struct cpumask *cpumask;202 struct path_info *pi;203 int i;204 205 cpu = get_cpu();206 207 pi = s->path_map[cpu];208 if (pi && !pi->failed)209 goto done;210 211 /*212 * Perf is not optimal, but we at least try the local node then just213 * try not to fail.214 */215 if (!pi)216 atomic_inc(&s->map_misses);217 218 node = cpu_to_node(cpu);219 cpumask = cpumask_of_node(node);220 for_each_cpu(i, cpumask) {221 pi = s->path_map[i];222 if (pi && !pi->failed)223 goto done;224 }225 226 for_each_cpu(i, s->path_mask) {227 pi = s->path_map[i];228 if (pi && !pi->failed)229 goto done;230 }231 pi = NULL;232 233done:234 put_cpu();235 return pi ? pi->path : NULL;236}237 238static struct path_selector_type ioa_ps = {239 .name = "io-affinity",240 .module = THIS_MODULE,241 .table_args = 1,242 .info_args = 1,243 .create = ioa_create,244 .destroy = ioa_destroy,245 .status = ioa_status,246 .add_path = ioa_add_path,247 .fail_path = ioa_fail_path,248 .reinstate_path = ioa_reinstate_path,249 .select_path = ioa_select_path,250};251 252static int __init dm_ioa_init(void)253{254 int ret = dm_register_path_selector(&ioa_ps);255 256 if (ret < 0)257 DMERR("register failed %d", ret);258 return ret;259}260 261static void __exit dm_ioa_exit(void)262{263 int ret = dm_unregister_path_selector(&ioa_ps);264 265 if (ret < 0)266 DMERR("unregister failed %d", ret);267}268 269module_init(dm_ioa_init);270module_exit(dm_ioa_exit);271 272MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on");273MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>");274MODULE_LICENSE("GPL");275