1338 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * System Control and Management Interface (SCMI) Performance Protocol4 *5 * Copyright (C) 2018-2023 ARM Ltd.6 */7 8#define pr_fmt(fmt) "SCMI Notifications PERF - " fmt9 10#include <linux/bits.h>11#include <linux/hashtable.h>12#include <linux/io.h>13#include <linux/log2.h>14#include <linux/module.h>15#include <linux/of.h>16#include <linux/platform_device.h>17#include <linux/pm_opp.h>18#include <linux/scmi_protocol.h>19#include <linux/sort.h>20#include <linux/xarray.h>21 22#include <trace/events/scmi.h>23 24#include "protocols.h"25#include "notify.h"26 27/* Updated only after ALL the mandatory features for that version are merged */28#define SCMI_PROTOCOL_SUPPORTED_VERSION 0x4000029 30#define MAX_OPPS 3231 32enum scmi_performance_protocol_cmd {33 PERF_DOMAIN_ATTRIBUTES = 0x3,34 PERF_DESCRIBE_LEVELS = 0x4,35 PERF_LIMITS_SET = 0x5,36 PERF_LIMITS_GET = 0x6,37 PERF_LEVEL_SET = 0x7,38 PERF_LEVEL_GET = 0x8,39 PERF_NOTIFY_LIMITS = 0x9,40 PERF_NOTIFY_LEVEL = 0xa,41 PERF_DESCRIBE_FASTCHANNEL = 0xb,42 PERF_DOMAIN_NAME_GET = 0xc,43};44 45enum {46 PERF_FC_LEVEL,47 PERF_FC_LIMIT,48 PERF_FC_MAX,49};50 51struct scmi_opp {52 u32 perf;53 u32 power;54 u32 trans_latency_us;55 u32 indicative_freq;56 u32 level_index;57 struct hlist_node hash;58};59 60struct scmi_msg_resp_perf_attributes {61 __le16 num_domains;62 __le16 flags;63#define POWER_SCALE_IN_MILLIWATT(x) ((x) & BIT(0))64#define POWER_SCALE_IN_MICROWATT(x) ((x) & BIT(1))65 __le32 stats_addr_low;66 __le32 stats_addr_high;67 __le32 stats_size;68};69 70struct scmi_msg_resp_perf_domain_attributes {71 __le32 flags;72#define SUPPORTS_SET_LIMITS(x) ((x) & BIT(31))73#define SUPPORTS_SET_PERF_LVL(x) ((x) & BIT(30))74#define SUPPORTS_PERF_LIMIT_NOTIFY(x) ((x) & BIT(29))75#define SUPPORTS_PERF_LEVEL_NOTIFY(x) ((x) & BIT(28))76#define SUPPORTS_PERF_FASTCHANNELS(x) ((x) & BIT(27))77#define SUPPORTS_EXTENDED_NAMES(x) ((x) & BIT(26))78#define SUPPORTS_LEVEL_INDEXING(x) ((x) & BIT(25))79 __le32 rate_limit_us;80 __le32 sustained_freq_khz;81 __le32 sustained_perf_level;82 u8 name[SCMI_SHORT_NAME_MAX_SIZE];83};84 85struct scmi_msg_perf_describe_levels {86 __le32 domain;87 __le32 level_index;88};89 90struct scmi_perf_set_limits {91 __le32 domain;92 __le32 max_level;93 __le32 min_level;94};95 96struct scmi_perf_get_limits {97 __le32 max_level;98 __le32 min_level;99};100 101struct scmi_perf_set_level {102 __le32 domain;103 __le32 level;104};105 106struct scmi_perf_notify_level_or_limits {107 __le32 domain;108 __le32 notify_enable;109};110 111struct scmi_perf_limits_notify_payld {112 __le32 agent_id;113 __le32 domain_id;114 __le32 range_max;115 __le32 range_min;116};117 118struct scmi_perf_level_notify_payld {119 __le32 agent_id;120 __le32 domain_id;121 __le32 performance_level;122};123 124struct scmi_msg_resp_perf_describe_levels {125 __le16 num_returned;126 __le16 num_remaining;127 struct {128 __le32 perf_val;129 __le32 power;130 __le16 transition_latency_us;131 __le16 reserved;132 } opp[];133};134 135struct scmi_msg_resp_perf_describe_levels_v4 {136 __le16 num_returned;137 __le16 num_remaining;138 struct {139 __le32 perf_val;140 __le32 power;141 __le16 transition_latency_us;142 __le16 reserved;143 __le32 indicative_freq;144 __le32 level_index;145 } opp[];146};147 148struct perf_dom_info {149 u32 id;150 bool set_limits;151 bool perf_limit_notify;152 bool perf_level_notify;153 bool perf_fastchannels;154 bool level_indexing_mode;155 u32 opp_count;156 u32 rate_limit_us;157 u32 sustained_freq_khz;158 u32 sustained_perf_level;159 unsigned long mult_factor;160 struct scmi_perf_domain_info info;161 struct scmi_opp opp[MAX_OPPS];162 struct scmi_fc_info *fc_info;163 struct xarray opps_by_idx;164 struct xarray opps_by_lvl;165 DECLARE_HASHTABLE(opps_by_freq, ilog2(MAX_OPPS));166};167 168#define LOOKUP_BY_FREQ(__htp, __freq) \169({ \170 /* u32 cast is needed to pick right hash func */ \171 u32 f_ = (u32)(__freq); \172 struct scmi_opp *_opp; \173 \174 hash_for_each_possible((__htp), _opp, hash, f_) \175 if (_opp->indicative_freq == f_) \176 break; \177 _opp; \178})179 180struct scmi_perf_info {181 u32 version;182 u16 num_domains;183 enum scmi_power_scale power_scale;184 u64 stats_addr;185 u32 stats_size;186 bool notify_lvl_cmd;187 bool notify_lim_cmd;188 struct perf_dom_info *dom_info;189};190 191static enum scmi_performance_protocol_cmd evt_2_cmd[] = {192 PERF_NOTIFY_LIMITS,193 PERF_NOTIFY_LEVEL,194};195 196static int scmi_perf_attributes_get(const struct scmi_protocol_handle *ph,197 struct scmi_perf_info *pi)198{199 int ret;200 struct scmi_xfer *t;201 struct scmi_msg_resp_perf_attributes *attr;202 203 ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES, 0,204 sizeof(*attr), &t);205 if (ret)206 return ret;207 208 attr = t->rx.buf;209 210 ret = ph->xops->do_xfer(ph, t);211 if (!ret) {212 u16 flags = le16_to_cpu(attr->flags);213 214 pi->num_domains = le16_to_cpu(attr->num_domains);215 216 if (POWER_SCALE_IN_MILLIWATT(flags))217 pi->power_scale = SCMI_POWER_MILLIWATTS;218 if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3)219 if (POWER_SCALE_IN_MICROWATT(flags))220 pi->power_scale = SCMI_POWER_MICROWATTS;221 222 pi->stats_addr = le32_to_cpu(attr->stats_addr_low) |223 (u64)le32_to_cpu(attr->stats_addr_high) << 32;224 pi->stats_size = le32_to_cpu(attr->stats_size);225 }226 227 ph->xops->xfer_put(ph, t);228 229 if (!ret) {230 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LEVEL, NULL))231 pi->notify_lvl_cmd = true;232 233 if (!ph->hops->protocol_msg_check(ph, PERF_NOTIFY_LIMITS, NULL))234 pi->notify_lim_cmd = true;235 }236 237 return ret;238}239 240static void scmi_perf_xa_destroy(void *data)241{242 int domain;243 struct scmi_perf_info *pinfo = data;244 245 for (domain = 0; domain < pinfo->num_domains; domain++) {246 xa_destroy(&((pinfo->dom_info + domain)->opps_by_idx));247 xa_destroy(&((pinfo->dom_info + domain)->opps_by_lvl));248 }249}250 251static int252scmi_perf_domain_attributes_get(const struct scmi_protocol_handle *ph,253 struct perf_dom_info *dom_info,254 bool notify_lim_cmd, bool notify_lvl_cmd,255 u32 version)256{257 int ret;258 u32 flags;259 struct scmi_xfer *t;260 struct scmi_msg_resp_perf_domain_attributes *attr;261 262 ret = ph->xops->xfer_get_init(ph, PERF_DOMAIN_ATTRIBUTES,263 sizeof(dom_info->id), sizeof(*attr), &t);264 if (ret)265 return ret;266 267 put_unaligned_le32(dom_info->id, t->tx.buf);268 attr = t->rx.buf;269 270 ret = ph->xops->do_xfer(ph, t);271 if (!ret) {272 flags = le32_to_cpu(attr->flags);273 274 dom_info->set_limits = SUPPORTS_SET_LIMITS(flags);275 dom_info->info.set_perf = SUPPORTS_SET_PERF_LVL(flags);276 if (notify_lim_cmd)277 dom_info->perf_limit_notify =278 SUPPORTS_PERF_LIMIT_NOTIFY(flags);279 if (notify_lvl_cmd)280 dom_info->perf_level_notify =281 SUPPORTS_PERF_LEVEL_NOTIFY(flags);282 dom_info->perf_fastchannels = SUPPORTS_PERF_FASTCHANNELS(flags);283 if (PROTOCOL_REV_MAJOR(version) >= 0x4)284 dom_info->level_indexing_mode =285 SUPPORTS_LEVEL_INDEXING(flags);286 dom_info->rate_limit_us = le32_to_cpu(attr->rate_limit_us) &287 GENMASK(19, 0);288 dom_info->sustained_freq_khz =289 le32_to_cpu(attr->sustained_freq_khz);290 dom_info->sustained_perf_level =291 le32_to_cpu(attr->sustained_perf_level);292 /*293 * sustained_freq_khz = mult_factor * sustained_perf_level294 * mult_factor must be non zero positive integer(not fraction)295 */296 if (!dom_info->sustained_freq_khz ||297 !dom_info->sustained_perf_level ||298 dom_info->level_indexing_mode) {299 /* CPUFreq converts to kHz, hence default 1000 */300 dom_info->mult_factor = 1000;301 } else {302 dom_info->mult_factor =303 (dom_info->sustained_freq_khz * 1000UL)304 / dom_info->sustained_perf_level;305 if ((dom_info->sustained_freq_khz * 1000UL) %306 dom_info->sustained_perf_level)307 dev_warn(ph->dev,308 "multiplier for domain %d rounded\n",309 dom_info->id);310 }311 if (!dom_info->mult_factor)312 dev_warn(ph->dev,313 "Wrong sustained perf/frequency(domain %d)\n",314 dom_info->id);315 316 strscpy(dom_info->info.name, attr->name,317 SCMI_SHORT_NAME_MAX_SIZE);318 }319 320 ph->xops->xfer_put(ph, t);321 322 /*323 * If supported overwrite short name with the extended one;324 * on error just carry on and use already provided short name.325 */326 if (!ret && PROTOCOL_REV_MAJOR(version) >= 0x3 &&327 SUPPORTS_EXTENDED_NAMES(flags))328 ph->hops->extended_name_get(ph, PERF_DOMAIN_NAME_GET,329 dom_info->id, NULL, dom_info->info.name,330 SCMI_MAX_STR_SIZE);331 332 xa_init(&dom_info->opps_by_lvl);333 if (dom_info->level_indexing_mode) {334 xa_init(&dom_info->opps_by_idx);335 hash_init(dom_info->opps_by_freq);336 }337 338 return ret;339}340 341static int opp_cmp_func(const void *opp1, const void *opp2)342{343 const struct scmi_opp *t1 = opp1, *t2 = opp2;344 345 return t1->perf - t2->perf;346}347 348struct scmi_perf_ipriv {349 u32 version;350 struct perf_dom_info *perf_dom;351};352 353static void iter_perf_levels_prepare_message(void *message,354 unsigned int desc_index,355 const void *priv)356{357 struct scmi_msg_perf_describe_levels *msg = message;358 const struct scmi_perf_ipriv *p = priv;359 360 msg->domain = cpu_to_le32(p->perf_dom->id);361 /* Set the number of OPPs to be skipped/already read */362 msg->level_index = cpu_to_le32(desc_index);363}364 365static int iter_perf_levels_update_state(struct scmi_iterator_state *st,366 const void *response, void *priv)367{368 const struct scmi_msg_resp_perf_describe_levels *r = response;369 370 st->num_returned = le16_to_cpu(r->num_returned);371 st->num_remaining = le16_to_cpu(r->num_remaining);372 373 return 0;374}375 376static inline int377process_response_opp(struct device *dev, struct perf_dom_info *dom,378 struct scmi_opp *opp, unsigned int loop_idx,379 const struct scmi_msg_resp_perf_describe_levels *r)380{381 int ret;382 383 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);384 opp->power = le32_to_cpu(r->opp[loop_idx].power);385 opp->trans_latency_us =386 le16_to_cpu(r->opp[loop_idx].transition_latency_us);387 388 ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL);389 if (ret) {390 dev_info(dev, FW_BUG "Failed to add opps_by_lvl at %d for %s - ret:%d\n",391 opp->perf, dom->info.name, ret);392 return ret;393 }394 395 return 0;396}397 398static inline int399process_response_opp_v4(struct device *dev, struct perf_dom_info *dom,400 struct scmi_opp *opp, unsigned int loop_idx,401 const struct scmi_msg_resp_perf_describe_levels_v4 *r)402{403 int ret;404 405 opp->perf = le32_to_cpu(r->opp[loop_idx].perf_val);406 opp->power = le32_to_cpu(r->opp[loop_idx].power);407 opp->trans_latency_us =408 le16_to_cpu(r->opp[loop_idx].transition_latency_us);409 410 ret = xa_insert(&dom->opps_by_lvl, opp->perf, opp, GFP_KERNEL);411 if (ret) {412 dev_info(dev, FW_BUG "Failed to add opps_by_lvl at %d for %s - ret:%d\n",413 opp->perf, dom->info.name, ret);414 return ret;415 }416 417 /* Note that PERF v4 reports always five 32-bit words */418 opp->indicative_freq = le32_to_cpu(r->opp[loop_idx].indicative_freq);419 if (dom->level_indexing_mode) {420 opp->level_index = le32_to_cpu(r->opp[loop_idx].level_index);421 422 ret = xa_insert(&dom->opps_by_idx, opp->level_index, opp,423 GFP_KERNEL);424 if (ret) {425 dev_warn(dev,426 "Failed to add opps_by_idx at %d for %s - ret:%d\n",427 opp->level_index, dom->info.name, ret);428 429 /* Cleanup by_lvl too */430 xa_erase(&dom->opps_by_lvl, opp->perf);431 432 return ret;433 }434 435 hash_add(dom->opps_by_freq, &opp->hash, opp->indicative_freq);436 }437 438 return 0;439}440 441static int442iter_perf_levels_process_response(const struct scmi_protocol_handle *ph,443 const void *response,444 struct scmi_iterator_state *st, void *priv)445{446 int ret;447 struct scmi_opp *opp;448 struct scmi_perf_ipriv *p = priv;449 450 opp = &p->perf_dom->opp[p->perf_dom->opp_count];451 if (PROTOCOL_REV_MAJOR(p->version) <= 0x3)452 ret = process_response_opp(ph->dev, p->perf_dom, opp,453 st->loop_idx, response);454 else455 ret = process_response_opp_v4(ph->dev, p->perf_dom, opp,456 st->loop_idx, response);457 458 /* Skip BAD duplicates received from firmware */459 if (ret)460 return ret == -EBUSY ? 0 : ret;461 462 p->perf_dom->opp_count++;463 464 dev_dbg(ph->dev, "Level %d Power %d Latency %dus Ifreq %d Index %d\n",465 opp->perf, opp->power, opp->trans_latency_us,466 opp->indicative_freq, opp->level_index);467 468 return 0;469}470 471static int472scmi_perf_describe_levels_get(const struct scmi_protocol_handle *ph,473 struct perf_dom_info *perf_dom, u32 version)474{475 int ret;476 void *iter;477 struct scmi_iterator_ops ops = {478 .prepare_message = iter_perf_levels_prepare_message,479 .update_state = iter_perf_levels_update_state,480 .process_response = iter_perf_levels_process_response,481 };482 struct scmi_perf_ipriv ppriv = {483 .version = version,484 .perf_dom = perf_dom,485 };486 487 iter = ph->hops->iter_response_init(ph, &ops, MAX_OPPS,488 PERF_DESCRIBE_LEVELS,489 sizeof(struct scmi_msg_perf_describe_levels),490 &ppriv);491 if (IS_ERR(iter))492 return PTR_ERR(iter);493 494 ret = ph->hops->iter_response_run(iter);495 if (ret)496 return ret;497 498 if (perf_dom->opp_count)499 sort(perf_dom->opp, perf_dom->opp_count,500 sizeof(struct scmi_opp), opp_cmp_func, NULL);501 502 return ret;503}504 505static int scmi_perf_num_domains_get(const struct scmi_protocol_handle *ph)506{507 struct scmi_perf_info *pi = ph->get_priv(ph);508 509 return pi->num_domains;510}511 512static inline struct perf_dom_info *513scmi_perf_domain_lookup(const struct scmi_protocol_handle *ph, u32 domain)514{515 struct scmi_perf_info *pi = ph->get_priv(ph);516 517 if (domain >= pi->num_domains)518 return ERR_PTR(-EINVAL);519 520 return pi->dom_info + domain;521}522 523static const struct scmi_perf_domain_info *524scmi_perf_info_get(const struct scmi_protocol_handle *ph, u32 domain)525{526 struct perf_dom_info *dom;527 528 dom = scmi_perf_domain_lookup(ph, domain);529 if (IS_ERR(dom))530 return ERR_PTR(-EINVAL);531 532 return &dom->info;533}534 535static int scmi_perf_msg_limits_set(const struct scmi_protocol_handle *ph,536 u32 domain, u32 max_perf, u32 min_perf)537{538 int ret;539 struct scmi_xfer *t;540 struct scmi_perf_set_limits *limits;541 542 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_SET,543 sizeof(*limits), 0, &t);544 if (ret)545 return ret;546 547 limits = t->tx.buf;548 limits->domain = cpu_to_le32(domain);549 limits->max_level = cpu_to_le32(max_perf);550 limits->min_level = cpu_to_le32(min_perf);551 552 ret = ph->xops->do_xfer(ph, t);553 554 ph->xops->xfer_put(ph, t);555 return ret;556}557 558static int __scmi_perf_limits_set(const struct scmi_protocol_handle *ph,559 struct perf_dom_info *dom, u32 max_perf,560 u32 min_perf)561{562 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].set_addr) {563 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];564 565 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LIMITS_SET,566 dom->id, min_perf, max_perf);567 iowrite32(max_perf, fci->set_addr);568 iowrite32(min_perf, fci->set_addr + 4);569 ph->hops->fastchannel_db_ring(fci->set_db);570 return 0;571 }572 573 return scmi_perf_msg_limits_set(ph, dom->id, max_perf, min_perf);574}575 576static int scmi_perf_limits_set(const struct scmi_protocol_handle *ph,577 u32 domain, u32 max_perf, u32 min_perf)578{579 struct scmi_perf_info *pi = ph->get_priv(ph);580 struct perf_dom_info *dom;581 582 dom = scmi_perf_domain_lookup(ph, domain);583 if (IS_ERR(dom))584 return PTR_ERR(dom);585 586 if (!dom->set_limits)587 return -EOPNOTSUPP;588 589 if (PROTOCOL_REV_MAJOR(pi->version) >= 0x3 && !max_perf && !min_perf)590 return -EINVAL;591 592 if (dom->level_indexing_mode) {593 struct scmi_opp *opp;594 595 if (min_perf) {596 opp = xa_load(&dom->opps_by_lvl, min_perf);597 if (!opp)598 return -EIO;599 600 min_perf = opp->level_index;601 }602 603 if (max_perf) {604 opp = xa_load(&dom->opps_by_lvl, max_perf);605 if (!opp)606 return -EIO;607 608 max_perf = opp->level_index;609 }610 }611 612 return __scmi_perf_limits_set(ph, dom, max_perf, min_perf);613}614 615static int scmi_perf_msg_limits_get(const struct scmi_protocol_handle *ph,616 u32 domain, u32 *max_perf, u32 *min_perf)617{618 int ret;619 struct scmi_xfer *t;620 struct scmi_perf_get_limits *limits;621 622 ret = ph->xops->xfer_get_init(ph, PERF_LIMITS_GET,623 sizeof(__le32), 0, &t);624 if (ret)625 return ret;626 627 put_unaligned_le32(domain, t->tx.buf);628 629 ret = ph->xops->do_xfer(ph, t);630 if (!ret) {631 limits = t->rx.buf;632 633 *max_perf = le32_to_cpu(limits->max_level);634 *min_perf = le32_to_cpu(limits->min_level);635 }636 637 ph->xops->xfer_put(ph, t);638 return ret;639}640 641static int __scmi_perf_limits_get(const struct scmi_protocol_handle *ph,642 struct perf_dom_info *dom, u32 *max_perf,643 u32 *min_perf)644{645 if (dom->fc_info && dom->fc_info[PERF_FC_LIMIT].get_addr) {646 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LIMIT];647 648 *max_perf = ioread32(fci->get_addr);649 *min_perf = ioread32(fci->get_addr + 4);650 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LIMITS_GET,651 dom->id, *min_perf, *max_perf);652 return 0;653 }654 655 return scmi_perf_msg_limits_get(ph, dom->id, max_perf, min_perf);656}657 658static int scmi_perf_limits_get(const struct scmi_protocol_handle *ph,659 u32 domain, u32 *max_perf, u32 *min_perf)660{661 int ret;662 struct perf_dom_info *dom;663 664 dom = scmi_perf_domain_lookup(ph, domain);665 if (IS_ERR(dom))666 return PTR_ERR(dom);667 668 ret = __scmi_perf_limits_get(ph, dom, max_perf, min_perf);669 if (ret)670 return ret;671 672 if (dom->level_indexing_mode) {673 struct scmi_opp *opp;674 675 opp = xa_load(&dom->opps_by_idx, *min_perf);676 if (!opp)677 return -EIO;678 679 *min_perf = opp->perf;680 681 opp = xa_load(&dom->opps_by_idx, *max_perf);682 if (!opp)683 return -EIO;684 685 *max_perf = opp->perf;686 }687 688 return 0;689}690 691static int scmi_perf_msg_level_set(const struct scmi_protocol_handle *ph,692 u32 domain, u32 level, bool poll)693{694 int ret;695 struct scmi_xfer *t;696 struct scmi_perf_set_level *lvl;697 698 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_SET, sizeof(*lvl), 0, &t);699 if (ret)700 return ret;701 702 t->hdr.poll_completion = poll;703 lvl = t->tx.buf;704 lvl->domain = cpu_to_le32(domain);705 lvl->level = cpu_to_le32(level);706 707 ret = ph->xops->do_xfer(ph, t);708 709 ph->xops->xfer_put(ph, t);710 return ret;711}712 713static int __scmi_perf_level_set(const struct scmi_protocol_handle *ph,714 struct perf_dom_info *dom, u32 level,715 bool poll)716{717 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr) {718 struct scmi_fc_info *fci = &dom->fc_info[PERF_FC_LEVEL];719 720 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LEVEL_SET,721 dom->id, level, 0);722 iowrite32(level, fci->set_addr);723 ph->hops->fastchannel_db_ring(fci->set_db);724 return 0;725 }726 727 return scmi_perf_msg_level_set(ph, dom->id, level, poll);728}729 730static int scmi_perf_level_set(const struct scmi_protocol_handle *ph,731 u32 domain, u32 level, bool poll)732{733 struct perf_dom_info *dom;734 735 dom = scmi_perf_domain_lookup(ph, domain);736 if (IS_ERR(dom))737 return PTR_ERR(dom);738 739 if (!dom->info.set_perf)740 return -EOPNOTSUPP;741 742 if (dom->level_indexing_mode) {743 struct scmi_opp *opp;744 745 opp = xa_load(&dom->opps_by_lvl, level);746 if (!opp)747 return -EIO;748 749 level = opp->level_index;750 }751 752 return __scmi_perf_level_set(ph, dom, level, poll);753}754 755static int scmi_perf_msg_level_get(const struct scmi_protocol_handle *ph,756 u32 domain, u32 *level, bool poll)757{758 int ret;759 struct scmi_xfer *t;760 761 ret = ph->xops->xfer_get_init(ph, PERF_LEVEL_GET,762 sizeof(u32), sizeof(u32), &t);763 if (ret)764 return ret;765 766 t->hdr.poll_completion = poll;767 put_unaligned_le32(domain, t->tx.buf);768 769 ret = ph->xops->do_xfer(ph, t);770 if (!ret)771 *level = get_unaligned_le32(t->rx.buf);772 773 ph->xops->xfer_put(ph, t);774 return ret;775}776 777static int __scmi_perf_level_get(const struct scmi_protocol_handle *ph,778 struct perf_dom_info *dom, u32 *level,779 bool poll)780{781 if (dom->fc_info && dom->fc_info[PERF_FC_LEVEL].get_addr) {782 *level = ioread32(dom->fc_info[PERF_FC_LEVEL].get_addr);783 trace_scmi_fc_call(SCMI_PROTOCOL_PERF, PERF_LEVEL_GET,784 dom->id, *level, 0);785 return 0;786 }787 788 return scmi_perf_msg_level_get(ph, dom->id, level, poll);789}790 791static int scmi_perf_level_get(const struct scmi_protocol_handle *ph,792 u32 domain, u32 *level, bool poll)793{794 int ret;795 struct perf_dom_info *dom;796 797 dom = scmi_perf_domain_lookup(ph, domain);798 if (IS_ERR(dom))799 return PTR_ERR(dom);800 801 ret = __scmi_perf_level_get(ph, dom, level, poll);802 if (ret)803 return ret;804 805 if (dom->level_indexing_mode) {806 struct scmi_opp *opp;807 808 opp = xa_load(&dom->opps_by_idx, *level);809 if (!opp)810 return -EIO;811 812 *level = opp->perf;813 }814 815 return 0;816}817 818static int scmi_perf_level_limits_notify(const struct scmi_protocol_handle *ph,819 u32 domain, int message_id,820 bool enable)821{822 int ret;823 struct scmi_xfer *t;824 struct scmi_perf_notify_level_or_limits *notify;825 826 ret = ph->xops->xfer_get_init(ph, message_id, sizeof(*notify), 0, &t);827 if (ret)828 return ret;829 830 notify = t->tx.buf;831 notify->domain = cpu_to_le32(domain);832 notify->notify_enable = enable ? cpu_to_le32(BIT(0)) : 0;833 834 ret = ph->xops->do_xfer(ph, t);835 836 ph->xops->xfer_put(ph, t);837 return ret;838}839 840static void scmi_perf_domain_init_fc(const struct scmi_protocol_handle *ph,841 struct perf_dom_info *dom)842{843 struct scmi_fc_info *fc;844 845 fc = devm_kcalloc(ph->dev, PERF_FC_MAX, sizeof(*fc), GFP_KERNEL);846 if (!fc)847 return;848 849 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,850 PERF_LEVEL_GET, 4, dom->id,851 &fc[PERF_FC_LEVEL].get_addr, NULL,852 &fc[PERF_FC_LEVEL].rate_limit);853 854 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,855 PERF_LIMITS_GET, 8, dom->id,856 &fc[PERF_FC_LIMIT].get_addr, NULL,857 &fc[PERF_FC_LIMIT].rate_limit);858 859 if (dom->info.set_perf)860 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,861 PERF_LEVEL_SET, 4, dom->id,862 &fc[PERF_FC_LEVEL].set_addr,863 &fc[PERF_FC_LEVEL].set_db,864 &fc[PERF_FC_LEVEL].rate_limit);865 866 if (dom->set_limits)867 ph->hops->fastchannel_init(ph, PERF_DESCRIBE_FASTCHANNEL,868 PERF_LIMITS_SET, 8, dom->id,869 &fc[PERF_FC_LIMIT].set_addr,870 &fc[PERF_FC_LIMIT].set_db,871 &fc[PERF_FC_LIMIT].rate_limit);872 873 dom->fc_info = fc;874}875 876static int scmi_dvfs_device_opps_add(const struct scmi_protocol_handle *ph,877 struct device *dev, u32 domain)878{879 int idx, ret;880 unsigned long freq;881 struct dev_pm_opp_data data = {};882 struct perf_dom_info *dom;883 884 dom = scmi_perf_domain_lookup(ph, domain);885 if (IS_ERR(dom))886 return PTR_ERR(dom);887 888 for (idx = 0; idx < dom->opp_count; idx++) {889 if (!dom->level_indexing_mode)890 freq = dom->opp[idx].perf * dom->mult_factor;891 else892 freq = dom->opp[idx].indicative_freq * dom->mult_factor;893 894 /* All OPPs above the sustained frequency are treated as turbo */895 data.turbo = freq > dom->sustained_freq_khz * 1000;896 897 data.level = dom->opp[idx].perf;898 data.freq = freq;899 900 ret = dev_pm_opp_add_dynamic(dev, &data);901 if (ret) {902 dev_warn(dev, "[%d][%s]: Failed to add OPP[%d] %lu\n",903 domain, dom->info.name, idx, freq);904 dev_pm_opp_remove_all_dynamic(dev);905 return ret;906 }907 908 dev_dbg(dev, "[%d][%s]:: Registered OPP[%d] %lu\n",909 domain, dom->info.name, idx, freq);910 }911 return 0;912}913 914static int915scmi_dvfs_transition_latency_get(const struct scmi_protocol_handle *ph,916 u32 domain)917{918 struct perf_dom_info *dom;919 920 dom = scmi_perf_domain_lookup(ph, domain);921 if (IS_ERR(dom))922 return PTR_ERR(dom);923 924 /* uS to nS */925 return dom->opp[dom->opp_count - 1].trans_latency_us * 1000;926}927 928static int929scmi_dvfs_rate_limit_get(const struct scmi_protocol_handle *ph,930 u32 domain, u32 *rate_limit)931{932 struct perf_dom_info *dom;933 934 if (!rate_limit)935 return -EINVAL;936 937 dom = scmi_perf_domain_lookup(ph, domain);938 if (IS_ERR(dom))939 return PTR_ERR(dom);940 941 *rate_limit = dom->rate_limit_us;942 return 0;943}944 945static int scmi_dvfs_freq_set(const struct scmi_protocol_handle *ph, u32 domain,946 unsigned long freq, bool poll)947{948 unsigned int level;949 struct perf_dom_info *dom;950 951 dom = scmi_perf_domain_lookup(ph, domain);952 if (IS_ERR(dom))953 return PTR_ERR(dom);954 955 if (!dom->level_indexing_mode) {956 level = freq / dom->mult_factor;957 } else {958 struct scmi_opp *opp;959 960 opp = LOOKUP_BY_FREQ(dom->opps_by_freq,961 freq / dom->mult_factor);962 if (!opp)963 return -EIO;964 965 level = opp->level_index;966 }967 968 return __scmi_perf_level_set(ph, dom, level, poll);969}970 971static int scmi_dvfs_freq_get(const struct scmi_protocol_handle *ph, u32 domain,972 unsigned long *freq, bool poll)973{974 int ret;975 u32 level;976 struct perf_dom_info *dom;977 978 dom = scmi_perf_domain_lookup(ph, domain);979 if (IS_ERR(dom))980 return PTR_ERR(dom);981 982 ret = __scmi_perf_level_get(ph, dom, &level, poll);983 if (ret)984 return ret;985 986 if (!dom->level_indexing_mode) {987 *freq = level * dom->mult_factor;988 } else {989 struct scmi_opp *opp;990 991 opp = xa_load(&dom->opps_by_idx, level);992 if (!opp)993 return -EIO;994 995 *freq = opp->indicative_freq * dom->mult_factor;996 }997 998 return ret;999}1000 1001static int scmi_dvfs_est_power_get(const struct scmi_protocol_handle *ph,1002 u32 domain, unsigned long *freq,1003 unsigned long *power)1004{1005 struct perf_dom_info *dom;1006 unsigned long opp_freq;1007 int idx, ret = -EINVAL;1008 struct scmi_opp *opp;1009 1010 dom = scmi_perf_domain_lookup(ph, domain);1011 if (IS_ERR(dom))1012 return PTR_ERR(dom);1013 1014 for (opp = dom->opp, idx = 0; idx < dom->opp_count; idx++, opp++) {1015 if (!dom->level_indexing_mode)1016 opp_freq = opp->perf * dom->mult_factor;1017 else1018 opp_freq = opp->indicative_freq * dom->mult_factor;1019 1020 if (opp_freq < *freq)1021 continue;1022 1023 *freq = opp_freq;1024 *power = opp->power;1025 ret = 0;1026 break;1027 }1028 1029 return ret;1030}1031 1032static bool scmi_fast_switch_possible(const struct scmi_protocol_handle *ph,1033 u32 domain)1034{1035 struct perf_dom_info *dom;1036 1037 dom = scmi_perf_domain_lookup(ph, domain);1038 if (IS_ERR(dom))1039 return false;1040 1041 return dom->fc_info && dom->fc_info[PERF_FC_LEVEL].set_addr;1042}1043 1044static int scmi_fast_switch_rate_limit(const struct scmi_protocol_handle *ph,1045 u32 domain, u32 *rate_limit)1046{1047 struct perf_dom_info *dom;1048 1049 if (!rate_limit)1050 return -EINVAL;1051 1052 dom = scmi_perf_domain_lookup(ph, domain);1053 if (IS_ERR(dom))1054 return PTR_ERR(dom);1055 1056 if (!dom->fc_info)1057 return -EINVAL;1058 1059 *rate_limit = dom->fc_info[PERF_FC_LEVEL].rate_limit;1060 return 0;1061}1062 1063static enum scmi_power_scale1064scmi_power_scale_get(const struct scmi_protocol_handle *ph)1065{1066 struct scmi_perf_info *pi = ph->get_priv(ph);1067 1068 return pi->power_scale;1069}1070 1071static const struct scmi_perf_proto_ops perf_proto_ops = {1072 .num_domains_get = scmi_perf_num_domains_get,1073 .info_get = scmi_perf_info_get,1074 .limits_set = scmi_perf_limits_set,1075 .limits_get = scmi_perf_limits_get,1076 .level_set = scmi_perf_level_set,1077 .level_get = scmi_perf_level_get,1078 .transition_latency_get = scmi_dvfs_transition_latency_get,1079 .rate_limit_get = scmi_dvfs_rate_limit_get,1080 .device_opps_add = scmi_dvfs_device_opps_add,1081 .freq_set = scmi_dvfs_freq_set,1082 .freq_get = scmi_dvfs_freq_get,1083 .est_power_get = scmi_dvfs_est_power_get,1084 .fast_switch_possible = scmi_fast_switch_possible,1085 .fast_switch_rate_limit = scmi_fast_switch_rate_limit,1086 .power_scale_get = scmi_power_scale_get,1087};1088 1089static bool scmi_perf_notify_supported(const struct scmi_protocol_handle *ph,1090 u8 evt_id, u32 src_id)1091{1092 bool supported;1093 struct perf_dom_info *dom;1094 1095 if (evt_id >= ARRAY_SIZE(evt_2_cmd))1096 return false;1097 1098 dom = scmi_perf_domain_lookup(ph, src_id);1099 if (IS_ERR(dom))1100 return false;1101 1102 if (evt_id == SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED)1103 supported = dom->perf_limit_notify;1104 else1105 supported = dom->perf_level_notify;1106 1107 return supported;1108}1109 1110static int scmi_perf_set_notify_enabled(const struct scmi_protocol_handle *ph,1111 u8 evt_id, u32 src_id, bool enable)1112{1113 int ret, cmd_id;1114 1115 if (evt_id >= ARRAY_SIZE(evt_2_cmd))1116 return -EINVAL;1117 1118 cmd_id = evt_2_cmd[evt_id];1119 ret = scmi_perf_level_limits_notify(ph, src_id, cmd_id, enable);1120 if (ret)1121 pr_debug("FAIL_ENABLED - evt[%X] dom[%d] - ret:%d\n",1122 evt_id, src_id, ret);1123 1124 return ret;1125}1126 1127static int1128scmi_perf_xlate_opp_to_freq(struct perf_dom_info *dom,1129 unsigned int index, unsigned long *freq)1130{1131 struct scmi_opp *opp;1132 1133 if (!dom || !freq)1134 return -EINVAL;1135 1136 if (!dom->level_indexing_mode) {1137 opp = xa_load(&dom->opps_by_lvl, index);1138 if (!opp)1139 return -ENODEV;1140 1141 *freq = opp->perf * dom->mult_factor;1142 } else {1143 opp = xa_load(&dom->opps_by_idx, index);1144 if (!opp)1145 return -ENODEV;1146 1147 *freq = opp->indicative_freq * dom->mult_factor;1148 }1149 1150 return 0;1151}1152 1153static void *scmi_perf_fill_custom_report(const struct scmi_protocol_handle *ph,1154 u8 evt_id, ktime_t timestamp,1155 const void *payld, size_t payld_sz,1156 void *report, u32 *src_id)1157{1158 int ret;1159 void *rep = NULL;1160 struct perf_dom_info *dom;1161 1162 switch (evt_id) {1163 case SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED:1164 {1165 const struct scmi_perf_limits_notify_payld *p = payld;1166 struct scmi_perf_limits_report *r = report;1167 unsigned long freq_min, freq_max;1168 1169 if (sizeof(*p) != payld_sz)1170 break;1171 1172 r->timestamp = timestamp;1173 r->agent_id = le32_to_cpu(p->agent_id);1174 r->domain_id = le32_to_cpu(p->domain_id);1175 r->range_max = le32_to_cpu(p->range_max);1176 r->range_min = le32_to_cpu(p->range_min);1177 /* Check if the reported domain exist at all */1178 dom = scmi_perf_domain_lookup(ph, r->domain_id);1179 if (IS_ERR(dom))1180 break;1181 /*1182 * Event will be reported from this point on...1183 * ...even if, later, xlated frequencies were not retrieved.1184 */1185 *src_id = r->domain_id;1186 rep = r;1187 1188 ret = scmi_perf_xlate_opp_to_freq(dom, r->range_max, &freq_max);1189 if (ret)1190 break;1191 1192 ret = scmi_perf_xlate_opp_to_freq(dom, r->range_min, &freq_min);1193 if (ret)1194 break;1195 1196 /* Report translated freqs ONLY if both available */1197 r->range_max_freq = freq_max;1198 r->range_min_freq = freq_min;1199 1200 break;1201 }1202 case SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED:1203 {1204 const struct scmi_perf_level_notify_payld *p = payld;1205 struct scmi_perf_level_report *r = report;1206 unsigned long freq;1207 1208 if (sizeof(*p) != payld_sz)1209 break;1210 1211 r->timestamp = timestamp;1212 r->agent_id = le32_to_cpu(p->agent_id);1213 r->domain_id = le32_to_cpu(p->domain_id);1214 /* Report translated freqs ONLY if available */1215 r->performance_level = le32_to_cpu(p->performance_level);1216 /* Check if the reported domain exist at all */1217 dom = scmi_perf_domain_lookup(ph, r->domain_id);1218 if (IS_ERR(dom))1219 break;1220 /*1221 * Event will be reported from this point on...1222 * ...even if, later, xlated frequencies were not retrieved.1223 */1224 *src_id = r->domain_id;1225 rep = r;1226 1227 /* Report translated freqs ONLY if available */1228 ret = scmi_perf_xlate_opp_to_freq(dom, r->performance_level,1229 &freq);1230 if (ret)1231 break;1232 1233 r->performance_level_freq = freq;1234 1235 break;1236 }1237 default:1238 break;1239 }1240 1241 return rep;1242}1243 1244static int scmi_perf_get_num_sources(const struct scmi_protocol_handle *ph)1245{1246 struct scmi_perf_info *pi = ph->get_priv(ph);1247 1248 if (!pi)1249 return -EINVAL;1250 1251 return pi->num_domains;1252}1253 1254static const struct scmi_event perf_events[] = {1255 {1256 .id = SCMI_EVENT_PERFORMANCE_LIMITS_CHANGED,1257 .max_payld_sz = sizeof(struct scmi_perf_limits_notify_payld),1258 .max_report_sz = sizeof(struct scmi_perf_limits_report),1259 },1260 {1261 .id = SCMI_EVENT_PERFORMANCE_LEVEL_CHANGED,1262 .max_payld_sz = sizeof(struct scmi_perf_level_notify_payld),1263 .max_report_sz = sizeof(struct scmi_perf_level_report),1264 },1265};1266 1267static const struct scmi_event_ops perf_event_ops = {1268 .is_notify_supported = scmi_perf_notify_supported,1269 .get_num_sources = scmi_perf_get_num_sources,1270 .set_notify_enabled = scmi_perf_set_notify_enabled,1271 .fill_custom_report = scmi_perf_fill_custom_report,1272};1273 1274static const struct scmi_protocol_events perf_protocol_events = {1275 .queue_sz = SCMI_PROTO_QUEUE_SZ,1276 .ops = &perf_event_ops,1277 .evts = perf_events,1278 .num_events = ARRAY_SIZE(perf_events),1279};1280 1281static int scmi_perf_protocol_init(const struct scmi_protocol_handle *ph)1282{1283 int domain, ret;1284 u32 version;1285 struct scmi_perf_info *pinfo;1286 1287 ret = ph->xops->version_get(ph, &version);1288 if (ret)1289 return ret;1290 1291 dev_dbg(ph->dev, "Performance Version %d.%d\n",1292 PROTOCOL_REV_MAJOR(version), PROTOCOL_REV_MINOR(version));1293 1294 pinfo = devm_kzalloc(ph->dev, sizeof(*pinfo), GFP_KERNEL);1295 if (!pinfo)1296 return -ENOMEM;1297 1298 pinfo->version = version;1299 1300 ret = scmi_perf_attributes_get(ph, pinfo);1301 if (ret)1302 return ret;1303 1304 pinfo->dom_info = devm_kcalloc(ph->dev, pinfo->num_domains,1305 sizeof(*pinfo->dom_info), GFP_KERNEL);1306 if (!pinfo->dom_info)1307 return -ENOMEM;1308 1309 for (domain = 0; domain < pinfo->num_domains; domain++) {1310 struct perf_dom_info *dom = pinfo->dom_info + domain;1311 1312 dom->id = domain;1313 scmi_perf_domain_attributes_get(ph, dom, pinfo->notify_lim_cmd,1314 pinfo->notify_lvl_cmd, version);1315 scmi_perf_describe_levels_get(ph, dom, version);1316 1317 if (dom->perf_fastchannels)1318 scmi_perf_domain_init_fc(ph, dom);1319 }1320 1321 ret = devm_add_action_or_reset(ph->dev, scmi_perf_xa_destroy, pinfo);1322 if (ret)1323 return ret;1324 1325 return ph->set_priv(ph, pinfo, version);1326}1327 1328static const struct scmi_protocol scmi_perf = {1329 .id = SCMI_PROTOCOL_PERF,1330 .owner = THIS_MODULE,1331 .instance_init = &scmi_perf_protocol_init,1332 .ops = &perf_proto_ops,1333 .events = &perf_protocol_events,1334 .supported_version = SCMI_PROTOCOL_SUPPORTED_VERSION,1335};1336 1337DEFINE_SCMI_PROTOCOL_REGISTER_UNREGISTER(perf, scmi_perf)1338