brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/*3 * System Control and Management Interface (SCMI) Message Protocol driver4 *5 * SCMI Message Protocol is used between the System Control Processor(SCP)6 * and the Application Processors(AP). The Message Handling Unit(MHU)7 * provides a mechanism for inter-processor communication between SCP's8 * Cortex M3 and AP.9 *10 * SCP offers control and management of the core/cluster power states,11 * various power domain DVFS including the core/cluster, certain system12 * clocks configuration, thermal sensors and many others.13 *14 * Copyright (C) 2018-2024 ARM Ltd.15 */16 17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt18 19#include <linux/bitmap.h>20#include <linux/debugfs.h>21#include <linux/device.h>22#include <linux/export.h>23#include <linux/idr.h>24#include <linux/io.h>25#include <linux/io-64-nonatomic-hi-lo.h>26#include <linux/kernel.h>27#include <linux/ktime.h>28#include <linux/hashtable.h>29#include <linux/list.h>30#include <linux/module.h>31#include <linux/of.h>32#include <linux/platform_device.h>33#include <linux/processor.h>34#include <linux/refcount.h>35#include <linux/slab.h>36#include <linux/xarray.h>37 38#include "common.h"39#include "notify.h"40 41#include "raw_mode.h"42 43#define CREATE_TRACE_POINTS44#include <trace/events/scmi.h>45 46static DEFINE_IDA(scmi_id);47 48static DEFINE_XARRAY(scmi_protocols);49 50/* List of all SCMI devices active in system */51static LIST_HEAD(scmi_list);52/* Protection for the entire list */53static DEFINE_MUTEX(scmi_list_mutex);54/* Track the unique id for the transfers for debug & profiling purpose */55static atomic_t transfer_last_id;56 57static struct dentry *scmi_top_dentry;58 59/**60 * struct scmi_xfers_info - Structure to manage transfer information61 *62 * @xfer_alloc_table: Bitmap table for allocated messages.63 *	Index of this bitmap table is also used for message64 *	sequence identifier.65 * @xfer_lock: Protection for message allocation66 * @max_msg: Maximum number of messages that can be pending67 * @free_xfers: A free list for available to use xfers. It is initialized with68 *		a number of xfers equal to the maximum allowed in-flight69 *		messages.70 * @pending_xfers: An hashtable, indexed by msg_hdr.seq, used to keep all the71 *		   currently in-flight messages.72 */73struct scmi_xfers_info {74	unsigned long *xfer_alloc_table;75	spinlock_t xfer_lock;76	int max_msg;77	struct hlist_head free_xfers;78	DECLARE_HASHTABLE(pending_xfers, SCMI_PENDING_XFERS_HT_ORDER_SZ);79};80 81/**82 * struct scmi_protocol_instance  - Describe an initialized protocol instance.83 * @handle: Reference to the SCMI handle associated to this protocol instance.84 * @proto: A reference to the protocol descriptor.85 * @gid: A reference for per-protocol devres management.86 * @users: A refcount to track effective users of this protocol.87 * @priv: Reference for optional protocol private data.88 * @version: Protocol version supported by the platform as detected at runtime.89 * @negotiated_version: When the platform supports a newer protocol version,90 *			the agent will try to negotiate with the platform the91 *			usage of the newest version known to it, since92 *			backward compatibility is NOT automatically assured.93 *			This field is NON-zero when a successful negotiation94 *			has completed.95 * @ph: An embedded protocol handle that will be passed down to protocol96 *	initialization code to identify this instance.97 *98 * Each protocol is initialized independently once for each SCMI platform in99 * which is defined by DT and implemented by the SCMI server fw.100 */101struct scmi_protocol_instance {102	const struct scmi_handle	*handle;103	const struct scmi_protocol	*proto;104	void				*gid;105	refcount_t			users;106	void				*priv;107	unsigned int			version;108	unsigned int			negotiated_version;109	struct scmi_protocol_handle	ph;110};111 112#define ph_to_pi(h)	container_of(h, struct scmi_protocol_instance, ph)113 114/**115 * struct scmi_debug_info  - Debug common info116 * @top_dentry: A reference to the top debugfs dentry117 * @name: Name of this SCMI instance118 * @type: Type of this SCMI instance119 * @is_atomic: Flag to state if the transport of this instance is atomic120 * @counters: An array of atomic_c's used for tracking statistics (if enabled)121 */122struct scmi_debug_info {123	struct dentry *top_dentry;124	const char *name;125	const char *type;126	bool is_atomic;127	atomic_t counters[SCMI_DEBUG_COUNTERS_LAST];128};129 130/**131 * struct scmi_info - Structure representing a SCMI instance132 *133 * @id: A sequence number starting from zero identifying this instance134 * @dev: Device pointer135 * @desc: SoC description for this instance136 * @version: SCMI revision information containing protocol version,137 *	implementation version and (sub-)vendor identification.138 * @handle: Instance of SCMI handle to send to clients139 * @tx_minfo: Universal Transmit Message management info140 * @rx_minfo: Universal Receive Message management info141 * @tx_idr: IDR object to map protocol id to Tx channel info pointer142 * @rx_idr: IDR object to map protocol id to Rx channel info pointer143 * @protocols: IDR for protocols' instance descriptors initialized for144 *	       this SCMI instance: populated on protocol's first attempted145 *	       usage.146 * @protocols_mtx: A mutex to protect protocols instances initialization.147 * @protocols_imp: List of protocols implemented, currently maximum of148 *		   scmi_revision_info.num_protocols elements allocated by the149 *		   base protocol150 * @active_protocols: IDR storing device_nodes for protocols actually defined151 *		      in the DT and confirmed as implemented by fw.152 * @atomic_threshold: Optional system wide DT-configured threshold, expressed153 *		      in microseconds, for atomic operations.154 *		      Only SCMI synchronous commands reported by the platform155 *		      to have an execution latency lesser-equal to the threshold156 *		      should be considered for atomic mode operation: such157 *		      decision is finally left up to the SCMI drivers.158 * @notify_priv: Pointer to private data structure specific to notifications.159 * @node: List head160 * @users: Number of users of this instance161 * @bus_nb: A notifier to listen for device bind/unbind on the scmi bus162 * @dev_req_nb: A notifier to listen for device request/unrequest on the scmi163 *		bus164 * @devreq_mtx: A mutex to serialize device creation for this SCMI instance165 * @dbg: A pointer to debugfs related data (if any)166 * @raw: An opaque reference handle used by SCMI Raw mode.167 */168struct scmi_info {169	int id;170	struct device *dev;171	const struct scmi_desc *desc;172	struct scmi_revision_info version;173	struct scmi_handle handle;174	struct scmi_xfers_info tx_minfo;175	struct scmi_xfers_info rx_minfo;176	struct idr tx_idr;177	struct idr rx_idr;178	struct idr protocols;179	/* Ensure mutual exclusive access to protocols instance array */180	struct mutex protocols_mtx;181	u8 *protocols_imp;182	struct idr active_protocols;183	unsigned int atomic_threshold;184	void *notify_priv;185	struct list_head node;186	int users;187	struct notifier_block bus_nb;188	struct notifier_block dev_req_nb;189	/* Serialize device creation process for this instance */190	struct mutex devreq_mtx;191	struct scmi_debug_info *dbg;192	void *raw;193};194 195#define handle_to_scmi_info(h)	container_of(h, struct scmi_info, handle)196#define bus_nb_to_scmi_info(nb)	container_of(nb, struct scmi_info, bus_nb)197#define req_nb_to_scmi_info(nb)	container_of(nb, struct scmi_info, dev_req_nb)198 199static void scmi_rx_callback(struct scmi_chan_info *cinfo,200			     u32 msg_hdr, void *priv);201static void scmi_bad_message_trace(struct scmi_chan_info *cinfo,202				   u32 msg_hdr, enum scmi_bad_msg err);203 204static struct scmi_transport_core_operations scmi_trans_core_ops = {205	.bad_message_trace = scmi_bad_message_trace,206	.rx_callback = scmi_rx_callback,207};208 209static unsigned long210scmi_vendor_protocol_signature(unsigned int protocol_id, char *vendor_id,211			       char *sub_vendor_id, u32 impl_ver)212{213	char *signature, *p;214	unsigned long hash = 0;215 216	/* vendor_id/sub_vendor_id guaranteed <= SCMI_SHORT_NAME_MAX_SIZE */217	signature = kasprintf(GFP_KERNEL, "%02X|%s|%s|0x%08X", protocol_id,218			      vendor_id ?: "", sub_vendor_id ?: "", impl_ver);219	if (!signature)220		return 0;221 222	p = signature;223	while (*p)224		hash = partial_name_hash(tolower(*p++), hash);225	hash = end_name_hash(hash);226 227	kfree(signature);228 229	return hash;230}231 232static unsigned long233scmi_protocol_key_calculate(int protocol_id, char *vendor_id,234			    char *sub_vendor_id, u32 impl_ver)235{236	if (protocol_id < SCMI_PROTOCOL_VENDOR_BASE)237		return protocol_id;238	else239		return scmi_vendor_protocol_signature(protocol_id, vendor_id,240						      sub_vendor_id, impl_ver);241}242 243static const struct scmi_protocol *244__scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id,245			      char *sub_vendor_id, u32 impl_ver)246{247	unsigned long key;248	struct scmi_protocol *proto = NULL;249 250	key = scmi_protocol_key_calculate(protocol_id, vendor_id,251					  sub_vendor_id, impl_ver);252	if (key)253		proto = xa_load(&scmi_protocols, key);254 255	return proto;256}257 258static const struct scmi_protocol *259scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id,260			    char *sub_vendor_id, u32 impl_ver)261{262	const struct scmi_protocol *proto = NULL;263 264	/* Searching for closest match ...*/265	proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id,266					      sub_vendor_id, impl_ver);267	if (proto)268		return proto;269 270	/* Any match just on vendor/sub_vendor ? */271	if (impl_ver) {272		proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id,273						      sub_vendor_id, 0);274		if (proto)275			return proto;276	}277 278	/* Any match just on the vendor ? */279	if (sub_vendor_id)280		proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id,281						      NULL, 0);282	return proto;283}284 285static const struct scmi_protocol *286scmi_protocol_get(int protocol_id, struct scmi_revision_info *version)287{288	const struct scmi_protocol *proto = NULL;289 290	if (protocol_id < SCMI_PROTOCOL_VENDOR_BASE)291		proto = xa_load(&scmi_protocols, protocol_id);292	else293		proto = scmi_vendor_protocol_lookup(protocol_id,294						    version->vendor_id,295						    version->sub_vendor_id,296						    version->impl_ver);297	if (!proto || !try_module_get(proto->owner)) {298		pr_warn("SCMI Protocol 0x%x not found!\n", protocol_id);299		return NULL;300	}301 302	pr_debug("Found SCMI Protocol 0x%x\n", protocol_id);303 304	if (protocol_id >= SCMI_PROTOCOL_VENDOR_BASE)305		pr_info("Loaded SCMI Vendor Protocol 0x%x - %s %s %X\n",306			protocol_id, proto->vendor_id ?: "",307			proto->sub_vendor_id ?: "", proto->impl_ver);308 309	return proto;310}311 312static void scmi_protocol_put(const struct scmi_protocol *proto)313{314	if (proto)315		module_put(proto->owner);316}317 318static int scmi_vendor_protocol_check(const struct scmi_protocol *proto)319{320	if (!proto->vendor_id) {321		pr_err("missing vendor_id for protocol 0x%x\n", proto->id);322		return -EINVAL;323	}324 325	if (strlen(proto->vendor_id) >= SCMI_SHORT_NAME_MAX_SIZE) {326		pr_err("malformed vendor_id for protocol 0x%x\n", proto->id);327		return -EINVAL;328	}329 330	if (proto->sub_vendor_id &&331	    strlen(proto->sub_vendor_id) >= SCMI_SHORT_NAME_MAX_SIZE) {332		pr_err("malformed sub_vendor_id for protocol 0x%x\n",333		       proto->id);334		return -EINVAL;335	}336 337	return 0;338}339 340int scmi_protocol_register(const struct scmi_protocol *proto)341{342	int ret;343	unsigned long key;344 345	if (!proto) {346		pr_err("invalid protocol\n");347		return -EINVAL;348	}349 350	if (!proto->instance_init) {351		pr_err("missing init for protocol 0x%x\n", proto->id);352		return -EINVAL;353	}354 355	if (proto->id >= SCMI_PROTOCOL_VENDOR_BASE &&356	    scmi_vendor_protocol_check(proto))357		return -EINVAL;358 359	/*360	 * Calculate a protocol key to register this protocol with the core;361	 * key value 0 is considered invalid.362	 */363	key = scmi_protocol_key_calculate(proto->id, proto->vendor_id,364					  proto->sub_vendor_id,365					  proto->impl_ver);366	if (!key)367		return -EINVAL;368 369	ret = xa_insert(&scmi_protocols, key, (void *)proto, GFP_KERNEL);370	if (ret) {371		pr_err("unable to allocate SCMI protocol slot for 0x%x - err %d\n",372		       proto->id, ret);373		return ret;374	}375 376	pr_debug("Registered SCMI Protocol 0x%x\n", proto->id);377 378	return 0;379}380EXPORT_SYMBOL_GPL(scmi_protocol_register);381 382void scmi_protocol_unregister(const struct scmi_protocol *proto)383{384	unsigned long key;385 386	key = scmi_protocol_key_calculate(proto->id, proto->vendor_id,387					  proto->sub_vendor_id,388					  proto->impl_ver);389	if (!key)390		return;391 392	xa_erase(&scmi_protocols, key);393 394	pr_debug("Unregistered SCMI Protocol 0x%x\n", proto->id);395}396EXPORT_SYMBOL_GPL(scmi_protocol_unregister);397 398/**399 * scmi_create_protocol_devices  - Create devices for all pending requests for400 * this SCMI instance.401 *402 * @np: The device node describing the protocol403 * @info: The SCMI instance descriptor404 * @prot_id: The protocol ID405 * @name: The optional name of the device to be created: if not provided this406 *	  call will lead to the creation of all the devices currently requested407 *	  for the specified protocol.408 */409static void scmi_create_protocol_devices(struct device_node *np,410					 struct scmi_info *info,411					 int prot_id, const char *name)412{413	struct scmi_device *sdev;414 415	mutex_lock(&info->devreq_mtx);416	sdev = scmi_device_create(np, info->dev, prot_id, name);417	if (name && !sdev)418		dev_err(info->dev,419			"failed to create device for protocol 0x%X (%s)\n",420			prot_id, name);421	mutex_unlock(&info->devreq_mtx);422}423 424static void scmi_destroy_protocol_devices(struct scmi_info *info,425					  int prot_id, const char *name)426{427	mutex_lock(&info->devreq_mtx);428	scmi_device_destroy(info->dev, prot_id, name);429	mutex_unlock(&info->devreq_mtx);430}431 432void scmi_notification_instance_data_set(const struct scmi_handle *handle,433					 void *priv)434{435	struct scmi_info *info = handle_to_scmi_info(handle);436 437	info->notify_priv = priv;438	/* Ensure updated protocol private date are visible */439	smp_wmb();440}441 442void *scmi_notification_instance_data_get(const struct scmi_handle *handle)443{444	struct scmi_info *info = handle_to_scmi_info(handle);445 446	/* Ensure protocols_private_data has been updated */447	smp_rmb();448	return info->notify_priv;449}450 451/**452 * scmi_xfer_token_set  - Reserve and set new token for the xfer at hand453 *454 * @minfo: Pointer to Tx/Rx Message management info based on channel type455 * @xfer: The xfer to act upon456 *457 * Pick the next unused monotonically increasing token and set it into458 * xfer->hdr.seq: picking a monotonically increasing value avoids immediate459 * reuse of freshly completed or timed-out xfers, thus mitigating the risk460 * of incorrect association of a late and expired xfer with a live in-flight461 * transaction, both happening to re-use the same token identifier.462 *463 * Since platform is NOT required to answer our request in-order we should464 * account for a few rare but possible scenarios:465 *466 *  - exactly 'next_token' may be NOT available so pick xfer_id >= next_token467 *    using find_next_zero_bit() starting from candidate next_token bit468 *469 *  - all tokens ahead upto (MSG_TOKEN_ID_MASK - 1) are used in-flight but we470 *    are plenty of free tokens at start, so try a second pass using471 *    find_next_zero_bit() and starting from 0.472 *473 *  X = used in-flight474 *475 * Normal476 * ------477 *478 *		|- xfer_id picked479 *   -----------+----------------------------------------------------------480 *   | | |X|X|X| | | | | | ... ... ... ... ... ... ... ... ... ... ...|X|X|481 *   ----------------------------------------------------------------------482 *		^483 *		|- next_token484 *485 * Out-of-order pending at start486 * -----------------------------487 *488 *	  |- xfer_id picked, last_token fixed489 *   -----+----------------------------------------------------------------490 *   |X|X| | | | |X|X| ... ... ... ... ... ... ... ... ... ... ... ...|X| |491 *   ----------------------------------------------------------------------492 *    ^493 *    |- next_token494 *495 *496 * Out-of-order pending at end497 * ---------------------------498 *499 *	  |- xfer_id picked, last_token fixed500 *   -----+----------------------------------------------------------------501 *   |X|X| | | | |X|X| ... ... ... ... ... ... ... ... ... ... |X|X|X||X|X|502 *   ----------------------------------------------------------------------503 *								^504 *								|- next_token505 *506 * Context: Assumes to be called with @xfer_lock already acquired.507 *508 * Return: 0 on Success or error509 */510static int scmi_xfer_token_set(struct scmi_xfers_info *minfo,511			       struct scmi_xfer *xfer)512{513	unsigned long xfer_id, next_token;514 515	/*516	 * Pick a candidate monotonic token in range [0, MSG_TOKEN_MAX - 1]517	 * using the pre-allocated transfer_id as a base.518	 * Note that the global transfer_id is shared across all message types519	 * so there could be holes in the allocated set of monotonic sequence520	 * numbers, but that is going to limit the effectiveness of the521	 * mitigation only in very rare limit conditions.522	 */523	next_token = (xfer->transfer_id & (MSG_TOKEN_MAX - 1));524 525	/* Pick the next available xfer_id >= next_token */526	xfer_id = find_next_zero_bit(minfo->xfer_alloc_table,527				     MSG_TOKEN_MAX, next_token);528	if (xfer_id == MSG_TOKEN_MAX) {529		/*530		 * After heavily out-of-order responses, there are no free531		 * tokens ahead, but only at start of xfer_alloc_table so532		 * try again from the beginning.533		 */534		xfer_id = find_next_zero_bit(minfo->xfer_alloc_table,535					     MSG_TOKEN_MAX, 0);536		/*537		 * Something is wrong if we got here since there can be a538		 * maximum number of (MSG_TOKEN_MAX - 1) in-flight messages539		 * but we have not found any free token [0, MSG_TOKEN_MAX - 1].540		 */541		if (WARN_ON_ONCE(xfer_id == MSG_TOKEN_MAX))542			return -ENOMEM;543	}544 545	/* Update +/- last_token accordingly if we skipped some hole */546	if (xfer_id != next_token)547		atomic_add((int)(xfer_id - next_token), &transfer_last_id);548 549	xfer->hdr.seq = (u16)xfer_id;550 551	return 0;552}553 554/**555 * scmi_xfer_token_clear  - Release the token556 *557 * @minfo: Pointer to Tx/Rx Message management info based on channel type558 * @xfer: The xfer to act upon559 */560static inline void scmi_xfer_token_clear(struct scmi_xfers_info *minfo,561					 struct scmi_xfer *xfer)562{563	clear_bit(xfer->hdr.seq, minfo->xfer_alloc_table);564}565 566/**567 * scmi_xfer_inflight_register_unlocked  - Register the xfer as in-flight568 *569 * @xfer: The xfer to register570 * @minfo: Pointer to Tx/Rx Message management info based on channel type571 *572 * Note that this helper assumes that the xfer to be registered as in-flight573 * had been built using an xfer sequence number which still corresponds to a574 * free slot in the xfer_alloc_table.575 *576 * Context: Assumes to be called with @xfer_lock already acquired.577 */578static inline void579scmi_xfer_inflight_register_unlocked(struct scmi_xfer *xfer,580				     struct scmi_xfers_info *minfo)581{582	/* Set in-flight */583	set_bit(xfer->hdr.seq, minfo->xfer_alloc_table);584	hash_add(minfo->pending_xfers, &xfer->node, xfer->hdr.seq);585	xfer->pending = true;586}587 588/**589 * scmi_xfer_inflight_register  - Try to register an xfer as in-flight590 *591 * @xfer: The xfer to register592 * @minfo: Pointer to Tx/Rx Message management info based on channel type593 *594 * Note that this helper does NOT assume anything about the sequence number595 * that was baked into the provided xfer, so it checks at first if it can596 * be mapped to a free slot and fails with an error if another xfer with the597 * same sequence number is currently still registered as in-flight.598 *599 * Return: 0 on Success or -EBUSY if sequence number embedded in the xfer600 *	   could not rbe mapped to a free slot in the xfer_alloc_table.601 */602static int scmi_xfer_inflight_register(struct scmi_xfer *xfer,603				       struct scmi_xfers_info *minfo)604{605	int ret = 0;606	unsigned long flags;607 608	spin_lock_irqsave(&minfo->xfer_lock, flags);609	if (!test_bit(xfer->hdr.seq, minfo->xfer_alloc_table))610		scmi_xfer_inflight_register_unlocked(xfer, minfo);611	else612		ret = -EBUSY;613	spin_unlock_irqrestore(&minfo->xfer_lock, flags);614 615	return ret;616}617 618/**619 * scmi_xfer_raw_inflight_register  - An helper to register the given xfer as in620 * flight on the TX channel, if possible.621 *622 * @handle: Pointer to SCMI entity handle623 * @xfer: The xfer to register624 *625 * Return: 0 on Success, error otherwise626 */627int scmi_xfer_raw_inflight_register(const struct scmi_handle *handle,628				    struct scmi_xfer *xfer)629{630	struct scmi_info *info = handle_to_scmi_info(handle);631 632	return scmi_xfer_inflight_register(xfer, &info->tx_minfo);633}634 635/**636 * scmi_xfer_pending_set  - Pick a proper sequence number and mark the xfer637 * as pending in-flight638 *639 * @xfer: The xfer to act upon640 * @minfo: Pointer to Tx/Rx Message management info based on channel type641 *642 * Return: 0 on Success or error otherwise643 */644static inline int scmi_xfer_pending_set(struct scmi_xfer *xfer,645					struct scmi_xfers_info *minfo)646{647	int ret;648	unsigned long flags;649 650	spin_lock_irqsave(&minfo->xfer_lock, flags);651	/* Set a new monotonic token as the xfer sequence number */652	ret = scmi_xfer_token_set(minfo, xfer);653	if (!ret)654		scmi_xfer_inflight_register_unlocked(xfer, minfo);655	spin_unlock_irqrestore(&minfo->xfer_lock, flags);656 657	return ret;658}659 660/**661 * scmi_xfer_get() - Allocate one message662 *663 * @handle: Pointer to SCMI entity handle664 * @minfo: Pointer to Tx/Rx Message management info based on channel type665 *666 * Helper function which is used by various message functions that are667 * exposed to clients of this driver for allocating a message traffic event.668 *669 * Picks an xfer from the free list @free_xfers (if any available) and perform670 * a basic initialization.671 *672 * Note that, at this point, still no sequence number is assigned to the673 * allocated xfer, nor it is registered as a pending transaction.674 *675 * The successfully initialized xfer is refcounted.676 *677 * Context: Holds @xfer_lock while manipulating @free_xfers.678 *679 * Return: An initialized xfer if all went fine, else pointer error.680 */681static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle,682				       struct scmi_xfers_info *minfo)683{684	unsigned long flags;685	struct scmi_xfer *xfer;686 687	spin_lock_irqsave(&minfo->xfer_lock, flags);688	if (hlist_empty(&minfo->free_xfers)) {689		spin_unlock_irqrestore(&minfo->xfer_lock, flags);690		return ERR_PTR(-ENOMEM);691	}692 693	/* grab an xfer from the free_list */694	xfer = hlist_entry(minfo->free_xfers.first, struct scmi_xfer, node);695	hlist_del_init(&xfer->node);696 697	/*698	 * Allocate transfer_id early so that can be used also as base for699	 * monotonic sequence number generation if needed.700	 */701	xfer->transfer_id = atomic_inc_return(&transfer_last_id);702 703	refcount_set(&xfer->users, 1);704	atomic_set(&xfer->busy, SCMI_XFER_FREE);705	spin_unlock_irqrestore(&minfo->xfer_lock, flags);706 707	return xfer;708}709 710/**711 * scmi_xfer_raw_get  - Helper to get a bare free xfer from the TX channel712 *713 * @handle: Pointer to SCMI entity handle714 *715 * Note that xfer is taken from the TX channel structures.716 *717 * Return: A valid xfer on Success, or an error-pointer otherwise718 */719struct scmi_xfer *scmi_xfer_raw_get(const struct scmi_handle *handle)720{721	struct scmi_xfer *xfer;722	struct scmi_info *info = handle_to_scmi_info(handle);723 724	xfer = scmi_xfer_get(handle, &info->tx_minfo);725	if (!IS_ERR(xfer))726		xfer->flags |= SCMI_XFER_FLAG_IS_RAW;727 728	return xfer;729}730 731/**732 * scmi_xfer_raw_channel_get  - Helper to get a reference to the proper channel733 * to use for a specific protocol_id Raw transaction.734 *735 * @handle: Pointer to SCMI entity handle736 * @protocol_id: Identifier of the protocol737 *738 * Note that in a regular SCMI stack, usually, a protocol has to be defined in739 * the DT to have an associated channel and be usable; but in Raw mode any740 * protocol in range is allowed, re-using the Base channel, so as to enable741 * fuzzing on any protocol without the need of a fully compiled DT.742 *743 * Return: A reference to the channel to use, or an ERR_PTR744 */745struct scmi_chan_info *746scmi_xfer_raw_channel_get(const struct scmi_handle *handle, u8 protocol_id)747{748	struct scmi_chan_info *cinfo;749	struct scmi_info *info = handle_to_scmi_info(handle);750 751	cinfo = idr_find(&info->tx_idr, protocol_id);752	if (!cinfo) {753		if (protocol_id == SCMI_PROTOCOL_BASE)754			return ERR_PTR(-EINVAL);755		/* Use Base channel for protocols not defined for DT */756		cinfo = idr_find(&info->tx_idr, SCMI_PROTOCOL_BASE);757		if (!cinfo)758			return ERR_PTR(-EINVAL);759		dev_warn_once(handle->dev,760			      "Using Base channel for protocol 0x%X\n",761			      protocol_id);762	}763 764	return cinfo;765}766 767/**768 * __scmi_xfer_put() - Release a message769 *770 * @minfo: Pointer to Tx/Rx Message management info based on channel type771 * @xfer: message that was reserved by scmi_xfer_get772 *773 * After refcount check, possibly release an xfer, clearing the token slot,774 * removing xfer from @pending_xfers and putting it back into free_xfers.775 *776 * This holds a spinlock to maintain integrity of internal data structures.777 */778static void779__scmi_xfer_put(struct scmi_xfers_info *minfo, struct scmi_xfer *xfer)780{781	unsigned long flags;782 783	spin_lock_irqsave(&minfo->xfer_lock, flags);784	if (refcount_dec_and_test(&xfer->users)) {785		if (xfer->pending) {786			scmi_xfer_token_clear(minfo, xfer);787			hash_del(&xfer->node);788			xfer->pending = false;789		}790		hlist_add_head(&xfer->node, &minfo->free_xfers);791	}792	spin_unlock_irqrestore(&minfo->xfer_lock, flags);793}794 795/**796 * scmi_xfer_raw_put  - Release an xfer that was taken by @scmi_xfer_raw_get797 *798 * @handle: Pointer to SCMI entity handle799 * @xfer: A reference to the xfer to put800 *801 * Note that as with other xfer_put() handlers the xfer is really effectively802 * released only if there are no more users on the system.803 */804void scmi_xfer_raw_put(const struct scmi_handle *handle, struct scmi_xfer *xfer)805{806	struct scmi_info *info = handle_to_scmi_info(handle);807 808	xfer->flags &= ~SCMI_XFER_FLAG_IS_RAW;809	xfer->flags &= ~SCMI_XFER_FLAG_CHAN_SET;810	return __scmi_xfer_put(&info->tx_minfo, xfer);811}812 813/**814 * scmi_xfer_lookup_unlocked  -  Helper to lookup an xfer_id815 *816 * @minfo: Pointer to Tx/Rx Message management info based on channel type817 * @xfer_id: Token ID to lookup in @pending_xfers818 *819 * Refcounting is untouched.820 *821 * Context: Assumes to be called with @xfer_lock already acquired.822 *823 * Return: A valid xfer on Success or error otherwise824 */825static struct scmi_xfer *826scmi_xfer_lookup_unlocked(struct scmi_xfers_info *minfo, u16 xfer_id)827{828	struct scmi_xfer *xfer = NULL;829 830	if (test_bit(xfer_id, minfo->xfer_alloc_table))831		xfer = XFER_FIND(minfo->pending_xfers, xfer_id);832 833	return xfer ?: ERR_PTR(-EINVAL);834}835 836/**837 * scmi_bad_message_trace  - A helper to trace weird messages838 *839 * @cinfo: A reference to the channel descriptor on which the message was840 *	   received841 * @msg_hdr: Message header to track842 * @err: A specific error code used as a status value in traces.843 *844 * This helper can be used to trace any kind of weird, incomplete, unexpected,845 * timed-out message that arrives and as such, can be traced only referring to846 * the header content, since the payload is missing/unreliable.847 */848static void scmi_bad_message_trace(struct scmi_chan_info *cinfo, u32 msg_hdr,849				   enum scmi_bad_msg err)850{851	char *tag;852	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);853 854	switch (MSG_XTRACT_TYPE(msg_hdr)) {855	case MSG_TYPE_COMMAND:856		tag = "!RESP";857		break;858	case MSG_TYPE_DELAYED_RESP:859		tag = "!DLYD";860		break;861	case MSG_TYPE_NOTIFICATION:862		tag = "!NOTI";863		break;864	default:865		tag = "!UNKN";866		break;867	}868 869	trace_scmi_msg_dump(info->id, cinfo->id,870			    MSG_XTRACT_PROT_ID(msg_hdr),871			    MSG_XTRACT_ID(msg_hdr), tag,872			    MSG_XTRACT_TOKEN(msg_hdr), err, NULL, 0);873}874 875/**876 * scmi_msg_response_validate  - Validate message type against state of related877 * xfer878 *879 * @cinfo: A reference to the channel descriptor.880 * @msg_type: Message type to check881 * @xfer: A reference to the xfer to validate against @msg_type882 *883 * This function checks if @msg_type is congruent with the current state of884 * a pending @xfer; if an asynchronous delayed response is received before the885 * related synchronous response (Out-of-Order Delayed Response) the missing886 * synchronous response is assumed to be OK and completed, carrying on with the887 * Delayed Response: this is done to address the case in which the underlying888 * SCMI transport can deliver such out-of-order responses.889 *890 * Context: Assumes to be called with xfer->lock already acquired.891 *892 * Return: 0 on Success, error otherwise893 */894static inline int scmi_msg_response_validate(struct scmi_chan_info *cinfo,895					     u8 msg_type,896					     struct scmi_xfer *xfer)897{898	/*899	 * Even if a response was indeed expected on this slot at this point,900	 * a buggy platform could wrongly reply feeding us an unexpected901	 * delayed response we're not prepared to handle: bail-out safely902	 * blaming firmware.903	 */904	if (msg_type == MSG_TYPE_DELAYED_RESP && !xfer->async_done) {905		dev_err(cinfo->dev,906			"Delayed Response for %d not expected! Buggy F/W ?\n",907			xfer->hdr.seq);908		return -EINVAL;909	}910 911	switch (xfer->state) {912	case SCMI_XFER_SENT_OK:913		if (msg_type == MSG_TYPE_DELAYED_RESP) {914			/*915			 * Delayed Response expected but delivered earlier.916			 * Assume message RESPONSE was OK and skip state.917			 */918			xfer->hdr.status = SCMI_SUCCESS;919			xfer->state = SCMI_XFER_RESP_OK;920			complete(&xfer->done);921			dev_warn(cinfo->dev,922				 "Received valid OoO Delayed Response for %d\n",923				 xfer->hdr.seq);924		}925		break;926	case SCMI_XFER_RESP_OK:927		if (msg_type != MSG_TYPE_DELAYED_RESP)928			return -EINVAL;929		break;930	case SCMI_XFER_DRESP_OK:931		/* No further message expected once in SCMI_XFER_DRESP_OK */932		return -EINVAL;933	}934 935	return 0;936}937 938/**939 * scmi_xfer_state_update  - Update xfer state940 *941 * @xfer: A reference to the xfer to update942 * @msg_type: Type of message being processed.943 *944 * Note that this message is assumed to have been already successfully validated945 * by @scmi_msg_response_validate(), so here we just update the state.946 *947 * Context: Assumes to be called on an xfer exclusively acquired using the948 *	    busy flag.949 */950static inline void scmi_xfer_state_update(struct scmi_xfer *xfer, u8 msg_type)951{952	xfer->hdr.type = msg_type;953 954	/* Unknown command types were already discarded earlier */955	if (xfer->hdr.type == MSG_TYPE_COMMAND)956		xfer->state = SCMI_XFER_RESP_OK;957	else958		xfer->state = SCMI_XFER_DRESP_OK;959}960 961static bool scmi_xfer_acquired(struct scmi_xfer *xfer)962{963	int ret;964 965	ret = atomic_cmpxchg(&xfer->busy, SCMI_XFER_FREE, SCMI_XFER_BUSY);966 967	return ret == SCMI_XFER_FREE;968}969 970/**971 * scmi_xfer_command_acquire  -  Helper to lookup and acquire a command xfer972 *973 * @cinfo: A reference to the channel descriptor.974 * @msg_hdr: A message header to use as lookup key975 *976 * When a valid xfer is found for the sequence number embedded in the provided977 * msg_hdr, reference counting is properly updated and exclusive access to this978 * xfer is granted till released with @scmi_xfer_command_release.979 *980 * Return: A valid @xfer on Success or error otherwise.981 */982static inline struct scmi_xfer *983scmi_xfer_command_acquire(struct scmi_chan_info *cinfo, u32 msg_hdr)984{985	int ret;986	unsigned long flags;987	struct scmi_xfer *xfer;988	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);989	struct scmi_xfers_info *minfo = &info->tx_minfo;990	u8 msg_type = MSG_XTRACT_TYPE(msg_hdr);991	u16 xfer_id = MSG_XTRACT_TOKEN(msg_hdr);992 993	/* Are we even expecting this? */994	spin_lock_irqsave(&minfo->xfer_lock, flags);995	xfer = scmi_xfer_lookup_unlocked(minfo, xfer_id);996	if (IS_ERR(xfer)) {997		dev_err(cinfo->dev,998			"Message for %d type %d is not expected!\n",999			xfer_id, msg_type);1000		spin_unlock_irqrestore(&minfo->xfer_lock, flags);1001 1002		scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNEXPECTED);1003		scmi_inc_count(info->dbg->counters, ERR_MSG_UNEXPECTED);1004 1005		return xfer;1006	}1007	refcount_inc(&xfer->users);1008	spin_unlock_irqrestore(&minfo->xfer_lock, flags);1009 1010	spin_lock_irqsave(&xfer->lock, flags);1011	ret = scmi_msg_response_validate(cinfo, msg_type, xfer);1012	/*1013	 * If a pending xfer was found which was also in a congruent state with1014	 * the received message, acquire exclusive access to it setting the busy1015	 * flag.1016	 * Spins only on the rare limit condition of concurrent reception of1017	 * RESP and DRESP for the same xfer.1018	 */1019	if (!ret) {1020		spin_until_cond(scmi_xfer_acquired(xfer));1021		scmi_xfer_state_update(xfer, msg_type);1022	}1023	spin_unlock_irqrestore(&xfer->lock, flags);1024 1025	if (ret) {1026		dev_err(cinfo->dev,1027			"Invalid message type:%d for %d - HDR:0x%X  state:%d\n",1028			msg_type, xfer_id, msg_hdr, xfer->state);1029 1030		scmi_bad_message_trace(cinfo, msg_hdr, MSG_INVALID);1031		scmi_inc_count(info->dbg->counters, ERR_MSG_INVALID);1032 1033		/* On error the refcount incremented above has to be dropped */1034		__scmi_xfer_put(minfo, xfer);1035		xfer = ERR_PTR(-EINVAL);1036	}1037 1038	return xfer;1039}1040 1041static inline void scmi_xfer_command_release(struct scmi_info *info,1042					     struct scmi_xfer *xfer)1043{1044	atomic_set(&xfer->busy, SCMI_XFER_FREE);1045	__scmi_xfer_put(&info->tx_minfo, xfer);1046}1047 1048static inline void scmi_clear_channel(struct scmi_info *info,1049				      struct scmi_chan_info *cinfo)1050{1051	if (!cinfo->is_p2a) {1052		dev_warn(cinfo->dev, "Invalid clear on A2P channel !\n");1053		return;1054	}1055 1056	if (info->desc->ops->clear_channel)1057		info->desc->ops->clear_channel(cinfo);1058}1059 1060static void scmi_handle_notification(struct scmi_chan_info *cinfo,1061				     u32 msg_hdr, void *priv)1062{1063	struct scmi_xfer *xfer;1064	struct device *dev = cinfo->dev;1065	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);1066	struct scmi_xfers_info *minfo = &info->rx_minfo;1067	ktime_t ts;1068 1069	ts = ktime_get_boottime();1070	xfer = scmi_xfer_get(cinfo->handle, minfo);1071	if (IS_ERR(xfer)) {1072		dev_err(dev, "failed to get free message slot (%ld)\n",1073			PTR_ERR(xfer));1074 1075		scmi_bad_message_trace(cinfo, msg_hdr, MSG_NOMEM);1076		scmi_inc_count(info->dbg->counters, ERR_MSG_NOMEM);1077 1078		scmi_clear_channel(info, cinfo);1079		return;1080	}1081 1082	unpack_scmi_header(msg_hdr, &xfer->hdr);1083	if (priv)1084		/* Ensure order between xfer->priv store and following ops */1085		smp_store_mb(xfer->priv, priv);1086	info->desc->ops->fetch_notification(cinfo, info->desc->max_msg_size,1087					    xfer);1088 1089	trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id,1090			    xfer->hdr.id, "NOTI", xfer->hdr.seq,1091			    xfer->hdr.status, xfer->rx.buf, xfer->rx.len);1092	scmi_inc_count(info->dbg->counters, NOTIFICATION_OK);1093 1094	scmi_notify(cinfo->handle, xfer->hdr.protocol_id,1095		    xfer->hdr.id, xfer->rx.buf, xfer->rx.len, ts);1096 1097	trace_scmi_rx_done(xfer->transfer_id, xfer->hdr.id,1098			   xfer->hdr.protocol_id, xfer->hdr.seq,1099			   MSG_TYPE_NOTIFICATION);1100 1101	if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) {1102		xfer->hdr.seq = MSG_XTRACT_TOKEN(msg_hdr);1103		scmi_raw_message_report(info->raw, xfer, SCMI_RAW_NOTIF_QUEUE,1104					cinfo->id);1105	}1106 1107	__scmi_xfer_put(minfo, xfer);1108 1109	scmi_clear_channel(info, cinfo);1110}1111 1112static void scmi_handle_response(struct scmi_chan_info *cinfo,1113				 u32 msg_hdr, void *priv)1114{1115	struct scmi_xfer *xfer;1116	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);1117 1118	xfer = scmi_xfer_command_acquire(cinfo, msg_hdr);1119	if (IS_ERR(xfer)) {1120		if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT))1121			scmi_raw_error_report(info->raw, cinfo, msg_hdr, priv);1122 1123		if (MSG_XTRACT_TYPE(msg_hdr) == MSG_TYPE_DELAYED_RESP)1124			scmi_clear_channel(info, cinfo);1125		return;1126	}1127 1128	/* rx.len could be shrunk in the sync do_xfer, so reset to maxsz */1129	if (xfer->hdr.type == MSG_TYPE_DELAYED_RESP)1130		xfer->rx.len = info->desc->max_msg_size;1131 1132	if (priv)1133		/* Ensure order between xfer->priv store and following ops */1134		smp_store_mb(xfer->priv, priv);1135	info->desc->ops->fetch_response(cinfo, xfer);1136 1137	trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id,1138			    xfer->hdr.id,1139			    xfer->hdr.type == MSG_TYPE_DELAYED_RESP ?1140			    (!SCMI_XFER_IS_RAW(xfer) ? "DLYD" : "dlyd") :1141			    (!SCMI_XFER_IS_RAW(xfer) ? "RESP" : "resp"),1142			    xfer->hdr.seq, xfer->hdr.status,1143			    xfer->rx.buf, xfer->rx.len);1144 1145	trace_scmi_rx_done(xfer->transfer_id, xfer->hdr.id,1146			   xfer->hdr.protocol_id, xfer->hdr.seq,1147			   xfer->hdr.type);1148 1149	if (xfer->hdr.type == MSG_TYPE_DELAYED_RESP) {1150		scmi_clear_channel(info, cinfo);1151		complete(xfer->async_done);1152		scmi_inc_count(info->dbg->counters, DELAYED_RESPONSE_OK);1153	} else {1154		complete(&xfer->done);1155		scmi_inc_count(info->dbg->counters, RESPONSE_OK);1156	}1157 1158	if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) {1159		/*1160		 * When in polling mode avoid to queue the Raw xfer on the IRQ1161		 * RX path since it will be already queued at the end of the TX1162		 * poll loop.1163		 */1164		if (!xfer->hdr.poll_completion)1165			scmi_raw_message_report(info->raw, xfer,1166						SCMI_RAW_REPLY_QUEUE,1167						cinfo->id);1168	}1169 1170	scmi_xfer_command_release(info, xfer);1171}1172 1173/**1174 * scmi_rx_callback() - callback for receiving messages1175 *1176 * @cinfo: SCMI channel info1177 * @msg_hdr: Message header1178 * @priv: Transport specific private data.1179 *1180 * Processes one received message to appropriate transfer information and1181 * signals completion of the transfer.1182 *1183 * NOTE: This function will be invoked in IRQ context, hence should be1184 * as optimal as possible.1185 */1186static void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr,1187			     void *priv)1188{1189	u8 msg_type = MSG_XTRACT_TYPE(msg_hdr);1190 1191	switch (msg_type) {1192	case MSG_TYPE_NOTIFICATION:1193		scmi_handle_notification(cinfo, msg_hdr, priv);1194		break;1195	case MSG_TYPE_COMMAND:1196	case MSG_TYPE_DELAYED_RESP:1197		scmi_handle_response(cinfo, msg_hdr, priv);1198		break;1199	default:1200		WARN_ONCE(1, "received unknown msg_type:%d\n", msg_type);1201		scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNKNOWN);1202		break;1203	}1204}1205 1206/**1207 * xfer_put() - Release a transmit message1208 *1209 * @ph: Pointer to SCMI protocol handle1210 * @xfer: message that was reserved by xfer_get_init1211 */1212static void xfer_put(const struct scmi_protocol_handle *ph,1213		     struct scmi_xfer *xfer)1214{1215	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1216	struct scmi_info *info = handle_to_scmi_info(pi->handle);1217 1218	__scmi_xfer_put(&info->tx_minfo, xfer);1219}1220 1221static bool scmi_xfer_done_no_timeout(struct scmi_chan_info *cinfo,1222				      struct scmi_xfer *xfer, ktime_t stop)1223{1224	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);1225 1226	/*1227	 * Poll also on xfer->done so that polling can be forcibly terminated1228	 * in case of out-of-order receptions of delayed responses1229	 */1230	return info->desc->ops->poll_done(cinfo, xfer) ||1231	       try_wait_for_completion(&xfer->done) ||1232	       ktime_after(ktime_get(), stop);1233}1234 1235static int scmi_wait_for_reply(struct device *dev, const struct scmi_desc *desc,1236			       struct scmi_chan_info *cinfo,1237			       struct scmi_xfer *xfer, unsigned int timeout_ms)1238{1239	int ret = 0;1240	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);1241 1242	if (xfer->hdr.poll_completion) {1243		/*1244		 * Real polling is needed only if transport has NOT declared1245		 * itself to support synchronous commands replies.1246		 */1247		if (!desc->sync_cmds_completed_on_ret) {1248			/*1249			 * Poll on xfer using transport provided .poll_done();1250			 * assumes no completion interrupt was available.1251			 */1252			ktime_t stop = ktime_add_ms(ktime_get(), timeout_ms);1253 1254			spin_until_cond(scmi_xfer_done_no_timeout(cinfo,1255								  xfer, stop));1256			if (ktime_after(ktime_get(), stop)) {1257				dev_err(dev,1258					"timed out in resp(caller: %pS) - polling\n",1259					(void *)_RET_IP_);1260				ret = -ETIMEDOUT;1261				scmi_inc_count(info->dbg->counters, XFERS_RESPONSE_POLLED_TIMEOUT);1262			}1263		}1264 1265		if (!ret) {1266			unsigned long flags;1267 1268			/*1269			 * Do not fetch_response if an out-of-order delayed1270			 * response is being processed.1271			 */1272			spin_lock_irqsave(&xfer->lock, flags);1273			if (xfer->state == SCMI_XFER_SENT_OK) {1274				desc->ops->fetch_response(cinfo, xfer);1275				xfer->state = SCMI_XFER_RESP_OK;1276			}1277			spin_unlock_irqrestore(&xfer->lock, flags);1278 1279			/* Trace polled replies. */1280			trace_scmi_msg_dump(info->id, cinfo->id,1281					    xfer->hdr.protocol_id, xfer->hdr.id,1282					    !SCMI_XFER_IS_RAW(xfer) ?1283					    "RESP" : "resp",1284					    xfer->hdr.seq, xfer->hdr.status,1285					    xfer->rx.buf, xfer->rx.len);1286			scmi_inc_count(info->dbg->counters, RESPONSE_POLLED_OK);1287 1288			if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) {1289				scmi_raw_message_report(info->raw, xfer,1290							SCMI_RAW_REPLY_QUEUE,1291							cinfo->id);1292			}1293		}1294	} else {1295		/* And we wait for the response. */1296		if (!wait_for_completion_timeout(&xfer->done,1297						 msecs_to_jiffies(timeout_ms))) {1298			dev_err(dev, "timed out in resp(caller: %pS)\n",1299				(void *)_RET_IP_);1300			ret = -ETIMEDOUT;1301			scmi_inc_count(info->dbg->counters, XFERS_RESPONSE_TIMEOUT);1302		}1303	}1304 1305	return ret;1306}1307 1308/**1309 * scmi_wait_for_message_response  - An helper to group all the possible ways of1310 * waiting for a synchronous message response.1311 *1312 * @cinfo: SCMI channel info1313 * @xfer: Reference to the transfer being waited for.1314 *1315 * Chooses waiting strategy (sleep-waiting vs busy-waiting) depending on1316 * configuration flags like xfer->hdr.poll_completion.1317 *1318 * Return: 0 on Success, error otherwise.1319 */1320static int scmi_wait_for_message_response(struct scmi_chan_info *cinfo,1321					  struct scmi_xfer *xfer)1322{1323	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);1324	struct device *dev = info->dev;1325 1326	trace_scmi_xfer_response_wait(xfer->transfer_id, xfer->hdr.id,1327				      xfer->hdr.protocol_id, xfer->hdr.seq,1328				      info->desc->max_rx_timeout_ms,1329				      xfer->hdr.poll_completion);1330 1331	return scmi_wait_for_reply(dev, info->desc, cinfo, xfer,1332				   info->desc->max_rx_timeout_ms);1333}1334 1335/**1336 * scmi_xfer_raw_wait_for_message_response  - An helper to wait for a message1337 * reply to an xfer raw request on a specific channel for the required timeout.1338 *1339 * @cinfo: SCMI channel info1340 * @xfer: Reference to the transfer being waited for.1341 * @timeout_ms: The maximum timeout in milliseconds1342 *1343 * Return: 0 on Success, error otherwise.1344 */1345int scmi_xfer_raw_wait_for_message_response(struct scmi_chan_info *cinfo,1346					    struct scmi_xfer *xfer,1347					    unsigned int timeout_ms)1348{1349	int ret;1350	struct scmi_info *info = handle_to_scmi_info(cinfo->handle);1351	struct device *dev = info->dev;1352 1353	ret = scmi_wait_for_reply(dev, info->desc, cinfo, xfer, timeout_ms);1354	if (ret)1355		dev_dbg(dev, "timed out in RAW response - HDR:%08X\n",1356			pack_scmi_header(&xfer->hdr));1357 1358	return ret;1359}1360 1361/**1362 * do_xfer() - Do one transfer1363 *1364 * @ph: Pointer to SCMI protocol handle1365 * @xfer: Transfer to initiate and wait for response1366 *1367 * Return: -ETIMEDOUT in case of no response, if transmit error,1368 *	return corresponding error, else if all goes well,1369 *	return 0.1370 */1371static int do_xfer(const struct scmi_protocol_handle *ph,1372		   struct scmi_xfer *xfer)1373{1374	int ret;1375	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1376	struct scmi_info *info = handle_to_scmi_info(pi->handle);1377	struct device *dev = info->dev;1378	struct scmi_chan_info *cinfo;1379 1380	/* Check for polling request on custom command xfers at first */1381	if (xfer->hdr.poll_completion &&1382	    !is_transport_polling_capable(info->desc)) {1383		dev_warn_once(dev,1384			      "Polling mode is not supported by transport.\n");1385		scmi_inc_count(info->dbg->counters, SENT_FAIL_POLLING_UNSUPPORTED);1386		return -EINVAL;1387	}1388 1389	cinfo = idr_find(&info->tx_idr, pi->proto->id);1390	if (unlikely(!cinfo)) {1391		scmi_inc_count(info->dbg->counters, SENT_FAIL_CHANNEL_NOT_FOUND);1392		return -EINVAL;1393	}1394	/* True ONLY if also supported by transport. */1395	if (is_polling_enabled(cinfo, info->desc))1396		xfer->hdr.poll_completion = true;1397 1398	/*1399	 * Initialise protocol id now from protocol handle to avoid it being1400	 * overridden by mistake (or malice) by the protocol code mangling with1401	 * the scmi_xfer structure prior to this.1402	 */1403	xfer->hdr.protocol_id = pi->proto->id;1404	reinit_completion(&xfer->done);1405 1406	trace_scmi_xfer_begin(xfer->transfer_id, xfer->hdr.id,1407			      xfer->hdr.protocol_id, xfer->hdr.seq,1408			      xfer->hdr.poll_completion);1409 1410	/* Clear any stale status */1411	xfer->hdr.status = SCMI_SUCCESS;1412	xfer->state = SCMI_XFER_SENT_OK;1413	/*1414	 * Even though spinlocking is not needed here since no race is possible1415	 * on xfer->state due to the monotonically increasing tokens allocation,1416	 * we must anyway ensure xfer->state initialization is not re-ordered1417	 * after the .send_message() to be sure that on the RX path an early1418	 * ISR calling scmi_rx_callback() cannot see an old stale xfer->state.1419	 */1420	smp_mb();1421 1422	ret = info->desc->ops->send_message(cinfo, xfer);1423	if (ret < 0) {1424		dev_dbg(dev, "Failed to send message %d\n", ret);1425		scmi_inc_count(info->dbg->counters, SENT_FAIL);1426		return ret;1427	}1428 1429	trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id,1430			    xfer->hdr.id, "CMND", xfer->hdr.seq,1431			    xfer->hdr.status, xfer->tx.buf, xfer->tx.len);1432	scmi_inc_count(info->dbg->counters, SENT_OK);1433 1434	ret = scmi_wait_for_message_response(cinfo, xfer);1435	if (!ret && xfer->hdr.status) {1436		ret = scmi_to_linux_errno(xfer->hdr.status);1437		scmi_inc_count(info->dbg->counters, ERR_PROTOCOL);1438	}1439 1440	if (info->desc->ops->mark_txdone)1441		info->desc->ops->mark_txdone(cinfo, ret, xfer);1442 1443	trace_scmi_xfer_end(xfer->transfer_id, xfer->hdr.id,1444			    xfer->hdr.protocol_id, xfer->hdr.seq, ret);1445 1446	return ret;1447}1448 1449static void reset_rx_to_maxsz(const struct scmi_protocol_handle *ph,1450			      struct scmi_xfer *xfer)1451{1452	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1453	struct scmi_info *info = handle_to_scmi_info(pi->handle);1454 1455	xfer->rx.len = info->desc->max_msg_size;1456}1457 1458/**1459 * do_xfer_with_response() - Do one transfer and wait until the delayed1460 *	response is received1461 *1462 * @ph: Pointer to SCMI protocol handle1463 * @xfer: Transfer to initiate and wait for response1464 *1465 * Using asynchronous commands in atomic/polling mode should be avoided since1466 * it could cause long busy-waiting here, so ignore polling for the delayed1467 * response and WARN if it was requested for this command transaction since1468 * upper layers should refrain from issuing such kind of requests.1469 *1470 * The only other option would have been to refrain from using any asynchronous1471 * command even if made available, when an atomic transport is detected, and1472 * instead forcibly use the synchronous version (thing that can be easily1473 * attained at the protocol layer), but this would also have led to longer1474 * stalls of the channel for synchronous commands and possibly timeouts.1475 * (in other words there is usually a good reason if a platform provides an1476 *  asynchronous version of a command and we should prefer to use it...just not1477 *  when using atomic/polling mode)1478 *1479 * Return: -ETIMEDOUT in case of no delayed response, if transmit error,1480 *	return corresponding error, else if all goes well, return 0.1481 */1482static int do_xfer_with_response(const struct scmi_protocol_handle *ph,1483				 struct scmi_xfer *xfer)1484{1485	int ret, timeout = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT);1486	DECLARE_COMPLETION_ONSTACK(async_response);1487 1488	xfer->async_done = &async_response;1489 1490	/*1491	 * Delayed responses should not be polled, so an async command should1492	 * not have been used when requiring an atomic/poll context; WARN and1493	 * perform instead a sleeping wait.1494	 * (Note Async + IgnoreDelayedResponses are sent via do_xfer)1495	 */1496	WARN_ON_ONCE(xfer->hdr.poll_completion);1497 1498	ret = do_xfer(ph, xfer);1499	if (!ret) {1500		if (!wait_for_completion_timeout(xfer->async_done, timeout)) {1501			dev_err(ph->dev,1502				"timed out in delayed resp(caller: %pS)\n",1503				(void *)_RET_IP_);1504			ret = -ETIMEDOUT;1505		} else if (xfer->hdr.status) {1506			ret = scmi_to_linux_errno(xfer->hdr.status);1507		}1508	}1509 1510	xfer->async_done = NULL;1511	return ret;1512}1513 1514/**1515 * xfer_get_init() - Allocate and initialise one message for transmit1516 *1517 * @ph: Pointer to SCMI protocol handle1518 * @msg_id: Message identifier1519 * @tx_size: transmit message size1520 * @rx_size: receive message size1521 * @p: pointer to the allocated and initialised message1522 *1523 * This function allocates the message using @scmi_xfer_get and1524 * initialise the header.1525 *1526 * Return: 0 if all went fine with @p pointing to message, else1527 *	corresponding error.1528 */1529static int xfer_get_init(const struct scmi_protocol_handle *ph,1530			 u8 msg_id, size_t tx_size, size_t rx_size,1531			 struct scmi_xfer **p)1532{1533	int ret;1534	struct scmi_xfer *xfer;1535	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1536	struct scmi_info *info = handle_to_scmi_info(pi->handle);1537	struct scmi_xfers_info *minfo = &info->tx_minfo;1538	struct device *dev = info->dev;1539 1540	/* Ensure we have sane transfer sizes */1541	if (rx_size > info->desc->max_msg_size ||1542	    tx_size > info->desc->max_msg_size)1543		return -ERANGE;1544 1545	xfer = scmi_xfer_get(pi->handle, minfo);1546	if (IS_ERR(xfer)) {1547		ret = PTR_ERR(xfer);1548		dev_err(dev, "failed to get free message slot(%d)\n", ret);1549		return ret;1550	}1551 1552	/* Pick a sequence number and register this xfer as in-flight */1553	ret = scmi_xfer_pending_set(xfer, minfo);1554	if (ret) {1555		dev_err(pi->handle->dev,1556			"Failed to get monotonic token %d\n", ret);1557		__scmi_xfer_put(minfo, xfer);1558		return ret;1559	}1560 1561	xfer->tx.len = tx_size;1562	xfer->rx.len = rx_size ? : info->desc->max_msg_size;1563	xfer->hdr.type = MSG_TYPE_COMMAND;1564	xfer->hdr.id = msg_id;1565	xfer->hdr.poll_completion = false;1566 1567	*p = xfer;1568 1569	return 0;1570}1571 1572/**1573 * version_get() - command to get the revision of the SCMI entity1574 *1575 * @ph: Pointer to SCMI protocol handle1576 * @version: Holds returned version of protocol.1577 *1578 * Updates the SCMI information in the internal data structure.1579 *1580 * Return: 0 if all went fine, else return appropriate error.1581 */1582static int version_get(const struct scmi_protocol_handle *ph, u32 *version)1583{1584	int ret;1585	__le32 *rev_info;1586	struct scmi_xfer *t;1587 1588	ret = xfer_get_init(ph, PROTOCOL_VERSION, 0, sizeof(*version), &t);1589	if (ret)1590		return ret;1591 1592	ret = do_xfer(ph, t);1593	if (!ret) {1594		rev_info = t->rx.buf;1595		*version = le32_to_cpu(*rev_info);1596	}1597 1598	xfer_put(ph, t);1599	return ret;1600}1601 1602/**1603 * scmi_set_protocol_priv  - Set protocol specific data at init time1604 *1605 * @ph: A reference to the protocol handle.1606 * @priv: The private data to set.1607 * @version: The detected protocol version for the core to register.1608 *1609 * Return: 0 on Success1610 */1611static int scmi_set_protocol_priv(const struct scmi_protocol_handle *ph,1612				  void *priv, u32 version)1613{1614	struct scmi_protocol_instance *pi = ph_to_pi(ph);1615 1616	pi->priv = priv;1617	pi->version = version;1618 1619	return 0;1620}1621 1622/**1623 * scmi_get_protocol_priv  - Set protocol specific data at init time1624 *1625 * @ph: A reference to the protocol handle.1626 *1627 * Return: Protocol private data if any was set.1628 */1629static void *scmi_get_protocol_priv(const struct scmi_protocol_handle *ph)1630{1631	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1632 1633	return pi->priv;1634}1635 1636static const struct scmi_xfer_ops xfer_ops = {1637	.version_get = version_get,1638	.xfer_get_init = xfer_get_init,1639	.reset_rx_to_maxsz = reset_rx_to_maxsz,1640	.do_xfer = do_xfer,1641	.do_xfer_with_response = do_xfer_with_response,1642	.xfer_put = xfer_put,1643};1644 1645struct scmi_msg_resp_domain_name_get {1646	__le32 flags;1647	u8 name[SCMI_MAX_STR_SIZE];1648};1649 1650/**1651 * scmi_common_extended_name_get  - Common helper to get extended resources name1652 * @ph: A protocol handle reference.1653 * @cmd_id: The specific command ID to use.1654 * @res_id: The specific resource ID to use.1655 * @flags: A pointer to specific flags to use, if any.1656 * @name: A pointer to the preallocated area where the retrieved name will be1657 *	  stored as a NULL terminated string.1658 * @len: The len in bytes of the @name char array.1659 *1660 * Return: 0 on Succcess1661 */1662static int scmi_common_extended_name_get(const struct scmi_protocol_handle *ph,1663					 u8 cmd_id, u32 res_id, u32 *flags,1664					 char *name, size_t len)1665{1666	int ret;1667	size_t txlen;1668	struct scmi_xfer *t;1669	struct scmi_msg_resp_domain_name_get *resp;1670 1671	txlen = !flags ? sizeof(res_id) : sizeof(res_id) + sizeof(*flags);1672	ret = ph->xops->xfer_get_init(ph, cmd_id, txlen, sizeof(*resp), &t);1673	if (ret)1674		goto out;1675 1676	put_unaligned_le32(res_id, t->tx.buf);1677	if (flags)1678		put_unaligned_le32(*flags, t->tx.buf + sizeof(res_id));1679	resp = t->rx.buf;1680 1681	ret = ph->xops->do_xfer(ph, t);1682	if (!ret)1683		strscpy(name, resp->name, len);1684 1685	ph->xops->xfer_put(ph, t);1686out:1687	if (ret)1688		dev_warn(ph->dev,1689			 "Failed to get extended name - id:%u (ret:%d). Using %s\n",1690			 res_id, ret, name);1691	return ret;1692}1693 1694/**1695 * scmi_common_get_max_msg_size  - Get maximum message size1696 * @ph: A protocol handle reference.1697 *1698 * Return: Maximum message size for the current protocol.1699 */1700static int scmi_common_get_max_msg_size(const struct scmi_protocol_handle *ph)1701{1702	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1703	struct scmi_info *info = handle_to_scmi_info(pi->handle);1704 1705	return info->desc->max_msg_size;1706}1707 1708/**1709 * struct scmi_iterator  - Iterator descriptor1710 * @msg: A reference to the message TX buffer; filled by @prepare_message with1711 *	 a proper custom command payload for each multi-part command request.1712 * @resp: A reference to the response RX buffer; used by @update_state and1713 *	  @process_response to parse the multi-part replies.1714 * @t: A reference to the underlying xfer initialized and used transparently by1715 *     the iterator internal routines.1716 * @ph: A reference to the associated protocol handle to be used.1717 * @ops: A reference to the custom provided iterator operations.1718 * @state: The current iterator state; used and updated in turn by the iterators1719 *	   internal routines and by the caller-provided @scmi_iterator_ops.1720 * @priv: A reference to optional private data as provided by the caller and1721 *	  passed back to the @@scmi_iterator_ops.1722 */1723struct scmi_iterator {1724	void *msg;1725	void *resp;1726	struct scmi_xfer *t;1727	const struct scmi_protocol_handle *ph;1728	struct scmi_iterator_ops *ops;1729	struct scmi_iterator_state state;1730	void *priv;1731};1732 1733static void *scmi_iterator_init(const struct scmi_protocol_handle *ph,1734				struct scmi_iterator_ops *ops,1735				unsigned int max_resources, u8 msg_id,1736				size_t tx_size, void *priv)1737{1738	int ret;1739	struct scmi_iterator *i;1740 1741	i = devm_kzalloc(ph->dev, sizeof(*i), GFP_KERNEL);1742	if (!i)1743		return ERR_PTR(-ENOMEM);1744 1745	i->ph = ph;1746	i->ops = ops;1747	i->priv = priv;1748 1749	ret = ph->xops->xfer_get_init(ph, msg_id, tx_size, 0, &i->t);1750	if (ret) {1751		devm_kfree(ph->dev, i);1752		return ERR_PTR(ret);1753	}1754 1755	i->state.max_resources = max_resources;1756	i->msg = i->t->tx.buf;1757	i->resp = i->t->rx.buf;1758 1759	return i;1760}1761 1762static int scmi_iterator_run(void *iter)1763{1764	int ret = -EINVAL;1765	struct scmi_iterator_ops *iops;1766	const struct scmi_protocol_handle *ph;1767	struct scmi_iterator_state *st;1768	struct scmi_iterator *i = iter;1769 1770	if (!i || !i->ops || !i->ph)1771		return ret;1772 1773	iops = i->ops;1774	ph = i->ph;1775	st = &i->state;1776 1777	do {1778		iops->prepare_message(i->msg, st->desc_index, i->priv);1779		ret = ph->xops->do_xfer(ph, i->t);1780		if (ret)1781			break;1782 1783		st->rx_len = i->t->rx.len;1784		ret = iops->update_state(st, i->resp, i->priv);1785		if (ret)1786			break;1787 1788		if (st->num_returned > st->max_resources - st->desc_index) {1789			dev_err(ph->dev,1790				"No. of resources can't exceed %d\n",1791				st->max_resources);1792			ret = -EINVAL;1793			break;1794		}1795 1796		for (st->loop_idx = 0; st->loop_idx < st->num_returned;1797		     st->loop_idx++) {1798			ret = iops->process_response(ph, i->resp, st, i->priv);1799			if (ret)1800				goto out;1801		}1802 1803		st->desc_index += st->num_returned;1804		ph->xops->reset_rx_to_maxsz(ph, i->t);1805		/*1806		 * check for both returned and remaining to avoid infinite1807		 * loop due to buggy firmware1808		 */1809	} while (st->num_returned && st->num_remaining);1810 1811out:1812	/* Finalize and destroy iterator */1813	ph->xops->xfer_put(ph, i->t);1814	devm_kfree(ph->dev, i);1815 1816	return ret;1817}1818 1819struct scmi_msg_get_fc_info {1820	__le32 domain;1821	__le32 message_id;1822};1823 1824struct scmi_msg_resp_desc_fc {1825	__le32 attr;1826#define SUPPORTS_DOORBELL(x)		((x) & BIT(0))1827#define DOORBELL_REG_WIDTH(x)		FIELD_GET(GENMASK(2, 1), (x))1828	__le32 rate_limit;1829	__le32 chan_addr_low;1830	__le32 chan_addr_high;1831	__le32 chan_size;1832	__le32 db_addr_low;1833	__le32 db_addr_high;1834	__le32 db_set_lmask;1835	__le32 db_set_hmask;1836	__le32 db_preserve_lmask;1837	__le32 db_preserve_hmask;1838};1839 1840static void1841scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph,1842			     u8 describe_id, u32 message_id, u32 valid_size,1843			     u32 domain, void __iomem **p_addr,1844			     struct scmi_fc_db_info **p_db, u32 *rate_limit)1845{1846	int ret;1847	u32 flags;1848	u64 phys_addr;1849	u8 size;1850	void __iomem *addr;1851	struct scmi_xfer *t;1852	struct scmi_fc_db_info *db = NULL;1853	struct scmi_msg_get_fc_info *info;1854	struct scmi_msg_resp_desc_fc *resp;1855	const struct scmi_protocol_instance *pi = ph_to_pi(ph);1856 1857	if (!p_addr) {1858		ret = -EINVAL;1859		goto err_out;1860	}1861 1862	ret = ph->xops->xfer_get_init(ph, describe_id,1863				      sizeof(*info), sizeof(*resp), &t);1864	if (ret)1865		goto err_out;1866 1867	info = t->tx.buf;1868	info->domain = cpu_to_le32(domain);1869	info->message_id = cpu_to_le32(message_id);1870 1871	/*1872	 * Bail out on error leaving fc_info addresses zeroed; this includes1873	 * the case in which the requested domain/message_id does NOT support1874	 * fastchannels at all.1875	 */1876	ret = ph->xops->do_xfer(ph, t);1877	if (ret)1878		goto err_xfer;1879 1880	resp = t->rx.buf;1881	flags = le32_to_cpu(resp->attr);1882	size = le32_to_cpu(resp->chan_size);1883	if (size != valid_size) {1884		ret = -EINVAL;1885		goto err_xfer;1886	}1887 1888	if (rate_limit)1889		*rate_limit = le32_to_cpu(resp->rate_limit) & GENMASK(19, 0);1890 1891	phys_addr = le32_to_cpu(resp->chan_addr_low);1892	phys_addr |= (u64)le32_to_cpu(resp->chan_addr_high) << 32;1893	addr = devm_ioremap(ph->dev, phys_addr, size);1894	if (!addr) {1895		ret = -EADDRNOTAVAIL;1896		goto err_xfer;1897	}1898 1899	*p_addr = addr;1900 1901	if (p_db && SUPPORTS_DOORBELL(flags)) {1902		db = devm_kzalloc(ph->dev, sizeof(*db), GFP_KERNEL);1903		if (!db) {1904			ret = -ENOMEM;1905			goto err_db;1906		}1907 1908		size = 1 << DOORBELL_REG_WIDTH(flags);1909		phys_addr = le32_to_cpu(resp->db_addr_low);1910		phys_addr |= (u64)le32_to_cpu(resp->db_addr_high) << 32;1911		addr = devm_ioremap(ph->dev, phys_addr, size);1912		if (!addr) {1913			ret = -EADDRNOTAVAIL;1914			goto err_db_mem;1915		}1916 1917		db->addr = addr;1918		db->width = size;1919		db->set = le32_to_cpu(resp->db_set_lmask);1920		db->set |= (u64)le32_to_cpu(resp->db_set_hmask) << 32;1921		db->mask = le32_to_cpu(resp->db_preserve_lmask);1922		db->mask |= (u64)le32_to_cpu(resp->db_preserve_hmask) << 32;1923 1924		*p_db = db;1925	}1926 1927	ph->xops->xfer_put(ph, t);1928 1929	dev_dbg(ph->dev,1930		"Using valid FC for protocol %X [MSG_ID:%u / RES_ID:%u]\n",1931		pi->proto->id, message_id, domain);1932 1933	return;1934 1935err_db_mem:1936	devm_kfree(ph->dev, db);1937 1938err_db:1939	*p_addr = NULL;1940 1941err_xfer:1942	ph->xops->xfer_put(ph, t);1943 1944err_out:1945	dev_warn(ph->dev,1946		 "Failed to get FC for protocol %X [MSG_ID:%u / RES_ID:%u] - ret:%d. Using regular messaging.\n",1947		 pi->proto->id, message_id, domain, ret);1948}1949 1950#define SCMI_PROTO_FC_RING_DB(w)			\1951do {							\1952	u##w val = 0;					\1953							\1954	if (db->mask)					\1955		val = ioread##w(db->addr) & db->mask;	\1956	iowrite##w((u##w)db->set | val, db->addr);	\1957} while (0)1958 1959static void scmi_common_fastchannel_db_ring(struct scmi_fc_db_info *db)1960{1961	if (!db || !db->addr)1962		return;1963 1964	if (db->width == 1)1965		SCMI_PROTO_FC_RING_DB(8);1966	else if (db->width == 2)1967		SCMI_PROTO_FC_RING_DB(16);1968	else if (db->width == 4)1969		SCMI_PROTO_FC_RING_DB(32);1970	else /* db->width == 8 */1971#ifdef CONFIG_64BIT1972		SCMI_PROTO_FC_RING_DB(64);1973#else1974	{1975		u64 val = 0;1976 1977		if (db->mask)1978			val = ioread64_hi_lo(db->addr) & db->mask;1979		iowrite64_hi_lo(db->set | val, db->addr);1980	}1981#endif1982}1983 1984/**1985 * scmi_protocol_msg_check  - Check protocol message attributes1986 *1987 * @ph: A reference to the protocol handle.1988 * @message_id: The ID of the message to check.1989 * @attributes: A parameter to optionally return the retrieved message1990 *		attributes, in case of Success.1991 *1992 * An helper to check protocol message attributes for a specific protocol1993 * and message pair.1994 *1995 * Return: 0 on SUCCESS1996 */1997static int scmi_protocol_msg_check(const struct scmi_protocol_handle *ph,1998				   u32 message_id, u32 *attributes)1999{2000	int ret;2001	struct scmi_xfer *t;2002 2003	ret = xfer_get_init(ph, PROTOCOL_MESSAGE_ATTRIBUTES,2004			    sizeof(__le32), 0, &t);2005	if (ret)2006		return ret;2007 2008	put_unaligned_le32(message_id, t->tx.buf);2009	ret = do_xfer(ph, t);2010	if (!ret && attributes)2011		*attributes = get_unaligned_le32(t->rx.buf);2012	xfer_put(ph, t);2013 2014	return ret;2015}2016 2017static const struct scmi_proto_helpers_ops helpers_ops = {2018	.extended_name_get = scmi_common_extended_name_get,2019	.get_max_msg_size = scmi_common_get_max_msg_size,2020	.iter_response_init = scmi_iterator_init,2021	.iter_response_run = scmi_iterator_run,2022	.protocol_msg_check = scmi_protocol_msg_check,2023	.fastchannel_init = scmi_common_fastchannel_init,2024	.fastchannel_db_ring = scmi_common_fastchannel_db_ring,2025};2026 2027/**2028 * scmi_revision_area_get  - Retrieve version memory area.2029 *2030 * @ph: A reference to the protocol handle.2031 *2032 * A helper to grab the version memory area reference during SCMI Base protocol2033 * initialization.2034 *2035 * Return: A reference to the version memory area associated to the SCMI2036 *	   instance underlying this protocol handle.2037 */2038struct scmi_revision_info *2039scmi_revision_area_get(const struct scmi_protocol_handle *ph)2040{2041	const struct scmi_protocol_instance *pi = ph_to_pi(ph);2042 2043	return pi->handle->version;2044}2045 2046/**2047 * scmi_protocol_version_negotiate  - Negotiate protocol version2048 *2049 * @ph: A reference to the protocol handle.2050 *2051 * An helper to negotiate a protocol version different from the latest2052 * advertised as supported from the platform: on Success backward2053 * compatibility is assured by the platform.2054 *2055 * Return: 0 on Success2056 */2057static int scmi_protocol_version_negotiate(struct scmi_protocol_handle *ph)2058{2059	int ret;2060	struct scmi_xfer *t;2061	struct scmi_protocol_instance *pi = ph_to_pi(ph);2062 2063	/* At first check if NEGOTIATE_PROTOCOL_VERSION is supported ... */2064	ret = scmi_protocol_msg_check(ph, NEGOTIATE_PROTOCOL_VERSION, NULL);2065	if (ret)2066		return ret;2067 2068	/* ... then attempt protocol version negotiation */2069	ret = xfer_get_init(ph, NEGOTIATE_PROTOCOL_VERSION,2070			    sizeof(__le32), 0, &t);2071	if (ret)2072		return ret;2073 2074	put_unaligned_le32(pi->proto->supported_version, t->tx.buf);2075	ret = do_xfer(ph, t);2076	if (!ret)2077		pi->negotiated_version = pi->proto->supported_version;2078 2079	xfer_put(ph, t);2080 2081	return ret;2082}2083 2084/**2085 * scmi_alloc_init_protocol_instance  - Allocate and initialize a protocol2086 * instance descriptor.2087 * @info: The reference to the related SCMI instance.2088 * @proto: The protocol descriptor.2089 *2090 * Allocate a new protocol instance descriptor, using the provided @proto2091 * description, against the specified SCMI instance @info, and initialize it;2092 * all resources management is handled via a dedicated per-protocol devres2093 * group.2094 *2095 * Context: Assumes to be called with @protocols_mtx already acquired.2096 * Return: A reference to a freshly allocated and initialized protocol instance2097 *	   or ERR_PTR on failure. On failure the @proto reference is at first2098 *	   put using @scmi_protocol_put() before releasing all the devres group.2099 */2100static struct scmi_protocol_instance *2101scmi_alloc_init_protocol_instance(struct scmi_info *info,2102				  const struct scmi_protocol *proto)2103{2104	int ret = -ENOMEM;2105	void *gid;2106	struct scmi_protocol_instance *pi;2107	const struct scmi_handle *handle = &info->handle;2108 2109	/* Protocol specific devres group */2110	gid = devres_open_group(handle->dev, NULL, GFP_KERNEL);2111	if (!gid) {2112		scmi_protocol_put(proto);2113		goto out;2114	}2115 2116	pi = devm_kzalloc(handle->dev, sizeof(*pi), GFP_KERNEL);2117	if (!pi)2118		goto clean;2119 2120	pi->gid = gid;2121	pi->proto = proto;2122	pi->handle = handle;2123	pi->ph.dev = handle->dev;2124	pi->ph.xops = &xfer_ops;2125	pi->ph.hops = &helpers_ops;2126	pi->ph.set_priv = scmi_set_protocol_priv;2127	pi->ph.get_priv = scmi_get_protocol_priv;2128	refcount_set(&pi->users, 1);2129	/* proto->init is assured NON NULL by scmi_protocol_register */2130	ret = pi->proto->instance_init(&pi->ph);2131	if (ret)2132		goto clean;2133 2134	ret = idr_alloc(&info->protocols, pi, proto->id, proto->id + 1,2135			GFP_KERNEL);2136	if (ret != proto->id)2137		goto clean;2138 2139	/*2140	 * Warn but ignore events registration errors since we do not want2141	 * to skip whole protocols if their notifications are messed up.2142	 */2143	if (pi->proto->events) {2144		ret = scmi_register_protocol_events(handle, pi->proto->id,2145						    &pi->ph,2146						    pi->proto->events);2147		if (ret)2148			dev_warn(handle->dev,2149				 "Protocol:%X - Events Registration Failed - err:%d\n",2150				 pi->proto->id, ret);2151	}2152 2153	devres_close_group(handle->dev, pi->gid);2154	dev_dbg(handle->dev, "Initialized protocol: 0x%X\n", pi->proto->id);2155 2156	if (pi->version > proto->supported_version) {2157		ret = scmi_protocol_version_negotiate(&pi->ph);2158		if (!ret) {2159			dev_info(handle->dev,2160				 "Protocol 0x%X successfully negotiated version 0x%X\n",2161				 proto->id, pi->negotiated_version);2162		} else {2163			dev_warn(handle->dev,2164				 "Detected UNSUPPORTED higher version 0x%X for protocol 0x%X.\n",2165				 pi->version, pi->proto->id);2166			dev_warn(handle->dev,2167				 "Trying version 0x%X. Backward compatibility is NOT assured.\n",2168				 pi->proto->supported_version);2169		}2170	}2171 2172	return pi;2173 2174clean:2175	/* Take care to put the protocol module's owner before releasing all */2176	scmi_protocol_put(proto);2177	devres_release_group(handle->dev, gid);2178out:2179	return ERR_PTR(ret);2180}2181 2182/**2183 * scmi_get_protocol_instance  - Protocol initialization helper.2184 * @handle: A reference to the SCMI platform instance.2185 * @protocol_id: The protocol being requested.2186 *2187 * In case the required protocol has never been requested before for this2188 * instance, allocate and initialize all the needed structures while handling2189 * resource allocation with a dedicated per-protocol devres subgroup.2190 *2191 * Return: A reference to an initialized protocol instance or error on failure:2192 *	   in particular returns -EPROBE_DEFER when the desired protocol could2193 *	   NOT be found.2194 */2195static struct scmi_protocol_instance * __must_check2196scmi_get_protocol_instance(const struct scmi_handle *handle, u8 protocol_id)2197{2198	struct scmi_protocol_instance *pi;2199	struct scmi_info *info = handle_to_scmi_info(handle);2200 2201	mutex_lock(&info->protocols_mtx);2202	pi = idr_find(&info->protocols, protocol_id);2203 2204	if (pi) {2205		refcount_inc(&pi->users);2206	} else {2207		const struct scmi_protocol *proto;2208 2209		/* Fails if protocol not registered on bus */2210		proto = scmi_protocol_get(protocol_id, &info->version);2211		if (proto)2212			pi = scmi_alloc_init_protocol_instance(info, proto);2213		else2214			pi = ERR_PTR(-EPROBE_DEFER);2215	}2216	mutex_unlock(&info->protocols_mtx);2217 2218	return pi;2219}2220 2221/**2222 * scmi_protocol_acquire  - Protocol acquire2223 * @handle: A reference to the SCMI platform instance.2224 * @protocol_id: The protocol being requested.2225 *2226 * Register a new user for the requested protocol on the specified SCMI2227 * platform instance, possibly triggering its initialization on first user.2228 *2229 * Return: 0 if protocol was acquired successfully.2230 */2231int scmi_protocol_acquire(const struct scmi_handle *handle, u8 protocol_id)2232{2233	return PTR_ERR_OR_ZERO(scmi_get_protocol_instance(handle, protocol_id));2234}2235 2236/**2237 * scmi_protocol_release  - Protocol de-initialization helper.2238 * @handle: A reference to the SCMI platform instance.2239 * @protocol_id: The protocol being requested.2240 *2241 * Remove one user for the specified protocol and triggers de-initialization2242 * and resources de-allocation once the last user has gone.2243 */2244void scmi_protocol_release(const struct scmi_handle *handle, u8 protocol_id)2245{2246	struct scmi_info *info = handle_to_scmi_info(handle);2247	struct scmi_protocol_instance *pi;2248 2249	mutex_lock(&info->protocols_mtx);2250	pi = idr_find(&info->protocols, protocol_id);2251	if (WARN_ON(!pi))2252		goto out;2253 2254	if (refcount_dec_and_test(&pi->users)) {2255		void *gid = pi->gid;2256 2257		if (pi->proto->events)2258			scmi_deregister_protocol_events(handle, protocol_id);2259 2260		if (pi->proto->instance_deinit)2261			pi->proto->instance_deinit(&pi->ph);2262 2263		idr_remove(&info->protocols, protocol_id);2264 2265		scmi_protocol_put(pi->proto);2266 2267		devres_release_group(handle->dev, gid);2268		dev_dbg(handle->dev, "De-Initialized protocol: 0x%X\n",2269			protocol_id);2270	}2271 2272out:2273	mutex_unlock(&info->protocols_mtx);2274}2275 2276void scmi_setup_protocol_implemented(const struct scmi_protocol_handle *ph,2277				     u8 *prot_imp)2278{2279	const struct scmi_protocol_instance *pi = ph_to_pi(ph);2280	struct scmi_info *info = handle_to_scmi_info(pi->handle);2281 2282	info->protocols_imp = prot_imp;2283}2284 2285static bool2286scmi_is_protocol_implemented(const struct scmi_handle *handle, u8 prot_id)2287{2288	int i;2289	struct scmi_info *info = handle_to_scmi_info(handle);2290	struct scmi_revision_info *rev = handle->version;2291 2292	if (!info->protocols_imp)2293		return false;2294 2295	for (i = 0; i < rev->num_protocols; i++)2296		if (info->protocols_imp[i] == prot_id)2297			return true;2298	return false;2299}2300 2301struct scmi_protocol_devres {2302	const struct scmi_handle *handle;2303	u8 protocol_id;2304};2305 2306static void scmi_devm_release_protocol(struct device *dev, void *res)2307{2308	struct scmi_protocol_devres *dres = res;2309 2310	scmi_protocol_release(dres->handle, dres->protocol_id);2311}2312 2313static struct scmi_protocol_instance __must_check *2314scmi_devres_protocol_instance_get(struct scmi_device *sdev, u8 protocol_id)2315{2316	struct scmi_protocol_instance *pi;2317	struct scmi_protocol_devres *dres;2318 2319	dres = devres_alloc(scmi_devm_release_protocol,2320			    sizeof(*dres), GFP_KERNEL);2321	if (!dres)2322		return ERR_PTR(-ENOMEM);2323 2324	pi = scmi_get_protocol_instance(sdev->handle, protocol_id);2325	if (IS_ERR(pi)) {2326		devres_free(dres);2327		return pi;2328	}2329 2330	dres->handle = sdev->handle;2331	dres->protocol_id = protocol_id;2332	devres_add(&sdev->dev, dres);2333 2334	return pi;2335}2336 2337/**2338 * scmi_devm_protocol_get  - Devres managed get protocol operations and handle2339 * @sdev: A reference to an scmi_device whose embedded struct device is to2340 *	  be used for devres accounting.2341 * @protocol_id: The protocol being requested.2342 * @ph: A pointer reference used to pass back the associated protocol handle.2343 *2344 * Get hold of a protocol accounting for its usage, eventually triggering its2345 * initialization, and returning the protocol specific operations and related2346 * protocol handle which will be used as first argument in most of the2347 * protocols operations methods.2348 * Being a devres based managed method, protocol hold will be automatically2349 * released, and possibly de-initialized on last user, once the SCMI driver2350 * owning the scmi_device is unbound from it.2351 *2352 * Return: A reference to the requested protocol operations or error.2353 *	   Must be checked for errors by caller.2354 */2355static const void __must_check *2356scmi_devm_protocol_get(struct scmi_device *sdev, u8 protocol_id,2357		       struct scmi_protocol_handle **ph)2358{2359	struct scmi_protocol_instance *pi;2360 2361	if (!ph)2362		return ERR_PTR(-EINVAL);2363 2364	pi = scmi_devres_protocol_instance_get(sdev, protocol_id);2365	if (IS_ERR(pi))2366		return pi;2367 2368	*ph = &pi->ph;2369 2370	return pi->proto->ops;2371}2372 2373/**2374 * scmi_devm_protocol_acquire  - Devres managed helper to get hold of a protocol2375 * @sdev: A reference to an scmi_device whose embedded struct device is to2376 *	  be used for devres accounting.2377 * @protocol_id: The protocol being requested.2378 *2379 * Get hold of a protocol accounting for its usage, possibly triggering its2380 * initialization but without getting access to its protocol specific operations2381 * and handle.2382 *2383 * Being a devres based managed method, protocol hold will be automatically2384 * released, and possibly de-initialized on last user, once the SCMI driver2385 * owning the scmi_device is unbound from it.2386 *2387 * Return: 0 on SUCCESS2388 */2389static int __must_check scmi_devm_protocol_acquire(struct scmi_device *sdev,2390						   u8 protocol_id)2391{2392	struct scmi_protocol_instance *pi;2393 2394	pi = scmi_devres_protocol_instance_get(sdev, protocol_id);2395	if (IS_ERR(pi))2396		return PTR_ERR(pi);2397 2398	return 0;2399}2400 2401static int scmi_devm_protocol_match(struct device *dev, void *res, void *data)2402{2403	struct scmi_protocol_devres *dres = res;2404 2405	if (WARN_ON(!dres || !data))2406		return 0;2407 2408	return dres->protocol_id == *((u8 *)data);2409}2410 2411/**2412 * scmi_devm_protocol_put  - Devres managed put protocol operations and handle2413 * @sdev: A reference to an scmi_device whose embedded struct device is to2414 *	  be used for devres accounting.2415 * @protocol_id: The protocol being requested.2416 *2417 * Explicitly release a protocol hold previously obtained calling the above2418 * @scmi_devm_protocol_get.2419 */2420static void scmi_devm_protocol_put(struct scmi_device *sdev, u8 protocol_id)2421{2422	int ret;2423 2424	ret = devres_release(&sdev->dev, scmi_devm_release_protocol,2425			     scmi_devm_protocol_match, &protocol_id);2426	WARN_ON(ret);2427}2428 2429/**2430 * scmi_is_transport_atomic  - Method to check if underlying transport for an2431 * SCMI instance is configured as atomic.2432 *2433 * @handle: A reference to the SCMI platform instance.2434 * @atomic_threshold: An optional return value for the system wide currently2435 *		      configured threshold for atomic operations.2436 *2437 * Return: True if transport is configured as atomic2438 */2439static bool scmi_is_transport_atomic(const struct scmi_handle *handle,2440				     unsigned int *atomic_threshold)2441{2442	bool ret;2443	struct scmi_info *info = handle_to_scmi_info(handle);2444 2445	ret = info->desc->atomic_enabled &&2446		is_transport_polling_capable(info->desc);2447	if (ret && atomic_threshold)2448		*atomic_threshold = info->atomic_threshold;2449 2450	return ret;2451}2452 2453/**2454 * scmi_handle_get() - Get the SCMI handle for a device2455 *2456 * @dev: pointer to device for which we want SCMI handle2457 *2458 * NOTE: The function does not track individual clients of the framework2459 * and is expected to be maintained by caller of SCMI protocol library.2460 * scmi_handle_put must be balanced with successful scmi_handle_get2461 *2462 * Return: pointer to handle if successful, NULL on error2463 */2464static struct scmi_handle *scmi_handle_get(struct device *dev)2465{2466	struct list_head *p;2467	struct scmi_info *info;2468	struct scmi_handle *handle = NULL;2469 2470	mutex_lock(&scmi_list_mutex);2471	list_for_each(p, &scmi_list) {2472		info = list_entry(p, struct scmi_info, node);2473		if (dev->parent == info->dev) {2474			info->users++;2475			handle = &info->handle;2476			break;2477		}2478	}2479	mutex_unlock(&scmi_list_mutex);2480 2481	return handle;2482}2483 2484/**2485 * scmi_handle_put() - Release the handle acquired by scmi_handle_get2486 *2487 * @handle: handle acquired by scmi_handle_get2488 *2489 * NOTE: The function does not track individual clients of the framework2490 * and is expected to be maintained by caller of SCMI protocol library.2491 * scmi_handle_put must be balanced with successful scmi_handle_get2492 *2493 * Return: 0 is successfully released2494 *	if null was passed, it returns -EINVAL;2495 */2496static int scmi_handle_put(const struct scmi_handle *handle)2497{2498	struct scmi_info *info;2499 2500	if (!handle)2501		return -EINVAL;2502 2503	info = handle_to_scmi_info(handle);2504	mutex_lock(&scmi_list_mutex);2505	if (!WARN_ON(!info->users))2506		info->users--;2507	mutex_unlock(&scmi_list_mutex);2508 2509	return 0;2510}2511 2512static void scmi_device_link_add(struct device *consumer,2513				 struct device *supplier)2514{2515	struct device_link *link;2516 2517	link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER);2518 2519	WARN_ON(!link);2520}2521 2522static void scmi_set_handle(struct scmi_device *scmi_dev)2523{2524	scmi_dev->handle = scmi_handle_get(&scmi_dev->dev);2525	if (scmi_dev->handle)2526		scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev);2527}2528 2529static int __scmi_xfer_info_init(struct scmi_info *sinfo,2530				 struct scmi_xfers_info *info)2531{2532	int i;2533	struct scmi_xfer *xfer;2534	struct device *dev = sinfo->dev;2535	const struct scmi_desc *desc = sinfo->desc;2536 2537	/* Pre-allocated messages, no more than what hdr.seq can support */2538	if (WARN_ON(!info->max_msg || info->max_msg > MSG_TOKEN_MAX)) {2539		dev_err(dev,2540			"Invalid maximum messages %d, not in range [1 - %lu]\n",2541			info->max_msg, MSG_TOKEN_MAX);2542		return -EINVAL;2543	}2544 2545	hash_init(info->pending_xfers);2546 2547	/* Allocate a bitmask sized to hold MSG_TOKEN_MAX tokens */2548	info->xfer_alloc_table = devm_bitmap_zalloc(dev, MSG_TOKEN_MAX,2549						    GFP_KERNEL);2550	if (!info->xfer_alloc_table)2551		return -ENOMEM;2552 2553	/*2554	 * Preallocate a number of xfers equal to max inflight messages,2555	 * pre-initialize the buffer pointer to pre-allocated buffers and2556	 * attach all of them to the free list2557	 */2558	INIT_HLIST_HEAD(&info->free_xfers);2559	for (i = 0; i < info->max_msg; i++) {2560		xfer = devm_kzalloc(dev, sizeof(*xfer), GFP_KERNEL);2561		if (!xfer)2562			return -ENOMEM;2563 2564		xfer->rx.buf = devm_kcalloc(dev, sizeof(u8), desc->max_msg_size,2565					    GFP_KERNEL);2566		if (!xfer->rx.buf)2567			return -ENOMEM;2568 2569		xfer->tx.buf = xfer->rx.buf;2570		init_completion(&xfer->done);2571		spin_lock_init(&xfer->lock);2572 2573		/* Add initialized xfer to the free list */2574		hlist_add_head(&xfer->node, &info->free_xfers);2575	}2576 2577	spin_lock_init(&info->xfer_lock);2578 2579	return 0;2580}2581 2582static int scmi_channels_max_msg_configure(struct scmi_info *sinfo)2583{2584	const struct scmi_desc *desc = sinfo->desc;2585 2586	if (!desc->ops->get_max_msg) {2587		sinfo->tx_minfo.max_msg = desc->max_msg;2588		sinfo->rx_minfo.max_msg = desc->max_msg;2589	} else {2590		struct scmi_chan_info *base_cinfo;2591 2592		base_cinfo = idr_find(&sinfo->tx_idr, SCMI_PROTOCOL_BASE);2593		if (!base_cinfo)2594			return -EINVAL;2595		sinfo->tx_minfo.max_msg = desc->ops->get_max_msg(base_cinfo);2596 2597		/* RX channel is optional so can be skipped */2598		base_cinfo = idr_find(&sinfo->rx_idr, SCMI_PROTOCOL_BASE);2599		if (base_cinfo)2600			sinfo->rx_minfo.max_msg =2601				desc->ops->get_max_msg(base_cinfo);2602	}2603 2604	return 0;2605}2606 2607static int scmi_xfer_info_init(struct scmi_info *sinfo)2608{2609	int ret;2610 2611	ret = scmi_channels_max_msg_configure(sinfo);2612	if (ret)2613		return ret;2614 2615	ret = __scmi_xfer_info_init(sinfo, &sinfo->tx_minfo);2616	if (!ret && !idr_is_empty(&sinfo->rx_idr))2617		ret = __scmi_xfer_info_init(sinfo, &sinfo->rx_minfo);2618 2619	return ret;2620}2621 2622static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node,2623			   int prot_id, bool tx)2624{2625	int ret, idx;2626	char name[32];2627	struct scmi_chan_info *cinfo;2628	struct idr *idr;2629	struct scmi_device *tdev = NULL;2630 2631	/* Transmit channel is first entry i.e. index 0 */2632	idx = tx ? 0 : 1;2633	idr = tx ? &info->tx_idr : &info->rx_idr;2634 2635	if (!info->desc->ops->chan_available(of_node, idx)) {2636		cinfo = idr_find(idr, SCMI_PROTOCOL_BASE);2637		if (unlikely(!cinfo)) /* Possible only if platform has no Rx */2638			return -EINVAL;2639		goto idr_alloc;2640	}2641 2642	cinfo = devm_kzalloc(info->dev, sizeof(*cinfo), GFP_KERNEL);2643	if (!cinfo)2644		return -ENOMEM;2645 2646	cinfo->is_p2a = !tx;2647	cinfo->rx_timeout_ms = info->desc->max_rx_timeout_ms;2648 2649	/* Create a unique name for this transport device */2650	snprintf(name, 32, "__scmi_transport_device_%s_%02X",2651		 idx ? "rx" : "tx", prot_id);2652	/* Create a uniquely named, dedicated transport device for this chan */2653	tdev = scmi_device_create(of_node, info->dev, prot_id, name);2654	if (!tdev) {2655		dev_err(info->dev,2656			"failed to create transport device (%s)\n", name);2657		devm_kfree(info->dev, cinfo);2658		return -EINVAL;2659	}2660	of_node_get(of_node);2661 2662	cinfo->id = prot_id;2663	cinfo->dev = &tdev->dev;2664	ret = info->desc->ops->chan_setup(cinfo, info->dev, tx);2665	if (ret) {2666		of_node_put(of_node);2667		scmi_device_destroy(info->dev, prot_id, name);2668		devm_kfree(info->dev, cinfo);2669		return ret;2670	}2671 2672	if (tx && is_polling_required(cinfo, info->desc)) {2673		if (is_transport_polling_capable(info->desc))2674			dev_info(&tdev->dev,2675				 "Enabled polling mode TX channel - prot_id:%d\n",2676				 prot_id);2677		else2678			dev_warn(&tdev->dev,2679				 "Polling mode NOT supported by transport.\n");2680	}2681 2682idr_alloc:2683	ret = idr_alloc(idr, cinfo, prot_id, prot_id + 1, GFP_KERNEL);2684	if (ret != prot_id) {2685		dev_err(info->dev,2686			"unable to allocate SCMI idr slot err %d\n", ret);2687		/* Destroy channel and device only if created by this call. */2688		if (tdev) {2689			of_node_put(of_node);2690			scmi_device_destroy(info->dev, prot_id, name);2691			devm_kfree(info->dev, cinfo);2692		}2693		return ret;2694	}2695 2696	cinfo->handle = &info->handle;2697	return 0;2698}2699 2700static inline int2701scmi_txrx_setup(struct scmi_info *info, struct device_node *of_node,2702		int prot_id)2703{2704	int ret = scmi_chan_setup(info, of_node, prot_id, true);2705 2706	if (!ret) {2707		/* Rx is optional, report only memory errors */2708		ret = scmi_chan_setup(info, of_node, prot_id, false);2709		if (ret && ret != -ENOMEM)2710			ret = 0;2711	}2712 2713	if (ret)2714		dev_err(info->dev,2715			"failed to setup channel for protocol:0x%X\n", prot_id);2716 2717	return ret;2718}2719 2720/**2721 * scmi_channels_setup  - Helper to initialize all required channels2722 *2723 * @info: The SCMI instance descriptor.2724 *2725 * Initialize all the channels found described in the DT against the underlying2726 * configured transport using custom defined dedicated devices instead of2727 * borrowing devices from the SCMI drivers; this way channels are initialized2728 * upfront during core SCMI stack probing and are no more coupled with SCMI2729 * devices used by SCMI drivers.2730 *2731 * Note that, even though a pair of TX/RX channels is associated to each2732 * protocol defined in the DT, a distinct freshly initialized channel is2733 * created only if the DT node for the protocol at hand describes a dedicated2734 * channel: in all the other cases the common BASE protocol channel is reused.2735 *2736 * Return: 0 on Success2737 */2738static int scmi_channels_setup(struct scmi_info *info)2739{2740	int ret;2741	struct device_node *top_np = info->dev->of_node;2742 2743	/* Initialize a common generic channel at first */2744	ret = scmi_txrx_setup(info, top_np, SCMI_PROTOCOL_BASE);2745	if (ret)2746		return ret;2747 2748	for_each_available_child_of_node_scoped(top_np, child) {2749		u32 prot_id;2750 2751		if (of_property_read_u32(child, "reg", &prot_id))2752			continue;2753 2754		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id))2755			dev_err(info->dev,2756				"Out of range protocol %d\n", prot_id);2757 2758		ret = scmi_txrx_setup(info, child, prot_id);2759		if (ret)2760			return ret;2761	}2762 2763	return 0;2764}2765 2766static int scmi_chan_destroy(int id, void *p, void *idr)2767{2768	struct scmi_chan_info *cinfo = p;2769 2770	if (cinfo->dev) {2771		struct scmi_info *info = handle_to_scmi_info(cinfo->handle);2772		struct scmi_device *sdev = to_scmi_dev(cinfo->dev);2773 2774		of_node_put(cinfo->dev->of_node);2775		scmi_device_destroy(info->dev, id, sdev->name);2776		cinfo->dev = NULL;2777	}2778 2779	idr_remove(idr, id);2780 2781	return 0;2782}2783 2784static void scmi_cleanup_channels(struct scmi_info *info, struct idr *idr)2785{2786	/* At first free all channels at the transport layer ... */2787	idr_for_each(idr, info->desc->ops->chan_free, idr);2788 2789	/* ...then destroy all underlying devices */2790	idr_for_each(idr, scmi_chan_destroy, idr);2791 2792	idr_destroy(idr);2793}2794 2795static void scmi_cleanup_txrx_channels(struct scmi_info *info)2796{2797	scmi_cleanup_channels(info, &info->tx_idr);2798 2799	scmi_cleanup_channels(info, &info->rx_idr);2800}2801 2802static int scmi_bus_notifier(struct notifier_block *nb,2803			     unsigned long action, void *data)2804{2805	struct scmi_info *info = bus_nb_to_scmi_info(nb);2806	struct scmi_device *sdev = to_scmi_dev(data);2807 2808	/* Skip transport devices and devices of different SCMI instances */2809	if (!strncmp(sdev->name, "__scmi_transport_device", 23) ||2810	    sdev->dev.parent != info->dev)2811		return NOTIFY_DONE;2812 2813	switch (action) {2814	case BUS_NOTIFY_BIND_DRIVER:2815		/* setup handle now as the transport is ready */2816		scmi_set_handle(sdev);2817		break;2818	case BUS_NOTIFY_UNBOUND_DRIVER:2819		scmi_handle_put(sdev->handle);2820		sdev->handle = NULL;2821		break;2822	default:2823		return NOTIFY_DONE;2824	}2825 2826	dev_dbg(info->dev, "Device %s (%s) is now %s\n", dev_name(&sdev->dev),2827		sdev->name, action == BUS_NOTIFY_BIND_DRIVER ?2828		"about to be BOUND." : "UNBOUND.");2829 2830	return NOTIFY_OK;2831}2832 2833static int scmi_device_request_notifier(struct notifier_block *nb,2834					unsigned long action, void *data)2835{2836	struct device_node *np;2837	struct scmi_device_id *id_table = data;2838	struct scmi_info *info = req_nb_to_scmi_info(nb);2839 2840	np = idr_find(&info->active_protocols, id_table->protocol_id);2841	if (!np)2842		return NOTIFY_DONE;2843 2844	dev_dbg(info->dev, "%sRequested device (%s) for protocol 0x%x\n",2845		action == SCMI_BUS_NOTIFY_DEVICE_REQUEST ? "" : "UN-",2846		id_table->name, id_table->protocol_id);2847 2848	switch (action) {2849	case SCMI_BUS_NOTIFY_DEVICE_REQUEST:2850		scmi_create_protocol_devices(np, info, id_table->protocol_id,2851					     id_table->name);2852		break;2853	case SCMI_BUS_NOTIFY_DEVICE_UNREQUEST:2854		scmi_destroy_protocol_devices(info, id_table->protocol_id,2855					      id_table->name);2856		break;2857	default:2858		return NOTIFY_DONE;2859	}2860 2861	return NOTIFY_OK;2862}2863 2864static const char * const dbg_counter_strs[] = {2865	"sent_ok",2866	"sent_fail",2867	"sent_fail_polling_unsupported",2868	"sent_fail_channel_not_found",2869	"response_ok",2870	"notification_ok",2871	"delayed_response_ok",2872	"xfers_response_timeout",2873	"xfers_response_polled_timeout",2874	"response_polled_ok",2875	"err_msg_unexpected",2876	"err_msg_invalid",2877	"err_msg_nomem",2878	"err_protocol",2879};2880 2881static ssize_t reset_all_on_write(struct file *filp, const char __user *buf,2882				  size_t count, loff_t *ppos)2883{2884	struct scmi_debug_info *dbg = filp->private_data;2885 2886	for (int i = 0; i < SCMI_DEBUG_COUNTERS_LAST; i++)2887		atomic_set(&dbg->counters[i], 0);2888 2889	return count;2890}2891 2892static const struct file_operations fops_reset_counts = {2893	.owner = THIS_MODULE,2894	.open = simple_open,2895	.write = reset_all_on_write,2896};2897 2898static void scmi_debugfs_counters_setup(struct scmi_debug_info *dbg,2899					struct dentry *trans)2900{2901	struct dentry *counters;2902	int idx;2903 2904	counters = debugfs_create_dir("counters", trans);2905 2906	for (idx = 0; idx < SCMI_DEBUG_COUNTERS_LAST; idx++)2907		debugfs_create_atomic_t(dbg_counter_strs[idx], 0600, counters,2908					&dbg->counters[idx]);2909 2910	debugfs_create_file("reset", 0200, counters, dbg, &fops_reset_counts);2911}2912 2913static void scmi_debugfs_common_cleanup(void *d)2914{2915	struct scmi_debug_info *dbg = d;2916 2917	if (!dbg)2918		return;2919 2920	debugfs_remove_recursive(dbg->top_dentry);2921	kfree(dbg->name);2922	kfree(dbg->type);2923}2924 2925static struct scmi_debug_info *scmi_debugfs_common_setup(struct scmi_info *info)2926{2927	char top_dir[16];2928	struct dentry *trans, *top_dentry;2929	struct scmi_debug_info *dbg;2930	const char *c_ptr = NULL;2931 2932	dbg = devm_kzalloc(info->dev, sizeof(*dbg), GFP_KERNEL);2933	if (!dbg)2934		return NULL;2935 2936	dbg->name = kstrdup(of_node_full_name(info->dev->of_node), GFP_KERNEL);2937	if (!dbg->name) {2938		devm_kfree(info->dev, dbg);2939		return NULL;2940	}2941 2942	of_property_read_string(info->dev->of_node, "compatible", &c_ptr);2943	dbg->type = kstrdup(c_ptr, GFP_KERNEL);2944	if (!dbg->type) {2945		kfree(dbg->name);2946		devm_kfree(info->dev, dbg);2947		return NULL;2948	}2949 2950	snprintf(top_dir, 16, "%d", info->id);2951	top_dentry = debugfs_create_dir(top_dir, scmi_top_dentry);2952	trans = debugfs_create_dir("transport", top_dentry);2953 2954	dbg->is_atomic = info->desc->atomic_enabled &&2955				is_transport_polling_capable(info->desc);2956 2957	debugfs_create_str("instance_name", 0400, top_dentry,2958			   (char **)&dbg->name);2959 2960	debugfs_create_u32("atomic_threshold_us", 0400, top_dentry,2961			   &info->atomic_threshold);2962 2963	debugfs_create_str("type", 0400, trans, (char **)&dbg->type);2964 2965	debugfs_create_bool("is_atomic", 0400, trans, &dbg->is_atomic);2966 2967	debugfs_create_u32("max_rx_timeout_ms", 0400, trans,2968			   (u32 *)&info->desc->max_rx_timeout_ms);2969 2970	debugfs_create_u32("max_msg_size", 0400, trans,2971			   (u32 *)&info->desc->max_msg_size);2972 2973	debugfs_create_u32("tx_max_msg", 0400, trans,2974			   (u32 *)&info->tx_minfo.max_msg);2975 2976	debugfs_create_u32("rx_max_msg", 0400, trans,2977			   (u32 *)&info->rx_minfo.max_msg);2978 2979	if (IS_ENABLED(CONFIG_ARM_SCMI_DEBUG_COUNTERS))2980		scmi_debugfs_counters_setup(dbg, trans);2981 2982	dbg->top_dentry = top_dentry;2983 2984	if (devm_add_action_or_reset(info->dev,2985				     scmi_debugfs_common_cleanup, dbg))2986		return NULL;2987 2988	return dbg;2989}2990 2991static int scmi_debugfs_raw_mode_setup(struct scmi_info *info)2992{2993	int id, num_chans = 0, ret = 0;2994	struct scmi_chan_info *cinfo;2995	u8 channels[SCMI_MAX_CHANNELS] = {};2996	DECLARE_BITMAP(protos, SCMI_MAX_CHANNELS) = {};2997 2998	if (!info->dbg)2999		return -EINVAL;3000 3001	/* Enumerate all channels to collect their ids */3002	idr_for_each_entry(&info->tx_idr, cinfo, id) {3003		/*3004		 * Cannot happen, but be defensive.3005		 * Zero as num_chans is ok, warn and carry on.3006		 */3007		if (num_chans >= SCMI_MAX_CHANNELS || !cinfo) {3008			dev_warn(info->dev,3009				 "SCMI RAW - Error enumerating channels\n");3010			break;3011		}3012 3013		if (!test_bit(cinfo->id, protos)) {3014			channels[num_chans++] = cinfo->id;3015			set_bit(cinfo->id, protos);3016		}3017	}3018 3019	info->raw = scmi_raw_mode_init(&info->handle, info->dbg->top_dentry,3020				       info->id, channels, num_chans,3021				       info->desc, info->tx_minfo.max_msg);3022	if (IS_ERR(info->raw)) {3023		dev_err(info->dev, "Failed to initialize SCMI RAW Mode !\n");3024		ret = PTR_ERR(info->raw);3025		info->raw = NULL;3026	}3027 3028	return ret;3029}3030 3031static const struct scmi_desc *scmi_transport_setup(struct device *dev)3032{3033	struct scmi_transport *trans;3034	int ret;3035 3036	trans = dev_get_platdata(dev);3037	if (!trans || !trans->desc || !trans->supplier || !trans->core_ops)3038		return NULL;3039 3040	if (!device_link_add(dev, trans->supplier, DL_FLAG_AUTOREMOVE_CONSUMER)) {3041		dev_err(dev,3042			"Adding link to supplier transport device failed\n");3043		return NULL;3044	}3045 3046	/* Provide core transport ops */3047	*trans->core_ops = &scmi_trans_core_ops;3048 3049	dev_info(dev, "Using %s\n", dev_driver_string(trans->supplier));3050 3051	ret = of_property_read_u32(dev->of_node, "arm,max-rx-timeout-ms",3052				   &trans->desc->max_rx_timeout_ms);3053	if (ret && ret != -EINVAL)3054		dev_err(dev, "Malformed arm,max-rx-timeout-ms DT property.\n");3055 3056	dev_info(dev, "SCMI max-rx-timeout: %dms\n",3057		 trans->desc->max_rx_timeout_ms);3058 3059	return trans->desc;3060}3061 3062static int scmi_probe(struct platform_device *pdev)3063{3064	int ret;3065	char *err_str = "probe failure\n";3066	struct scmi_handle *handle;3067	const struct scmi_desc *desc;3068	struct scmi_info *info;3069	bool coex = IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT_COEX);3070	struct device *dev = &pdev->dev;3071	struct device_node *child, *np = dev->of_node;3072 3073	desc = scmi_transport_setup(dev);3074	if (!desc) {3075		err_str = "transport invalid\n";3076		ret = -EINVAL;3077		goto out_err;3078	}3079 3080	info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL);3081	if (!info)3082		return -ENOMEM;3083 3084	info->id = ida_alloc_min(&scmi_id, 0, GFP_KERNEL);3085	if (info->id < 0)3086		return info->id;3087 3088	info->dev = dev;3089	info->desc = desc;3090	info->bus_nb.notifier_call = scmi_bus_notifier;3091	info->dev_req_nb.notifier_call = scmi_device_request_notifier;3092	INIT_LIST_HEAD(&info->node);3093	idr_init(&info->protocols);3094	mutex_init(&info->protocols_mtx);3095	idr_init(&info->active_protocols);3096	mutex_init(&info->devreq_mtx);3097 3098	platform_set_drvdata(pdev, info);3099	idr_init(&info->tx_idr);3100	idr_init(&info->rx_idr);3101 3102	handle = &info->handle;3103	handle->dev = info->dev;3104	handle->version = &info->version;3105	handle->devm_protocol_acquire = scmi_devm_protocol_acquire;3106	handle->devm_protocol_get = scmi_devm_protocol_get;3107	handle->devm_protocol_put = scmi_devm_protocol_put;3108 3109	/* System wide atomic threshold for atomic ops .. if any */3110	if (!of_property_read_u32(np, "atomic-threshold-us",3111				  &info->atomic_threshold))3112		dev_info(dev,3113			 "SCMI System wide atomic threshold set to %d us\n",3114			 info->atomic_threshold);3115	handle->is_transport_atomic = scmi_is_transport_atomic;3116 3117	/* Setup all channels described in the DT at first */3118	ret = scmi_channels_setup(info);3119	if (ret) {3120		err_str = "failed to setup channels\n";3121		goto clear_ida;3122	}3123 3124	ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb);3125	if (ret) {3126		err_str = "failed to register bus notifier\n";3127		goto clear_txrx_setup;3128	}3129 3130	ret = blocking_notifier_chain_register(&scmi_requested_devices_nh,3131					       &info->dev_req_nb);3132	if (ret) {3133		err_str = "failed to register device notifier\n";3134		goto clear_bus_notifier;3135	}3136 3137	ret = scmi_xfer_info_init(info);3138	if (ret) {3139		err_str = "failed to init xfers pool\n";3140		goto clear_dev_req_notifier;3141	}3142 3143	if (scmi_top_dentry) {3144		info->dbg = scmi_debugfs_common_setup(info);3145		if (!info->dbg)3146			dev_warn(dev, "Failed to setup SCMI debugfs.\n");3147 3148		if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) {3149			ret = scmi_debugfs_raw_mode_setup(info);3150			if (!coex) {3151				if (ret)3152					goto clear_dev_req_notifier;3153 3154				/* Bail out anyway when coex disabled. */3155				return 0;3156			}3157 3158			/* Coex enabled, carry on in any case. */3159			dev_info(dev, "SCMI RAW Mode COEX enabled !\n");3160		}3161	}3162 3163	if (scmi_notification_init(handle))3164		dev_err(dev, "SCMI Notifications NOT available.\n");3165 3166	if (info->desc->atomic_enabled &&3167	    !is_transport_polling_capable(info->desc))3168		dev_err(dev,3169			"Transport is not polling capable. Atomic mode not supported.\n");3170 3171	/*3172	 * Trigger SCMI Base protocol initialization.3173	 * It's mandatory and won't be ever released/deinit until the3174	 * SCMI stack is shutdown/unloaded as a whole.3175	 */3176	ret = scmi_protocol_acquire(handle, SCMI_PROTOCOL_BASE);3177	if (ret) {3178		err_str = "unable to communicate with SCMI\n";3179		if (coex) {3180			dev_err(dev, "%s", err_str);3181			return 0;3182		}3183		goto notification_exit;3184	}3185 3186	mutex_lock(&scmi_list_mutex);3187	list_add_tail(&info->node, &scmi_list);3188	mutex_unlock(&scmi_list_mutex);3189 3190	for_each_available_child_of_node(np, child) {3191		u32 prot_id;3192 3193		if (of_property_read_u32(child, "reg", &prot_id))3194			continue;3195 3196		if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id))3197			dev_err(dev, "Out of range protocol %d\n", prot_id);3198 3199		if (!scmi_is_protocol_implemented(handle, prot_id)) {3200			dev_err(dev, "SCMI protocol %d not implemented\n",3201				prot_id);3202			continue;3203		}3204 3205		/*3206		 * Save this valid DT protocol descriptor amongst3207		 * @active_protocols for this SCMI instance/3208		 */3209		ret = idr_alloc(&info->active_protocols, child,3210				prot_id, prot_id + 1, GFP_KERNEL);3211		if (ret != prot_id) {3212			dev_err(dev, "SCMI protocol %d already activated. Skip\n",3213				prot_id);3214			continue;3215		}3216 3217		of_node_get(child);3218		scmi_create_protocol_devices(child, info, prot_id, NULL);3219	}3220 3221	return 0;3222 3223notification_exit:3224	if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT))3225		scmi_raw_mode_cleanup(info->raw);3226	scmi_notification_exit(&info->handle);3227clear_dev_req_notifier:3228	blocking_notifier_chain_unregister(&scmi_requested_devices_nh,3229					   &info->dev_req_nb);3230clear_bus_notifier:3231	bus_unregister_notifier(&scmi_bus_type, &info->bus_nb);3232clear_txrx_setup:3233	scmi_cleanup_txrx_channels(info);3234clear_ida:3235	ida_free(&scmi_id, info->id);3236 3237out_err:3238	return dev_err_probe(dev, ret, "%s", err_str);3239}3240 3241static void scmi_remove(struct platform_device *pdev)3242{3243	int id;3244	struct scmi_info *info = platform_get_drvdata(pdev);3245	struct device_node *child;3246 3247	if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT))3248		scmi_raw_mode_cleanup(info->raw);3249 3250	mutex_lock(&scmi_list_mutex);3251	if (info->users)3252		dev_warn(&pdev->dev,3253			 "Still active SCMI users will be forcibly unbound.\n");3254	list_del(&info->node);3255	mutex_unlock(&scmi_list_mutex);3256 3257	scmi_notification_exit(&info->handle);3258 3259	mutex_lock(&info->protocols_mtx);3260	idr_destroy(&info->protocols);3261	mutex_unlock(&info->protocols_mtx);3262 3263	idr_for_each_entry(&info->active_protocols, child, id)3264		of_node_put(child);3265	idr_destroy(&info->active_protocols);3266 3267	blocking_notifier_chain_unregister(&scmi_requested_devices_nh,3268					   &info->dev_req_nb);3269	bus_unregister_notifier(&scmi_bus_type, &info->bus_nb);3270 3271	/* Safe to free channels since no more users */3272	scmi_cleanup_txrx_channels(info);3273 3274	ida_free(&scmi_id, info->id);3275}3276 3277static ssize_t protocol_version_show(struct device *dev,3278				     struct device_attribute *attr, char *buf)3279{3280	struct scmi_info *info = dev_get_drvdata(dev);3281 3282	return sprintf(buf, "%u.%u\n", info->version.major_ver,3283		       info->version.minor_ver);3284}3285static DEVICE_ATTR_RO(protocol_version);3286 3287static ssize_t firmware_version_show(struct device *dev,3288				     struct device_attribute *attr, char *buf)3289{3290	struct scmi_info *info = dev_get_drvdata(dev);3291 3292	return sprintf(buf, "0x%x\n", info->version.impl_ver);3293}3294static DEVICE_ATTR_RO(firmware_version);3295 3296static ssize_t vendor_id_show(struct device *dev,3297			      struct device_attribute *attr, char *buf)3298{3299	struct scmi_info *info = dev_get_drvdata(dev);3300 3301	return sprintf(buf, "%s\n", info->version.vendor_id);3302}3303static DEVICE_ATTR_RO(vendor_id);3304 3305static ssize_t sub_vendor_id_show(struct device *dev,3306				  struct device_attribute *attr, char *buf)3307{3308	struct scmi_info *info = dev_get_drvdata(dev);3309 3310	return sprintf(buf, "%s\n", info->version.sub_vendor_id);3311}3312static DEVICE_ATTR_RO(sub_vendor_id);3313 3314static struct attribute *versions_attrs[] = {3315	&dev_attr_firmware_version.attr,3316	&dev_attr_protocol_version.attr,3317	&dev_attr_vendor_id.attr,3318	&dev_attr_sub_vendor_id.attr,3319	NULL,3320};3321ATTRIBUTE_GROUPS(versions);3322 3323static struct platform_driver scmi_driver = {3324	.driver = {3325		   .name = "arm-scmi",3326		   .suppress_bind_attrs = true,3327		   .dev_groups = versions_groups,3328		   },3329	.probe = scmi_probe,3330	.remove_new = scmi_remove,3331};3332 3333static struct dentry *scmi_debugfs_init(void)3334{3335	struct dentry *d;3336 3337	d = debugfs_create_dir("scmi", NULL);3338	if (IS_ERR(d)) {3339		pr_err("Could NOT create SCMI top dentry.\n");3340		return NULL;3341	}3342 3343	return d;3344}3345 3346static int __init scmi_driver_init(void)3347{3348	/* Bail out if no SCMI transport was configured */3349	if (WARN_ON(!IS_ENABLED(CONFIG_ARM_SCMI_HAVE_TRANSPORT)))3350		return -EINVAL;3351 3352	if (IS_ENABLED(CONFIG_ARM_SCMI_HAVE_SHMEM))3353		scmi_trans_core_ops.shmem = scmi_shared_mem_operations_get();3354 3355	if (IS_ENABLED(CONFIG_ARM_SCMI_HAVE_MSG))3356		scmi_trans_core_ops.msg = scmi_message_operations_get();3357 3358	if (IS_ENABLED(CONFIG_ARM_SCMI_NEED_DEBUGFS))3359		scmi_top_dentry = scmi_debugfs_init();3360 3361	scmi_base_register();3362 3363	scmi_clock_register();3364	scmi_perf_register();3365	scmi_power_register();3366	scmi_reset_register();3367	scmi_sensors_register();3368	scmi_voltage_register();3369	scmi_system_register();3370	scmi_powercap_register();3371	scmi_pinctrl_register();3372 3373	return platform_driver_register(&scmi_driver);3374}3375module_init(scmi_driver_init);3376 3377static void __exit scmi_driver_exit(void)3378{3379	scmi_base_unregister();3380 3381	scmi_clock_unregister();3382	scmi_perf_unregister();3383	scmi_power_unregister();3384	scmi_reset_unregister();3385	scmi_sensors_unregister();3386	scmi_voltage_unregister();3387	scmi_system_unregister();3388	scmi_powercap_unregister();3389	scmi_pinctrl_unregister();3390 3391	platform_driver_unregister(&scmi_driver);3392 3393	debugfs_remove_recursive(scmi_top_dentry);3394}3395module_exit(scmi_driver_exit);3396 3397MODULE_ALIAS("platform:arm-scmi");3398MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>");3399MODULE_DESCRIPTION("ARM SCMI protocol driver");3400MODULE_LICENSE("GPL v2");3401