brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/*3 * intel_pt_decoder.c: Intel Processor Trace support4 * Copyright (c) 2013-2014, Intel Corporation.5 */6 7#ifndef _GNU_SOURCE8#define _GNU_SOURCE9#endif10#include <stdlib.h>11#include <stdbool.h>12#include <string.h>13#include <errno.h>14#include <stdint.h>15#include <inttypes.h>16#include <linux/compiler.h>17#include <linux/string.h>18#include <linux/zalloc.h>19 20#include "../auxtrace.h"21 22#include "intel-pt-insn-decoder.h"23#include "intel-pt-pkt-decoder.h"24#include "intel-pt-decoder.h"25#include "intel-pt-log.h"26 27#define BITULL(x) (1ULL << (x))28 29/* IA32_RTIT_CTL MSR bits */30#define INTEL_PT_CYC_ENABLE		BITULL(1)31#define INTEL_PT_CYC_THRESHOLD		(BITULL(22) | BITULL(21) | BITULL(20) | BITULL(19))32#define INTEL_PT_CYC_THRESHOLD_SHIFT	1933 34#define INTEL_PT_BLK_SIZE 102435 36#define BIT63 (((uint64_t)1 << 63))37 38#define SEVEN_BYTES 0xffffffffffffffULL39 40#define NO_VMCS 0xffffffffffULL41 42#define INTEL_PT_RETURN 143 44/*45 * Default maximum number of loops with no packets consumed i.e. stuck in a46 * loop.47 */48#define INTEL_PT_MAX_LOOPS 10000049 50struct intel_pt_blk {51	struct intel_pt_blk *prev;52	uint64_t ip[INTEL_PT_BLK_SIZE];53};54 55struct intel_pt_stack {56	struct intel_pt_blk *blk;57	struct intel_pt_blk *spare;58	int pos;59};60 61enum intel_pt_p_once {62	INTEL_PT_PRT_ONCE_UNK_VMCS,63	INTEL_PT_PRT_ONCE_ERANGE,64};65 66enum intel_pt_pkt_state {67	INTEL_PT_STATE_NO_PSB,68	INTEL_PT_STATE_NO_IP,69	INTEL_PT_STATE_ERR_RESYNC,70	INTEL_PT_STATE_IN_SYNC,71	INTEL_PT_STATE_TNT_CONT,72	INTEL_PT_STATE_TNT,73	INTEL_PT_STATE_TIP,74	INTEL_PT_STATE_TIP_PGD,75	INTEL_PT_STATE_FUP,76	INTEL_PT_STATE_FUP_NO_TIP,77	INTEL_PT_STATE_FUP_IN_PSB,78	INTEL_PT_STATE_RESAMPLE,79	INTEL_PT_STATE_VM_TIME_CORRELATION,80};81 82static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)83{84	switch (pkt_state) {85	case INTEL_PT_STATE_NO_PSB:86	case INTEL_PT_STATE_NO_IP:87	case INTEL_PT_STATE_ERR_RESYNC:88	case INTEL_PT_STATE_IN_SYNC:89	case INTEL_PT_STATE_TNT_CONT:90	case INTEL_PT_STATE_RESAMPLE:91	case INTEL_PT_STATE_VM_TIME_CORRELATION:92		return true;93	case INTEL_PT_STATE_TNT:94	case INTEL_PT_STATE_TIP:95	case INTEL_PT_STATE_TIP_PGD:96	case INTEL_PT_STATE_FUP:97	case INTEL_PT_STATE_FUP_NO_TIP:98	case INTEL_PT_STATE_FUP_IN_PSB:99		return false;100	default:101		return true;102	};103}104 105#ifdef INTEL_PT_STRICT106#define INTEL_PT_STATE_ERR1	INTEL_PT_STATE_NO_PSB107#define INTEL_PT_STATE_ERR2	INTEL_PT_STATE_NO_PSB108#define INTEL_PT_STATE_ERR3	INTEL_PT_STATE_NO_PSB109#define INTEL_PT_STATE_ERR4	INTEL_PT_STATE_NO_PSB110#else111#define INTEL_PT_STATE_ERR1	(decoder->pkt_state)112#define INTEL_PT_STATE_ERR2	INTEL_PT_STATE_NO_IP113#define INTEL_PT_STATE_ERR3	INTEL_PT_STATE_ERR_RESYNC114#define INTEL_PT_STATE_ERR4	INTEL_PT_STATE_IN_SYNC115#endif116 117struct intel_pt_decoder {118	int (*get_trace)(struct intel_pt_buffer *buffer, void *data);119	int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,120			 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,121			 uint64_t max_insn_cnt, void *data);122	bool (*pgd_ip)(uint64_t ip, void *data);123	int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data);124	struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs);125	void *data;126	struct intel_pt_state state;127	const unsigned char *buf;128	size_t len;129	bool return_compression;130	bool branch_enable;131	bool mtc_insn;132	bool pge;133	bool have_tma;134	bool have_cyc;135	bool fixup_last_mtc;136	bool have_last_ip;137	bool in_psb;138	bool hop;139	bool leap;140	bool emulated_ptwrite;141	bool vm_time_correlation;142	bool vm_tm_corr_dry_run;143	bool vm_tm_corr_reliable;144	bool vm_tm_corr_same_buf;145	bool vm_tm_corr_continuous;146	bool nr;147	bool next_nr;148	bool iflag;149	bool next_iflag;150	enum intel_pt_param_flags flags;151	uint64_t pos;152	uint64_t last_ip;153	uint64_t ip;154	uint64_t pip_payload;155	uint64_t timestamp;156	uint64_t tsc_timestamp;157	uint64_t ref_timestamp;158	uint64_t buf_timestamp;159	uint64_t sample_timestamp;160	uint64_t ret_addr;161	uint64_t ctc_timestamp;162	uint64_t ctc_delta;163	uint64_t cycle_cnt;164	uint64_t cyc_ref_timestamp;165	uint64_t first_timestamp;166	uint64_t last_reliable_timestamp;167	uint64_t vmcs;168	uint64_t print_once;169	uint64_t last_ctc;170	uint32_t last_mtc;171	uint32_t tsc_ctc_ratio_n;172	uint32_t tsc_ctc_ratio_d;173	uint32_t tsc_ctc_mult;174	uint32_t tsc_slip;175	uint32_t ctc_rem_mask;176	int mtc_shift;177	struct intel_pt_stack stack;178	enum intel_pt_pkt_state pkt_state;179	enum intel_pt_pkt_ctx pkt_ctx;180	enum intel_pt_pkt_ctx prev_pkt_ctx;181	enum intel_pt_blk_type blk_type;182	int blk_type_pos;183	struct intel_pt_pkt packet;184	struct intel_pt_pkt tnt;185	int pkt_step;186	int pkt_len;187	int last_packet_type;188	unsigned int cbr;189	unsigned int cbr_seen;190	unsigned int max_non_turbo_ratio;191	double max_non_turbo_ratio_fp;192	double cbr_cyc_to_tsc;193	double calc_cyc_to_tsc;194	bool have_calc_cyc_to_tsc;195	int exec_mode;196	unsigned int insn_bytes;197	uint64_t period;198	enum intel_pt_period_type period_type;199	uint64_t tot_insn_cnt;200	uint64_t period_insn_cnt;201	uint64_t period_mask;202	uint64_t period_ticks;203	uint64_t last_masked_timestamp;204	uint64_t tot_cyc_cnt;205	uint64_t sample_tot_cyc_cnt;206	uint64_t base_cyc_cnt;207	uint64_t cyc_cnt_timestamp;208	uint64_t ctl;209	uint64_t cyc_threshold;210	double tsc_to_cyc;211	bool continuous_period;212	bool overflow;213	bool set_fup_tx_flags;214	bool set_fup_ptw;215	bool set_fup_mwait;216	bool set_fup_pwre;217	bool set_fup_exstop;218	bool set_fup_bep;219	bool set_fup_cfe_ip;220	bool set_fup_cfe;221	bool set_fup_mode_exec;222	bool sample_cyc;223	unsigned int fup_tx_flags;224	unsigned int tx_flags;225	uint64_t fup_ptw_payload;226	uint64_t fup_mwait_payload;227	uint64_t fup_pwre_payload;228	uint64_t cbr_payload;229	uint64_t timestamp_insn_cnt;230	uint64_t sample_insn_cnt;231	uint64_t stuck_ip;232	struct intel_pt_pkt fup_cfe_pkt;233	int max_loops;234	int no_progress;235	int stuck_ip_prd;236	int stuck_ip_cnt;237	uint64_t psb_ip;238	const unsigned char *next_buf;239	size_t next_len;240	unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];241	int evd_cnt;242	struct intel_pt_evd evd[INTEL_PT_MAX_EVDS];243};244 245static uint64_t intel_pt_lower_power_of_2(uint64_t x)246{247	int i;248 249	for (i = 0; x != 1; i++)250		x >>= 1;251 252	return x << i;253}254 255__printf(1, 2)256static void p_log(const char *fmt, ...)257{258	char buf[512];259	va_list args;260 261	va_start(args, fmt);262	vsnprintf(buf, sizeof(buf), fmt, args);263	va_end(args);264 265	fprintf(stderr, "%s\n", buf);266	intel_pt_log("%s\n", buf);267}268 269static bool intel_pt_print_once(struct intel_pt_decoder *decoder,270				enum intel_pt_p_once id)271{272	uint64_t bit = 1ULL << id;273 274	if (decoder->print_once & bit)275		return false;276	decoder->print_once |= bit;277	return true;278}279 280static uint64_t intel_pt_cyc_threshold(uint64_t ctl)281{282	if (!(ctl & INTEL_PT_CYC_ENABLE))283		return 0;284 285	return (ctl & INTEL_PT_CYC_THRESHOLD) >> INTEL_PT_CYC_THRESHOLD_SHIFT;286}287 288static void intel_pt_setup_period(struct intel_pt_decoder *decoder)289{290	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {291		uint64_t period;292 293		period = intel_pt_lower_power_of_2(decoder->period);294		decoder->period_mask  = ~(period - 1);295		decoder->period_ticks = period;296	}297}298 299static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)300{301	if (!d)302		return 0;303	return (t / d) * n + ((t % d) * n) / d;304}305 306struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)307{308	struct intel_pt_decoder *decoder;309 310	if (!params->get_trace || !params->walk_insn)311		return NULL;312 313	decoder = zalloc(sizeof(struct intel_pt_decoder));314	if (!decoder)315		return NULL;316 317	decoder->get_trace          = params->get_trace;318	decoder->walk_insn          = params->walk_insn;319	decoder->pgd_ip             = params->pgd_ip;320	decoder->lookahead          = params->lookahead;321	decoder->findnew_vmcs_info  = params->findnew_vmcs_info;322	decoder->data               = params->data;323	decoder->return_compression = params->return_compression;324	decoder->branch_enable      = params->branch_enable;325	decoder->hop                = params->quick >= 1;326	decoder->leap               = params->quick >= 2;327	decoder->vm_time_correlation = params->vm_time_correlation;328	decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;329	decoder->first_timestamp    = params->first_timestamp;330	decoder->last_reliable_timestamp = params->first_timestamp;331	decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;332 333	decoder->flags              = params->flags;334 335	decoder->ctl                = params->ctl;336	decoder->period             = params->period;337	decoder->period_type        = params->period_type;338 339	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;340	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;341 342	decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);343 344	intel_pt_setup_period(decoder);345 346	decoder->mtc_shift = params->mtc_period;347	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;348 349	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;350	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;351 352	if (!decoder->tsc_ctc_ratio_n)353		decoder->tsc_ctc_ratio_d = 0;354 355	if (decoder->tsc_ctc_ratio_d) {356		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))357			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /358						decoder->tsc_ctc_ratio_d;359	}360 361	/*362	 * A TSC packet can slip past MTC packets so that the timestamp appears363	 * to go backwards. One estimate is that can be up to about 40 CPU364	 * cycles, which is certainly less than 0x1000 TSC ticks, but accept365	 * slippage an order of magnitude more to be on the safe side.366	 */367	decoder->tsc_slip = 0x10000;368 369	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);370	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);371	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);372	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);373	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);374 375	if (decoder->hop)376		intel_pt_log("Hop mode: decoding FUP and TIPs, but not TNT\n");377 378	return decoder;379}380 381void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,382				  uint64_t first_timestamp)383{384	decoder->first_timestamp = first_timestamp;385}386 387static void intel_pt_pop_blk(struct intel_pt_stack *stack)388{389	struct intel_pt_blk *blk = stack->blk;390 391	stack->blk = blk->prev;392	if (!stack->spare)393		stack->spare = blk;394	else395		free(blk);396}397 398static uint64_t intel_pt_pop(struct intel_pt_stack *stack)399{400	if (!stack->pos) {401		if (!stack->blk)402			return 0;403		intel_pt_pop_blk(stack);404		if (!stack->blk)405			return 0;406		stack->pos = INTEL_PT_BLK_SIZE;407	}408	return stack->blk->ip[--stack->pos];409}410 411static int intel_pt_alloc_blk(struct intel_pt_stack *stack)412{413	struct intel_pt_blk *blk;414 415	if (stack->spare) {416		blk = stack->spare;417		stack->spare = NULL;418	} else {419		blk = malloc(sizeof(struct intel_pt_blk));420		if (!blk)421			return -ENOMEM;422	}423 424	blk->prev = stack->blk;425	stack->blk = blk;426	stack->pos = 0;427	return 0;428}429 430static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)431{432	int err;433 434	if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {435		err = intel_pt_alloc_blk(stack);436		if (err)437			return err;438	}439 440	stack->blk->ip[stack->pos++] = ip;441	return 0;442}443 444static void intel_pt_clear_stack(struct intel_pt_stack *stack)445{446	while (stack->blk)447		intel_pt_pop_blk(stack);448	stack->pos = 0;449}450 451static void intel_pt_free_stack(struct intel_pt_stack *stack)452{453	intel_pt_clear_stack(stack);454	zfree(&stack->blk);455	zfree(&stack->spare);456}457 458void intel_pt_decoder_free(struct intel_pt_decoder *decoder)459{460	intel_pt_free_stack(&decoder->stack);461	free(decoder);462}463 464static int intel_pt_ext_err(int code)465{466	switch (code) {467	case -ENOMEM:468		return INTEL_PT_ERR_NOMEM;469	case -ENOSYS:470		return INTEL_PT_ERR_INTERN;471	case -EBADMSG:472		return INTEL_PT_ERR_BADPKT;473	case -ENODATA:474		return INTEL_PT_ERR_NODATA;475	case -EILSEQ:476		return INTEL_PT_ERR_NOINSN;477	case -ENOENT:478		return INTEL_PT_ERR_MISMAT;479	case -EOVERFLOW:480		return INTEL_PT_ERR_OVR;481	case -ENOSPC:482		return INTEL_PT_ERR_LOST;483	case -ELOOP:484		return INTEL_PT_ERR_NELOOP;485	case -ECONNRESET:486		return INTEL_PT_ERR_EPTW;487	default:488		return INTEL_PT_ERR_UNK;489	}490}491 492static const char *intel_pt_err_msgs[] = {493	[INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",494	[INTEL_PT_ERR_INTERN] = "Internal error",495	[INTEL_PT_ERR_BADPKT] = "Bad packet",496	[INTEL_PT_ERR_NODATA] = "No more data",497	[INTEL_PT_ERR_NOINSN] = "Failed to get instruction",498	[INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",499	[INTEL_PT_ERR_OVR]    = "Overflow packet",500	[INTEL_PT_ERR_LOST]   = "Lost trace data",501	[INTEL_PT_ERR_UNK]    = "Unknown error!",502	[INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)",503	[INTEL_PT_ERR_EPTW]   = "Broken emulated ptwrite",504};505 506int intel_pt__strerror(int code, char *buf, size_t buflen)507{508	if (code < 1 || code >= INTEL_PT_ERR_MAX)509		code = INTEL_PT_ERR_UNK;510	strlcpy(buf, intel_pt_err_msgs[code], buflen);511	return 0;512}513 514static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,515				 uint64_t last_ip)516{517	uint64_t ip;518 519	switch (packet->count) {520	case 1:521		ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |522		     packet->payload;523		break;524	case 2:525		ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |526		     packet->payload;527		break;528	case 3:529		ip = packet->payload;530		/* Sign-extend 6-byte ip */531		if (ip & (uint64_t)0x800000000000ULL)532			ip |= (uint64_t)0xffff000000000000ULL;533		break;534	case 4:535		ip = (last_ip & (uint64_t)0xffff000000000000ULL) |536		     packet->payload;537		break;538	case 6:539		ip = packet->payload;540		break;541	default:542		return 0;543	}544 545	return ip;546}547 548static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)549{550	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);551	decoder->have_last_ip = true;552}553 554static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)555{556	intel_pt_set_last_ip(decoder);557	decoder->ip = decoder->last_ip;558}559 560static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)561{562	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,563			    decoder->buf);564}565 566static int intel_pt_bug(struct intel_pt_decoder *decoder)567{568	intel_pt_log("ERROR: Internal error\n");569	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;570	return -ENOSYS;571}572 573static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)574{575	decoder->tx_flags = 0;576}577 578static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)579{580	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;581}582 583static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)584{585	decoder->pip_payload = decoder->packet.payload;586}587 588static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)589{590	decoder->next_nr = decoder->pip_payload & 1;591}592 593static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)594{595	decoder->nr = decoder->pip_payload & 1;596	decoder->next_nr = decoder->nr;597}598 599static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)600{601	intel_pt_update_pip(decoder);602	intel_pt_set_nr(decoder);603}604 605static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)606{607	intel_pt_clear_tx_flags(decoder);608	decoder->have_tma = false;609	decoder->pkt_len = 1;610	decoder->pkt_step = 1;611	intel_pt_decoder_log_packet(decoder);612	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {613		intel_pt_log("ERROR: Bad packet\n");614		decoder->pkt_state = INTEL_PT_STATE_ERR1;615	}616	return -EBADMSG;617}618 619static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)620{621	decoder->sample_timestamp = decoder->timestamp;622	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;623	decoder->state.cycles = decoder->tot_cyc_cnt;624}625 626static void intel_pt_reposition(struct intel_pt_decoder *decoder)627{628	decoder->ip = 0;629	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;630	decoder->timestamp = 0;631	decoder->have_tma = false;632}633 634static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)635{636	struct intel_pt_buffer buffer = { .buf = 0, };637	int ret;638 639	decoder->pkt_step = 0;640 641	intel_pt_log("Getting more data\n");642	ret = decoder->get_trace(&buffer, decoder->data);643	if (ret)644		return ret;645	decoder->buf = buffer.buf;646	decoder->len = buffer.len;647	if (!decoder->len) {648		intel_pt_log("No more data\n");649		return -ENODATA;650	}651	decoder->buf_timestamp = buffer.ref_timestamp;652	if (!buffer.consecutive || reposition) {653		intel_pt_reposition(decoder);654		decoder->ref_timestamp = buffer.ref_timestamp;655		decoder->state.trace_nr = buffer.trace_nr;656		decoder->vm_tm_corr_same_buf = false;657		intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",658			     decoder->ref_timestamp);659		return -ENOLINK;660	}661 662	return 0;663}664 665static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,666				  bool reposition)667{668	if (!decoder->next_buf)669		return intel_pt_get_data(decoder, reposition);670 671	decoder->buf = decoder->next_buf;672	decoder->len = decoder->next_len;673	decoder->next_buf = 0;674	decoder->next_len = 0;675	return 0;676}677 678static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)679{680	unsigned char *buf = decoder->temp_buf;681	size_t old_len, len, n;682	int ret;683 684	old_len = decoder->len;685	len = decoder->len;686	memcpy(buf, decoder->buf, len);687 688	ret = intel_pt_get_data(decoder, false);689	if (ret) {690		decoder->pos += old_len;691		return ret < 0 ? ret : -EINVAL;692	}693 694	n = INTEL_PT_PKT_MAX_SZ - len;695	if (n > decoder->len)696		n = decoder->len;697	memcpy(buf + len, decoder->buf, n);698	len += n;699 700	decoder->prev_pkt_ctx = decoder->pkt_ctx;701	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);702	if (ret < (int)old_len) {703		decoder->next_buf = decoder->buf;704		decoder->next_len = decoder->len;705		decoder->buf = buf;706		decoder->len = old_len;707		return intel_pt_bad_packet(decoder);708	}709 710	decoder->next_buf = decoder->buf + (ret - old_len);711	decoder->next_len = decoder->len - (ret - old_len);712 713	decoder->buf = buf;714	decoder->len = ret;715 716	return ret;717}718 719struct intel_pt_pkt_info {720	struct intel_pt_decoder	  *decoder;721	struct intel_pt_pkt       packet;722	uint64_t                  pos;723	int                       pkt_len;724	int                       last_packet_type;725	void                      *data;726};727 728typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);729 730/* Lookahead packets in current buffer */731static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,732				  intel_pt_pkt_cb_t cb, void *data)733{734	struct intel_pt_pkt_info pkt_info;735	const unsigned char *buf = decoder->buf;736	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;737	size_t len = decoder->len;738	int ret;739 740	pkt_info.decoder          = decoder;741	pkt_info.pos              = decoder->pos;742	pkt_info.pkt_len          = decoder->pkt_step;743	pkt_info.last_packet_type = decoder->last_packet_type;744	pkt_info.data             = data;745 746	while (1) {747		do {748			pkt_info.pos += pkt_info.pkt_len;749			buf          += pkt_info.pkt_len;750			len          -= pkt_info.pkt_len;751 752			if (!len)753				return INTEL_PT_NEED_MORE_BYTES;754 755			ret = intel_pt_get_packet(buf, len, &pkt_info.packet,756						  &pkt_ctx);757			if (!ret)758				return INTEL_PT_NEED_MORE_BYTES;759			if (ret < 0)760				return ret;761 762			pkt_info.pkt_len = ret;763		} while (pkt_info.packet.type == INTEL_PT_PAD);764 765		ret = cb(&pkt_info);766		if (ret)767			return 0;768 769		pkt_info.last_packet_type = pkt_info.packet.type;770	}771}772 773struct intel_pt_calc_cyc_to_tsc_info {774	uint64_t        cycle_cnt;775	unsigned int    cbr;776	uint32_t        last_mtc;777	uint64_t        ctc_timestamp;778	uint64_t        ctc_delta;779	uint64_t        tsc_timestamp;780	uint64_t        timestamp;781	bool            have_tma;782	bool            fixup_last_mtc;783	bool            from_mtc;784	double          cbr_cyc_to_tsc;785};786 787/*788 * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower789 * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC790 * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA791 * packet by copying the missing bits from the current MTC assuming the least792 * difference between the two, and that the current MTC comes after last_mtc.793 */794static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,795				    uint32_t *last_mtc)796{797	uint32_t first_missing_bit = 1U << (16 - mtc_shift);798	uint32_t mask = ~(first_missing_bit - 1);799 800	*last_mtc |= mtc & mask;801	if (*last_mtc >= mtc) {802		*last_mtc -= first_missing_bit;803		*last_mtc &= 0xff;804	}805}806 807static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)808{809	struct intel_pt_decoder *decoder = pkt_info->decoder;810	struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;811	uint64_t timestamp;812	double cyc_to_tsc;813	unsigned int cbr;814	uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;815 816	switch (pkt_info->packet.type) {817	case INTEL_PT_TNT:818	case INTEL_PT_TIP_PGE:819	case INTEL_PT_TIP:820	case INTEL_PT_FUP:821	case INTEL_PT_PSB:822	case INTEL_PT_PIP:823	case INTEL_PT_MODE_EXEC:824	case INTEL_PT_MODE_TSX:825	case INTEL_PT_PSBEND:826	case INTEL_PT_PAD:827	case INTEL_PT_VMCS:828	case INTEL_PT_MNT:829	case INTEL_PT_PTWRITE:830	case INTEL_PT_PTWRITE_IP:831	case INTEL_PT_BBP:832	case INTEL_PT_BIP:833	case INTEL_PT_BEP:834	case INTEL_PT_BEP_IP:835	case INTEL_PT_CFE:836	case INTEL_PT_CFE_IP:837	case INTEL_PT_EVD:838		return 0;839 840	case INTEL_PT_MTC:841		if (!data->have_tma)842			return 0;843 844		mtc = pkt_info->packet.payload;845		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {846			data->fixup_last_mtc = false;847			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,848						&data->last_mtc);849		}850		if (mtc > data->last_mtc)851			mtc_delta = mtc - data->last_mtc;852		else853			mtc_delta = mtc + 256 - data->last_mtc;854		data->ctc_delta += mtc_delta << decoder->mtc_shift;855		data->last_mtc = mtc;856 857		if (decoder->tsc_ctc_mult) {858			timestamp = data->ctc_timestamp +859				data->ctc_delta * decoder->tsc_ctc_mult;860		} else {861			timestamp = data->ctc_timestamp +862				multdiv(data->ctc_delta,863					decoder->tsc_ctc_ratio_n,864					decoder->tsc_ctc_ratio_d);865		}866 867		if (timestamp < data->timestamp)868			return 1;869 870		if (pkt_info->last_packet_type != INTEL_PT_CYC) {871			data->timestamp = timestamp;872			return 0;873		}874 875		break;876 877	case INTEL_PT_TSC:878		/*879		 * For now, do not support using TSC packets - refer880		 * intel_pt_calc_cyc_to_tsc().881		 */882		if (data->from_mtc)883			return 1;884		timestamp = pkt_info->packet.payload |885			    (data->timestamp & (0xffULL << 56));886		if (data->from_mtc && timestamp < data->timestamp &&887		    data->timestamp - timestamp < decoder->tsc_slip)888			return 1;889		if (timestamp < data->timestamp)890			timestamp += (1ULL << 56);891		if (pkt_info->last_packet_type != INTEL_PT_CYC) {892			if (data->from_mtc)893				return 1;894			data->tsc_timestamp = timestamp;895			data->timestamp = timestamp;896			return 0;897		}898		break;899 900	case INTEL_PT_TMA:901		if (data->from_mtc)902			return 1;903 904		if (!decoder->tsc_ctc_ratio_d)905			return 0;906 907		ctc = pkt_info->packet.payload;908		fc = pkt_info->packet.count;909		ctc_rem = ctc & decoder->ctc_rem_mask;910 911		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;912 913		data->ctc_timestamp = data->tsc_timestamp - fc;914		if (decoder->tsc_ctc_mult) {915			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;916		} else {917			data->ctc_timestamp -=918				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,919					decoder->tsc_ctc_ratio_d);920		}921 922		data->ctc_delta = 0;923		data->have_tma = true;924		data->fixup_last_mtc = true;925 926		return 0;927 928	case INTEL_PT_CYC:929		data->cycle_cnt += pkt_info->packet.payload;930		return 0;931 932	case INTEL_PT_CBR:933		cbr = pkt_info->packet.payload;934		if (data->cbr && data->cbr != cbr)935			return 1;936		data->cbr = cbr;937		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;938		return 0;939 940	case INTEL_PT_TIP_PGD:941	case INTEL_PT_TRACESTOP:942	case INTEL_PT_EXSTOP:943	case INTEL_PT_EXSTOP_IP:944	case INTEL_PT_MWAIT:945	case INTEL_PT_PWRE:946	case INTEL_PT_PWRX:947	case INTEL_PT_OVF:948	case INTEL_PT_BAD: /* Does not happen */949	default:950		return 1;951	}952 953	if (!data->cbr && decoder->cbr) {954		data->cbr = decoder->cbr;955		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;956	}957 958	if (!data->cycle_cnt)959		return 1;960 961	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;962 963	if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&964	    cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {965		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",966			     cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);967		return 1;968	}969 970	decoder->calc_cyc_to_tsc = cyc_to_tsc;971	decoder->have_calc_cyc_to_tsc = true;972 973	if (data->cbr) {974		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",975			     cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);976	} else {977		intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",978			     cyc_to_tsc, pkt_info->pos);979	}980 981	return 1;982}983 984static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,985				     bool from_mtc)986{987	struct intel_pt_calc_cyc_to_tsc_info data = {988		.cycle_cnt      = 0,989		.cbr            = 0,990		.last_mtc       = decoder->last_mtc,991		.ctc_timestamp  = decoder->ctc_timestamp,992		.ctc_delta      = decoder->ctc_delta,993		.tsc_timestamp  = decoder->tsc_timestamp,994		.timestamp      = decoder->timestamp,995		.have_tma       = decoder->have_tma,996		.fixup_last_mtc = decoder->fixup_last_mtc,997		.from_mtc       = from_mtc,998		.cbr_cyc_to_tsc = 0,999	};1000 1001	/*1002	 * For now, do not support using TSC packets for at least the reasons:1003	 * 1) timing might have stopped1004	 * 2) TSC packets within PSB+ can slip against CYC packets1005	 */1006	if (!from_mtc)1007		return;1008 1009	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);1010}1011 1012static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)1013{1014	int ret;1015 1016	decoder->last_packet_type = decoder->packet.type;1017 1018	do {1019		decoder->pos += decoder->pkt_step;1020		decoder->buf += decoder->pkt_step;1021		decoder->len -= decoder->pkt_step;1022 1023		if (!decoder->len) {1024			ret = intel_pt_get_next_data(decoder, false);1025			if (ret)1026				return ret;1027		}1028 1029		decoder->prev_pkt_ctx = decoder->pkt_ctx;1030		ret = intel_pt_get_packet(decoder->buf, decoder->len,1031					  &decoder->packet, &decoder->pkt_ctx);1032		if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&1033		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {1034			ret = intel_pt_get_split_packet(decoder);1035			if (ret < 0)1036				return ret;1037		}1038		if (ret <= 0)1039			return intel_pt_bad_packet(decoder);1040 1041		decoder->pkt_len = ret;1042		decoder->pkt_step = ret;1043		intel_pt_decoder_log_packet(decoder);1044	} while (decoder->packet.type == INTEL_PT_PAD);1045 1046	return 0;1047}1048 1049static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)1050{1051	uint64_t timestamp, masked_timestamp;1052 1053	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;1054	masked_timestamp = timestamp & decoder->period_mask;1055	if (decoder->continuous_period) {1056		if (masked_timestamp > decoder->last_masked_timestamp)1057			return 1;1058	} else {1059		timestamp += 1;1060		masked_timestamp = timestamp & decoder->period_mask;1061		if (masked_timestamp > decoder->last_masked_timestamp) {1062			decoder->last_masked_timestamp = masked_timestamp;1063			decoder->continuous_period = true;1064		}1065	}1066 1067	if (masked_timestamp < decoder->last_masked_timestamp)1068		return decoder->period_ticks;1069 1070	return decoder->period_ticks - (timestamp - masked_timestamp);1071}1072 1073static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)1074{1075	switch (decoder->period_type) {1076	case INTEL_PT_PERIOD_INSTRUCTIONS:1077		return decoder->period - decoder->period_insn_cnt;1078	case INTEL_PT_PERIOD_TICKS:1079		return intel_pt_next_period(decoder);1080	case INTEL_PT_PERIOD_NONE:1081	case INTEL_PT_PERIOD_MTC:1082	default:1083		return 0;1084	}1085}1086 1087static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)1088{1089	uint64_t timestamp, masked_timestamp;1090 1091	switch (decoder->period_type) {1092	case INTEL_PT_PERIOD_INSTRUCTIONS:1093		decoder->period_insn_cnt = 0;1094		break;1095	case INTEL_PT_PERIOD_TICKS:1096		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;1097		masked_timestamp = timestamp & decoder->period_mask;1098		if (masked_timestamp > decoder->last_masked_timestamp)1099			decoder->last_masked_timestamp = masked_timestamp;1100		else1101			decoder->last_masked_timestamp += decoder->period_ticks;1102		break;1103	case INTEL_PT_PERIOD_NONE:1104	case INTEL_PT_PERIOD_MTC:1105	default:1106		break;1107	}1108 1109	decoder->state.type |= INTEL_PT_INSTRUCTION;1110}1111 1112/*1113 * Sample FUP instruction at the same time as reporting the FUP event, so the1114 * instruction sample gets the same flags as the FUP event.1115 */1116static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)1117{1118	struct intel_pt_insn intel_pt_insn;1119	uint64_t max_insn_cnt, insn_cnt = 0;1120	int err;1121 1122	decoder->state.insn_op = INTEL_PT_OP_OTHER;1123	decoder->state.insn_len = 0;1124 1125	if (!decoder->branch_enable || !decoder->pge || decoder->hop ||1126	    decoder->ip != decoder->last_ip)1127		return;1128 1129	if (!decoder->mtc_insn)1130		decoder->mtc_insn = true;1131 1132	max_insn_cnt = intel_pt_next_sample(decoder);1133	if (max_insn_cnt != 1)1134		return;1135 1136	err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,1137				 0, max_insn_cnt, decoder->data);1138	/* Ignore error, it will be reported next walk anyway */1139	if (err)1140		return;1141 1142	if (intel_pt_insn.branch != INTEL_PT_BR_NO_BRANCH) {1143		intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);1144		return;1145	}1146 1147	decoder->tot_insn_cnt += insn_cnt;1148	decoder->timestamp_insn_cnt += insn_cnt;1149	decoder->sample_insn_cnt += insn_cnt;1150	decoder->period_insn_cnt += insn_cnt;1151 1152	intel_pt_sample_insn(decoder);1153 1154	decoder->state.type |= INTEL_PT_INSTRUCTION;1155	decoder->ip += intel_pt_insn.length;1156}1157 1158static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,1159			      struct intel_pt_insn *intel_pt_insn, uint64_t ip)1160{1161	uint64_t max_insn_cnt, insn_cnt = 0;1162	int err;1163 1164	if (!decoder->mtc_insn)1165		decoder->mtc_insn = true;1166 1167	max_insn_cnt = intel_pt_next_sample(decoder);1168 1169	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,1170				 max_insn_cnt, decoder->data);1171 1172	decoder->tot_insn_cnt += insn_cnt;1173	decoder->timestamp_insn_cnt += insn_cnt;1174	decoder->sample_insn_cnt += insn_cnt;1175	decoder->period_insn_cnt += insn_cnt;1176 1177	if (err) {1178		decoder->no_progress = 0;1179		decoder->pkt_state = INTEL_PT_STATE_ERR2;1180		intel_pt_log_at("ERROR: Failed to get instruction",1181				decoder->ip);1182		if (err == -ENOENT)1183			return -ENOLINK;1184		return -EILSEQ;1185	}1186 1187	if (ip && decoder->ip == ip) {1188		err = -EAGAIN;1189		goto out;1190	}1191 1192	if (max_insn_cnt && insn_cnt >= max_insn_cnt)1193		intel_pt_sample_insn(decoder);1194 1195	if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {1196		decoder->state.type = INTEL_PT_INSTRUCTION;1197		decoder->state.from_ip = decoder->ip;1198		decoder->state.to_ip = 0;1199		decoder->ip += intel_pt_insn->length;1200		err = INTEL_PT_RETURN;1201		goto out;1202	}1203 1204	if (intel_pt_insn->op == INTEL_PT_OP_CALL) {1205		/* Zero-length calls are excluded */1206		if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||1207		    intel_pt_insn->rel) {1208			err = intel_pt_push(&decoder->stack, decoder->ip +1209					    intel_pt_insn->length);1210			if (err)1211				goto out;1212		}1213	} else if (intel_pt_insn->op == INTEL_PT_OP_RET) {1214		decoder->ret_addr = intel_pt_pop(&decoder->stack);1215	}1216 1217	if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {1218		int cnt = decoder->no_progress++;1219 1220		decoder->state.from_ip = decoder->ip;1221		decoder->ip += intel_pt_insn->length +1222				intel_pt_insn->rel;1223		decoder->state.to_ip = decoder->ip;1224		err = INTEL_PT_RETURN;1225 1226		/*1227		 * Check for being stuck in a loop.  This can happen if a1228		 * decoder error results in the decoder erroneously setting the1229		 * ip to an address that is itself in an infinite loop that1230		 * consumes no packets.  When that happens, there must be an1231		 * unconditional branch.1232		 */1233		if (cnt) {1234			if (cnt == 1) {1235				decoder->stuck_ip = decoder->state.to_ip;1236				decoder->stuck_ip_prd = 1;1237				decoder->stuck_ip_cnt = 1;1238			} else if (cnt > decoder->max_loops ||1239				   decoder->state.to_ip == decoder->stuck_ip) {1240				intel_pt_log_at("ERROR: Never-ending loop",1241						decoder->state.to_ip);1242				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;1243				err = -ELOOP;1244				goto out;1245			} else if (!--decoder->stuck_ip_cnt) {1246				decoder->stuck_ip_prd += 1;1247				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;1248				decoder->stuck_ip = decoder->state.to_ip;1249			}1250		}1251		goto out_no_progress;1252	}1253out:1254	decoder->no_progress = 0;1255out_no_progress:1256	decoder->state.insn_op = intel_pt_insn->op;1257	decoder->state.insn_len = intel_pt_insn->length;1258	memcpy(decoder->state.insn, intel_pt_insn->buf,1259	       INTEL_PT_INSN_BUF_SZ);1260 1261	if (decoder->tx_flags & INTEL_PT_IN_TX)1262		decoder->state.flags |= INTEL_PT_IN_TX;1263 1264	return err;1265}1266 1267static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)1268{1269	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;1270 1271	decoder->exec_mode = decoder->packet.payload;1272	decoder->iflag = iflag;1273	decoder->next_iflag = iflag;1274	decoder->state.from_iflag = iflag;1275	decoder->state.to_iflag = iflag;1276}1277 1278static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)1279{1280	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;1281 1282	decoder->exec_mode = decoder->packet.payload;1283	decoder->next_iflag = iflag;1284}1285 1286static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)1287{1288	decoder->state.type |= INTEL_PT_IFLAG_CHG;1289	decoder->state.from_iflag = decoder->iflag;1290	decoder->state.to_iflag = decoder->next_iflag;1291	decoder->iflag = decoder->next_iflag;1292}1293 1294static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)1295{1296	if (decoder->iflag != decoder->next_iflag)1297		intel_pt_sample_iflag(decoder);1298}1299 1300static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)1301{1302	decoder->set_fup_tx_flags = false;1303	decoder->set_fup_ptw = false;1304	decoder->set_fup_mwait = false;1305	decoder->set_fup_pwre = false;1306	decoder->set_fup_exstop = false;1307	decoder->set_fup_bep = false;1308	decoder->set_fup_cfe_ip = false;1309	decoder->set_fup_cfe = false;1310	decoder->evd_cnt = 0;1311	decoder->set_fup_mode_exec = false;1312	decoder->iflag = decoder->next_iflag;1313}1314 1315static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)1316{1317	enum intel_pt_sample_type type = decoder->state.type;1318	bool sample_fup_insn = false;1319	bool ret = false;1320 1321	decoder->state.type &= ~INTEL_PT_BRANCH;1322	decoder->state.insn_op = INTEL_PT_OP_OTHER;1323	decoder->state.insn_len = 0;1324 1325	if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) {1326		bool ip = decoder->set_fup_cfe_ip;1327 1328		decoder->set_fup_cfe_ip = false;1329		decoder->set_fup_cfe = false;1330		decoder->state.type |= INTEL_PT_EVT;1331		if (!ip && decoder->pge)1332			decoder->state.type |= INTEL_PT_BRANCH;1333		decoder->state.cfe_type = decoder->fup_cfe_pkt.count;1334		decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;1335		decoder->state.evd_cnt = decoder->evd_cnt;1336		decoder->state.evd = decoder->evd;1337		decoder->evd_cnt = 0;1338		if (ip || decoder->pge)1339			decoder->state.flags |= INTEL_PT_FUP_IP;1340		ret = true;1341	}1342	if (decoder->set_fup_mode_exec) {1343		decoder->set_fup_mode_exec = false;1344		intel_pt_sample_iflag(decoder);1345		sample_fup_insn = no_tip;1346		ret = true;1347	}1348	if (decoder->set_fup_tx_flags) {1349		decoder->set_fup_tx_flags = false;1350		decoder->tx_flags = decoder->fup_tx_flags;1351		decoder->state.type |= INTEL_PT_TRANSACTION;1352		if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)1353			decoder->state.type |= INTEL_PT_BRANCH;1354		decoder->state.flags = decoder->fup_tx_flags;1355		ret = true;1356	}1357	if (decoder->set_fup_ptw) {1358		decoder->set_fup_ptw = false;1359		decoder->state.type |= INTEL_PT_PTW;1360		decoder->state.flags |= INTEL_PT_FUP_IP;1361		decoder->state.ptw_payload = decoder->fup_ptw_payload;1362		ret = true;1363	}1364	if (decoder->set_fup_mwait) {1365		decoder->set_fup_mwait = false;1366		decoder->state.type |= INTEL_PT_MWAIT_OP;1367		decoder->state.mwait_payload = decoder->fup_mwait_payload;1368		ret = true;1369	}1370	if (decoder->set_fup_pwre) {1371		decoder->set_fup_pwre = false;1372		decoder->state.type |= INTEL_PT_PWR_ENTRY;1373		decoder->state.pwre_payload = decoder->fup_pwre_payload;1374		ret = true;1375	}1376	if (decoder->set_fup_exstop) {1377		decoder->set_fup_exstop = false;1378		decoder->state.type |= INTEL_PT_EX_STOP;1379		decoder->state.flags |= INTEL_PT_FUP_IP;1380		ret = true;1381	}1382	if (decoder->set_fup_bep) {1383		decoder->set_fup_bep = false;1384		decoder->state.type |= INTEL_PT_BLK_ITEMS;1385		ret = true;1386	}1387	if (decoder->overflow) {1388		decoder->overflow = false;1389		if (!ret && !decoder->pge) {1390			if (decoder->hop) {1391				decoder->state.type = 0;1392				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;1393			}1394			decoder->pge = true;1395			decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;1396			decoder->state.from_ip = 0;1397			decoder->state.to_ip = decoder->ip;1398			return true;1399		}1400	}1401	if (ret) {1402		decoder->state.from_ip = decoder->ip;1403		decoder->state.to_ip = 0;1404		if (sample_fup_insn)1405			intel_pt_sample_fup_insn(decoder);1406	} else {1407		decoder->state.type = type;1408	}1409	return ret;1410}1411 1412static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,1413					  struct intel_pt_insn *intel_pt_insn,1414					  uint64_t ip, int err)1415{1416	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&1417	       intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&1418	       ip == decoder->ip + intel_pt_insn->length;1419}1420 1421static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)1422{1423	struct intel_pt_insn intel_pt_insn;1424	uint64_t ip;1425	int err;1426 1427	ip = decoder->last_ip;1428 1429	while (1) {1430		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);1431		if (err == INTEL_PT_RETURN)1432			return 0;1433		if (err == -EAGAIN ||1434		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {1435			bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;1436 1437			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1438			if (intel_pt_fup_event(decoder, no_tip) && no_tip)1439				return 0;1440			return -EAGAIN;1441		}1442		decoder->set_fup_tx_flags = false;1443		if (err)1444			return err;1445 1446		if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {1447			intel_pt_log_at("ERROR: Unexpected indirect branch",1448					decoder->ip);1449			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;1450			return -ENOENT;1451		}1452 1453		if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {1454			intel_pt_log_at("ERROR: Unexpected conditional branch",1455					decoder->ip);1456			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;1457			return -ENOENT;1458		}1459 1460		intel_pt_bug(decoder);1461	}1462}1463 1464static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)1465{1466	struct intel_pt_insn intel_pt_insn;1467	int err;1468 1469	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);1470	if (err == INTEL_PT_RETURN &&1471	    decoder->pgd_ip &&1472	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&1473	    (decoder->state.type & INTEL_PT_BRANCH) &&1474	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {1475		/* Unconditional branch leaving filter region */1476		decoder->no_progress = 0;1477		decoder->pge = false;1478		decoder->continuous_period = false;1479		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1480		decoder->state.type |= INTEL_PT_TRACE_END;1481		intel_pt_update_nr(decoder);1482		return 0;1483	}1484	if (err == INTEL_PT_RETURN)1485		return 0;1486	if (err)1487		return err;1488 1489	intel_pt_update_nr(decoder);1490	intel_pt_sample_iflag_chg(decoder);1491 1492	if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {1493		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {1494			decoder->pge = false;1495			decoder->continuous_period = false;1496			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1497			decoder->state.from_ip = decoder->ip;1498			if (decoder->packet.count == 0) {1499				decoder->state.to_ip = 0;1500			} else {1501				decoder->state.to_ip = decoder->last_ip;1502				decoder->ip = decoder->last_ip;1503			}1504			decoder->state.type |= INTEL_PT_TRACE_END;1505		} else {1506			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1507			decoder->state.from_ip = decoder->ip;1508			if (decoder->packet.count == 0) {1509				decoder->state.to_ip = 0;1510			} else {1511				decoder->state.to_ip = decoder->last_ip;1512				decoder->ip = decoder->last_ip;1513			}1514		}1515		return 0;1516	}1517 1518	if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {1519		uint64_t to_ip = decoder->ip + intel_pt_insn.length +1520				 intel_pt_insn.rel;1521 1522		if (decoder->pgd_ip &&1523		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&1524		    decoder->pgd_ip(to_ip, decoder->data)) {1525			/* Conditional branch leaving filter region */1526			decoder->pge = false;1527			decoder->continuous_period = false;1528			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1529			decoder->ip = to_ip;1530			decoder->state.from_ip = decoder->ip;1531			decoder->state.to_ip = to_ip;1532			decoder->state.type |= INTEL_PT_TRACE_END;1533			return 0;1534		}1535		intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",1536				decoder->ip);1537		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;1538		return -ENOENT;1539	}1540 1541	return intel_pt_bug(decoder);1542}1543 1544struct eptw_data {1545	int bit_countdown;1546	uint64_t payload;1547};1548 1549static int intel_pt_eptw_lookahead_cb(struct intel_pt_pkt_info *pkt_info)1550{1551	struct eptw_data *data = pkt_info->data;1552	int nr_bits;1553 1554	switch (pkt_info->packet.type) {1555	case INTEL_PT_PAD:1556	case INTEL_PT_MNT:1557	case INTEL_PT_MODE_EXEC:1558	case INTEL_PT_MODE_TSX:1559	case INTEL_PT_MTC:1560	case INTEL_PT_FUP:1561	case INTEL_PT_CYC:1562	case INTEL_PT_CBR:1563	case INTEL_PT_TSC:1564	case INTEL_PT_TMA:1565	case INTEL_PT_PIP:1566	case INTEL_PT_VMCS:1567	case INTEL_PT_PSB:1568	case INTEL_PT_PSBEND:1569	case INTEL_PT_PTWRITE:1570	case INTEL_PT_PTWRITE_IP:1571	case INTEL_PT_EXSTOP:1572	case INTEL_PT_EXSTOP_IP:1573	case INTEL_PT_MWAIT:1574	case INTEL_PT_PWRE:1575	case INTEL_PT_PWRX:1576	case INTEL_PT_BBP:1577	case INTEL_PT_BIP:1578	case INTEL_PT_BEP:1579	case INTEL_PT_BEP_IP:1580	case INTEL_PT_CFE:1581	case INTEL_PT_CFE_IP:1582	case INTEL_PT_EVD:1583		break;1584 1585	case INTEL_PT_TNT:1586		nr_bits = data->bit_countdown;1587		if (nr_bits > pkt_info->packet.count)1588			nr_bits = pkt_info->packet.count;1589		data->payload <<= nr_bits;1590		data->payload |= pkt_info->packet.payload >> (64 - nr_bits);1591		data->bit_countdown -= nr_bits;1592		return !data->bit_countdown;1593 1594	case INTEL_PT_TIP_PGE:1595	case INTEL_PT_TIP_PGD:1596	case INTEL_PT_TIP:1597	case INTEL_PT_BAD:1598	case INTEL_PT_OVF:1599	case INTEL_PT_TRACESTOP:1600	default:1601		return 1;1602	}1603 1604	return 0;1605}1606 1607static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)1608{1609	int n = 64 - decoder->tnt.count;1610	struct eptw_data data = {1611		.bit_countdown = n,1612		.payload = decoder->tnt.payload >> n,1613	};1614 1615	decoder->emulated_ptwrite = false;1616	intel_pt_log("Emulated ptwrite detected\n");1617 1618	intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);1619	if (data.bit_countdown)1620		return -ECONNRESET;1621 1622	decoder->state.type = INTEL_PT_PTW;1623	decoder->state.from_ip = decoder->ip;1624	decoder->state.to_ip = 0;1625	decoder->state.ptw_payload = data.payload;1626	return 0;1627}1628 1629static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)1630{1631	struct intel_pt_insn intel_pt_insn;1632	int err;1633 1634	while (1) {1635		if (decoder->emulated_ptwrite)1636			return intel_pt_emulated_ptwrite(decoder);1637		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);1638		if (err == INTEL_PT_RETURN) {1639			decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;1640			return 0;1641		}1642		if (err) {1643			decoder->emulated_ptwrite = false;1644			return err;1645		}1646 1647		if (intel_pt_insn.op == INTEL_PT_OP_RET) {1648			if (!decoder->return_compression) {1649				intel_pt_log_at("ERROR: RET when expecting conditional branch",1650						decoder->ip);1651				decoder->pkt_state = INTEL_PT_STATE_ERR3;1652				return -ENOENT;1653			}1654			if (!decoder->ret_addr) {1655				intel_pt_log_at("ERROR: Bad RET compression (stack empty)",1656						decoder->ip);1657				decoder->pkt_state = INTEL_PT_STATE_ERR3;1658				return -ENOENT;1659			}1660			if (!(decoder->tnt.payload & BIT63)) {1661				intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",1662						decoder->ip);1663				decoder->pkt_state = INTEL_PT_STATE_ERR3;1664				return -ENOENT;1665			}1666			decoder->tnt.count -= 1;1667			if (decoder->tnt.count)1668				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;1669			else1670				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1671			decoder->tnt.payload <<= 1;1672			decoder->state.from_ip = decoder->ip;1673			decoder->ip = decoder->ret_addr;1674			decoder->state.to_ip = decoder->ip;1675			return 0;1676		}1677 1678		if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {1679			/* Handle deferred TIPs */1680			err = intel_pt_get_next_packet(decoder);1681			if (err)1682				return err;1683			if (decoder->packet.type != INTEL_PT_TIP ||1684			    decoder->packet.count == 0) {1685				intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",1686						decoder->ip);1687				decoder->pkt_state = INTEL_PT_STATE_ERR3;1688				decoder->pkt_step = 0;1689				return -ENOENT;1690			}1691			intel_pt_set_last_ip(decoder);1692			decoder->state.from_ip = decoder->ip;1693			decoder->state.to_ip = decoder->last_ip;1694			decoder->ip = decoder->last_ip;1695			intel_pt_update_nr(decoder);1696			intel_pt_sample_iflag_chg(decoder);1697			return 0;1698		}1699 1700		if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {1701			decoder->tnt.count -= 1;1702			if (decoder->tnt.count)1703				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;1704			else1705				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1706			if (decoder->tnt.payload & BIT63) {1707				decoder->tnt.payload <<= 1;1708				decoder->state.from_ip = decoder->ip;1709				decoder->ip += intel_pt_insn.length +1710					       intel_pt_insn.rel;1711				decoder->state.to_ip = decoder->ip;1712				return 0;1713			}1714			/* Instruction sample for a non-taken branch */1715			if (decoder->state.type & INTEL_PT_INSTRUCTION) {1716				decoder->tnt.payload <<= 1;1717				decoder->state.type = INTEL_PT_INSTRUCTION;1718				decoder->state.from_ip = decoder->ip;1719				decoder->state.to_ip = 0;1720				decoder->ip += intel_pt_insn.length;1721				return 0;1722			}1723			decoder->sample_cyc = false;1724			decoder->ip += intel_pt_insn.length;1725			if (!decoder->tnt.count) {1726				intel_pt_update_sample_time(decoder);1727				return -EAGAIN;1728			}1729			decoder->tnt.payload <<= 1;1730			continue;1731		}1732 1733		return intel_pt_bug(decoder);1734	}1735}1736 1737static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)1738{1739	unsigned int fup_tx_flags;1740	int err;1741 1742	fup_tx_flags = decoder->packet.payload &1743		       (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);1744	err = intel_pt_get_next_packet(decoder);1745	if (err)1746		return err;1747	if (decoder->packet.type == INTEL_PT_FUP) {1748		decoder->fup_tx_flags = fup_tx_flags;1749		decoder->set_fup_tx_flags = true;1750		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))1751			*no_tip = true;1752	} else {1753		intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",1754				decoder->pos);1755		intel_pt_update_in_tx(decoder);1756	}1757	return 0;1758}1759 1760static int intel_pt_evd(struct intel_pt_decoder *decoder)1761{1762	if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {1763		intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);1764		return -ENOSYS;1765	}1766	decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){1767		.type = decoder->packet.count,1768		.payload = decoder->packet.payload,1769	};1770	return 0;1771}1772 1773static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp)1774{1775	timestamp |= (ref_timestamp & (0xffULL << 56));1776 1777	if (timestamp < ref_timestamp) {1778		if (ref_timestamp - timestamp > (1ULL << 55))1779			timestamp += (1ULL << 56);1780	} else {1781		if (timestamp - ref_timestamp > (1ULL << 55))1782			timestamp -= (1ULL << 56);1783	}1784 1785	return timestamp;1786}1787 1788/* For use only when decoder->vm_time_correlation is true */1789static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,1790				   uint64_t timestamp)1791{1792	uint64_t max_timestamp = decoder->buf_timestamp;1793 1794	if (!max_timestamp) {1795		max_timestamp = decoder->last_reliable_timestamp +1796				0x400000000ULL;1797	}1798	return timestamp >= decoder->last_reliable_timestamp &&1799	       timestamp < decoder->buf_timestamp;1800}1801 1802static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)1803{1804	uint64_t timestamp;1805	bool bad = false;1806 1807	decoder->have_tma = false;1808 1809	if (decoder->ref_timestamp) {1810		timestamp = intel_pt_8b_tsc(decoder->packet.payload,1811					    decoder->ref_timestamp);1812		decoder->tsc_timestamp = timestamp;1813		decoder->timestamp = timestamp;1814		decoder->ref_timestamp = 0;1815		decoder->timestamp_insn_cnt = 0;1816	} else if (decoder->timestamp) {1817		timestamp = decoder->packet.payload |1818			    (decoder->timestamp & (0xffULL << 56));1819		decoder->tsc_timestamp = timestamp;1820		if (timestamp < decoder->timestamp &&1821		    decoder->timestamp - timestamp < decoder->tsc_slip) {1822			intel_pt_log_to("Suppressing backwards timestamp",1823					timestamp);1824			timestamp = decoder->timestamp;1825		}1826		if (timestamp < decoder->timestamp) {1827			if (!decoder->buf_timestamp ||1828			    (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {1829				intel_pt_log_to("Wraparound timestamp", timestamp);1830				timestamp += (1ULL << 56);1831				decoder->tsc_timestamp = timestamp;1832			} else {1833				intel_pt_log_to("Suppressing bad timestamp", timestamp);1834				timestamp = decoder->timestamp;1835				bad = true;1836			}1837		}1838		if (decoder->vm_time_correlation &&1839		    (bad || !intel_pt_time_in_range(decoder, timestamp)) &&1840		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))1841			p_log("Timestamp out of range");1842		decoder->timestamp = timestamp;1843		decoder->timestamp_insn_cnt = 0;1844	}1845 1846	if (decoder->last_packet_type == INTEL_PT_CYC) {1847		decoder->cyc_ref_timestamp = decoder->timestamp;1848		decoder->cycle_cnt = 0;1849		decoder->have_calc_cyc_to_tsc = false;1850		intel_pt_calc_cyc_to_tsc(decoder, false);1851	}1852 1853	intel_pt_log_to("Setting timestamp", decoder->timestamp);1854}1855 1856static int intel_pt_overflow(struct intel_pt_decoder *decoder)1857{1858	intel_pt_log("ERROR: Buffer overflow\n");1859	intel_pt_clear_tx_flags(decoder);1860	intel_pt_set_nr(decoder);1861	decoder->timestamp_insn_cnt = 0;1862	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;1863	decoder->state.from_ip = decoder->ip;1864	decoder->ip = 0;1865	decoder->pge = false;1866	intel_pt_clear_fup_event(decoder);1867	decoder->overflow = true;1868	return -EOVERFLOW;1869}1870 1871static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)1872{1873	if (decoder->have_cyc)1874		return;1875 1876	decoder->cyc_cnt_timestamp = decoder->timestamp;1877	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;1878}1879 1880static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)1881{1882	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;1883 1884	if (decoder->pge)1885		intel_pt_mtc_cyc_cnt_pge(decoder);1886}1887 1888static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)1889{1890	uint64_t tot_cyc_cnt, tsc_delta;1891 1892	if (decoder->have_cyc)1893		return;1894 1895	decoder->sample_cyc = true;1896 1897	if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)1898		return;1899 1900	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;1901	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;1902 1903	if (tot_cyc_cnt > decoder->tot_cyc_cnt)1904		decoder->tot_cyc_cnt = tot_cyc_cnt;1905}1906 1907static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)1908{1909	uint32_t ctc = decoder->packet.payload;1910	uint32_t fc = decoder->packet.count;1911	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;1912 1913	if (!decoder->tsc_ctc_ratio_d)1914		return;1915 1916	if (decoder->pge && !decoder->in_psb)1917		intel_pt_mtc_cyc_cnt_pge(decoder);1918	else1919		intel_pt_mtc_cyc_cnt_upd(decoder);1920 1921	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;1922	decoder->last_ctc = ctc - ctc_rem;1923	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;1924	if (decoder->tsc_ctc_mult) {1925		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;1926	} else {1927		decoder->ctc_timestamp -= multdiv(ctc_rem,1928						  decoder->tsc_ctc_ratio_n,1929						  decoder->tsc_ctc_ratio_d);1930	}1931	decoder->ctc_delta = 0;1932	decoder->have_tma = true;1933	decoder->fixup_last_mtc = true;1934	intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",1935		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);1936}1937 1938static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)1939{1940	uint64_t timestamp;1941	uint32_t mtc, mtc_delta;1942 1943	if (!decoder->have_tma)1944		return;1945 1946	mtc = decoder->packet.payload;1947 1948	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {1949		decoder->fixup_last_mtc = false;1950		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,1951					&decoder->last_mtc);1952	}1953 1954	if (mtc > decoder->last_mtc)1955		mtc_delta = mtc - decoder->last_mtc;1956	else1957		mtc_delta = mtc + 256 - decoder->last_mtc;1958 1959	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;1960 1961	if (decoder->tsc_ctc_mult) {1962		timestamp = decoder->ctc_timestamp +1963			    decoder->ctc_delta * decoder->tsc_ctc_mult;1964	} else {1965		timestamp = decoder->ctc_timestamp +1966			    multdiv(decoder->ctc_delta,1967				    decoder->tsc_ctc_ratio_n,1968				    decoder->tsc_ctc_ratio_d);1969	}1970 1971	if (timestamp < decoder->timestamp)1972		intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",1973			     timestamp, decoder->timestamp);1974	else1975		decoder->timestamp = timestamp;1976 1977	intel_pt_mtc_cyc_cnt_upd(decoder);1978 1979	decoder->timestamp_insn_cnt = 0;1980	decoder->last_mtc = mtc;1981 1982	if (decoder->last_packet_type == INTEL_PT_CYC) {1983		decoder->cyc_ref_timestamp = decoder->timestamp;1984		decoder->cycle_cnt = 0;1985		decoder->have_calc_cyc_to_tsc = false;1986		intel_pt_calc_cyc_to_tsc(decoder, true);1987	}1988 1989	intel_pt_log_to("Setting timestamp", decoder->timestamp);1990}1991 1992static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)1993{1994	unsigned int cbr = decoder->packet.payload & 0xff;1995 1996	decoder->cbr_payload = decoder->packet.payload;1997 1998	if (decoder->cbr == cbr)1999		return;2000 2001	decoder->cbr = cbr;2002	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;2003	decoder->cyc_ref_timestamp = decoder->timestamp;2004	decoder->cycle_cnt = 0;2005 2006	intel_pt_mtc_cyc_cnt_cbr(decoder);2007}2008 2009static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)2010{2011	uint64_t timestamp = decoder->cyc_ref_timestamp;2012 2013	decoder->have_cyc = true;2014 2015	decoder->cycle_cnt += decoder->packet.payload;2016	if (decoder->pge)2017		decoder->tot_cyc_cnt += decoder->packet.payload;2018	decoder->sample_cyc = true;2019 2020	if (!decoder->cyc_ref_timestamp)2021		return;2022 2023	if (decoder->have_calc_cyc_to_tsc)2024		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;2025	else if (decoder->cbr)2026		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;2027	else2028		return;2029 2030	if (timestamp < decoder->timestamp)2031		intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",2032			     timestamp, decoder->timestamp);2033	else2034		decoder->timestamp = timestamp;2035 2036	decoder->timestamp_insn_cnt = 0;2037 2038	intel_pt_log_to("Setting timestamp", decoder->timestamp);2039}2040 2041static void intel_pt_bbp(struct intel_pt_decoder *decoder)2042{2043	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {2044		memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));2045		decoder->state.items.is_32_bit = false;2046	}2047	decoder->blk_type = decoder->packet.payload;2048	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);2049	if (decoder->blk_type == INTEL_PT_GP_REGS)2050		decoder->state.items.is_32_bit = decoder->packet.count;2051	if (decoder->blk_type_pos < 0) {2052		intel_pt_log("WARNING: Unknown block type %u\n",2053			     decoder->blk_type);2054	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {2055		intel_pt_log("WARNING: Duplicate block type %u\n",2056			     decoder->blk_type);2057	}2058}2059 2060static void intel_pt_bip(struct intel_pt_decoder *decoder)2061{2062	uint32_t id = decoder->packet.count;2063	uint32_t bit = 1 << id;2064	int pos = decoder->blk_type_pos;2065 2066	if (pos < 0 || id >= INTEL_PT_BLK_ITEM_ID_CNT) {2067		intel_pt_log("WARNING: Unknown block item %u type %d\n",2068			     id, decoder->blk_type);2069		return;2070	}2071 2072	if (decoder->state.items.mask[pos] & bit) {2073		intel_pt_log("WARNING: Duplicate block item %u type %d\n",2074			     id, decoder->blk_type);2075	}2076 2077	decoder->state.items.mask[pos] |= bit;2078	decoder->state.items.val[pos][id] = decoder->packet.payload;2079}2080 2081/* Walk PSB+ packets when already in sync. */2082static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)2083{2084	int err;2085 2086	decoder->in_psb = true;2087 2088	while (1) {2089		err = intel_pt_get_next_packet(decoder);2090		if (err)2091			goto out;2092 2093		switch (decoder->packet.type) {2094		case INTEL_PT_PSBEND:2095			err = 0;2096			goto out;2097 2098		case INTEL_PT_TIP_PGD:2099		case INTEL_PT_TIP_PGE:2100		case INTEL_PT_TIP:2101		case INTEL_PT_TNT:2102		case INTEL_PT_TRACESTOP:2103		case INTEL_PT_BAD:2104		case INTEL_PT_PSB:2105		case INTEL_PT_PTWRITE:2106		case INTEL_PT_PTWRITE_IP:2107		case INTEL_PT_EXSTOP:2108		case INTEL_PT_EXSTOP_IP:2109		case INTEL_PT_MWAIT:2110		case INTEL_PT_PWRE:2111		case INTEL_PT_PWRX:2112		case INTEL_PT_BBP:2113		case INTEL_PT_BIP:2114		case INTEL_PT_BEP:2115		case INTEL_PT_BEP_IP:2116		case INTEL_PT_CFE:2117		case INTEL_PT_CFE_IP:2118		case INTEL_PT_EVD:2119			decoder->have_tma = false;2120			intel_pt_log("ERROR: Unexpected packet\n");2121			err = -EAGAIN;2122			goto out;2123 2124		case INTEL_PT_OVF:2125			err = intel_pt_overflow(decoder);2126			goto out;2127 2128		case INTEL_PT_TSC:2129			intel_pt_calc_tsc_timestamp(decoder);2130			break;2131 2132		case INTEL_PT_TMA:2133			intel_pt_calc_tma(decoder);2134			break;2135 2136		case INTEL_PT_CBR:2137			intel_pt_calc_cbr(decoder);2138			break;2139 2140		case INTEL_PT_MODE_EXEC:2141			intel_pt_mode_exec_status(decoder);2142			break;2143 2144		case INTEL_PT_PIP:2145			intel_pt_set_pip(decoder);2146			break;2147 2148		case INTEL_PT_FUP:2149			decoder->pge = true;2150			if (decoder->packet.count) {2151				intel_pt_set_last_ip(decoder);2152				decoder->psb_ip = decoder->last_ip;2153			}2154			break;2155 2156		case INTEL_PT_MODE_TSX:2157			intel_pt_update_in_tx(decoder);2158			break;2159 2160		case INTEL_PT_MTC:2161			intel_pt_calc_mtc_timestamp(decoder);2162			if (decoder->period_type == INTEL_PT_PERIOD_MTC)2163				decoder->state.type |= INTEL_PT_INSTRUCTION;2164			break;2165 2166		case INTEL_PT_CYC:2167			intel_pt_calc_cyc_timestamp(decoder);2168			break;2169 2170		case INTEL_PT_VMCS:2171		case INTEL_PT_MNT:2172		case INTEL_PT_PAD:2173		default:2174			break;2175		}2176	}2177out:2178	decoder->in_psb = false;2179 2180	return err;2181}2182 2183static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)2184{2185	int err;2186 2187	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {2188		decoder->tx_flags = 0;2189		decoder->state.flags &= ~INTEL_PT_IN_TX;2190		decoder->state.flags |= INTEL_PT_ABORT_TX;2191	} else {2192		decoder->state.flags |= INTEL_PT_ASYNC;2193	}2194 2195	while (1) {2196		err = intel_pt_get_next_packet(decoder);2197		if (err)2198			return err;2199 2200		switch (decoder->packet.type) {2201		case INTEL_PT_TNT:2202		case INTEL_PT_FUP:2203		case INTEL_PT_TRACESTOP:2204		case INTEL_PT_PSB:2205		case INTEL_PT_TSC:2206		case INTEL_PT_TMA:2207		case INTEL_PT_MODE_TSX:2208		case INTEL_PT_BAD:2209		case INTEL_PT_PSBEND:2210		case INTEL_PT_PTWRITE:2211		case INTEL_PT_PTWRITE_IP:2212		case INTEL_PT_EXSTOP:2213		case INTEL_PT_EXSTOP_IP:2214		case INTEL_PT_MWAIT:2215		case INTEL_PT_PWRE:2216		case INTEL_PT_PWRX:2217		case INTEL_PT_BBP:2218		case INTEL_PT_BIP:2219		case INTEL_PT_BEP:2220		case INTEL_PT_BEP_IP:2221		case INTEL_PT_CFE:2222		case INTEL_PT_CFE_IP:2223		case INTEL_PT_EVD:2224			intel_pt_log("ERROR: Missing TIP after FUP\n");2225			decoder->pkt_state = INTEL_PT_STATE_ERR3;2226			decoder->pkt_step = 0;2227			return -ENOENT;2228 2229		case INTEL_PT_CBR:2230			intel_pt_calc_cbr(decoder);2231			break;2232 2233		case INTEL_PT_OVF:2234			return intel_pt_overflow(decoder);2235 2236		case INTEL_PT_TIP_PGD:2237			decoder->state.from_ip = decoder->ip;2238			if (decoder->packet.count == 0) {2239				decoder->state.to_ip = 0;2240			} else {2241				intel_pt_set_ip(decoder);2242				decoder->state.to_ip = decoder->ip;2243			}2244			decoder->pge = false;2245			decoder->continuous_period = false;2246			decoder->state.type |= INTEL_PT_TRACE_END;2247			intel_pt_update_nr(decoder);2248			return 0;2249 2250		case INTEL_PT_TIP_PGE:2251			decoder->pge = true;2252			intel_pt_log("Omitting PGE ip " x64_fmt "\n",2253				     decoder->ip);2254			decoder->state.from_ip = 0;2255			if (decoder->packet.count == 0) {2256				decoder->state.to_ip = 0;2257			} else {2258				intel_pt_set_ip(decoder);2259				decoder->state.to_ip = decoder->ip;2260			}2261			decoder->state.type |= INTEL_PT_TRACE_BEGIN;2262			intel_pt_mtc_cyc_cnt_pge(decoder);2263			intel_pt_set_nr(decoder);2264			return 0;2265 2266		case INTEL_PT_TIP:2267			decoder->state.from_ip = decoder->ip;2268			if (decoder->packet.count == 0) {2269				decoder->state.to_ip = 0;2270			} else {2271				intel_pt_set_ip(decoder);2272				decoder->state.to_ip = decoder->ip;2273			}2274			intel_pt_update_nr(decoder);2275			intel_pt_sample_iflag_chg(decoder);2276			return 0;2277 2278		case INTEL_PT_PIP:2279			intel_pt_update_pip(decoder);2280			break;2281 2282		case INTEL_PT_MTC:2283			intel_pt_calc_mtc_timestamp(decoder);2284			if (decoder->period_type == INTEL_PT_PERIOD_MTC)2285				decoder->state.type |= INTEL_PT_INSTRUCTION;2286			break;2287 2288		case INTEL_PT_CYC:2289			intel_pt_calc_cyc_timestamp(decoder);2290			break;2291 2292		case INTEL_PT_MODE_EXEC:2293			intel_pt_mode_exec(decoder);2294			break;2295 2296		case INTEL_PT_VMCS:2297		case INTEL_PT_MNT:2298		case INTEL_PT_PAD:2299			break;2300 2301		default:2302			return intel_pt_bug(decoder);2303		}2304	}2305}2306 2307static int intel_pt_resample(struct intel_pt_decoder *decoder)2308{2309	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;2310	decoder->state.type = INTEL_PT_INSTRUCTION;2311	decoder->state.from_ip = decoder->ip;2312	decoder->state.to_ip = 0;2313	return 0;2314}2315 2316struct intel_pt_vm_tsc_info {2317	struct intel_pt_pkt pip_packet;2318	struct intel_pt_pkt vmcs_packet;2319	struct intel_pt_pkt tma_packet;2320	bool tsc, pip, vmcs, tma, psbend;2321	uint64_t ctc_delta;2322	uint64_t last_ctc;2323	int max_lookahead;2324};2325 2326/* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */2327static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)2328{2329	struct intel_pt_vm_tsc_info *data = pkt_info->data;2330 2331	switch (pkt_info->packet.type) {2332	case INTEL_PT_PAD:2333	case INTEL_PT_MNT:2334	case INTEL_PT_MODE_EXEC:2335	case INTEL_PT_MODE_TSX:2336	case INTEL_PT_MTC:2337	case INTEL_PT_FUP:2338	case INTEL_PT_CYC:2339	case INTEL_PT_CBR:2340		break;2341 2342	case INTEL_PT_TSC:2343		data->tsc = true;2344		break;2345 2346	case INTEL_PT_TMA:2347		data->tma_packet = pkt_info->packet;2348		data->tma = true;2349		break;2350 2351	case INTEL_PT_PIP:2352		data->pip_packet = pkt_info->packet;2353		data->pip = true;2354		break;2355 2356	case INTEL_PT_VMCS:2357		data->vmcs_packet = pkt_info->packet;2358		data->vmcs = true;2359		break;2360 2361	case INTEL_PT_PSBEND:2362		data->psbend = true;2363		return 1;2364 2365	case INTEL_PT_TIP_PGE:2366	case INTEL_PT_PTWRITE:2367	case INTEL_PT_PTWRITE_IP:2368	case INTEL_PT_EXSTOP:2369	case INTEL_PT_EXSTOP_IP:2370	case INTEL_PT_MWAIT:2371	case INTEL_PT_PWRE:2372	case INTEL_PT_PWRX:2373	case INTEL_PT_BBP:2374	case INTEL_PT_BIP:2375	case INTEL_PT_BEP:2376	case INTEL_PT_BEP_IP:2377	case INTEL_PT_OVF:2378	case INTEL_PT_BAD:2379	case INTEL_PT_TNT:2380	case INTEL_PT_TIP_PGD:2381	case INTEL_PT_TIP:2382	case INTEL_PT_PSB:2383	case INTEL_PT_TRACESTOP:2384	case INTEL_PT_CFE:2385	case INTEL_PT_CFE_IP:2386	case INTEL_PT_EVD:2387	default:2388		return 1;2389	}2390 2391	return 0;2392}2393 2394struct intel_pt_ovf_fup_info {2395	int max_lookahead;2396	bool found;2397};2398 2399/* Lookahead to detect a FUP packet after OVF */2400static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info)2401{2402	struct intel_pt_ovf_fup_info *data = pkt_info->data;2403 2404	if (pkt_info->packet.type == INTEL_PT_CYC ||2405	    pkt_info->packet.type == INTEL_PT_MTC ||2406	    pkt_info->packet.type == INTEL_PT_TSC)2407		return !--(data->max_lookahead);2408	data->found = pkt_info->packet.type == INTEL_PT_FUP;2409	return 1;2410}2411 2412static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)2413{2414	struct intel_pt_ovf_fup_info data = {2415		.max_lookahead = 16,2416		.found = false,2417	};2418 2419	intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);2420	return data.found;2421}2422 2423/* Lookahead and get the TMA packet after TSC */2424static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info)2425{2426	struct intel_pt_vm_tsc_info *data = pkt_info->data;2427 2428	if (pkt_info->packet.type == INTEL_PT_CYC ||2429	    pkt_info->packet.type == INTEL_PT_MTC)2430		return !--(data->max_lookahead);2431 2432	if (pkt_info->packet.type == INTEL_PT_TMA) {2433		data->tma_packet = pkt_info->packet;2434		data->tma = true;2435	}2436	return 1;2437}2438 2439static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)2440{2441	if (decoder->tsc_ctc_mult)2442		return ctc * decoder->tsc_ctc_mult;2443	else2444		return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);2445}2446 2447static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,2448					   uint32_t ctc,2449					   uint32_t fc,2450					   uint64_t last_ctc_timestamp,2451					   uint64_t ctc_delta,2452					   uint32_t last_ctc)2453{2454	/* Number of CTC ticks from last_ctc_timestamp to last_mtc */2455	uint64_t last_mtc_ctc = last_ctc + ctc_delta;2456	/*2457	 * Number of CTC ticks from there until current TMA packet. We would2458	 * expect last_mtc_ctc to be before ctc, but the TSC packet can slip2459	 * past an MTC, so a sign-extended value is used.2460	 */2461	uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc);2462	/* Total CTC ticks from last_ctc_timestamp to current TMA packet */2463	uint64_t new_ctc_delta = ctc_delta + delta;2464	uint64_t expected_tsc;2465 2466	/*2467	 * Convert CTC ticks to TSC ticks, add the starting point2468	 * (last_ctc_timestamp) and the fast counter from the TMA packet.2469	 */2470	expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;2471 2472	if (intel_pt_enable_logging) {2473		intel_pt_log_x64(last_mtc_ctc);2474		intel_pt_log_x32(last_ctc);2475		intel_pt_log_x64(ctc_delta);2476		intel_pt_log_x64(delta);2477		intel_pt_log_x32(ctc);2478		intel_pt_log_x64(new_ctc_delta);2479		intel_pt_log_x64(last_ctc_timestamp);2480		intel_pt_log_x32(fc);2481		intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));2482		intel_pt_log_x64(expected_tsc);2483	}2484 2485	return expected_tsc;2486}2487 2488static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,2489				      struct intel_pt_vm_tsc_info *data)2490{2491	uint32_t ctc = data->tma_packet.payload;2492	uint32_t fc = data->tma_packet.count;2493 2494	return intel_pt_calc_expected_tsc(decoder, ctc, fc,2495					  decoder->ctc_timestamp,2496					  data->ctc_delta, data->last_ctc);2497}2498 2499static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,2500				      struct intel_pt_vmcs_info *vmcs_info)2501{2502	uint64_t payload = decoder->packet.payload;2503 2504	/* VMX adds the TSC Offset, so subtract to get host TSC */2505	decoder->packet.payload -= vmcs_info->tsc_offset;2506	/* TSC packet has only 7 bytes */2507	decoder->packet.payload &= SEVEN_BYTES;2508 2509	/*2510	 * The buffer is mmapped from the data file, so this also updates the2511	 * data file.2512	 */2513	if (!decoder->vm_tm_corr_dry_run)2514		memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);2515 2516	intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx642517		     "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",2518		     payload, decoder->packet.payload, vmcs_info->vmcs,2519		     vmcs_info->tsc_offset);2520}2521 2522static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,2523					     uint64_t tsc_offset)2524{2525	struct intel_pt_vmcs_info vmcs_info = {2526		.vmcs = NO_VMCS,2527		.tsc_offset = tsc_offset2528	};2529 2530	intel_pt_translate_vm_tsc(decoder, &vmcs_info);2531}2532 2533static inline bool in_vm(uint64_t pip_payload)2534{2535	return pip_payload & 1;2536}2537 2538static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet)2539{2540	return pip_packet->payload & 1;2541}2542 2543static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info)2544{2545	p_log("VMCS: %#" PRIx64 "  TSC Offset %#" PRIx64,2546	      vmcs_info->vmcs, vmcs_info->tsc_offset);2547}2548 2549static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,2550				    struct intel_pt_vm_tsc_info *data)2551{2552	memset(data, 0, sizeof(*data));2553	data->ctc_delta = decoder->ctc_delta;2554	data->last_ctc = decoder->last_ctc;2555	intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);2556	if (data->tsc && !data->psbend)2557		p_log("ERROR: PSB without PSBEND");2558	decoder->in_psb = data->psbend;2559}2560 2561static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,2562					  struct intel_pt_vm_tsc_info *data,2563					  struct intel_pt_vmcs_info *vmcs_info,2564					  uint64_t host_tsc)2565{2566	if (!decoder->in_psb) {2567		/* Can't happen */2568		p_log("ERROR: First TSC is not in PSB+");2569	}2570 2571	if (data->pip) {2572		if (pip_in_vm(&data->pip_packet)) { /* Guest */2573			if (vmcs_info && vmcs_info->tsc_offset) {2574				intel_pt_translate_vm_tsc(decoder, vmcs_info);2575				decoder->vm_tm_corr_reliable = true;2576			} else {2577				p_log("ERROR: First TSC, unknown TSC Offset");2578			}2579		} else { /* Host */2580			decoder->vm_tm_corr_reliable = true;2581		}2582	} else { /* Host or Guest */2583		decoder->vm_tm_corr_reliable = false;2584		if (intel_pt_time_in_range(decoder, host_tsc)) {2585			/* Assume Host */2586		} else {2587			/* Assume Guest */2588			if (vmcs_info && vmcs_info->tsc_offset)2589				intel_pt_translate_vm_tsc(decoder, vmcs_info);2590			else2591				p_log("ERROR: First TSC, no PIP, unknown TSC Offset");2592		}2593	}2594}2595 2596static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,2597				    struct intel_pt_vm_tsc_info *data)2598{2599	struct intel_pt_vmcs_info *vmcs_info;2600	uint64_t tsc_offset = 0;2601	uint64_t vmcs;2602	bool reliable = true;2603	uint64_t expected_tsc;2604	uint64_t host_tsc;2605	uint64_t ref_timestamp;2606 2607	bool assign = false;2608	bool assign_reliable = false;2609 2610	/* Already have 'data' for the in_psb case */2611	if (!decoder->in_psb) {2612		memset(data, 0, sizeof(*data));2613		data->ctc_delta = decoder->ctc_delta;2614		data->last_ctc = decoder->last_ctc;2615		data->max_lookahead = 16;2616		intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);2617		if (decoder->pge) {2618			data->pip = true;2619			data->pip_packet.payload = decoder->pip_payload;2620		}2621	}2622 2623	/* Calculations depend on having TMA packets */2624	if (!data->tma) {2625		p_log("ERROR: TSC without TMA");2626		return;2627	}2628 2629	vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;2630	if (vmcs == NO_VMCS)2631		vmcs = 0;2632 2633	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);2634 2635	ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;2636	host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);2637 2638	if (!decoder->ctc_timestamp) {2639		intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);2640		return;2641	}2642 2643	expected_tsc = intel_pt_expected_tsc(decoder, data);2644 2645	tsc_offset = host_tsc - expected_tsc;2646 2647	/* Determine if TSC is from Host or Guest */2648	if (data->pip) {2649		if (pip_in_vm(&data->pip_packet)) { /* Guest */2650			if (!vmcs_info) {2651				/* PIP NR=1 without VMCS cannot happen */2652				p_log("ERROR: Missing VMCS");2653				intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);2654				decoder->vm_tm_corr_reliable = false;2655				return;2656			}2657		} else { /* Host */2658			decoder->last_reliable_timestamp = host_tsc;2659			decoder->vm_tm_corr_reliable = true;2660			return;2661		}2662	} else { /* Host or Guest */2663		reliable = false; /* Host/Guest is a guess, so not reliable */2664		if (decoder->in_psb) {2665			if (!tsc_offset)2666				return; /* Zero TSC Offset, assume Host */2667			/*2668			 * TSC packet has only 7 bytes of TSC. We have no2669			 * information about the Guest's 8th byte, but it2670			 * doesn't matter because we only need 7 bytes.2671			 * Here, since the 8th byte is unreliable and2672			 * irrelevant, compare only 7 byes.2673			 */2674			if (vmcs_info &&2675			    (tsc_offset & SEVEN_BYTES) ==2676			    (vmcs_info->tsc_offset & SEVEN_BYTES)) {2677				/* Same TSC Offset as last VMCS, assume Guest */2678				goto guest;2679			}2680		}2681		/*2682		 * Check if the host_tsc is within the expected range.2683		 * Note, we could narrow the range more by looking ahead for2684		 * the next host TSC in the same buffer, but we don't bother to2685		 * do that because this is probably good enough.2686		 */2687		if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {2688			/* Within expected range for Host TSC, assume Host */2689			decoder->vm_tm_corr_reliable = false;2690			return;2691		}2692	}2693 2694guest: /* Assuming Guest */2695 2696	/* Determine whether to assign TSC Offset */2697	if (vmcs_info && vmcs_info->vmcs) {2698		if (vmcs_info->tsc_offset && vmcs_info->reliable) {2699			assign = false;2700		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&2701			   decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {2702			/* Continuous tracing, TSC in a PSB is not a time loss */2703			assign = true;2704			assign_reliable = true;2705		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {2706			/*2707			 * Unlikely to be a time loss TSC in a PSB which is not2708			 * at the start of a buffer.2709			 */2710			assign = true;2711			assign_reliable = false;2712		}2713	}2714 2715	/* Record VMCS TSC Offset */2716	if (assign && (vmcs_info->tsc_offset != tsc_offset ||2717		       vmcs_info->reliable != assign_reliable)) {2718		bool print = vmcs_info->tsc_offset != tsc_offset;2719 2720		vmcs_info->tsc_offset = tsc_offset;2721		vmcs_info->reliable = assign_reliable;2722		if (print)2723			intel_pt_print_vmcs_info(vmcs_info);2724	}2725 2726	/* Determine what TSC Offset to use */2727	if (vmcs_info && vmcs_info->tsc_offset) {2728		if (!vmcs_info->reliable)2729			reliable = false;2730		intel_pt_translate_vm_tsc(decoder, vmcs_info);2731	} else {2732		reliable = false;2733		if (vmcs_info) {2734			if (!vmcs_info->error_printed) {2735				p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64,2736				      vmcs_info->vmcs);2737				vmcs_info->error_printed = true;2738			}2739		} else {2740			if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))2741				p_log("ERROR: Unknown VMCS");2742		}2743		intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);2744	}2745 2746	decoder->vm_tm_corr_reliable = reliable;2747}2748 2749static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)2750{2751	uint64_t host_tsc = decoder->packet.payload;2752	uint64_t guest_tsc = decoder->packet.payload;2753	struct intel_pt_vmcs_info *vmcs_info;2754	uint64_t vmcs;2755 2756	vmcs = decoder->vmcs;2757	if (vmcs == NO_VMCS)2758		vmcs = 0;2759 2760	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);2761 2762	if (decoder->pge) {2763		if (in_vm(decoder->pip_payload)) { /* Guest */2764			if (!vmcs_info) {2765				/* PIP NR=1 without VMCS cannot happen */2766				p_log("ERROR: Missing VMCS");2767			}2768		} else { /* Host */2769			return;2770		}2771	} else { /* Host or Guest */2772		if (intel_pt_time_in_range(decoder, host_tsc)) {2773			/* Within expected range for Host TSC, assume Host */2774			return;2775		}2776	}2777 2778	if (vmcs_info) {2779		/* Translate Guest TSC to Host TSC */2780		host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES;2781		host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);2782		intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx642783			     "    VMCS %#" PRIx64 "    TSC Offset %#" PRIx64 "\n",2784			     guest_tsc, host_tsc, vmcs_info->vmcs,2785			     vmcs_info->tsc_offset);2786		if (!intel_pt_time_in_range(decoder, host_tsc) &&2787		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))2788			p_log("Timestamp out of range");2789	} else {2790		if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))2791			p_log("ERROR: Unknown VMCS");2792		host_tsc = decoder->timestamp;2793	}2794 2795	decoder->packet.payload = host_tsc;2796 2797	if (!decoder->vm_tm_corr_dry_run)2798		memcpy((void *)decoder->buf + 1, &host_tsc, 8);2799}2800 2801static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)2802{2803	struct intel_pt_vm_tsc_info data = { .psbend = false };2804	bool pge;2805	int err;2806 2807	if (decoder->in_psb)2808		intel_pt_vm_tm_corr_psb(decoder, &data);2809 2810	while (1) {2811		err = intel_pt_get_next_packet(decoder);2812		if (err == -ENOLINK)2813			continue;2814		if (err)2815			break;2816 2817		switch (decoder->packet.type) {2818		case INTEL_PT_TIP_PGD:2819			decoder->pge = false;2820			decoder->vm_tm_corr_continuous = false;2821			break;2822 2823		case INTEL_PT_TNT:2824		case INTEL_PT_TIP:2825		case INTEL_PT_TIP_PGE:2826			decoder->pge = true;2827			break;2828 2829		case INTEL_PT_OVF:2830			decoder->in_psb = false;2831			pge = decoder->pge;2832			decoder->pge = intel_pt_ovf_fup_lookahead(decoder);2833			if (pge != decoder->pge)2834				intel_pt_log("Surprising PGE change in OVF!");2835			if (!decoder->pge)2836				decoder->vm_tm_corr_continuous = false;2837			break;2838 2839		case INTEL_PT_FUP:2840			if (decoder->in_psb)2841				decoder->pge = true;2842			break;2843 2844		case INTEL_PT_TRACESTOP:2845			decoder->pge = false;2846			decoder->vm_tm_corr_continuous = false;2847			decoder->have_tma = false;2848			break;2849 2850		case INTEL_PT_PSB:2851			intel_pt_vm_tm_corr_psb(decoder, &data);2852			break;2853 2854		case INTEL_PT_PIP:2855			decoder->pip_payload = decoder->packet.payload;2856			break;2857 2858		case INTEL_PT_MTC:2859			intel_pt_calc_mtc_timestamp(decoder);2860			break;2861 2862		case INTEL_PT_TSC:2863			intel_pt_vm_tm_corr_tsc(decoder, &data);2864			intel_pt_calc_tsc_timestamp(decoder);2865			decoder->vm_tm_corr_same_buf = true;2866			decoder->vm_tm_corr_continuous = decoder->pge;2867			break;2868 2869		case INTEL_PT_TMA:2870			intel_pt_calc_tma(decoder);2871			break;2872 2873		case INTEL_PT_CYC:2874			intel_pt_calc_cyc_timestamp(decoder);2875			break;2876 2877		case INTEL_PT_CBR:2878			intel_pt_calc_cbr(decoder);2879			break;2880 2881		case INTEL_PT_PSBEND:2882			decoder->in_psb = false;2883			data.psbend = false;2884			break;2885 2886		case INTEL_PT_VMCS:2887			if (decoder->packet.payload != NO_VMCS)2888				decoder->vmcs = decoder->packet.payload;2889			break;2890 2891		case INTEL_PT_BBP:2892			decoder->blk_type = decoder->packet.payload;2893			break;2894 2895		case INTEL_PT_BIP:2896			if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&2897			    decoder->packet.count == 2)2898				intel_pt_vm_tm_corr_pebs_tsc(decoder);2899			break;2900 2901		case INTEL_PT_BEP:2902		case INTEL_PT_BEP_IP:2903			decoder->blk_type = 0;2904			break;2905 2906		case INTEL_PT_CFE:2907		case INTEL_PT_CFE_IP:2908		case INTEL_PT_EVD:2909		case INTEL_PT_MODE_EXEC:2910		case INTEL_PT_MODE_TSX:2911		case INTEL_PT_MNT:2912		case INTEL_PT_PAD:2913		case INTEL_PT_PTWRITE_IP:2914		case INTEL_PT_PTWRITE:2915		case INTEL_PT_MWAIT:2916		case INTEL_PT_PWRE:2917		case INTEL_PT_EXSTOP_IP:2918		case INTEL_PT_EXSTOP:2919		case INTEL_PT_PWRX:2920		case INTEL_PT_BAD: /* Does not happen */2921		default:2922			break;2923		}2924	}2925 2926	return err;2927}2928 2929#define HOP_PROCESS	02930#define HOP_IGNORE	12931#define HOP_RETURN	22932#define HOP_AGAIN	32933 2934static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);2935 2936/* Hop mode: Ignore TNT, do not walk code, but get ip from FUPs and TIPs */2937static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)2938{2939	*err = 0;2940 2941	/* Leap from PSB to PSB, getting ip from FUP within PSB+ */2942	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {2943		*err = intel_pt_scan_for_psb(decoder);2944		if (*err)2945			return HOP_RETURN;2946	}2947 2948	switch (decoder->packet.type) {2949	case INTEL_PT_TNT:2950		return HOP_IGNORE;2951 2952	case INTEL_PT_TIP_PGD:2953		decoder->pge = false;2954		if (!decoder->packet.count) {2955			intel_pt_set_nr(decoder);2956			return HOP_IGNORE;2957		}2958		intel_pt_set_ip(decoder);2959		decoder->state.type |= INTEL_PT_TRACE_END;2960		decoder->state.from_ip = 0;2961		decoder->state.to_ip = decoder->ip;2962		intel_pt_update_nr(decoder);2963		return HOP_RETURN;2964 2965	case INTEL_PT_TIP:2966		if (!decoder->packet.count) {2967			intel_pt_set_nr(decoder);2968			return HOP_IGNORE;2969		}2970		intel_pt_set_ip(decoder);2971		decoder->state.type = INTEL_PT_INSTRUCTION;2972		decoder->state.from_ip = decoder->ip;2973		decoder->state.to_ip = 0;2974		intel_pt_update_nr(decoder);2975		intel_pt_sample_iflag_chg(decoder);2976		return HOP_RETURN;2977 2978	case INTEL_PT_FUP:2979		if (!decoder->packet.count)2980			return HOP_IGNORE;2981		intel_pt_set_ip(decoder);2982		if (decoder->set_fup_mwait || decoder->set_fup_pwre)2983			*no_tip = true;2984		if (!decoder->branch_enable || !decoder->pge)2985			*no_tip = true;2986		if (*no_tip) {2987			decoder->state.type = INTEL_PT_INSTRUCTION;2988			decoder->state.from_ip = decoder->ip;2989			decoder->state.to_ip = 0;2990			intel_pt_fup_event(decoder, *no_tip);2991			return HOP_RETURN;2992		}2993		intel_pt_fup_event(decoder, *no_tip);2994		decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;2995		*err = intel_pt_walk_fup_tip(decoder);2996		if (!*err && decoder->state.to_ip)2997			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;2998		return HOP_RETURN;2999 3000	case INTEL_PT_PSB:3001		decoder->state.psb_offset = decoder->pos;3002		decoder->psb_ip = 0;3003		decoder->last_ip = 0;3004		decoder->have_last_ip = true;3005		*err = intel_pt_walk_psbend(decoder);3006		if (*err == -EAGAIN)3007			return HOP_AGAIN;3008		if (*err)3009			return HOP_RETURN;3010		decoder->state.type = INTEL_PT_PSB_EVT;3011		if (decoder->psb_ip) {3012			decoder->state.type |= INTEL_PT_INSTRUCTION;3013			decoder->ip = decoder->psb_ip;3014		}3015		decoder->state.from_ip = decoder->psb_ip;3016		decoder->state.to_ip = 0;3017		return HOP_RETURN;3018 3019	case INTEL_PT_BAD:3020	case INTEL_PT_PAD:3021	case INTEL_PT_TIP_PGE:3022	case INTEL_PT_TSC:3023	case INTEL_PT_TMA:3024	case INTEL_PT_MODE_EXEC:3025	case INTEL_PT_MODE_TSX:3026	case INTEL_PT_MTC:3027	case INTEL_PT_CYC:3028	case INTEL_PT_VMCS:3029	case INTEL_PT_PSBEND:3030	case INTEL_PT_CBR:3031	case INTEL_PT_TRACESTOP:3032	case INTEL_PT_PIP:3033	case INTEL_PT_OVF:3034	case INTEL_PT_MNT:3035	case INTEL_PT_PTWRITE:3036	case INTEL_PT_PTWRITE_IP:3037	case INTEL_PT_EXSTOP:3038	case INTEL_PT_EXSTOP_IP:3039	case INTEL_PT_MWAIT:3040	case INTEL_PT_PWRE:3041	case INTEL_PT_PWRX:3042	case INTEL_PT_BBP:3043	case INTEL_PT_BIP:3044	case INTEL_PT_BEP:3045	case INTEL_PT_BEP_IP:3046	case INTEL_PT_CFE:3047	case INTEL_PT_CFE_IP:3048	case INTEL_PT_EVD:3049	default:3050		return HOP_PROCESS;3051	}3052}3053 3054struct intel_pt_psb_info {3055	struct intel_pt_pkt fup_packet;3056	bool fup;3057	int after_psbend;3058};3059 3060/* Lookahead and get the FUP packet from PSB+ */3061static int intel_pt_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info)3062{3063	struct intel_pt_psb_info *data = pkt_info->data;3064 3065	switch (pkt_info->packet.type) {3066	case INTEL_PT_PAD:3067	case INTEL_PT_MNT:3068	case INTEL_PT_TSC:3069	case INTEL_PT_TMA:3070	case INTEL_PT_MODE_EXEC:3071	case INTEL_PT_MODE_TSX:3072	case INTEL_PT_MTC:3073	case INTEL_PT_CYC:3074	case INTEL_PT_VMCS:3075	case INTEL_PT_CBR:3076	case INTEL_PT_PIP:3077		if (data->after_psbend) {3078			data->after_psbend -= 1;3079			if (!data->after_psbend)3080				return 1;3081		}3082		break;3083 3084	case INTEL_PT_FUP:3085		if (data->after_psbend)3086			return 1;3087		if (data->fup || pkt_info->packet.count == 0)3088			return 1;3089		data->fup_packet = pkt_info->packet;3090		data->fup = true;3091		break;3092 3093	case INTEL_PT_PSBEND:3094		if (!data->fup)3095			return 1;3096		/* Keep going to check for a TIP.PGE */3097		data->after_psbend = 6;3098		break;3099 3100	case INTEL_PT_TIP_PGE:3101		/* Ignore FUP in PSB+ if followed by TIP.PGE */3102		if (data->after_psbend)3103			data->fup = false;3104		return 1;3105 3106	case INTEL_PT_PTWRITE:3107	case INTEL_PT_PTWRITE_IP:3108	case INTEL_PT_EXSTOP:3109	case INTEL_PT_EXSTOP_IP:3110	case INTEL_PT_MWAIT:3111	case INTEL_PT_PWRE:3112	case INTEL_PT_PWRX:3113	case INTEL_PT_BBP:3114	case INTEL_PT_BIP:3115	case INTEL_PT_BEP:3116	case INTEL_PT_BEP_IP:3117	case INTEL_PT_CFE:3118	case INTEL_PT_CFE_IP:3119	case INTEL_PT_EVD:3120		if (data->after_psbend) {3121			data->after_psbend -= 1;3122			if (!data->after_psbend)3123				return 1;3124			break;3125		}3126		return 1;3127 3128	case INTEL_PT_OVF:3129	case INTEL_PT_BAD:3130	case INTEL_PT_TNT:3131	case INTEL_PT_TIP_PGD:3132	case INTEL_PT_TIP:3133	case INTEL_PT_PSB:3134	case INTEL_PT_TRACESTOP:3135	default:3136		return 1;3137	}3138 3139	return 0;3140}3141 3142static int intel_pt_psb(struct intel_pt_decoder *decoder)3143{3144	int err;3145 3146	decoder->last_ip = 0;3147	decoder->psb_ip = 0;3148	decoder->have_last_ip = true;3149	intel_pt_clear_stack(&decoder->stack);3150	err = intel_pt_walk_psbend(decoder);3151	if (err)3152		return err;3153	decoder->state.type = INTEL_PT_PSB_EVT;3154	decoder->state.from_ip = decoder->psb_ip;3155	decoder->state.to_ip = 0;3156	return 0;3157}3158 3159static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)3160{3161	int err;3162 3163	if (decoder->ip != decoder->last_ip) {3164		err = intel_pt_walk_fup(decoder);3165		if (!err || err != -EAGAIN)3166			return err;3167	}3168 3169	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;3170	err = intel_pt_psb(decoder);3171	if (err) {3172		decoder->pkt_state = INTEL_PT_STATE_ERR3;3173		return -ENOENT;3174	}3175 3176	return 0;3177}3178 3179static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)3180{3181	struct intel_pt_psb_info data = { .fup = false };3182 3183	if (!decoder->branch_enable)3184		return false;3185 3186	intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);3187	if (!data.fup)3188		return false;3189 3190	decoder->packet = data.fup_packet;3191	intel_pt_set_last_ip(decoder);3192	decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;3193 3194	*err = intel_pt_fup_in_psb(decoder);3195 3196	return true;3197}3198 3199static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)3200{3201	int last_packet_type = INTEL_PT_PAD;3202	bool no_tip = false;3203	int err;3204 3205	while (1) {3206		err = intel_pt_get_next_packet(decoder);3207		if (err)3208			return err;3209next:3210		err = 0;3211		if (decoder->cyc_threshold) {3212			if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)3213				decoder->sample_cyc = false;3214			last_packet_type = decoder->packet.type;3215		}3216 3217		if (decoder->hop) {3218			switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {3219			case HOP_IGNORE:3220				continue;3221			case HOP_RETURN:3222				return err;3223			case HOP_AGAIN:3224				goto next;3225			default:3226				break;3227			}3228		}3229 3230		switch (decoder->packet.type) {3231		case INTEL_PT_TNT:3232			if (!decoder->packet.count)3233				break;3234			decoder->tnt = decoder->packet;3235			decoder->pkt_state = INTEL_PT_STATE_TNT;3236			err = intel_pt_walk_tnt(decoder);3237			if (err == -EAGAIN)3238				break;3239			return err;3240 3241		case INTEL_PT_TIP_PGD:3242			if (decoder->packet.count != 0)3243				intel_pt_set_last_ip(decoder);3244			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;3245			return intel_pt_walk_tip(decoder);3246 3247		case INTEL_PT_TIP_PGE: {3248			decoder->pge = true;3249			decoder->overflow = false;3250			intel_pt_mtc_cyc_cnt_pge(decoder);3251			intel_pt_set_nr(decoder);3252			if (decoder->packet.count == 0) {3253				intel_pt_log_at("Skipping zero TIP.PGE",3254						decoder->pos);3255				break;3256			}3257			intel_pt_sample_iflag_chg(decoder);3258			intel_pt_set_ip(decoder);3259			decoder->state.from_ip = 0;3260			decoder->state.to_ip = decoder->ip;3261			decoder->state.type |= INTEL_PT_TRACE_BEGIN;3262			/*3263			 * In hop mode, resample to get the to_ip as an3264			 * "instruction" sample.3265			 */3266			if (decoder->hop)3267				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;3268			return 0;3269		}3270 3271		case INTEL_PT_OVF:3272			return intel_pt_overflow(decoder);3273 3274		case INTEL_PT_TIP:3275			if (decoder->packet.count != 0)3276				intel_pt_set_last_ip(decoder);3277			decoder->pkt_state = INTEL_PT_STATE_TIP;3278			return intel_pt_walk_tip(decoder);3279 3280		case INTEL_PT_FUP:3281			if (decoder->packet.count == 0) {3282				intel_pt_log_at("Skipping zero FUP",3283						decoder->pos);3284				no_tip = false;3285				break;3286			}3287			intel_pt_set_last_ip(decoder);3288			if (!decoder->branch_enable || !decoder->pge) {3289				decoder->ip = decoder->last_ip;3290				if (intel_pt_fup_event(decoder, no_tip))3291					return 0;3292				no_tip = false;3293				break;3294			}3295			if (decoder->set_fup_mwait)3296				no_tip = true;3297			if (no_tip)3298				decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;3299			else3300				decoder->pkt_state = INTEL_PT_STATE_FUP;3301			err = intel_pt_walk_fup(decoder);3302			if (err != -EAGAIN)3303				return err;3304			if (no_tip) {3305				no_tip = false;3306				break;3307			}3308			return intel_pt_walk_fup_tip(decoder);3309 3310		case INTEL_PT_TRACESTOP:3311			decoder->pge = false;3312			decoder->continuous_period = false;3313			intel_pt_clear_tx_flags(decoder);3314			decoder->have_tma = false;3315			break;3316 3317		case INTEL_PT_PSB:3318			decoder->state.psb_offset = decoder->pos;3319			decoder->psb_ip = 0;3320			if (intel_pt_psb_with_fup(decoder, &err))3321				return err;3322			err = intel_pt_psb(decoder);3323			if (err == -EAGAIN)3324				goto next;3325			return err;3326 3327		case INTEL_PT_PIP:3328			intel_pt_update_pip(decoder);3329			break;3330 3331		case INTEL_PT_MTC:3332			intel_pt_calc_mtc_timestamp(decoder);3333			if (decoder->period_type != INTEL_PT_PERIOD_MTC)3334				break;3335			/*3336			 * Ensure that there has been an instruction since the3337			 * last MTC.3338			 */3339			if (!decoder->mtc_insn)3340				break;3341			decoder->mtc_insn = false;3342			/* Ensure that there is a timestamp */3343			if (!decoder->timestamp)3344				break;3345			decoder->state.type = INTEL_PT_INSTRUCTION;3346			decoder->state.from_ip = decoder->ip;3347			decoder->state.to_ip = 0;3348			decoder->mtc_insn = false;3349			return 0;3350 3351		case INTEL_PT_TSC:3352			intel_pt_calc_tsc_timestamp(decoder);3353			break;3354 3355		case INTEL_PT_TMA:3356			intel_pt_calc_tma(decoder);3357			break;3358 3359		case INTEL_PT_CYC:3360			intel_pt_calc_cyc_timestamp(decoder);3361			break;3362 3363		case INTEL_PT_CBR:3364			intel_pt_calc_cbr(decoder);3365			if (decoder->cbr != decoder->cbr_seen) {3366				decoder->state.type = 0;3367				return 0;3368			}3369			break;3370 3371		case INTEL_PT_MODE_EXEC:3372			intel_pt_mode_exec(decoder);3373			err = intel_pt_get_next_packet(decoder);3374			if (err)3375				return err;3376			if (decoder->packet.type == INTEL_PT_FUP) {3377				decoder->set_fup_mode_exec = true;3378				no_tip = true;3379			}3380			goto next;3381 3382		case INTEL_PT_MODE_TSX:3383			/* MODE_TSX need not be followed by FUP */3384			if (!decoder->pge || decoder->in_psb) {3385				intel_pt_update_in_tx(decoder);3386				break;3387			}3388			err = intel_pt_mode_tsx(decoder, &no_tip);3389			if (err)3390				return err;3391			goto next;3392 3393		case INTEL_PT_BAD: /* Does not happen */3394			return intel_pt_bug(decoder);3395 3396		case INTEL_PT_PSBEND:3397		case INTEL_PT_VMCS:3398		case INTEL_PT_MNT:3399		case INTEL_PT_PAD:3400			break;3401 3402		case INTEL_PT_PTWRITE_IP:3403			decoder->fup_ptw_payload = decoder->packet.payload;3404			err = intel_pt_get_next_packet(decoder);3405			if (err)3406				return err;3407			if (decoder->packet.type == INTEL_PT_FUP) {3408				decoder->set_fup_ptw = true;3409				no_tip = true;3410			} else {3411				intel_pt_log_at("ERROR: Missing FUP after PTWRITE",3412						decoder->pos);3413			}3414			goto next;3415 3416		case INTEL_PT_PTWRITE:3417			decoder->state.type = INTEL_PT_PTW;3418			decoder->state.from_ip = decoder->ip;3419			decoder->state.to_ip = 0;3420			decoder->state.ptw_payload = decoder->packet.payload;3421			return 0;3422 3423		case INTEL_PT_MWAIT:3424			decoder->fup_mwait_payload = decoder->packet.payload;3425			decoder->set_fup_mwait = true;3426			break;3427 3428		case INTEL_PT_PWRE:3429			if (decoder->set_fup_mwait) {3430				decoder->fup_pwre_payload =3431							decoder->packet.payload;3432				decoder->set_fup_pwre = true;3433				break;3434			}3435			decoder->state.type = INTEL_PT_PWR_ENTRY;3436			decoder->state.from_ip = decoder->ip;3437			decoder->state.to_ip = 0;3438			decoder->state.pwrx_payload = decoder->packet.payload;3439			return 0;3440 3441		case INTEL_PT_EXSTOP_IP:3442			err = intel_pt_get_next_packet(decoder);3443			if (err)3444				return err;3445			if (decoder->packet.type == INTEL_PT_FUP) {3446				decoder->set_fup_exstop = true;3447				no_tip = true;3448			} else {3449				intel_pt_log_at("ERROR: Missing FUP after EXSTOP",3450						decoder->pos);3451			}3452			goto next;3453 3454		case INTEL_PT_EXSTOP:3455			decoder->state.type = INTEL_PT_EX_STOP;3456			decoder->state.from_ip = decoder->ip;3457			decoder->state.to_ip = 0;3458			return 0;3459 3460		case INTEL_PT_PWRX:3461			decoder->state.type = INTEL_PT_PWR_EXIT;3462			decoder->state.from_ip = decoder->ip;3463			decoder->state.to_ip = 0;3464			decoder->state.pwrx_payload = decoder->packet.payload;3465			return 0;3466 3467		case INTEL_PT_BBP:3468			intel_pt_bbp(decoder);3469			break;3470 3471		case INTEL_PT_BIP:3472			intel_pt_bip(decoder);3473			break;3474 3475		case INTEL_PT_BEP:3476			decoder->state.type = INTEL_PT_BLK_ITEMS;3477			decoder->state.from_ip = decoder->ip;3478			decoder->state.to_ip = 0;3479			return 0;3480 3481		case INTEL_PT_BEP_IP:3482			err = intel_pt_get_next_packet(decoder);3483			if (err)3484				return err;3485			if (decoder->packet.type == INTEL_PT_FUP) {3486				decoder->set_fup_bep = true;3487				no_tip = true;3488			} else {3489				intel_pt_log_at("ERROR: Missing FUP after BEP",3490						decoder->pos);3491			}3492			goto next;3493 3494		case INTEL_PT_CFE:3495			decoder->fup_cfe_pkt = decoder->packet;3496			decoder->set_fup_cfe = true;3497			if (!decoder->pge) {3498				intel_pt_fup_event(decoder, true);3499				return 0;3500			}3501			break;3502 3503		case INTEL_PT_CFE_IP:3504			decoder->fup_cfe_pkt = decoder->packet;3505			err = intel_pt_get_next_packet(decoder);3506			if (err)3507				return err;3508			if (decoder->packet.type == INTEL_PT_FUP) {3509				decoder->set_fup_cfe_ip = true;3510				no_tip = true;3511			} else {3512				intel_pt_log_at("ERROR: Missing FUP after CFE",3513						decoder->pos);3514			}3515			goto next;3516 3517		case INTEL_PT_EVD:3518			err = intel_pt_evd(decoder);3519			if (err)3520				return err;3521			break;3522 3523		default:3524			return intel_pt_bug(decoder);3525		}3526	}3527}3528 3529static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)3530{3531	return decoder->packet.count &&3532	       (decoder->have_last_ip || decoder->packet.count == 3 ||3533		decoder->packet.count == 6);3534}3535 3536/* Walk PSB+ packets to get in sync. */3537static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)3538{3539	int err;3540 3541	decoder->in_psb = true;3542 3543	while (1) {3544		err = intel_pt_get_next_packet(decoder);3545		if (err)3546			goto out;3547 3548		switch (decoder->packet.type) {3549		case INTEL_PT_TIP_PGD:3550			decoder->continuous_period = false;3551			fallthrough;3552		case INTEL_PT_TIP_PGE:3553		case INTEL_PT_TIP:3554		case INTEL_PT_PTWRITE:3555		case INTEL_PT_PTWRITE_IP:3556		case INTEL_PT_EXSTOP:3557		case INTEL_PT_EXSTOP_IP:3558		case INTEL_PT_MWAIT:3559		case INTEL_PT_PWRE:3560		case INTEL_PT_PWRX:3561		case INTEL_PT_BBP:3562		case INTEL_PT_BIP:3563		case INTEL_PT_BEP:3564		case INTEL_PT_BEP_IP:3565		case INTEL_PT_CFE:3566		case INTEL_PT_CFE_IP:3567		case INTEL_PT_EVD:3568			intel_pt_log("ERROR: Unexpected packet\n");3569			err = -ENOENT;3570			goto out;3571 3572		case INTEL_PT_FUP:3573			decoder->pge = true;3574			if (intel_pt_have_ip(decoder)) {3575				uint64_t current_ip = decoder->ip;3576 3577				intel_pt_set_ip(decoder);3578				decoder->psb_ip = decoder->ip;3579				if (current_ip)3580					intel_pt_log_to("Setting IP",3581							decoder->ip);3582			}3583			break;3584 3585		case INTEL_PT_MTC:3586			intel_pt_calc_mtc_timestamp(decoder);3587			break;3588 3589		case INTEL_PT_TSC:3590			intel_pt_calc_tsc_timestamp(decoder);3591			break;3592 3593		case INTEL_PT_TMA:3594			intel_pt_calc_tma(decoder);3595			break;3596 3597		case INTEL_PT_CYC:3598			intel_pt_calc_cyc_timestamp(decoder);3599			break;3600 3601		case INTEL_PT_CBR:3602			intel_pt_calc_cbr(decoder);3603			break;3604 3605		case INTEL_PT_PIP:3606			intel_pt_set_pip(decoder);3607			break;3608 3609		case INTEL_PT_MODE_EXEC:3610			intel_pt_mode_exec_status(decoder);3611			break;3612 3613		case INTEL_PT_MODE_TSX:3614			intel_pt_update_in_tx(decoder);3615			break;3616 3617		case INTEL_PT_TRACESTOP:3618			decoder->pge = false;3619			decoder->continuous_period = false;3620			intel_pt_clear_tx_flags(decoder);3621			fallthrough;3622 3623		case INTEL_PT_TNT:3624			decoder->have_tma = false;3625			intel_pt_log("ERROR: Unexpected packet\n");3626			if (decoder->ip)3627				decoder->pkt_state = INTEL_PT_STATE_ERR4;3628			else3629				decoder->pkt_state = INTEL_PT_STATE_ERR3;3630			err = -ENOENT;3631			goto out;3632 3633		case INTEL_PT_BAD: /* Does not happen */3634			err = intel_pt_bug(decoder);3635			goto out;3636 3637		case INTEL_PT_OVF:3638			err = intel_pt_overflow(decoder);3639			goto out;3640 3641		case INTEL_PT_PSBEND:3642			err = 0;3643			goto out;3644 3645		case INTEL_PT_PSB:3646		case INTEL_PT_VMCS:3647		case INTEL_PT_MNT:3648		case INTEL_PT_PAD:3649		default:3650			break;3651		}3652	}3653out:3654	decoder->in_psb = false;3655 3656	return err;3657}3658 3659static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)3660{3661	int err;3662 3663	while (1) {3664		err = intel_pt_get_next_packet(decoder);3665		if (err)3666			return err;3667 3668		switch (decoder->packet.type) {3669		case INTEL_PT_TIP_PGD:3670			decoder->continuous_period = false;3671			decoder->pge = false;3672			if (intel_pt_have_ip(decoder))3673				intel_pt_set_ip(decoder);3674			if (!decoder->ip)3675				break;3676			decoder->state.type |= INTEL_PT_TRACE_END;3677			return 0;3678 3679		case INTEL_PT_TIP_PGE:3680			decoder->pge = true;3681			intel_pt_mtc_cyc_cnt_pge(decoder);3682			if (intel_pt_have_ip(decoder))3683				intel_pt_set_ip(decoder);3684			if (!decoder->ip)3685				break;3686			decoder->state.type |= INTEL_PT_TRACE_BEGIN;3687			return 0;3688 3689		case INTEL_PT_TIP:3690			decoder->pge = true;3691			if (intel_pt_have_ip(decoder))3692				intel_pt_set_ip(decoder);3693			if (!decoder->ip)3694				break;3695			return 0;3696 3697		case INTEL_PT_FUP:3698			if (intel_pt_have_ip(decoder))3699				intel_pt_set_ip(decoder);3700			if (decoder->ip)3701				return 0;3702			break;3703 3704		case INTEL_PT_MTC:3705			intel_pt_calc_mtc_timestamp(decoder);3706			break;3707 3708		case INTEL_PT_TSC:3709			intel_pt_calc_tsc_timestamp(decoder);3710			break;3711 3712		case INTEL_PT_TMA:3713			intel_pt_calc_tma(decoder);3714			break;3715 3716		case INTEL_PT_CYC:3717			intel_pt_calc_cyc_timestamp(decoder);3718			break;3719 3720		case INTEL_PT_CBR:3721			intel_pt_calc_cbr(decoder);3722			break;3723 3724		case INTEL_PT_PIP:3725			intel_pt_set_pip(decoder);3726			break;3727 3728		case INTEL_PT_MODE_EXEC:3729			intel_pt_mode_exec_status(decoder);3730			break;3731 3732		case INTEL_PT_MODE_TSX:3733			intel_pt_update_in_tx(decoder);3734			break;3735 3736		case INTEL_PT_OVF:3737			return intel_pt_overflow(decoder);3738 3739		case INTEL_PT_BAD: /* Does not happen */3740			return intel_pt_bug(decoder);3741 3742		case INTEL_PT_TRACESTOP:3743			decoder->pge = false;3744			decoder->continuous_period = false;3745			intel_pt_clear_tx_flags(decoder);3746			decoder->have_tma = false;3747			break;3748 3749		case INTEL_PT_PSB:3750			decoder->state.psb_offset = decoder->pos;3751			decoder->psb_ip = 0;3752			decoder->last_ip = 0;3753			decoder->have_last_ip = true;3754			intel_pt_clear_stack(&decoder->stack);3755			err = intel_pt_walk_psb(decoder);3756			if (err)3757				return err;3758			decoder->state.type = INTEL_PT_PSB_EVT;3759			decoder->state.from_ip = decoder->psb_ip;3760			decoder->state.to_ip = 0;3761			return 0;3762 3763		case INTEL_PT_TNT:3764		case INTEL_PT_PSBEND:3765		case INTEL_PT_VMCS:3766		case INTEL_PT_MNT:3767		case INTEL_PT_PAD:3768		case INTEL_PT_PTWRITE:3769		case INTEL_PT_PTWRITE_IP:3770		case INTEL_PT_EXSTOP:3771		case INTEL_PT_EXSTOP_IP:3772		case INTEL_PT_MWAIT:3773		case INTEL_PT_PWRE:3774		case INTEL_PT_PWRX:3775		case INTEL_PT_BBP:3776		case INTEL_PT_BIP:3777		case INTEL_PT_BEP:3778		case INTEL_PT_BEP_IP:3779		case INTEL_PT_CFE:3780		case INTEL_PT_CFE_IP:3781		case INTEL_PT_EVD:3782		default:3783			break;3784		}3785	}3786}3787 3788static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)3789{3790	int err;3791 3792	intel_pt_clear_fup_event(decoder);3793	decoder->overflow = false;3794 3795	if (!decoder->branch_enable) {3796		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;3797		decoder->state.type = 0; /* Do not have a sample */3798		return 0;3799	}3800 3801	intel_pt_log("Scanning for full IP\n");3802	err = intel_pt_walk_to_ip(decoder);3803	if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))3804		return err;3805 3806	/* In hop mode, resample to get the to_ip as an "instruction" sample */3807	if (decoder->hop)3808		decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;3809	else3810		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;3811 3812	decoder->state.from_ip = 0;3813	decoder->state.to_ip = decoder->ip;3814	intel_pt_log_to("Setting IP", decoder->ip);3815 3816	return 0;3817}3818 3819static int intel_pt_part_psb(struct intel_pt_decoder *decoder)3820{3821	const unsigned char *end = decoder->buf + decoder->len;3822	size_t i;3823 3824	for (i = INTEL_PT_PSB_LEN - 1; i; i--) {3825		if (i > decoder->len)3826			continue;3827		if (!memcmp(end - i, INTEL_PT_PSB_STR, i))3828			return i;3829	}3830	return 0;3831}3832 3833static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)3834{3835	size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;3836	const char *psb = INTEL_PT_PSB_STR;3837 3838	if (rest_psb > decoder->len ||3839	    memcmp(decoder->buf, psb + part_psb, rest_psb))3840		return 0;3841 3842	return rest_psb;3843}3844 3845static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,3846				  int part_psb)3847{3848	int rest_psb, ret;3849 3850	decoder->pos += decoder->len;3851	decoder->len = 0;3852 3853	ret = intel_pt_get_next_data(decoder, false);3854	if (ret)3855		return ret;3856 3857	rest_psb = intel_pt_rest_psb(decoder, part_psb);3858	if (!rest_psb)3859		return 0;3860 3861	decoder->pos -= part_psb;3862	decoder->next_buf = decoder->buf + rest_psb;3863	decoder->next_len = decoder->len - rest_psb;3864	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);3865	decoder->buf = decoder->temp_buf;3866	decoder->len = INTEL_PT_PSB_LEN;3867 3868	return 0;3869}3870 3871static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)3872{3873	unsigned char *next;3874	int ret;3875 3876	intel_pt_log("Scanning for PSB\n");3877	while (1) {3878		if (!decoder->len) {3879			ret = intel_pt_get_next_data(decoder, false);3880			if (ret)3881				return ret;3882		}3883 3884		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,3885			      INTEL_PT_PSB_LEN);3886		if (!next) {3887			int part_psb;3888 3889			part_psb = intel_pt_part_psb(decoder);3890			if (part_psb) {3891				ret = intel_pt_get_split_psb(decoder, part_psb);3892				if (ret)3893					return ret;3894			} else {3895				decoder->pos += decoder->len;3896				decoder->len = 0;3897			}3898			continue;3899		}3900 3901		decoder->pkt_step = next - decoder->buf;3902		return intel_pt_get_next_packet(decoder);3903	}3904}3905 3906static int intel_pt_sync(struct intel_pt_decoder *decoder)3907{3908	int err;3909 3910	decoder->pge = false;3911	decoder->continuous_period = false;3912	decoder->have_last_ip = false;3913	decoder->last_ip = 0;3914	decoder->psb_ip = 0;3915	decoder->ip = 0;3916	intel_pt_clear_stack(&decoder->stack);3917 3918	err = intel_pt_scan_for_psb(decoder);3919	if (err)3920		return err;3921 3922	if (decoder->vm_time_correlation) {3923		decoder->in_psb = true;3924		if (!decoder->timestamp)3925			decoder->timestamp = 1;3926		decoder->state.type = 0;3927		decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;3928		return 0;3929	}3930 3931	decoder->have_last_ip = true;3932	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;3933 3934	err = intel_pt_walk_psb(decoder);3935	if (err)3936		return err;3937 3938	decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */3939	decoder->state.from_ip = decoder->psb_ip;3940	decoder->state.to_ip = 0;3941 3942	if (decoder->ip) {3943		/*3944		 * In hop mode, resample to get the PSB FUP ip as an3945		 * "instruction" sample.3946		 */3947		if (decoder->hop)3948			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;3949		else3950			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;3951	}3952 3953	return 0;3954}3955 3956static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)3957{3958	uint64_t est = decoder->sample_insn_cnt << 1;3959 3960	if (!decoder->cbr || !decoder->max_non_turbo_ratio)3961		goto out;3962 3963	est *= decoder->max_non_turbo_ratio;3964	est /= decoder->cbr;3965out:3966	return decoder->sample_timestamp + est;3967}3968 3969const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)3970{3971	int err;3972 3973	do {3974		decoder->state.type = INTEL_PT_BRANCH;3975		decoder->state.flags = 0;3976 3977		switch (decoder->pkt_state) {3978		case INTEL_PT_STATE_NO_PSB:3979			err = intel_pt_sync(decoder);3980			break;3981		case INTEL_PT_STATE_NO_IP:3982			decoder->have_last_ip = false;3983			decoder->last_ip = 0;3984			decoder->ip = 0;3985			fallthrough;3986		case INTEL_PT_STATE_ERR_RESYNC:3987			err = intel_pt_sync_ip(decoder);3988			break;3989		case INTEL_PT_STATE_IN_SYNC:3990			err = intel_pt_walk_trace(decoder);3991			break;3992		case INTEL_PT_STATE_TNT:3993		case INTEL_PT_STATE_TNT_CONT:3994			err = intel_pt_walk_tnt(decoder);3995			if (err == -EAGAIN)3996				err = intel_pt_walk_trace(decoder);3997			break;3998		case INTEL_PT_STATE_TIP:3999		case INTEL_PT_STATE_TIP_PGD:4000			err = intel_pt_walk_tip(decoder);4001			break;4002		case INTEL_PT_STATE_FUP:4003			err = intel_pt_walk_fup(decoder);4004			if (err == -EAGAIN)4005				err = intel_pt_walk_fup_tip(decoder);4006			break;4007		case INTEL_PT_STATE_FUP_NO_TIP:4008			err = intel_pt_walk_fup(decoder);4009			if (err == -EAGAIN)4010				err = intel_pt_walk_trace(decoder);4011			break;4012		case INTEL_PT_STATE_FUP_IN_PSB:4013			err = intel_pt_fup_in_psb(decoder);4014			break;4015		case INTEL_PT_STATE_RESAMPLE:4016			err = intel_pt_resample(decoder);4017			break;4018		case INTEL_PT_STATE_VM_TIME_CORRELATION:4019			err = intel_pt_vm_time_correlation(decoder);4020			break;4021		default:4022			err = intel_pt_bug(decoder);4023			break;4024		}4025	} while (err == -ENOLINK);4026 4027	if (err) {4028		decoder->state.err = intel_pt_ext_err(err);4029		if (err != -EOVERFLOW)4030			decoder->state.from_ip = decoder->ip;4031		intel_pt_update_sample_time(decoder);4032		decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;4033		intel_pt_set_nr(decoder);4034	} else {4035		decoder->state.err = 0;4036		if (decoder->cbr != decoder->cbr_seen) {4037			decoder->cbr_seen = decoder->cbr;4038			if (!decoder->state.type) {4039				decoder->state.from_ip = decoder->ip;4040				decoder->state.to_ip = 0;4041			}4042			decoder->state.type |= INTEL_PT_CBR_CHG;4043			decoder->state.cbr_payload = decoder->cbr_payload;4044			decoder->state.cbr = decoder->cbr;4045		}4046		if (intel_pt_sample_time(decoder->pkt_state)) {4047			intel_pt_update_sample_time(decoder);4048			if (decoder->sample_cyc) {4049				decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;4050				decoder->state.flags |= INTEL_PT_SAMPLE_IPC;4051				decoder->sample_cyc = false;4052			}4053		}4054		/*4055		 * When using only TSC/MTC to compute cycles, IPC can be4056		 * sampled as soon as the cycle count changes.4057		 */4058		if (!decoder->have_cyc)4059			decoder->state.flags |= INTEL_PT_SAMPLE_IPC;4060	}4061 4062	 /* Let PSB event always have TSC timestamp */4063	if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)4064		decoder->sample_timestamp = decoder->tsc_timestamp;4065 4066	decoder->state.from_nr = decoder->nr;4067	decoder->state.to_nr = decoder->next_nr;4068	decoder->nr = decoder->next_nr;4069 4070	decoder->state.timestamp = decoder->sample_timestamp;4071	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);4072	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;4073	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;4074 4075	return &decoder->state;4076}4077 4078/**4079 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.4080 * @buf: pointer to buffer pointer4081 * @len: size of buffer4082 *4083 * Updates the buffer pointer to point to the start of the next PSB packet if4084 * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,4085 * @len is adjusted accordingly.4086 *4087 * Return: %true if a PSB packet is found, %false otherwise.4088 */4089static bool intel_pt_next_psb(unsigned char **buf, size_t *len)4090{4091	unsigned char *next;4092 4093	next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);4094	if (next) {4095		*len -= next - *buf;4096		*buf = next;4097		return true;4098	}4099	return false;4100}4101 4102/**4103 * intel_pt_step_psb - move buffer pointer to the start of the following PSB4104 *                     packet.4105 * @buf: pointer to buffer pointer4106 * @len: size of buffer4107 *4108 * Updates the buffer pointer to point to the start of the following PSB packet4109 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer4110 * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.4111 *4112 * Return: %true if a PSB packet is found, %false otherwise.4113 */4114static bool intel_pt_step_psb(unsigned char **buf, size_t *len)4115{4116	unsigned char *next;4117 4118	if (!*len)4119		return false;4120 4121	next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);4122	if (next) {4123		*len -= next - *buf;4124		*buf = next;4125		return true;4126	}4127	return false;4128}4129 4130/**4131 * intel_pt_last_psb - find the last PSB packet in a buffer.4132 * @buf: buffer4133 * @len: size of buffer4134 *4135 * This function finds the last PSB in a buffer.4136 *4137 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.4138 */4139static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)4140{4141	const char *n = INTEL_PT_PSB_STR;4142	unsigned char *p;4143	size_t k;4144 4145	if (len < INTEL_PT_PSB_LEN)4146		return NULL;4147 4148	k = len - INTEL_PT_PSB_LEN + 1;4149	while (1) {4150		p = memrchr(buf, n[0], k);4151		if (!p)4152			return NULL;4153		if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))4154			return p;4155		k = p - buf;4156		if (!k)4157			return NULL;4158	}4159}4160 4161/**4162 * intel_pt_next_tsc - find and return next TSC.4163 * @buf: buffer4164 * @len: size of buffer4165 * @tsc: TSC value returned4166 * @rem: returns remaining size when TSC is found4167 *4168 * Find a TSC packet in @buf and return the TSC value.  This function assumes4169 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a4170 * PSBEND packet is found.4171 *4172 * Return: %true if TSC is found, false otherwise.4173 */4174static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,4175			      size_t *rem)4176{4177	enum intel_pt_pkt_ctx ctx = INTEL_PT_NO_CTX;4178	struct intel_pt_pkt packet;4179	int ret;4180 4181	while (len) {4182		ret = intel_pt_get_packet(buf, len, &packet, &ctx);4183		if (ret <= 0)4184			return false;4185		if (packet.type == INTEL_PT_TSC) {4186			*tsc = packet.payload;4187			*rem = len;4188			return true;4189		}4190		if (packet.type == INTEL_PT_PSBEND)4191			return false;4192		buf += ret;4193		len -= ret;4194	}4195	return false;4196}4197 4198/**4199 * intel_pt_tsc_cmp - compare 7-byte TSCs.4200 * @tsc1: first TSC to compare4201 * @tsc2: second TSC to compare4202 *4203 * This function compares 7-byte TSC values allowing for the possibility that4204 * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped4205 * around so for that purpose this function assumes the absolute difference is4206 * less than half the maximum difference.4207 *4208 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is4209 * after @tsc2.4210 */4211static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)4212{4213	const uint64_t halfway = (1ULL << 55);4214 4215	if (tsc1 == tsc2)4216		return 0;4217 4218	if (tsc1 < tsc2) {4219		if (tsc2 - tsc1 < halfway)4220			return -1;4221		else4222			return 1;4223	} else {4224		if (tsc1 - tsc2 < halfway)4225			return 1;4226		else4227			return -1;4228	}4229}4230 4231#define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)4232 4233/**4234 * adj_for_padding - adjust overlap to account for padding.4235 * @buf_b: second buffer4236 * @buf_a: first buffer4237 * @len_a: size of first buffer4238 *4239 * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap4240 * accordingly.4241 *4242 * Return: A pointer into @buf_b from where non-overlapped data starts4243 */4244static unsigned char *adj_for_padding(unsigned char *buf_b,4245				      unsigned char *buf_a, size_t len_a)4246{4247	unsigned char *p = buf_b - MAX_PADDING;4248	unsigned char *q = buf_a + len_a - MAX_PADDING;4249	int i;4250 4251	for (i = MAX_PADDING; i; i--, p++, q++) {4252		if (*p != *q)4253			break;4254	}4255 4256	return p;4257}4258 4259/**4260 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data4261 *                             using TSC.4262 * @buf_a: first buffer4263 * @len_a: size of first buffer4264 * @buf_b: second buffer4265 * @len_b: size of second buffer4266 * @consecutive: returns true if there is data in buf_b that is consecutive4267 *               to buf_a4268 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling4269 *4270 * If the trace contains TSC we can look at the last TSC of @buf_a and the4271 * first TSC of @buf_b in order to determine if the buffers overlap, and then4272 * walk forward in @buf_b until a later TSC is found.  A precondition is that4273 * @buf_a and @buf_b are positioned at a PSB.4274 *4275 * Return: A pointer into @buf_b from where non-overlapped data starts, or4276 * @buf_b + @len_b if there is no non-overlapped data.4277 */4278static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,4279						size_t len_a,4280						unsigned char *buf_b,4281						size_t len_b, bool *consecutive,4282						bool ooo_tsc)4283{4284	uint64_t tsc_a, tsc_b;4285	unsigned char *p;4286	size_t len, rem_a, rem_b;4287 4288	p = intel_pt_last_psb(buf_a, len_a);4289	if (!p)4290		return buf_b; /* No PSB in buf_a => no overlap */4291 4292	len = len_a - (p - buf_a);4293	if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {4294		/* The last PSB+ in buf_a is incomplete, so go back one more */4295		len_a -= len;4296		p = intel_pt_last_psb(buf_a, len_a);4297		if (!p)4298			return buf_b; /* No full PSB+ => assume no overlap */4299		len = len_a - (p - buf_a);4300		if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))4301			return buf_b; /* No TSC in buf_a => assume no overlap */4302	}4303 4304	while (1) {4305		/* Ignore PSB+ with no TSC */4306		if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {4307			int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);4308 4309			/* Same TSC, so buffers are consecutive */4310			if (!cmp && rem_b >= rem_a) {4311				unsigned char *start;4312 4313				*consecutive = true;4314				start = buf_b + len_b - (rem_b - rem_a);4315				return adj_for_padding(start, buf_a, len_a);4316			}4317			if (cmp < 0 && !ooo_tsc)4318				return buf_b; /* tsc_a < tsc_b => no overlap */4319		}4320 4321		if (!intel_pt_step_psb(&buf_b, &len_b))4322			return buf_b + len_b; /* No PSB in buf_b => no data */4323	}4324}4325 4326/**4327 * intel_pt_find_overlap - determine start of non-overlapped trace data.4328 * @buf_a: first buffer4329 * @len_a: size of first buffer4330 * @buf_b: second buffer4331 * @len_b: size of second buffer4332 * @have_tsc: can use TSC packets to detect overlap4333 * @consecutive: returns true if there is data in buf_b that is consecutive4334 *               to buf_a4335 * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling4336 *4337 * When trace samples or snapshots are recorded there is the possibility that4338 * the data overlaps.  Note that, for the purposes of decoding, data is only4339 * useful if it begins with a PSB packet.4340 *4341 * Return: A pointer into @buf_b from where non-overlapped data starts, or4342 * @buf_b + @len_b if there is no non-overlapped data.4343 */4344unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,4345				     unsigned char *buf_b, size_t len_b,4346				     bool have_tsc, bool *consecutive,4347				     bool ooo_tsc)4348{4349	unsigned char *found;4350 4351	/* Buffer 'b' must start at PSB so throw away everything before that */4352	if (!intel_pt_next_psb(&buf_b, &len_b))4353		return buf_b + len_b; /* No PSB */4354 4355	if (!intel_pt_next_psb(&buf_a, &len_a))4356		return buf_b; /* No overlap */4357 4358	if (have_tsc) {4359		found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,4360						  consecutive, ooo_tsc);4361		if (found)4362			return found;4363	}4364 4365	/*4366	 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,4367	 * we can ignore the first part of buffer 'a'.4368	 */4369	while (len_b < len_a) {4370		if (!intel_pt_step_psb(&buf_a, &len_a))4371			return buf_b; /* No overlap */4372	}4373 4374	/* Now len_b >= len_a */4375	while (1) {4376		/* Potential overlap so check the bytes */4377		found = memmem(buf_a, len_a, buf_b, len_a);4378		if (found) {4379			*consecutive = true;4380			return adj_for_padding(buf_b + len_a, buf_a, len_a);4381		}4382 4383		/* Try again at next PSB in buffer 'a' */4384		if (!intel_pt_step_psb(&buf_a, &len_a))4385			return buf_b; /* No overlap */4386	}4387}4388 4389/**4390 * struct fast_forward_data - data used by intel_pt_ff_cb().4391 * @timestamp: timestamp to fast forward towards4392 * @buf_timestamp: buffer timestamp of last buffer with trace data earlier than4393 *                 the fast forward timestamp.4394 */4395struct fast_forward_data {4396	uint64_t timestamp;4397	uint64_t buf_timestamp;4398};4399 4400/**4401 * intel_pt_ff_cb - fast forward lookahead callback.4402 * @buffer: Intel PT trace buffer4403 * @data: opaque pointer to fast forward data (struct fast_forward_data)4404 *4405 * Determine if @buffer trace is past the fast forward timestamp.4406 *4407 * Return: 1 (stop lookahead) if @buffer trace is past the fast forward4408 *         timestamp, and 0 otherwise.4409 */4410static int intel_pt_ff_cb(struct intel_pt_buffer *buffer, void *data)4411{4412	struct fast_forward_data *d = data;4413	unsigned char *buf;4414	uint64_t tsc;4415	size_t rem;4416	size_t len;4417 4418	buf = (unsigned char *)buffer->buf;4419	len = buffer->len;4420 4421	if (!intel_pt_next_psb(&buf, &len) ||4422	    !intel_pt_next_tsc(buf, len, &tsc, &rem))4423		return 0;4424 4425	tsc = intel_pt_8b_tsc(tsc, buffer->ref_timestamp);4426 4427	intel_pt_log("Buffer 1st timestamp " x64_fmt " ref timestamp " x64_fmt "\n",4428		     tsc, buffer->ref_timestamp);4429 4430	/*4431	 * If the buffer contains a timestamp earlier that the fast forward4432	 * timestamp, then record it, else stop.4433	 */4434	if (tsc < d->timestamp)4435		d->buf_timestamp = buffer->ref_timestamp;4436	else4437		return 1;4438 4439	return 0;4440}4441 4442/**4443 * intel_pt_fast_forward - reposition decoder forwards.4444 * @decoder: Intel PT decoder4445 * @timestamp: timestamp to fast forward towards4446 *4447 * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.4448 *4449 * Return: 0 on success or negative error code on failure.4450 */4451int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)4452{4453	struct fast_forward_data d = { .timestamp = timestamp };4454	unsigned char *buf;4455	size_t len;4456	int err;4457 4458	intel_pt_log("Fast forward towards timestamp " x64_fmt "\n", timestamp);4459 4460	/* Find buffer timestamp of buffer to fast forward to */4461	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);4462	if (err < 0)4463		return err;4464 4465	/* Walk to buffer with same buffer timestamp */4466	if (d.buf_timestamp) {4467		do {4468			decoder->pos += decoder->len;4469			decoder->len = 0;4470			err = intel_pt_get_next_data(decoder, true);4471			/* -ENOLINK means non-consecutive trace */4472			if (err && err != -ENOLINK)4473				return err;4474		} while (decoder->buf_timestamp != d.buf_timestamp);4475	}4476 4477	if (!decoder->buf)4478		return 0;4479 4480	buf = (unsigned char *)decoder->buf;4481	len = decoder->len;4482 4483	if (!intel_pt_next_psb(&buf, &len))4484		return 0;4485 4486	/*4487	 * Walk PSBs while the PSB timestamp is less than the fast forward4488	 * timestamp.4489	 */4490	do {4491		uint64_t tsc;4492		size_t rem;4493 4494		if (!intel_pt_next_tsc(buf, len, &tsc, &rem))4495			break;4496		tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);4497		/*4498		 * A TSC packet can slip past MTC packets but, after fast4499		 * forward, decoding starts at the TSC timestamp. That means4500		 * the timestamps may not be exactly the same as the timestamps4501		 * that would have been decoded without fast forward.4502		 */4503		if (tsc < timestamp) {4504			intel_pt_log("Fast forward to next PSB timestamp " x64_fmt "\n", tsc);4505			decoder->pos += decoder->len - len;4506			decoder->buf = buf;4507			decoder->len = len;4508			intel_pt_reposition(decoder);4509		} else {4510			break;4511		}4512	} while (intel_pt_step_psb(&buf, &len));4513 4514	return 0;4515}4516