470 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2/*3 * Stage 1 of the trace events.4 *5 * Override the macros in the event tracepoint header <trace/events/XXX.h>6 * to include the following:7 *8 * struct trace_event_raw_<call> {9 * struct trace_entry ent;10 * <type> <item>;11 * <type2> <item2>[<len>];12 * [...]13 * };14 *15 * The <type> <item> is created by the __field(type, item) macro or16 * the __array(type2, item2, len) macro.17 * We simply do "type item;", and that will create the fields18 * in the structure.19 */20 21#include <linux/trace_events.h>22 23#ifndef TRACE_SYSTEM_VAR24#define TRACE_SYSTEM_VAR TRACE_SYSTEM25#endif26 27#include "stages/init.h"28 29/*30 * DECLARE_EVENT_CLASS can be used to add a generic function31 * handlers for events. That is, if all events have the same32 * parameters and just have distinct trace points.33 * Each tracepoint can be defined with DEFINE_EVENT and that34 * will map the DECLARE_EVENT_CLASS to the tracepoint.35 *36 * TRACE_EVENT is a one to one mapping between tracepoint and template.37 */38#undef TRACE_EVENT39#define TRACE_EVENT(name, proto, args, tstruct, assign, print) \40 DECLARE_EVENT_CLASS(name, \41 PARAMS(proto), \42 PARAMS(args), \43 PARAMS(tstruct), \44 PARAMS(assign), \45 PARAMS(print)); \46 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args));47 48#include "stages/stage1_struct_define.h"49 50#undef DECLARE_EVENT_CLASS51#define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \52 struct trace_event_raw_##name { \53 struct trace_entry ent; \54 tstruct \55 char __data[]; \56 }; \57 \58 static struct trace_event_class event_class_##name;59 60#undef DEFINE_EVENT61#define DEFINE_EVENT(template, name, proto, args) \62 static struct trace_event_call __used \63 __attribute__((__aligned__(4))) event_##name64 65#undef DEFINE_EVENT_FN66#define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \67 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))68 69#undef DEFINE_EVENT_PRINT70#define DEFINE_EVENT_PRINT(template, name, proto, args, print) \71 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args))72 73/* Callbacks are meaningless to ftrace. */74#undef TRACE_EVENT_FN75#define TRACE_EVENT_FN(name, proto, args, tstruct, \76 assign, print, reg, unreg) \77 TRACE_EVENT(name, PARAMS(proto), PARAMS(args), \78 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \79 80#undef TRACE_EVENT_FN_COND81#define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \82 assign, print, reg, unreg) \83 TRACE_EVENT_CONDITION(name, PARAMS(proto), PARAMS(args), PARAMS(cond), \84 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \85 86#undef TRACE_EVENT_FLAGS87#define TRACE_EVENT_FLAGS(name, value) \88 __TRACE_EVENT_FLAGS(name, value)89 90#undef TRACE_EVENT_PERF_PERM91#define TRACE_EVENT_PERF_PERM(name, expr...) \92 __TRACE_EVENT_PERF_PERM(name, expr)93 94#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)95 96/*97 * Stage 2 of the trace events.98 *99 * Include the following:100 *101 * struct trace_event_data_offsets_<call> {102 * u32 <item1>;103 * u32 <item2>;104 * [...]105 * };106 *107 * The __dynamic_array() macro will create each u32 <item>, this is108 * to keep the offset of each array from the beginning of the event.109 * The size of an array is also encoded, in the higher 16 bits of <item>.110 */111 112#include "stages/stage2_data_offsets.h"113 114#undef DECLARE_EVENT_CLASS115#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \116 struct trace_event_data_offsets_##call { \117 tstruct; \118 };119 120#undef DEFINE_EVENT121#define DEFINE_EVENT(template, name, proto, args)122 123#undef DEFINE_EVENT_PRINT124#define DEFINE_EVENT_PRINT(template, name, proto, args, print)125 126#undef TRACE_EVENT_FLAGS127#define TRACE_EVENT_FLAGS(event, flag)128 129#undef TRACE_EVENT_PERF_PERM130#define TRACE_EVENT_PERF_PERM(event, expr...)131 132#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)133 134/*135 * Stage 3 of the trace events.136 *137 * Override the macros in the event tracepoint header <trace/events/XXX.h>138 * to include the following:139 *140 * enum print_line_t141 * trace_raw_output_<call>(struct trace_iterator *iter, int flags)142 * {143 * struct trace_seq *s = &iter->seq;144 * struct trace_event_raw_<call> *field; <-- defined in stage 1145 * struct trace_seq *p = &iter->tmp_seq;146 *147 * -------(for event)-------148 *149 * struct trace_entry *entry;150 *151 * entry = iter->ent;152 *153 * if (entry->type != event_<call>->event.type) {154 * WARN_ON_ONCE(1);155 * return TRACE_TYPE_UNHANDLED;156 * }157 *158 * field = (typeof(field))entry;159 *160 * trace_seq_init(p);161 * return trace_output_call(iter, <call>, <TP_printk> "\n");162 *163 * ------(or, for event class)------164 *165 * int ret;166 *167 * field = (typeof(field))iter->ent;168 *169 * ret = trace_raw_output_prep(iter, trace_event);170 * if (ret != TRACE_TYPE_HANDLED)171 * return ret;172 *173 * trace_event_printf(iter, <TP_printk> "\n");174 *175 * return trace_handle_return(s);176 * -------177 * }178 *179 * This is the method used to print the raw event to the trace180 * output format. Note, this is not needed if the data is read181 * in binary.182 */183 184#include "stages/stage3_trace_output.h"185 186#undef DECLARE_EVENT_CLASS187#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \188static notrace enum print_line_t \189trace_raw_output_##call(struct trace_iterator *iter, int flags, \190 struct trace_event *trace_event) \191{ \192 struct trace_seq *s = &iter->seq; \193 struct trace_seq __maybe_unused *p = &iter->tmp_seq; \194 struct trace_event_raw_##call *field; \195 int ret; \196 \197 field = (typeof(field))iter->ent; \198 \199 ret = trace_raw_output_prep(iter, trace_event); \200 if (ret != TRACE_TYPE_HANDLED) \201 return ret; \202 \203 trace_event_printf(iter, print); \204 \205 return trace_handle_return(s); \206} \207static struct trace_event_functions trace_event_type_funcs_##call = { \208 .trace = trace_raw_output_##call, \209};210 211#undef DEFINE_EVENT_PRINT212#define DEFINE_EVENT_PRINT(template, call, proto, args, print) \213static notrace enum print_line_t \214trace_raw_output_##call(struct trace_iterator *iter, int flags, \215 struct trace_event *event) \216{ \217 struct trace_event_raw_##template *field; \218 struct trace_entry *entry; \219 struct trace_seq *p = &iter->tmp_seq; \220 \221 entry = iter->ent; \222 \223 if (entry->type != event_##call.event.type) { \224 WARN_ON_ONCE(1); \225 return TRACE_TYPE_UNHANDLED; \226 } \227 \228 field = (typeof(field))entry; \229 \230 trace_seq_init(p); \231 return trace_output_call(iter, #call, print); \232} \233static struct trace_event_functions trace_event_type_funcs_##call = { \234 .trace = trace_raw_output_##call, \235};236 237#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)238 239#include "stages/stage4_event_fields.h"240 241#undef DECLARE_EVENT_CLASS242#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \243static struct trace_event_fields trace_event_fields_##call[] = { \244 tstruct \245 {} };246 247#undef DEFINE_EVENT_PRINT248#define DEFINE_EVENT_PRINT(template, name, proto, args, print)249 250#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)251 252#include "stages/stage5_get_offsets.h"253 254#undef DECLARE_EVENT_CLASS255#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \256static inline notrace int trace_event_get_offsets_##call( \257 struct trace_event_data_offsets_##call *__data_offsets, proto) \258{ \259 int __data_size = 0; \260 int __maybe_unused __item_length; \261 struct trace_event_raw_##call __maybe_unused *entry; \262 \263 tstruct; \264 \265 return __data_size; \266}267 268#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)269 270/*271 * Stage 4 of the trace events.272 *273 * Override the macros in the event tracepoint header <trace/events/XXX.h>274 * to include the following:275 *276 * For those macros defined with TRACE_EVENT:277 *278 * static struct trace_event_call event_<call>;279 *280 * static void trace_event_raw_event_<call>(void *__data, proto)281 * {282 * struct trace_event_file *trace_file = __data;283 * struct trace_event_call *event_call = trace_file->event_call;284 * struct trace_event_data_offsets_<call> __maybe_unused __data_offsets;285 * unsigned long eflags = trace_file->flags;286 * enum event_trigger_type __tt = ETT_NONE;287 * struct ring_buffer_event *event;288 * struct trace_event_raw_<call> *entry; <-- defined in stage 1289 * struct trace_buffer *buffer;290 * unsigned long irq_flags;291 * int __data_size;292 * int pc;293 *294 * if (!(eflags & EVENT_FILE_FL_TRIGGER_COND)) {295 * if (eflags & EVENT_FILE_FL_TRIGGER_MODE)296 * event_triggers_call(trace_file, NULL);297 * if (eflags & EVENT_FILE_FL_SOFT_DISABLED)298 * return;299 * }300 *301 * local_save_flags(irq_flags);302 * pc = preempt_count();303 *304 * __data_size = trace_event_get_offsets_<call>(&__data_offsets, args);305 *306 * event = trace_event_buffer_lock_reserve(&buffer, trace_file,307 * event_<call>->event.type,308 * sizeof(*entry) + __data_size,309 * irq_flags, pc);310 * if (!event)311 * return;312 * entry = ring_buffer_event_data(event);313 *314 * { <assign>; } <-- Here we assign the entries by the __field and315 * __array macros.316 *317 * if (eflags & EVENT_FILE_FL_TRIGGER_COND)318 * __tt = event_triggers_call(trace_file, entry);319 *320 * if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT,321 * &trace_file->flags))322 * ring_buffer_discard_commit(buffer, event);323 * else if (!filter_check_discard(trace_file, entry, buffer, event))324 * trace_buffer_unlock_commit(buffer, event, irq_flags, pc);325 *326 * if (__tt)327 * event_triggers_post_call(trace_file, __tt);328 * }329 *330 * static struct trace_event ftrace_event_type_<call> = {331 * .trace = trace_raw_output_<call>, <-- stage 2332 * };333 *334 * static char print_fmt_<call>[] = <TP_printk>;335 *336 * static struct trace_event_class __used event_class_<template> = {337 * .system = "<system>",338 * .fields_array = trace_event_fields_<call>,339 * .fields = LIST_HEAD_INIT(event_class_##call.fields),340 * .raw_init = trace_event_raw_init,341 * .probe = trace_event_raw_event_##call,342 * .reg = trace_event_reg,343 * };344 *345 * static struct trace_event_call event_<call> = {346 * .class = event_class_<template>,347 * {348 * .tp = &__tracepoint_<call>,349 * },350 * .event = &ftrace_event_type_<call>,351 * .print_fmt = print_fmt_<call>,352 * .flags = TRACE_EVENT_FL_TRACEPOINT,353 * };354 * // its only safe to use pointers when doing linker tricks to355 * // create an array.356 * static struct trace_event_call __used357 * __section("_ftrace_events") *__event_<call> = &event_<call>;358 *359 */360 361#ifdef CONFIG_PERF_EVENTS362 363#define _TRACE_PERF_PROTO(call, proto) \364 static notrace void \365 perf_trace_##call(void *__data, proto);366 367#define _TRACE_PERF_INIT(call) \368 .perf_probe = perf_trace_##call,369 370#else371#define _TRACE_PERF_PROTO(call, proto)372#define _TRACE_PERF_INIT(call)373#endif /* CONFIG_PERF_EVENTS */374 375#include "stages/stage6_event_callback.h"376 377#undef DECLARE_EVENT_CLASS378#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \379 \380static notrace void \381trace_event_raw_event_##call(void *__data, proto) \382{ \383 struct trace_event_file *trace_file = __data; \384 struct trace_event_data_offsets_##call __maybe_unused __data_offsets;\385 struct trace_event_buffer fbuffer; \386 struct trace_event_raw_##call *entry; \387 int __data_size; \388 \389 if (trace_trigger_soft_disabled(trace_file)) \390 return; \391 \392 __data_size = trace_event_get_offsets_##call(&__data_offsets, args); \393 \394 entry = trace_event_buffer_reserve(&fbuffer, trace_file, \395 sizeof(*entry) + __data_size); \396 \397 if (!entry) \398 return; \399 \400 tstruct \401 \402 { assign; } \403 \404 trace_event_buffer_commit(&fbuffer); \405}406/*407 * The ftrace_test_probe is compiled out, it is only here as a build time check408 * to make sure that if the tracepoint handling changes, the ftrace probe will409 * fail to compile unless it too is updated.410 */411 412#undef DEFINE_EVENT413#define DEFINE_EVENT(template, call, proto, args) \414static inline void ftrace_test_probe_##call(void) \415{ \416 check_trace_callback_type_##call(trace_event_raw_event_##template); \417}418 419#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)420 421#include "stages/stage7_class_define.h"422 423#undef DECLARE_EVENT_CLASS424#define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \425_TRACE_PERF_PROTO(call, PARAMS(proto)); \426static char print_fmt_##call[] = print; \427static struct trace_event_class __used __refdata event_class_##call = { \428 .system = TRACE_SYSTEM_STRING, \429 .fields_array = trace_event_fields_##call, \430 .fields = LIST_HEAD_INIT(event_class_##call.fields),\431 .raw_init = trace_event_raw_init, \432 .probe = trace_event_raw_event_##call, \433 .reg = trace_event_reg, \434 _TRACE_PERF_INIT(call) \435};436 437#undef DEFINE_EVENT438#define DEFINE_EVENT(template, call, proto, args) \439 \440static struct trace_event_call __used event_##call = { \441 .class = &event_class_##template, \442 { \443 .tp = &__tracepoint_##call, \444 }, \445 .event.funcs = &trace_event_type_funcs_##template, \446 .print_fmt = print_fmt_##template, \447 .flags = TRACE_EVENT_FL_TRACEPOINT, \448}; \449static struct trace_event_call __used \450__section("_ftrace_events") *__event_##call = &event_##call451 452#undef DEFINE_EVENT_PRINT453#define DEFINE_EVENT_PRINT(template, call, proto, args, print) \454 \455static char print_fmt_##call[] = print; \456 \457static struct trace_event_call __used event_##call = { \458 .class = &event_class_##template, \459 { \460 .tp = &__tracepoint_##call, \461 }, \462 .event.funcs = &trace_event_type_funcs_##call, \463 .print_fmt = print_fmt_##call, \464 .flags = TRACE_EVENT_FL_TRACEPOINT, \465}; \466static struct trace_event_call __used \467__section("_ftrace_events") *__event_##call = &event_##call468 469#include TRACE_INCLUDE(TRACE_INCLUDE_FILE)470