1	// SPDX-License-Identifier: GPL-2.0
2
3	#ifndef _LINUX_KERNEL_TRACE_H
4	#define _LINUX_KERNEL_TRACE_H
5
6	#include <linux/fs.h>
7	#include <linux/atomic.h>
8	#include <linux/sched.h>
9	#include <linux/clocksource.h>
10	#include <linux/ring_buffer.h>
11	#include <linux/mmiotrace.h>
12	#include <linux/tracepoint.h>
13	#include <linux/ftrace.h>
14	#include <linux/trace.h>
15	#include <linux/hw_breakpoint.h>
16	#include <linux/trace_seq.h>
17	#include <linux/trace_events.h>
18	#include <linux/compiler.h>
19	#include <linux/glob.h>
20	#include <linux/irq_work.h>
21	#include <linux/workqueue.h>
22	#include <linux/ctype.h>
23	#include <linux/once_lite.h>
24	#include <linux/ftrace_regs.h>
25	#include <linux/llist.h>
26
27	#include "pid_list.h"
28
29	#ifdef CONFIG_FTRACE_SYSCALLS
30	#include <asm/unistd.h> /* For NR_syscalls */
31	#include <asm/syscall.h> /* some archs define it here */
32	#endif
33
34	#define TRACE_MODE_WRITE 0640
35	#define TRACE_MODE_READ 0440
36
37	enum trace_type {
38	__TRACE_FIRST_TYPE = 0,
39
40	TRACE_FN,
41	TRACE_CTX,
42	TRACE_WAKE,
43	TRACE_STACK,
44	TRACE_PRINT,
45	TRACE_BPRINT,
46	TRACE_MMIO_RW,
47	TRACE_MMIO_MAP,
48	TRACE_BRANCH,
49	TRACE_GRAPH_RET,
50	TRACE_GRAPH_ENT,
51	TRACE_GRAPH_RETADDR_ENT,
52	TRACE_USER_STACK,
53	TRACE_BLK,
54	TRACE_BPUTS,
55	TRACE_HWLAT,
56	TRACE_OSNOISE,
57	TRACE_TIMERLAT,
58	TRACE_RAW_DATA,
59	TRACE_FUNC_REPEATS,
60
61	__TRACE_LAST_TYPE,
62	};
63
64
65	#undef __field
66	#define __field(type, item) type item;
67
68	#undef __field_fn
69	#define __field_fn(type, item) type item;
70
71	#undef __field_packed
72	#define __field_packed(type, item) type item;
73
74	#undef __field_struct
75	#define __field_struct(type, item) __field(type, item)
76
77	#undef __field_desc
78	#define __field_desc(type, container, item)
79
80	#undef __field_desc_packed
81	#define __field_desc_packed(type, container, item)
82
83	#undef __array
84	#define __array(type, item, size) type item[size];
85
86	/*
87	* For backward compatibility, older user space expects to see the
88	* kernel_stack event with a fixed size caller field. But today the fix
89	* size is ignored by the kernel, and the real structure is dynamic.
90	* Expose to user space: "unsigned long caller[8];" but the real structure
91	* will be "unsigned long caller[] __counted_by(size)"
92	*/
93	#undef __stack_array
94	#define __stack_array(type, item, size, field) type item[] __counted_by(field);
95
96	#undef __array_desc
97	#define __array_desc(type, container, item, size)
98
99	#undef __dynamic_array
100	#define __dynamic_array(type, item) type item[];
101
102	#undef __rel_dynamic_array
103	#define __rel_dynamic_array(type, item) type item[];
104
105	#undef F_STRUCT
106	#define F_STRUCT(args...) args
107
108	#undef FTRACE_ENTRY
109	#define FTRACE_ENTRY(name, struct_name, id, tstruct, print) \
110	struct struct_name { \
111	struct trace_entry ent; \
112	tstruct \
113	}
114
115	#undef FTRACE_ENTRY_DUP
116	#define FTRACE_ENTRY_DUP(name, name_struct, id, tstruct, printk)
117
118	#undef FTRACE_ENTRY_REG
119	#define FTRACE_ENTRY_REG(name, struct_name, id, tstruct, print, regfn) \
120	FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print))
121
122	#undef FTRACE_ENTRY_PACKED
123	#define FTRACE_ENTRY_PACKED(name, struct_name, id, tstruct, print) \
124	FTRACE_ENTRY(name, struct_name, id, PARAMS(tstruct), PARAMS(print)) __packed
125
126	#include "trace_entries.h"
127
128	/* Use this for memory failure errors */
129	#define MEM_FAIL(condition, fmt, ...) \
130	DO_ONCE_LITE_IF(condition, pr_err, "ERROR: " fmt, ##__VA_ARGS__)
131
132	#define FAULT_STRING "(fault)"
133
134	#define HIST_STACKTRACE_DEPTH 16
135	#define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
136	#define HIST_STACKTRACE_SKIP 5
137
138	#define SYSCALL_FAULT_USER_MAX 165
139
140	/*
141	* syscalls are special, and need special handling, this is why
142	* they are not included in trace_entries.h
143	*/
144	struct syscall_trace_enter {
145	struct trace_entry ent;
146	int nr;
147	unsigned long args[];
148	};
149
150	struct syscall_trace_exit {
151	struct trace_entry ent;
152	int nr;
153	long ret;
154	};
155
156	struct kprobe_trace_entry_head {
157	struct trace_entry ent;
158	unsigned long ip;
159	};
160
161	struct eprobe_trace_entry_head {
162	struct trace_entry ent;
163	};
164
165	struct kretprobe_trace_entry_head {
166	struct trace_entry ent;
167	unsigned long func;
168	unsigned long ret_ip;
169	};
170
171	struct fentry_trace_entry_head {
172	struct trace_entry ent;
173	unsigned long ip;
174	};
175
176	struct fexit_trace_entry_head {
177	struct trace_entry ent;
178	unsigned long func;
179	unsigned long ret_ip;
180	};
181
182	#define TRACE_BUF_SIZE 1024
183
184	struct trace_array;
185
186	/*
187	* The CPU trace array - it consists of thousands of trace entries
188	* plus some other descriptor data: (for example which task started
189	* the trace, etc.)
190	*/
191	struct trace_array_cpu {
192	local_t disabled;
193
194	unsigned long entries;
195	unsigned long saved_latency;
196	unsigned long critical_start;
197	unsigned long critical_end;
198	unsigned long critical_sequence;
199	unsigned long nice;
200	unsigned long policy;
201	unsigned long rt_priority;
202	unsigned long skipped_entries;
203	u64 preempt_timestamp;
204	pid_t pid;
205	kuid_t uid;
206	char comm[TASK_COMM_LEN];
207
208	#ifdef CONFIG_FUNCTION_TRACER
209	int ftrace_ignore_pid;
210	#endif
211	bool ignore_pid;
212	};
213
214	struct tracer;
215	struct trace_option_dentry;
216
217	struct array_buffer {
218	struct trace_array *tr;
219	struct trace_buffer *buffer;
220	struct trace_array_cpu __percpu *data;
221	u64 time_start;
222	int cpu;
223	};
224
225	#define TRACE_FLAGS_MAX_SIZE 64
226
227	struct trace_options {
228	struct tracer *tracer;
229	struct trace_option_dentry *topts;
230	};
231
232	struct trace_pid_list *trace_pid_list_alloc(void);
233	void trace_pid_list_free(struct trace_pid_list *pid_list);
234	bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid);
235	int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid);
236	int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid);
237	int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid);
238	int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
239	unsigned int *next);
240
241	enum {
242	TRACE_PIDS = BIT(0),
243	TRACE_NO_PIDS = BIT(1),
244	};
245
246	static inline bool pid_type_enabled(int type, struct trace_pid_list *pid_list,
247	struct trace_pid_list *no_pid_list)
248	{
249	/* Return true if the pid list in type has pids */
250	return ((type & TRACE_PIDS) && pid_list) ||
251	((type & TRACE_NO_PIDS) && no_pid_list);
252	}
253
254	static inline bool still_need_pid_events(int type, struct trace_pid_list *pid_list,
255	struct trace_pid_list *no_pid_list)
256	{
257	/*
258	* Turning off what is in @type, return true if the "other"
259	* pid list, still has pids in it.
260	*/
261	return (!(type & TRACE_PIDS) && pid_list) ||
262	(!(type & TRACE_NO_PIDS) && no_pid_list);
263	}
264
265	typedef bool (*cond_update_fn_t)(struct trace_array *tr, void *cond_data);
266
267	/**
268	* struct cond_snapshot - conditional snapshot data and callback
269	*
270	* The cond_snapshot structure encapsulates a callback function and
271	* data associated with the snapshot for a given tracing instance.
272	*
273	* When a snapshot is taken conditionally, by invoking
274	* tracing_snapshot_cond(tr, cond_data), the cond_data passed in is
275	* passed in turn to the cond_snapshot.update() function. That data
276	* can be compared by the update() implementation with the cond_data
277	* contained within the struct cond_snapshot instance associated with
278	* the trace_array. Because the tr->max_lock is held throughout the
279	* update() call, the update() function can directly retrieve the
280	* cond_snapshot and cond_data associated with the per-instance
281	* snapshot associated with the trace_array.
282	*
283	* The cond_snapshot.update() implementation can save data to be
284	* associated with the snapshot if it decides to, and returns 'true'
285	* in that case, or it returns 'false' if the conditional snapshot
286	* shouldn't be taken.
287	*
288	* The cond_snapshot instance is created and associated with the
289	* user-defined cond_data by tracing_cond_snapshot_enable().
290	* Likewise, the cond_snapshot instance is destroyed and is no longer
291	* associated with the trace instance by
292	* tracing_cond_snapshot_disable().
293	*
294	* The method below is required.
295	*
296	* @update: When a conditional snapshot is invoked, the update()
297	* callback function is invoked with the tr->max_lock held. The
298	* update() implementation signals whether or not to actually
299	* take the snapshot, by returning 'true' if so, 'false' if no
300	* snapshot should be taken. Because the max_lock is held for
301	* the duration of update(), the implementation is safe to
302	* directly retrieved and save any implementation data it needs
303	* to in association with the snapshot.
304	*/
305	struct cond_snapshot {
306	void *cond_data;
307	cond_update_fn_t update;
308	};
309
310	/*
311	* struct trace_func_repeats - used to keep track of the consecutive
312	* (on the same CPU) calls of a single function.
313	*/
314	struct trace_func_repeats {
315	unsigned long ip;
316	unsigned long parent_ip;
317	unsigned long count;
318	u64 ts_last_call;
319	};
320
321	struct trace_module_delta {
322	struct rcu_head rcu;
323	long delta[];
324	};
325
326	/*
327	* The trace array - an array of per-CPU trace arrays. This is the
328	* highest level data structure that individual tracers deal with.
329	* They have on/off state as well:
330	*/
331	struct trace_array {
332	struct list_head list;
333	char *name;
334	struct array_buffer array_buffer;
335	#ifdef CONFIG_TRACER_MAX_TRACE
336	/*
337	* The max_buffer is used to snapshot the trace when a maximum
338	* latency is reached, or when the user initiates a snapshot.
339	* Some tracers will use this to store a maximum trace while
340	* it continues examining live traces.
341	*
342	* The buffers for the max_buffer are set up the same as the array_buffer
343	* When a snapshot is taken, the buffer of the max_buffer is swapped
344	* with the buffer of the array_buffer and the buffers are reset for
345	* the array_buffer so the tracing can continue.
346	*/
347	struct array_buffer max_buffer;
348	bool allocated_snapshot;
349	spinlock_t snapshot_trigger_lock;
350	unsigned int snapshot;
351	unsigned long max_latency;
352	#ifdef CONFIG_FSNOTIFY
353	struct dentry *d_max_latency;
354	struct work_struct fsnotify_work;
355	struct irq_work fsnotify_irqwork;
356	#endif
357	#endif
358	/* The below is for memory mapped ring buffer */
359	unsigned int mapped;
360	unsigned long range_addr_start;
361	unsigned long range_addr_size;
362	char *range_name;
363	long text_delta;
364	struct trace_module_delta *module_delta;
365	void *scratch; /* pointer in persistent memory */
366	int scratch_size;
367
368	int buffer_disabled;
369
370	struct trace_pid_list __rcu *filtered_pids;
371	struct trace_pid_list __rcu *filtered_no_pids;
372	/*
373	* max_lock is used to protect the swapping of buffers
374	* when taking a max snapshot. The buffers themselves are
375	* protected by per_cpu spinlocks. But the action of the swap
376	* needs its own lock.
377	*
378	* This is defined as a arch_spinlock_t in order to help
379	* with performance when lockdep debugging is enabled.
380	*
381	* It is also used in other places outside the update_max_tr
382	* so it needs to be defined outside of the
383	* CONFIG_TRACER_MAX_TRACE.
384	*/
385	arch_spinlock_t max_lock;
386	#ifdef CONFIG_FTRACE_SYSCALLS
387	int sys_refcount_enter;
388	int sys_refcount_exit;
389	struct trace_event_file *enter_syscall_files[NR_syscalls];
390	struct trace_event_file *exit_syscall_files[NR_syscalls];
391	#endif
392	int stop_count;
393	int clock_id;
394	int nr_topts;
395	bool clear_trace;
396	int buffer_percent;
397	unsigned int n_err_log_entries;
398	struct tracer *current_trace;
399	struct tracer_flags *current_trace_flags;
400	u64 trace_flags;
401	unsigned char trace_flags_index[TRACE_FLAGS_MAX_SIZE];
402	unsigned int flags;
403	raw_spinlock_t start_lock;
404	const char *system_names;
405	struct list_head err_log;
406	struct dentry *dir;
407	struct dentry *options;
408	struct dentry *percpu_dir;
409	struct eventfs_inode *event_dir;
410	struct trace_options *topts;
411	struct list_head systems;
412	struct list_head events;
413	struct list_head marker_list;
414	struct list_head tracers;
415	struct trace_event_file *trace_marker_file;
416	cpumask_var_t tracing_cpumask; /* only trace on set CPUs */
417	/* one per_cpu trace_pipe can be opened by only one user */
418	cpumask_var_t pipe_cpumask;
419	int ref;
420	int trace_ref;
421	#ifdef CONFIG_MODULES
422	struct list_head mod_events;
423	#endif
424	#ifdef CONFIG_FUNCTION_TRACER
425	struct ftrace_ops *ops;
426	struct trace_pid_list __rcu *function_pids;
427	struct trace_pid_list __rcu *function_no_pids;
428	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
429	struct fgraph_ops *gops;
430	#endif
431	#ifdef CONFIG_DYNAMIC_FTRACE
432	/* All of these are protected by the ftrace_lock */
433	struct list_head func_probes;
434	struct list_head mod_trace;
435	struct list_head mod_notrace;
436	#endif
437	/* function tracing enabled */
438	int function_enabled;
439	#endif
440	int no_filter_buffering_ref;
441	unsigned int syscall_buf_sz;
442	struct list_head hist_vars;
443	#ifdef CONFIG_TRACER_SNAPSHOT
444	struct cond_snapshot *cond_snapshot;
445	#endif
446	struct trace_func_repeats __percpu *last_func_repeats;
447	/*
448	* On boot up, the ring buffer is set to the minimum size, so that
449	* we do not waste memory on systems that are not using tracing.
450	*/
451	bool ring_buffer_expanded;
452	};
453
454	enum {
455	TRACE_ARRAY_FL_GLOBAL = BIT(0),
456	TRACE_ARRAY_FL_BOOT = BIT(1),
457	TRACE_ARRAY_FL_LAST_BOOT = BIT(2),
458	TRACE_ARRAY_FL_MOD_INIT = BIT(3),
459	TRACE_ARRAY_FL_MEMMAP = BIT(4),
460	TRACE_ARRAY_FL_VMALLOC = BIT(5),
461	};
462
463	#ifdef CONFIG_MODULES
464	bool module_exists(const char *module);
465	#else
466	static inline bool module_exists(const char *module)
467	{
468	return false;
469	}
470	#endif
471
472	extern struct list_head ftrace_trace_arrays;
473
474	extern struct mutex trace_types_lock;
475
476	extern int trace_array_get(struct trace_array *tr);
477	extern int tracing_check_open_get_tr(struct trace_array *tr);
478	extern struct trace_array *trace_array_find(const char *instance);
479	extern struct trace_array *trace_array_find_get(const char *instance);
480
481	extern u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe);
482	extern int tracing_set_filter_buffering(struct trace_array *tr, bool set);
483	extern int tracing_set_clock(struct trace_array *tr, const char *clockstr);
484
485	extern bool trace_clock_in_ns(struct trace_array *tr);
486
487	extern unsigned long trace_adjust_address(struct trace_array *tr, unsigned long addr);
488
489	/*
490	* The global tracer (top) should be the first trace array added,
491	* but we check the flag anyway.
492	*/
493	static inline struct trace_array *top_trace_array(void)
494	{
495	struct trace_array *tr;
496
497	if (list_empty(head: &ftrace_trace_arrays))
498	return NULL;
499
500	tr = list_entry(ftrace_trace_arrays.prev,
501	typeof(*tr), list);
502	WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL));
503	return tr;
504	}
505
506	#define FTRACE_CMP_TYPE(var, type) \
507	__builtin_types_compatible_p(typeof(var), type *)
508
509	#undef IF_ASSIGN
510	#define IF_ASSIGN(var, entry, etype, id) \
511	if (FTRACE_CMP_TYPE(var, etype)) { \
512	var = (typeof(var))(entry); \
513	WARN_ON(id != 0 && (entry)->type != id); \
514	break; \
515	}
516
517	/* Will cause compile errors if type is not found. */
518	extern void __ftrace_bad_type(void);
519
520	/*
521	* The trace_assign_type is a verifier that the entry type is
522	* the same as the type being assigned. To add new types simply
523	* add a line with the following format:
524	*
525	* IF_ASSIGN(var, ent, type, id);
526	*
527	* Where "type" is the trace type that includes the trace_entry
528	* as the "ent" item. And "id" is the trace identifier that is
529	* used in the trace_type enum.
530	*
531	* If the type can have more than one id, then use zero.
532	*/
533	#define trace_assign_type(var, ent) \
534	do { \
535	IF_ASSIGN(var, ent, struct ftrace_entry, TRACE_FN); \
536	IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
537	IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
538	IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
539	IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
540	IF_ASSIGN(var, ent, struct bprint_entry, TRACE_BPRINT); \
541	IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS); \
542	IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT); \
543	IF_ASSIGN(var, ent, struct osnoise_entry, TRACE_OSNOISE);\
544	IF_ASSIGN(var, ent, struct timerlat_entry, TRACE_TIMERLAT);\
545	IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\
546	IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \
547	TRACE_MMIO_RW); \
548	IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \
549	TRACE_MMIO_MAP); \
550	IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
551	IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry, \
552	TRACE_GRAPH_ENT); \
553	IF_ASSIGN(var, ent, struct fgraph_retaddr_ent_entry,\
554	TRACE_GRAPH_RETADDR_ENT); \
555	IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \
556	TRACE_GRAPH_RET); \
557	IF_ASSIGN(var, ent, struct func_repeats_entry, \
558	TRACE_FUNC_REPEATS); \
559	__ftrace_bad_type(); \
560	} while (0)
561
562	/*
563	* An option specific to a tracer. This is a boolean value.
564	* The bit is the bit index that sets its value on the
565	* flags value in struct tracer_flags.
566	*/
567	struct tracer_opt {
568	const char *name; /* Will appear on the trace_options file */
569	u32 bit; /* Mask assigned in val field in tracer_flags */
570	};
571
572	/*
573	* The set of specific options for a tracer. Your tracer
574	* have to set the initial value of the flags val.
575	*/
576	struct tracer_flags {
577	u32 val;
578	struct tracer_opt *opts;
579	struct tracer *trace;
580	};
581
582	/* Makes more easy to define a tracer opt */
583	#define TRACER_OPT(s, b) .name = #s, .bit = b
584
585
586	struct trace_option_dentry {
587	struct tracer_opt *opt;
588	struct tracer_flags *flags;
589	struct trace_array *tr;
590	struct dentry *entry;
591	};
592
593	/**
594	* struct tracer - a specific tracer and its callbacks to interact with tracefs
595	* @name: the name chosen to select it on the available_tracers file
596	* @init: called when one switches to this tracer (echo name > current_tracer)
597	* @reset: called when one switches to another tracer
598	* @start: called when tracing is unpaused (echo 1 > tracing_on)
599	* @stop: called when tracing is paused (echo 0 > tracing_on)
600	* @update_thresh: called when tracing_thresh is updated
601	* @open: called when the trace file is opened
602	* @pipe_open: called when the trace_pipe file is opened
603	* @close: called when the trace file is released
604	* @pipe_close: called when the trace_pipe file is released
605	* @read: override the default read callback on trace_pipe
606	* @splice_read: override the default splice_read callback on trace_pipe
607	* @selftest: selftest to run on boot (see trace_selftest.c)
608	* @print_headers: override the first lines that describe your columns
609	* @print_line: callback that prints a trace
610	* @set_flag: signals one of your private flags changed (trace_options file)
611	* @flags: your private flags
612	*/
613	struct tracer {
614	const char *name;
615	int (*init)(struct trace_array *tr);
616	void (*reset)(struct trace_array *tr);
617	void (*start)(struct trace_array *tr);
618	void (*stop)(struct trace_array *tr);
619	int (*update_thresh)(struct trace_array *tr);
620	void (*open)(struct trace_iterator *iter);
621	void (*pipe_open)(struct trace_iterator *iter);
622	void (*close)(struct trace_iterator *iter);
623	void (*pipe_close)(struct trace_iterator *iter);
624	ssize_t (*read)(struct trace_iterator *iter,
625	struct file *filp, char __user *ubuf,
626	size_t cnt, loff_t *ppos);
627	ssize_t (*splice_read)(struct trace_iterator *iter,
628	struct file *filp,
629	loff_t *ppos,
630	struct pipe_inode_info *pipe,
631	size_t len,
632	unsigned int flags);
633	#ifdef CONFIG_FTRACE_STARTUP_TEST
634	int (*selftest)(struct tracer *trace,
635	struct trace_array *tr);
636	#endif
637	void (*print_header)(struct seq_file *m);
638	enum print_line_t (*print_line)(struct trace_iterator *iter);
639	/* If you handled the flag setting, return 0 */
640	int (*set_flag)(struct trace_array *tr,
641	u32 old_flags, u32 bit, int set);
642	/* Return 0 if OK with change, else return non-zero */
643	int (*flag_changed)(struct trace_array *tr,
644	u64 mask, int set);
645	struct tracer *next;
646	struct tracer_flags *flags;
647	struct tracer_flags *default_flags;
648	int enabled;
649	bool print_max;
650	bool allow_instances;
651	#ifdef CONFIG_TRACER_MAX_TRACE
652	bool use_max_tr;
653	#endif
654	/* True if tracer cannot be enabled in kernel param */
655	bool noboot;
656	};
657
658	static inline struct ring_buffer_iter *
659	trace_buffer_iter(struct trace_iterator *iter, int cpu)
660	{
661	return iter->buffer_iter ? iter->buffer_iter[cpu] : NULL;
662	}
663
664	int tracer_init(struct tracer *t, struct trace_array *tr);
665	int tracing_is_enabled(void);
666	void tracing_reset_online_cpus(struct array_buffer *buf);
667	void tracing_reset_all_online_cpus(void);
668	void tracing_reset_all_online_cpus_unlocked(void);
669	int tracing_open_generic(struct inode *inode, struct file *filp);
670	int tracing_open_generic_tr(struct inode *inode, struct file *filp);
671	int tracing_release_generic_tr(struct inode *inode, struct file *file);
672	int tracing_open_file_tr(struct inode *inode, struct file *filp);
673	int tracing_release_file_tr(struct inode *inode, struct file *filp);
674	int tracing_single_release_file_tr(struct inode *inode, struct file *filp);
675	bool tracing_is_disabled(void);
676	bool tracer_tracing_is_on(struct trace_array *tr);
677	void tracer_tracing_on(struct trace_array *tr);
678	void tracer_tracing_off(struct trace_array *tr);
679	void tracer_tracing_disable(struct trace_array *tr);
680	void tracer_tracing_enable(struct trace_array *tr);
681	struct dentry *trace_create_file(const char *name,
682	umode_t mode,
683	struct dentry *parent,
684	void *data,
685	const struct file_operations *fops);
686
687
688	/**
689	* tracer_tracing_is_on_cpu - show real state of ring buffer enabled on for a cpu
690	* @tr : the trace array to know if ring buffer is enabled
691	* @cpu: The cpu buffer to check if enabled
692	*
693	* Shows real state of the per CPU buffer if it is enabled or not.
694	*/
695	static inline bool tracer_tracing_is_on_cpu(struct trace_array *tr, int cpu)
696	{
697	if (tr->array_buffer.buffer)
698	return ring_buffer_record_is_on_cpu(buffer: tr->array_buffer.buffer, cpu);
699	return false;
700	}
701
702	int tracing_init_dentry(void);
703
704	struct ring_buffer_event;
705
706	struct ring_buffer_event *
707	trace_buffer_lock_reserve(struct trace_buffer *buffer,
708	int type,
709	unsigned long len,
710	unsigned int trace_ctx);
711
712	int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu);
713
714	struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
715	struct trace_array_cpu *data);
716
717	struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
718	int *ent_cpu, u64 *ent_ts);
719
720	void trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer,
721	struct ring_buffer_event *event);
722
723	bool trace_is_tracepoint_string(const char *str);
724	const char *trace_event_format(struct trace_iterator *iter, const char *fmt);
725	char *trace_iter_expand_format(struct trace_iterator *iter);
726	bool ignore_event(struct trace_iterator *iter);
727
728	int trace_empty(struct trace_iterator *iter);
729
730	void *trace_find_next_entry_inc(struct trace_iterator *iter);
731
732	void trace_init_global_iter(struct trace_iterator *iter);
733
734	void tracing_iter_reset(struct trace_iterator *iter, int cpu);
735
736	unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu);
737	unsigned long trace_total_entries(struct trace_array *tr);
738
739	void trace_function(struct trace_array *tr,
740	unsigned long ip,
741	unsigned long parent_ip,
742	unsigned int trace_ctx,
743	struct ftrace_regs *regs);
744	void trace_graph_function(struct trace_array *tr,
745	unsigned long ip,
746	unsigned long parent_ip,
747	unsigned int trace_ctx);
748	void trace_latency_header(struct seq_file *m);
749	void trace_default_header(struct seq_file *m);
750	void print_trace_header(struct seq_file *m, struct trace_iterator *iter);
751
752	void trace_graph_return(struct ftrace_graph_ret *trace, struct fgraph_ops *gops,
753	struct ftrace_regs *fregs);
754	int trace_graph_entry(struct ftrace_graph_ent *trace, struct fgraph_ops *gops,
755	struct ftrace_regs *fregs);
756
757	void tracing_start_cmdline_record(void);
758	void tracing_stop_cmdline_record(void);
759	void tracing_start_tgid_record(void);
760	void tracing_stop_tgid_record(void);
761
762	int register_tracer(struct tracer *type);
763	int is_tracing_stopped(void);
764
765	loff_t tracing_lseek(struct file *file, loff_t offset, int whence);
766
767	extern cpumask_var_t __read_mostly tracing_buffer_mask;
768
769	#define for_each_tracing_cpu(cpu) \
770	for_each_cpu(cpu, tracing_buffer_mask)
771
772	extern unsigned long nsecs_to_usecs(unsigned long nsecs);
773
774	extern unsigned long tracing_thresh;
775
776	/* PID filtering */
777
778	bool trace_find_filtered_pid(struct trace_pid_list *filtered_pids,
779	pid_t search_pid);
780	bool trace_ignore_this_task(struct trace_pid_list *filtered_pids,
781	struct trace_pid_list *filtered_no_pids,
782	struct task_struct *task);
783	void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
784	struct task_struct *self,
785	struct task_struct *task);
786	void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos);
787	void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos);
788	int trace_pid_show(struct seq_file *m, void *v);
789	int trace_pid_write(struct trace_pid_list *filtered_pids,
790	struct trace_pid_list **new_pid_list,
791	const char __user *ubuf, size_t cnt);
792
793	#ifdef CONFIG_TRACER_MAX_TRACE
794	void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu,
795	void *cond_data);
796	void update_max_tr_single(struct trace_array *tr,
797	struct task_struct *tsk, int cpu);
798
799	#ifdef CONFIG_FSNOTIFY
800	#define LATENCY_FS_NOTIFY
801	#endif
802	#endif /* CONFIG_TRACER_MAX_TRACE */
803
804	#ifdef LATENCY_FS_NOTIFY
805	void latency_fsnotify(struct trace_array *tr);
806	#else
807	static inline void latency_fsnotify(struct trace_array *tr) { }
808	#endif
809
810	#ifdef CONFIG_STACKTRACE
811	void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, int skip);
812	#else
813	static inline void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
814	int skip)
815	{
816	}
817	#endif /* CONFIG_STACKTRACE */
818
819	void trace_last_func_repeats(struct trace_array *tr,
820	struct trace_func_repeats *last_info,
821	unsigned int trace_ctx);
822
823	extern u64 ftrace_now(int cpu);
824
825	extern void trace_find_cmdline(int pid, char comm[]);
826	extern int trace_find_tgid(int pid);
827	extern void trace_event_follow_fork(struct trace_array *tr, bool enable);
828
829	extern int trace_events_enabled(struct trace_array *tr, const char *system);
830
831	#ifdef CONFIG_DYNAMIC_FTRACE
832	extern unsigned long ftrace_update_tot_cnt;
833	extern unsigned long ftrace_number_of_pages;
834	extern unsigned long ftrace_number_of_groups;
835	extern u64 ftrace_update_time;
836	extern u64 ftrace_total_mod_time;
837	void ftrace_init_trace_array(struct trace_array *tr);
838	#else
839	static inline void ftrace_init_trace_array(struct trace_array *tr) { }
840	#endif
841	#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
842	extern int DYN_FTRACE_TEST_NAME(void);
843	#define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
844	extern int DYN_FTRACE_TEST_NAME2(void);
845
846	extern void trace_set_ring_buffer_expanded(struct trace_array *tr);
847	extern bool tracing_selftest_disabled;
848
849	#ifdef CONFIG_FTRACE_STARTUP_TEST
850	extern void __init disable_tracing_selftest(const char *reason);
851
852	extern int trace_selftest_startup_function(struct tracer *trace,
853	struct trace_array *tr);
854	extern int trace_selftest_startup_function_graph(struct tracer *trace,
855	struct trace_array *tr);
856	extern int trace_selftest_startup_irqsoff(struct tracer *trace,
857	struct trace_array *tr);
858	extern int trace_selftest_startup_preemptoff(struct tracer *trace,
859	struct trace_array *tr);
860	extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
861	struct trace_array *tr);
862	extern int trace_selftest_startup_wakeup(struct tracer *trace,
863	struct trace_array *tr);
864	extern int trace_selftest_startup_nop(struct tracer *trace,
865	struct trace_array *tr);
866	extern int trace_selftest_startup_branch(struct tracer *trace,
867	struct trace_array *tr);
868	/*
869	* Tracer data references selftest functions that only occur
870	* on boot up. These can be __init functions. Thus, when selftests
871	* are enabled, then the tracers need to reference __init functions.
872	*/
873	#define __tracer_data __refdata
874	#else
875	static inline void __init disable_tracing_selftest(const char *reason)
876	{
877	}
878	/* Tracers are seldom changed. Optimize when selftests are disabled. */
879	#define __tracer_data __read_mostly
880	#endif /* CONFIG_FTRACE_STARTUP_TEST */
881
882	extern void *head_page(struct trace_array_cpu *data);
883	extern unsigned long long ns2usecs(u64 nsec);
884
885	__printf(2, 0)
886	int trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
887	__printf(2, 0)
888	int trace_vprintk(unsigned long ip, const char *fmt, va_list args);
889	__printf(3, 0)
890	int trace_array_vprintk(struct trace_array *tr,
891	unsigned long ip, const char *fmt, va_list args);
892	__printf(3, 4)
893	int trace_array_printk_buf(struct trace_buffer *buffer,
894	unsigned long ip, const char *fmt, ...);
895	void trace_printk_seq(struct trace_seq *s);
896	enum print_line_t print_trace_line(struct trace_iterator *iter);
897
898	extern char trace_find_mark(unsigned long long duration);
899
900	struct ftrace_hash;
901
902	struct ftrace_mod_load {
903	struct list_head list;
904	char *func;
905	char *module;
906	int enable;
907	};
908
909	enum {
910	FTRACE_HASH_FL_MOD = (1 << 0),
911	};
912
913	struct ftrace_hash {
914	unsigned long size_bits;
915	struct hlist_head *buckets;
916	unsigned long count;
917	unsigned long flags;
918	struct rcu_head rcu;
919	};
920
921	struct ftrace_func_entry *
922	ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip);
923
924	static __always_inline bool ftrace_hash_empty(struct ftrace_hash *hash)
925	{
926	return !hash || !(hash->count || (hash->flags & FTRACE_HASH_FL_MOD));
927	}
928
929	/* Standard output formatting function used for function return traces */
930	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
931
932	/* Flag options */
933	#define TRACE_GRAPH_PRINT_OVERRUN 0x1
934	#define TRACE_GRAPH_PRINT_CPU 0x2
935	#define TRACE_GRAPH_PRINT_OVERHEAD 0x4
936	#define TRACE_GRAPH_PRINT_PROC 0x8
937	#define TRACE_GRAPH_PRINT_DURATION 0x10
938	#define TRACE_GRAPH_PRINT_ABS_TIME 0x20
939	#define TRACE_GRAPH_PRINT_REL_TIME 0x40
940	#define TRACE_GRAPH_PRINT_IRQS 0x80
941	#define TRACE_GRAPH_PRINT_TAIL 0x100
942	#define TRACE_GRAPH_SLEEP_TIME 0x200
943	#define TRACE_GRAPH_GRAPH_TIME 0x400
944	#define TRACE_GRAPH_PRINT_RETVAL 0x800
945	#define TRACE_GRAPH_PRINT_RETVAL_HEX 0x1000
946	#define TRACE_GRAPH_PRINT_RETADDR 0x2000
947	#define TRACE_GRAPH_ARGS 0x4000
948	#define TRACE_GRAPH_PRINT_FILL_SHIFT 28
949	#define TRACE_GRAPH_PRINT_FILL_MASK (0x3 << TRACE_GRAPH_PRINT_FILL_SHIFT)
950
951	#ifdef CONFIG_FUNCTION_PROFILER
952	extern void ftrace_graph_graph_time_control(bool enable);
953	#else
954	static inline void ftrace_graph_graph_time_control(bool enable) { }
955	#endif
956
957	extern enum print_line_t
958	print_graph_function_flags(struct trace_iterator *iter, u32 flags);
959	extern void print_graph_headers_flags(struct seq_file *s, u32 flags);
960	extern void
961	trace_print_graph_duration(unsigned long long duration, struct trace_seq *s);
962	extern void graph_trace_open(struct trace_iterator *iter);
963	extern void graph_trace_close(struct trace_iterator *iter);
964	extern int __trace_graph_entry(struct trace_array *tr,
965	struct ftrace_graph_ent *trace,
966	unsigned int trace_ctx);
967	extern int __trace_graph_retaddr_entry(struct trace_array *tr,
968	struct ftrace_graph_ent *trace,
969	unsigned int trace_ctx,
970	unsigned long retaddr,
971	struct ftrace_regs *fregs);
972	extern void __trace_graph_return(struct trace_array *tr,
973	struct ftrace_graph_ret *trace,
974	unsigned int trace_ctx,
975	u64 calltime, u64 rettime);
976
977	extern void init_array_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops);
978	extern int allocate_fgraph_ops(struct trace_array *tr, struct ftrace_ops *ops);
979	extern void free_fgraph_ops(struct trace_array *tr);
980
981	enum {
982	TRACE_GRAPH_FL = 1,
983
984	/*
985	* In the very unlikely case that an interrupt came in
986	* at a start of graph tracing, and we want to trace
987	* the function in that interrupt, the depth can be greater
988	* than zero, because of the preempted start of a previous
989	* trace. In an even more unlikely case, depth could be 2
990	* if a softirq interrupted the start of graph tracing,
991	* followed by an interrupt preempting a start of graph
992	* tracing in the softirq, and depth can even be 3
993	* if an NMI came in at the start of an interrupt function
994	* that preempted a softirq start of a function that
995	* preempted normal context!!!! Luckily, it can't be
996	* greater than 3, so the next two bits are a mask
997	* of what the depth is when we set TRACE_GRAPH_FL
998	*/
999
1000	TRACE_GRAPH_DEPTH_START_BIT,
1001	TRACE_GRAPH_DEPTH_END_BIT,
1002
1003	/*
1004	* To implement set_graph_notrace, if this bit is set, we ignore
1005	* function graph tracing of called functions, until the return
1006	* function is called to clear it.
1007	*/
1008	TRACE_GRAPH_NOTRACE_BIT,
1009	};
1010
1011	#define TRACE_GRAPH_NOTRACE (1 << TRACE_GRAPH_NOTRACE_BIT)
1012
1013	static inline unsigned long ftrace_graph_depth(unsigned long *task_var)
1014	{
1015	return (*task_var >> TRACE_GRAPH_DEPTH_START_BIT) & 3;
1016	}
1017
1018	static inline void ftrace_graph_set_depth(unsigned long *task_var, int depth)
1019	{
1020	*task_var &= ~(3 << TRACE_GRAPH_DEPTH_START_BIT);
1021	*task_var |= (depth & 3) << TRACE_GRAPH_DEPTH_START_BIT;
1022	}
1023
1024	#ifdef CONFIG_DYNAMIC_FTRACE
1025	extern struct ftrace_hash __rcu *ftrace_graph_hash;
1026	extern struct ftrace_hash __rcu *ftrace_graph_notrace_hash;
1027
1028	static inline int
1029	ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace)
1030	{
1031	unsigned long addr = trace->func;
1032	int ret = 0;
1033	struct ftrace_hash *hash;
1034
1035	preempt_disable_notrace();
1036
1037	/*
1038	* Have to open code "rcu_dereference_sched()" because the
1039	* function graph tracer can be called when RCU is not
1040	* "watching".
1041	* Protected with schedule_on_each_cpu(ftrace_sync)
1042	*/
1043	hash = rcu_dereference_protected(ftrace_graph_hash, !preemptible());
1044
1045	if (ftrace_hash_empty(hash)) {
1046	ret = 1;
1047	goto out;
1048	}
1049
1050	if (ftrace_lookup_ip(hash, ip: addr)) {
1051	/*
1052	* This needs to be cleared on the return functions
1053	* when the depth is zero.
1054	*/
1055	*task_var |= TRACE_GRAPH_FL;
1056	ftrace_graph_set_depth(task_var, depth: trace->depth);
1057
1058	/*
1059	* If no irqs are to be traced, but a set_graph_function
1060	* is set, and called by an interrupt handler, we still
1061	* want to trace it.
1062	*/
1063	if (in_hardirq())
1064	trace_recursion_set(TRACE_IRQ_BIT);
1065	else
1066	trace_recursion_clear(TRACE_IRQ_BIT);
1067	ret = 1;
1068	}
1069
1070	out:
1071	preempt_enable_notrace();
1072	return ret;
1073	}
1074
1075	static inline void
1076	ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace)
1077	{
1078	unsigned long *task_var = fgraph_get_task_var(gops);
1079
1080	if ((*task_var & TRACE_GRAPH_FL) &&
1081	trace->depth == ftrace_graph_depth(task_var))
1082	*task_var &= ~TRACE_GRAPH_FL;
1083	}
1084
1085	static inline int ftrace_graph_notrace_addr(unsigned long addr)
1086	{
1087	int ret = 0;
1088	struct ftrace_hash *notrace_hash;
1089
1090	preempt_disable_notrace();
1091
1092	/*
1093	* Have to open code "rcu_dereference_sched()" because the
1094	* function graph tracer can be called when RCU is not
1095	* "watching".
1096	* Protected with schedule_on_each_cpu(ftrace_sync)
1097	*/
1098	notrace_hash = rcu_dereference_protected(ftrace_graph_notrace_hash,
1099	!preemptible());
1100
1101	if (ftrace_lookup_ip(hash: notrace_hash, ip: addr))
1102	ret = 1;
1103
1104	preempt_enable_notrace();
1105	return ret;
1106	}
1107	#else
1108	static inline int ftrace_graph_addr(unsigned long *task_var, struct ftrace_graph_ent *trace)
1109	{
1110	return 1;
1111	}
1112
1113	static inline int ftrace_graph_notrace_addr(unsigned long addr)
1114	{
1115	return 0;
1116	}
1117	static inline void ftrace_graph_addr_finish(struct fgraph_ops *gops, struct ftrace_graph_ret *trace)
1118	{ }
1119	#endif /* CONFIG_DYNAMIC_FTRACE */
1120
1121	extern unsigned int fgraph_max_depth;
1122	extern int fgraph_no_sleep_time;
1123	extern bool fprofile_no_sleep_time;
1124
1125	static inline bool
1126	ftrace_graph_ignore_func(struct fgraph_ops *gops, struct ftrace_graph_ent *trace)
1127	{
1128	unsigned long *task_var = fgraph_get_task_var(gops);
1129
1130	/* trace it when it is-nested-in or is a function enabled. */
1131	return !((*task_var & TRACE_GRAPH_FL) ||
1132	ftrace_graph_addr(task_var, trace)) ||
1133	(trace->depth < 0) ||
1134	(fgraph_max_depth && trace->depth >= fgraph_max_depth);
1135	}
1136
1137	void fgraph_init_ops(struct ftrace_ops *dst_ops,
1138	struct ftrace_ops *src_ops);
1139
1140	#else /* CONFIG_FUNCTION_GRAPH_TRACER */
1141	static inline enum print_line_t
1142	print_graph_function_flags(struct trace_iterator *iter, u32 flags)
1143	{
1144	return TRACE_TYPE_UNHANDLED;
1145	}
1146	static inline void free_fgraph_ops(struct trace_array *tr) { }
1147	/* ftrace_ops may not be defined */
1148	#define init_array_fgraph_ops(tr, ops) do { } while (0)
1149	#define allocate_fgraph_ops(tr, ops) ({ 0; })
1150	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1151
1152	extern struct list_head ftrace_pids;
1153
1154	#ifdef CONFIG_FUNCTION_TRACER
1155
1156	#define FTRACE_PID_IGNORE -1
1157	#define FTRACE_PID_TRACE -2
1158
1159	struct ftrace_func_command {
1160	struct list_head list;
1161	char *name;
1162	int (*func)(struct trace_array *tr,
1163	struct ftrace_hash *hash,
1164	char *func, char *cmd,
1165	char *params, int enable);
1166	};
1167	extern bool ftrace_filter_param __initdata;
1168	extern int ftrace_is_dead(void);
1169	int ftrace_create_function_files(struct trace_array *tr,
1170	struct dentry *parent);
1171	void ftrace_destroy_function_files(struct trace_array *tr);
1172	int ftrace_allocate_ftrace_ops(struct trace_array *tr);
1173	void ftrace_free_ftrace_ops(struct trace_array *tr);
1174	void ftrace_init_global_array_ops(struct trace_array *tr);
1175	struct trace_array *trace_get_global_array(void);
1176	void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func);
1177	void ftrace_reset_array_ops(struct trace_array *tr);
1178	void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer);
1179	void ftrace_init_tracefs_toplevel(struct trace_array *tr,
1180	struct dentry *d_tracer);
1181	void ftrace_clear_pids(struct trace_array *tr);
1182	int init_function_trace(void);
1183	void ftrace_pid_follow_fork(struct trace_array *tr, bool enable);
1184	#else
1185	static inline int ftrace_is_dead(void) { return 0; }
1186	static inline int
1187	ftrace_create_function_files(struct trace_array *tr,
1188	struct dentry *parent)
1189	{
1190	return 0;
1191	}
1192	static inline int ftrace_allocate_ftrace_ops(struct trace_array *tr)
1193	{
1194	return 0;
1195	}
1196	static inline void ftrace_free_ftrace_ops(struct trace_array *tr) { }
1197	static inline void ftrace_destroy_function_files(struct trace_array *tr) { }
1198	static inline __init void
1199	ftrace_init_global_array_ops(struct trace_array *tr) { }
1200	static inline void ftrace_reset_array_ops(struct trace_array *tr) { }
1201	static inline void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d) { }
1202	static inline void ftrace_init_tracefs_toplevel(struct trace_array *tr, struct dentry *d) { }
1203	static inline void ftrace_clear_pids(struct trace_array *tr) { }
1204	static inline int init_function_trace(void) { return 0; }
1205	static inline void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) { }
1206	/* ftace_func_t type is not defined, use macro instead of static inline */
1207	#define ftrace_init_array_ops(tr, func) do { } while (0)
1208	#endif /* CONFIG_FUNCTION_TRACER */
1209
1210	#if defined(CONFIG_FUNCTION_TRACER) && defined(CONFIG_DYNAMIC_FTRACE)
1211
1212	struct ftrace_probe_ops {
1213	void (*func)(unsigned long ip,
1214	unsigned long parent_ip,
1215	struct trace_array *tr,
1216	struct ftrace_probe_ops *ops,
1217	void *data);
1218	int (*init)(struct ftrace_probe_ops *ops,
1219	struct trace_array *tr,
1220	unsigned long ip, void *init_data,
1221	void **data);
1222	void (*free)(struct ftrace_probe_ops *ops,
1223	struct trace_array *tr,
1224	unsigned long ip, void *data);
1225	int (*print)(struct seq_file *m,
1226	unsigned long ip,
1227	struct ftrace_probe_ops *ops,
1228	void *data);
1229	};
1230
1231	struct ftrace_func_mapper;
1232	typedef int (*ftrace_mapper_func)(void *data);
1233
1234	struct ftrace_func_mapper *allocate_ftrace_func_mapper(void);
1235	void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper,
1236	unsigned long ip);
1237	int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper,
1238	unsigned long ip, void *data);
1239	void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper,
1240	unsigned long ip);
1241	void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper,
1242	ftrace_mapper_func free_func);
1243
1244	extern int
1245	register_ftrace_function_probe(char *glob, struct trace_array *tr,
1246	struct ftrace_probe_ops *ops, void *data);
1247	extern int
1248	unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr,
1249	struct ftrace_probe_ops *ops);
1250	extern void clear_ftrace_function_probes(struct trace_array *tr);
1251
1252	int register_ftrace_command(struct ftrace_func_command *cmd);
1253	int unregister_ftrace_command(struct ftrace_func_command *cmd);
1254
1255	void ftrace_create_filter_files(struct ftrace_ops *ops,
1256	struct dentry *parent);
1257	void ftrace_destroy_filter_files(struct ftrace_ops *ops);
1258
1259	extern int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
1260	int len, int reset);
1261	extern int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
1262	int len, int reset);
1263	#else
1264	struct ftrace_func_command;
1265
1266	static inline __init int register_ftrace_command(struct ftrace_func_command *cmd)
1267	{
1268	return -EINVAL;
1269	}
1270	static inline __init int unregister_ftrace_command(char *cmd_name)
1271	{
1272	return -EINVAL;
1273	}
1274	static inline void clear_ftrace_function_probes(struct trace_array *tr)
1275	{
1276	}
1277
1278	/*
1279	* The ops parameter passed in is usually undefined.
1280	* This must be a macro.
1281	*/
1282	#define ftrace_create_filter_files(ops, parent) do { } while (0)
1283	#define ftrace_destroy_filter_files(ops) do { } while (0)
1284	#endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */
1285
1286	bool ftrace_event_is_function(struct trace_event_call *call);
1287
1288	/*
1289	* struct trace_parser - servers for reading the user input separated by spaces
1290	* @cont: set if the input is not complete - no final space char was found
1291	* @buffer: holds the parsed user input
1292	* @idx: user input length
1293	* @size: buffer size
1294	*/
1295	struct trace_parser {
1296	bool cont;
1297	bool fail;
1298	char *buffer;
1299	unsigned idx;
1300	unsigned size;
1301	};
1302
1303	static inline bool trace_parser_loaded(struct trace_parser *parser)
1304	{
1305	return !parser->fail && parser->idx != 0;
1306	}
1307
1308	static inline bool trace_parser_cont(struct trace_parser *parser)
1309	{
1310	return parser->cont;
1311	}
1312
1313	static inline void trace_parser_clear(struct trace_parser *parser)
1314	{
1315	parser->cont = false;
1316	parser->idx = 0;
1317	}
1318
1319	static inline void trace_parser_fail(struct trace_parser *parser)
1320	{
1321	parser->fail = true;
1322	}
1323
1324	extern int trace_parser_get_init(struct trace_parser *parser, int size);
1325	extern void trace_parser_put(struct trace_parser *parser);
1326	extern int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1327	size_t cnt, loff_t *ppos);
1328
1329	/*
1330	* Only create function graph options if function graph is configured.
1331	*/
1332	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1333	# define FGRAPH_FLAGS \
1334	C(DISPLAY_GRAPH, "display-graph"),
1335	#else
1336	# define FGRAPH_FLAGS
1337	#endif
1338
1339	#ifdef CONFIG_BRANCH_TRACER
1340	# define BRANCH_FLAGS \
1341	C(BRANCH, "branch"),
1342	#else
1343	# define BRANCH_FLAGS
1344	#endif
1345
1346	#ifdef CONFIG_FUNCTION_TRACER
1347	# define FUNCTION_FLAGS \
1348	C(FUNCTION, "function-trace"), \
1349	C(FUNC_FORK, "function-fork"),
1350	# define FUNCTION_DEFAULT_FLAGS TRACE_ITER(FUNCTION)
1351	#else
1352	# define FUNCTION_FLAGS
1353	# define FUNCTION_DEFAULT_FLAGS 0UL
1354	# define TRACE_ITER_FUNC_FORK_BIT -1
1355	#endif
1356
1357	#ifdef CONFIG_STACKTRACE
1358	# define STACK_FLAGS \
1359	C(STACKTRACE, "stacktrace"),
1360	#else
1361	# define STACK_FLAGS
1362	#endif
1363
1364	#ifdef CONFIG_FUNCTION_PROFILER
1365	# define PROFILER_FLAGS \
1366	C(PROF_TEXT_OFFSET, "prof-text-offset"),
1367	# ifdef CONFIG_FUNCTION_GRAPH_TRACER
1368	# define FPROFILE_FLAGS \
1369	C(GRAPH_TIME, "graph-time"),
1370	# define FPROFILE_DEFAULT_FLAGS TRACE_ITER(GRAPH_TIME)
1371	# else
1372	# define FPROFILE_FLAGS
1373	# define FPROFILE_DEFAULT_FLAGS 0UL
1374	# endif
1375	#else
1376	# define PROFILER_FLAGS
1377	# define FPROFILE_FLAGS
1378	# define FPROFILE_DEFAULT_FLAGS 0UL
1379	# define TRACE_ITER_PROF_TEXT_OFFSET_BIT -1
1380	#endif
1381
1382	/*
1383	* trace_iterator_flags is an enumeration that defines bit
1384	* positions into trace_flags that controls the output.
1385	*
1386	* NOTE: These bits must match the trace_options array in
1387	* trace.c (this macro guarantees it).
1388	*/
1389	#define TRACE_FLAGS \
1390	C(PRINT_PARENT, "print-parent"), \
1391	C(SYM_OFFSET, "sym-offset"), \
1392	C(SYM_ADDR, "sym-addr"), \
1393	C(VERBOSE, "verbose"), \
1394	C(RAW, "raw"), \
1395	C(HEX, "hex"), \
1396	C(BIN, "bin"), \
1397	C(BLOCK, "block"), \
1398	C(FIELDS, "fields"), \
1399	C(PRINTK, "trace_printk"), \
1400	C(ANNOTATE, "annotate"), \
1401	C(USERSTACKTRACE, "userstacktrace"), \
1402	C(SYM_USEROBJ, "sym-userobj"), \
1403	C(PRINTK_MSGONLY, "printk-msg-only"), \
1404	C(CONTEXT_INFO, "context-info"), /* Print pid/cpu/time */ \
1405	C(LATENCY_FMT, "latency-format"), \
1406	C(RECORD_CMD, "record-cmd"), \
1407	C(RECORD_TGID, "record-tgid"), \
1408	C(OVERWRITE, "overwrite"), \
1409	C(STOP_ON_FREE, "disable_on_free"), \
1410	C(IRQ_INFO, "irq-info"), \
1411	C(MARKERS, "markers"), \
1412	C(EVENT_FORK, "event-fork"), \
1413	C(TRACE_PRINTK, "trace_printk_dest"), \
1414	C(COPY_MARKER, "copy_trace_marker"), \
1415	C(PAUSE_ON_TRACE, "pause-on-trace"), \
1416	C(HASH_PTR, "hash-ptr"), /* Print hashed pointer */ \
1417	FUNCTION_FLAGS \
1418	FGRAPH_FLAGS \
1419	STACK_FLAGS \
1420	BRANCH_FLAGS \
1421	PROFILER_FLAGS \
1422	FPROFILE_FLAGS
1423
1424	/*
1425	* By defining C, we can make TRACE_FLAGS a list of bit names
1426	* that will define the bits for the flag masks.
1427	*/
1428	#undef C
1429	#define C(a, b) TRACE_ITER_##a##_BIT
1430
1431	enum trace_iterator_bits {
1432	TRACE_FLAGS
1433	/* Make sure we don't go more than we have bits for */
1434	TRACE_ITER_LAST_BIT
1435	};
1436
1437	/*
1438	* And use TRACE_ITER(flag) to define the bit masks.
1439	*/
1440	#define TRACE_ITER(flag) \
1441	(TRACE_ITER_##flag##_BIT < 0 ? 0 : 1ULL << (TRACE_ITER_##flag##_BIT))
1442
1443	/*
1444	* TRACE_ITER_SYM_MASK masks the options in trace_flags that
1445	* control the output of kernel symbols.
1446	*/
1447	#define TRACE_ITER_SYM_MASK \
1448	(TRACE_ITER(PRINT_PARENT)|TRACE_ITER(SYM_OFFSET)|TRACE_ITER(SYM_ADDR))
1449
1450	extern struct tracer nop_trace;
1451
1452	#ifdef CONFIG_BRANCH_TRACER
1453	extern int enable_branch_tracing(struct trace_array *tr);
1454	extern void disable_branch_tracing(void);
1455	static inline int trace_branch_enable(struct trace_array *tr)
1456	{
1457	if (tr->trace_flags & TRACE_ITER(BRANCH))
1458	return enable_branch_tracing(tr);
1459	return 0;
1460	}
1461	static inline void trace_branch_disable(void)
1462	{
1463	/* due to races, always disable */
1464	disable_branch_tracing();
1465	}
1466	#else
1467	static inline int trace_branch_enable(struct trace_array *tr)
1468	{
1469	return 0;
1470	}
1471	static inline void trace_branch_disable(void)
1472	{
1473	}
1474	#endif /* CONFIG_BRANCH_TRACER */
1475
1476	/* set ring buffers to default size if not already done so */
1477	int tracing_update_buffers(struct trace_array *tr);
1478
1479	union trace_synth_field {
1480	u8 as_u8;
1481	u16 as_u16;
1482	u32 as_u32;
1483	u64 as_u64;
1484	struct trace_dynamic_info as_dynamic;
1485	};
1486
1487	struct ftrace_event_field {
1488	struct list_head link;
1489	const char *name;
1490	const char *type;
1491	int filter_type;
1492	int offset;
1493	int size;
1494	unsigned int is_signed:1;
1495	unsigned int needs_test:1;
1496	int len;
1497	};
1498
1499	struct prog_entry;
1500
1501	struct event_filter {
1502	struct prog_entry __rcu *prog;
1503	char *filter_string;
1504	};
1505
1506	struct event_subsystem {
1507	struct list_head list;
1508	const char *name;
1509	struct event_filter *filter;
1510	int ref_count;
1511	};
1512
1513	struct trace_subsystem_dir {
1514	struct list_head list;
1515	struct event_subsystem *subsystem;
1516	struct trace_array *tr;
1517	struct eventfs_inode *ei;
1518	int ref_count;
1519	int nr_events;
1520	};
1521
1522	void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1523	struct trace_buffer *buffer,
1524	struct ring_buffer_event *event,
1525	unsigned int trcace_ctx,
1526	struct pt_regs *regs);
1527
1528	static inline void trace_buffer_unlock_commit(struct trace_array *tr,
1529	struct trace_buffer *buffer,
1530	struct ring_buffer_event *event,
1531	unsigned int trace_ctx)
1532	{
1533	trace_buffer_unlock_commit_regs(tr, buffer, event, trcace_ctx: trace_ctx, NULL);
1534	}
1535
1536	DECLARE_PER_CPU(bool, trace_taskinfo_save);
1537	int trace_save_cmdline(struct task_struct *tsk);
1538	int trace_create_savedcmd(void);
1539	int trace_alloc_tgid_map(void);
1540	void trace_free_saved_cmdlines_buffer(void);
1541
1542	extern const struct file_operations tracing_saved_cmdlines_fops;
1543	extern const struct file_operations tracing_saved_tgids_fops;
1544	extern const struct file_operations tracing_saved_cmdlines_size_fops;
1545
1546	DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1547	DECLARE_PER_CPU(int, trace_buffered_event_cnt);
1548	void trace_buffered_event_disable(void);
1549	void trace_buffered_event_enable(void);
1550
1551	void early_enable_events(struct trace_array *tr, char *buf, bool disable_first);
1552
1553	struct trace_user_buf;
1554	struct trace_user_buf_info {
1555	struct trace_user_buf __percpu *tbuf;
1556	size_t size;
1557	int ref;
1558	};
1559
1560	typedef int (*trace_user_buf_copy)(char *dst, const char __user *src,
1561	size_t size, void *data);
1562	int trace_user_fault_init(struct trace_user_buf_info *tinfo, size_t size);
1563	int trace_user_fault_get(struct trace_user_buf_info *tinfo);
1564	int trace_user_fault_put(struct trace_user_buf_info *tinfo);
1565	void trace_user_fault_destroy(struct trace_user_buf_info *tinfo);
1566	char *trace_user_fault_read(struct trace_user_buf_info *tinfo,
1567	const char __user *ptr, size_t size,
1568	trace_user_buf_copy copy_func, void *data);
1569
1570	static inline void
1571	__trace_event_discard_commit(struct trace_buffer *buffer,
1572	struct ring_buffer_event *event)
1573	{
1574	if (this_cpu_read(trace_buffered_event) == event) {
1575	/* Simply release the temp buffer and enable preemption */
1576	this_cpu_dec(trace_buffered_event_cnt);
1577	preempt_enable_notrace();
1578	return;
1579	}
1580	/* ring_buffer_discard_commit() enables preemption */
1581	ring_buffer_discard_commit(buffer, event);
1582	}
1583
1584	/*
1585	* Helper function for event_trigger_unlock_commit{_regs}().
1586	* If there are event triggers attached to this event that requires
1587	* filtering against its fields, then they will be called as the
1588	* entry already holds the field information of the current event.
1589	*
1590	* It also checks if the event should be discarded or not.
1591	* It is to be discarded if the event is soft disabled and the
1592	* event was only recorded to process triggers, or if the event
1593	* filter is active and this event did not match the filters.
1594	*
1595	* Returns true if the event is discarded, false otherwise.
1596	*/
1597	static inline bool
1598	__event_trigger_test_discard(struct trace_event_file *file,
1599	struct trace_buffer *buffer,
1600	struct ring_buffer_event *event,
1601	void *entry,
1602	enum event_trigger_type *tt)
1603	{
1604	unsigned long eflags = file->flags;
1605
1606	if (eflags & EVENT_FILE_FL_TRIGGER_COND)
1607	*tt = event_triggers_call(file, buffer, rec: entry, event);
1608
1609	if (likely(!(file->flags & (EVENT_FILE_FL_SOFT_DISABLED |
1610	EVENT_FILE_FL_FILTERED |
1611	EVENT_FILE_FL_PID_FILTER))))
1612	return false;
1613
1614	if (file->flags & EVENT_FILE_FL_SOFT_DISABLED)
1615	goto discard;
1616
1617	if (file->flags & EVENT_FILE_FL_FILTERED &&
1618	!filter_match_preds(filter: file->filter, rec: entry))
1619	goto discard;
1620
1621	if ((file->flags & EVENT_FILE_FL_PID_FILTER) &&
1622	trace_event_ignore_this_pid(trace_file: file))
1623	goto discard;
1624
1625	return false;
1626	discard:
1627	__trace_event_discard_commit(buffer, event);
1628	return true;
1629	}
1630
1631	/**
1632	* event_trigger_unlock_commit - handle triggers and finish event commit
1633	* @file: The file pointer associated with the event
1634	* @buffer: The ring buffer that the event is being written to
1635	* @event: The event meta data in the ring buffer
1636	* @entry: The event itself
1637	* @trace_ctx: The tracing context flags.
1638	*
1639	* This is a helper function to handle triggers that require data
1640	* from the event itself. It also tests the event against filters and
1641	* if the event is soft disabled and should be discarded.
1642	*/
1643	static inline void
1644	event_trigger_unlock_commit(struct trace_event_file *file,
1645	struct trace_buffer *buffer,
1646	struct ring_buffer_event *event,
1647	void *entry, unsigned int trace_ctx)
1648	{
1649	enum event_trigger_type tt = ETT_NONE;
1650
1651	if (!__event_trigger_test_discard(file, buffer, event, entry, tt: &tt))
1652	trace_buffer_unlock_commit(tr: file->tr, buffer, event, trace_ctx);
1653
1654	if (tt)
1655	event_triggers_post_call(file, tt);
1656	}
1657
1658	#define FILTER_PRED_INVALID ((unsigned short)-1)
1659	#define FILTER_PRED_IS_RIGHT (1 << 15)
1660	#define FILTER_PRED_FOLD (1 << 15)
1661
1662	/*
1663	* The max preds is the size of unsigned short with
1664	* two flags at the MSBs. One bit is used for both the IS_RIGHT
1665	* and FOLD flags. The other is reserved.
1666	*
1667	* 2^14 preds is way more than enough.
1668	*/
1669	#define MAX_FILTER_PRED 16384
1670
1671	struct filter_pred;
1672	struct regex;
1673
1674	typedef int (*regex_match_func)(char *str, struct regex *r, int len);
1675
1676	enum regex_type {
1677	MATCH_FULL = 0,
1678	MATCH_FRONT_ONLY,
1679	MATCH_MIDDLE_ONLY,
1680	MATCH_END_ONLY,
1681	MATCH_GLOB,
1682	MATCH_INDEX,
1683	};
1684
1685	struct regex {
1686	char pattern[MAX_FILTER_STR_VAL];
1687	int len;
1688	int field_len;
1689	regex_match_func match;
1690	};
1691
1692	static inline bool is_string_field(struct ftrace_event_field *field)
1693	{
1694	return field->filter_type == FILTER_DYN_STRING ||
1695	field->filter_type == FILTER_RDYN_STRING ||
1696	field->filter_type == FILTER_STATIC_STRING ||
1697	field->filter_type == FILTER_PTR_STRING ||
1698	field->filter_type == FILTER_COMM;
1699	}
1700
1701	static inline bool is_function_field(struct ftrace_event_field *field)
1702	{
1703	return field->filter_type == FILTER_TRACE_FN;
1704	}
1705
1706	extern enum regex_type
1707	filter_parse_regex(char *buff, int len, char **search, int *not);
1708	extern void print_event_filter(struct trace_event_file *file,
1709	struct trace_seq *s);
1710	extern int apply_event_filter(struct trace_event_file *file,
1711	char *filter_string);
1712	extern int apply_subsystem_event_filter(struct trace_subsystem_dir *dir,
1713	char *filter_string);
1714	extern void print_subsystem_event_filter(struct event_subsystem *system,
1715	struct trace_seq *s);
1716	extern int filter_assign_type(const char *type);
1717	extern int create_event_filter(struct trace_array *tr,
1718	struct trace_event_call *call,
1719	char *filter_str, bool set_str,
1720	struct event_filter **filterp);
1721	extern void free_event_filter(struct event_filter *filter);
1722
1723	struct ftrace_event_field *
1724	trace_find_event_field(struct trace_event_call *call, char *name);
1725
1726	extern void trace_event_enable_cmd_record(bool enable);
1727	extern void trace_event_enable_tgid_record(bool enable);
1728
1729	extern int event_trace_init(void);
1730	extern int init_events(void);
1731	extern int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr);
1732	extern int event_trace_del_tracer(struct trace_array *tr);
1733	extern void __trace_early_add_events(struct trace_array *tr);
1734
1735	extern struct trace_event_file *__find_event_file(struct trace_array *tr,
1736	const char *system,
1737	const char *event);
1738	extern struct trace_event_file *find_event_file(struct trace_array *tr,
1739	const char *system,
1740	const char *event);
1741
1742	static inline void *event_file_data(struct file *filp)
1743	{
1744	return READ_ONCE(file_inode(filp)->i_private);
1745	}
1746
1747	extern struct mutex event_mutex;
1748	extern struct list_head ftrace_events;
1749
1750	/*
1751	* When the trace_event_file is the filp->i_private pointer,
1752	* it must be taken under the event_mutex lock, and then checked
1753	* if the EVENT_FILE_FL_FREED flag is set. If it is, then the
1754	* data pointed to by the trace_event_file can not be trusted.
1755	*
1756	* Use the event_file_file() to access the trace_event_file from
1757	* the filp the first time under the event_mutex and check for
1758	* NULL. If it is needed to be retrieved again and the event_mutex
1759	* is still held, then the event_file_data() can be used and it
1760	* is guaranteed to be valid.
1761	*/
1762	static inline struct trace_event_file *event_file_file(struct file *filp)
1763	{
1764	struct trace_event_file *file;
1765
1766	lockdep_assert_held(&event_mutex);
1767	file = READ_ONCE(file_inode(filp)->i_private);
1768	if (!file || file->flags & EVENT_FILE_FL_FREED)
1769	return NULL;
1770	return file;
1771	}
1772
1773	extern const struct file_operations event_trigger_fops;
1774	extern const struct file_operations event_hist_fops;
1775	extern const struct file_operations event_hist_debug_fops;
1776	extern const struct file_operations event_inject_fops;
1777
1778	#ifdef CONFIG_HIST_TRIGGERS
1779	extern int register_trigger_hist_cmd(void);
1780	extern int register_trigger_hist_enable_disable_cmds(void);
1781	#else
1782	static inline int register_trigger_hist_cmd(void) { return 0; }
1783	static inline int register_trigger_hist_enable_disable_cmds(void) { return 0; }
1784	#endif
1785
1786	extern int register_trigger_cmds(void);
1787	extern void clear_event_triggers(struct trace_array *tr);
1788
1789	enum {
1790	EVENT_TRIGGER_FL_PROBE = BIT(0),
1791	EVENT_TRIGGER_FL_COUNT = BIT(1),
1792	};
1793
1794	struct event_trigger_data {
1795	unsigned long count;
1796	int ref;
1797	int flags;
1798	struct event_command *cmd_ops;
1799	struct event_filter __rcu *filter;
1800	char *filter_str;
1801	void *private_data;
1802	bool paused;
1803	bool paused_tmp;
1804	struct list_head list;
1805	char *name;
1806	struct list_head named_list;
1807	struct event_trigger_data *named_data;
1808	struct llist_node llist;
1809	};
1810
1811	/* Avoid typos */
1812	#define ENABLE_EVENT_STR "enable_event"
1813	#define DISABLE_EVENT_STR "disable_event"
1814	#define ENABLE_HIST_STR "enable_hist"
1815	#define DISABLE_HIST_STR "disable_hist"
1816
1817	struct enable_trigger_data {
1818	struct trace_event_file *file;
1819	bool enable;
1820	bool hist;
1821	};
1822
1823	bool event_trigger_count(struct event_trigger_data *data,
1824	struct trace_buffer *buffer, void *rec,
1825	struct ring_buffer_event *event);
1826
1827	extern int event_enable_trigger_print(struct seq_file *m,
1828	struct event_trigger_data *data);
1829	extern void event_enable_trigger_free(struct event_trigger_data *data);
1830	extern int event_enable_trigger_parse(struct event_command *cmd_ops,
1831	struct trace_event_file *file,
1832	char *glob, char *cmd,
1833	char *param_and_filter);
1834	extern int event_enable_register_trigger(char *glob,
1835	struct event_trigger_data *data,
1836	struct trace_event_file *file);
1837	extern void event_enable_unregister_trigger(char *glob,
1838	struct event_trigger_data *test,
1839	struct trace_event_file *file);
1840	extern struct event_trigger_data *
1841	trigger_data_alloc(struct event_command *cmd_ops, char *cmd, char *param,
1842	void *private_data);
1843	extern void trigger_data_free(struct event_trigger_data *data);
1844	extern int event_trigger_init(struct event_trigger_data *data);
1845	extern int trace_event_trigger_enable_disable(struct trace_event_file *file,
1846	int trigger_enable);
1847	extern void update_cond_flag(struct trace_event_file *file);
1848	extern int set_trigger_filter(char *filter_str,
1849	struct event_trigger_data *trigger_data,
1850	struct trace_event_file *file);
1851	extern struct event_trigger_data *find_named_trigger(const char *name);
1852	extern bool is_named_trigger(struct event_trigger_data *test);
1853	extern int save_named_trigger(const char *name,
1854	struct event_trigger_data *data);
1855	extern void del_named_trigger(struct event_trigger_data *data);
1856	extern void pause_named_trigger(struct event_trigger_data *data);
1857	extern void unpause_named_trigger(struct event_trigger_data *data);
1858	extern void set_named_trigger_data(struct event_trigger_data *data,
1859	struct event_trigger_data *named_data);
1860	extern struct event_trigger_data *
1861	get_named_trigger_data(struct event_trigger_data *data);
1862	extern int register_event_command(struct event_command *cmd);
1863	extern int unregister_event_command(struct event_command *cmd);
1864	extern int register_trigger_hist_enable_disable_cmds(void);
1865	extern bool event_trigger_check_remove(const char *glob);
1866	extern bool event_trigger_empty_param(const char *param);
1867	extern int event_trigger_separate_filter(char *param_and_filter, char **param,
1868	char **filter, bool param_required);
1869	extern int event_trigger_parse_num(char *trigger,
1870	struct event_trigger_data *trigger_data);
1871	extern int event_trigger_set_filter(struct event_command *cmd_ops,
1872	struct trace_event_file *file,
1873	char *param,
1874	struct event_trigger_data *trigger_data);
1875	extern void event_trigger_reset_filter(struct event_command *cmd_ops,
1876	struct event_trigger_data *trigger_data);
1877	extern int event_trigger_register(struct event_command *cmd_ops,
1878	struct trace_event_file *file,
1879	char *glob,
1880	struct event_trigger_data *trigger_data);
1881	extern void event_trigger_unregister(struct event_command *cmd_ops,
1882	struct trace_event_file *file,
1883	char *glob,
1884	struct event_trigger_data *trigger_data);
1885
1886	extern void event_file_get(struct trace_event_file *file);
1887	extern void event_file_put(struct trace_event_file *file);
1888
1889	/**
1890	* struct event_command - callbacks and data members for event commands
1891	*
1892	* Event commands are invoked by users by writing the command name
1893	* into the 'trigger' file associated with a trace event. The
1894	* parameters associated with a specific invocation of an event
1895	* command are used to create an event trigger instance, which is
1896	* added to the list of trigger instances associated with that trace
1897	* event. When the event is hit, the set of triggers associated with
1898	* that event is invoked.
1899	*
1900	* The data members in this structure provide per-event command data
1901	* for various event commands.
1902	*
1903	* All the data members below, except for @post_trigger, must be set
1904	* for each event command.
1905	*
1906	* @name: The unique name that identifies the event command. This is
1907	* the name used when setting triggers via trigger files.
1908	*
1909	* @trigger_type: A unique id that identifies the event command
1910	* 'type'. This value has two purposes, the first to ensure that
1911	* only one trigger of the same type can be set at a given time
1912	* for a particular event e.g. it doesn't make sense to have both
1913	* a traceon and traceoff trigger attached to a single event at
1914	* the same time, so traceon and traceoff have the same type
1915	* though they have different names. The @trigger_type value is
1916	* also used as a bit value for deferring the actual trigger
1917	* action until after the current event is finished. Some
1918	* commands need to do this if they themselves log to the trace
1919	* buffer (see the @post_trigger() member below). @trigger_type
1920	* values are defined by adding new values to the trigger_type
1921	* enum in include/linux/trace_events.h.
1922	*
1923	* @flags: See the enum event_command_flags below.
1924	*
1925	* All the methods below, except for @set_filter() and @unreg_all(),
1926	* must be implemented.
1927	*
1928	* @parse: The callback function responsible for parsing and
1929	* registering the trigger written to the 'trigger' file by the
1930	* user. It allocates the trigger instance and registers it with
1931	* the appropriate trace event. It makes use of the other
1932	* event_command callback functions to orchestrate this, and is
1933	* usually implemented by the generic utility function
1934	* @event_trigger_callback() (see trace_event_triggers.c).
1935	*
1936	* @reg: Adds the trigger to the list of triggers associated with the
1937	* event, and enables the event trigger itself, after
1938	* initializing it (via the event_command @init() function).
1939	* This is also where commands can use the @trigger_type value to
1940	* make the decision as to whether or not multiple instances of
1941	* the trigger should be allowed. This is usually implemented by
1942	* the generic utility function @register_trigger() (see
1943	* trace_event_triggers.c).
1944	*
1945	* @unreg: Removes the trigger from the list of triggers associated
1946	* with the event, and disables the event trigger itself, after
1947	* initializing it (via the event_command @free() function).
1948	* This is usually implemented by the generic utility function
1949	* @unregister_trigger() (see trace_event_triggers.c).
1950	*
1951	* @unreg_all: An optional function called to remove all the triggers
1952	* from the list of triggers associated with the event. Called
1953	* when a trigger file is opened in truncate mode.
1954	*
1955	* @set_filter: An optional function called to parse and set a filter
1956	* for the trigger. If no @set_filter() method is set for the
1957	* event command, filters set by the user for the command will be
1958	* ignored. This is usually implemented by the generic utility
1959	* function @set_trigger_filter() (see trace_event_triggers.c).
1960	*
1961	* All the methods below, except for @init() and @free(), must be
1962	* implemented.
1963	*
1964	* @trigger: The trigger 'probe' function called when the triggering
1965	* event occurs. The data passed into this callback is the data
1966	* that was supplied to the event_command @reg() function that
1967	* registered the trigger (see struct event_command) along with
1968	* the trace record, rec.
1969	*
1970	* @count_func: If defined and a numeric parameter is passed to the
1971	* trigger, then this function will be called before @trigger
1972	* is called. If this function returns false, then @trigger is not
1973	* executed.
1974	*
1975	* @init: An optional initialization function called for the trigger
1976	* when the trigger is registered (via the event_command reg()
1977	* function). This can be used to perform per-trigger
1978	* initialization such as incrementing a per-trigger reference
1979	* count, for instance. This is usually implemented by the
1980	* generic utility function @event_trigger_init() (see
1981	* trace_event_triggers.c).
1982	*
1983	* @free: An optional de-initialization function called for the
1984	* trigger when the trigger is unregistered (via the
1985	* event_command @reg() function). This can be used to perform
1986	* per-trigger de-initialization such as decrementing a
1987	* per-trigger reference count and freeing corresponding trigger
1988	* data, for instance. This is usually implemented by the
1989	* generic utility function @event_trigger_free() (see
1990	* trace_event_triggers.c).
1991	*
1992	* @print: The callback function invoked to have the trigger print
1993	* itself. This is usually implemented by a wrapper function
1994	* that calls the generic utility function @event_trigger_print()
1995	* (see trace_event_triggers.c).
1996	*/
1997	struct event_command {
1998	struct list_head list;
1999	char *name;
2000	enum event_trigger_type trigger_type;
2001	int flags;
2002	int (*parse)(struct event_command *cmd_ops,
2003	struct trace_event_file *file,
2004	char *glob, char *cmd,
2005	char *param_and_filter);
2006	int (*reg)(char *glob,
2007	struct event_trigger_data *data,
2008	struct trace_event_file *file);
2009	void (*unreg)(char *glob,
2010	struct event_trigger_data *data,
2011	struct trace_event_file *file);
2012	void (*unreg_all)(struct trace_event_file *file);
2013	int (*set_filter)(char *filter_str,
2014	struct event_trigger_data *data,
2015	struct trace_event_file *file);
2016	void (*trigger)(struct event_trigger_data *data,
2017	struct trace_buffer *buffer,
2018	void *rec,
2019	struct ring_buffer_event *rbe);
2020	bool (*count_func)(struct event_trigger_data *data,
2021	struct trace_buffer *buffer,
2022	void *rec,
2023	struct ring_buffer_event *rbe);
2024	int (*init)(struct event_trigger_data *data);
2025	void (*free)(struct event_trigger_data *data);
2026	int (*print)(struct seq_file *m,
2027	struct event_trigger_data *data);
2028	};
2029
2030	/**
2031	* enum event_command_flags - flags for struct event_command
2032	*
2033	* @POST_TRIGGER: A flag that says whether or not this command needs
2034	* to have its action delayed until after the current event has
2035	* been closed. Some triggers need to avoid being invoked while
2036	* an event is currently in the process of being logged, since
2037	* the trigger may itself log data into the trace buffer. Thus
2038	* we make sure the current event is committed before invoking
2039	* those triggers. To do that, the trigger invocation is split
2040	* in two - the first part checks the filter using the current
2041	* trace record; if a command has the @post_trigger flag set, it
2042	* sets a bit for itself in the return value, otherwise it
2043	* directly invokes the trigger. Once all commands have been
2044	* either invoked or set their return flag, the current record is
2045	* either committed or discarded. At that point, if any commands
2046	* have deferred their triggers, those commands are finally
2047	* invoked following the close of the current event. In other
2048	* words, if the event_command @func() probe implementation
2049	* itself logs to the trace buffer, this flag should be set,
2050	* otherwise it can be left unspecified.
2051	*
2052	* @NEEDS_REC: A flag that says whether or not this command needs
2053	* access to the trace record in order to perform its function,
2054	* regardless of whether or not it has a filter associated with
2055	* it (filters make a trigger require access to the trace record
2056	* but are not always present).
2057	*/
2058	enum event_command_flags {
2059	EVENT_CMD_FL_POST_TRIGGER = 1,
2060	EVENT_CMD_FL_NEEDS_REC = 2,
2061	};
2062
2063	static inline bool event_command_post_trigger(struct event_command *cmd_ops)
2064	{
2065	return cmd_ops->flags & EVENT_CMD_FL_POST_TRIGGER;
2066	}
2067
2068	static inline bool event_command_needs_rec(struct event_command *cmd_ops)
2069	{
2070	return cmd_ops->flags & EVENT_CMD_FL_NEEDS_REC;
2071	}
2072
2073	extern int trace_event_enable_disable(struct trace_event_file *file,
2074	int enable, int soft_disable);
2075	extern int tracing_alloc_snapshot(void);
2076	extern void tracing_snapshot_cond(struct trace_array *tr, void *cond_data);
2077	extern int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update);
2078
2079	extern int tracing_snapshot_cond_disable(struct trace_array *tr);
2080	extern void *tracing_cond_snapshot_data(struct trace_array *tr);
2081
2082	extern const char *__start___trace_bprintk_fmt[];
2083	extern const char *__stop___trace_bprintk_fmt[];
2084
2085	extern const char *__start___tracepoint_str[];
2086	extern const char *__stop___tracepoint_str[];
2087
2088	void trace_printk_control(bool enabled);
2089	void trace_printk_start_comm(void);
2090	int trace_keep_overwrite(struct tracer *tracer, u64 mask, int set);
2091	int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled);
2092
2093	/* Used from boot time tracer */
2094	extern int trace_set_options(struct trace_array *tr, char *option);
2095	extern int tracing_set_tracer(struct trace_array *tr, const char *buf);
2096	extern ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
2097	unsigned long size, int cpu_id);
2098	extern int tracing_set_cpumask(struct trace_array *tr,
2099	cpumask_var_t tracing_cpumask_new);
2100
2101
2102	#define MAX_EVENT_NAME_LEN 64
2103
2104	extern ssize_t trace_parse_run_command(struct file *file,
2105	const char __user *buffer, size_t count, loff_t *ppos,
2106	int (*createfn)(const char *));
2107
2108	extern unsigned int err_pos(char *cmd, const char *str);
2109	extern void tracing_log_err(struct trace_array *tr,
2110	const char *loc, const char *cmd,
2111	const char **errs, u8 type, u16 pos);
2112
2113	/*
2114	* Normal trace_printk() and friends allocates special buffers
2115	* to do the manipulation, as well as saves the print formats
2116	* into sections to display. But the trace infrastructure wants
2117	* to use these without the added overhead at the price of being
2118	* a bit slower (used mainly for warnings, where we don't care
2119	* about performance). The internal_trace_puts() is for such
2120	* a purpose.
2121	*/
2122	#define internal_trace_puts(str) __trace_puts(_THIS_IP_, str, strlen(str))
2123
2124	#undef FTRACE_ENTRY
2125	#define FTRACE_ENTRY(call, struct_name, id, tstruct, print) \
2126	extern struct trace_event_call \
2127	__aligned(4) event_##call;
2128	#undef FTRACE_ENTRY_DUP
2129	#define FTRACE_ENTRY_DUP(call, struct_name, id, tstruct, print) \
2130	FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
2131	#undef FTRACE_ENTRY_PACKED
2132	#define FTRACE_ENTRY_PACKED(call, struct_name, id, tstruct, print) \
2133	FTRACE_ENTRY(call, struct_name, id, PARAMS(tstruct), PARAMS(print))
2134
2135	#include "trace_entries.h"
2136
2137	#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_FUNCTION_TRACER)
2138	int perf_ftrace_event_register(struct trace_event_call *call,
2139	enum trace_reg type, void *data);
2140	#else
2141	#define perf_ftrace_event_register NULL
2142	#endif
2143
2144	#ifdef CONFIG_FTRACE_SYSCALLS
2145	void init_ftrace_syscalls(void);
2146	const char *get_syscall_name(int syscall);
2147	#else
2148	static inline void init_ftrace_syscalls(void) { }
2149	static inline const char *get_syscall_name(int syscall)
2150	{
2151	return NULL;
2152	}
2153	#endif
2154
2155	#ifdef CONFIG_EVENT_TRACING
2156	void trace_event_init(void);
2157	void trace_event_update_all(struct trace_eval_map **map, int len);
2158	/* Used from boot time tracer */
2159	extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set);
2160	extern int trigger_process_regex(struct trace_event_file *file, char *buff);
2161	#else
2162	static inline void __init trace_event_init(void) { }
2163	static inline void trace_event_update_all(struct trace_eval_map **map, int len) { }
2164	#endif
2165
2166	#ifdef CONFIG_TRACER_SNAPSHOT
2167	void tracing_snapshot_instance(struct trace_array *tr);
2168	int tracing_alloc_snapshot_instance(struct trace_array *tr);
2169	int tracing_arm_snapshot(struct trace_array *tr);
2170	void tracing_disarm_snapshot(struct trace_array *tr);
2171	#else
2172	static inline void tracing_snapshot_instance(struct trace_array *tr) { }
2173	static inline int tracing_alloc_snapshot_instance(struct trace_array *tr)
2174	{
2175	return 0;
2176	}
2177	static inline int tracing_arm_snapshot(struct trace_array *tr) { return 0; }
2178	static inline void tracing_disarm_snapshot(struct trace_array *tr) { }
2179	#endif
2180
2181	#ifdef CONFIG_PREEMPT_TRACER
2182	void tracer_preempt_on(unsigned long a0, unsigned long a1);
2183	void tracer_preempt_off(unsigned long a0, unsigned long a1);
2184	#else
2185	static inline void tracer_preempt_on(unsigned long a0, unsigned long a1) { }
2186	static inline void tracer_preempt_off(unsigned long a0, unsigned long a1) { }
2187	#endif
2188	#ifdef CONFIG_IRQSOFF_TRACER
2189	void tracer_hardirqs_on(unsigned long a0, unsigned long a1);
2190	void tracer_hardirqs_off(unsigned long a0, unsigned long a1);
2191	#else
2192	static inline void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { }
2193	static inline void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { }
2194	#endif
2195
2196	/*
2197	* Reset the state of the trace_iterator so that it can read consumed data.
2198	* Normally, the trace_iterator is used for reading the data when it is not
2199	* consumed, and must retain state.
2200	*/
2201	static __always_inline void trace_iterator_reset(struct trace_iterator *iter)
2202	{
2203	memset_startat(iter, 0, seq);
2204	iter->pos = -1;
2205	}
2206
2207	/* Check the name is good for event/group/fields */
2208	static inline bool __is_good_name(const char *name, bool hash_ok)
2209	{
2210	if (!isalpha(*name) && *name != '_' && (!hash_ok || *name != '-'))
2211	return false;
2212	while (*++name != '\0') {
2213	if (!isalpha(*name) && !isdigit(c: *name) && *name != '_' &&
2214	(!hash_ok || *name != '-'))
2215	return false;
2216	}
2217	return true;
2218	}
2219
2220	/* Check the name is good for event/group/fields */
2221	static inline bool is_good_name(const char *name)
2222	{
2223	return __is_good_name(name, hash_ok: false);
2224	}
2225
2226	/* Check the name is good for system */
2227	static inline bool is_good_system_name(const char *name)
2228	{
2229	return __is_good_name(name, hash_ok: true);
2230	}
2231
2232	/* Convert certain expected symbols into '_' when generating event names */
2233	static inline void sanitize_event_name(char *name)
2234	{
2235	while (*name++ != '\0')
2236	if (*name == ':' || *name == '.' || *name == '*')
2237	*name = '_';
2238	}
2239
2240	/*
2241	* This is a generic way to read and write a u64 value from a file in tracefs.
2242	*
2243	* The value is stored on the variable pointed by *val. The value needs
2244	* to be at least *min and at most *max. The write is protected by an
2245	* existing *lock.
2246	*/
2247	struct trace_min_max_param {
2248	struct mutex *lock;
2249	u64 *val;
2250	u64 *min;
2251	u64 *max;
2252	};
2253
2254	#define U64_STR_SIZE 24 /* 20 digits max */
2255
2256	extern const struct file_operations trace_min_max_fops;
2257
2258	#ifdef CONFIG_RV
2259	extern int rv_init_interface(void);
2260	#else
2261	static inline int rv_init_interface(void)
2262	{
2263	return 0;
2264	}
2265	#endif
2266
2267	/*
2268	* This is used only to distinguish
2269	* function address from trampoline code.
2270	* So this value has no meaning.
2271	*/
2272	#define FTRACE_TRAMPOLINE_MARKER ((unsigned long) INT_MAX)
2273
2274	/*
2275	* This is used to get the address of the args array based on
2276	* the type of the entry.
2277	*/
2278	#define FGRAPH_ENTRY_ARGS(e) \
2279	({ \
2280	unsigned long *_args; \
2281	struct ftrace_graph_ent_entry *_e = e; \
2282	\
2283	if (IS_ENABLED(CONFIG_FUNCTION_GRAPH_RETADDR) && \
2284	e->ent.type == TRACE_GRAPH_RETADDR_ENT) { \
2285	struct fgraph_retaddr_ent_entry *_re; \
2286	\
2287	_re = (typeof(_re))_e; \
2288	_args = _re->args; \
2289	} else { \
2290	_args = _e->args; \
2291	} \
2292	_args; \
2293	})
2294
2295	#endif /* _LINUX_KERNEL_TRACE_H */
2296