diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/trace/Makefile | 1 | ||||
| -rw-r--r-- | kernel/trace/trace.c | 1192 | ||||
| -rw-r--r-- | kernel/trace/trace.h | 105 | ||||
| -rw-r--r-- | kernel/trace/trace_snapshot.c | 1067 |
4 files changed, 1188 insertions, 1177 deletions
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 04096c21d06b..83aeb5c77008 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -69,6 +69,7 @@ obj-$(CONFIG_TRACING) += trace_seq.o obj-$(CONFIG_TRACING) += trace_stat.o obj-$(CONFIG_TRACING) += trace_printk.o obj-$(CONFIG_TRACING) += trace_pid.o +obj-$(CONFIG_TRACER_SNAPSHOT) += trace_snapshot.o obj-$(CONFIG_TRACING) += pid_list.o obj-$(CONFIG_TRACING_MAP) += tracing_map.o obj-$(CONFIG_PREEMPTIRQ_DELAY_TEST) += preemptirq_delay_test.o diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index cf48fe23e71f..850f62032fd2 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -47,7 +47,6 @@ #include <linux/trace.h> #include <linux/sched/clock.h> #include <linux/sched/rt.h> -#include <linux/fsnotify.h> #include <linux/irq_work.h> #include <linux/workqueue.h> #include <linux/sort.h> @@ -219,15 +218,9 @@ static void ftrace_trace_userstack(struct trace_array *tr, static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; static char *default_bootup_tracer; -static bool allocate_snapshot; -static bool snapshot_at_boot; - static char boot_instance_info[COMMAND_LINE_SIZE] __initdata; static int boot_instance_index; -static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata; -static int boot_snapshot_index; - static int __init set_cmdline_ftrace(char *str) { strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); @@ -276,38 +269,6 @@ static int __init stop_trace_on_warning(char *str) } __setup("traceoff_on_warning", stop_trace_on_warning); -static int __init boot_alloc_snapshot(char *str) -{ - char *slot = boot_snapshot_info + boot_snapshot_index; - int left = sizeof(boot_snapshot_info) - boot_snapshot_index; - int ret; - - if (str[0] == '=') { - str++; - if (strlen(str) >= left) - return -1; - - ret = snprintf(slot, left, "%s\t", str); - boot_snapshot_index += ret; - } else { - allocate_snapshot = true; - /* We also need the main ring buffer expanded */ - trace_set_ring_buffer_expanded(NULL); - } - return 1; -} -__setup("alloc_snapshot", boot_alloc_snapshot); - - -static int __init boot_snapshot(char *str) -{ - snapshot_at_boot = true; - boot_alloc_snapshot(str); - return 1; -} -__setup("ftrace_boot_snapshot", boot_snapshot); - - static int __init boot_instance(char *str) { char *slot = boot_instance_info + boot_instance_index; @@ -807,47 +768,6 @@ void tracing_on(void) EXPORT_SYMBOL_GPL(tracing_on); #ifdef CONFIG_TRACER_SNAPSHOT -static void tracing_snapshot_instance_cond(struct trace_array *tr, - void *cond_data) -{ - unsigned long flags; - - if (in_nmi()) { - trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); - trace_array_puts(tr, "*** snapshot is being ignored ***\n"); - return; - } - - if (!tr->allocated_snapshot) { - trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n"); - trace_array_puts(tr, "*** stopping trace here! ***\n"); - tracer_tracing_off(tr); - return; - } - - if (tr->mapped) { - trace_array_puts(tr, "*** BUFFER MEMORY MAPPED ***\n"); - trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); - return; - } - - /* Note, snapshot can not be used when the tracer uses it */ - if (tracer_uses_snapshot(tr->current_trace)) { - trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n"); - trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); - return; - } - - local_irq_save(flags); - update_max_tr(tr, current, smp_processor_id(), cond_data); - local_irq_restore(flags); -} - -void tracing_snapshot_instance(struct trace_array *tr) -{ - tracing_snapshot_instance_cond(tr, NULL); -} - /** * tracing_snapshot - take a snapshot of the current buffer. * @@ -871,138 +791,6 @@ void tracing_snapshot(void) EXPORT_SYMBOL_GPL(tracing_snapshot); /** - * tracing_snapshot_cond - conditionally take a snapshot of the current buffer. - * @tr: The tracing instance to snapshot - * @cond_data: The data to be tested conditionally, and possibly saved - * - * This is the same as tracing_snapshot() except that the snapshot is - * conditional - the snapshot will only happen if the - * cond_snapshot.update() implementation receiving the cond_data - * returns true, which means that the trace array's cond_snapshot - * update() operation used the cond_data to determine whether the - * snapshot should be taken, and if it was, presumably saved it along - * with the snapshot. - */ -void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) -{ - tracing_snapshot_instance_cond(tr, cond_data); -} -EXPORT_SYMBOL_GPL(tracing_snapshot_cond); - -/** - * tracing_cond_snapshot_data - get the user data associated with a snapshot - * @tr: The tracing instance - * - * When the user enables a conditional snapshot using - * tracing_snapshot_cond_enable(), the user-defined cond_data is saved - * with the snapshot. This accessor is used to retrieve it. - * - * Should not be called from cond_snapshot.update(), since it takes - * the tr->max_lock lock, which the code calling - * cond_snapshot.update() has already done. - * - * Returns the cond_data associated with the trace array's snapshot. - */ -void *tracing_cond_snapshot_data(struct trace_array *tr) -{ - void *cond_data = NULL; - - local_irq_disable(); - arch_spin_lock(&tr->max_lock); - - if (tr->cond_snapshot) - cond_data = tr->cond_snapshot->cond_data; - - arch_spin_unlock(&tr->max_lock); - local_irq_enable(); - - return cond_data; -} -EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); - -static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, - struct array_buffer *size_buf, int cpu_id); -static void set_buffer_entries(struct array_buffer *buf, unsigned long val); - -int tracing_alloc_snapshot_instance(struct trace_array *tr) -{ - int order; - int ret; - - if (!tr->allocated_snapshot) { - - /* Make the snapshot buffer have the same order as main buffer */ - order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); - ret = ring_buffer_subbuf_order_set(tr->snapshot_buffer.buffer, order); - if (ret < 0) - return ret; - - /* allocate spare buffer */ - ret = resize_buffer_duplicate_size(&tr->snapshot_buffer, - &tr->array_buffer, RING_BUFFER_ALL_CPUS); - if (ret < 0) - return ret; - - tr->allocated_snapshot = true; - } - - return 0; -} - -static void free_snapshot(struct trace_array *tr) -{ - /* - * We don't free the ring buffer. instead, resize it because - * The max_tr ring buffer has some state (e.g. ring->clock) and - * we want preserve it. - */ - ring_buffer_subbuf_order_set(tr->snapshot_buffer.buffer, 0); - ring_buffer_resize(tr->snapshot_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); - set_buffer_entries(&tr->snapshot_buffer, 1); - tracing_reset_online_cpus(&tr->snapshot_buffer); - tr->allocated_snapshot = false; -} - -static int tracing_arm_snapshot_locked(struct trace_array *tr) -{ - int ret; - - lockdep_assert_held(&trace_types_lock); - - spin_lock(&tr->snapshot_trigger_lock); - if (tr->snapshot == UINT_MAX || tr->mapped) { - spin_unlock(&tr->snapshot_trigger_lock); - return -EBUSY; - } - - tr->snapshot++; - spin_unlock(&tr->snapshot_trigger_lock); - - ret = tracing_alloc_snapshot_instance(tr); - if (ret) { - spin_lock(&tr->snapshot_trigger_lock); - tr->snapshot--; - spin_unlock(&tr->snapshot_trigger_lock); - } - - return ret; -} - -int tracing_arm_snapshot(struct trace_array *tr) -{ - guard(mutex)(&trace_types_lock); - return tracing_arm_snapshot_locked(tr); -} - -void tracing_disarm_snapshot(struct trace_array *tr) -{ - spin_lock(&tr->snapshot_trigger_lock); - if (!WARN_ON(!tr->snapshot)) - tr->snapshot--; - spin_unlock(&tr->snapshot_trigger_lock); -} - -/** * tracing_alloc_snapshot - allocate snapshot buffer. * * This only allocates the snapshot buffer if it isn't already @@ -1023,129 +811,12 @@ int tracing_alloc_snapshot(void) return ret; } EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); - -/** - * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer. - * - * This is similar to tracing_snapshot(), but it will allocate the - * snapshot buffer if it isn't already allocated. Use this only - * where it is safe to sleep, as the allocation may sleep. - * - * This causes a swap between the snapshot buffer and the current live - * tracing buffer. You can use this to take snapshots of the live - * trace when some condition is triggered, but continue to trace. - */ -void tracing_snapshot_alloc(void) -{ - int ret; - - ret = tracing_alloc_snapshot(); - if (ret < 0) - return; - - tracing_snapshot(); -} -EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); - -/** - * tracing_snapshot_cond_enable - enable conditional snapshot for an instance - * @tr: The tracing instance - * @cond_data: User data to associate with the snapshot - * @update: Implementation of the cond_snapshot update function - * - * Check whether the conditional snapshot for the given instance has - * already been enabled, or if the current tracer is already using a - * snapshot; if so, return -EBUSY, else create a cond_snapshot and - * save the cond_data and update function inside. - * - * Returns 0 if successful, error otherwise. - */ -int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, - cond_update_fn_t update) -{ - struct cond_snapshot *cond_snapshot __free(kfree) = - kzalloc_obj(*cond_snapshot); - int ret; - - if (!cond_snapshot) - return -ENOMEM; - - cond_snapshot->cond_data = cond_data; - cond_snapshot->update = update; - - guard(mutex)(&trace_types_lock); - - if (tracer_uses_snapshot(tr->current_trace)) - return -EBUSY; - - /* - * The cond_snapshot can only change to NULL without the - * trace_types_lock. We don't care if we race with it going - * to NULL, but we want to make sure that it's not set to - * something other than NULL when we get here, which we can - * do safely with only holding the trace_types_lock and not - * having to take the max_lock. - */ - if (tr->cond_snapshot) - return -EBUSY; - - ret = tracing_arm_snapshot_locked(tr); - if (ret) - return ret; - - local_irq_disable(); - arch_spin_lock(&tr->max_lock); - tr->cond_snapshot = no_free_ptr(cond_snapshot); - arch_spin_unlock(&tr->max_lock); - local_irq_enable(); - - return 0; -} -EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); - -/** - * tracing_snapshot_cond_disable - disable conditional snapshot for an instance - * @tr: The tracing instance - * - * Check whether the conditional snapshot for the given instance is - * enabled; if so, free the cond_snapshot associated with it, - * otherwise return -EINVAL. - * - * Returns 0 if successful, error otherwise. - */ -int tracing_snapshot_cond_disable(struct trace_array *tr) -{ - int ret = 0; - - local_irq_disable(); - arch_spin_lock(&tr->max_lock); - - if (!tr->cond_snapshot) - ret = -EINVAL; - else { - kfree(tr->cond_snapshot); - tr->cond_snapshot = NULL; - } - - arch_spin_unlock(&tr->max_lock); - local_irq_enable(); - - tracing_disarm_snapshot(tr); - - return ret; -} -EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); #else void tracing_snapshot(void) { WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); } EXPORT_SYMBOL_GPL(tracing_snapshot); -void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) -{ - WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used"); -} -EXPORT_SYMBOL_GPL(tracing_snapshot_cond); int tracing_alloc_snapshot(void) { WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); @@ -1158,23 +829,6 @@ void tracing_snapshot_alloc(void) tracing_snapshot(); } EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); -void *tracing_cond_snapshot_data(struct trace_array *tr) -{ - return NULL; -} -EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); -int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update) -{ - return -ENODEV; -} -EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); -int tracing_snapshot_cond_disable(struct trace_array *tr) -{ - return false; -} -EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); -#define free_snapshot(tr) do { } while (0) -#define tracing_arm_snapshot_locked(tr) ({ -EBUSY; }) #endif /* CONFIG_TRACER_SNAPSHOT */ void tracer_tracing_off(struct trace_array *tr) @@ -1487,206 +1141,6 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) unsigned long __read_mostly tracing_thresh; -#ifdef CONFIG_TRACER_MAX_TRACE -#ifdef LATENCY_FS_NOTIFY -static struct workqueue_struct *fsnotify_wq; - -static void latency_fsnotify_workfn(struct work_struct *work) -{ - struct trace_array *tr = container_of(work, struct trace_array, - fsnotify_work); - fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY); -} - -static void latency_fsnotify_workfn_irq(struct irq_work *iwork) -{ - struct trace_array *tr = container_of(iwork, struct trace_array, - fsnotify_irqwork); - queue_work(fsnotify_wq, &tr->fsnotify_work); -} - -__init static int latency_fsnotify_init(void) -{ - fsnotify_wq = alloc_workqueue("tr_max_lat_wq", - WQ_UNBOUND | WQ_HIGHPRI, 0); - if (!fsnotify_wq) { - pr_err("Unable to allocate tr_max_lat_wq\n"); - return -ENOMEM; - } - return 0; -} - -late_initcall_sync(latency_fsnotify_init); - -void latency_fsnotify(struct trace_array *tr) -{ - if (!fsnotify_wq) - return; - /* - * We cannot call queue_work(&tr->fsnotify_work) from here because it's - * possible that we are called from __schedule() or do_idle(), which - * could cause a deadlock. - */ - irq_work_queue(&tr->fsnotify_irqwork); -} -#endif /* !LATENCY_FS_NOTIFY */ - -static const struct file_operations tracing_max_lat_fops; - -static void trace_create_maxlat_file(struct trace_array *tr, - struct dentry *d_tracer) -{ -#ifdef LATENCY_FS_NOTIFY - INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); - init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); -#endif - tr->d_max_latency = trace_create_file("tracing_max_latency", - TRACE_MODE_WRITE, - d_tracer, tr, - &tracing_max_lat_fops); -} - -/* - * Copy the new maximum trace into the separate maximum-trace - * structure. (this way the maximum trace is permanently saved, - * for later retrieval via /sys/kernel/tracing/tracing_max_latency) - */ -static void -__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) -{ - struct array_buffer *trace_buf = &tr->array_buffer; - struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); - struct array_buffer *max_buf = &tr->snapshot_buffer; - struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); - - max_buf->cpu = cpu; - max_buf->time_start = data->preempt_timestamp; - - max_data->saved_latency = tr->max_latency; - max_data->critical_start = data->critical_start; - max_data->critical_end = data->critical_end; - - strscpy(max_data->comm, tsk->comm); - max_data->pid = tsk->pid; - /* - * If tsk == current, then use current_uid(), as that does not use - * RCU. The irq tracer can be called out of RCU scope. - */ - if (tsk == current) - max_data->uid = current_uid(); - else - max_data->uid = task_uid(tsk); - - max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; - max_data->policy = tsk->policy; - max_data->rt_priority = tsk->rt_priority; - - /* record this tasks comm */ - tracing_record_cmdline(tsk); - latency_fsnotify(tr); -} -#else -static inline void trace_create_maxlat_file(struct trace_array *tr, - struct dentry *d_tracer) { } -static inline void __update_max_tr(struct trace_array *tr, - struct task_struct *tsk, int cpu) { } -#endif /* CONFIG_TRACER_MAX_TRACE */ - -#ifdef CONFIG_TRACER_SNAPSHOT -/** - * update_max_tr - snapshot all trace buffers from global_trace to max_tr - * @tr: tracer - * @tsk: the task with the latency - * @cpu: The cpu that initiated the trace. - * @cond_data: User data associated with a conditional snapshot - * - * Flip the buffers between the @tr and the max_tr and record information - * about which task was the cause of this latency. - */ -void -update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, - void *cond_data) -{ - if (tr->stop_count) - return; - - WARN_ON_ONCE(!irqs_disabled()); - - if (!tr->allocated_snapshot) { - /* Only the nop tracer should hit this when disabling */ - WARN_ON_ONCE(tr->current_trace != &nop_trace); - return; - } - - arch_spin_lock(&tr->max_lock); - - /* Inherit the recordable setting from array_buffer */ - if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) - ring_buffer_record_on(tr->snapshot_buffer.buffer); - else - ring_buffer_record_off(tr->snapshot_buffer.buffer); - - if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) { - arch_spin_unlock(&tr->max_lock); - return; - } - - swap(tr->array_buffer.buffer, tr->snapshot_buffer.buffer); - - __update_max_tr(tr, tsk, cpu); - - arch_spin_unlock(&tr->max_lock); - - /* Any waiters on the old snapshot buffer need to wake up */ - ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS); -} - -/** - * update_max_tr_single - only copy one trace over, and reset the rest - * @tr: tracer - * @tsk: task with the latency - * @cpu: the cpu of the buffer to copy. - * - * Flip the trace of a single CPU buffer between the @tr and the max_tr. - */ -void -update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) -{ - int ret; - - if (tr->stop_count) - return; - - WARN_ON_ONCE(!irqs_disabled()); - if (!tr->allocated_snapshot) { - /* Only the nop tracer should hit this when disabling */ - WARN_ON_ONCE(tr->current_trace != &nop_trace); - return; - } - - arch_spin_lock(&tr->max_lock); - - ret = ring_buffer_swap_cpu(tr->snapshot_buffer.buffer, tr->array_buffer.buffer, cpu); - - if (ret == -EBUSY) { - /* - * We failed to swap the buffer due to a commit taking - * place on this CPU. We fail to record, but we reset - * the max trace buffer (no one writes directly to it) - * and flag that it failed. - * Another reason is resize is in progress. - */ - trace_array_printk_buf(tr->snapshot_buffer.buffer, _THIS_IP_, - "Failed to swap buffers due to commit or resize in progress\n"); - } - - WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); - - __update_max_tr(tr, tsk, cpu); - arch_spin_unlock(&tr->max_lock); -} -#endif /* CONFIG_TRACER_SNAPSHOT */ - struct pipe_wait { struct trace_iterator *iter; int wait_index; @@ -1995,7 +1449,7 @@ int __init register_tracer(struct tracer *type) return 0; } -static void tracing_reset_cpu(struct array_buffer *buf, int cpu) +void tracing_reset_cpu(struct array_buffer *buf, int cpu) { struct trace_buffer *buffer = buf->buffer; @@ -3760,50 +3214,6 @@ static void test_ftrace_alive(struct seq_file *m) "# MAY BE MISSING FUNCTION EVENTS\n"); } -#ifdef CONFIG_TRACER_SNAPSHOT -static void show_snapshot_main_help(struct seq_file *m) -{ - seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" - "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" - "# Takes a snapshot of the main buffer.\n" - "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" - "# (Doesn't have to be '2' works with any number that\n" - "# is not a '0' or '1')\n"); -} - -static void show_snapshot_percpu_help(struct seq_file *m) -{ - seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); -#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP - seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" - "# Takes a snapshot of the main buffer for this cpu.\n"); -#else - seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" - "# Must use main snapshot file to allocate.\n"); -#endif - seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" - "# (Doesn't have to be '2' works with any number that\n" - "# is not a '0' or '1')\n"); -} - -static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) -{ - if (iter->tr->allocated_snapshot) - seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); - else - seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); - - seq_puts(m, "# Snapshot commands:\n"); - if (iter->cpu_file == RING_BUFFER_ALL_CPUS) - show_snapshot_main_help(m); - else - show_snapshot_percpu_help(m); -} -#else -/* Should never be called */ -static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } -#endif - static int s_show(struct seq_file *m, void *v) { struct trace_iterator *iter = v; @@ -3852,17 +3262,6 @@ static int s_show(struct seq_file *m, void *v) return 0; } -/* - * Should be used after trace_array_get(), trace_types_lock - * ensures that i_cdev was already initialized. - */ -static inline int tracing_get_cpu(struct inode *inode) -{ - if (inode->i_cdev) /* See trace_create_cpu_file() */ - return (long)inode->i_cdev - 1; - return RING_BUFFER_ALL_CPUS; -} - static const struct seq_operations tracer_seq_ops = { .start = s_start, .next = s_next, @@ -3889,7 +3288,7 @@ static void free_trace_iter_content(struct trace_iterator *iter) free_cpumask_var(iter->started); } -static struct trace_iterator * +struct trace_iterator * __tracing_open(struct inode *inode, struct file *file, bool snapshot) { struct trace_array *tr = inode->i_private; @@ -4069,7 +3468,7 @@ int tracing_single_release_file_tr(struct inode *inode, struct file *filp) return single_release(inode, filp); } -static int tracing_release(struct inode *inode, struct file *file) +int tracing_release(struct inode *inode, struct file *file) { struct trace_array *tr = inode->i_private; struct seq_file *m = file->private_data; @@ -5220,7 +4619,7 @@ int tracer_init(struct tracer *t, struct trace_array *tr) return t->init(tr); } -static void set_buffer_entries(struct array_buffer *buf, unsigned long val) +void trace_set_buffer_entries(struct array_buffer *buf, unsigned long val) { int cpu; @@ -5231,40 +4630,12 @@ static void set_buffer_entries(struct array_buffer *buf, unsigned long val) static void update_buffer_entries(struct array_buffer *buf, int cpu) { if (cpu == RING_BUFFER_ALL_CPUS) { - set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0)); + trace_set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0)); } else { per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu); } } -#ifdef CONFIG_TRACER_SNAPSHOT -/* resize @tr's buffer to the size of @size_tr's entries */ -static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, - struct array_buffer *size_buf, int cpu_id) -{ - int cpu, ret = 0; - - if (cpu_id == RING_BUFFER_ALL_CPUS) { - for_each_tracing_cpu(cpu) { - ret = ring_buffer_resize(trace_buf->buffer, - per_cpu_ptr(size_buf->data, cpu)->entries, cpu); - if (ret < 0) - break; - per_cpu_ptr(trace_buf->data, cpu)->entries = - per_cpu_ptr(size_buf->data, cpu)->entries; - } - } else { - ret = ring_buffer_resize(trace_buf->buffer, - per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); - if (ret == 0) - per_cpu_ptr(trace_buf->data, cpu_id)->entries = - per_cpu_ptr(size_buf->data, cpu_id)->entries; - } - - return ret; -} -#endif /* CONFIG_TRACER_SNAPSHOT */ - static int __tracing_resize_ring_buffer(struct trace_array *tr, unsigned long size, int cpu) { @@ -5683,9 +5054,8 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf, return ret; } -static ssize_t -tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, - size_t cnt, loff_t *ppos) +ssize_t tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, + size_t cnt, loff_t *ppos) { char buf[64]; int r; @@ -5697,9 +5067,8 @@ tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); } -static ssize_t -tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, - size_t cnt, loff_t *ppos) +ssize_t tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, + size_t cnt, loff_t *ppos) { unsigned long val; int ret; @@ -5741,28 +5110,6 @@ tracing_thresh_write(struct file *filp, const char __user *ubuf, return cnt; } -#ifdef CONFIG_TRACER_MAX_TRACE - -static ssize_t -tracing_max_lat_read(struct file *filp, char __user *ubuf, - size_t cnt, loff_t *ppos) -{ - struct trace_array *tr = filp->private_data; - - return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos); -} - -static ssize_t -tracing_max_lat_write(struct file *filp, const char __user *ubuf, - size_t cnt, loff_t *ppos) -{ - struct trace_array *tr = filp->private_data; - - return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos); -} - -#endif - static int open_pipe_on_cpu(struct trace_array *tr, int cpu) { if (cpu == RING_BUFFER_ALL_CPUS) { @@ -7140,194 +6487,6 @@ u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_eve return ring_buffer_event_time_stamp(buffer, rbe); } -struct ftrace_buffer_info { - struct trace_iterator iter; - void *spare; - unsigned int spare_cpu; - unsigned int spare_size; - unsigned int read; -}; - -#ifdef CONFIG_TRACER_SNAPSHOT -static int tracing_snapshot_open(struct inode *inode, struct file *file) -{ - struct trace_array *tr = inode->i_private; - struct trace_iterator *iter; - struct seq_file *m; - int ret; - - ret = tracing_check_open_get_tr(tr); - if (ret) - return ret; - - if (file->f_mode & FMODE_READ) { - iter = __tracing_open(inode, file, true); - if (IS_ERR(iter)) - ret = PTR_ERR(iter); - } else { - /* Writes still need the seq_file to hold the private data */ - ret = -ENOMEM; - m = kzalloc_obj(*m); - if (!m) - goto out; - iter = kzalloc_obj(*iter); - if (!iter) { - kfree(m); - goto out; - } - ret = 0; - - iter->tr = tr; - iter->array_buffer = &tr->snapshot_buffer; - iter->cpu_file = tracing_get_cpu(inode); - m->private = iter; - file->private_data = m; - } -out: - if (ret < 0) - trace_array_put(tr); - - return ret; -} - -static void tracing_swap_cpu_buffer(void *tr) -{ - update_max_tr_single((struct trace_array *)tr, current, smp_processor_id()); -} - -static ssize_t -tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, - loff_t *ppos) -{ - struct seq_file *m = filp->private_data; - struct trace_iterator *iter = m->private; - struct trace_array *tr = iter->tr; - unsigned long val; - int ret; - - ret = tracing_update_buffers(tr); - if (ret < 0) - return ret; - - ret = kstrtoul_from_user(ubuf, cnt, 10, &val); - if (ret) - return ret; - - guard(mutex)(&trace_types_lock); - - if (tracer_uses_snapshot(tr->current_trace)) - return -EBUSY; - - local_irq_disable(); - arch_spin_lock(&tr->max_lock); - if (tr->cond_snapshot) - ret = -EBUSY; - arch_spin_unlock(&tr->max_lock); - local_irq_enable(); - if (ret) - return ret; - - switch (val) { - case 0: - if (iter->cpu_file != RING_BUFFER_ALL_CPUS) - return -EINVAL; - if (tr->allocated_snapshot) - free_snapshot(tr); - break; - case 1: -/* Only allow per-cpu swap if the ring buffer supports it */ -#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP - if (iter->cpu_file != RING_BUFFER_ALL_CPUS) - return -EINVAL; -#endif - if (tr->allocated_snapshot) - ret = resize_buffer_duplicate_size(&tr->snapshot_buffer, - &tr->array_buffer, iter->cpu_file); - - ret = tracing_arm_snapshot_locked(tr); - if (ret) - return ret; - - /* Now, we're going to swap */ - if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { - local_irq_disable(); - update_max_tr(tr, current, smp_processor_id(), NULL); - local_irq_enable(); - } else { - smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer, - (void *)tr, 1); - } - tracing_disarm_snapshot(tr); - break; - default: - if (tr->allocated_snapshot) { - if (iter->cpu_file == RING_BUFFER_ALL_CPUS) - tracing_reset_online_cpus(&tr->snapshot_buffer); - else - tracing_reset_cpu(&tr->snapshot_buffer, iter->cpu_file); - } - break; - } - - if (ret >= 0) { - *ppos += cnt; - ret = cnt; - } - - return ret; -} - -static int tracing_snapshot_release(struct inode *inode, struct file *file) -{ - struct seq_file *m = file->private_data; - int ret; - - ret = tracing_release(inode, file); - - if (file->f_mode & FMODE_READ) - return ret; - - /* If write only, the seq_file is just a stub */ - if (m) - kfree(m->private); - kfree(m); - - return 0; -} - -static int tracing_buffers_open(struct inode *inode, struct file *filp); -static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, - size_t count, loff_t *ppos); -static int tracing_buffers_release(struct inode *inode, struct file *file); -static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, unsigned int flags); - -static int snapshot_raw_open(struct inode *inode, struct file *filp) -{ - struct ftrace_buffer_info *info; - int ret; - - /* The following checks for tracefs lockdown */ - ret = tracing_buffers_open(inode, filp); - if (ret < 0) - return ret; - - info = filp->private_data; - - if (tracer_uses_snapshot(info->iter.trace)) { - tracing_buffers_release(inode, filp); - return -EBUSY; - } - - info->iter.snapshot = true; - info->iter.array_buffer = &info->iter.tr->snapshot_buffer; - - return ret; -} - -#endif /* CONFIG_TRACER_SNAPSHOT */ - - static const struct file_operations tracing_thresh_fops = { .open = tracing_open_generic, .read = tracing_thresh_read, @@ -7335,16 +6494,6 @@ static const struct file_operations tracing_thresh_fops = { .llseek = generic_file_llseek, }; -#ifdef CONFIG_TRACER_MAX_TRACE -static const struct file_operations tracing_max_lat_fops = { - .open = tracing_open_generic_tr, - .read = tracing_max_lat_read, - .write = tracing_max_lat_write, - .llseek = generic_file_llseek, - .release = tracing_release_generic_tr, -}; -#endif - static const struct file_operations set_tracer_fops = { .open = tracing_open_generic_tr, .read = tracing_set_trace_read, @@ -7431,24 +6580,6 @@ static const struct file_operations last_boot_fops = { .release = tracing_seq_release, }; -#ifdef CONFIG_TRACER_SNAPSHOT -static const struct file_operations snapshot_fops = { - .open = tracing_snapshot_open, - .read = seq_read, - .write = tracing_snapshot_write, - .llseek = tracing_lseek, - .release = tracing_snapshot_release, -}; - -static const struct file_operations snapshot_raw_fops = { - .open = snapshot_raw_open, - .read = tracing_buffers_read, - .release = tracing_buffers_release, - .splice_read = tracing_buffers_splice_read, -}; - -#endif /* CONFIG_TRACER_SNAPSHOT */ - /* * trace_min_max_write - Write a u64 value to a trace_min_max_param struct * @filp: The active open file structure @@ -7808,7 +6939,7 @@ static const struct file_operations tracing_err_log_fops = { .release = tracing_err_log_release, }; -static int tracing_buffers_open(struct inode *inode, struct file *filp) +int tracing_buffers_open(struct inode *inode, struct file *filp) { struct trace_array *tr = inode->i_private; struct ftrace_buffer_info *info; @@ -7856,9 +6987,8 @@ tracing_buffers_poll(struct file *filp, poll_table *poll_table) return trace_poll(iter, filp, poll_table); } -static ssize_t -tracing_buffers_read(struct file *filp, char __user *ubuf, - size_t count, loff_t *ppos) +ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) { struct ftrace_buffer_info *info = filp->private_data; struct trace_iterator *iter = &info->iter; @@ -7959,7 +7089,7 @@ static int tracing_buffers_flush(struct file *file, fl_owner_t id) return 0; } -static int tracing_buffers_release(struct inode *inode, struct file *file) +int tracing_buffers_release(struct inode *inode, struct file *file) { struct ftrace_buffer_info *info = file->private_data; struct trace_iterator *iter = &info->iter; @@ -8033,10 +7163,9 @@ static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) spd->partial[i].private = 0; } -static ssize_t -tracing_buffers_splice_read(struct file *file, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags) +ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) { struct ftrace_buffer_info *info = file->p |