| From 6f30383e38c4fe5853189e005a97f424031dd854 Mon Sep 17 00:00:00 2001 |
| From: Cheng Jian <cj.chengjian@huawei.com> |
| Date: Sat, 4 May 2019 19:39:39 +0800 |
| Subject: ftrace: Enable trampoline when rec count returns back to one |
| |
| [ Upstream commit a124692b698b00026a58d89831ceda2331b2e1d0 ] |
| |
| Custom trampolines can only be enabled if there is only a single ops |
| attached to it. If there's only a single callback registered to a function, |
| and the ops has a trampoline registered for it, then we can call the |
| trampoline directly. This is very useful for improving the performance of |
| ftrace and livepatch. |
| |
| If more than one callback is registered to a function, the general |
| trampoline is used, and the custom trampoline is not restored back to the |
| direct call even if all the other callbacks were unregistered and we are |
| back to one callback for the function. |
| |
| To fix this, set FTRACE_FL_TRAMP flag if rec count is decremented |
| to one, and the ops that left has a trampoline. |
| |
| Testing After this patch : |
| |
| insmod livepatch_unshare_files.ko |
| cat /sys/kernel/debug/tracing/enabled_functions |
| |
| unshare_files (1) R I tramp: 0xffffffffc0000000(klp_ftrace_handler+0x0/0xa0) ->ftrace_ops_assist_func+0x0/0xf0 |
| |
| echo unshare_files > /sys/kernel/debug/tracing/set_ftrace_filter |
| echo function > /sys/kernel/debug/tracing/current_tracer |
| cat /sys/kernel/debug/tracing/enabled_functions |
| |
| unshare_files (2) R I ->ftrace_ops_list_func+0x0/0x150 |
| |
| echo nop > /sys/kernel/debug/tracing/current_tracer |
| cat /sys/kernel/debug/tracing/enabled_functions |
| |
| unshare_files (1) R I tramp: 0xffffffffc0000000(klp_ftrace_handler+0x0/0xa0) ->ftrace_ops_assist_func+0x0/0xf0 |
| |
| Link: http://lkml.kernel.org/r/1556969979-111047-1-git-send-email-cj.chengjian@huawei.com |
| |
| Signed-off-by: Cheng Jian <cj.chengjian@huawei.com> |
| Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org> |
| Signed-off-by: Sasha Levin <sashal@kernel.org> |
| --- |
| kernel/trace/ftrace.c | 28 +++++++++++++++------------- |
| 1 file changed, 15 insertions(+), 13 deletions(-) |
| |
| diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c |
| index 0043aef0ed8dc..77109b9cf7332 100644 |
| --- a/kernel/trace/ftrace.c |
| +++ b/kernel/trace/ftrace.c |
| @@ -1631,6 +1631,11 @@ static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) |
| return keep_regs; |
| } |
| |
| +static struct ftrace_ops * |
| +ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); |
| +static struct ftrace_ops * |
| +ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); |
| + |
| static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, |
| int filter_hash, |
| bool inc) |
| @@ -1759,15 +1764,17 @@ static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, |
| } |
| |
| /* |
| - * If the rec had TRAMP enabled, then it needs to |
| - * be cleared. As TRAMP can only be enabled iff |
| - * there is only a single ops attached to it. |
| - * In otherwords, always disable it on decrementing. |
| - * In the future, we may set it if rec count is |
| - * decremented to one, and the ops that is left |
| - * has a trampoline. |
| + * The TRAMP needs to be set only if rec count |
| + * is decremented to one, and the ops that is |
| + * left has a trampoline. As TRAMP can only be |
| + * enabled if there is only a single ops attached |
| + * to it. |
| */ |
| - rec->flags &= ~FTRACE_FL_TRAMP; |
| + if (ftrace_rec_count(rec) == 1 && |
| + ftrace_find_tramp_ops_any(rec)) |
| + rec->flags |= FTRACE_FL_TRAMP; |
| + else |
| + rec->flags &= ~FTRACE_FL_TRAMP; |
| |
| /* |
| * flags will be cleared in ftrace_check_record() |
| @@ -1960,11 +1967,6 @@ static void print_ip_ins(const char *fmt, const unsigned char *p) |
| printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); |
| } |
| |
| -static struct ftrace_ops * |
| -ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); |
| -static struct ftrace_ops * |
| -ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); |
| - |
| enum ftrace_bug_type ftrace_bug_type; |
| const void *ftrace_expected; |
| |
| -- |
| 2.20.1 |
| |