// SPDX-License-Identifier: GPL-2.0-only
/*
* eBPF JIT compiler
*
* Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
* IBM Corporation
*
* Based on the powerpc classic BPF JIT compiler by Matt Evans
*/
#include <linux/moduleloader.h>
#include <asm/cacheflush.h>
#include <asm/asm-compat.h>
#include <linux/netdevice.h>
#include <linux/filter.h>
#include <linux/if_vlan.h>
#include <linux/kernel.h>
#include <linux/memory.h>
#include <linux/bpf.h>
#include <asm/kprobes.h>
#include <asm/text-patching.h>
#include "bpf_jit.h"
/* These offsets are from bpf prog end and stay the same across progs */
static int bpf_jit_ool_stub, bpf_jit_long_branch_stub;
static void bpf_jit_fill_ill_insns(void *area, unsigned int size)
{
memset32(area, BREAKPOINT_INSTRUCTION, size / 4);
}
void dummy_tramp(void);
asm (
" .pushsection .text, \"ax\", @progbits ;"
" .global dummy_tramp ;"
" .type dummy_tramp, @function ;"
"dummy_tramp: ;"
#ifdef CONFIG_PPC_FTRACE_OUT_OF_LINE
" blr ;"
#else
/* LR is always in r11, so we don't need a 'mflr r11' here */
" mtctr 11 ;"
" mtlr 0 ;"
" bctr ;"
#endif
" .size dummy_tramp, .-dummy_tramp ;"
" .popsection ;"
);
void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx)
{
int ool_stub_idx, long_branch_stub_idx;
/*
* Out-of-line stub:
* mflr r0
* [b|bl] tramp
* mtlr r0 // only with CONFIG_PPC_FTRACE_OUT_OF_LINE
* b bpf_func + 4
*/
ool_stub_idx = ctx->idx;
EMIT(PPC_RAW_MFLR(_R0));
EMIT(PPC_RAW_NOP());
if (IS_ENABLED(CONFIG_PPC_FTRACE_OUT_OF_LINE))
EMIT(PPC_RAW_MTLR(_R0));
WARN_ON_ONCE(!is_offset_in_branch_range(4 - (long)ctx->idx * 4));
EMIT(PPC_RAW_BRANCH(4 - (long)ctx->idx * 4));
/*
* Long branch stub:
* .long <dummy_tramp_addr>
* mflr r11
* bcl 20,31,$+4
* mflr r12
* ld r12, -8-SZL(r12)
* mtctr r12
* mtlr r11 // needed to retain ftrace ABI
* bctr
*/
if (image)
*((unsigned long *)&image[ctx->idx]) = (unsigned long)dummy_tramp;
ctx->idx += SZL / 4;
long_branch_stub_idx = ctx->idx;
EMIT(PPC_RAW_MFLR(_R11));
EMIT(PPC_RAW_BCL4());
EMIT(PPC_RAW_MFLR(_R12));
EMIT(PPC_RAW_LL(_R12, _R12, -8-SZL));
EMIT(PPC_RAW_MTCTR(_R12));
EMIT(PPC_RAW_MTLR(_R11));
EMIT(PPC_RAW_BCTR());
if (!bpf_jit_ool_stub) {
bpf_jit_ool_stub = (ctx->idx - ool_stub_idx) * 4;
bpf_jit_long_branch_stub = (ctx->idx - long_branch_stub_idx) * 4;
}
}
int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr)
{
if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) {
PPC_JMP(exit_addr);
} else if (ctx->alt_exit_addr) </