lkml.org 
[lkml]   [2018]   [May]   [24]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH 4.16 063/161] s390: use expoline thunks in the BPF JIT
    Date
    4.16-stable review patch.  If anyone has any objections, please let me know.

    ------------------

    From: Martin Schwidefsky <schwidefsky@de.ibm.com>

    [ Upstream commit de5cb6eb514ebe241e3edeb290cb41deb380b81d ]

    The BPF JIT need safe guarding against spectre v2 in the sk_load_xxx
    assembler stubs and the indirect branches generated by the JIT itself
    need to be converted to expolines.

    Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
    Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
    ---
    arch/s390/net/bpf_jit.S | 16 ++++++----
    arch/s390/net/bpf_jit_comp.c | 63 +++++++++++++++++++++++++++++++++++++++++--
    2 files changed, 71 insertions(+), 8 deletions(-)

    --- a/arch/s390/net/bpf_jit.S
    +++ b/arch/s390/net/bpf_jit.S
    @@ -9,6 +9,7 @@
    */

    #include <linux/linkage.h>
    +#include <asm/nospec-insn.h>
    #include "bpf_jit.h"

    /*
    @@ -54,7 +55,7 @@ ENTRY(sk_load_##NAME##_pos); \
    clg %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */ \
    jh sk_load_##NAME##_slow; \
    LOAD %r14,-SIZE(%r3,%r12); /* Get data from skb */ \
    - b OFF_OK(%r6); /* Return */ \
    + B_EX OFF_OK,%r6; /* Return */ \
    \
    sk_load_##NAME##_slow:; \
    lgr %r2,%r7; /* Arg1 = skb pointer */ \
    @@ -64,11 +65,14 @@ sk_load_##NAME##_slow:; \
    brasl %r14,skb_copy_bits; /* Get data from skb */ \
    LOAD %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */ \
    ltgr %r2,%r2; /* Set cc to (%r2 != 0) */ \
    - br %r6; /* Return */
    + BR_EX %r6; /* Return */

    sk_load_common(word, 4, llgf) /* r14 = *(u32 *) (skb->data+offset) */
    sk_load_common(half, 2, llgh) /* r14 = *(u16 *) (skb->data+offset) */

    + GEN_BR_THUNK %r6
    + GEN_B_THUNK OFF_OK,%r6
    +
    /*
    * Load 1 byte from SKB (optimized version)
    */
    @@ -80,7 +84,7 @@ ENTRY(sk_load_byte_pos)
    clg %r3,STK_OFF_HLEN(%r15) # Offset >= hlen?
    jnl sk_load_byte_slow
    llgc %r14,0(%r3,%r12) # Get byte from skb
    - b OFF_OK(%r6) # Return OK
    + B_EX OFF_OK,%r6 # Return OK

    sk_load_byte_slow:
    lgr %r2,%r7 # Arg1 = skb pointer
    @@ -90,7 +94,7 @@ sk_load_byte_slow:
    brasl %r14,skb_copy_bits # Get data from skb
    llgc %r14,STK_OFF_TMP(%r15) # Load result from temp buffer
    ltgr %r2,%r2 # Set cc to (%r2 != 0)
    - br %r6 # Return cc
    + BR_EX %r6 # Return cc

    #define sk_negative_common(NAME, SIZE, LOAD) \
    sk_load_##NAME##_slow_neg:; \
    @@ -104,7 +108,7 @@ sk_load_##NAME##_slow_neg:; \
    jz bpf_error; \
    LOAD %r14,0(%r2); /* Get data from pointer */ \
    xr %r3,%r3; /* Set cc to zero */ \
    - br %r6; /* Return cc */
    + BR_EX %r6; /* Return cc */

    sk_negative_common(word, 4, llgf)
    sk_negative_common(half, 2, llgh)
    @@ -113,4 +117,4 @@ sk_negative_common(byte, 1, llgc)
    bpf_error:
    # force a return 0 from jit handler
    ltgr %r15,%r15 # Set condition code
    - br %r6
    + BR_EX %r6
    --- a/arch/s390/net/bpf_jit_comp.c
    +++ b/arch/s390/net/bpf_jit_comp.c
    @@ -25,6 +25,8 @@
    #include <linux/bpf.h>
    #include <asm/cacheflush.h>
    #include <asm/dis.h>
    +#include <asm/facility.h>
    +#include <asm/nospec-branch.h>
    #include <asm/set_memory.h>
    #include "bpf_jit.h"

    @@ -41,6 +43,8 @@ struct bpf_jit {
    int base_ip; /* Base address for literal pool */
    int ret0_ip; /* Address of return 0 */
    int exit_ip; /* Address of exit */
    + int r1_thunk_ip; /* Address of expoline thunk for 'br %r1' */
    + int r14_thunk_ip; /* Address of expoline thunk for 'br %r14' */
    int tail_call_start; /* Tail call start offset */
    int labels[1]; /* Labels for local jumps */
    };
    @@ -250,6 +254,19 @@ static inline void reg_set_seen(struct b
    REG_SET_SEEN(b2); \
    })

    +#define EMIT6_PCREL_RILB(op, b, target) \
    +({ \
    + int rel = (target - jit->prg) / 2; \
    + _EMIT6(op | reg_high(b) << 16 | rel >> 16, rel & 0xffff); \
    + REG_SET_SEEN(b); \
    +})
    +
    +#define EMIT6_PCREL_RIL(op, target) \
    +({ \
    + int rel = (target - jit->prg) / 2; \
    + _EMIT6(op | rel >> 16, rel & 0xffff); \
    +})
    +
    #define _EMIT6_IMM(op, imm) \
    ({ \
    unsigned int __imm = (imm); \
    @@ -469,8 +486,45 @@ static void bpf_jit_epilogue(struct bpf_
    EMIT4(0xb9040000, REG_2, BPF_REG_0);
    /* Restore registers */
    save_restore_regs(jit, REGS_RESTORE, stack_depth);
    + if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
    + jit->r14_thunk_ip = jit->prg;
    + /* Generate __s390_indirect_jump_r14 thunk */
    + if (test_facility(35)) {
    + /* exrl %r0,.+10 */
    + EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
    + } else {
    + /* larl %r1,.+14 */
    + EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
    + /* ex 0,0(%r1) */
    + EMIT4_DISP(0x44000000, REG_0, REG_1, 0);
    + }
    + /* j . */
    + EMIT4_PCREL(0xa7f40000, 0);
    + }
    /* br %r14 */
    _EMIT2(0x07fe);
    +
    + if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable &&
    + (jit->seen & SEEN_FUNC)) {
    + jit->r1_thunk_ip = jit->prg;
    + /* Generate __s390_indirect_jump_r1 thunk */
    + if (test_facility(35)) {
    + /* exrl %r0,.+10 */
    + EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
    + /* j . */
    + EMIT4_PCREL(0xa7f40000, 0);
    + /* br %r1 */
    + _EMIT2(0x07f1);
    + } else {
    + /* larl %r1,.+14 */
    + EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
    + /* ex 0,S390_lowcore.br_r1_tampoline */
    + EMIT4_DISP(0x44000000, REG_0, REG_0,
    + offsetof(struct lowcore, br_r1_trampoline));
    + /* j . */
    + EMIT4_PCREL(0xa7f40000, 0);
    + }
    + }
    }

    /*
    @@ -966,8 +1020,13 @@ static noinline int bpf_jit_insn(struct
    /* lg %w1,<d(imm)>(%l) */
    EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L,
    EMIT_CONST_U64(func));
    - /* basr %r14,%w1 */
    - EMIT2(0x0d00, REG_14, REG_W1);
    + if (IS_ENABLED(CC_USING_EXPOLINE) && !nospec_disable) {
    + /* brasl %r14,__s390_indirect_jump_r1 */
    + EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
    + } else {
    + /* basr %r14,%w1 */
    + EMIT2(0x0d00, REG_14, REG_W1);
    + }
    /* lgr %b0,%r2: load return value into %b0 */
    EMIT4(0xb9040000, BPF_REG_0, REG_2);
    if ((jit->seen & SEEN_SKB) &&

    \
     
     \ /
      Last update: 2018-05-24 12:22    [W:2.597 / U:0.008 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site