1/* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12/* 13 * Copyright 2019 Joyent, Inc. 14 * Copyright 2024 MNX Cloud, Inc. 15 */ 16 17 .file "retpoline.s" 18 19/* 20 * This file implements the various hooks that are needed for retpolines and 21 * return stack buffer (RSB) stuffing. For more information, please see the 22 * 'Speculative Execution CPU Side Channel Security' section of the 23 * uts/i86pc/os/cpuid.c big theory statement. 24 */ 25 26#include <sys/asm_linkage.h> 27#include <sys/x86_archext.h> 28 29#if defined(__amd64) 30 31/* 32 * This macro generates the default retpoline entry point that the compiler 33 * expects. It implements the expected retpoline form. 34 */ 35#define RETPOLINE_MKTHUNK(reg) \ 36 ENTRY(__x86_indirect_thunk_##reg) \ 37 call 2f; \ 381: \ 39 pause; \ 40 lfence; \ 41 jmp 1b; \ 422: \ 43 movq %##reg, (%rsp); \ 44 ret; \ 45 SET_SIZE(__x86_indirect_thunk_##reg) 46 47/* 48 * This macro generates the default retpoline form. It exists in addition to the 49 * thunk so if we need to restore the default retpoline behavior to the thunk 50 * we can. 51 */ 52#define RETPOLINE_MKGENERIC(reg) \ 53 ENTRY(__x86_indirect_thunk_gen_##reg) \ 54 call 2f; \ 551: \ 56 pause; \ 57 lfence; \ 58 jmp 1b; \ 592: \ 60 movq %##reg, (%rsp); \ 61 ret; \ 62 SET_SIZE(__x86_indirect_thunk_gen_##reg) 63 64/* 65 * This macro generates the no-op form of the retpoline which will be used if we 66 * either need to disable retpolines because we have enhanced IBRS or because we 67 * have been asked to disable mitigations. 68 */ 69#define RETPOLINE_MKJUMP(reg) \ 70 ENTRY(__x86_indirect_thunk_jmp_##reg) \ 71 jmp *%##reg; \ 72 SET_SIZE(__x86_indirect_thunk_jmp_##reg) 73 74 RETPOLINE_MKTHUNK(rax) 75 RETPOLINE_MKTHUNK(rbx) 76 RETPOLINE_MKTHUNK(rcx) 77 RETPOLINE_MKTHUNK(rdx) 78 RETPOLINE_MKTHUNK(rdi) 79 RETPOLINE_MKTHUNK(rsi) 80 RETPOLINE_MKTHUNK(rbp) 81 RETPOLINE_MKTHUNK(r8) 82 RETPOLINE_MKTHUNK(r9) 83 RETPOLINE_MKTHUNK(r10) 84 RETPOLINE_MKTHUNK(r11) 85 RETPOLINE_MKTHUNK(r12) 86 RETPOLINE_MKTHUNK(r13) 87 RETPOLINE_MKTHUNK(r14) 88 RETPOLINE_MKTHUNK(r15) 89 90 RETPOLINE_MKGENERIC(rax) 91 RETPOLINE_MKGENERIC(rbx) 92 RETPOLINE_MKGENERIC(rcx) 93 RETPOLINE_MKGENERIC(rdx) 94 RETPOLINE_MKGENERIC(rdi) 95 RETPOLINE_MKGENERIC(rsi) 96 RETPOLINE_MKGENERIC(rbp) 97 RETPOLINE_MKGENERIC(r8) 98 RETPOLINE_MKGENERIC(r9) 99 RETPOLINE_MKGENERIC(r10) 100 RETPOLINE_MKGENERIC(r11) 101 RETPOLINE_MKGENERIC(r12) 102 RETPOLINE_MKGENERIC(r13) 103 RETPOLINE_MKGENERIC(r14) 104 RETPOLINE_MKGENERIC(r15) 105 106 RETPOLINE_MKJUMP(rax) 107 RETPOLINE_MKJUMP(rbx) 108 RETPOLINE_MKJUMP(rcx) 109 RETPOLINE_MKJUMP(rdx) 110 RETPOLINE_MKJUMP(rdi) 111 RETPOLINE_MKJUMP(rsi) 112 RETPOLINE_MKJUMP(rbp) 113 RETPOLINE_MKJUMP(r8) 114 RETPOLINE_MKJUMP(r9) 115 RETPOLINE_MKJUMP(r10) 116 RETPOLINE_MKJUMP(r11) 117 RETPOLINE_MKJUMP(r12) 118 RETPOLINE_MKJUMP(r13) 119 RETPOLINE_MKJUMP(r14) 120 RETPOLINE_MKJUMP(r15) 121 122 /* 123 * The x86_rsb_stuff{,_vmexit} functions can be called from pretty 124 * arbitrary contexts. It's much easier for us to save and restore all 125 * the registers we touch rather than clobber them for callers. You 126 * must preserve this property or the system will panic at best. 127 * 128 * The reason for two entry points is because the need to RSB stuff 129 * on Intel depends greatly on factors that are different in the 130 * VMEXIT case, vs. the other context-switching cases 131 * 132 * See cpuid.c's cpuid_patch_rsb() for where the two entry points' 133 * NOPs actually get patched with one-byte RETs as need be, and the 134 * rules we use to determine which gets disabled with a RET, and which 135 * maintain their NOP to proceed to executing the stuffing sequence. 136 */ 137 ENTRY_NP(x86_rsb_stuff_vmexit) 138 nop 139 ALTENTRY(x86_rsb_stuff) 140 nop 141 pushq %rdi 142 pushq %rax 143 movl $16, %edi 144 movq %rsp, %rax 145rsb_loop: 146 call 2f 1471: 148 pause 149 call 1b 1502: 151 call 2f 1521: 153 pause 154 call 1b 1552: 156 subl $1, %edi 157 jnz rsb_loop 158 movq %rax, %rsp 159 popq %rax 160 popq %rdi 161 ret 162 SET_SIZE(x86_rsb_stuff) 163 SET_SIZE(x86_rsb_stuff_vmexit) 164 165#elif defined(__i386) 166 167/* 168 * While the kernel is 64-bit only, dboot is still 32-bit, so there are a 169 * limited number of variants that are used for 32-bit. However as dboot is 170 * short lived and uses them sparingly, we only do the full variant and do not 171 * have an AMD specific version. 172 */ 173 174#define RETPOLINE_MKTHUNK(reg) \ 175 ENTRY(__x86_indirect_thunk_##reg) \ 176 call 2f; \ 1771: \ 178 pause; \ 179 lfence; \ 180 jmp 1b; \ 1812: \ 182 movl %##reg, (%esp); \ 183 ret; \ 184 SET_SIZE(__x86_indirect_thunk_##reg) 185 186 RETPOLINE_MKTHUNK(edi) 187 RETPOLINE_MKTHUNK(eax) 188 189#else 190#error "Your architecture is in another castle." 191#endif 192