Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: #ifndef _ASM_X86_ALTERNATIVE_H 2: #define _ASM_X86_ALTERNATIVE_H 3: 4: #include <linux/types.h> 5: #include <linux/stddef.h> 6: #include <linux/stringify.h> 7: #include <asm/asm.h> 8: #include <asm/ptrace.h> 9: 10: /* 11: * Alternative inline assembly for SMP. 12: * 13: * The LOCK_PREFIX macro defined here replaces the LOCK and 14: * LOCK_PREFIX macros used everywhere in the source tree. 15: * 16: * SMP alternatives use the same data structures as the other 17: * alternatives and the X86_FEATURE_UP flag to indicate the case of a 18: * UP system running a SMP kernel. The existing apply_alternatives() 19: * works fine for patching a SMP kernel for UP. 20: * 21: * The SMP alternative tables can be kept after boot and contain both 22: * UP and SMP versions of the instructions to allow switching back to 23: * SMP at runtime, when hotplugging in a new CPU, which is especially 24: * useful in virtualized environments. 25: * 26: * The very common lock prefix is handled as special case in a 27: * separate table which is a pure address list without replacement ptr 28: * and size information. That keeps the table sizes small. 29: */ 30: 31: #ifdef CONFIG_SMP 32: #define LOCK_PREFIX_HERE \ 33: ".pushsection .smp_locks,\"a\"\n" \ 34: ".balign 4\n" \ 35: ".long 671f - .\n" /* offset */ \ 36: ".popsection\n" \ 37: "671:" 38: 39: #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " 40: 41: #else /* ! CONFIG_SMP */ 42: #define LOCK_PREFIX_HERE "" 43: #define LOCK_PREFIX "" 44: #endif 45: 46: struct alt_instr { 47: s32 instr_offset; /* original instruction */ 48: s32 repl_offset; /* offset to replacement instruction */ 49: u16 cpuid; /* cpuid bit set for replacement */ 50: u8 instrlen; /* length of original instruction */ 51: u8 replacementlen; /* length of new instruction, <= instrlen */ 52: }; 53: 54: extern void alternative_instructions(void); 55: extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); 56: 57: struct module; 58: 59: #ifdef CONFIG_SMP 60: extern void alternatives_smp_module_add(struct module *mod, char *name, 61: void *locks, void *locks_end, 62: void *text, void *text_end); 63: extern void alternatives_smp_module_del(struct module *mod); 64: extern void alternatives_enable_smp(void); 65: extern int alternatives_text_reserved(void *start, void *end); 66: extern bool skip_smp_alternatives; 67: #else 68: static inline void alternatives_smp_module_add(struct module *mod, char *name, 69: void *locks, void *locks_end, 70: void *text, void *text_end) {} 71: static inline void alternatives_smp_module_del(struct module *mod) {} 72: static inline void alternatives_enable_smp(void) {} 73: static inline int alternatives_text_reserved(void *start, void *end) 74: { 75: return 0; 76: } 77: #endif /* CONFIG_SMP */ 78: 79: #define OLDINSTR(oldinstr) "661:\n\t" oldinstr "\n662:\n" 80: 81: #define b_replacement(number) "663"#number 82: #define e_replacement(number) "664"#number 83: 84: #define alt_slen "662b-661b" 85: #define alt_rlen(number) e_replacement(number)"f-"b_replacement(number)"f" 86: 87: #define ALTINSTR_ENTRY(feature, number) \ 88: " .long 661b - .\n" /* label */ \ 89: " .long " b_replacement(number)"f - .\n" /* new instruction */ \ 90: " .word " __stringify(feature) "\n" /* feature bit */ \ 91: " .byte " alt_slen "\n" /* source len */ \ 92: " .byte " alt_rlen(number) "\n" /* replacement len */ 93: 94: #define DISCARD_ENTRY(number) /* rlen <= slen */ \ 95: " .byte 0xff + (" alt_rlen(number) ") - (" alt_slen ")\n" 96: 97: #define ALTINSTR_REPLACEMENT(newinstr, feature, number) /* replacement */ \ 98: b_replacement(number)":\n\t" newinstr "\n" e_replacement(number) ":\n\t" 99: 100: /* alternative assembly primitive: */ 101: #define ALTERNATIVE(oldinstr, newinstr, feature) \ 102: OLDINSTR(oldinstr) \ 103: ".pushsection .altinstructions,\"a\"\n" \ 104: ALTINSTR_ENTRY(feature, 1) \ 105: ".popsection\n" \ 106: ".pushsection .discard,\"aw\",@progbits\n" \ 107: DISCARD_ENTRY(1) \ 108: ".popsection\n" \ 109: ".pushsection .altinstr_replacement, \"ax\"\n" \ 110: ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ 111: ".popsection" 112: 113: #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ OLDINSTR(oldinstr) \ 115: ".pushsection .altinstructions,\"a\"\n" \ 116: ALTINSTR_ENTRY(feature1, 1) \ 117: ALTINSTR_ENTRY(feature2, 2) \ 118: ".popsection\n" \ 119: ".pushsection .discard,\"aw\",@progbits\n" \ 120: DISCARD_ENTRY(1) \ 121: DISCARD_ENTRY(2) \ 122: ".popsection\n" \ 123: ".pushsection .altinstr_replacement, \"ax\"\n" \ 124: ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ 125: ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ 126: ".popsection" 127: 128: /* 129: * This must be included *after* the definition of ALTERNATIVE due to 130: * <asm/arch_hweight.h> 131: */ 132: #include <asm/cpufeature.h> 133: 134: /* 135: * Alternative instructions for different CPU types or capabilities. 136: * 137: * This allows to use optimized instructions even on generic binary 138: * kernels. 139: * 140: * length of oldinstr must be longer or equal the length of newinstr 141: * It can be padded with nops as needed. 142: * 143: * For non barrier like inlines please define new variants 144: * without volatile and memory clobber. 145: */ 146: #define alternative(oldinstr, newinstr, feature) \ 147: asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory") 148: 149: /* 150: * Alternative inline assembly with input. 151: * 152: * Pecularities: 153: * No memory clobber here. 154: * Argument numbers start with 1. 155: * Best is to use constraints that are fixed size (like (%1) ... "r") 156: * If you use variable sized constraints like "m" or "g" in the 157: * replacement make sure to pad to the worst case length. 158: * Leaving an unused argument 0 to keep API compatibility. 159: */ 160: #define alternative_input(oldinstr, newinstr, feature, input...) \ 161: asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 162: : : "i" (0), ## input) 163: 164: /* Like alternative_input, but with a single output argument */ 165: #define alternative_io(oldinstr, newinstr, feature, output, input...) \ 166: asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \ 167: : output : "i" (0), ## input) 168: 169: /* Like alternative_io, but for replacing a direct call with another one. */ 170: #define alternative_call(oldfunc, newfunc, feature, output, input...) \ 171: asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \ 172: : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input) 173: 174: /* 175: * Like alternative_call, but there are two features and respective functions. 176: * If CPU has feature2, function2 is used. 177: * Otherwise, if CPU has feature1, function1 is used. 178: * Otherwise, old function is used. 179: */ 180: #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \ 181: output, input...) \ 182: asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\ "call %P[new2]", feature2) \ 184: : output : [old] "i" (oldfunc), [new1] "i" (newfunc1), \ 185: [new2] "i" (newfunc2), ## input) 186: 187: /* 188: * use this macro(s) if you need more than one output parameter 189: * in alternative_io 190: */ 191: #define ASM_OUTPUT2(a...) a 192: 193: /* 194: * use this macro if you need clobbers but no inputs in 195: * alternative_{input,io,call}() 196: */ 197: #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr 198: 199: struct paravirt_patch_site; 200: #ifdef CONFIG_PARAVIRT 201: void apply_paravirt(struct paravirt_patch_site *start, 202: struct paravirt_patch_site *end); 203: #else 204: static inline void apply_paravirt(struct paravirt_patch_site *start, 205: struct paravirt_patch_site *end) 206: {} 207: #define __parainstructions NULL 208: #define __parainstructions_end NULL 209: #endif 210: 211: extern void *text_poke_early(void *addr, const void *opcode, size_t len); 212: 213: /* 214: * Clear and restore the kernel write-protection flag on the local CPU. 215: * Allows the kernel to edit read-only pages. 216: * Side-effect: any interrupt handler running between save and restore will have 217: * the ability to write to read-only pages. 218: * 219: * Warning: 220: * Code patching in the UP case is safe if NMIs and MCE handlers are stopped and 221: * no thread can be preempted in the instructions being modified (no iret to an 222: * invalid instruction possible) or if the instructions are changed from a 223: * consistent state to another consistent state atomically. 224: * On the local CPU you need to be protected again NMI or MCE handlers seeing an 225: * inconsistent instruction while you patch. 226: */ 227: extern void *text_poke(void *addr, const void *opcode, size_t len); 228: extern int poke_int3_handler(struct pt_regs *regs); 229: extern void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler); 230: 231: #endif /* _ASM_X86_ALTERNATIVE_H */ 232: