Green shading in the line number column means the source is part of the translation unit, red means it is conditionally excluded. Highlighted line numbers link to the translation unit page. Highlighted macros link to the macro page.
1: #ifndef _LINUX_JUMP_LABEL_H 2: #define _LINUX_JUMP_LABEL_H 3: 4: /* 5: * Jump label support 6: * 7: * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com> 8: * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com> 9: * 10: * Jump labels provide an interface to generate dynamic branches using 11: * self-modifying code. Assuming toolchain and architecture support the result 12: * of a "if (static_key_false(&key))" statement is a unconditional branch (which 13: * defaults to false - and the true block is placed out of line). 14: * 15: * However at runtime we can change the branch target using 16: * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key 17: * object and for as long as there are references all branches referring to 18: * that particular key will point to the (out of line) true block. 19: * 20: * Since this relies on modifying code the static_key_slow_{inc,dec}() functions 21: * must be considered absolute slow paths (machine wide synchronization etc.). 22: * OTOH, since the affected branches are unconditional their runtime overhead 23: * will be absolutely minimal, esp. in the default (off) case where the total 24: * effect is a single NOP of appropriate size. The on case will patch in a jump 25: * to the out-of-line block. 26: * 27: * When the control is directly exposed to userspace it is prudent to delay the 28: * decrement to avoid high frequency code modifications which can (and do) 29: * cause significant performance degradation. Struct static_key_deferred and 30: * static_key_slow_dec_deferred() provide for this. 31: * 32: * Lacking toolchain and or architecture support, it falls back to a simple 33: * conditional branch. 34: * 35: * struct static_key my_key = STATIC_KEY_INIT_TRUE; 36: * 37: * if (static_key_true(&my_key)) { 38: * } 39: * 40: * will result in the true case being in-line and starts the key with a single 41: * reference. Mixing static_key_true() and static_key_false() on the same key is not 42: * allowed. 43: * 44: * Not initializing the key (static data is initialized to 0s anyway) is the 45: * same as using STATIC_KEY_INIT_FALSE. 46: * 47: */ 48: 49: #include <linux/types.h> 50: #include <linux/compiler.h> 51: #include <linux/bug.h> 52: 53: extern bool static_key_initialized; 54: 55: #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \ 56: "%s used before call to jump_label_init", \ 57: __func__) 58: 59: #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL) 60: 61: struct static_key { 62: atomic_t enabled; 63: /* Set lsb bit to 1 if branch is default true, 0 ot */ 64: struct jump_entry *entries; 65: #ifdef CONFIG_MODULES 66: struct static_key_mod *next; 67: #endif 68: }; 69: 70: # include <asm/jump_label.h> 71: # define HAVE_JUMP_LABEL 72: #endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ 73: 74: enum jump_label_type { 75: JUMP_LABEL_DISABLE = 0, 76: JUMP_LABEL_ENABLE, 77: }; 78: 79: struct module; 80: 81: #include <linux/atomic.h> 82: #ifdef HAVE_JUMP_LABEL 83: 84: #define JUMP_LABEL_TRUE_BRANCH 1UL 85: 86: static 87: inline struct jump_entry *jump_label_get_entries(struct static_key *key) 88: { 89: return (struct jump_entry *)((unsigned long)key->entries 90: & ~JUMP_LABEL_TRUE_BRANCH); 91: } 92: 93: static inline bool jump_label_get_branch_default(struct static_key *key) 94: { 95: if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH) 96: return true; 97: return false; 98: } 99: 100: static __always_inline bool static_key_false(struct static_key *key) 101: { 102: return arch_static_branch(key); 103: } 104: 105: static __always_inline bool static_key_true(struct static_key *key) 106: { 107: return !static_key_false(key); 108: } 109: 110: extern struct jump_entry __start___jump_table[]; 111: extern struct jump_entry __stop___jump_table[]; 112: 113: extern void jump_label_init(void); 114: extern void jump_label_lock(void); 115: extern void jump_label_unlock(void); 116: extern void arch_jump_label_transform(struct jump_entry *entry, 117: enum jump_label_type type); 118: extern void arch_jump_label_transform_static(struct jump_entry *entry, 119: enum jump_label_type type); 120: extern int jump_label_text_reserved(void *start, void *end); 121: extern void static_key_slow_inc(struct static_key *key); 122: extern void static_key_slow_dec(struct static_key *key); 123: extern void jump_label_apply_nops(struct module *mod); 124: 125: #define STATIC_KEY_INIT_TRUE ((struct static_key) \ 126: { .enabled = ATOMIC_INIT(1), .entries = (void *)1 }) 127: #define STATIC_KEY_INIT_FALSE ((struct static_key) \ 128: { .enabled = ATOMIC_INIT(0), .entries = (void *)0 }) 129: 130: #else /* !HAVE_JUMP_LABEL */ 131: 132: struct static_key { 133: atomic_t enabled; 134: }; 135: 136: static __always_inline void jump_label_init(void) 137: { 138: static_key_initialized = true; 139: } 140: 141: static __always_inline bool static_key_false(struct static_key *key) 142: { 143: if (unlikely(atomic_read(&key->enabled) > 0)) 144: return true; 145: return false; 146: } 147: 148: static __always_inline bool static_key_true(struct static_key *key) 149: { 150: if (likely(atomic_read(&key->enabled) > 0)) 151: return true; 152: return false; 153: } 154: 155: static inline void static_key_slow_inc(struct static_key *key) 156: { 157: STATIC_KEY_CHECK_USE(); 158: atomic_inc(&key->enabled); 159: } 160: 161: static inline void static_key_slow_dec(struct static_key *key) 162: { 163: STATIC_KEY_CHECK_USE(); 164: atomic_dec(&key->enabled); 165: } 166: 167: static inline int jump_label_text_reserved(void *start, void *end) 168: { 169: return 0; 170: } 171: 172: static inline void jump_label_lock(void) {} 173: static inline void jump_label_unlock(void) {} 174: 175: static inline int jump_label_apply_nops(struct module *mod) 176: { 177: return 0; 178: } 179: 180: #define STATIC_KEY_INIT_TRUE ((struct static_key) \ 181: { .enabled = ATOMIC_INIT(1) }) 182: #define STATIC_KEY_INIT_FALSE ((struct static_key) \ 183: { .enabled = ATOMIC_INIT(0) }) 184: 185: #endif /* HAVE_JUMP_LABEL */ 186: 187: #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE 188: #define jump_label_enabled static_key_enabled 189: 190: static inline bool static_key_enabled(struct static_key *key) 191: { 192: return (atomic_read(&key->enabled) > 0); 193: } 194: 195: #endif /* _LINUX_JUMP_LABEL_H */ 196: