jump_label (setup_arch)

setup_arch jump_label_init 분석

// arch/arm64/kernel/setup.c
void __init __no_sanitize_address setup_arch(char **cmdline_p)
{
    jump_label_init();
}

jump_label_init

• 구조체 정의

// arch/arm/include/asm/jump_label.h
typedef u32 jump_label_t;

struct jump_entry {
    jump_label_t code;
    jump_label_t target;
    jump_label_t key; 
};


struct static_key {
    atomic_t enabled;
#ifdef CONFIG_JUMP_LABEL
/*
 * Note:
 *   To make anonymous unions work with old compilers, the static
 *   initialization of them requires brackets. This creates a dependency
 *   on the order of the struct with the initializers. If any fields
 *   are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
 *   to be modified.
 *
 * bit 0 => 1 if key is initially true
 *        0 if initially false
 * bit 1 => 1 if points to struct static_key_mod
 *        0 if points to struct jump_entry
 */
    union {
        unsigned long type;
        struct jump_entry *entries;
        struct static_key_mod *next;
    };
#endif    /* CONFIG_JUMP_LABEL */
};

• jump label 이란 if 분기를 어셈블리 레벨에서 더 빠르게 처리할 수 있도록 처리하는 방법
• section 에 jump_table 용 공간이 추가됨

// arch/arm64/include/asm/jump_label.h
#define JUMP_TABLE_ENTRY(key, label)            \
    ".pushsection    __jump_table, \"aw\"\n\t"    \
    ".align        3\n\t"                \
    ".long        1b - ., " label " - .\n\t"    \
    ".quad        " key " - .\n\t"        \
    ".popsection\n\t"

void __init jump_label_init(void)
{
    // JUMP TABLE 엔트리 시작, 끝 섹션 주소를 가져옴
    // [    0.000000] iter_start: 0xffff800081c43d70, iter_enx: 0xffff800081c4b210
    struct jump_entry *iter_start = __start___jump_table;
    struct jump_entry *iter_stop = __stop___jump_table;
    struct static_key *key = NULL;
    struct jump_entry *iter;

    /*
     * Since we are initializing the static_key.enabled field with
     * with the 'raw' int values (to avoid pulling in atomic.h) in
     * jump_label.h, let's make sure that is safe. There are only two
     * cases to check since we initialize to 0 or 1.
     */
    BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
    BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);

    if (static_key_initialized)
        return;

    cpus_read_lock();
    jump_label_lock();
    // key, code 순으로 정렬
    jump_label_sort_entries(iter_start, iter_stop);

    for (iter = iter_start; iter < iter_stop; iter++) {
        struct static_key *iterk;
        bool in_init;

        /* rewrite NOPs */
        if (jump_label_type(iter) == JUMP_LABEL_NOP)
            arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); // 아키텍처 별로 nop 이 달라서, nop 을 직접 추가

        in_init = init_section_contains((void *)jump_entry_code(iter), 1);
        jump_entry_set_init(iter, in_init);


        // 중복 처리 하지 않음
        iterk = jump_entry_key(iter);
        if (iterk == key)
            continue;

        key = iterk;
        // jump entry 에 key 의 초기값을 가져와서 설정
        static_key_set_entries(key, iter);
    }
    static_key_initialized = true;
    jump_label_unlock();
    cpus_read_unlock();
}

static void static_key_set_entries(struct static_key *key,
                   struct jump_entry *entries)
{
    unsigned long type;

    WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);
    type = key->type & JUMP_TYPE_MASK;
    key->entries = entries;
    key->type |= type;
}

jump_entry key

static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
{
    long offset = entry->key & ~3L;
    return (struct static_key *)((unsigned long)&entry->key + offset);
}

// key 의 첫번째 비트는 branch 인지 여부로 사용
static inline bool jump_entry_is_branch(const struct jump_entry *entry)
{
    return (unsigned long)entry->key & 1UL;
}

// key 의 두번째 비트는 init 영역에 있는지 여부
static inline void jump_entry_set_init(struct jump_entry *entry, bool set)
{
    if (set)
        entry->key |= 2;
    else
        entry->key &= ~2;
}

jump_label_type

static enum jump_label_type jump_label_type(struct jump_entry *entry)
{
    struct static_key *key = jump_entry_key(entry);
    bool enabled = static_key_enabled(key);
    bool branch = jump_entry_is_branch(entry);

    /* See the comment in linux/jump_label.h */
    return enabled ^ branch;
}

enum jump_label_type {
    JUMP_LABEL_NOP = 0,
    JUMP_LABEL_JMP,
};

• jump_label.h 의 주석을 살펴보자

이름	설명
type	정적 초기값
enabled	동적으로 변경된 값
branch	1 = likely, 0 = unlikely

jump_label_type 은 런타임에 호출되는 값이기 때문에, enabled ^ branch 로 jump_label_type 을 획득 할 수 있다.

/*
 * Combine the right initial value (type) with the right branch order
 * to generate the desired result.
 *
 *
 * type\branch|    likely (1)          |    unlikely (0)
 * -----------+-----------------------+------------------
 *            |                       |
 *  true (1)  |       ...              |       ...
 *            |    NOP              |       JMP L
 *            |    <br-stmts>          |    1: ...
 *            |    L: ...              |
 *            |                  |
 *            |                  |    L: <br-stmts>
 *            |                  |       jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *            |                       |
 *  false (0) |       ...              |       ...
 *            |    JMP L          |       NOP
 *            |    <br-stmts>          |    1: ...
 *            |    L: ...              |
 *            |                  |
 *            |                  |    L: <br-stmts>
 *            |                  |       jmp 1b
 *            |                       |
 * -----------+-----------------------+------------------
 *
 * The initial value is encoded in the LSB of static_key::entries,
 * type: 0 = false, 1 = true.
 *
 * The branch type is encoded in the LSB of jump_entry::key,
 * branch: 0 = unlikely, 1 = likely.
 *
 * This gives the following logic table:
 *
 *    enabled    type    branch      instuction
 * -----------------------------+-----------
 *    0    0    0    | NOP
 *    0    0    1    | JMP
 *    0    1    0    | NOP
 *    0    1    1    | JMP
 *
 *    1    0    0    | JMP
 *    1    0    1    | NOP
 *    1    1    0    | JMP
 *    1    1    1    | NOP
 *
 * Which gives the following functions:
 *
 *   dynamic: instruction = enabled ^ branch
 *   static:  instruction = type ^ branch
 *
 * See jump_label_type() / jump_label_init_type().
 */

사용법

이름	설명
DEFINE_STATIC_KEY_XXX	정적 키 초기화
DECLARE_STATIC_KEY_XXX	외부의 정적키 가져오기
static_key_enabled	return static_key === true
static_key_disbled	return static_key === false
static_branch_likely, static_branch_unlikely	키의 true, false 를 그대로 리턴
static_branch_disable	key 를 false 로 설정
static_branch_enable	key 를 true 로 설정

static DEFINE_STATIC_KEY_TRUE(sk_true);
static DEFINE_STATIC_KEY_FALSE(sk_false);

static __init int jump_label_test(void)
{
    int i;

    for (i = 0; i < 2; i++) {
        WARN_ON(static_key_enabled(&sk_true.key) != true);
        WARN_ON(static_key_enabled(&sk_false.key) != false);

        WARN_ON(!static_branch_likely(&sk_true));
        WARN_ON(!static_branch_unlikely(&sk_true));
        WARN_ON(static_branch_likely(&sk_false));
        WARN_ON(static_branch_unlikely(&sk_false));

        static_branch_disable(&sk_true);
        static_branch_enable(&sk_false);

        WARN_ON(static_key_enabled(&sk_true.key) == true);
        WARN_ON(static_key_enabled(&sk_false.key) == false);

        WARN_ON(static_branch_likely(&sk_true));
        WARN_ON(static_branch_unlikely(&sk_true));
        WARN_ON(!static_branch_likely(&sk_false));
        WARN_ON(!static_branch_unlikely(&sk_false));

        static_branch_enable(&sk_true);
        static_branch_disable(&sk_false);
    }

    return 0;
}

총정리

이름	설명
static_key	현재 키가 켜져있는지 꺼져있는지
jump_entry	분기 명령 위치, 분기할 주소 위치, key 와 초기 설정값 저장
static_key_enabled	이 둘을 연결하여 분기를 수행하는 함수

if (static_branch_likely(&key))
    do_something();

• static_branch_likely 이므로 branch = 1

key 가 true 였다면

if (NOP) // enabled (1) ^ branch (1) = 0 (NOP)
  do_something();

key 가 false 였다면

if (JMP) // enabled (0) ^ branch (1) = 1 (JMP)
  do_something();

브랜치에 점프 테이블을 등록해두고 기본 값으로 설정

#define static_branch_likely(x)                            \
({                                        \
    bool branch;                                \
    if (__builtin_types_compatible_p(typeof(*x), struct static_key_true))    \ // static_branch_likely 에서 true type 인 경우
        branch = !arch_static_branch(&(x)->key, true);            \ // branch 를 안타는게 기본. if 를 바로 실행할테니
    else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
        branch = !arch_static_branch_jump(&(x)->key, true);        \
    else                                    \
        branch = ____wrong_branch_error();                \
    likely_notrace(branch);                                \
})

/* This macro is also expanded on the Rust side. */
#define ARCH_STATIC_BRANCH_ASM(key, label)        \
    "1:    nop\n\t"                \
    JUMP_TABLE_ENTRY(key, label)

// 기본이 NOP 인 브랜치. 그래도 점프 테이블에 등록해야 나중에 변경 가능
static __always_inline bool arch_static_branch(struct static_key * const key,
                           const bool branch)
{
    char *k = &((char *)key)[branch];

    asm goto(
        ARCH_STATIC_BRANCH_ASM("%c0", "%l[l_yes]")
        :  :  "i"(k) :  : l_yes
        );

    return false;
l_yes:
    return true;
}

// 기본이  JMP 인 브랜치.
static __always_inline bool arch_static_branch_jump(struct static_key *key,
                            bool branch)
{
    asm goto(".balign "__stringify(JUMP_LABEL_NOP_SIZE)"        \n"
         "1:                            \n" // 1번 라벨
         "b %l[l_yes]                        \n" // likely 니까 점프 할거야
         ".pushsection __jump_table, \"aw\"            \n" // jump table 에 등록
         ".word 1b, %l[l_yes], %c0                \n" // 1번 라벨에서 l_yes 로 뛸거고 key 값 설정
         ".popsection                        \n"
         : : "i" (&((char *)key)[branch]) : : l_yes);

    return false;
l_yes:
    return true;
}

asm goto (assembly_code_string
          : output operands
          : input operands
          : clobbers
          : goto labels);

jump_entry 가 변경될때 text 영역에서 nop <-> jmp 를 변경

bool arch_jump_label_transform_queue(struct jump_entry *entry,
                     enum jump_label_type type)
{
    void *addr = (void *)jump_entry_code(entry);
    u32 insn;

    if (type == JUMP_LABEL_JMP) {
        insn = aarch64_insn_gen_branch_imm(jump_entry_code(entry),
                           jump_entry_target(entry),
                           AARCH64_INSN_BRANCH_NOLINK);
    } else {
        insn = aarch64_insn_gen_nop();
    }

    aarch64_insn_patch_text_nosync(addr, insn);
    return true;
}