mm/kfence/kfence.h

root/mm/kfence/kfence.h
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Kernel Electric-Fence (KFENCE). For more info please see
 * Documentation/dev-tools/kfence.rst.
 *
 * Copyright (C) 2020, Google LLC.
 */

#ifndef MM_KFENCE_KFENCE_H
#define MM_KFENCE_KFENCE_H

#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>

#include "../slab.h" /* for struct kmem_cache */

/*
 * Get the canary byte pattern for @addr. Use a pattern that varies based on the
 * lower 3 bits of the address, to detect memory corruptions with higher
 * probability, where similar constants are used.
 */
#define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))

/*
 * Define a continuous 8-byte canary starting from a multiple of 8. The canary
 * of each byte is only related to the lowest three bits of its address, so the
 * canary of every 8 bytes is the same. 64-bit memory can be filled and checked
 * at a time instead of byte by byte to improve performance.
 */
#define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(le64_to_cpu(0x0706050403020100)))

/* Maximum stack depth for reports. */
#define KFENCE_STACK_DEPTH 64

extern raw_spinlock_t kfence_freelist_lock;

/* KFENCE object states. */
enum kfence_object_state {
        KFENCE_OBJECT_UNUSED,           /* Object is unused. */
        KFENCE_OBJECT_ALLOCATED,        /* Object is currently allocated. */
        KFENCE_OBJECT_RCU_FREEING,      /* Object was allocated, and then being freed by rcu. */
        KFENCE_OBJECT_FREED,            /* Object was allocated, and then freed. */
};

/* Alloc/free tracking information. */
struct kfence_track {
        pid_t pid;
        int cpu;
        u64 ts_nsec;
        int num_stack_entries;
        unsigned long stack_entries[KFENCE_STACK_DEPTH];
};

/* KFENCE metadata per guarded allocation. */
struct kfence_metadata {
        struct list_head list __guarded_by(&kfence_freelist_lock);      /* Freelist node. */
        struct rcu_head rcu_head;       /* For delayed freeing. */

        /*
         * Lock protecting below data; to ensure consistency of the below data,
         * since the following may execute concurrently: __kfence_alloc(),
         * __kfence_free(), kfence_handle_page_fault(). However, note that we
         * cannot grab the same metadata off the freelist twice, and multiple
         * __kfence_alloc() cannot run concurrently on the same metadata.
         */
        raw_spinlock_t lock;

        /* The current state of the object; see above. */
        enum kfence_object_state state;

        /*
         * Allocated object address; cannot be calculated from size, because of
         * alignment requirements.
         *
         * Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
         */
        unsigned long addr;

        /*
         * The size of the original allocation.
         */
        size_t size;

        /*
         * The kmem_cache cache of the last allocation; NULL if never allocated
         * or the cache has already been destroyed.
         */
        struct kmem_cache *cache;

        /*
         * In case of an invalid access, the page that was unprotected; we
         * optimistically only store one address.
         */
        unsigned long unprotected_page __guarded_by(&lock);

        /* Allocation and free stack information. */
        struct kfence_track alloc_track __guarded_by(&lock);
        struct kfence_track free_track __guarded_by(&lock);
        /* For updating alloc_covered on frees. */
        u32 alloc_stack_hash __guarded_by(&lock);
#ifdef CONFIG_MEMCG
        struct slabobj_ext obj_exts;
#endif
};

#define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \
                                        CONFIG_KFENCE_NUM_OBJECTS)

extern struct kfence_metadata *kfence_metadata;

static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
{
        long index;

        /* The checks do not affect performance; only called from slow-paths. */

        if (!is_kfence_address((void *)addr))
                return NULL;

        /*
         * May be an invalid index if called with an address at the edge of
         * __kfence_pool, in which case we would report an "invalid access"
         * error.
         */
        index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
        if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
                return NULL;

        return &kfence_metadata[index];
}

/* KFENCE error types for report generation. */
enum kfence_error_type {
        KFENCE_ERROR_OOB,               /* Detected a out-of-bounds access. */
        KFENCE_ERROR_UAF,               /* Detected a use-after-free access. */
        KFENCE_ERROR_CORRUPTION,        /* Detected a memory corruption on free. */
        KFENCE_ERROR_INVALID,           /* Invalid access of unknown type. */
        KFENCE_ERROR_INVALID_FREE,      /* Invalid free. */
};

void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
                         const struct kfence_metadata *meta, enum kfence_error_type type);

void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) __must_hold(&meta->lock);

#endif /* MM_KFENCE_KFENCE_H */
Linux
