/* $NetBSD: tls.c,v 1.29 2026/03/15 13:56:17 skrll Exp $ */ /*- * Copyright (c) 2011 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Joerg Sonnenberger. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include <sys/cdefs.h> __RCSID("$NetBSD: tls.c,v 1.29 2026/03/15 13:56:17 skrll Exp $"); /* * Thread-local storage * * Reference: * * [ELFTLS] Ulrich Drepper, `ELF Handling For Thread-Local * Storage', Version 0.21, 2023-08-22. * https://akkadia.org/drepper/tls.pdf * https://web.archive.org/web/20240718081934/https://akkadia.org/drepper/tls.pdf */ #include <sys/param.h> #include <sys/ucontext.h> #include <lwp.h> #include <stdalign.h> #include <stddef.h> #include <string.h> #include "debug.h" #include "rtld.h" #include <machine/lwp_private.h> #if defined(__HAVE_TLS_VARIANT_I) || defined(__HAVE_TLS_VARIANT_II) static struct tls_tcb *_rtld_tls_allocate_locked(void); static void *_rtld_tls_module_allocate(struct tls_tcb *, size_t); /* A macro to test correct alignment of a pointer. */ #define ALIGNED_P(ptr, algnmt) ((algnmt) == 0 || ((uintptr_t)(ptr) & ((algnmt) - 1)) == 0) /* * DTV offset * * On some architectures (m68k, mips, or1k, powerpc, and riscv), * the DTV offsets passed to __tls_get_addr have a bias relative * to the start of the DTV, in order to maximize the range of TLS * offsets that can be used by instruction encodings with signed * displacements. */ #ifndef TLS_DTV_OFFSET #define TLS_DTV_OFFSET 0 #endif /* * Alignment of the static data * * In Variant I, the thread pointer (TP) can be anchored in three ways * depending on the architecture. It either points * * - directly to the TCB (e.g. Arm, and AArch64); or * - at a specific "biased" offset within the data (e.g. PowerPC), or * directly at the data (e.g. RISC-V). These architectures define * __HAVE___LWP_SETTCB to handle the offset * * An area of memory with the correct alignment is allocated and the * struct tcb placed as follows for each case above * * - at the start of the aligned memory with data starting at the first * object's required alignment. * - just below the second max alignment boundary so that data starts * on the second max alignment boundary. * * The code is written such that obj->tlsoffset is always relative to * the end of the struct tcb. Maybe this is suboptimal? * */ #if defined(__HAVE_TLS_VARIANT_I) && !defined(__HAVE___LWP_SETTCB) #define _RTLD_TLS_INITIAL_OFFSET sizeof(struct tls_tcb) #endif #ifndef _RTLD_TLS_INITIAL_OFFSET #define _RTLD_TLS_INITIAL_OFFSET 0 #endif static size_t _rtld_tls_static_space; /* Static TLS space allocated */ static size_t _rtld_tls_static_offset = _RTLD_TLS_INITIAL_OFFSET; /* Next offset for static TLS to use */ static size_t _rtld_tls_static_max_align; size_t _rtld_tls_dtv_generation = 1; /* Bumped on each load of obj w/ TLS */ size_t _rtld_tls_max_index = 1; /* Max index into up-to-date DTV */ /* * DTV -- Dynamic Thread Vector * * The DTV is a per-thread array that maps each module with * thread-local storage to a pointer into part of the thread's TCB * (thread control block), or dynamically loaded TLS blocks, * reserved for that module's storage. * * The TCB itself, struct tls_tcb, has a pointer to the DTV at * tcb->tcb_dtv. * * The layout is: * * +---------------+ * | max index | -1 max index i for which dtv[i] is alloced * +---------------+ * | generation | 0 void **dtv points here * +---------------+ * | obj 1 tls ptr | 1 TLS pointer for obj w/ obj->tlsindex 1 * +---------------+ * | obj 2 tls ptr | 2 TLS pointer for obj w/ obj->tlsindex 2 * +---------------+ * . * . * . * * The values of obj->tlsindex start at 1; this way, * dtv[obj->tlsindex] works, when dtv[0] is the generation. The * TLS pointers go either into the static thread-local storage, * for the initial objects (i.e., those loaded at startup), or * into TLS blocks dynamically allocated for objects that * dynamically loaded by dlopen. * * The generation field is a cache of the global generation number * _rtld_tls_dtv_generation, which is bumped every time an object * with TLS is loaded in _rtld_map_object, and cached by * __tls_get_addr (via _rtld_tls_get_addr) when a newly loaded * module lies outside the bounds of the current DTV. * * XXX Why do we keep max index and generation separately? They * appear to be initialized the same, always incremented together, * and always stored together. * * XXX Why is this not a struct? * * struct dtv { * size_t dtv_gen; * void *dtv_module[]; * }; */ #define DTV_GENERATION(dtv) ((size_t)((dtv)[0])) #define DTV_MAX_INDEX(dtv) ((size_t)((dtv)[-1])) #define SET_DTV_GENERATION(dtv, val) (dtv)[0] = (void *)(size_t)(val) #define SET_DTV_MAX_INDEX(dtv, val) (dtv)[-1] = (void *)(size_t)(val) /* * _rtld_tls_get_addr(tcb, idx, offset) * * Slow path for __tls_get_addr (see below), called to allocate * TLS space if needed for the object obj with obj->tlsindex idx, * at offset, which must be below obj->tlssize. * * This may allocate a DTV if the current one is too old, and it * may allocate a dynamically loaded TLS block if there isn't one * already allocated for it. * * XXX Why is the first argument passed as `void *tls' instead of * just `struct tls_tcb *tcb'? */ void * _rtld_tls_get_addr(void *tls, size_t idx, size_t offset) { struct tls_tcb *tcb = tls; void **dtv, **new_dtv; sigset_t mask; _rtld_exclusive_enter(&mask); dtv = tcb->tcb_dtv; /* * If the generation number has changed, we have to allocate a * new DTV. * * XXX Do we really? Isn't it enough to check whether idx <= * DTV_MAX_INDEX(dtv)? */ if (__predict_false(DTV_GENERATION(dtv) != _rtld_tls_dtv_generation)) { size_t to_copy = DTV_MAX_INDEX(dtv); /* * "2 +" because the first element is the generation and * the second one is the maximum index. */ new_dtv = xcalloc((2 + _rtld_tls_max_index) * sizeof(*dtv)); ++new_dtv; /* advance past DTV_MAX_INDEX */ if (to_copy > _rtld_tls_max_index) /* XXX How? */ to_copy = _rtld_tls_max_index; memcpy(new_dtv + 1, dtv + 1, to_copy * sizeof(*dtv)); xfree(dtv - 1); /* retreat back to DTV_MAX_INDEX */ dtv = tcb->tcb_dtv = new_dtv; SET_DTV_MAX_INDEX(dtv, _rtld_tls_max_index); SET_DTV_GENERATION(dtv, _rtld_tls_dtv_generation); } if (__predict_false(dtv[idx] == NULL)) dtv[idx] = _rtld_tls_module_allocate(tcb, idx); _rtld_exclusive_exit(&mask); return (uint8_t *)dtv[idx] + offset; } /* * _rtld_tls_initial_allocation() * * Allocate the TCB (thread control block) for the initial thread, * once the static TLS space usage has been determined (plus some * slop to allow certain special cases like Mesa to be dlopened). * * This must be done _after_ all initial objects (i.e., those * loaded at startup, as opposed to objects dynamically loaded by * dlopen) have had TLS offsets allocated if need be by * _rtld_tls_offset_allocate, and have had relocations processed. */ void _rtld_tls_initial_allocation(void) { struct tls_tcb *tcb; _rtld_tls_static_space = _rtld_tls_static_offset + RTLD_STATIC_TLS_RESERVATION; #ifndef __HAVE_TLS_VARIANT_I _rtld_tls_static_space = roundup2(_rtld_tls_static_space, alignof(max_align_t)); #ifdef __HAVE___LWP_SETTCB if (_rtld_tls_static_max_align > sizeof(struct tls_tcb)) _rtld_tls_static_space += _rtld_tls_static_max_align - sizeof(struct tls_tcb); #endif #endif dbg(("_rtld_tls_static_space %zu", _rtld_tls_static_space)); tcb = _rtld_tls_allocate_locked(); #ifdef __HAVE___LWP_SETTCB __lwp_settcb(tcb); #else _lwp_setprivate(tcb); #endif } /* * _rtld_tls_allocate_locked() * * Internal subroutine to allocate a TCB (thread control block) * for the current thread. * * This allocates a DTV and a TCB that points to it, including * static space in the TCB for the TLS of the initial objects. * TLS blocks for dynamically loaded objects are allocated lazily. * * Caller must either be single-threaded (at startup via * _rtld_tls_initial_allocation) or hold the rtld exclusive lock * (via _rtld_tls_allocate). */ static struct tls_tcb * _rtld_tls_allocate_locked(void) { Obj_Entry *obj; struct tls_tcb *tcb; uint8_t *p, *q; p = xmalloc_aligned(_rtld_tls_static_space + sizeof(struct tls_tcb), _rtld_tls_static_max_align, 0); memset(p, 0, _rtld_tls_static_space + sizeof(struct tls_tcb)); #ifdef __HAVE_TLS_VARIANT_I #ifdef __HAVE___LWP_SETTCB if (_rtld_tls_static_max_align > sizeof(struct tls_tcb)) p += _rtld_tls_static_max_align - sizeof(struct tls_tcb); #endif tcb = (struct tls_tcb *)p; p += sizeof(struct tls_tcb); #else p += _rtld_tls_static_space; tcb = (struct tls_tcb *)p; tcb->tcb_self = tcb; #endif dbg(("lwp %d tls tcb %p", _lwp_self(), tcb)); /* * "2 +" because the first element is the generation and the second * one is the maximum index. */ tcb->tcb_dtv = xcalloc(sizeof(*tcb->tcb_dtv) * (2 + _rtld_tls_max_index)); ++tcb->tcb_dtv; /* advance past DTV_MAX_INDEX */ SET_DTV_MAX_INDEX(tcb->tcb_dtv, _rtld_tls_max_index); SET_DTV_GENERATION(tcb->tcb_dtv, _rtld_tls_dtv_generation); for (obj = _rtld_objlist; obj != NULL; obj = obj->next) { if (obj->tls_static) { #ifdef __HAVE_TLS_VARIANT_I q = p + obj->tlsoffset; #else q = p - obj->tlsoffset; #endif dbg(("%s: [lwp %d] tls dtv %p-%p index %zu " "offset %zx alignment %zx tlsinit %p%s", obj->path, _lwp_self(), q, q + obj->tlsinitsize, obj->tlsindex, obj->tlsoffset, obj->tlsalign, obj->tlsinit, ALIGNED_P(q, obj->tlsalign) ? "" : " BAD ALIGNMENT")); assert(ALIGNED_P(q, obj->tlsalign)); if (obj->tlsinitsize) memcpy(q, obj->tlsinit, obj->tlsinitsize); tcb->tcb_dtv[obj->tlsindex] = q; } } return tcb; } /* * _rtld_tls_allocate() * * Allocate a TCB (thread control block) for the current thread. * * Called by pthread_create for non-initial threads. (The initial * thread's TCB is allocated by _rtld_tls_initial_allocation.) */ struct tls_tcb * _rtld_tls_allocate(void) { struct tls_tcb *tcb; sigset_t mask; _rtld_exclusive_enter(&mask); tcb = _rtld_tls_allocate_locked(); _rtld_exclusive_exit(&mask); return tcb; } /* * _rtld_tls_free(tcb) * * Free a TCB allocated with _rtld_tls_allocate. * * Frees any TLS blocks for dynamically loaded objects that tcb's * DTV points to, and frees tcb's DTV, and frees tcb. */ void _rtld_tls_free(struct tls_tcb *tcb) { size_t i, max_index; uint8_t *p, *p_end; sigset_t mask; _rtld_exclusive_enter(&mask); #ifdef __HAVE_TLS_VARIANT_I p = (uint8_t *)tcb; #else p = (uint8_t *)tcb - _rtld_tls_static_space; #endif p_end = p + _rtld_tls_static_space; max_index = DTV_MAX_INDEX(tcb->tcb_dtv); for (i = 1; i <= max_index; ++i) { if ((uint8_t *)tcb->tcb_dtv[i] < p || (uint8_t *)tcb->tcb_dtv[i] >= p_end) xfree(tcb->tcb_dtv[i]); } xfree(tcb->tcb_dtv - 1); /* retreat back to DTV_MAX_INDEX */ xfree(p); _rtld_exclusive_exit(&mask); } /* * _rtld_tls_module_allocate(tcb, idx) * * Allocate thread-local storage in the thread with the given TCB * (thread control block) for the object obj whose obj->tlsindex * is idx. * * If obj has had space in static TLS reserved (obj->tls_static), * return a pointer into that. Otherwise, allocate a TLS block, * mark obj as having a TLS block allocated (obj->tls_dynamic), * and return it. * * Called by _rtld_tls_get_addr to get the thread-local storage * for an object the first time around. */ static void * _rtld_tls_module_allocate(struct tls_tcb *tcb, size_t idx) { Obj_Entry *obj; uint8_t *p; for (obj = _rtld_objlist; obj != NULL; obj = obj->next) { if (obj->tlsindex == idx) break; } if (obj == NULL) { _rtld_error("Module for TLS index %zu missing", idx); _rtld_die(); } if (obj->tls_static) { #ifdef __HAVE_TLS_VARIANT_I p = (uint8_t *)tcb + obj->tlsoffset + sizeof(struct tls_tcb); #else p = (uint8_t *)tcb - obj->tlsoffset; #endif return p; } p = xmalloc_aligned(obj->tlssize, obj->tlsalign, 0); memcpy(p, obj->tlsinit, obj->tlsinitsize); memset(p + obj->tlsinitsize, 0, obj->tlssize - obj->tlsinitsize); obj->tls_dynamic = 1; return p; } /* * _rtld_tls_offset_allocate(obj) * * Allocate a static thread-local storage offset for obj. * * Called by _rtld at startup for all initial objects. Called * also by MD relocation logic, which is allowed (for Mesa) to * allocate an additional 64 bytes (RTLD_STATIC_TLS_RESERVATION) * of static thread-local storage in dlopened objects. */ int _rtld_tls_offset_allocate(Obj_Entry *obj) { size_t offset, next_offset; if (obj->tls_dynamic) return -1; if (obj->tls_static) return 0; if (obj->tlssize == 0) { obj->tlsoffset = 0; obj->tls_static = 1; return 0; } #ifdef __HAVE_TLS_VARIANT_I offset = roundup2(_rtld_tls_static_offset, obj->tlsalign); next_offset = offset + obj->tlssize; #ifndef __HAVE___LWP_GETTCB_FAST offset -= sizeof(struct tls_tcb); #endif #else offset = roundup2(_rtld_tls_static_offset + obj->tlssize, obj->tlsalign); next_offset = offset; #endif /* * Check if the static allocation was already done. * This happens if dynamically loaded modules want to use * static TLS space. * * XXX Keep an actual free list and callbacks for initialisation. */ if (_rtld_tls_static_space) { if (obj->tlsinitsize) { _rtld_error("%s: Use of initialized " "Thread Local Storage with model initial-exec " "and dlopen is not supported", obj->path); return -1; } if (next_offset > _rtld_tls_static_space) { _rtld_error("%s: No space available " "for static Thread Local Storage", obj->path); return -1; } } if (obj->tlsalign > _rtld_tls_static_max_align) { _rtld_tls_static_max_align = obj->tlsalign; } obj->tlsoffset = offset; dbg(("%s: static tls offset 0x%zx size %zu align %zu (%zx/%zx)", obj->path, obj->tlsoffset, obj->tlssize, obj->tlsalign, _rtld_tls_static_offset, next_offset)); _rtld_tls_static_offset = next_offset; obj->tls_static = 1; return 0; } /* * _rtld_tls_offset_free(obj) * * Free a static thread-local storage offset for obj. * * Called by dlclose (via _rtld_unload_object -> _rtld_obj_free). * * Since static thread-local storage is normally not used by * dlopened objects (with the exception of Mesa), this doesn't do * anything to recycle the space right now. */ void _rtld_tls_offset_free(Obj_Entry *obj) { /* * XXX See above. */ obj->tls_static = 0; return; } #if defined(__HAVE_COMMON___TLS_GET_ADDR) && defined(RTLD_LOADER) /* * __tls_get_addr(tlsindex) * * Symbol directly called by code generated by the compiler for * references thread-local storage in the general-dynamic or * local-dynamic TLS models (but not initial-exec or local-exec). * * The argument is a pointer to * * struct { * unsigned long int ti_module; * unsigned long int ti_offset; * }; * * as in, e.g., [ELFTLS] Sec. 3.4.3. This coincides with the * type size_t[2] on all architectures that use this common * __tls_get_addr definition (XXX but why do we write it as * size_t[2]?). * * ti_module, i.e., arg[0], is the obj->tlsindex assigned at * load-time by _rtld_map_object, and ti_offset, i.e., arg[1], is * assigned at link-time by ld(1), possibly adjusted by * TLS_DTV_OFFSET. * * Some architectures -- specifically IA-64 -- use a different * calling convention. Some architectures -- specifically i386 * -- also use another entry point ___tls_get_addr (that's three * leading underscores) with a different calling convention. */ void * __tls_get_addr(void *arg_) { size_t *arg = (size_t *)arg_; void **dtv; #ifdef __HAVE___LWP_GETTCB_FAST struct tls_tcb * const tcb = __lwp_gettcb_fast(); #else struct tls_tcb * const tcb = __lwp_getprivate_fast(); #endif size_t idx = arg[0], offset = arg[1] + TLS_DTV_OFFSET; dtv = tcb->tcb_dtv; /* * Fast path: access to an already allocated DTV entry. This * checks the current limit and the entry without needing any * locking. Entries are only freed on dlclose() and it is an * application bug if code of the module is still running at * that point. */ if (__predict_true(idx <= DTV_MAX_INDEX(dtv) && dtv[idx] != NULL)) return (uint8_t *)dtv[idx] + offset; return _rtld_tls_get_addr(tcb, idx, offset); } #endif #endif /* __HAVE_TLS_VARIANT_I || __HAVE_TLS_VARIANT_II */
