/* * Copyright (c) 2009 The DragonFly Project. All rights reserved. * * This code is derived from software contributed to The DragonFly Project * by Matthew Dillon <dillon@backplane.com> * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * 3. Neither the name of The DragonFly Project nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific, prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SYS_MUTEX2_H_ #define _SYS_MUTEX2_H_ #ifndef _SYS_MUTEX_H_ #include <sys/mutex.h> #endif #ifndef _SYS_THREAD2_H_ #include <sys/thread2.h> #endif #ifndef _SYS_GLOBALDATA_H_ #include <sys/globaldata.h> #endif #include <machine/atomic.h> /* * Initialize a new mutex, placing it in an unlocked state with no refs. */ static __inline void mtx_init(mtx_t *mtx, const char *ident) { mtx->mtx_lock = 0; mtx->mtx_flags = 0; mtx->mtx_owner = NULL; mtx->mtx_exlink = NULL; mtx->mtx_shlink = NULL; mtx->mtx_ident = ident; } static __inline void mtx_init_flags(mtx_t *mtx, const char *ident, uint32_t flags) { mtx->mtx_lock = 0; mtx->mtx_flags = flags; mtx->mtx_owner = NULL; mtx->mtx_exlink = NULL; mtx->mtx_shlink = NULL; mtx->mtx_ident = ident; } /* * Initialize a mtx link structure for deeper control over the mutex * operation. */ static __inline void mtx_link_init(mtx_link_t *link) { link->state = MTX_LINK_IDLE; link->callback = NULL; link->arg = NULL; } /* * A link structure initialized this way causes mutex operations to not block, * caller must specify a callback. Caller may still abort the mutex via * the link. */ static __inline void mtx_link_init_async(mtx_link_t *link, void (*callback)(mtx_link_t *link, void *arg, int error), void *arg) { link->state = MTX_LINK_IDLE; link->callback = callback; link->arg = arg; } /* * Deinitialize a mutex */ static __inline void mtx_uninit(mtx_t *mtx) { /* empty */ } /* * Exclusive-lock a mutex, block until acquired or aborted. Recursion * is allowed. * * This version of the function allows the mtx_link to be passed in, thus * giving the caller visibility for the link structure which is required * when calling mtx_abort_ex_link() or when requesting an asynchronous lock. * * The mutex may be aborted at any time while the passed link structure * is valid. */ static __inline int mtx_lock_ex_link(mtx_t *mtx, mtx_link_t *link, int flags, int to) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) return(_mtx_lock_ex_link(mtx, link, flags, to)); mtx->mtx_owner = curthread; link->state = MTX_LINK_ACQUIRED; return(0); } /* * Short-form exclusive-lock a mutex, block until acquired. Recursion is * allowed. This is equivalent to mtx_lock_ex(mtx, 0, 0). */ static __inline void mtx_lock(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) { _mtx_lock_ex(mtx, 0, 0); return; } mtx->mtx_owner = curthread; } /* * Exclusive-lock a mutex, block until acquired. Recursion is allowed. * * Returns 0 on success, or the tsleep() return code on failure. * An error can only be returned if PCATCH is specified in the flags. */ static __inline int mtx_lock_ex(mtx_t *mtx, int flags, int to) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) return(_mtx_lock_ex(mtx, flags, to)); mtx->mtx_owner = curthread; return(0); } static __inline int mtx_lock_ex_quick(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) return(_mtx_lock_ex_quick(mtx)); mtx->mtx_owner = curthread; return(0); } static __inline int mtx_lock_sh_link(mtx_t *mtx, mtx_link_t *link, int flags, int to) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0) return(_mtx_lock_sh_link(mtx, link, flags, to)); link->state = MTX_LINK_ACQUIRED; return(0); } /* * Share-lock a mutex, block until acquired. Recursion is allowed. * * Returns 0 on success, or the tsleep() return code on failure. * An error can only be returned if PCATCH is specified in the flags. */ static __inline int mtx_lock_sh(mtx_t *mtx, int flags, int to) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0) return(_mtx_lock_sh(mtx, flags, to)); return(0); } static __inline int mtx_lock_sh_quick(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0) return(_mtx_lock_sh_quick(mtx)); return(0); } /* * Adds a shared lock reference to a lock already locked shared, * does not block on pending exclusive request. */ static __inline void mtx_lock_sh_again(mtx_t *mtx) { KKASSERT((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 && (mtx->mtx_lock & MTX_MASK) > 0); atomic_add_int(&mtx->mtx_lock, 1); } /* * Short-form exclusive spinlock a mutex. Must be paired with * mtx_spinunlock(). */ static __inline void mtx_spinlock(mtx_t *mtx) { globaldata_t gd = mycpu; /* * Predispose a hard critical section */ crit_enter_quick(gd->gd_curthread); ++gd->gd_spinlocks; cpu_ccfence(); /* * If we cannot get it trivially get it the hard way. * * Note that mtx_owner will be set twice if we fail to get it * trivially, but there's no point conditionalizing it as a * conditional will be slower. */ if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) _mtx_spinlock(mtx); mtx->mtx_owner = gd->gd_curthread; } static __inline int mtx_spinlock_try(mtx_t *mtx) { globaldata_t gd = mycpu; /* * Predispose a hard critical section */ crit_enter_quick(gd->gd_curthread); ++gd->gd_spinlocks; cpu_ccfence(); /* * If we cannot get it trivially call _mtx_spinlock_try(). This * function will clean up the hard critical section if it fails. */ if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) return(_mtx_spinlock_try(mtx)); mtx->mtx_owner = gd->gd_curthread; return (0); } /* * Short-form exclusive-lock a mutex, spin until acquired. Recursion is * allowed. This form is identical to mtx_spinlock_ex(). * * Attempt to exclusive-lock a mutex, return 0 on success and * EAGAIN on failure. */ static __inline int mtx_lock_ex_try(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, MTX_EXCLUSIVE | 1) == 0) return (_mtx_lock_ex_try(mtx)); mtx->mtx_owner = curthread; return (0); } /* * Attempt to share-lock a mutex, return 0 on success and * EAGAIN on failure. */ static __inline int mtx_lock_sh_try(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 0, 1) == 0) return (_mtx_lock_sh_try(mtx)); return (0); } /* * If the lock is held exclusively it must be owned by the caller. If the * lock is already a shared lock this operation is a NOP. A panic will * occur if the lock is not held either shared or exclusive. * * The exclusive count is converted to a shared count. */ static __inline void mtx_downgrade(mtx_t *mtx) { globaldata_t gd __debugvar = mycpu; KKASSERT((mtx->mtx_lock & MTX_EXCLUSIVE) && mtx->mtx_owner == gd->gd_curthread); mtx->mtx_owner = NULL; if (atomic_cmpset_int(&mtx->mtx_lock, MTX_EXCLUSIVE | 1, 1) == 0) _mtx_downgrade(mtx); } /* * Upgrade a shared lock to an exclusive lock. The upgrade will fail if * the shared lock has a count other then 1. Optimize the most likely case * but note that a single cmpset can fail due to WANTED races. * * If the lock is held exclusively it must be owned by the caller and * this function will simply return without doing anything. A panic will * occur if the lock is held exclusively by someone other then the caller. * * Returns 0 on success, EDEADLK on failure. */ static __inline int mtx_upgrade_try(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 1, MTX_EXCLUSIVE | 1)) { mtx->mtx_owner = curthread; return(0); } return (_mtx_upgrade_try(mtx)); } /* * Optimized unlock cases. * * NOTE: mtx_unlock() handles any type of mutex: exclusive, shared, and * both blocking and spin methods. * * The mtx_unlock_ex/sh() forms are optimized for exclusive or shared * mutexes and produce less code, but it is ok for code to just use * mtx_unlock() and, in fact, if code uses the short-form mtx_lock() * or mtx_spinlock() to lock it should also use mtx_unlock() to unlock. */ static __inline void mtx_unlock(mtx_t *mtx) { globaldata_t gd __debugvar = mycpu; u_int lock = mtx->mtx_lock; KKASSERT((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 || mtx->mtx_owner == gd->gd_curthread); if (lock == (MTX_EXCLUSIVE | 1)) { mtx->mtx_owner = NULL; if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0) _mtx_unlock(mtx); } else if (lock == 1) { if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0) _mtx_unlock(mtx); } else { _mtx_unlock(mtx); } } static __inline void mtx_unlock_ex(mtx_t *mtx) { globaldata_t gd __debugvar = mycpu; u_int lock = mtx->mtx_lock; KKASSERT((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 || mtx->mtx_owner == gd->gd_curthread); if (lock == (MTX_EXCLUSIVE | 1)) { mtx->mtx_owner = NULL; if (atomic_cmpset_int(&mtx->mtx_lock, lock, 0) == 0) _mtx_unlock(mtx); } else { _mtx_unlock(mtx); } } static __inline void mtx_unlock_sh(mtx_t *mtx) { if (atomic_cmpset_int(&mtx->mtx_lock, 1, 0) == 0) _mtx_unlock(mtx); } /* * NOTE: spinlocks are exclusive-only */ static __inline void mtx_spinunlock(mtx_t *mtx) { globaldata_t gd = mycpu; mtx_unlock(mtx); cpu_ccfence(); --gd->gd_spinlocks; crit_exit_quick(gd->gd_curthread); } /* * Return TRUE (non-zero) if the mutex is locked shared or exclusive by * anyone, including the owner. */ static __inline int mtx_islocked(mtx_t *mtx) { return(mtx->mtx_lock != 0); } /* * Return TRUE (non-zero) if the mutex is locked exclusively by anyone, * including the owner. Returns FALSE (0) if the mutex is unlocked or * if it is locked shared by one or more entities. * * A caller wishing to check whether a lock is owned exclusively by it * should use mtx_owned(). */ static __inline int mtx_islocked_ex(mtx_t *mtx) { return((mtx->mtx_lock & MTX_EXCLUSIVE) != 0); } /* * Return TRUE (non-zero) if the mutex is not locked. */ static __inline int mtx_notlocked(mtx_t *mtx) { return(mtx->mtx_lock == 0); } /* * Return TRUE (non-zero) if the mutex is not locked exclusively. * The mutex may in an unlocked or shared lock state. */ static __inline int mtx_notlocked_ex(mtx_t *mtx) { return((mtx->mtx_lock & MTX_EXCLUSIVE) != 0); } /* * Return TRUE (non-zero) if the mutex is exclusively locked by * the caller. */ static __inline int mtx_owned(mtx_t *mtx) { return((mtx->mtx_lock & MTX_EXCLUSIVE) && mtx->mtx_owner == curthread); } /* * Return TRUE (non-zero) if the mutex is not exclusively locked by * the caller. */ static __inline int mtx_notowned(mtx_t *mtx) { return((mtx->mtx_lock & MTX_EXCLUSIVE) == 0 || mtx->mtx_owner != curthread); } /* * Return the shared or exclusive lock count. A return value of 0 * indicate that the mutex is not locked. * * NOTE: If the mutex is held exclusively by someone other then the * caller the lock count for the other owner is still returned. */ static __inline int mtx_lockrefs(mtx_t *mtx) { return(mtx->mtx_lock & MTX_MASK); } /* * Lock must held and will be released on return. Returns state * which can be passed to mtx_lock_temp_restore() to return the * lock to its previous state. */ static __inline mtx_state_t mtx_lock_temp_release(mtx_t *mtx) { mtx_state_t state; state = (mtx->mtx_lock & MTX_EXCLUSIVE); mtx_unlock(mtx); return state; } /* * Restore the previous state of a lock released with * mtx_lock_temp_release() or mtx_lock_upgrade(). */ static __inline void mtx_lock_temp_restore(mtx_t *mtx, mtx_state_t state) { if (state & MTX_EXCLUSIVE) mtx_lock_ex_quick(mtx); else mtx_lock_sh_quick(mtx); } #endif
