tests/lib/libc/sys/t_futex_ops.c

root/tests/lib/libc/sys/t_futex_ops.c
/* $NetBSD: t_futex_ops.c,v 1.14 2025/03/05 12:02:00 riastradh Exp $ */

/*-
 * Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__COPYRIGHT("@(#) Copyright (c) 2019, 2020\
 The NetBSD Foundation, inc. All rights reserved.");
__RCSID("$NetBSD: t_futex_ops.c,v 1.14 2025/03/05 12:02:00 riastradh Exp $");

#include <sys/fcntl.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <atomic.h>
#include <errno.h>
#include <lwp.h>
#include <stdlib.h>
#include <stdio.h>
#include <signal.h>
#include <time.h>
#include <limits.h>
#include <sched.h>
#include <unistd.h>

#include <atf-c.h>

#include <libc/include/futex_private.h>

#include "h_macros.h"

#define LOAD(x)         (*(volatile int *)(x))
#define STORE(x, y)     *(volatile int *)(x) = (y)

#if 0
#define DPRINTF(x)      printf x
#else
#define DPRINTF(x)      __nothing
#endif

#define STACK_SIZE      65536

static volatile int futex_word;
static volatile int futex_word1;

static volatile unsigned int nlwps_running;

struct lwp_data {
        ucontext_t      context;
        void            (*func)(void *);
        void            *stack_base;
        lwpid_t         lwpid;
        pid_t           child;
        lwpid_t         threadid;
        int             wait_op;
        int             op_flags;
        int             bitset;
        volatile int    *futex_ptr;
        volatile int    *error_ptr;
        int             block_val;

        void            (*exit_func)(void);

        int             futex_error;
};

#define WAITER_LWP0             0
#define WAITER_LWP1             1
#define WAITER_LWP2             2
#define WAITER_LWP3             3
#define WAITER_LWP4             4
#define WAITER_LWP5             5
#define NLWPS                   6

struct lwp_data lwp_data[NLWPS];

static const char *bs_path = "t_futex_ops_backing_store";
static int bs_fd = -1;
static int *bs_addr = MAP_FAILED;
static void *bs_source_buffer = NULL;
static void *bs_verify_buffer = NULL;
static long bs_pagesize;

static void
create_lwp_waiter(struct lwp_data *d)
{
        RL(_lwp_create(&d->context, 0, &d->lwpid));
}

static void
exit_lwp_waiter(void)
{
        _lwp_exit();
}

static void
reap_lwp_waiter(struct lwp_data *d)
{
        RL(_lwp_wait(d->lwpid, NULL));
}

static void
create_proc_waiter(struct lwp_data *d)
{
        pid_t pid;

        RL(pid = fork());
        if (pid == 0) {
                (*d->func)(d);
                _exit(666);             /* backstop */
        } else
                d->child = pid;
}

static void
exit_proc_waiter(void)
{
        _exit(0);
}

static void
reap_proc_waiter(struct lwp_data *d)
{
        pid_t pid;
        int status;

        RL(pid = waitpid(d->child, &status, 0));
        ATF_CHECK_EQ_MSG(pid, d->child,
            "pid=%lld d->child=%lld", (long long)pid, (long long)d->child);
        ATF_CHECK_MSG(WIFEXITED(status), "status=0x%x", status);
        ATF_CHECK_EQ_MSG(WEXITSTATUS(status), 0, "status=0x%x", status);
}

static void
setup_lwp_context(struct lwp_data *d, void (*func)(void *))
{

        memset(d, 0, sizeof(*d));
        REQUIRE_LIBC(d->stack_base = mmap(NULL, STACK_SIZE,
                PROT_READ | PROT_WRITE, MAP_ANON | MAP_STACK | MAP_PRIVATE,
                -1, 0),
            MAP_FAILED);
        _lwp_makecontext(&d->context, func, d, NULL, d->stack_base,
            STACK_SIZE);
        d->threadid = 0;
        d->func = func;
}

static void
simple_test_waiter_lwp(void *arg)
{
        struct lwp_data *d = arg;

        d->threadid = _lwp_self();

        membar_producer();
        atomic_inc_uint(&nlwps_running);
        membar_sync();

        if (__futex(d->futex_ptr, d->wait_op | d->op_flags,
                d->block_val, NULL, NULL, 0, d->bitset) == -1) {
                d->futex_error = errno;
                membar_sync();
                atomic_dec_uint(&nlwps_running);
                _lwp_exit();
        } else {
                d->futex_error = 0;
        }

        membar_sync();
        atomic_dec_uint(&nlwps_running);

        _lwp_exit();
}

static bool
verify_zero_bs(void)
{
        ssize_t nread;

        if (bs_verify_buffer == NULL) {
                REQUIRE_LIBC(bs_verify_buffer = malloc(bs_pagesize), NULL);
        }

        RL(nread = pread(bs_fd, bs_verify_buffer, bs_pagesize, 0));
        ATF_REQUIRE_EQ_MSG(nread, bs_pagesize, "nread=%zu bs_pagesize=%lu",
            nread, bs_pagesize);

        return (memcmp(bs_verify_buffer, bs_source_buffer, bs_pagesize) == 0);
}

static void
create_bs(int map_flags)
{
        ssize_t nwrit;

        bs_pagesize = sysconf(_SC_PAGESIZE);
        ATF_REQUIRE_MSG(bs_pagesize > 0, "bs_pagesize=%ld", bs_pagesize);

        if ((map_flags & (MAP_FILE | MAP_ANON)) == MAP_FILE) {
                REQUIRE_LIBC(bs_source_buffer = calloc(1, bs_pagesize), NULL);

                RL(bs_fd = open(bs_path, O_RDWR | O_CREAT | O_EXCL, 0644));
                RL(nwrit = pwrite(bs_fd, bs_source_buffer, bs_pagesize, 0));
                ATF_REQUIRE_EQ_MSG(nwrit, bs_pagesize,
                    "nwrit=%zu bs_pagesize=%lu", nwrit, bs_pagesize);
                ATF_REQUIRE(verify_zero_bs());
        }

        REQUIRE_LIBC(bs_addr = mmap(NULL, bs_pagesize, PROT_READ | PROT_WRITE,
                map_flags | MAP_HASSEMAPHORE, bs_fd, 0),
            MAP_FAILED);
}

static void
cleanup_bs(void)
{

        if (bs_fd != -1) {
                (void) close(bs_fd);
                bs_fd = -1;
                (void) unlink(bs_path);
        }
        if (bs_source_buffer != NULL) {
                free(bs_source_buffer);
                bs_source_buffer = NULL;
        }
        if (bs_verify_buffer != NULL) {
                free(bs_verify_buffer);
                bs_verify_buffer = NULL;
        }
        if (bs_addr != MAP_FAILED) {
                munmap(bs_addr, bs_pagesize);
                bs_addr = MAP_FAILED;
        }
}

static void
do_cleanup(void)
{
        int i;

        for (i = 0; i < NLWPS; i++) {
                struct lwp_data *d = &lwp_data[i];
                if (d->stack_base != NULL && d->stack_base != MAP_FAILED) {
                        (void) munmap(d->stack_base, STACK_SIZE);
                }
        }
        memset(lwp_data, 0, sizeof(lwp_data));
        STORE(&futex_word, 0);
        STORE(&futex_word1, 0);
        nlwps_running = 0;

        cleanup_bs();
}

/*****************************************************************************/

static void
wait_wake_test_waiter_lwp(void *arg)
{
        struct lwp_data *d = arg;

        d->threadid = _lwp_self();

        STORE(d->futex_ptr, 1);
        membar_sync();

        /* This will block because *futex_ptr == 1. */
        if (__futex(d->futex_ptr, FUTEX_WAIT | d->op_flags,
                1, NULL, NULL, 0, 0) == -1) {
                STORE(d->error_ptr, errno);
                (*d->exit_func)();
        } else {
                STORE(d->error_ptr, 0);
        }

        do {
                membar_sync();
                sleep(1);
        } while (LOAD(d->futex_ptr) != 0);

        STORE(d->futex_ptr, 2);
        membar_sync();

        do {
                membar_sync();
                sleep(1);
        } while (LOAD(d->futex_ptr) != 3);

        /* This will not block because futex_word != 666. */
        if (__futex(d->futex_ptr, FUTEX_WAIT | d->op_flags,
                666, NULL, NULL, 0, 0) == -1) {
                /* This SHOULD be EAGAIN. */
                STORE(d->error_ptr, errno);
        }

        STORE(d->futex_ptr, 4);
        membar_sync();

        (*d->exit_func)();
}

static void
do_futex_wait_wake_test(volatile int *futex_ptr, volatile int *error_ptr,
    void (*create_func)(struct lwp_data *),
    void (*exit_func)(void),
    void (*reap_func)(struct lwp_data *),
    int flags)
{
        struct lwp_data *wlwp = &lwp_data[WAITER_LWP0];
        int tries;
        int n;

        if (error_ptr == NULL)
                error_ptr = &wlwp->futex_error;

        if (create_func == NULL)
                create_func = create_lwp_waiter;
        if (exit_func == NULL)
                exit_func = exit_lwp_waiter;
        if (reap_func == NULL)
                reap_func = reap_lwp_waiter;

        setup_lwp_context(wlwp, wait_wake_test_waiter_lwp);

        DPRINTF(("futex_basic_wait_wake: testing with flags 0x%x\n", flags));
        wlwp->op_flags = flags;
        wlwp->error_ptr = error_ptr;
        STORE(error_ptr, -1);
        wlwp->futex_ptr = futex_ptr;
        STORE(futex_ptr, 0);
        wlwp->exit_func = exit_func;
        membar_sync();

        DPRINTF(("futex_basic_wait_wake: creating watier LWP\n"));
        (*create_func)(wlwp);

        DPRINTF(("futex_basic_wait_wake: waiting for LWP %d to enter futex\n",
            wlwp->lwpid));
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (LOAD(futex_ptr) == 1)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG((n = LOAD(futex_ptr)), 1, "LOAD(futex_ptr)=%d", n);

        /*
         * If the LWP is blocked in the futex, it will not have yet
         * modified *error_ptr.
         */
        DPRINTF(("futex_basic_wait_wake: checking for successful wait (%d)\n",
            LOAD(error_ptr)));
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (LOAD(error_ptr) == -1)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG((n = LOAD(error_ptr)), -1, "error=%d", n);

        /* Make sure invalid #wakes in rejected. */
        ATF_REQUIRE_ERRNO(EINVAL,
            __futex(futex_ptr, FUTEX_WAKE | flags,
                -1, NULL, NULL, 0, 0) == -1);

        DPRINTF(("futex_basic_wait_wake: waking 1 waiter\n"));
        RL(n = __futex(futex_ptr, FUTEX_WAKE | flags, 1, NULL, NULL, 0, 0));
        ATF_REQUIRE_EQ_MSG(n, 1, "n=%d wakeups", n);

        DPRINTF(("futex_basic_wait_wake: checking for successful wake (%d)\n",
            LOAD(error_ptr)));
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (LOAD(error_ptr) == 0)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG((n = LOAD(error_ptr)), 0, "error=%d", n);

        STORE(futex_ptr, 0);
        membar_sync();

        DPRINTF(("futex_basic_wait_wake: waiting for LWP to advance (2)\n"));
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (LOAD(futex_ptr) == 2)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG((n = LOAD(futex_ptr)), 2, "LOAD(futex_ptr)=%d", n);

        STORE(futex_ptr, 3);
        membar_sync();

        DPRINTF(("futex_basic_wait_wake: waiting for LWP to advance (4)\n"));
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (LOAD(futex_ptr) == 4)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG((n = LOAD(futex_ptr)), 4, "error=%d", n);

        DPRINTF(("futex_basic_wait_wake: checking for expected EGAIN\n"));
        ATF_REQUIRE_EQ_MSG((n = LOAD(error_ptr)), EAGAIN, "error=%d", n);

        DPRINTF(("futex_basic_wait_wake: reaping LWP %d\n", wlwp->lwpid));
        (*reap_func)(wlwp);
}

ATF_TC_WITH_CLEANUP(futex_basic_wait_wake_private);
ATF_TC_HEAD(futex_basic_wait_wake_private, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests basic futex WAIT + WAKE operations (PRIVATE)");
}
ATF_TC_BODY(futex_basic_wait_wake_private, tc)
{
        do_futex_wait_wake_test(&futex_word, NULL,
            NULL, NULL, NULL,
            FUTEX_PRIVATE_FLAG);
}
ATF_TC_CLEANUP(futex_basic_wait_wake_private, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_basic_wait_wake_shared);
ATF_TC_HEAD(futex_basic_wait_wake_shared, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests basic futex WAIT + WAKE operations (SHARED)");
}
ATF_TC_BODY(futex_basic_wait_wake_shared, tc)
{
        do_futex_wait_wake_test(&futex_word, NULL,
            NULL, NULL, NULL,
            0);
}
ATF_TC_CLEANUP(futex_basic_wait_wake_shared, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_anon_bs_private);
ATF_TC_HEAD(futex_wait_wake_anon_bs_private, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT + WAKE operations (MAP_ANON + PRIVATE)");
}
ATF_TC_BODY(futex_wait_wake_anon_bs_private, tc)
{
        create_bs(MAP_ANON | MAP_PRIVATE);
        do_futex_wait_wake_test(&bs_addr[0], NULL,
            NULL, NULL, NULL,
            FUTEX_PRIVATE_FLAG);
}
ATF_TC_CLEANUP(futex_wait_wake_anon_bs_private, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_anon_bs_shared);
ATF_TC_HEAD(futex_wait_wake_anon_bs_shared, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT + WAKE operations (MAP_ANON + SHARED)");
}
ATF_TC_BODY(futex_wait_wake_anon_bs_shared, tc)
{
        create_bs(MAP_ANON | MAP_PRIVATE);
        do_futex_wait_wake_test(&bs_addr[0], NULL,
            NULL, NULL, NULL,
            0);
}
ATF_TC_CLEANUP(futex_wait_wake_anon_bs_shared, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_private);
ATF_TC_HEAD(futex_wait_wake_file_bs_private, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT + WAKE operations (MAP_FILE + PRIVATE)");
}
ATF_TC_BODY(futex_wait_wake_file_bs_private, tc)
{
        /*
         * This combination (non-COW mapped file + PRIVATE futex)
         * doesn't really make sense, but we should make sure it
         * works as expected.
         */
        create_bs(MAP_FILE | MAP_SHARED);
        do_futex_wait_wake_test(&bs_addr[0], NULL,
            NULL, NULL, NULL,
            FUTEX_PRIVATE_FLAG);
        ATF_REQUIRE(!verify_zero_bs());
}
ATF_TC_CLEANUP(futex_wait_wake_file_bs_private, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_cow_private);
ATF_TC_HEAD(futex_wait_wake_file_bs_cow_private, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT + WAKE operations (MAP_FILE COW + PRIVATE)");
}
ATF_TC_BODY(futex_wait_wake_file_bs_cow_private, tc)
{
        create_bs(MAP_FILE | MAP_PRIVATE);
        do_futex_wait_wake_test(&bs_addr[0], NULL,
            NULL, NULL, NULL,
            FUTEX_PRIVATE_FLAG);
        ATF_REQUIRE(verify_zero_bs());
}
ATF_TC_CLEANUP(futex_wait_wake_file_bs_cow_private, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_shared);
ATF_TC_HEAD(futex_wait_wake_file_bs_shared, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT + WAKE operations (MAP_FILE + SHARED)");
}
ATF_TC_BODY(futex_wait_wake_file_bs_shared, tc)
{
        create_bs(MAP_FILE | MAP_SHARED);
        do_futex_wait_wake_test(&bs_addr[0], NULL,
            NULL, NULL, NULL,
            0);
        ATF_REQUIRE(!verify_zero_bs());
}
ATF_TC_CLEANUP(futex_wait_wake_file_bs_shared, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_cow_shared);
ATF_TC_HEAD(futex_wait_wake_file_bs_cow_shared, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT + WAKE operations (MAP_FILE COW + SHARED)");
}
ATF_TC_BODY(futex_wait_wake_file_bs_cow_shared, tc)
{
        /*
         * This combination (COW mapped file + SHARED futex)
         * doesn't really make sense, but we should make sure it
         * works as expected.
         */
        create_bs(MAP_FILE | MAP_PRIVATE);
        do_futex_wait_wake_test(&bs_addr[0], NULL,
            NULL, NULL, NULL,
            0);
        ATF_REQUIRE(verify_zero_bs());
}
ATF_TC_CLEANUP(futex_wait_wake_file_bs_cow_shared, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_anon_bs_shared_proc);
ATF_TC_HEAD(futex_wait_wake_anon_bs_shared_proc, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests multiproc futex WAIT + WAKE operations (MAP_ANON + SHARED)");
}
ATF_TC_BODY(futex_wait_wake_anon_bs_shared_proc, tc)
{
        create_bs(MAP_ANON | MAP_SHARED);
        do_futex_wait_wake_test(&bs_addr[0], &bs_addr[1],
            create_proc_waiter,
            exit_proc_waiter,
            reap_proc_waiter,
            0);
}
ATF_TC_CLEANUP(futex_wait_wake_anon_bs_shared_proc, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_shared_proc);
ATF_TC_HEAD(futex_wait_wake_file_bs_shared_proc, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests multiproc futex WAIT + WAKE operations (MAP_ANON + SHARED)");
}
ATF_TC_BODY(futex_wait_wake_file_bs_shared_proc, tc)
{
        create_bs(MAP_FILE | MAP_SHARED);
        do_futex_wait_wake_test(&bs_addr[0], &bs_addr[1],
            create_proc_waiter,
            exit_proc_waiter,
            reap_proc_waiter,
            0);
}
ATF_TC_CLEANUP(futex_wait_wake_file_bs_shared_proc, tc)
{
        do_cleanup();
}

/*****************************************************************************/

ATF_TC(futex_wait_pointless_bitset);
ATF_TC_HEAD(futex_wait_pointless_bitset, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests basic futex WAIT + WAKE operations (SHARED)");
}
ATF_TC_BODY(futex_wait_pointless_bitset, tc)
{

        futex_word = 1;
        ATF_REQUIRE_ERRNO(EINVAL,
            __futex(&futex_word, FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG,
                1, NULL, NULL, 0, 0) == -1);
}

static void
do_futex_wait_wake_bitset_test(int flags)
{
        struct lwp_data *wlwp0 = &lwp_data[WAITER_LWP0];
        struct lwp_data *wlwp1 = &lwp_data[WAITER_LWP1];
        int i, tries, n;

        for (i = WAITER_LWP0; i <= WAITER_LWP1; i++) {
                setup_lwp_context(&lwp_data[i], simple_test_waiter_lwp);
                lwp_data[i].op_flags = flags;
                lwp_data[i].futex_error = -1;
                lwp_data[i].bitset = __BIT(i);
                lwp_data[i].wait_op = FUTEX_WAIT_BITSET;
                lwp_data[i].futex_ptr = &futex_word;
                lwp_data[i].block_val = 1;
        }

        STORE(&futex_word, 1);
        membar_sync();

        RL(_lwp_create(&wlwp0->context, 0, &wlwp0->lwpid));
        RL(_lwp_create(&wlwp1->context, 0, &wlwp1->lwpid));

        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 2)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 2,
            "waiters failed to start, nlwps_running=%u", nlwps_running);

        /* Ensure they're blocked. */
        ATF_REQUIRE_EQ_MSG(wlwp0->futex_error, -1, "wlwp0->futex_error=%d",
            wlwp0->futex_error);
        ATF_REQUIRE_EQ_MSG(wlwp1->futex_error, -1, "wlwp1->futex_error=%d",
            wlwp1->futex_error);

        /* Make sure invalid #wakes in rejected. */
        ATF_REQUIRE_ERRNO(EINVAL,
            __futex(&futex_word, FUTEX_WAKE_BITSET | flags,
                -1, NULL, NULL, 0, 0) == -1);

        /* This should result in no wakeups because no bits are set. */
        RL(n = __futex(&futex_word, FUTEX_WAKE_BITSET | flags,
                INT_MAX, NULL, NULL, 0, 0));
        ATF_REQUIRE_EQ_MSG(n, 0, "n=%d wakeups", n);

        /* This should result in no wakeups because the wrongs bits are set. */
        RL(n = __futex(&futex_word, FUTEX_WAKE_BITSET | flags,
                INT_MAX, NULL, NULL, 0,
                ~(wlwp0->bitset | wlwp1->bitset)));
        ATF_REQUIRE_EQ_MSG(n, 0, "n=%d wakeups", n);

        /* Trust, but verify. */
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 2)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 2,
            "waiters exited unexpectedly, nlwps_running=%u", nlwps_running);

        /* Wake up the first LWP. */
        RL(n = __futex(&futex_word, FUTEX_WAKE_BITSET | flags,
                INT_MAX, NULL, NULL, 0, wlwp0->bitset));
        ATF_REQUIRE_EQ_MSG(n, 1, "n=%d wakeups", n);
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 1)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 1, "nlwps_running=%u",
            nlwps_running);
        ATF_REQUIRE_EQ_MSG(wlwp0->futex_error, 0, "wlwp0->futex_error=%d",
            wlwp0->futex_error);
        RL(_lwp_wait(wlwp0->lwpid, NULL));

        /* Wake up the second LWP. */
        RL(n = __futex(&futex_word, FUTEX_WAKE_BITSET | flags,
                INT_MAX, NULL, NULL, 0, wlwp1->bitset));
        ATF_REQUIRE_EQ_MSG(n, 1, "n=%d wakeups", n);
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 0)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 0, "nlwps_running=%u",
            nlwps_running);
        ATF_REQUIRE_EQ_MSG(wlwp1->futex_error, 0, "wlwp1->futex_error=%d",
            wlwp1->futex_error);
        RL(_lwp_wait(wlwp1->lwpid, NULL));
}

ATF_TC_WITH_CLEANUP(futex_wait_wake_bitset);
ATF_TC_HEAD(futex_wait_wake_bitset, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT_BITSET + WAKE_BITSET operations");
}
ATF_TC_BODY(futex_wait_wake_bitset, tc)
{
        do_futex_wait_wake_bitset_test(FUTEX_PRIVATE_FLAG);
}
ATF_TC_CLEANUP(futex_wait_wake_bitset, tc)
{
        do_cleanup();
}

/*****************************************************************************/

static void
do_futex_requeue_test(int flags, int op)
{
        struct lwp_data *wlwp0 = &lwp_data[WAITER_LWP0];
        struct lwp_data *wlwp1 = &lwp_data[WAITER_LWP1];
        struct lwp_data *wlwp2 = &lwp_data[WAITER_LWP2];
        struct lwp_data *wlwp3 = &lwp_data[WAITER_LWP3];
        const int good_val3 = (op == FUTEX_CMP_REQUEUE) ?   1 : 0;
        const int bad_val3  = (op == FUTEX_CMP_REQUEUE) ? 666 : 0;
        int i, tries, n;

        for (i = WAITER_LWP0; i <= WAITER_LWP3; i++) {
                setup_lwp_context(&lwp_data[i], simple_test_waiter_lwp);
                lwp_data[i].op_flags = flags;
                lwp_data[i].futex_error = -1;
                lwp_data[i].futex_ptr = &futex_word;
                lwp_data[i].block_val = 1;
                lwp_data[i].bitset = 0;
                lwp_data[i].wait_op = FUTEX_WAIT;
        }

        STORE(&futex_word, 1);
        STORE(&futex_word1, 1);
        membar_sync();

        RL(_lwp_create(&wlwp0->context, 0, &wlwp0->lwpid));
        RL(_lwp_create(&wlwp1->context, 0, &wlwp1->lwpid));
        RL(_lwp_create(&wlwp2->context, 0, &wlwp2->lwpid));
        RL(_lwp_create(&wlwp3->context, 0, &wlwp3->lwpid));

        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 4)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 4,
            "waiters failed to start, nlwps_running=%u", nlwps_running);

        /* Ensure they're blocked. */
        ATF_REQUIRE_EQ_MSG(wlwp0->futex_error, -1, "wlwp0->futex_error=%d",
            wlwp0->futex_error);
        ATF_REQUIRE_EQ_MSG(wlwp1->futex_error, -1, "wlwp1->futex_error=%d",
            wlwp1->futex_error);
        ATF_REQUIRE_EQ_MSG(wlwp2->futex_error, -1, "wlwp2->futex_error=%d",
            wlwp2->futex_error);
        ATF_REQUIRE_EQ_MSG(wlwp3->futex_error, -1, "wlwp3->futex_error=%d",
            wlwp3->futex_error);

        /* Make sure invalid #wakes and #requeues are rejected. */
        ATF_REQUIRE_ERRNO(EINVAL,
            __futex(&futex_word, op | flags,
                -1, NULL, &futex_word1, INT_MAX, bad_val3) == -1);

        ATF_REQUIRE_ERRNO(EINVAL,
            __futex(&futex_word, op | flags,
                0, NULL, &futex_word1, -1, bad_val3) == -1);

        /*
         * FUTEX 0: 4 LWPs
         * FUTEX 1: 0 LWPs
         */

        if (op == FUTEX_CMP_REQUEUE) {
                /* This should fail because the futex_word value is 1. */
                ATF_REQUIRE_ERRNO(EAGAIN,
                    __futex(&futex_word, op | flags,
                        0, NULL, &futex_word1, INT_MAX, bad_val3) == -1);
        }

        /*
         * FUTEX 0: 4 LWPs
         * FUTEX 1: 0 LWPs
         */

        /* Move all waiters from 0 to 1. */
        RL(n = __futex(&futex_word, op | flags, 0, NULL, &futex_word1,
                INT_MAX, good_val3));
        ATF_CHECK_EQ_MSG(n, 4, "n=%d woken or requeued", n);

        /*
         * FUTEX 0: 0 LWPs
         * FUTEX 1: 4 LWPs
         */

        if (op == FUTEX_CMP_REQUEUE) {
                /* This should fail because the futex_word1 value is 1. */
                ATF_REQUIRE_ERRNO(EAGAIN,
                    __futex(&futex_word1, op | flags,
                        1, NULL, &futex_word, 1, bad_val3) == -1);
        }

        /*
         * FUTEX 0: 0 LWPs
         * FUTEX 1: 4 LWPs
         */

        /* Wake one waiter on 1, move one waiter to 0. */
        RL(n = __futex(&futex_word1, op | flags, 1, NULL, &futex_word,
                1, good_val3));
        ATF_CHECK_EQ_MSG(n, 2, "n=%d woken or requeued", n);

        /*
         * FUTEX 0: 1 LWP
         * FUTEX 1: 2 LWPs
         */

        /* Wake all waiters on 0 (should be 1). */
        RL(n = __futex(&futex_word, FUTEX_WAKE | flags, INT_MAX, NULL, NULL,
                0, 0));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);

        /* Wake all waiters on 1 (should be 2). */
        RL(n = __futex(&futex_word1, FUTEX_WAKE | flags, INT_MAX, NULL, NULL,
                0, 0));
        ATF_CHECK_EQ_MSG(n, 2, "n=%d woken", n);

        /* Trust, but verify. */
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 0)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 0,
            "waiters failed to exit, nlwps_running=%u", nlwps_running);

        RL(_lwp_wait(wlwp0->lwpid, NULL));
        RL(_lwp_wait(wlwp1->lwpid, NULL));
        RL(_lwp_wait(wlwp2->lwpid, NULL));
        RL(_lwp_wait(wlwp3->lwpid, NULL));
}

ATF_TC_WITH_CLEANUP(futex_requeue);
ATF_TC_HEAD(futex_requeue, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex REQUEUE operations");
}
ATF_TC_BODY(futex_requeue, tc)
{
        do_futex_requeue_test(FUTEX_PRIVATE_FLAG, FUTEX_REQUEUE);
}
ATF_TC_CLEANUP(futex_requeue, tc)
{
        do_cleanup();
}

ATF_TC_WITH_CLEANUP(futex_cmp_requeue);
ATF_TC_HEAD(futex_cmp_requeue, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex CMP_REQUEUE operations");
}
ATF_TC_BODY(futex_cmp_requeue, tc)
{
        do_futex_requeue_test(FUTEX_PRIVATE_FLAG, FUTEX_CMP_REQUEUE);
}
ATF_TC_CLEANUP(futex_cmp_requeue, tc)
{
        do_cleanup();
}

ATF_TC(futex_cmp_requeue_trivial);
ATF_TC_HEAD(futex_cmp_requeue_trivial, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests trivial cases of futex CMP_REQUEUE operations");
}
ATF_TC_BODY(futex_cmp_requeue_trivial, tc)
{
        int nwoken;

        futex_word = 123;
        futex_word1 = 456;      /* should be ignored */
        ATF_CHECK_ERRNO(EAGAIN, __futex(&futex_word, FUTEX_CMP_REQUEUE,
                /*nwake*/1, NULL, &futex_word1, /*nrequeue*/1, 0) == -1);
        ATF_CHECK_ERRNO(EAGAIN, __futex(&futex_word, FUTEX_CMP_REQUEUE,
                /*nwake*/1, NULL, &futex_word1, /*nrequeue*/1, 122) == -1);
        nwoken = __futex(&futex_word, FUTEX_CMP_REQUEUE,
            /*nwake*/1, NULL, &futex_word1, /*nrequeue*/1, 123);
        ATF_CHECK_MSG(nwoken != -1, "errno=%d (%s)", errno, strerror(errno));
        ATF_CHECK_EQ_MSG(nwoken, 0, "nwoken=%d", nwoken);
        ATF_CHECK_EQ_MSG(futex_word, 123, "futex_word=%d", futex_word);
        ATF_CHECK_EQ_MSG(futex_word1, 456, "futex_word1=%d", futex_word1);
}

/*****************************************************************************/

static void
do_futex_wake_op_op_test(int flags)
{
        int op, n;

        futex_word = 0;
        futex_word1 = 0;

        /*
         * The op= operations should work even if there are no waiters.
         */

        /*
         * Because these operations use both futex addresses, exercise
         * rejecting unaligned futex addresses here.
         */
        op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_EQ, 0);
        ATF_CHECK_ERRNO(EINVAL,
            __futex((int *)1, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op) == -1);
        ATF_CHECK_EQ_MSG(futex_word1, 0, "futex_word1=%d", futex_word1);

        ATF_CHECK_ERRNO(EINVAL,
            __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, (int *)1, 0, op) == -1);
        ATF_CHECK_EQ_MSG(futex_word, 0, "futex_word=%d", futex_word);

        /* Check unmapped uaddr2 handling, too. */
        ATF_CHECK_ERRNO(EFAULT,
            __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, NULL, 0, op) == -1);
        ATF_CHECK_EQ_MSG(futex_word, 0, "futex_word=%d", futex_word);

        op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 1, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_ADD, 1, FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_OR, 2, FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        /* This should fail because of invalid shift value 32. */
        op = FUTEX_OP(FUTEX_OP_OR | FUTEX_OP_OPARG_SHIFT, 32,
            FUTEX_OP_CMP_EQ, 0);
        ATF_CHECK_ERRNO(EINVAL,
            __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op) == -1);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_OR | FUTEX_OP_OPARG_SHIFT, 31,
            FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, (int)0x80000002,
            "futex_word1=0x%x", futex_word1);

        op = FUTEX_OP(FUTEX_OP_ANDN | FUTEX_OP_OPARG_SHIFT, 31,
            FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_XOR, 2, FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 0, "futex_word1=%d", futex_word1);

        /*
         * Verify oparg is sign-extended.
         */
        futex_word1 = 0;
        op = FUTEX_OP(FUTEX_OP_SET, 0xfff, FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, -1, "futex_word1=%d", futex_word1);

        futex_word1 = 0;
        op = FUTEX_OP(FUTEX_OP_SET, -1, FUTEX_OP_CMP_EQ, 0);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 0, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, -1, "futex_word1=%d", futex_word1);
}

ATF_TC_WITH_CLEANUP(futex_wake_op_op);
ATF_TC_HEAD(futex_wake_op_op, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAKE_OP OP operations");
}
ATF_TC_BODY(futex_wake_op_op, tc)
{
        do_futex_wake_op_op_test(FUTEX_PRIVATE_FLAG);
}
ATF_TC_CLEANUP(futex_wake_op_op, tc)
{
        do_cleanup();
}

static void
create_wake_op_test_lwps(int flags)
{
        int i;

        futex_word1 = 0;
        membar_sync();

        for (i = WAITER_LWP0; i <= WAITER_LWP5; i++) {
                setup_lwp_context(&lwp_data[i], simple_test_waiter_lwp);
                lwp_data[i].op_flags = flags;
                lwp_data[i].futex_error = -1;
                lwp_data[i].futex_ptr = &futex_word1;
                lwp_data[i].block_val = 0;
                lwp_data[i].bitset = 0;
                lwp_data[i].wait_op = FUTEX_WAIT;
                RL(_lwp_create(&lwp_data[i].context, 0, &lwp_data[i].lwpid));
        }

        for (i = 0; i < 5; i++) {
                membar_sync();
                if (nlwps_running == 6)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 6,
            "waiters failed to start, nlwps_running=%u", nlwps_running);

        /* Ensure they're blocked. */
        for (i = WAITER_LWP0; i <= WAITER_LWP5; i++) {
                ATF_REQUIRE_EQ_MSG(lwp_data[i].futex_error, -1,
                    "i=%d lwp_data[i].futex_error=%d",
                    i, lwp_data[i].futex_error);
        }
}

static void
reap_wake_op_test_lwps(void)
{
        int i;

        for (i = WAITER_LWP0; i <= WAITER_LWP5; i++) {
                RL(_lwp_wait(lwp_data[i].lwpid, NULL));
        }
}

static void
do_futex_wake_op_cmp_test(int flags)
{
        int tries, op, n;

        futex_word = 0;
        membar_sync();

        /*
         * Verify and negative and positive for each individual
         * compare.
         */

        create_wake_op_test_lwps(flags);

        /* #LWPs = 6 */
        futex_word1 = 0xfff;
        op = FUTEX_OP(FUTEX_OP_SET, 0, FUTEX_OP_CMP_EQ, 0xfff);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 0, "futex_word1=%d", futex_word1);

        futex_word1 = 0xfff;
        op = FUTEX_OP(FUTEX_OP_SET, 0, FUTEX_OP_CMP_EQ, -1);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 0, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_SET, 0, FUTEX_OP_CMP_EQ, 1);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 0, "futex_word1=%d", futex_word1);

        futex_word1 = -1;
        op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_EQ, 0xfff);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 1, "futex_word1=%d", futex_word1);

        /* #LWPs = 5 */
        op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_NE, 1);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 1, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_NE, 2);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        /* #LWPs = 4 */
        op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_LT, 2);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_LT, 3);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        /* #LWPs = 3 */
        op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_LE, 1);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 1, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_LE, 1);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 1, "futex_word1=%d", futex_word1);

        /* #LWPs = 2 */
        op = FUTEX_OP(FUTEX_OP_SET, 3, FUTEX_OP_CMP_GT, 3);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 3, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_GT, 2);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        /* #LWPs = 1 */
        op = FUTEX_OP(FUTEX_OP_SET, 3, FUTEX_OP_CMP_GE, 4);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 0, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 3, "futex_word1=%d", futex_word1);

        op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_GE, 3);
        RL(n = __futex(&futex_word, FUTEX_WAKE_OP | flags,
                0, NULL, &futex_word1, 1, op));
        ATF_CHECK_EQ_MSG(n, 1, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word1, 2, "futex_word1=%d", futex_word1);

        /* #LWPs = 0 */

        /* Trust, but verify. */
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 0)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 0,
            "waiters failed to exit, nlwps_running=%u", nlwps_running);

        reap_wake_op_test_lwps();

        /*
         * Verify wakes on uaddr work even if the uaddr2 comparison
         * fails.
         */

        create_wake_op_test_lwps(flags);

        /* #LWPs = 6 */
        ATF_CHECK_EQ_MSG(futex_word, 0, "futex_word=%d", futex_word);
        op = FUTEX_OP(FUTEX_OP_SET, 0, FUTEX_OP_CMP_EQ, 666);
        RL(n = __futex(&futex_word1, FUTEX_WAKE_OP | flags,
                INT_MAX, NULL, &futex_word, 0, op));
        ATF_CHECK_EQ_MSG(n, 6, "n=%d woken", n);
        ATF_CHECK_EQ_MSG(futex_word, 0, "futex_word=%d", futex_word);

        /* #LWPs = 0 */

        /* Trust, but verify. */
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 0)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 0,
            "waiters failed to exit, nlwps_running=%u", nlwps_running);

        reap_wake_op_test_lwps();
}

ATF_TC_WITH_CLEANUP(futex_wake_op_cmp);
ATF_TC_HEAD(futex_wake_op_cmp, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAKE_OP CMP operations");
}
ATF_TC_BODY(futex_wake_op_cmp, tc)
{
        do_futex_wake_op_cmp_test(FUTEX_PRIVATE_FLAG);
}
ATF_TC_CLEANUP(futex_wake_op_cmp, tc)
{
        do_cleanup();
}

/*****************************************************************************/



/*****************************************************************************/

static void
do_futex_wait_timeout(bool relative, clockid_t clock)
{
        struct timespec ts;
        struct timespec deadline;
        int op = relative ? FUTEX_WAIT : FUTEX_WAIT_BITSET;

        if (clock == CLOCK_REALTIME)
                op |= FUTEX_CLOCK_REALTIME;

        RL(clock_gettime(clock, &deadline));
        deadline.tv_sec += 2;
        if (relative) {
                ts.tv_sec = 2;
                ts.tv_nsec = 0;
        } else {
                ts = deadline;
        }

        futex_word = 1;
        ATF_REQUIRE_ERRNO(ETIMEDOUT,
            __futex(&futex_word, op | FUTEX_PRIVATE_FLAG,
                1, &ts, NULL, 0, FUTEX_BITSET_MATCH_ANY) == -1);

        /* Can't reliably check CLOCK_REALTIME in the presence of NTP. */
        if (clock != CLOCK_REALTIME) {
                RL(clock_gettime(clock, &ts));
                ATF_CHECK_MSG(ts.tv_sec >= deadline.tv_sec,
                    "ts=%lld.%09ldsec deadline=%lld.%09ldsec",
                    (long long)ts.tv_sec, ts.tv_nsec,
                    (long long)deadline.tv_sec, deadline.tv_nsec);
                ATF_CHECK_MSG((ts.tv_sec > deadline.tv_sec ||
                        ts.tv_nsec >= deadline.tv_nsec),
                    "ts=%lld.%09ldsec deadline=%lld.%09ldsec",
                    (long long)ts.tv_sec, ts.tv_nsec,
                    (long long)deadline.tv_sec, deadline.tv_nsec);
        }
}

ATF_TC(futex_wait_timeout_relative);
ATF_TC_HEAD(futex_wait_timeout_relative, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT with relative timeout");
}
ATF_TC_BODY(futex_wait_timeout_relative, tc)
{
        do_futex_wait_timeout(true, CLOCK_MONOTONIC);
}

ATF_TC(futex_wait_timeout_relative_rt);
ATF_TC_HEAD(futex_wait_timeout_relative_rt, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT with relative timeout (REALTIME)");
}
ATF_TC_BODY(futex_wait_timeout_relative_rt, tc)
{
        do_futex_wait_timeout(true, CLOCK_REALTIME);
}

ATF_TC(futex_wait_timeout_deadline);
ATF_TC_HEAD(futex_wait_timeout_deadline, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT with absolute deadline");
}
ATF_TC_BODY(futex_wait_timeout_deadline, tc)
{
        do_futex_wait_timeout(false, CLOCK_MONOTONIC);
}

ATF_TC(futex_wait_timeout_deadline_rt);
ATF_TC_HEAD(futex_wait_timeout_deadline_rt, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT with absolute deadline (REALTIME)");
}
ATF_TC_BODY(futex_wait_timeout_deadline_rt, tc)
{
        do_futex_wait_timeout(false, CLOCK_REALTIME);
}

/*****************************************************************************/

static void
sig_noop(int sig __unused)
{
}

static void (*old_act)(int) = SIG_DFL;

static void
do_futex_wait_evil_unmapped(int map_flags)
{
        int i;

        create_bs(map_flags);

        REQUIRE_LIBC(signal(SIGUSR1, sig_noop), SIG_ERR);

        setup_lwp_context(&lwp_data[0], simple_test_waiter_lwp);
        lwp_data[0].op_flags = 0;
        lwp_data[0].futex_error = -1;
        lwp_data[0].futex_ptr = &bs_addr[0];
        lwp_data[0].block_val = 0;
        lwp_data[0].bitset = 0;
        lwp_data[0].wait_op = FUTEX_WAIT;
        RL(_lwp_create(&lwp_data[0].context, 0, &lwp_data[0].lwpid));

        for (i = 0; i < 5; i++) {
                membar_sync();
                if (nlwps_running == 1)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 1,
            "waiters failed to start, nlwps_running=%u", nlwps_running);

        /* Ensure it's blocked. */
        ATF_REQUIRE_EQ_MSG(lwp_data[0].futex_error, -1,
            "lwp_data[0].futex_error=%d", lwp_data[0].futex_error);

        /* Rudely unmap the backing store. */
        cleanup_bs();

        /* Signal the waiter so that it leaves the futex. */
        RL(_lwp_kill(lwp_data[0].threadid, SIGUSR1));

        /* Yay! No panic! */

        reap_lwp_waiter(&lwp_data[0]);
}

ATF_TC_WITH_CLEANUP(futex_wait_evil_unmapped_anon);
ATF_TC_HEAD(futex_wait_evil_unmapped_anon, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests futex WAIT while futex is unmapped - anon memory");
}
ATF_TC_BODY(futex_wait_evil_unmapped_anon, tc)
{
        do_futex_wait_evil_unmapped(MAP_ANON);
}
ATF_TC_CLEANUP(futex_wait_evil_unmapped_anon, tc)
{
        signal(SIGUSR1, old_act);
        do_cleanup();
}

/*****************************************************************************/

static int pri_min;
static int pri_max;

static void
lowpri_simple_test_waiter_lwp(void *arg)
{
        struct lwp_data *d = arg;
        struct sched_param sp;
        int policy;

        d->threadid = _lwp_self();

        RL(_sched_getparam(getpid(), d->threadid, &policy, &sp));
        policy = SCHED_RR;
        sp.sched_priority = pri_min;
        RL(_sched_setparam(getpid(), d->threadid, policy, &sp));

        simple_test_waiter_lwp(arg);
}

static void
highpri_simple_test_waiter_lwp(void *arg)
{
        struct lwp_data *d = arg;
        struct sched_param sp;
        int policy;

        d->threadid = _lwp_self();

        RL(_sched_getparam(getpid(), d->threadid, &policy, &sp));
        policy = SCHED_RR;
        sp.sched_priority = pri_max;
        RL(_sched_setparam(getpid(), d->threadid, policy, &sp));

        simple_test_waiter_lwp(arg);
}

static void
do_test_wake_highest_pri(void)
{
        lwpid_t waiter;
        int tries;
        long pri;
        int n;

        RL(pri = sysconf(_SC_SCHED_PRI_MIN));
        pri_min = (int)pri;
        RL(pri = sysconf(_SC_SCHED_PRI_MAX));
        pri_max = (int)pri;

        futex_word = 0;
        membar_sync();

        setup_lwp_context(&lwp_data[0], lowpri_simple_test_waiter_lwp);
        lwp_data[0].op_flags = FUTEX_PRIVATE_FLAG;
        lwp_data[0].futex_error = -1;
        lwp_data[0].futex_ptr = &futex_word;
        lwp_data[0].block_val = 0;
        lwp_data[0].bitset = 0;
        lwp_data[0].wait_op = FUTEX_WAIT;
        RL(_lwp_create(&lwp_data[0].context, 0, &lwp_data[0].lwpid));

        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 1)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 1,
            "lowpri waiter failed to start, nlwps_running=%u", nlwps_running);

        /* Ensure it's blocked. */
        ATF_REQUIRE_EQ_MSG(lwp_data[0].futex_error, -1,
            "lwp_data[0].futex_error=%d", lwp_data[0].futex_error);

        setup_lwp_context(&lwp_data[1], highpri_simple_test_waiter_lwp);
        lwp_data[1].op_flags = FUTEX_PRIVATE_FLAG;
        lwp_data[1].futex_error = -1;
        lwp_data[1].futex_ptr = &futex_word;
        lwp_data[1].block_val = 0;
        lwp_data[1].bitset = 0;
        lwp_data[1].wait_op = FUTEX_WAIT;
        RL(_lwp_create(&lwp_data[1].context, 0, &lwp_data[1].lwpid));

        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 2)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 2,
            "highpri waiter failed to start, nlwps_running=%u", nlwps_running);

        /* Ensure it's blocked. */
        ATF_REQUIRE_EQ_MSG(lwp_data[1].futex_error, -1,
            "lwp_data[1].futex_error=%d", lwp_data[1].futex_error);

        /* Wake the first LWP.  We should get the highpri thread. */
        RL(n = __futex(&futex_word, FUTEX_WAKE | FUTEX_PRIVATE_FLAG,
                1, NULL, NULL, 0, 0));
        ATF_REQUIRE_EQ_MSG(n, 1, "n=%d woken", n);
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 1)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 1, "nlwps_running=%u",
            nlwps_running);
        RL(_lwp_wait(0, &waiter));
        ATF_REQUIRE_EQ_MSG(waiter, lwp_data[1].threadid,
            "waiter=%ld lwp_data[1].threadid=%ld",
            (long)waiter, (long)lwp_data[1].threadid);

        /* Wake the second LWP.  We should get the lowpri thread. */
        RL(n = __futex(&futex_word, FUTEX_WAKE | FUTEX_PRIVATE_FLAG,
                1, NULL, NULL, 0, 0));
        ATF_REQUIRE_EQ_MSG(n, 1, "n=%d woken", n);
        sleep(1);
        for (tries = 0; tries < 5; tries++) {
                membar_sync();
                if (nlwps_running == 0)
                        break;
                sleep(1);
        }
        membar_sync();
        ATF_REQUIRE_EQ_MSG(nlwps_running, 0, "nlwps_running=%u",
            nlwps_running);
        RL(_lwp_wait(0, &waiter));
        ATF_REQUIRE_EQ_MSG(waiter, lwp_data[0].threadid,
            "waiter=%ld lwp_data[0].threadid=%ld",
            (long)waiter, (long)lwp_data[0].threadid);
}

ATF_TC_WITH_CLEANUP(futex_wake_highest_pri);
ATF_TC_HEAD(futex_wake_highest_pri, tc)
{
        atf_tc_set_md_var(tc, "descr",
            "tests that futex WAKE wakes the highest priority waiter");
        atf_tc_set_md_var(tc, "require.user", "root");
}
ATF_TC_BODY(futex_wake_highest_pri, tc)
{
        atf_tc_expect_fail("PR kern/55230");
        do_test_wake_highest_pri();
}
ATF_TC_CLEANUP(futex_wake_highest_pri, tc)
{
        do_cleanup();
}

/*****************************************************************************/

ATF_TP_ADD_TCS(tp)
{
        ATF_TP_ADD_TC(tp, futex_basic_wait_wake_private);
        ATF_TP_ADD_TC(tp, futex_basic_wait_wake_shared);
        ATF_TP_ADD_TC(tp, futex_wait_wake_anon_bs_private);
        ATF_TP_ADD_TC(tp, futex_wait_wake_anon_bs_shared);
        ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_private);
        ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_shared);
        ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_cow_private);
        ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_cow_shared);

        ATF_TP_ADD_TC(tp, futex_wait_wake_anon_bs_shared_proc);
        ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_shared_proc);

        ATF_TP_ADD_TC(tp, futex_wait_pointless_bitset);
        ATF_TP_ADD_TC(tp, futex_wait_wake_bitset);

        ATF_TP_ADD_TC(tp, futex_wait_timeout_relative);
        ATF_TP_ADD_TC(tp, futex_wait_timeout_relative_rt);
        ATF_TP_ADD_TC(tp, futex_wait_timeout_deadline);
        ATF_TP_ADD_TC(tp, futex_wait_timeout_deadline_rt);

        ATF_TP_ADD_TC(tp, futex_wait_evil_unmapped_anon);

        ATF_TP_ADD_TC(tp, futex_requeue);
        ATF_TP_ADD_TC(tp, futex_cmp_requeue);
        ATF_TP_ADD_TC(tp, futex_cmp_requeue_trivial);

        ATF_TP_ADD_TC(tp, futex_wake_op_op);
        ATF_TP_ADD_TC(tp, futex_wake_op_cmp);

        ATF_TP_ADD_TC(tp, futex_wake_highest_pri);

        return atf_no_error();
}