/*
 * Copyright (c) 2008-2016 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@backplane.com>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#ifndef _CPU_CPUMASK_H_
#define _CPU_CPUMASK_H_

#include <machine/stdint.h>
#ifdef _KERNEL
#include <cpu/atomic.h>
#endif

/*
 * cpumask_t - a mask representing a set of cpus and supporting routines.
 *
 * WARNING! It is recommended that this mask NOT be made variably-sized
 *          because it affects a huge number of system structures.  However,
 *          kernel code (non-module) can be optimized to not operate on the
 *          whole mask.
 */

typedef struct {
        __uint64_t      ary[4];
} __cpumask_t;

#define CPUMASK_ELEMENTS        4

#define CPUMASK_INITIALIZER_ALLONES     { .ary = { (__uint64_t)-1, \
                                          (__uint64_t)-1, \
                                          (__uint64_t)-1, \
                                          (__uint64_t)-1 } }
#define CPUMASK_INITIALIZER_ONLYONE     { .ary = { 1, 0, 0, 0 } }

#define CPUMASK_SIMPLE(cpu)     ((__uint64_t)1 << (cpu))

#define CPUMASK_ADDR(mask, cpu) (&(mask).ary[((cpu) >> 6) & 3])

#define BSRCPUMASK(val)         ((val).ary[3] ? 192 + bsrq((val).ary[3]) : \
                                ((val).ary[2] ? 128 + bsrq((val).ary[2]) : \
                                ((val).ary[1] ? 64 + bsrq((val).ary[1]) : \
                                                bsrq((val).ary[0]))))

#define BSFCPUMASK(val)         ((val).ary[0] ? bsfq((val).ary[0]) : \
                                ((val).ary[1] ? 64 + bsfq((val).ary[1]) : \
                                ((val).ary[2] ? 128 + bsfq((val).ary[2]) : \
                                                192 + bsfq((val).ary[3]))))

#define CPUMASK_CMPMASKEQ(val1, val2)   ((val1).ary[0] == (val2).ary[0] && \
                                         (val1).ary[1] == (val2).ary[1] && \
                                         (val1).ary[2] == (val2).ary[2] && \
                                         (val1).ary[3] == (val2).ary[3])

#define CPUMASK_CMPMASKNEQ(val1, val2)  ((val1).ary[0] != (val2).ary[0] || \
                                         (val1).ary[1] != (val2).ary[1] || \
                                         (val1).ary[2] != (val2).ary[2] || \
                                         (val1).ary[3] != (val2).ary[3])

#define CPUMASK_ISUP(val)               ((val).ary[0] == 1 && \
                                         (val).ary[1] == 0 && \
                                         (val).ary[2] == 0 && \
                                         (val).ary[3] == 0)

#define CPUMASK_TESTZERO(val)           ((val).ary[0] == 0 && \
                                         (val).ary[1] == 0 && \
                                         (val).ary[2] == 0 && \
                                         (val).ary[3] == 0)

#define CPUMASK_TESTNZERO(val)          ((val).ary[0] != 0 || \
                                         (val).ary[1] != 0 || \
                                         (val).ary[2] != 0 || \
                                         (val).ary[3] != 0)

#define CPUMASK_TESTBIT(val, i)         ((val).ary[((i) >> 6) & 3] & \
                                         CPUMASK_SIMPLE((i) & 63))

#define CPUMASK_TESTMASK(val1, val2)    (((val1).ary[0] & (val2.ary[0])) || \
                                         ((val1).ary[1] & (val2.ary[1])) || \
                                         ((val1).ary[2] & (val2.ary[2])) || \
                                         ((val1).ary[3] & (val2.ary[3])))

#define CPUMASK_LOWMASK(val)            ((val).ary[0])

#define CPUMASK_ORBIT(mask, i)          ((mask).ary[((i) >> 6) & 3] |= \
                                         CPUMASK_SIMPLE((i) & 63))

#define CPUMASK_ANDBIT(mask, i)         ((mask).ary[((i) >> 6) & 3] &= \
                                         CPUMASK_SIMPLE((i) & 63))

#define CPUMASK_NANDBIT(mask, i)        ((mask).ary[((i) >> 6) & 3] &= \
                                         ~CPUMASK_SIMPLE((i) & 63))

#define CPUMASK_ASSZERO(mask)           do {                            \
                                        (mask).ary[0] = 0;              \
                                        (mask).ary[1] = 0;              \
                                        (mask).ary[2] = 0;              \
                                        (mask).ary[3] = 0;              \
                                        } while(0)

#define CPUMASK_ASSALLONES(mask)        do {                            \
                                        (mask).ary[0] = (__uint64_t)-1; \
                                        (mask).ary[1] = (__uint64_t)-1; \
                                        (mask).ary[2] = (__uint64_t)-1; \
                                        (mask).ary[3] = (__uint64_t)-1; \
                                        } while(0)

#define CPUMASK_ASSBIT(mask, i)         do {                            \
                                                CPUMASK_ASSZERO(mask);  \
                                                CPUMASK_ORBIT(mask, i); \
                                        } while(0)

#define CPUMASK_ASSBMASK(mask, i)       do {                            \
                if (i < 64) {                                           \
                        (mask).ary[0] = CPUMASK_SIMPLE(i) - 1;          \
                        (mask).ary[1] = 0;                              \
                        (mask).ary[2] = 0;                              \
                        (mask).ary[3] = 0;                              \
                } else if (i < 128) {                                   \
                        (mask).ary[0] = (__uint64_t)-1;                 \
                        (mask).ary[1] = CPUMASK_SIMPLE((i) - 64) - 1;   \
                        (mask).ary[2] = 0;                              \
                        (mask).ary[3] = 0;                              \
                } else if (i < 192) {                                   \
                        (mask).ary[0] = (__uint64_t)-1;                 \
                        (mask).ary[1] = (__uint64_t)-1;                 \
                        (mask).ary[2] = CPUMASK_SIMPLE((i) - 128) - 1;  \
                        (mask).ary[3] = 0;                              \
                } else {                                                \
                        (mask).ary[0] = (__uint64_t)-1;                 \
                        (mask).ary[1] = (__uint64_t)-1;                 \
                        (mask).ary[2] = (__uint64_t)-1;                 \
                        (mask).ary[3] = CPUMASK_SIMPLE((i) - 192) - 1;  \
                }                                                       \
                                        } while(0)

#define CPUMASK_ASSNBMASK(mask, i)      do {                            \
                if (i < 64) {                                           \
                        (mask).ary[0] = ~(CPUMASK_SIMPLE(i) - 1);       \
                        (mask).ary[1] = (__uint64_t)-1;                 \
                        (mask).ary[2] = (__uint64_t)-1;                 \
                        (mask).ary[3] = (__uint64_t)-1;                 \
                } else if (i < 128) {                                   \
                        (mask).ary[0] = 0;                              \
                        (mask).ary[1] = ~(CPUMASK_SIMPLE((i) - 64) - 1);\
                        (mask).ary[2] = (__uint64_t)-1;                 \
                        (mask).ary[3] = (__uint64_t)-1;                 \
                } else if (i < 192) {                                   \
                        (mask).ary[0] = 0;                              \
                        (mask).ary[1] = 0;                              \
                        (mask).ary[2] = ~(CPUMASK_SIMPLE((i) - 128) - 1);\
                        (mask).ary[3] = (__uint64_t)-1;                 \
                } else {                                                \
                        (mask).ary[0] = 0;                              \
                        (mask).ary[1] = 0;                              \
                        (mask).ary[2] = 0;                              \
                        (mask).ary[3] = ~(CPUMASK_SIMPLE((i) - 192) - 1);\
                }                                                       \
                                        } while(0)

#define CPUMASK_ANDMASK(mask, val)      do {                            \
                                        (mask).ary[0] &= (val).ary[0];  \
                                        (mask).ary[1] &= (val).ary[1];  \
                                        (mask).ary[2] &= (val).ary[2];  \
                                        (mask).ary[3] &= (val).ary[3];  \
                                        } while(0)

#define CPUMASK_NANDMASK(mask, val)     do {                            \
                                        (mask).ary[0] &= ~(val).ary[0]; \
                                        (mask).ary[1] &= ~(val).ary[1]; \
                                        (mask).ary[2] &= ~(val).ary[2]; \
                                        (mask).ary[3] &= ~(val).ary[3]; \
                                        } while(0)

#define CPUMASK_ORMASK(mask, val)       do {                            \
                                        (mask).ary[0] |= (val).ary[0];  \
                                        (mask).ary[1] |= (val).ary[1];  \
                                        (mask).ary[2] |= (val).ary[2];  \
                                        (mask).ary[3] |= (val).ary[3];  \
                                        } while(0)

#define CPUMASK_XORMASK(mask, val)      do {                            \
                                        (mask).ary[0] ^= (val).ary[0];  \
                                        (mask).ary[1] ^= (val).ary[1];  \
                                        (mask).ary[2] ^= (val).ary[2];  \
                                        (mask).ary[3] ^= (val).ary[3];  \
                                        } while(0)

#define CPUMASK_INVMASK(mask)           do {                            \
                                        (mask).ary[0] ^= -1L;           \
                                        (mask).ary[1] ^= -1L;           \
                                        (mask).ary[2] ^= -1L;           \
                                        (mask).ary[3] ^= -1L;           \
                                        } while(0)

#ifndef _KERNEL
#define __CPU_SETSIZE           ((int)(sizeof(cpumask_t) * 8))

#define __CPU_COUNT(set)        (                                       \
                                __builtin_popcountl((set)->ary[0]) +    \
                                __builtin_popcountl((set)->ary[1]) +    \
                                __builtin_popcountl((set)->ary[2]) +    \
                                __builtin_popcountl((set)->ary[3]))

#define __CPU_CLR(cpu, set)     CPUMASK_NANDBIT(*set, cpu)
#define __CPU_EQUAL(set1, set2) CPUMASK_CMPMASKEQ(*set1, *set2)
#define __CPU_ISSET(cpu, set)   CPUMASK_TESTBIT(*set, cpu)
#define __CPU_SET(cpu, set)     CPUMASK_ORBIT(*set, cpu)
#define __CPU_ZERO(set)         CPUMASK_ASSZERO(*set)

#define __CPU_AND(dst, set1, set2)                                      \
                        do {                                            \
                                if (dst == set1) {                      \
                                        CPUMASK_ANDMASK(*dst, *set2);   \
                                } else {                                \
                                        *dst = *set2;                   \
                                        CPUMASK_ANDMASK(*dst, *set1);   \
                                }                                       \
                        } while (0)

#define __CPU_OR(dst, set1, set2)                                       \
                        do {                                            \
                                if (dst == set1) {                      \
                                        CPUMASK_ORMASK(*dst, *set2);    \
                                } else {                                \
                                        *dst = *set2;                   \
                                        CPUMASK_ORMASK(*dst, *set1);    \
                                }                                       \
                        } while (0)

#define __CPU_XOR(dst, set1, set2)                                      \
                        do {                                            \
                                if (dst == set1) {                      \
                                        CPUMASK_XORMASK(*dst, *set2);   \
                                } else {                                \
                                        *dst = *set2;                   \
                                        CPUMASK_XORMASK(*dst, *set1);   \
                                }                                       \
                        } while (0)
#endif

#ifdef _KERNEL
#define ATOMIC_CPUMASK_ORBIT(mask, i)                                     \
                        atomic_set_cpumask(&(mask).ary[((i) >> 6) & 3],   \
                                           CPUMASK_SIMPLE((i) & 63))

#define ATOMIC_CPUMASK_NANDBIT(mask, i)                                   \
                        atomic_clear_cpumask(&(mask).ary[((i) >> 6) & 3], \
                                           CPUMASK_SIMPLE((i) & 63))

#define ATOMIC_CPUMASK_TESTANDSET(mask, i)                                \
                atomic_testandset_long(&(mask).ary[((i) >> 6) & 3], (i))

#define ATOMIC_CPUMASK_TESTANDCLR(mask, i)                                \
                atomic_testandclear_long(&(mask).ary[((i) >> 6) & 3], (i))

#define ATOMIC_CPUMASK_ORMASK(mask, val) do {                             \
                        atomic_set_cpumask(&(mask).ary[0], (val).ary[0]); \
                        atomic_set_cpumask(&(mask).ary[1], (val).ary[1]); \
                        atomic_set_cpumask(&(mask).ary[2], (val).ary[2]); \
                        atomic_set_cpumask(&(mask).ary[3], (val).ary[3]); \
                                         } while(0)

#define ATOMIC_CPUMASK_NANDMASK(mask, val) do {                             \
                        atomic_clear_cpumask(&(mask).ary[0], (val).ary[0]); \
                        atomic_clear_cpumask(&(mask).ary[1], (val).ary[1]); \
                        atomic_clear_cpumask(&(mask).ary[2], (val).ary[2]); \
                        atomic_clear_cpumask(&(mask).ary[3], (val).ary[3]); \
                                         } while(0)

#define ATOMIC_CPUMASK_COPY(mask, val) do {                                 \
                        atomic_store_rel_cpumask(&(mask).ary[0], (val).ary[0]);\
                        atomic_store_rel_cpumask(&(mask).ary[1], (val).ary[1]);\
                        atomic_store_rel_cpumask(&(mask).ary[2], (val).ary[2]);\
                        atomic_store_rel_cpumask(&(mask).ary[3], (val).ary[3]);\
                                         } while(0)
#endif

#endif /* !_CPU_CPUMASK_H_ */