dmb
dmb();
dmb();
ALT_SMP(dmb ish)
ALT_SMP(W(dmb) ish)
ALT_SMP(mcr p15, 0, r0, c7, c10, 5) @ dmb
dmb ish
W(dmb) ish
mcr p15, 0, r0, c7, c10, 5 @ dmb
#define dma_rmb() dmb(osh)
#define dma_wmb() dmb(oshst)
#define __smp_mb() dmb(ish)
#define __smp_wmb() dmb(ishst)
dmb();
dmb();
dmb();
dmb();
ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\
ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) \
ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \
ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \
__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K)
__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K)
__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K)
__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L)
__CMPXCHG128(_mb, dmb ish, l, "memory")
ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\
ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\
#define __smp_mb() dmb(ish)
#define __smp_rmb() dmb(ishld)
#define __smp_wmb() dmb(ishst)
#define __dma_mb() dmb(osh)
#define __dma_rmb() dmb(oshld)
#define __dma_wmb() dmb(oshst)
__XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory")
__XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory")
__AARCH64_INSN_FUNCS(dmb, 0xFFFFF0FF, 0xD50330BF)
#ifdef dmb
#define aarch32_smp_mb() dmb(ish)
#define aarch32_smp_rmb() dmb(ishld)
#define aarch32_smp_wmb() dmb(ishst)
dmb(sy);
dmb(osh);
dmb();
dmb();
dmb->idx = dmb_node->sba_idx;
dmb->dmb_tok = dmb_node->token;
dmb->cpu_addr = dmb_node->cpu_addr;
dmb->dma_addr = dmb_node->dma_addr;
dmb->dmb_len = dmb_node->len;
static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
if (tmp_node->token == dmb->dmb_tok) {
static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
if (tmp_node->token == dmb->dmb_tok) {
dmb->idx = dmb_node->sba_idx;
dmb->dmb_tok = dmb_node->token;
dmb->cpu_addr = dmb_node->cpu_addr;
dmb->dma_addr = dmb_node->dma_addr;
dmb->dmb_len = dmb_node->len;
static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb,
sba_idx = dmb->idx;
dmb_node->len = dmb->dmb_len;
dmb(sy);
dmb(sy);
dmb(ishst);
dmb(ishst);
u64 dmb;
#define ISM_CREATE_REQ(dmb, idx, sf, offset) \
((dmb) | (idx) << 24 | (sf) << 23 | (offset))
static void ism_free_dmb(struct ism_dev *ism, struct dibs_dmb *dmb)
clear_bit(dmb->idx, ism->sba_bitmap);
dma_unmap_page(&ism->pdev->dev, dmb->dma_addr, dmb->dmb_len,
folio_put(virt_to_folio(dmb->cpu_addr));
static int ism_alloc_dmb(struct ism_dev *ism, struct dibs_dmb *dmb)
if (PAGE_ALIGN(dmb->dmb_len) > dma_get_max_seg_size(&ism->pdev->dev))
if (!dmb->idx) {
dmb->idx = bit;
if (dmb->idx < ISM_DMB_BIT_OFFSET ||
test_and_set_bit(dmb->idx, ism->sba_bitmap))
__GFP_NORETRY, get_order(dmb->dmb_len));
dmb->cpu_addr = folio_address(folio);
dmb->dma_addr = dma_map_page(&ism->pdev->dev,
virt_to_page(dmb->cpu_addr), 0,
dmb->dmb_len, DMA_FROM_DEVICE);
if (dma_mapping_error(&ism->pdev->dev, dmb->dma_addr)) {
kfree(dmb->cpu_addr);
clear_bit(dmb->idx, ism->sba_bitmap);
static int ism_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb,
ret = ism_alloc_dmb(ism, dmb);
cmd.request.dmb = dmb->dma_addr;
cmd.request.dmb_len = dmb->dmb_len;
cmd.request.sba_idx = dmb->idx;
cmd.request.vlan_valid = dmb->vlan_valid;
cmd.request.vlan_id = dmb->vlan_id;
memcpy(&cmd.request.rgid, &dmb->rgid, sizeof(u64));
ism_free_dmb(ism, dmb);
dmb->dmb_tok = cmd.response.dmb_tok;
dibs->dmb_clientid_arr[dmb->idx - ISM_DMB_BIT_OFFSET] = client->id;
static int ism_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
cmd.request.dmb_tok = dmb->dmb_tok;
dibs->dmb_clientid_arr[dmb->idx - ISM_DMB_BIT_OFFSET] = NO_DIBS_CLIENT;
ism_free_dmb(ism, dmb);
int (*register_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb,
int (*unregister_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb);
int (*attach_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb);
struct dibs_dmb dmb;
memset(&dmb, 0, sizeof(dmb));
dmb.dmb_tok = dmb_desc->token;
dmb.idx = dmb_desc->sba_idx;
dmb.cpu_addr = dmb_desc->cpu_addr;
dmb.dma_addr = dmb_desc->dma_addr;
dmb.dmb_len = dmb_desc->len;
smcd->dibs->ops->unregister_dmb(smcd->dibs, &dmb);
struct dibs_dmb dmb;
memset(&dmb, 0, sizeof(dmb));
dmb.dmb_len = dmb_len;
dmb.idx = dmb_desc->sba_idx;
dmb.vlan_id = lgr->vlan_id;
copy_to_dibsgid(&dmb.rgid, &lgr->peer_gid);
rc = dibs->ops->register_dmb(dibs, &dmb, &smc_dibs_client);
dmb_desc->sba_idx = dmb.idx;
dmb_desc->token = dmb.dmb_tok;
dmb_desc->cpu_addr = dmb.cpu_addr;
dmb_desc->dma_addr = dmb.dma_addr;
dmb_desc->len = dmb.dmb_len;
struct dibs_dmb dmb;
memset(&dmb, 0, sizeof(dmb));
dmb.dmb_tok = token;
rc = dev->dibs->ops->attach_dmb(dev->dibs, &dmb);
dmb_desc->sba_idx = dmb.idx;
dmb_desc->token = dmb.dmb_tok;
dmb_desc->cpu_addr = dmb.cpu_addr;
dmb_desc->dma_addr = dmb.dma_addr;
dmb_desc->len = dmb.dmb_len;
#define dma_wmb() dmb(oshst)
#define dma_rmb() dmb(oshld)
#define virt_rmb() dmb(ishld)
#define virt_wmb() dmb(ishst)
#define virt_store_mb(var, value) do { WRITE_ONCE(var, value); dmb(ish); } while (0)