root/usr/src/lib/udapl/udapl_tavor/common/dapl_cr_callback.c
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 *
 * MODULE: dapls_cr_callback.c
 *
 * PURPOSE: implements passive side connection callbacks
 *
 * Description: Accepts asynchronous callbacks from the Communications Manager
 *              for EVDs that have been specified as the connection_evd.
 *
 * $Id: dapl_cr_callback.c,v 1.58 2003/08/20 14:55:39 sjs2 Exp $
 */

#include "dapl.h"
#include "dapl_evd_util.h"
#include "dapl_cr_util.h"
#include "dapl_ia_util.h"
#include "dapl_sp_util.h"
#include "dapl_ep_util.h"
#include "dapl_adapter_util.h"


/*
 * Prototypes
 */
DAT_RETURN dapli_connection_request(
        IN ib_cm_handle_t ib_cm_handle,
        IN DAPL_SP *sp_ptr,
        IN DAPL_PRIVATE *prd_ptr,
        IN DAPL_EVD *evd_ptr);

DAPL_EP * dapli_get_sp_ep(
        IN ib_cm_handle_t ib_cm_handle,
        IN DAPL_SP *sp_ptr,
        IN const ib_cm_events_t ib_cm_event);

/*
 * dapls_cr_callback
 *
 * The callback function registered with verbs for passive side of
 * connection requests. The interface is specified by cm_api.h
 *
 *
 * Input:
 *      ib_cm_handle,           Handle to CM
 *      ib_cm_event             Specific CM event
 *      instant_data            Private data with DAT ADDRESS header
 *      context                 SP pointer
 *
 * Output:
 *      None
 *
 */
void
dapls_cr_callback(
        IN ib_cm_handle_t ib_cm_handle,
        IN const ib_cm_events_t ib_cm_event,
        IN const void *private_data_ptr, /* event data */
        IN const void *context)
{
        DAPL_EP         *ep_ptr;
        DAPL_EVD        *evd_ptr;
        DAPL_SP         *sp_ptr;
        DAPL_PRIVATE    *prd_ptr;
        DAT_EVENT_NUMBER        event_type;
        DAT_RETURN              dat_status;

        dapl_dbg_log(DAPL_DBG_TYPE_CM,
            "--> dapls_cr_callback! context: 0x%p "
            "event: %d cm_handle 0x%llx magic 0x%x\n",
            context, ib_cm_event, ib_cm_handle,
            ((DAPL_HEADER *)context)->magic);

        if (((DAPL_HEADER *)context)->magic == DAPL_MAGIC_INVALID) {
                return;
        }
        /*
         * Passive side of the connection, context is a SP and
         * we need to look up the EP.
         */
        dapl_os_assert(((DAPL_HEADER *)context)->magic == DAPL_MAGIC_PSP ||
            ((DAPL_HEADER *)context)->magic == DAPL_MAGIC_RSP);
        sp_ptr = (DAPL_SP *) context;

        /*
         * CONNECT_REQUEST events create an event on the PSP
         * EVD, which will trigger connection processing. The
         * sequence is:
         * CONNECT_REQUEST Event to SP
         * CONNECTED Event to EP
         * DISCONNECT Event to EP
         *
         * Obtain the EP if required and set an event up on the correct EVD.
         */
        if (ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING ||
            ib_cm_event == IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA) {
                ep_ptr = NULL;
                evd_ptr = sp_ptr->evd_handle;
        } else {
                ep_ptr = dapli_get_sp_ep(ib_cm_handle, sp_ptr, ib_cm_event);
                dapl_os_assert(ep_ptr != NULL);
                evd_ptr = (DAPL_EVD *) ep_ptr->param.connect_evd_handle;
                dapl_dbg_log(DAPL_DBG_TYPE_CM,
                    "    dapls_cr_callback cont: ep 0x%p evd 0x%p\n",
                    ep_ptr, evd_ptr);
        }

        prd_ptr = (DAPL_PRIVATE *)private_data_ptr;
        dat_status = DAT_INTERNAL_ERROR;        /* init to ERR */

        switch (ib_cm_event) {
        case IB_CME_CONNECTION_REQUEST_PENDING:
        case IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA: {
                /*
                 * Requests arriving on a disabled SP are immediatly rejected
                 */

                dapl_os_lock(&sp_ptr->header.lock);
                if (sp_ptr->listening == DAT_FALSE) {
                        dapl_os_unlock(&sp_ptr->header.lock);
                        dapl_dbg_log(DAPL_DBG_TYPE_CM,
                        "---> dapls_cr_callback: conn event on down SP\n");
                        return;
                }

                if (sp_ptr->header.handle_type == DAT_HANDLE_TYPE_RSP) {
                /*
                 * RSP connections only allow a single connection. Close
                 * it down NOW so we reject any further connections.
                 */
                        sp_ptr->listening = DAT_FALSE;
                }
                dapl_os_unlock(&sp_ptr->header.lock);

                /*
                 * Only occurs on the passive side of a connection
                 * dapli_connection_request will post the connection
                 * event if appropriate.
                 */
                dat_status = dapli_connection_request(ib_cm_handle,
                    sp_ptr, prd_ptr, evd_ptr);
                break;
        }
        case IB_CME_CONNECTED: {
                /*
                 * This is just a notification the connection is now
                 * established, there isn't any private data to deal with.
                 *
                 * Update the EP state and cache a copy of the cm handle,
                 * then let the user know we are ready to go.
                 */
                dapl_os_lock(&ep_ptr->header.lock);
                if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECT_PENDING) {
                /*
                 * If someone pulled the plug on the connection, just
                 * exit
                 */
                        dapl_os_unlock(&ep_ptr->header.lock);
                        dat_status = DAT_SUCCESS;
                        break;
                }
                dapls_ib_connected(ep_ptr);
                ep_ptr->param.ep_state = DAT_EP_STATE_CONNECTED;
                ep_ptr->cm_handle = ib_cm_handle;
                dapl_os_unlock(&ep_ptr->header.lock);

                dat_status = dapls_evd_post_connection_event(
                    evd_ptr,
                    DAT_CONNECTION_EVENT_ESTABLISHED,
                    (DAT_HANDLE) ep_ptr,
                    ((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data_size,
                    ((DAPL_CR *)ep_ptr->cr_ptr)->param.private_data);
                /*
                 * post them to the recv evd now.
                 * there is a race here - if events arrive after we change
                 * the ep state to connected and before we process premature
                 * events
                 */
                dapls_evd_post_premature_events(ep_ptr);
                break;
        }
        case IB_CME_DISCONNECTED:
        case IB_CME_DISCONNECTED_ON_LINK_DOWN: {
                /*
                 * EP is now fully disconnected; initiate any post processing
                 * to reset the underlying QP and get the EP ready for
                 * another connection
                 */
                dapl_os_lock(&ep_ptr->header.lock);
                if (ep_ptr->param.ep_state == DAT_EP_STATE_DISCONNECTED) {
                        /* DTO error caused this */
                        event_type = DAT_CONNECTION_EVENT_BROKEN;
                } else {
                        ep_ptr->param.ep_state  = DAT_EP_STATE_DISCONNECTED;
                        dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE,
                            ib_cm_event);
                        event_type = DAT_CONNECTION_EVENT_DISCONNECTED;
                }
                dapls_evd_post_premature_events(ep_ptr);

                ep_ptr->cr_ptr = NULL;
                dapl_os_unlock(&ep_ptr->header.lock);

                /*
                 * If the user has done an ep_free of the EP, we have been
                 * waiting for the disconnect event; just clean it up now.
                 */
                if (ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT) {
                        (void) dapl_ep_free(ep_ptr);
                }

                /* If the EP has been freed, the evd_ptr will be NULL */
                if (evd_ptr != NULL) {
                        dat_status = dapls_evd_post_connection_event(
                            evd_ptr, event_type, (DAT_HANDLE) ep_ptr, 0, 0);
                }

                break;
        }
        case IB_CME_DESTINATION_REJECT:
        case IB_CME_DESTINATION_REJECT_PRIVATE_DATA:
        case IB_CME_DESTINATION_UNREACHABLE: {
                /*
                 * After posting an accept the requesting node has
                 * stopped talking.
                 */
                dapl_os_lock(&ep_ptr->header.lock);
                ep_ptr->param.ep_state  = DAT_EP_STATE_DISCONNECTED;
                ep_ptr->cm_handle = IB_INVALID_HANDLE;
                dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
                dapl_os_unlock(&ep_ptr->header.lock);
                dat_status = dapls_evd_post_connection_event(
                    evd_ptr,
                    DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR,
                    (DAT_HANDLE) ep_ptr, 0, 0);

                break;
        }
        case IB_CME_TOO_MANY_CONNECTION_REQUESTS: {
                /*
                 * DAPL does not deal with this IB error. There is a
                 * separate OVERFLOW event error if we try to post too many
                 * events, but we don't propagate this provider error.  Not
                 * all providers generate this error.
                 */
                break;
        }
        case IB_CME_LOCAL_FAILURE: {
                ep_ptr->param.ep_state  = DAT_EP_STATE_DISCONNECTED;
                dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
                dat_status = dapls_evd_post_connection_event(
                    evd_ptr,
                    DAT_CONNECTION_EVENT_BROKEN,
                    (DAT_HANDLE) ep_ptr, 0, 0);

                break;
        }
        case IB_CME_TIMED_OUT: {
                ep_ptr->param.ep_state = DAT_EP_STATE_DISCONNECTED;
                dapls_ib_disconnect_clean(ep_ptr, DAT_FALSE, ib_cm_event);
                dat_status = dapls_evd_post_connection_event(
                    evd_ptr,
                    DAT_CONNECTION_EVENT_TIMED_OUT,
                    (DAT_HANDLE) ep_ptr, 0, 0);

                break;
        }
        default:
                dapl_os_assert(0);              /* shouldn't happen */
                break;
        }

        if (dat_status != DAT_SUCCESS) {
                /* The event post failed; take appropriate action.  */
                (void) dapls_ib_reject_connection(ib_cm_handle,
                    IB_CME_LOCAL_FAILURE, sp_ptr);
                return;
        }
}


/*
 * dapli_connection_request
 *
 * Process a connection request on the Passive side of a connection.
 * Create a CR record and link it on to the SP so we can update it
 * and free it later. Create an EP if specified by the PSP flags.
 *
 * Input:
 *      ib_cm_handle,
 *      sp_ptr
 *      event_ptr
 *      prd_ptr
 *
 * Output:
 *      None
 *
 * Returns
 *      DAT_INSUFFICIENT_RESOURCES
 *      DAT_SUCCESS
 *
 */
DAT_RETURN
dapli_connection_request(
        IN  ib_cm_handle_t ib_cm_handle,
        IN  DAPL_SP *sp_ptr,
        IN  DAPL_PRIVATE *prd_ptr,
        IN  DAPL_EVD *evd_ptr)
{
        DAT_RETURN      dat_status;
        DAPL_CR         *cr_ptr;
        DAPL_EP         *ep_ptr;
        DAPL_IA         *ia_ptr;
        DAT_SP_HANDLE   sp_handle;
        struct sockaddr_in *sv4;
        struct sockaddr_in6 *sv6;
        uint8_t         *sadata;
        DAT_COUNT       length;

        cr_ptr = dapls_cr_alloc(sp_ptr->header.owner_ia);
        if (cr_ptr == NULL) {
                /* Invoking function will call dapls_ib_cm_reject() */
                return (DAT_INSUFFICIENT_RESOURCES);
        }

        /*
         * Set up the CR
         */
        cr_ptr->sp_ptr = sp_ptr; /* maintain sp_ptr in case of reject */
        cr_ptr->ib_cm_handle = ib_cm_handle;
        /*
         * Copy the remote address and private data out of the private_data
         * payload and put them in a local structure
         */
        cr_ptr->param.private_data = cr_ptr->private_data;
        cr_ptr->param.remote_ia_address_ptr =
            (DAT_IA_ADDRESS_PTR)&cr_ptr->remote_ia_address;
        cr_ptr->param.remote_port_qual =
            (DAT_PORT_QUAL) prd_ptr->hello_msg.hi_port;
        length = (DAT_COUNT) prd_ptr->hello_msg.hi_clen;
        cr_ptr->param.private_data_size = length;
        (void) dapl_os_memcpy(cr_ptr->private_data,
            prd_ptr->private_data, length);
        switch (prd_ptr->hello_msg.hi_ipv) {
        case AF_INET:
                sv4 = (struct sockaddr_in *)&cr_ptr->remote_ia_address;
                sv4->sin_family = AF_INET;
                sv4->sin_port = prd_ptr->hello_msg.hi_port;
                sv4->sin_addr = prd_ptr->hello_msg.hi_v4ipaddr;
                break;
        case AF_INET6:
                sv6 = (struct sockaddr_in6 *)&cr_ptr->remote_ia_address;
                sv6->sin6_family = AF_INET6;
                sv6->sin6_port = prd_ptr->hello_msg.hi_port;
                sv6->sin6_addr = prd_ptr->hello_msg.hi_v6ipaddr;
                break;
        default:
                sadata = (uint8_t *)&cr_ptr->remote_ia_address;
                (void) dapl_os_memcpy(sadata, prd_ptr->hello_msg.hi_saaddr,
                    DAPL_ATS_NBYTES);
                break;
        }

        /* EP will be NULL unless RSP service point */
        ep_ptr = (DAPL_EP *) sp_ptr->ep_handle;

        if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) {
                /*
                 * Never true for RSP connections
                 *
                 * Create an EP for the user. If we can't allocate an
                 * EP we are out of resources and need to tell the
                 * requestor that we cant help them.
                 */
                ia_ptr = sp_ptr->header.owner_ia;
                ep_ptr = dapl_ep_alloc(ia_ptr, NULL, DAT_FALSE);
                if (ep_ptr == NULL) {
                        dapls_cr_free(cr_ptr);
                        /* Invoking function will call dapls_ib_cm_reject() */
                        return (DAT_INSUFFICIENT_RESOURCES);
                }
                /* Link the EP onto the IA */
                dapl_ia_link_ep(ia_ptr, ep_ptr);
        }

        cr_ptr->param.local_ep_handle = ep_ptr;

        if (ep_ptr != NULL) {
                /* Assign valid EP fields: RSP and PSP_PROVIDER_FLAG only */
                if (sp_ptr->psp_flags == DAT_PSP_PROVIDER_FLAG) {
                        ep_ptr->param.ep_state =
                            DAT_EP_STATE_TENTATIVE_CONNECTION_PENDING;
                } else { /* RSP */
                        dapl_os_assert(sp_ptr->header.handle_type ==
                            DAT_HANDLE_TYPE_RSP);
                        ep_ptr->param.ep_state =
                            DAT_EP_STATE_PASSIVE_CONNECTION_PENDING;
                }
                ep_ptr->cm_handle = ib_cm_handle;
        }

        /* Post the event.  */
        /* assign sp_ptr to union to avoid typecast errors from compilers */
        sp_handle.psp_handle = (DAT_PSP_HANDLE)sp_ptr;
        dat_status = dapls_evd_post_cr_arrival_event(
            evd_ptr,
            DAT_CONNECTION_REQUEST_EVENT,
            sp_handle,
            (DAT_IA_ADDRESS_PTR)&sp_ptr->header.owner_ia->hca_ptr->hca_address,
            sp_ptr->conn_qual,
            (DAT_CR_HANDLE)cr_ptr);
        if (dat_status != DAT_SUCCESS) {
                dapls_cr_free(cr_ptr);
                (void) dapls_ib_reject_connection(ib_cm_handle,
                    IB_CME_LOCAL_FAILURE, sp_ptr);
                return (DAT_INSUFFICIENT_RESOURCES);
        }

        /* link the CR onto the SP so we can pick it up later */
        dapl_sp_link_cr(sp_ptr, cr_ptr);

        return (DAT_SUCCESS);
}


/*
 * dapli_get_sp_ep
 *
 * Passive side of a connection is now fully established. Clean
 * up resources and obtain the EP pointer associated with a CR in
 * the SP
 *
 * Input:
 *      ib_cm_handle,
 *      sp_ptr
 *
 * Output:
 *      none
 *
 * Returns
 *      ep_ptr
 *
 */
DAPL_EP *
dapli_get_sp_ep(
        IN ib_cm_handle_t ib_cm_handle,
        IN DAPL_SP *sp_ptr,
        IN const ib_cm_events_t ib_cm_event)
{
        DAPL_CR         *cr_ptr;
        DAPL_EP         *ep_ptr;

        /*
         * There are potentially multiple connections in progress. Need to
         * go through the list and find the one we are interested
         * in. There is no guarantee of order. dapl_sp_search_cr
         * leaves the CR on the SP queue.
         */
        cr_ptr = dapl_sp_search_cr(sp_ptr, ib_cm_handle);
        if (cr_ptr == NULL) {
                dapl_os_assert(0);
                return (NULL);
        }

        ep_ptr = (DAPL_EP *)cr_ptr->param.local_ep_handle;

        dapl_os_assert(!(DAPL_BAD_HANDLE(ep_ptr, DAPL_MAGIC_EP)) ||
            ep_ptr->header.magic == DAPL_MAGIC_EP_EXIT);

        if (ib_cm_event == IB_CME_DISCONNECTED ||
            ib_cm_event == IB_CME_DISCONNECTED_ON_LINK_DOWN) {
                /* Remove the CR from the queue */
                dapl_sp_remove_cr(sp_ptr, cr_ptr);
                /*
                 * Last event, time to clean up and dispose of the resource
                 */
                dapls_cr_free(cr_ptr);

                /*
                 * If this SP has been removed from service, free it
                 * up after the last CR is removed
                 */
                dapl_os_lock(&sp_ptr->header.lock);
                if (sp_ptr->listening != DAT_TRUE &&
                    sp_ptr->cr_list_count == 0 &&
                    sp_ptr->state != DAPL_SP_STATE_FREE) {
                        dapl_dbg_log(DAPL_DBG_TYPE_CM,
                            "--> dapli_get_sp_ep! disconnect dump sp: %p \n",
                            sp_ptr);
                        sp_ptr->state = DAPL_SP_STATE_FREE;
                        dapl_os_unlock(&sp_ptr->header.lock);
                        (void) dapls_ib_remove_conn_listener(sp_ptr->
                            header.owner_ia, sp_ptr);
                        dapls_ia_unlink_sp((DAPL_IA *)sp_ptr->header.owner_ia,
                            sp_ptr);
                        dapls_sp_free_sp(sp_ptr);
                } else {
                        dapl_os_unlock(&sp_ptr->header.lock);
                }
        }
        return (ep_ptr);
}