/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 *
 * MODULE: dapl_ia_open.c
 *
 * PURPOSE: Interface Adapter management
 * Description: Interfaces in this file are completely described in
 *              the DAPL 1.1 API, Chapter 6, section 2
 *
 * $Id: dapl_ia_open.c,v 1.30 2003/07/31 14:04:17 jlentini Exp $
 */

#include "dapl.h"
#include "dapl_provider.h"
#include "dapl_evd_util.h"
#include "dapl_hca_util.h"
#include "dapl_ia_util.h"
#include "dapl_adapter_util.h"
#include <sys/systeminfo.h>
#include <libdevinfo.h>


/*
 * LOCAL PROTOTYPES
 */
#if defined(IBHOSTS_NAMING)
void dapli_assign_hca_ip_address(
        DAPL_HCA                *hca_ptr,
        char                    *device_name);
#endif /* IBHOSTS_NAMING */

static void dapli_hca_cleanup(DAPL_HCA *hca_ptr, DAT_BOOLEAN dec_ref);

/*
 * Determine whether the platform supports RO (Relaxed ordering)
 * Return B_TRUE if it does support RO and B_FALSE if it does not support RO
 *
 * udapl_ro_disallowed is an out paramter returning whether or not
 * relaxed ordering should be disabled (regardless of whether the platform
 * is capable of supporting relaxed ordering)
 *
 */
static boolean_t
dapl_ro_disallowed(void)
{
        static const char * const non_ro_capable_platforms[] = {
                "i86pc",
                "i86xpv",
                "SUNW,Sun-Fire-V215",
                "SUNW,Sun-Fire-V245",
                "SUNW,Sun-Fire-V445",
                "SUNW,Sun-Fire-T1000",
                "SUNW,Sun-Fire-T200",
                "SUNW,Sun-Blade-T6300",
                "SUNW,Sun-Blade-T6320",
                "SUNW,SPARC-Enterprise-T1000",
                "SUNW,SPARC-Enterprise-T2000",
                "SUNW,SPARC-Enterprise-T5120",
                "SUNW,SPARC-Enterprise-T5220",
                NULL
        };
        char platform[256 + 1];
        register int i;
        register const char *cp;
        int ret;
        di_node_t root_node, node;
        boolean_t ro_disallowed;
        static const char *ro_disallowed_property =
            "pci-relaxed-ordering-disallowed";
        int bool;
        int *boolp = &bool;

        ret = sysinfo(SI_PLATFORM, platform, sizeof (platform));
        if ((ret != -1) && (ret <= sizeof (platform))) {
                for (i = 0; (cp = non_ro_capable_platforms[i]) != NULL; ++i) {
                        if (strcmp(platform, cp) == 0)
                                return (B_TRUE);
                }
        }

        /*
         * This function only finds and looks at the FIRST udapl node.
         * It is assumed that there can only be one such node.
         */
        if ((root_node = di_init("/", DINFOSUBTREE | DINFOPROP)) == DI_NODE_NIL)
                return (B_FALSE);

        node = di_drv_first_node("daplt", root_node);
        if (node != DI_NODE_NIL) {
                ret = di_prop_lookup_ints(DDI_DEV_T_ANY, node,
                    ro_disallowed_property, &boolp);
                switch (ret) {
                case 0:
                case 1:
                        ro_disallowed = B_TRUE;
                        break;
                default:
                        ro_disallowed = B_FALSE;
                        break;
                }

        }
        else
                ro_disallowed = B_FALSE;

        di_fini(root_node);

        return (ro_disallowed);
}

/*
 * dapl_ia_open
 *
 * DAPL Requirements Version xxx, 6.2.1.1
 *
 * Open a provider and return a handle. The handle enables the user
 * to invoke operations on this provider.
 *
 * The dat_ia_open  call is actually part of the DAT registration module.
 * That function maps the DAT_NAME parameter of dat_ia_open to a DAT_PROVIDER,
 * and calls this function.
 *
 * Input:
 *      provider
 *      async_evd_qlen
 *      async_evd_handle_ptr
 *
 * Output:
 *      async_evd_handle
 *      ia_handle
 *
 * Return Values:
 *      DAT_SUCCESS
 *      DAT_INSUFFICIENT_RESOURCES
 *      DAT_INVALID_PARAMETER
 *      DAT_INVALID_HANDLE
 *      DAT_NAME_NOT_FOUND      (returned by dat registry if necessary)
 */
DAT_RETURN
dapl_ia_open(
        IN      const DAT_NAME_PTR      name,
        IN      DAT_COUNT               async_evd_qlen,
        INOUT   DAT_EVD_HANDLE          *async_evd_handle_ptr,
        OUT     DAT_IA_HANDLE           *ia_handle_ptr,
        IN      boolean_t               ro_aware_client)
{
        DAT_RETURN      dat_status;
        DAT_PROVIDER    *provider;
        DAPL_HCA        *hca_ptr;
        DAPL_IA         *ia_ptr;
        DAPL_EVD        *evd_ptr;
        boolean_t       ro_disallowed;

        dat_status = DAT_SUCCESS;
        hca_ptr = NULL;
        ia_ptr = NULL;

        dapl_dbg_log(DAPL_DBG_TYPE_API,
            "dapl_ia_open(%s, %d, %p, %p, %d)\n",
            name,
            async_evd_qlen,
            async_evd_handle_ptr,
            ia_handle_ptr,
            ro_aware_client);

        dat_status = dapl_provider_list_search(name, &provider);
        if (DAT_SUCCESS != dat_status) {
                dapl_dbg_log(DAPL_DBG_TYPE_API,
                    "dapl_ia_open: dapl_provider_list_search(\"%s\") returned "
                    "%d\n",
                    name,
                    dat_status);

                dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1);
                goto bail;
        }

        /* ia_handle_ptr and async_evd_handle_ptr cannot be NULL */
        if (ia_handle_ptr == NULL) {
                dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
                goto bail;
        }
        if (async_evd_handle_ptr == NULL) {
                dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG3);
                goto bail;
        }

        /* initialize the caller's OUT param */
        *ia_handle_ptr = DAT_HANDLE_NULL;

        /* get the hca_ptr */
        hca_ptr = (DAPL_HCA *)provider->extension;

        /*
         * Open the HCA if it has not been done before.
         */
        dapl_os_lock(&hca_ptr->lock);
        if (hca_ptr->ib_hca_handle == IB_INVALID_HANDLE) {
                /* register with the HW */
                dat_status = dapls_ib_open_hca(hca_ptr,
                    &hca_ptr->ib_hca_handle);

                if (dat_status != DAT_SUCCESS) {
                        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                            "dapls_ib_open_hca failed %d\n", dat_status);
                        dapl_os_unlock(&hca_ptr->lock);
                        goto bail;
                }

                /* create a cq domain for this HCA */
                dat_status = dapls_ib_cqd_create(hca_ptr);

                if (dat_status != DAT_SUCCESS) {
                        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                            "ERR: Cannot allocate CQD: err %x\n", dat_status);
                        dapli_hca_cleanup(hca_ptr, DAT_FALSE);
                        dapl_os_unlock(&hca_ptr->lock);
                        goto bail;
                }
                /*
                 * Obtain the IP address associated with this name and HCA.
                 */

#ifdef IBHOSTS_NAMING
                dapli_assign_hca_ip_address(hca_ptr, name);
#endif /* IBHOSTS_NAMING */

                /*
                 * Obtain IA attributes from the HCA to limit certain
                 * operations.
                 * If using DAPL_ATS naming, ib_query_hca will also set the ip
                 * address.
                 */
                dat_status = dapls_ib_query_hca(hca_ptr,
                    &hca_ptr->ia_attr,
                    NULL,
                    &hca_ptr->hca_address, NULL);
                if (dat_status != DAT_SUCCESS) {
                        dapli_hca_cleanup(hca_ptr, DAT_FALSE);
                        dapl_os_unlock(&hca_ptr->lock);
                        goto bail;
                }
        }

        /* is the IA going to use the ConnectX? */
        if (hca_ptr->hermon_resize_cq != 0) {
                /*
                 * We are running with a ConnectX.
                 * Determine whether platform is RO capable.
                 * If platform support RO and client does not
                 * support RO and we are not disabling RO, reject the open.
                 */
                ro_disallowed = dapl_ro_disallowed();

                if (! ro_aware_client && ! ro_disallowed) {
                        dapl_dbg_log(DAPL_DBG_TYPE_API,
                            "dapl_ia_open: failing ro_disallowed %d "
                            "ro_aware_client %d \n",
                            ro_disallowed, ro_aware_client);

                        dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
                            DAT_INVALID_RO_COOKIE);
                        dapli_hca_cleanup(hca_ptr, DAT_FALSE);
                        dapl_os_unlock(&hca_ptr->lock);
                        goto bail;
                }
        } else {
                /* We are not running with a Connect X */
                ro_disallowed = B_TRUE;
        }


        /* Take a reference on the hca_handle */
        dapl_os_atomic_inc(&hca_ptr->handle_ref_count);
        dapl_os_unlock(&hca_ptr->lock);

        /* Allocate and initialize ia structure */
        ia_ptr = dapl_ia_alloc(provider, hca_ptr);
        if (!ia_ptr) {
                dapl_os_lock(&hca_ptr->lock);
                dapli_hca_cleanup(hca_ptr, DAT_TRUE);
                dapl_os_unlock(&hca_ptr->lock);
                dat_status = DAT_ERROR(DAT_INSUFFICIENT_RESOURCES,
                    DAT_RESOURCE_MEMORY);
                goto bail;
        }

        /*
         * Note when we should be disabling relaxed ordering.
         * If the property indicates that we should not use relaxed ordering
         * we remember that fact.  If the platform is supposed to be
         * non relaxed ordering capable, we disable relaxed ordering as
         * well, just in case the property or the list indicating that
         * this platform is not relaxed ordering capable is mistaken.
         */
        if (ro_disallowed)
                ia_ptr->dapl_flags |= DAPL_DISABLE_RO;

        /*
         * we need an async EVD for this IA
         * use the one passed in (if non-NULL) or create one
         */

        evd_ptr = (DAPL_EVD *) *async_evd_handle_ptr;
        if (evd_ptr) {
                if (DAPL_BAD_HANDLE(evd_ptr, DAPL_MAGIC_EVD) ||
                    ! (evd_ptr->evd_flags & DAT_EVD_ASYNC_FLAG)) {
                        dat_status = DAT_ERROR(DAT_INVALID_HANDLE,
                            DAT_INVALID_HANDLE_EVD_ASYNC);
                        goto bail;
                }
                /*
                 * InfiniBand allows only 1 asychronous event handler per HCA
                 * (see InfiniBand Spec, release 1.1, vol I, section 11.5.2,
                 *  page 559).
                 *
                 * We only need to make sure that this EVD's CQ belongs to
                 * the same HCA as is being opened.
                 */

                if (evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle !=
                    hca_ptr->ib_hca_handle) {
                        dat_status = DAT_ERROR(DAT_INVALID_HANDLE,
                            DAT_INVALID_HANDLE_EVD_ASYNC);
                        goto bail;
                }

                ia_ptr->cleanup_async_error_evd = DAT_FALSE;
                ia_ptr->async_error_evd = evd_ptr;
        } else {
                /*
                 * Verify we have >0 length, and let the provider check the
                 * size
                 */
                if (async_evd_qlen <= 0) {
                        dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
                            DAT_INVALID_ARG2);
                        goto bail;
                }
                dat_status = dapls_evd_internal_create(ia_ptr,
                    NULL,       /* CNO ptr */
                    async_evd_qlen,
                    DAT_EVD_ASYNC_FLAG,
                    &evd_ptr);
                if (dat_status != DAT_SUCCESS) {
                        goto bail;
                }

                dapl_os_atomic_inc(&evd_ptr->evd_ref_count);

                dapl_os_lock(&hca_ptr->lock);
                if (hca_ptr->async_evd != (DAPL_EVD *) 0) {
#if 0
                        /*
                         * The async EVD for this HCA has already been assigned.
                         * It's an error to try and assign another one.
                         *
                         * However, we need to somehow allow multiple IAs
                         * off of the same HCA.  The right way to do this
                         * is by dispatching events off the HCA to the
                         * appropriate IA, but we aren't there yet.  So for
                         * now we create the EVD but don't connect it to
                         * anything.
                         */
                        dapl_os_atomic_dec(&evd_ptr->evd_ref_count);
                        dapl_evd_free(evd_ptr);
                        dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
                            DAT_INVALID_ARG4);
                        goto bail;
#endif
                        dapl_os_unlock(&hca_ptr->lock);
                } else {
                        hca_ptr->async_evd = evd_ptr;
                        dapl_os_unlock(&hca_ptr->lock);

                        /*
                         * Register the handlers associated with the async EVD.
                         */
                        dat_status = dapls_ia_setup_callbacks(ia_ptr, evd_ptr);
                        if (dat_status != DAT_SUCCESS) {
                                /* Assign the EVD so it gets cleaned up */
                                ia_ptr->cleanup_async_error_evd = DAT_TRUE;
                                ia_ptr->async_error_evd = evd_ptr;
                                goto bail;
                        }
                }

                ia_ptr->cleanup_async_error_evd = DAT_TRUE;
                ia_ptr->async_error_evd = evd_ptr;
        }

        dat_status = DAT_SUCCESS;
        *ia_handle_ptr = ia_ptr;
        *async_evd_handle_ptr = evd_ptr;

bail:
        if (dat_status != DAT_SUCCESS) {
                if (ia_ptr) {
                        /* This will release the async EVD if needed.  */
                        (void) dapl_ia_close(ia_ptr, DAT_CLOSE_ABRUPT_FLAG);
                }
        }

        dapl_dbg_log(DAPL_DBG_TYPE_RTN,
            "dapl_ia_open () returns 0x%x\n",
            dat_status);

        return (dat_status);
}

/*
 * dapli_hca_cleanup
 *
 * Clean up partially allocated HCA stuff. Strictly to make cleanup
 * simple.
 */
void
dapli_hca_cleanup(
        DAPL_HCA        *hca_ptr,
        DAT_BOOLEAN     dec_ref)
{
        (void) dapls_ib_close_hca(hca_ptr->ib_hca_handle);
        hca_ptr->ib_hca_handle = IB_INVALID_HANDLE;
        if (dec_ref == DAT_TRUE) {
                dapl_os_atomic_dec(&hca_ptr->handle_ref_count);
        }
}

#if defined(IBHOSTS_NAMING)

char *dapli_get_adapter_num(
        char                    *device_name);

void dapli_setup_dummy_addr(
        IN  DAPL_HCA            *hca_ptr,
        IN  char                *hca_name);
/*
 * dapli_assign_hca_ip_address
 *
 * Obtain the IP address of the passed in name, which represents a
 * port on the hca. There are three methods here to obtain the
 * appropriate IP address, each with their own shortcoming:
 * 1) IPOIB_NAMING. Requires the implementation of the IPoIB
 *    interface defined in include/dapl/ipoib_names.h. This is
 *    not the recommended interface as IPoIB is limited at
 *    the point we need to obtain an IP address on the
 *    passive side of a connection. The code supporting this
 *    implementation has been removed.
 *
 * 2) IBHOSTS. An entry exists in DNS and in the /etc/dapl/ibhosts
 *    file. The immediate drawback here is that we must dictate
 *    how to name the interface, which is a stated DAPL non-goal.
 *    In the broader perspective, this method requires us to xmit
 *    the IP address in the private data of a connection, which has
 *    other fun problems. This is the default method and is known to
 *    work, but it has problems.
 *
 * 3) Obtain the IP address from the driver, which has registered
 *    the address with the SA for retrieval.
 *
 *
 * Input:
 *      hca_ptr                 Pointer to HCA structure
 *      device_name             Name of device as reported by the provider
 *
 * Output:
 *      none
 *
 * Returns:
 *      char * to string number
 */
void
dapli_assign_hca_ip_address(
        DAPL_HCA                *hca_ptr,
        char                    *device_name)
{
        char            *adapter_num;
#define NAMELEN 128
        struct addrinfo *addr;
        char            hostname[NAMELEN];
        char            *str;
        int             rc;

        /*
         * Obtain the IP address of the adapter. This is a simple
         * scheme that creates a name that must appear available to
         * DNS, e.g. it must be in the local site DNS or in the local
         * /etc/hosts file, etc.
         *
         *      <hostname>-ib<index>
         *
         * This scheme obviously doesn't work with adapters from
         * multiple vendors, but will suffice in common installations.
         */

        rc = gethostname(hostname, NAMELEN);
        /*
         * Strip off domain info if it exists (e.g. mynode.mydomain.com)
         */
        for (str = hostname; *str && *str != '.'; ) {
                str++;
        }
        if (*str == '.') {
                *str = '\0';
        }
        dapl_os_strcat(hostname, "-ib");
        adapter_num = dapli_get_adapter_num(device_name);
        dapl_os_strcat(hostname, adapter_num);

        rc = dapls_osd_getaddrinfo(hostname, &addr);

        if (rc != 0) {
                /* Not registered in DNS, provide a dummy value */
                dapli_setup_dummy_addr(hca_ptr, hostname);
        } else {
                /*
                 * hca_address is defined as a DAT_SOCK_ADDR6 whereas ai_addr
                 * is a sockaddr
                 */
                (void) dapl_os_memcpy((void *)&hca_ptr->hca_address,
                    (void *)(addr->ai_addr), sizeof (DAT_SOCK_ADDR6));
        }
}


/*
 * dapli_stup_dummy_addr
 *
 * Set up a dummy local address for the HCA. Things are not going
 * to work too well if this happens.
 * We call this routine if:
 *  - remote host adapter name is not in DNS
 *  - IPoIB implementation is not correctly set up
 *  - Similar nonsense.
 *
 * Input:
 *      hca_ptr
 *      rhost_name              Name of remote adapter
 *
 * Output:
 *      none
 *
 * Returns:
 *      none
 */
void
dapli_setup_dummy_addr(
        IN  DAPL_HCA            *hca_ptr,
        IN  char                *rhost_name)
{
        struct sockaddr_in      *si;

        /* Not registered in DNS, provide a dummy value */
        dapl_dbg_log(DAPL_DBG_TYPE_ERR, "WARNING: <%s> not registered in DNS,"
            " using dummy IP value\n", rhost_name);
        si = (struct sockaddr_in *)&hca_ptr->hca_address;
        si->sin_family = AF_INET;
        si->sin_addr.s_addr = 0x01020304;
}


/*
 * dapls_get_adapter_num
 *
 * Given a device name, return a string of the device number
 *
 * Input:
 *      device_name             Name of device as reported by the provider
 *
 * Output:
 *      none
 *
 * Returns:
 *      char * to string number
 */
char *
dapli_get_adapter_num(
        char            *device_name)
{
        static char     *zero = "0";
        char            *p;

        /*
         * Optimisticaly simple algorithm: the device number appears at
         * the end of the device name string. Device that do not end
         * in a number are by default "0".
         */

        for (p = device_name; *p; p++) {
                if (isdigit(*p)) {
                        return (p);
                }
        }

        return (zero);
}
#endif /* IBHOSTS_NAMING */


/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 *  tab-width: 8
 * End:
 */