root/usr/src/lib/rpcsec_gss/rpcsec_gss_misc.c
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1986-1995, 1997, 2001 by Sun Microsystems, Inc.
 * All rights reserved.
 */

/*
 * Copyright 1993 OpenVision Technologies, Inc., All Rights Reserved.
 *
 * $Header:
 * /afs/gza.com/product/secure/rel-eng/src/1.1/rpc/RCS/auth_gssapi_misc.c,v
 * 1.10 1994/10/27 12:39:23 jik Exp $
 */

#include <stdlib.h>
#include <gssapi/gssapi.h>
#include <rpc/rpc.h>
#include <rpc/rpcsec_defs.h>

/*
 * Miscellaneous XDR routines.
 */
bool_t
__xdr_gss_buf(xdrs, buf)
        XDR             *xdrs;
        gss_buffer_t    buf;
{
        u_int cast_len, bound_len;

        /*
         * We go through this contortion because size_t is a now a ulong,
         * GSS-API uses ulongs.
         */

        if (xdrs->x_op != XDR_DECODE) {
                bound_len = cast_len = (u_int) buf->length;
        } else {
                bound_len = (u_int)-1;
        }

        if (xdr_bytes(xdrs, (char **)&buf->value, &cast_len,
            bound_len) == TRUE) {
                if (xdrs->x_op == XDR_DECODE)
                        buf->length = cast_len;

                return (TRUE);
        }

        return (FALSE);
}

bool_t
__xdr_rpc_gss_creds(xdrs, creds)
        XDR                     *xdrs;
        rpc_gss_creds           *creds;
{
        if (!xdr_u_int(xdrs, &creds->version) ||
                                !xdr_u_int(xdrs, &creds->gss_proc) ||
                                !xdr_u_int(xdrs, &creds->seq_num) ||
                                !xdr_u_int(xdrs, (u_int *)&creds->service) ||
                                !__xdr_gss_buf(xdrs, &creds->ctx_handle))
                return (FALSE);
        return (TRUE);
}

bool_t
__xdr_rpc_gss_init_arg(xdrs, init_arg)
        XDR                     *xdrs;
        rpc_gss_init_arg        *init_arg;
{
        if (!__xdr_gss_buf(xdrs, init_arg))
                return (FALSE);
        return (TRUE);
}

bool_t
__xdr_rpc_gss_init_res(xdrs, init_res)
        XDR                     *xdrs;
        rpc_gss_init_res        *init_res;
{
        if (!__xdr_gss_buf(xdrs, &init_res->ctx_handle) ||
                        !xdr_u_int(xdrs, (u_int *)&init_res->gss_major) ||
                        !xdr_u_int(xdrs, (u_int *)&init_res->gss_minor) ||
                        !xdr_u_int(xdrs, (u_int *)&init_res->seq_window) ||
                        !__xdr_gss_buf(xdrs, &init_res->token))
                return (FALSE);
        return (TRUE);
}

/*
 * Generic routine to wrap data used by client and server sides.
 */
bool_t
__rpc_gss_wrap_data(service, qop, context, seq_num, out_xdrs, xdr_func,
                                                        xdr_ptr)
        OM_uint32               qop;
        rpc_gss_service_t       service;
        gss_ctx_id_t            context;
        u_int                   seq_num;
        XDR                     *out_xdrs;
        bool_t                  (*xdr_func)();
        caddr_t                 xdr_ptr;
{
        OM_uint32               minor;
        gss_buffer_desc         in_buf, out_buf;
        XDR                     temp_xdrs;
        bool_t                  conf_state;
        bool_t                  ret = FALSE;
        u_int                   bufsiz;
        char                    *buf;

        /*
         * Create a temporary XDR/buffer to hold the data to be wrapped.
         */
        out_buf.length = 0;
        bufsiz = xdr_sizeof(xdr_func, xdr_ptr) +
                xdr_sizeof(xdr_u_int, &seq_num);
        if ((buf = (char *)malloc(bufsiz)) == NULL) {
                fprintf(stderr, dgettext(TEXT_DOMAIN, "malloc failed in "
                        "__rpc_gss_wrap_data\n"));
                return (FALSE);
        }
        xdrmem_create(&temp_xdrs, buf, bufsiz, XDR_ENCODE);

        /*
         * serialize the sequence number into tmp memory
         */
        if (!xdr_u_int(&temp_xdrs, &seq_num))
                goto fail;

        /*
         * serialize the arguments into tmp memory
         */
        if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
                goto fail;

        /*
         * Data to be wrapped goes in in_buf.  If privacy is used,
         * out_buf will have wrapped data (in_buf will no longer be
         * needed).  If integrity is used, out_buf will have checksum
         * which will follow the data in in_buf.
         */
        in_buf.length = xdr_getpos(&temp_xdrs);
        in_buf.value = temp_xdrs.x_base;

        switch (service) {
        case rpc_gss_svc_privacy:
                if (gss_seal(&minor, context, TRUE, qop, &in_buf,
                                &conf_state, &out_buf) != GSS_S_COMPLETE)
                        goto fail;
                in_buf.length = 0;      /* in_buf not needed */
                if (!conf_state)
                        goto fail;
                break;
        case rpc_gss_svc_integrity:
                if (gss_sign(&minor, context, qop, &in_buf,
                                                &out_buf) != GSS_S_COMPLETE)
                        goto fail;
                break;
        default:
                goto fail;
        }

        /*
         * write out in_buf and out_buf as needed
         */
        if (in_buf.length != 0) {
                if (!__xdr_gss_buf(out_xdrs, &in_buf))
                        goto fail;
        }

        if (!__xdr_gss_buf(out_xdrs, &out_buf))
                goto fail;
        ret = TRUE;
fail:
        XDR_DESTROY(&temp_xdrs);
        if (buf)
                (void) free(buf);
        if (out_buf.length != 0)
                (void) gss_release_buffer(&minor, &out_buf);
        return (ret);
}

/*
 * Generic routine to unwrap data used by client and server sides.
 */
bool_t
__rpc_gss_unwrap_data(service, context, seq_num, qop_check, in_xdrs, xdr_func,
                                                                xdr_ptr)
        rpc_gss_service_t       service;
        gss_ctx_id_t            context;
        u_int                   seq_num;
        OM_uint32               qop_check;
        XDR                     *in_xdrs;
        bool_t                  (*xdr_func)();
        caddr_t                 xdr_ptr;
{
        gss_buffer_desc         in_buf, out_buf;
        XDR                     temp_xdrs;
        u_int                   seq_num2;
        bool_t                  conf;
        OM_uint32               major = GSS_S_COMPLETE, minor = 0;
        int                     qop;

        in_buf.value = NULL;
        out_buf.value = NULL;

        /*
         * Pull out wrapped data.  For privacy service, this is the
         * encrypted data.  For integrity service, this is the data
         * followed by a checksum.
         */
        if (!__xdr_gss_buf(in_xdrs, &in_buf))
                return (FALSE);

        if (service == rpc_gss_svc_privacy) {
                major = gss_unseal(&minor, context, &in_buf, &out_buf, &conf,
                                                        &qop);
                free(in_buf.value);
                if (major != GSS_S_COMPLETE)
                        return (FALSE);
                /*
                 * Keep the returned token (unencrypted data) in in_buf.
                 */
                in_buf.length = out_buf.length;
                in_buf.value = out_buf.value;

                /*
                 * If privacy was not used, or if QOP is not what we are
                 * expecting, fail.
                 */
                if (!conf || qop != qop_check)
                        goto fail;

        } else if (service == rpc_gss_svc_integrity) {
                if (!__xdr_gss_buf(in_xdrs, &out_buf))
                        return (FALSE);
                major = gss_verify(&minor, context, &in_buf, &out_buf, &qop);
                free(out_buf.value);
                if (major != GSS_S_COMPLETE) {
                        free(in_buf.value);
                        return (FALSE);
                }

                /*
                 * If QOP is not what we are expecting, fail.
                 */
                if (qop != qop_check)
                        goto fail;
        }

        xdrmem_create(&temp_xdrs, in_buf.value, in_buf.length, XDR_DECODE);

        /*
         * The data consists of the sequence number followed by the
         * arguments.  Make sure sequence number is what we are
         * expecting (i.e., the value in the header).
         */
        if (!xdr_u_int(&temp_xdrs, &seq_num2))
                goto fail;
        if (seq_num2 != seq_num)
                goto fail;

        /*
         * Deserialize the arguments into xdr_ptr, and release in_buf.
         */
        if (!(*xdr_func)(&temp_xdrs, xdr_ptr))
                goto fail;

        if (service == rpc_gss_svc_privacy)
                (void) gss_release_buffer(&minor, &in_buf);
        else
                free(in_buf.value);
        XDR_DESTROY(&temp_xdrs);
        return (TRUE);
fail:
        XDR_DESTROY(&temp_xdrs);
        if (service == rpc_gss_svc_privacy)
                (void) gss_release_buffer(&minor, &in_buf);
        else
                free(in_buf.value);
        return (FALSE);
}

/*ARGSUSED*/
int
__find_max_data_length(service, context, qop, max_tp_unit_len)
        rpc_gss_service_t service;
        gss_ctx_id_t    context;
        OM_uint32       qop;
        int             max_tp_unit_len;
{
        int             conf;
        OM_uint32       maj_stat = GSS_S_COMPLETE, min_stat = 0;
        OM_uint32       max_input_size;
        int             ret_val = 0;

        if (service == rpc_gss_svc_integrity || service == rpc_gss_svc_default)
                conf = 0;
        else if (service == rpc_gss_svc_privacy)
                conf = 1;
        else if (service == rpc_gss_svc_none)
                return (max_tp_unit_len);

        maj_stat = gss_wrap_size_limit(&min_stat,
                context, conf, qop,
                max_tp_unit_len, &max_input_size);

        /*
         * max_input_size may result in negative value
         */
        if (maj_stat == GSS_S_COMPLETE) {
                if ((int)max_input_size <= 0)
                        ret_val = 0;
                else
                        ret_val = (int)(max_input_size);
        } else {
                fprintf(stderr, dgettext(TEXT_DOMAIN,
                                        "gss_wrap_size_limit failed in "
                                        "__find_max_data_length\n"));
        }

        return (ret_val);
}