xref: /illumos-gate/usr/src/lib/gss_mechs/mech_krb5/mech/util_cksum.c (revision 581cede61ac9c14d8d4ea452562a567189eead78)
1 /*
2  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 
6 #pragma ident	"%Z%%M%	%I%	%E% SMI"
7 
8 /*
9  * Copyright 1993 by OpenVision Technologies, Inc.
10  *
11  * Permission to use, copy, modify, distribute, and sell this software
12  * and its documentation for any purpose is hereby granted without fee,
13  * provided that the above copyright notice appears in all copies and
14  * that both that copyright notice and this permission notice appear in
15  * supporting documentation, and that the name of OpenVision not be used
16  * in advertising or publicity pertaining to distribution of the software
17  * without specific, written prior permission. OpenVision makes no
18  * representations about the suitability of this software for any
19  * purpose.  It is provided "as is" without express or implied warranty.
20  *
21  * OPENVISION DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
22  * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
23  * EVENT SHALL OPENVISION BE LIABLE FOR ANY SPECIAL, INDIRECT OR
24  * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF
25  * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
26  * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
27  * PERFORMANCE OF THIS SOFTWARE.
28  */
29 
30 #include "gssapiP_krb5.h"
31 #ifdef HAVE_MEMORY_H
32 #include <memory.h>
33 #endif
34 
35 /* Checksumming the channel bindings always uses plain MD5.  */
36 krb5_error_code
37 kg_checksum_channel_bindings(context, cb, cksum, bigend)
38      krb5_context context;
39      gss_channel_bindings_t cb;
40      krb5_checksum *cksum;
41      int bigend;
42 {
43    size_t len;
44    char *buf = 0;
45    char *ptr;
46    size_t sumlen;
47    krb5_data plaind;
48    krb5_error_code code;
49    void *temp;
50 
51    /* initialize the the cksum */
52    code = krb5_c_checksum_length(context, CKSUMTYPE_RSA_MD5, &sumlen);
53    if (code)
54        return(code);
55 
56    cksum->checksum_type = CKSUMTYPE_RSA_MD5;
57    cksum->length = sumlen;
58 
59    /* generate a buffer full of zeros if no cb specified */
60 
61    if (cb == GSS_C_NO_CHANNEL_BINDINGS) {
62        if ((cksum->contents = (krb5_octet *) xmalloc(cksum->length)) == NULL) {
63 	   return(ENOMEM);
64        }
65        memset(cksum->contents, '\0', cksum->length);
66        return(0);
67    }
68 
69    /* create the buffer to checksum into */
70 
71    len = (sizeof(krb5_int32)*5+
72 	  cb->initiator_address.length+
73 	  cb->acceptor_address.length+
74 	  cb->application_data.length);
75 
76    if ((buf = (char *) xmalloc(len)) == NULL)
77       return(ENOMEM);
78 
79    /* helper macros.  This code currently depends on a long being 32
80       bits, and htonl dtrt. */
81 
82    ptr = buf;
83 
84    TWRITE_INT(ptr, cb->initiator_addrtype, bigend);
85    TWRITE_BUF(ptr, cb->initiator_address, bigend);
86    TWRITE_INT(ptr, cb->acceptor_addrtype, bigend);
87    TWRITE_BUF(ptr, cb->acceptor_address, bigend);
88    TWRITE_BUF(ptr, cb->application_data, bigend);
89 
90    /* checksum the data */
91 
92    plaind.length = len;
93    plaind.data = buf;
94 
95 #if 0
96    /*
97     * SUNW15resync
98     * MIT 1.5-6 seems/is wrong here in 2 ways
99     *   - why free then alloc contents again?
100     *   - calling krb5_free_checksum_contents results in cksum->length
101     *     getting set to 0 which causes ftp to fail
102     * so lets stick w/oldey-but-goodey code.
103     */
104    code = krb5_c_make_checksum(context, CKSUMTYPE_RSA_MD5, 0, 0,
105 			       &plaind, cksum);
106    if (code)
107        goto cleanup;
108 
109    if ((temp = xmalloc(cksum->length)) == NULL) {
110        krb5_free_checksum_contents(context, cksum);
111        code = ENOMEM;
112        goto cleanup;
113    }
114 
115    memcpy(temp, cksum->contents, cksum->length);
116    krb5_free_checksum_contents(context, cksum);
117    cksum->contents = (krb5_octet *)temp;
118    /* SUNW15resync - need to reset cksum->length here */
119 
120    /* success */
121  cleanup:
122    if (buf)
123        xfree(buf);
124 #endif /* 0 */
125 
126    if (code = krb5_c_make_checksum(context, CKSUMTYPE_RSA_MD5, 0, 0,
127                                    &plaind, cksum)) {
128       xfree(cksum->contents); /* SUNW15resync -just in case not already free */
129       xfree(buf);
130       return(code);
131    }
132 
133    /* success */
134 
135    xfree(buf);
136    return code;
137 }
138