xref: /illumos-gate/usr/src/cmd/boot/bootadm/bootadm_digest.c (revision 2f7f7a62d7a3e8a2e75eb88b95bc65871b6b90cb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 /*
26  * Copyright 2016 Toomas Soome <tsoome@me.com>
27  */
28 
29 /*
30  * Create sha1 hash for file.
31  */
32 
33 #include <stdio.h>
34 #include <errno.h>
35 #include <string.h>
36 #include <sys/types.h>
37 #include <sys/stat.h>
38 #include <fcntl.h>
39 #include <security/cryptoki.h>
40 #include <cryptoutil.h>
41 #include <locale.h>
42 #include "bootadm.h"
43 
44 #define	BUFFERSIZE	(1024 * 64)
45 #define	RESULTLEN	(512)
46 static CK_BYTE buf[BUFFERSIZE];
47 
48 /*
49  * do_digest - Compute digest of a file. Borrowed from digest.
50  *
51  *  hSession - session
52  *  pmech - ptr to mechanism to be used for digest
53  *  fd  - file descriptor
54  *  pdigest - buffer  where digest result is returned
55  *  pdigestlen - length of digest buffer on input,
56  *               length of result on output
57  */
58 static CK_RV
59 do_digest(CK_SESSION_HANDLE hSession, CK_MECHANISM_PTR pmech,
60     int fd, CK_BYTE_PTR *pdigest, CK_ULONG_PTR pdigestlen)
61 {
62 	CK_RV rv;
63 	ssize_t nread;
64 	int err;
65 
66 	if ((rv = C_DigestInit(hSession, pmech)) != CKR_OK) {
67 		return (rv);
68 	}
69 
70 	while ((nread = read(fd, buf, sizeof (buf))) > 0) {
71 		/* Get the digest */
72 		rv = C_DigestUpdate(hSession, buf, (CK_ULONG)nread);
73 		if (rv != CKR_OK)
74 			return (rv);
75 	}
76 
77 	/* There was a read error */
78 	if (nread == -1) {
79 		err = errno;
80 		bam_print(gettext("error reading file: %s\n"), strerror(err));
81 		return (CKR_GENERAL_ERROR);
82 	}
83 
84 	rv = C_DigestFinal(hSession, *pdigest, pdigestlen);
85 
86 	/* result too big to fit? Allocate a bigger buffer */
87 	if (rv == CKR_BUFFER_TOO_SMALL) {
88 		*pdigest = realloc(*pdigest, *pdigestlen);
89 
90 		if (*pdigest == NULL) {
91 			err = errno;
92 			bam_print(gettext("realloc: %s\n"), strerror(err));
93 			return (CKR_HOST_MEMORY);
94 		}
95 
96 		rv = C_DigestFinal(hSession, *pdigest, pdigestlen);
97 	}
98 
99 	return (rv);
100 }
101 
102 int
103 bootadm_digest(const char *filename, char **result)
104 {
105 	int fd;
106 	CK_RV rv;
107 	CK_ULONG slotcount;
108 	CK_SLOT_ID slotID;
109 	CK_SLOT_ID_PTR pSlotList = NULL;
110 	CK_MECHANISM_TYPE mech_type = CKM_SHA_1;
111 	CK_MECHANISM_INFO info;
112 	CK_MECHANISM mech;
113 	CK_SESSION_HANDLE hSession = CK_INVALID_HANDLE;
114 	CK_BYTE_PTR resultbuf = NULL;
115 	CK_ULONG resultlen;
116 	char *resultstr = NULL;
117 	int resultstrlen;
118 	int i, exitcode;
119 
120 	/* Initialize, and get list of slots */
121 	rv = C_Initialize(NULL);
122 	if (rv != CKR_OK && rv != CKR_CRYPTOKI_ALREADY_INITIALIZED) {
123 		bam_print(gettext(
124 		    "failed to initialize PKCS #11 framework: %s\n"),
125 		    pkcs11_strerror(rv));
126 		return (BAM_ERROR);
127 	}
128 
129 	/* Get slot count */
130 	rv = C_GetSlotList(0, NULL, &slotcount);
131 	if (rv != CKR_OK || slotcount == 0) {
132 		bam_print(gettext(
133 		    "failed to find any cryptographic provider: %s\n"),
134 		    pkcs11_strerror(rv));
135 		exitcode = BAM_ERROR;
136 		goto cleanup;
137 	}
138 
139 	/* Found at least one slot, allocate memory for slot list */
140 	pSlotList = malloc(slotcount * sizeof (CK_SLOT_ID));
141 	if (pSlotList == NULL) {
142 		bam_print(gettext("out of memory\n"));
143 		exitcode = BAM_ERROR;
144 		goto cleanup;
145 	}
146 
147 	/* Get the list of slots */
148 	if ((rv = C_GetSlotList(0, pSlotList, &slotcount)) != CKR_OK) {
149 		bam_print(gettext(
150 		    "failed to find any cryptographic provider; "
151 		    "please check with your system administrator: %s\n"),
152 		    pkcs11_strerror(rv));
153 		exitcode = BAM_ERROR;
154 		goto cleanup;
155 	}
156 
157 	/* Find a slot with matching mechanism */
158 	for (i = 0; i < slotcount; i++) {
159 		slotID = pSlotList[i];
160 		rv = C_GetMechanismInfo(slotID, mech_type, &info);
161 		if (rv != CKR_OK) {
162 			continue; /* to the next slot */
163 		} else {
164 			if (info.flags & CKF_DIGEST)
165 				break;
166 		}
167 	}
168 
169 	/* Show error if no matching mechanism found */
170 	if (i == slotcount) {
171 		bam_print(gettext("no cryptographic provider was "
172 		    "found for sha1\n"));
173 		exitcode = BAM_ERROR;
174 		goto cleanup;
175 	}
176 
177 	/* Mechanism is supported. Go ahead & open a session */
178 	rv = C_OpenSession(slotID, CKF_SERIAL_SESSION,
179 	    NULL, NULL, &hSession);
180 
181 	if (rv != CKR_OK) {
182 		bam_print(gettext("can not open PKCS#11 session: %s\n"),
183 		    pkcs11_strerror(rv));
184 		exitcode = BAM_ERROR;
185 		goto cleanup;
186 	}
187 
188 	/* Allocate a buffer to store result. */
189 	resultlen = RESULTLEN;
190 	if ((resultbuf = malloc(resultlen)) == NULL) {
191 		bam_print(gettext("out of memory\n"));
192 		exitcode = BAM_ERROR;
193 		goto cleanup;
194 	}
195 
196 	mech.mechanism = mech_type;
197 	mech.pParameter = NULL;
198 	mech.ulParameterLen = 0;
199 	exitcode = BAM_SUCCESS;
200 
201 	if ((fd = open(filename, O_RDONLY | O_NONBLOCK)) == -1) {
202 		bam_print(gettext("can not open input file %s\n"), filename);
203 		exitcode = BAM_ERROR;
204 		goto cleanup;
205 	}
206 
207 	rv = do_digest(hSession, &mech, fd, &resultbuf, &resultlen);
208 
209 	if (rv != CKR_OK) {
210 		bam_print(gettext("crypto operation failed for "
211 		    "file %s: %s\n"), filename, pkcs11_strerror(rv));
212 		exitcode = BAM_ERROR;
213 		goto cleanup;
214 	}
215 
216 	/* Allocate a buffer to store result string */
217 	resultstrlen = 2 * resultlen + 1;
218 	if ((resultstr = malloc(resultstrlen)) == NULL) {
219 		bam_print(gettext("out of memory\n"));
220 		exitcode = BAM_ERROR;
221 		goto cleanup;
222 	}
223 
224 	tohexstr(resultbuf, resultlen, resultstr, resultstrlen);
225 
226 	(void) close(fd);
227 cleanup:
228 	if (exitcode == BAM_ERROR) {
229 		free(resultstr);
230 		resultstr = NULL;
231 	}
232 
233 	free(resultbuf);
234 	free(pSlotList);
235 
236 	if (hSession != CK_INVALID_HANDLE)
237 		(void) C_CloseSession(hSession);
238 
239 	(void) C_Finalize(NULL);
240 
241 	*result = resultstr;
242 	return (exitcode);
243 }
244