xref: /illumos-gate/usr/src/lib/pkcs11/pkcs11_softtoken/common/softEncryptUtil.c (revision 581cede61ac9c14d8d4ea452562a567189eead78)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <pthread.h>
27 #include <stdlib.h>
28 #include <string.h>
29 #include <strings.h>
30 #include <sys/types.h>
31 #include <security/cryptoki.h>
32 #include <modes/modes.h>
33 #include <arcfour.h>
34 #include "softSession.h"
35 #include "softObject.h"
36 #include "softOps.h"
37 #include "softCrypt.h"
38 #include "softRSA.h"
39 
40 /*
41  * Add padding bytes with the value of length of padding.
42  */
43 void
44 soft_add_pkcs7_padding(CK_BYTE *buf, int block_size, CK_ULONG data_len)
45 {
46 
47 	ulong_t i, pad_len;
48 	CK_BYTE pad_value;
49 
50 	pad_len = block_size - (data_len % block_size);
51 	pad_value = (CK_BYTE)pad_len;
52 
53 	for (i = 0; i < pad_len; i++)
54 		buf[i] = pad_value;
55 }
56 
57 /*
58  * Perform encrypt init operation internally for the support of
59  * CKM_DES_MAC and CKM_DES_MAC_GENERAL
60  *
61  * This function is called with the session being held, and without
62  * its mutex taken.
63  */
64 CK_RV
65 soft_encrypt_init_internal(soft_session_t *session_p, CK_MECHANISM_PTR
66 	pMechanism, soft_object_t *key_p)
67 {
68 	CK_RV rv;
69 
70 	(void) pthread_mutex_lock(&session_p->session_mutex);
71 
72 	/* Check to see if encrypt operation is already active */
73 	if (session_p->encrypt.flags & CRYPTO_OPERATION_ACTIVE) {
74 		(void) pthread_mutex_unlock(&session_p->session_mutex);
75 		return (CKR_OPERATION_ACTIVE);
76 	}
77 
78 	session_p->encrypt.flags = CRYPTO_OPERATION_ACTIVE;
79 
80 	(void) pthread_mutex_unlock(&session_p->session_mutex);
81 
82 	rv = soft_encrypt_init(session_p, pMechanism, key_p);
83 
84 	if (rv != CKR_OK) {
85 		(void) pthread_mutex_lock(&session_p->session_mutex);
86 		session_p->encrypt.flags &= ~CRYPTO_OPERATION_ACTIVE;
87 		(void) pthread_mutex_unlock(&session_p->session_mutex);
88 	}
89 
90 	return (rv);
91 }
92 
93 /*
94  * soft_encrypt_init()
95  *
96  * Arguments:
97  *	session_p:	pointer to soft_session_t struct
98  *	pMechanism:	pointer to CK_MECHANISM struct provided by application
99  *	key_p:		pointer to key soft_object_t struct
100  *
101  * Description:
102  *	called by C_EncryptInit(). This function calls the corresponding
103  *	encrypt init routine based on the mechanism.
104  *
105  * Returns:
106  *	CKR_OK: success
107  *	CKR_HOST_MEMORY: run out of system memory
108  *	CKR_MECHANISM_PARAM_INVALID: invalid parameters in mechanism
109  *	CKR_MECHANISM_INVALID: invalid mechanism type
110  *	CKR_KEY_TYPE_INCONSISTENT: incorrect type of key to use
111  *		with the specified mechanism
112  */
113 CK_RV
114 soft_encrypt_init(soft_session_t *session_p, CK_MECHANISM_PTR pMechanism,
115     soft_object_t *key_p)
116 {
117 
118 	CK_RV rv;
119 
120 	switch (pMechanism->mechanism) {
121 
122 	case CKM_DES_ECB:
123 
124 		if (key_p->key_type != CKK_DES) {
125 			return (CKR_KEY_TYPE_INCONSISTENT);
126 		}
127 		goto ecb_common;
128 
129 	case CKM_DES3_ECB:
130 
131 		if ((key_p->key_type != CKK_DES2) &&
132 		    (key_p->key_type != CKK_DES3)) {
133 			return (CKR_KEY_TYPE_INCONSISTENT);
134 		}
135 
136 ecb_common:
137 		return (soft_des_crypt_init_common(session_p, pMechanism,
138 		    key_p, B_TRUE));
139 
140 	case CKM_DES_CBC:
141 	case CKM_DES_CBC_PAD:
142 
143 		if (key_p->key_type != CKK_DES) {
144 			return (CKR_KEY_TYPE_INCONSISTENT);
145 		}
146 
147 		goto cbc_common;
148 
149 	case CKM_DES3_CBC:
150 	case CKM_DES3_CBC_PAD:
151 	{
152 
153 		soft_des_ctx_t *soft_des_ctx;
154 
155 		if ((key_p->key_type != CKK_DES2) &&
156 		    (key_p->key_type != CKK_DES3)) {
157 			return (CKR_KEY_TYPE_INCONSISTENT);
158 		}
159 
160 cbc_common:
161 		if ((pMechanism->pParameter == NULL) ||
162 		    (pMechanism->ulParameterLen != DES_BLOCK_LEN)) {
163 			return (CKR_MECHANISM_PARAM_INVALID);
164 		}
165 
166 		rv = soft_des_crypt_init_common(session_p, pMechanism,
167 		    key_p, B_TRUE);
168 
169 		if (rv != CKR_OK)
170 			return (rv);
171 
172 		(void) pthread_mutex_lock(&session_p->session_mutex);
173 
174 		soft_des_ctx = (soft_des_ctx_t *)session_p->encrypt.context;
175 		/* Copy Initialization Vector (IV) into the context. */
176 		(void) memcpy(soft_des_ctx->ivec, pMechanism->pParameter,
177 		    DES_BLOCK_LEN);
178 
179 		/* Allocate a context for DES cipher-block chaining. */
180 		soft_des_ctx->des_cbc = (void *)des_cbc_ctx_init(
181 		    soft_des_ctx->key_sched, soft_des_ctx->keysched_len,
182 		    soft_des_ctx->ivec, key_p->key_type);
183 
184 		if (soft_des_ctx->des_cbc == NULL) {
185 			bzero(soft_des_ctx->key_sched,
186 			    soft_des_ctx->keysched_len);
187 			free(soft_des_ctx->key_sched);
188 			free(session_p->encrypt.context);
189 			session_p->encrypt.context = NULL;
190 			rv = CKR_HOST_MEMORY;
191 		}
192 
193 		(void) pthread_mutex_unlock(&session_p->session_mutex);
194 
195 		return (rv);
196 	}
197 	case CKM_AES_ECB:
198 
199 		if (key_p->key_type != CKK_AES) {
200 			return (CKR_KEY_TYPE_INCONSISTENT);
201 		}
202 
203 		return (soft_aes_crypt_init_common(session_p, pMechanism,
204 		    key_p, B_TRUE));
205 
206 	case CKM_AES_CBC:
207 	case CKM_AES_CBC_PAD:
208 	{
209 		soft_aes_ctx_t *soft_aes_ctx;
210 
211 		if (key_p->key_type != CKK_AES) {
212 			return (CKR_KEY_TYPE_INCONSISTENT);
213 		}
214 
215 		if ((pMechanism->pParameter == NULL) ||
216 		    (pMechanism->ulParameterLen != AES_BLOCK_LEN)) {
217 			return (CKR_MECHANISM_PARAM_INVALID);
218 		}
219 
220 		rv = soft_aes_crypt_init_common(session_p, pMechanism,
221 		    key_p, B_TRUE);
222 
223 		if (rv != CKR_OK)
224 			return (rv);
225 
226 		(void) pthread_mutex_lock(&session_p->session_mutex);
227 
228 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
229 		/* Copy Initialization Vector (IV) into the context. */
230 		(void) memcpy(soft_aes_ctx->ivec, pMechanism->pParameter,
231 		    AES_BLOCK_LEN);
232 
233 		/* Allocate a context for AES cipher-block chaining. */
234 		soft_aes_ctx->aes_cbc = (void *)aes_cbc_ctx_init(
235 		    soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len,
236 		    soft_aes_ctx->ivec);
237 
238 		if (soft_aes_ctx->aes_cbc == NULL) {
239 			bzero(soft_aes_ctx->key_sched,
240 			    soft_aes_ctx->keysched_len);
241 			free(soft_aes_ctx->key_sched);
242 			free(session_p->encrypt.context);
243 			session_p->encrypt.context = NULL;
244 			rv = CKR_HOST_MEMORY;
245 		}
246 
247 		(void) pthread_mutex_unlock(&session_p->session_mutex);
248 
249 		return (rv);
250 	}
251 	case CKM_AES_CTR:
252 	{
253 		soft_aes_ctx_t *soft_aes_ctx;
254 
255 		if (key_p->key_type != CKK_AES) {
256 			return (CKR_KEY_TYPE_INCONSISTENT);
257 		}
258 
259 		if (pMechanism->pParameter == NULL ||
260 		    pMechanism->ulParameterLen != sizeof (CK_AES_CTR_PARAMS)) {
261 			return (CKR_MECHANISM_PARAM_INVALID);
262 		}
263 
264 		rv = soft_aes_crypt_init_common(session_p, pMechanism,
265 		    key_p, B_TRUE);
266 
267 		if (rv != CKR_OK)
268 			return (rv);
269 
270 		(void) pthread_mutex_lock(&session_p->session_mutex);
271 
272 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
273 		soft_aes_ctx->aes_cbc = aes_ctr_ctx_init(
274 		    soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len,
275 		    pMechanism->pParameter);
276 
277 		if (soft_aes_ctx->aes_cbc == NULL) {
278 			bzero(soft_aes_ctx->key_sched,
279 			    soft_aes_ctx->keysched_len);
280 			free(soft_aes_ctx->key_sched);
281 			free(session_p->encrypt.context);
282 			session_p->encrypt.context = NULL;
283 			rv = CKR_HOST_MEMORY;
284 		}
285 
286 		(void) pthread_mutex_unlock(&session_p->session_mutex);
287 
288 		return (rv);
289 	}
290 	case CKM_RC4:
291 
292 		if (key_p->key_type != CKK_RC4) {
293 			return (CKR_KEY_TYPE_INCONSISTENT);
294 		}
295 
296 		return (soft_arcfour_crypt_init(session_p, pMechanism, key_p,
297 		    B_TRUE));
298 
299 	case CKM_RSA_X_509:
300 	case CKM_RSA_PKCS:
301 
302 		if (key_p->key_type != CKK_RSA) {
303 			return (CKR_KEY_TYPE_INCONSISTENT);
304 		}
305 
306 		return (soft_rsa_crypt_init_common(session_p, pMechanism,
307 		    key_p, B_TRUE));
308 
309 	case CKM_BLOWFISH_CBC:
310 	{
311 		soft_blowfish_ctx_t *soft_blowfish_ctx;
312 
313 		if (key_p->key_type != CKK_BLOWFISH)
314 			return (CKR_KEY_TYPE_INCONSISTENT);
315 
316 		if ((pMechanism->pParameter == NULL) ||
317 		    (pMechanism->ulParameterLen != BLOWFISH_BLOCK_LEN))
318 			return (CKR_MECHANISM_PARAM_INVALID);
319 
320 		rv = soft_blowfish_crypt_init_common(session_p, pMechanism,
321 		    key_p, B_TRUE);
322 
323 		if (rv != CKR_OK)
324 			return (rv);
325 
326 		(void) pthread_mutex_lock(&session_p->session_mutex);
327 
328 		soft_blowfish_ctx =
329 		    (soft_blowfish_ctx_t *)session_p->encrypt.context;
330 		/* Copy Initialization Vector (IV) into the context. */
331 		(void) memcpy(soft_blowfish_ctx->ivec, pMechanism->pParameter,
332 		    BLOWFISH_BLOCK_LEN);
333 
334 		/* Allocate a context for Blowfish cipher-block chaining */
335 		soft_blowfish_ctx->blowfish_cbc =
336 		    (void *)blowfish_cbc_ctx_init(soft_blowfish_ctx->key_sched,
337 		    soft_blowfish_ctx->keysched_len,
338 		    soft_blowfish_ctx->ivec);
339 
340 		if (soft_blowfish_ctx->blowfish_cbc == NULL) {
341 			bzero(soft_blowfish_ctx->key_sched,
342 			    soft_blowfish_ctx->keysched_len);
343 			free(soft_blowfish_ctx->key_sched);
344 			free(session_p->encrypt.context);
345 			session_p->encrypt.context = NULL;
346 			rv = CKR_HOST_MEMORY;
347 		}
348 
349 		(void) pthread_mutex_unlock(&session_p->session_mutex);
350 
351 		return (rv);
352 	}
353 	default:
354 		return (CKR_MECHANISM_INVALID);
355 	}
356 }
357 
358 
359 /*
360  * soft_encrypt_common()
361  *
362  * Arguments:
363  *      session_p:	pointer to soft_session_t struct
364  *	pData:		pointer to the input data to be encrypted
365  *	ulDataLen:	length of the input data
366  *	pEncrypted:	pointer to the output data after encryption
367  *	pulEncryptedLen: pointer to the length of the output data
368  *	update:		boolean flag indicates caller is soft_encrypt
369  *			or soft_encrypt_update
370  *
371  * Description:
372  *      This function calls the corresponding encrypt routine based
373  *	on the mechanism.
374  *
375  * Returns:
376  *	see corresponding encrypt routine.
377  */
378 CK_RV
379 soft_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
380     CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
381     CK_ULONG_PTR pulEncryptedLen, boolean_t update)
382 {
383 
384 	CK_MECHANISM_TYPE mechanism = session_p->encrypt.mech.mechanism;
385 
386 	switch (mechanism) {
387 
388 	case CKM_DES_ECB:
389 	case CKM_DES_CBC:
390 	case CKM_DES3_ECB:
391 	case CKM_DES3_CBC:
392 
393 		if (ulDataLen == 0) {
394 			*pulEncryptedLen = 0;
395 			return (CKR_OK);
396 		}
397 		/* FALLTHROUGH */
398 
399 	case CKM_DES_CBC_PAD:
400 	case CKM_DES3_CBC_PAD:
401 
402 		return (soft_des_encrypt_common(session_p, pData,
403 		    ulDataLen, pEncrypted, pulEncryptedLen, update));
404 
405 	case CKM_AES_ECB:
406 	case CKM_AES_CBC:
407 	case CKM_AES_CTR:
408 
409 		if (ulDataLen == 0) {
410 			*pulEncryptedLen = 0;
411 			return (CKR_OK);
412 		}
413 		/* FALLTHROUGH */
414 
415 	case CKM_AES_CBC_PAD:
416 
417 		return (soft_aes_encrypt_common(session_p, pData,
418 		    ulDataLen, pEncrypted, pulEncryptedLen, update));
419 
420 	case CKM_BLOWFISH_CBC:
421 
422 		if (ulDataLen == 0) {
423 			*pulEncryptedLen = 0;
424 			return (CKR_OK);
425 		}
426 
427 		return (soft_blowfish_encrypt_common(session_p, pData,
428 		    ulDataLen, pEncrypted, pulEncryptedLen, update));
429 
430 	case CKM_RC4:
431 
432 		if (ulDataLen == 0) {
433 			*pulEncryptedLen = 0;
434 			return (CKR_OK);
435 		}
436 
437 		return (soft_arcfour_crypt(&(session_p->encrypt), pData,
438 		    ulDataLen, pEncrypted, pulEncryptedLen));
439 
440 	case CKM_RSA_X_509:
441 	case CKM_RSA_PKCS:
442 
443 		return (soft_rsa_encrypt_common(session_p, pData,
444 		    ulDataLen, pEncrypted, pulEncryptedLen, mechanism));
445 
446 	default:
447 		return (CKR_MECHANISM_INVALID);
448 	}
449 }
450 
451 
452 /*
453  * soft_encrypt()
454  *
455  * Arguments:
456  *      session_p:	pointer to soft_session_t struct
457  *	pData:		pointer to the input data to be encrypted
458  *	ulDataLen:	length of the input data
459  *	pEncryptedData:	pointer to the output data after encryption
460  *	pulEncryptedDataLen: pointer to the length of the output data
461  *
462  * Description:
463  *      called by C_Encrypt(). This function calls the soft_encrypt_common
464  *	routine.
465  *
466  * Returns:
467  *	see soft_encrypt_common().
468  */
469 CK_RV
470 soft_encrypt(soft_session_t *session_p, CK_BYTE_PTR pData,
471     CK_ULONG ulDataLen, CK_BYTE_PTR pEncryptedData,
472     CK_ULONG_PTR pulEncryptedDataLen)
473 {
474 
475 	return (soft_encrypt_common(session_p, pData, ulDataLen,
476 	    pEncryptedData, pulEncryptedDataLen, B_FALSE));
477 }
478 
479 
480 /*
481  * soft_encrypt_update()
482  *
483  * Arguments:
484  *      session_p:	pointer to soft_session_t struct
485  *      pPart:		pointer to the input data to be digested
486  *      ulPartLen:	length of the input data
487  *	pEncryptedPart:	pointer to the ciphertext
488  *	pulEncryptedPartLen: pointer to the length of the ciphertext
489  *
490  * Description:
491  *      called by C_EncryptUpdate(). This function calls the
492  *	soft_encrypt_common routine (with update flag on).
493  *
494  * Returns:
495  *	see soft_encrypt_common().
496  */
497 CK_RV
498 soft_encrypt_update(soft_session_t *session_p, CK_BYTE_PTR pPart,
499 	CK_ULONG ulPartLen, CK_BYTE_PTR pEncryptedPart,
500 	CK_ULONG_PTR pulEncryptedPartLen)
501 {
502 
503 	CK_MECHANISM_TYPE mechanism = session_p->encrypt.mech.mechanism;
504 
505 	switch (mechanism) {
506 
507 	case CKM_DES_ECB:
508 	case CKM_DES_CBC:
509 	case CKM_DES_CBC_PAD:
510 	case CKM_DES3_ECB:
511 	case CKM_DES3_CBC:
512 	case CKM_DES3_CBC_PAD:
513 	case CKM_AES_ECB:
514 	case CKM_AES_CBC:
515 	case CKM_AES_CBC_PAD:
516 	case CKM_AES_CTR:
517 	case CKM_BLOWFISH_CBC:
518 	case CKM_RC4:
519 
520 		return (soft_encrypt_common(session_p, pPart, ulPartLen,
521 		    pEncryptedPart, pulEncryptedPartLen, B_TRUE));
522 
523 	default:
524 		/* PKCS11: The mechanism only supports single-part operation. */
525 		return (CKR_MECHANISM_INVALID);
526 	}
527 }
528 
529 
530 /*
531  * soft_encrypt_final()
532  *
533  * Arguments:
534  *      session_p:		pointer to soft_session_t struct
535  *      pLastEncryptedPart:	pointer to the last encrypted data part
536  *      pulLastEncryptedPartLen: pointer to the length of the last
537  *				encrypted data part
538  *
539  * Description:
540  *      called by C_EncryptFinal().
541  *
542  * Returns:
543  *	CKR_OK: success
544  *	CKR_FUNCTION_FAILED: encrypt final function failed
545  *	CKR_DATA_LEN_RANGE: remaining buffer contains bad length
546  */
547 CK_RV
548 soft_encrypt_final(soft_session_t *session_p, CK_BYTE_PTR pLastEncryptedPart,
549 	CK_ULONG_PTR pulLastEncryptedPartLen)
550 {
551 
552 	CK_MECHANISM_TYPE mechanism = session_p->encrypt.mech.mechanism;
553 	CK_ULONG out_len;
554 	CK_RV rv = CKR_OK;
555 	int rc;
556 
557 	(void) pthread_mutex_lock(&session_p->session_mutex);
558 
559 	if (session_p->encrypt.context == NULL) {
560 		rv = CKR_OPERATION_NOT_INITIALIZED;
561 		*pulLastEncryptedPartLen = 0;
562 		goto clean1;
563 	}
564 	switch (mechanism) {
565 
566 	case CKM_DES_CBC_PAD:
567 	case CKM_DES3_CBC_PAD:
568 	{
569 		soft_des_ctx_t *soft_des_ctx;
570 
571 		soft_des_ctx = (soft_des_ctx_t *)session_p->encrypt.context;
572 		/*
573 		 * For CKM_DES_CBC_PAD, compute output length with
574 		 * padding. If the remaining buffer has one block
575 		 * of data, then output length will be two blocksize of
576 		 * ciphertext. If the remaining buffer has less than
577 		 * one block of data, then output length will be
578 		 * one blocksize.
579 		 */
580 		if (soft_des_ctx->remain_len == DES_BLOCK_LEN)
581 			out_len = 2 * DES_BLOCK_LEN;
582 		else
583 			out_len = DES_BLOCK_LEN;
584 
585 		if (pLastEncryptedPart == NULL) {
586 			/*
587 			 * Application asks for the length of the output
588 			 * buffer to hold the ciphertext.
589 			 */
590 			*pulLastEncryptedPartLen = out_len;
591 			goto clean1;
592 		} else {
593 			crypto_data_t out;
594 
595 			/* Copy remaining data to the output buffer. */
596 			(void) memcpy(pLastEncryptedPart, soft_des_ctx->data,
597 			    soft_des_ctx->remain_len);
598 
599 			/*
600 			 * Add padding bytes prior to encrypt final.
601 			 */
602 			soft_add_pkcs7_padding(pLastEncryptedPart +
603 			    soft_des_ctx->remain_len, DES_BLOCK_LEN,
604 			    soft_des_ctx->remain_len);
605 
606 			out.cd_format = CRYPTO_DATA_RAW;
607 			out.cd_offset = 0;
608 			out.cd_length = out_len;
609 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
610 			out.cd_raw.iov_len = out_len;
611 
612 			/* Encrypt multiple blocks of data. */
613 			rc = des_encrypt_contiguous_blocks(
614 			    (des_ctx_t *)soft_des_ctx->des_cbc,
615 			    (char *)pLastEncryptedPart, out_len, &out);
616 
617 			if (rc == 0) {
618 				*pulLastEncryptedPartLen = out_len;
619 			} else {
620 				*pulLastEncryptedPartLen = 0;
621 				rv = CKR_FUNCTION_FAILED;
622 			}
623 
624 			/* Cleanup memory space. */
625 			free(soft_des_ctx->des_cbc);
626 			bzero(soft_des_ctx->key_sched,
627 			    soft_des_ctx->keysched_len);
628 			free(soft_des_ctx->key_sched);
629 		}
630 
631 		break;
632 	}
633 	case CKM_DES_CBC:
634 	case CKM_DES_ECB:
635 	case CKM_DES3_CBC:
636 	case CKM_DES3_ECB:
637 	{
638 
639 		soft_des_ctx_t *soft_des_ctx;
640 
641 		soft_des_ctx = (soft_des_ctx_t *)session_p->encrypt.context;
642 		/*
643 		 * CKM_DES_CBC and CKM_DES_ECB does not do any padding,
644 		 * so when the final is called, the remaining buffer
645 		 * should not contain any more data.
646 		 */
647 		*pulLastEncryptedPartLen = 0;
648 		if (soft_des_ctx->remain_len != 0) {
649 			rv = CKR_DATA_LEN_RANGE;
650 		} else {
651 			if (pLastEncryptedPart == NULL)
652 				goto clean1;
653 		}
654 
655 		/* Cleanup memory space. */
656 		free(soft_des_ctx->des_cbc);
657 		bzero(soft_des_ctx->key_sched, soft_des_ctx->keysched_len);
658 		free(soft_des_ctx->key_sched);
659 
660 		break;
661 	}
662 	case CKM_AES_CBC_PAD:
663 	{
664 		soft_aes_ctx_t *soft_aes_ctx;
665 
666 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
667 		/*
668 		 * For CKM_AES_CBC_PAD, compute output length with
669 		 * padding. If the remaining buffer has one block
670 		 * of data, then output length will be two blocksize of
671 		 * ciphertext. If the remaining buffer has less than
672 		 * one block of data, then output length will be
673 		 * one blocksize.
674 		 */
675 		if (soft_aes_ctx->remain_len == AES_BLOCK_LEN)
676 			out_len = 2 * AES_BLOCK_LEN;
677 		else
678 			out_len = AES_BLOCK_LEN;
679 
680 		if (pLastEncryptedPart == NULL) {
681 			/*
682 			 * Application asks for the length of the output
683 			 * buffer to hold the ciphertext.
684 			 */
685 			*pulLastEncryptedPartLen = out_len;
686 			goto clean1;
687 		} else {
688 			crypto_data_t out;
689 
690 			/* Copy remaining data to the output buffer. */
691 			(void) memcpy(pLastEncryptedPart, soft_aes_ctx->data,
692 			    soft_aes_ctx->remain_len);
693 
694 			/*
695 			 * Add padding bytes prior to encrypt final.
696 			 */
697 			soft_add_pkcs7_padding(pLastEncryptedPart +
698 			    soft_aes_ctx->remain_len, AES_BLOCK_LEN,
699 			    soft_aes_ctx->remain_len);
700 
701 			out.cd_format = CRYPTO_DATA_RAW;
702 			out.cd_offset = 0;
703 			out.cd_length = out_len;
704 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
705 			out.cd_raw.iov_len = out_len;
706 
707 			/* Encrypt multiple blocks of data. */
708 			rc = aes_encrypt_contiguous_blocks(
709 			    (aes_ctx_t *)soft_aes_ctx->aes_cbc,
710 			    (char *)pLastEncryptedPart, out_len, &out);
711 
712 			if (rc == 0) {
713 				*pulLastEncryptedPartLen = out_len;
714 			} else {
715 				*pulLastEncryptedPartLen = 0;
716 				rv = CKR_FUNCTION_FAILED;
717 			}
718 
719 			/* Cleanup memory space. */
720 			free(soft_aes_ctx->aes_cbc);
721 			bzero(soft_aes_ctx->key_sched,
722 			    soft_aes_ctx->keysched_len);
723 			free(soft_aes_ctx->key_sched);
724 		}
725 
726 		break;
727 	}
728 	case CKM_AES_CBC:
729 	case CKM_AES_ECB:
730 	{
731 		soft_aes_ctx_t *soft_aes_ctx;
732 
733 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
734 		/*
735 		 * CKM_AES_CBC and CKM_AES_ECB does not do any padding,
736 		 * so when the final is called, the remaining buffer
737 		 * should not contain any more data.
738 		 */
739 		*pulLastEncryptedPartLen = 0;
740 		if (soft_aes_ctx->remain_len != 0) {
741 			rv = CKR_DATA_LEN_RANGE;
742 		} else {
743 			if (pLastEncryptedPart == NULL)
744 				goto clean1;
745 		}
746 
747 		/* Cleanup memory space. */
748 		free(soft_aes_ctx->aes_cbc);
749 		bzero(soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len);
750 		free(soft_aes_ctx->key_sched);
751 
752 		break;
753 	}
754 	case CKM_AES_CTR:
755 	{
756 		crypto_data_t out;
757 		soft_aes_ctx_t *soft_aes_ctx;
758 		ctr_ctx_t *ctr_ctx;
759 		size_t len;
760 
761 		soft_aes_ctx = (soft_aes_ctx_t *)session_p->encrypt.context;
762 		ctr_ctx = soft_aes_ctx->aes_cbc;
763 		len = ctr_ctx->ctr_remainder_len;
764 
765 		if (pLastEncryptedPart == NULL) {
766 			*pulLastEncryptedPartLen = len;
767 			goto clean1;
768 		}
769 		if (len > 0) {
770 			out.cd_format = CRYPTO_DATA_RAW;
771 			out.cd_offset = 0;
772 			out.cd_length = len;
773 			out.cd_raw.iov_base = (char *)pLastEncryptedPart;
774 			out.cd_raw.iov_len = len;
775 
776 			rv = ctr_mode_final(ctr_ctx, &out, aes_encrypt_block);
777 		}
778 		if (rv == CRYPTO_BUFFER_TOO_SMALL) {
779 			*pulLastEncryptedPartLen = len;
780 			goto clean1;
781 		}
782 
783 		/* Cleanup memory space. */
784 		free(ctr_ctx);
785 		bzero(soft_aes_ctx->key_sched, soft_aes_ctx->keysched_len);
786 		free(soft_aes_ctx->key_sched);
787 
788 		break;
789 	}
790 	case CKM_BLOWFISH_CBC:
791 	{
792 		soft_blowfish_ctx_t *soft_blowfish_ctx;
793 
794 		soft_blowfish_ctx =
795 		    (soft_blowfish_ctx_t *)session_p->encrypt.context;
796 		/*
797 		 * CKM_BLOWFISH_CBC does not do any padding, so when the
798 		 * final is called, the remaining buffer should not contain
799 		 * any more data
800 		 */
801 		*pulLastEncryptedPartLen = 0;
802 		if (soft_blowfish_ctx->remain_len != 0)
803 			rv = CKR_DATA_LEN_RANGE;
804 		else {
805 			if (pLastEncryptedPart == NULL)
806 				goto clean1;
807 		}
808 
809 		free(soft_blowfish_ctx->blowfish_cbc);
810 		bzero(soft_blowfish_ctx->key_sched,
811 		    soft_blowfish_ctx->keysched_len);
812 		free(soft_blowfish_ctx->key_sched);
813 		break;
814 	}
815 
816 	case CKM_RC4:
817 	{
818 		ARCFour_key *key = (ARCFour_key *)session_p->encrypt.context;
819 		/* Remaining data size is always zero for RC4. */
820 		*pulLastEncryptedPartLen = 0;
821 		if (pLastEncryptedPart == NULL)
822 			goto clean1;
823 		bzero(key, sizeof (*key));
824 		break;
825 	}
826 	default:
827 		/* PKCS11: The mechanism only supports single-part operation. */
828 		rv = CKR_MECHANISM_INVALID;
829 		break;
830 	}
831 
832 	free(session_p->encrypt.context);
833 	session_p->encrypt.context = NULL;
834 clean1:
835 	(void) pthread_mutex_unlock(&session_p->session_mutex);
836 
837 	return (rv);
838 }
839 
840 /*
841  * This function frees the allocated active crypto context and the
842  * lower level of allocated struct as needed.
843  * This function is called by the 1st tier of encrypt/decrypt routines
844  * or by the 2nd tier of session close routine. Since the 1st tier
845  * caller will always call this function without locking the session
846  * mutex and the 2nd tier caller will call with the lock, we add the
847  * third parameter "lock_held" to distiguish this case.
848  */
849 void
850 soft_crypt_cleanup(soft_session_t *session_p, boolean_t encrypt,
851 	boolean_t lock_held)
852 {
853 
854 	crypto_active_op_t *active_op;
855 	boolean_t lock_true = B_TRUE;
856 
857 	if (!lock_held)
858 		(void) pthread_mutex_lock(&session_p->session_mutex);
859 
860 	active_op = (encrypt) ? &(session_p->encrypt) : &(session_p->decrypt);
861 
862 	switch (active_op->mech.mechanism) {
863 
864 	case CKM_DES_CBC_PAD:
865 	case CKM_DES3_CBC_PAD:
866 	case CKM_DES_CBC:
867 	case CKM_DES_ECB:
868 	case CKM_DES3_CBC:
869 	case CKM_DES3_ECB:
870 	{
871 
872 		soft_des_ctx_t *soft_des_ctx =
873 		    (soft_des_ctx_t *)active_op->context;
874 		des_ctx_t *des_ctx;
875 
876 		if (soft_des_ctx != NULL) {
877 			des_ctx = (des_ctx_t *)soft_des_ctx->des_cbc;
878 			if (des_ctx != NULL) {
879 				bzero(des_ctx->dc_keysched,
880 				    des_ctx->dc_keysched_len);
881 				free(soft_des_ctx->des_cbc);
882 			}
883 			bzero(soft_des_ctx->key_sched,
884 			    soft_des_ctx->keysched_len);
885 			free(soft_des_ctx->key_sched);
886 		}
887 		break;
888 	}
889 
890 	case CKM_AES_CBC_PAD:
891 	case CKM_AES_CBC:
892 	case CKM_AES_ECB:
893 	{
894 		soft_aes_ctx_t *soft_aes_ctx =
895 		    (soft_aes_ctx_t *)active_op->context;
896 		aes_ctx_t *aes_ctx;
897 
898 		if (soft_aes_ctx != NULL) {
899 			aes_ctx = (aes_ctx_t *)soft_aes_ctx->aes_cbc;
900 			if (aes_ctx != NULL) {
901 				bzero(aes_ctx->ac_keysched,
902 				    aes_ctx->ac_keysched_len);
903 				free(soft_aes_ctx->aes_cbc);
904 			}
905 			bzero(soft_aes_ctx->key_sched,
906 			    soft_aes_ctx->keysched_len);
907 			free(soft_aes_ctx->key_sched);
908 		}
909 		break;
910 	}
911 
912 	case CKM_BLOWFISH_CBC:
913 	{
914 		soft_blowfish_ctx_t *soft_blowfish_ctx =
915 		    (soft_blowfish_ctx_t *)active_op->context;
916 		blowfish_ctx_t *blowfish_ctx;
917 
918 		if (soft_blowfish_ctx != NULL) {
919 			blowfish_ctx =
920 			    (blowfish_ctx_t *)soft_blowfish_ctx->blowfish_cbc;
921 			if (blowfish_ctx != NULL) {
922 				bzero(blowfish_ctx->bc_keysched,
923 				    blowfish_ctx->bc_keysched_len);
924 				free(soft_blowfish_ctx->blowfish_cbc);
925 			}
926 
927 			bzero(soft_blowfish_ctx->key_sched,
928 			    soft_blowfish_ctx->keysched_len);
929 			free(soft_blowfish_ctx->key_sched);
930 		}
931 		break;
932 	}
933 
934 	case CKM_RC4:
935 	{
936 		ARCFour_key *key = (ARCFour_key *)active_op->context;
937 
938 		if (key != NULL)
939 			bzero(key, sizeof (*key));
940 		break;
941 	}
942 
943 	case CKM_RSA_X_509:
944 	case CKM_RSA_PKCS:
945 	{
946 		soft_rsa_ctx_t *rsa_ctx =
947 		    (soft_rsa_ctx_t *)active_op->context;
948 
949 		if (rsa_ctx != NULL)
950 			if (rsa_ctx->key != NULL) {
951 				soft_cleanup_object(rsa_ctx->key);
952 				free(rsa_ctx->key);
953 			}
954 
955 		break;
956 	}
957 
958 	} /* switch */
959 
960 	if (active_op->context != NULL) {
961 		free(active_op->context);
962 		active_op->context = NULL;
963 	}
964 
965 	active_op->flags = 0;
966 
967 	if (!lock_held)
968 		SES_REFRELE(session_p, lock_true);
969 }
970