1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Marvell 4 * 5 * Antoine Tenart <antoine.tenart@free-electrons.com> 6 */ 7 8 #include <asm/unaligned.h> 9 #include <linux/device.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dmapool.h> 12 #include <crypto/aead.h> 13 #include <crypto/aes.h> 14 #include <crypto/authenc.h> 15 #include <crypto/chacha.h> 16 #include <crypto/ctr.h> 17 #include <crypto/internal/des.h> 18 #include <crypto/gcm.h> 19 #include <crypto/ghash.h> 20 #include <crypto/poly1305.h> 21 #include <crypto/sha.h> 22 #include <crypto/sm3.h> 23 #include <crypto/sm4.h> 24 #include <crypto/xts.h> 25 #include <crypto/skcipher.h> 26 #include <crypto/internal/aead.h> 27 #include <crypto/internal/skcipher.h> 28 29 #include "safexcel.h" 30 31 enum safexcel_cipher_direction { 32 SAFEXCEL_ENCRYPT, 33 SAFEXCEL_DECRYPT, 34 }; 35 36 enum safexcel_cipher_alg { 37 SAFEXCEL_DES, 38 SAFEXCEL_3DES, 39 SAFEXCEL_AES, 40 SAFEXCEL_CHACHA20, 41 SAFEXCEL_SM4, 42 }; 43 44 struct safexcel_cipher_ctx { 45 struct safexcel_context base; 46 struct safexcel_crypto_priv *priv; 47 48 u32 mode; 49 enum safexcel_cipher_alg alg; 50 u8 aead; /* !=0=AEAD, 2=IPSec ESP AEAD, 3=IPsec ESP GMAC */ 51 u8 xcm; /* 0=authenc, 1=GCM, 2 reserved for CCM */ 52 u8 aadskip; 53 u8 blocksz; 54 u32 ivmask; 55 u32 ctrinit; 56 57 __le32 key[16]; 58 u32 nonce; 59 unsigned int key_len, xts; 60 61 /* All the below is AEAD specific */ 62 u32 hash_alg; 63 u32 state_sz; 64 __be32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)]; 65 __be32 opad[SHA512_DIGEST_SIZE / sizeof(u32)]; 66 67 struct crypto_cipher *hkaes; 68 struct crypto_aead *fback; 69 }; 70 71 struct safexcel_cipher_req { 72 enum safexcel_cipher_direction direction; 73 /* Number of result descriptors associated to the request */ 74 unsigned int rdescs; 75 bool needs_inv; 76 int nr_src, nr_dst; 77 }; 78 79 static int safexcel_skcipher_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 80 struct safexcel_command_desc *cdesc) 81 { 82 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 83 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 84 /* 32 bit nonce */ 85 cdesc->control_data.token[0] = ctx->nonce; 86 /* 64 bit IV part */ 87 memcpy(&cdesc->control_data.token[1], iv, 8); 88 /* 32 bit counter, start at 0 or 1 (big endian!) */ 89 cdesc->control_data.token[3] = 90 (__force u32)cpu_to_be32(ctx->ctrinit); 91 return 4; 92 } 93 if (ctx->alg == SAFEXCEL_CHACHA20) { 94 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 95 /* 96 bit nonce part */ 96 memcpy(&cdesc->control_data.token[0], &iv[4], 12); 97 /* 32 bit counter */ 98 cdesc->control_data.token[3] = *(u32 *)iv; 99 return 4; 100 } 101 102 cdesc->control_data.options |= ctx->ivmask; 103 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 104 return ctx->blocksz / sizeof(u32); 105 } 106 107 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 108 struct safexcel_command_desc *cdesc, 109 struct safexcel_token *atoken, 110 u32 length) 111 { 112 struct safexcel_token *token; 113 int ivlen; 114 115 ivlen = safexcel_skcipher_iv(ctx, iv, cdesc); 116 if (ivlen == 4) { 117 /* No space in cdesc, instruction moves to atoken */ 118 cdesc->additional_cdata_size = 1; 119 token = atoken; 120 } else { 121 /* Everything fits in cdesc */ 122 token = (struct safexcel_token *)(cdesc->control_data.token + 2); 123 /* Need to pad with NOP */ 124 eip197_noop_token(&token[1]); 125 } 126 127 token->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 128 token->packet_length = length; 129 token->stat = EIP197_TOKEN_STAT_LAST_PACKET | 130 EIP197_TOKEN_STAT_LAST_HASH; 131 token->instructions = EIP197_TOKEN_INS_LAST | 132 EIP197_TOKEN_INS_TYPE_CRYPTO | 133 EIP197_TOKEN_INS_TYPE_OUTPUT; 134 } 135 136 static void safexcel_aead_iv(struct safexcel_cipher_ctx *ctx, u8 *iv, 137 struct safexcel_command_desc *cdesc) 138 { 139 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD || 140 ctx->aead & EIP197_AEAD_TYPE_IPSEC_ESP) { /* _ESP and _ESP_GMAC */ 141 /* 32 bit nonce */ 142 cdesc->control_data.token[0] = ctx->nonce; 143 /* 64 bit IV part */ 144 memcpy(&cdesc->control_data.token[1], iv, 8); 145 /* 32 bit counter, start at 0 or 1 (big endian!) */ 146 cdesc->control_data.token[3] = 147 (__force u32)cpu_to_be32(ctx->ctrinit); 148 return; 149 } 150 if (ctx->xcm == EIP197_XCM_MODE_GCM || ctx->alg == SAFEXCEL_CHACHA20) { 151 /* 96 bit IV part */ 152 memcpy(&cdesc->control_data.token[0], iv, 12); 153 /* 32 bit counter, start at 0 or 1 (big endian!) */ 154 cdesc->control_data.token[3] = 155 (__force u32)cpu_to_be32(ctx->ctrinit); 156 return; 157 } 158 /* CBC */ 159 memcpy(cdesc->control_data.token, iv, ctx->blocksz); 160 } 161 162 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, 163 struct safexcel_command_desc *cdesc, 164 struct safexcel_token *atoken, 165 enum safexcel_cipher_direction direction, 166 u32 cryptlen, u32 assoclen, u32 digestsize) 167 { 168 struct safexcel_token *aadref; 169 int atoksize = 2; /* Start with minimum size */ 170 int assocadj = assoclen - ctx->aadskip, aadalign; 171 172 /* Always 4 dwords of embedded IV for AEAD modes */ 173 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; 174 175 if (direction == SAFEXCEL_DECRYPT) 176 cryptlen -= digestsize; 177 178 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM)) { 179 /* Construct IV block B0 for the CBC-MAC */ 180 u8 *final_iv = (u8 *)cdesc->control_data.token; 181 u8 *cbcmaciv = (u8 *)&atoken[1]; 182 __le32 *aadlen = (__le32 *)&atoken[5]; 183 184 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 185 /* Length + nonce */ 186 cdesc->control_data.token[0] = ctx->nonce; 187 /* Fixup flags byte */ 188 *(__le32 *)cbcmaciv = 189 cpu_to_le32(ctx->nonce | 190 ((assocadj > 0) << 6) | 191 ((digestsize - 2) << 2)); 192 /* 64 bit IV part */ 193 memcpy(&cdesc->control_data.token[1], iv, 8); 194 memcpy(cbcmaciv + 4, iv, 8); 195 /* Start counter at 0 */ 196 cdesc->control_data.token[3] = 0; 197 /* Message length */ 198 *(__be32 *)(cbcmaciv + 12) = cpu_to_be32(cryptlen); 199 } else { 200 /* Variable length IV part */ 201 memcpy(final_iv, iv, 15 - iv[0]); 202 memcpy(cbcmaciv, iv, 15 - iv[0]); 203 /* Start variable length counter at 0 */ 204 memset(final_iv + 15 - iv[0], 0, iv[0] + 1); 205 memset(cbcmaciv + 15 - iv[0], 0, iv[0] - 1); 206 /* fixup flags byte */ 207 cbcmaciv[0] |= ((assocadj > 0) << 6) | 208 ((digestsize - 2) << 2); 209 /* insert lower 2 bytes of message length */ 210 cbcmaciv[14] = cryptlen >> 8; 211 cbcmaciv[15] = cryptlen & 255; 212 } 213 214 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 215 atoken->packet_length = AES_BLOCK_SIZE + 216 ((assocadj > 0) << 1); 217 atoken->stat = 0; 218 atoken->instructions = EIP197_TOKEN_INS_ORIGIN_TOKEN | 219 EIP197_TOKEN_INS_TYPE_HASH; 220 221 if (likely(assocadj)) { 222 *aadlen = cpu_to_le32((assocadj >> 8) | 223 (assocadj & 255) << 8); 224 atoken += 6; 225 atoksize += 7; 226 } else { 227 atoken += 5; 228 atoksize += 6; 229 } 230 231 /* Process AAD data */ 232 aadref = atoken; 233 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 234 atoken->packet_length = assocadj; 235 atoken->stat = 0; 236 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 237 atoken++; 238 239 /* For CCM only, align AAD data towards hash engine */ 240 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 241 aadalign = (assocadj + 2) & 15; 242 atoken->packet_length = assocadj && aadalign ? 243 16 - aadalign : 244 0; 245 if (likely(cryptlen)) { 246 atoken->stat = 0; 247 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 248 } else { 249 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 250 atoken->instructions = EIP197_TOKEN_INS_LAST | 251 EIP197_TOKEN_INS_TYPE_HASH; 252 } 253 } else { 254 safexcel_aead_iv(ctx, iv, cdesc); 255 256 /* Process AAD data */ 257 aadref = atoken; 258 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 259 atoken->packet_length = assocadj; 260 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 261 atoken->instructions = EIP197_TOKEN_INS_LAST | 262 EIP197_TOKEN_INS_TYPE_HASH; 263 } 264 atoken++; 265 266 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 267 /* For ESP mode (and not GMAC), skip over the IV */ 268 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 269 atoken->packet_length = EIP197_AEAD_IPSEC_IV_SIZE; 270 atoken->stat = 0; 271 atoken->instructions = 0; 272 atoken++; 273 atoksize++; 274 } else if (unlikely(ctx->alg == SAFEXCEL_CHACHA20 && 275 direction == SAFEXCEL_DECRYPT)) { 276 /* Poly-chacha decryption needs a dummy NOP here ... */ 277 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 278 atoken->packet_length = 16; /* According to Op Manual */ 279 atoken->stat = 0; 280 atoken->instructions = 0; 281 atoken++; 282 atoksize++; 283 } 284 285 if (ctx->xcm) { 286 /* For GCM and CCM, obtain enc(Y0) */ 287 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT_REMRES; 288 atoken->packet_length = 0; 289 atoken->stat = 0; 290 atoken->instructions = AES_BLOCK_SIZE; 291 atoken++; 292 293 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 294 atoken->packet_length = AES_BLOCK_SIZE; 295 atoken->stat = 0; 296 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 297 EIP197_TOKEN_INS_TYPE_CRYPTO; 298 atoken++; 299 atoksize += 2; 300 } 301 302 if (likely(cryptlen || ctx->alg == SAFEXCEL_CHACHA20)) { 303 /* Fixup stat field for AAD direction instruction */ 304 aadref->stat = 0; 305 306 /* Process crypto data */ 307 atoken->opcode = EIP197_TOKEN_OPCODE_DIRECTION; 308 atoken->packet_length = cryptlen; 309 310 if (unlikely(ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) { 311 /* Fixup instruction field for AAD dir instruction */ 312 aadref->instructions = EIP197_TOKEN_INS_TYPE_HASH; 313 314 /* Do not send to crypt engine in case of GMAC */ 315 atoken->instructions = EIP197_TOKEN_INS_LAST | 316 EIP197_TOKEN_INS_TYPE_HASH | 317 EIP197_TOKEN_INS_TYPE_OUTPUT; 318 } else { 319 atoken->instructions = EIP197_TOKEN_INS_LAST | 320 EIP197_TOKEN_INS_TYPE_CRYPTO | 321 EIP197_TOKEN_INS_TYPE_HASH | 322 EIP197_TOKEN_INS_TYPE_OUTPUT; 323 } 324 325 cryptlen &= 15; 326 if (unlikely(ctx->xcm == EIP197_XCM_MODE_CCM && cryptlen)) { 327 atoken->stat = 0; 328 /* For CCM only, pad crypto data to the hash engine */ 329 atoken++; 330 atoksize++; 331 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 332 atoken->packet_length = 16 - cryptlen; 333 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 334 atoken->instructions = EIP197_TOKEN_INS_TYPE_HASH; 335 } else { 336 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH; 337 } 338 atoken++; 339 atoksize++; 340 } 341 342 if (direction == SAFEXCEL_ENCRYPT) { 343 /* Append ICV */ 344 atoken->opcode = EIP197_TOKEN_OPCODE_INSERT; 345 atoken->packet_length = digestsize; 346 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 347 EIP197_TOKEN_STAT_LAST_PACKET; 348 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT | 349 EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 350 } else { 351 /* Extract ICV */ 352 atoken->opcode = EIP197_TOKEN_OPCODE_RETRIEVE; 353 atoken->packet_length = digestsize; 354 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 355 EIP197_TOKEN_STAT_LAST_PACKET; 356 atoken->instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST; 357 atoken++; 358 atoksize++; 359 360 /* Verify ICV */ 361 atoken->opcode = EIP197_TOKEN_OPCODE_VERIFY; 362 atoken->packet_length = digestsize | 363 EIP197_TOKEN_HASH_RESULT_VERIFY; 364 atoken->stat = EIP197_TOKEN_STAT_LAST_HASH | 365 EIP197_TOKEN_STAT_LAST_PACKET; 366 atoken->instructions = EIP197_TOKEN_INS_TYPE_OUTPUT; 367 } 368 369 /* Fixup length of the token in the command descriptor */ 370 cdesc->additional_cdata_size = atoksize; 371 } 372 373 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm, 374 const u8 *key, unsigned int len) 375 { 376 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 377 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 378 struct safexcel_crypto_priv *priv = ctx->priv; 379 struct crypto_aes_ctx aes; 380 int ret, i; 381 382 ret = aes_expandkey(&aes, key, len); 383 if (ret) 384 return ret; 385 386 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 387 for (i = 0; i < len / sizeof(u32); i++) { 388 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 389 ctx->base.needs_inv = true; 390 break; 391 } 392 } 393 } 394 395 for (i = 0; i < len / sizeof(u32); i++) 396 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 397 398 ctx->key_len = len; 399 400 memzero_explicit(&aes, sizeof(aes)); 401 return 0; 402 } 403 404 static int safexcel_aead_setkey(struct crypto_aead *ctfm, const u8 *key, 405 unsigned int len) 406 { 407 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 408 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 409 struct safexcel_ahash_export_state istate, ostate; 410 struct safexcel_crypto_priv *priv = ctx->priv; 411 struct crypto_authenc_keys keys; 412 struct crypto_aes_ctx aes; 413 int err = -EINVAL, i; 414 415 if (unlikely(crypto_authenc_extractkeys(&keys, key, len))) 416 goto badkey; 417 418 if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD) { 419 /* Must have at least space for the nonce here */ 420 if (unlikely(keys.enckeylen < CTR_RFC3686_NONCE_SIZE)) 421 goto badkey; 422 /* last 4 bytes of key are the nonce! */ 423 ctx->nonce = *(u32 *)(keys.enckey + keys.enckeylen - 424 CTR_RFC3686_NONCE_SIZE); 425 /* exclude the nonce here */ 426 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE; 427 } 428 429 /* Encryption key */ 430 switch (ctx->alg) { 431 case SAFEXCEL_DES: 432 err = verify_aead_des_key(ctfm, keys.enckey, keys.enckeylen); 433 if (unlikely(err)) 434 goto badkey; 435 break; 436 case SAFEXCEL_3DES: 437 err = verify_aead_des3_key(ctfm, keys.enckey, keys.enckeylen); 438 if (unlikely(err)) 439 goto badkey; 440 break; 441 case SAFEXCEL_AES: 442 err = aes_expandkey(&aes, keys.enckey, keys.enckeylen); 443 if (unlikely(err)) 444 goto badkey; 445 break; 446 case SAFEXCEL_SM4: 447 if (unlikely(keys.enckeylen != SM4_KEY_SIZE)) 448 goto badkey; 449 break; 450 default: 451 dev_err(priv->dev, "aead: unsupported cipher algorithm\n"); 452 goto badkey; 453 } 454 455 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 456 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) { 457 if (le32_to_cpu(ctx->key[i]) != 458 ((u32 *)keys.enckey)[i]) { 459 ctx->base.needs_inv = true; 460 break; 461 } 462 } 463 } 464 465 /* Auth key */ 466 switch (ctx->hash_alg) { 467 case CONTEXT_CONTROL_CRYPTO_ALG_SHA1: 468 if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey, 469 keys.authkeylen, &istate, &ostate)) 470 goto badkey; 471 break; 472 case CONTEXT_CONTROL_CRYPTO_ALG_SHA224: 473 if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey, 474 keys.authkeylen, &istate, &ostate)) 475 goto badkey; 476 break; 477 case CONTEXT_CONTROL_CRYPTO_ALG_SHA256: 478 if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey, 479 keys.authkeylen, &istate, &ostate)) 480 goto badkey; 481 break; 482 case CONTEXT_CONTROL_CRYPTO_ALG_SHA384: 483 if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey, 484 keys.authkeylen, &istate, &ostate)) 485 goto badkey; 486 break; 487 case CONTEXT_CONTROL_CRYPTO_ALG_SHA512: 488 if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey, 489 keys.authkeylen, &istate, &ostate)) 490 goto badkey; 491 break; 492 case CONTEXT_CONTROL_CRYPTO_ALG_SM3: 493 if (safexcel_hmac_setkey("safexcel-sm3", keys.authkey, 494 keys.authkeylen, &istate, &ostate)) 495 goto badkey; 496 break; 497 default: 498 dev_err(priv->dev, "aead: unsupported hash algorithm\n"); 499 goto badkey; 500 } 501 502 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma && 503 (memcmp(ctx->ipad, istate.state, ctx->state_sz) || 504 memcmp(ctx->opad, ostate.state, ctx->state_sz))) 505 ctx->base.needs_inv = true; 506 507 /* Now copy the keys into the context */ 508 for (i = 0; i < keys.enckeylen / sizeof(u32); i++) 509 ctx->key[i] = cpu_to_le32(((u32 *)keys.enckey)[i]); 510 ctx->key_len = keys.enckeylen; 511 512 memcpy(ctx->ipad, &istate.state, ctx->state_sz); 513 memcpy(ctx->opad, &ostate.state, ctx->state_sz); 514 515 memzero_explicit(&keys, sizeof(keys)); 516 return 0; 517 518 badkey: 519 memzero_explicit(&keys, sizeof(keys)); 520 return err; 521 } 522 523 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, 524 struct crypto_async_request *async, 525 struct safexcel_cipher_req *sreq, 526 struct safexcel_command_desc *cdesc) 527 { 528 struct safexcel_crypto_priv *priv = ctx->priv; 529 int ctrl_size = ctx->key_len / sizeof(u32); 530 531 cdesc->control_data.control1 = ctx->mode; 532 533 if (ctx->aead) { 534 /* Take in account the ipad+opad digests */ 535 if (ctx->xcm) { 536 ctrl_size += ctx->state_sz / sizeof(u32); 537 cdesc->control_data.control0 = 538 CONTEXT_CONTROL_KEY_EN | 539 CONTEXT_CONTROL_DIGEST_XCM | 540 ctx->hash_alg | 541 CONTEXT_CONTROL_SIZE(ctrl_size); 542 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 543 /* Chacha20-Poly1305 */ 544 cdesc->control_data.control0 = 545 CONTEXT_CONTROL_KEY_EN | 546 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20 | 547 (sreq->direction == SAFEXCEL_ENCRYPT ? 548 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT : 549 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN) | 550 ctx->hash_alg | 551 CONTEXT_CONTROL_SIZE(ctrl_size); 552 return 0; 553 } else { 554 ctrl_size += ctx->state_sz / sizeof(u32) * 2; 555 cdesc->control_data.control0 = 556 CONTEXT_CONTROL_KEY_EN | 557 CONTEXT_CONTROL_DIGEST_HMAC | 558 ctx->hash_alg | 559 CONTEXT_CONTROL_SIZE(ctrl_size); 560 } 561 562 if (sreq->direction == SAFEXCEL_ENCRYPT && 563 (ctx->xcm == EIP197_XCM_MODE_CCM || 564 ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP_GMAC)) 565 cdesc->control_data.control0 |= 566 CONTEXT_CONTROL_TYPE_HASH_ENCRYPT_OUT; 567 else if (sreq->direction == SAFEXCEL_ENCRYPT) 568 cdesc->control_data.control0 |= 569 CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT; 570 else if (ctx->xcm == EIP197_XCM_MODE_CCM) 571 cdesc->control_data.control0 |= 572 CONTEXT_CONTROL_TYPE_DECRYPT_HASH_IN; 573 else 574 cdesc->control_data.control0 |= 575 CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN; 576 } else { 577 if (sreq->direction == SAFEXCEL_ENCRYPT) 578 cdesc->control_data.control0 = 579 CONTEXT_CONTROL_TYPE_CRYPTO_OUT | 580 CONTEXT_CONTROL_KEY_EN | 581 CONTEXT_CONTROL_SIZE(ctrl_size); 582 else 583 cdesc->control_data.control0 = 584 CONTEXT_CONTROL_TYPE_CRYPTO_IN | 585 CONTEXT_CONTROL_KEY_EN | 586 CONTEXT_CONTROL_SIZE(ctrl_size); 587 } 588 589 if (ctx->alg == SAFEXCEL_DES) { 590 cdesc->control_data.control0 |= 591 CONTEXT_CONTROL_CRYPTO_ALG_DES; 592 } else if (ctx->alg == SAFEXCEL_3DES) { 593 cdesc->control_data.control0 |= 594 CONTEXT_CONTROL_CRYPTO_ALG_3DES; 595 } else if (ctx->alg == SAFEXCEL_AES) { 596 switch (ctx->key_len >> ctx->xts) { 597 case AES_KEYSIZE_128: 598 cdesc->control_data.control0 |= 599 CONTEXT_CONTROL_CRYPTO_ALG_AES128; 600 break; 601 case AES_KEYSIZE_192: 602 cdesc->control_data.control0 |= 603 CONTEXT_CONTROL_CRYPTO_ALG_AES192; 604 break; 605 case AES_KEYSIZE_256: 606 cdesc->control_data.control0 |= 607 CONTEXT_CONTROL_CRYPTO_ALG_AES256; 608 break; 609 default: 610 dev_err(priv->dev, "aes keysize not supported: %u\n", 611 ctx->key_len >> ctx->xts); 612 return -EINVAL; 613 } 614 } else if (ctx->alg == SAFEXCEL_CHACHA20) { 615 cdesc->control_data.control0 |= 616 CONTEXT_CONTROL_CRYPTO_ALG_CHACHA20; 617 } else if (ctx->alg == SAFEXCEL_SM4) { 618 cdesc->control_data.control0 |= 619 CONTEXT_CONTROL_CRYPTO_ALG_SM4; 620 } 621 622 return 0; 623 } 624 625 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring, 626 struct crypto_async_request *async, 627 struct scatterlist *src, 628 struct scatterlist *dst, 629 unsigned int cryptlen, 630 struct safexcel_cipher_req *sreq, 631 bool *should_complete, int *ret) 632 { 633 struct skcipher_request *areq = skcipher_request_cast(async); 634 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 635 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(skcipher); 636 struct safexcel_result_desc *rdesc; 637 int ndesc = 0; 638 639 *ret = 0; 640 641 if (unlikely(!sreq->rdescs)) 642 return 0; 643 644 while (sreq->rdescs--) { 645 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 646 if (IS_ERR(rdesc)) { 647 dev_err(priv->dev, 648 "cipher: result: could not retrieve the result descriptor\n"); 649 *ret = PTR_ERR(rdesc); 650 break; 651 } 652 653 if (likely(!*ret)) 654 *ret = safexcel_rdesc_check_errors(priv, rdesc); 655 656 ndesc++; 657 } 658 659 safexcel_complete(priv, ring); 660 661 if (src == dst) { 662 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 663 } else { 664 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 665 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 666 } 667 668 /* 669 * Update IV in req from last crypto output word for CBC modes 670 */ 671 if ((!ctx->aead) && (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 672 (sreq->direction == SAFEXCEL_ENCRYPT)) { 673 /* For encrypt take the last output word */ 674 sg_pcopy_to_buffer(dst, sreq->nr_dst, areq->iv, 675 crypto_skcipher_ivsize(skcipher), 676 (cryptlen - 677 crypto_skcipher_ivsize(skcipher))); 678 } 679 680 *should_complete = true; 681 682 return ndesc; 683 } 684 685 static int safexcel_send_req(struct crypto_async_request *base, int ring, 686 struct safexcel_cipher_req *sreq, 687 struct scatterlist *src, struct scatterlist *dst, 688 unsigned int cryptlen, unsigned int assoclen, 689 unsigned int digestsize, u8 *iv, int *commands, 690 int *results) 691 { 692 struct skcipher_request *areq = skcipher_request_cast(base); 693 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq); 694 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 695 struct safexcel_crypto_priv *priv = ctx->priv; 696 struct safexcel_command_desc *cdesc; 697 struct safexcel_command_desc *first_cdesc = NULL; 698 struct safexcel_result_desc *rdesc, *first_rdesc = NULL; 699 struct scatterlist *sg; 700 unsigned int totlen; 701 unsigned int totlen_src = cryptlen + assoclen; 702 unsigned int totlen_dst = totlen_src; 703 struct safexcel_token *atoken; 704 int n_cdesc = 0, n_rdesc = 0; 705 int queued, i, ret = 0; 706 bool first = true; 707 708 sreq->nr_src = sg_nents_for_len(src, totlen_src); 709 710 if (ctx->aead) { 711 /* 712 * AEAD has auth tag appended to output for encrypt and 713 * removed from the output for decrypt! 714 */ 715 if (sreq->direction == SAFEXCEL_DECRYPT) 716 totlen_dst -= digestsize; 717 else 718 totlen_dst += digestsize; 719 720 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32), 721 ctx->ipad, ctx->state_sz); 722 if (!ctx->xcm) 723 memcpy(ctx->base.ctxr->data + (ctx->key_len + 724 ctx->state_sz) / sizeof(u32), ctx->opad, 725 ctx->state_sz); 726 } else if ((ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) && 727 (sreq->direction == SAFEXCEL_DECRYPT)) { 728 /* 729 * Save IV from last crypto input word for CBC modes in decrypt 730 * direction. Need to do this first in case of inplace operation 731 * as it will be overwritten. 732 */ 733 sg_pcopy_to_buffer(src, sreq->nr_src, areq->iv, 734 crypto_skcipher_ivsize(skcipher), 735 (totlen_src - 736 crypto_skcipher_ivsize(skcipher))); 737 } 738 739 sreq->nr_dst = sg_nents_for_len(dst, totlen_dst); 740 741 /* 742 * Remember actual input length, source buffer length may be 743 * updated in case of inline operation below. 744 */ 745 totlen = totlen_src; 746 queued = totlen_src; 747 748 if (src == dst) { 749 sreq->nr_src = max(sreq->nr_src, sreq->nr_dst); 750 sreq->nr_dst = sreq->nr_src; 751 if (unlikely((totlen_src || totlen_dst) && 752 (sreq->nr_src <= 0))) { 753 dev_err(priv->dev, "In-place buffer not large enough (need %d bytes)!", 754 max(totlen_src, totlen_dst)); 755 return -EINVAL; 756 } 757 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 758 } else { 759 if (unlikely(totlen_src && (sreq->nr_src <= 0))) { 760 dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", 761 totlen_src); 762 return -EINVAL; 763 } 764 dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 765 766 if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { 767 dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", 768 totlen_dst); 769 dma_unmap_sg(priv->dev, src, sreq->nr_src, 770 DMA_TO_DEVICE); 771 return -EINVAL; 772 } 773 dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 774 } 775 776 memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); 777 778 if (!totlen) { 779 /* 780 * The EIP97 cannot deal with zero length input packets! 781 * So stuff a dummy command descriptor indicating a 1 byte 782 * (dummy) input packet, using the context record as source. 783 */ 784 first_cdesc = safexcel_add_cdesc(priv, ring, 785 1, 1, ctx->base.ctxr_dma, 786 1, 1, ctx->base.ctxr_dma, 787 &atoken); 788 if (IS_ERR(first_cdesc)) { 789 /* No space left in the command descriptor ring */ 790 ret = PTR_ERR(first_cdesc); 791 goto cdesc_rollback; 792 } 793 n_cdesc = 1; 794 goto skip_cdesc; 795 } 796 797 /* command descriptors */ 798 for_each_sg(src, sg, sreq->nr_src, i) { 799 int len = sg_dma_len(sg); 800 801 /* Do not overflow the request */ 802 if (queued < len) 803 len = queued; 804 805 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, 806 !(queued - len), 807 sg_dma_address(sg), len, totlen, 808 ctx->base.ctxr_dma, &atoken); 809 if (IS_ERR(cdesc)) { 810 /* No space left in the command descriptor ring */ 811 ret = PTR_ERR(cdesc); 812 goto cdesc_rollback; 813 } 814 815 if (!n_cdesc) 816 first_cdesc = cdesc; 817 818 n_cdesc++; 819 queued -= len; 820 if (!queued) 821 break; 822 } 823 skip_cdesc: 824 /* Add context control words and token to first command descriptor */ 825 safexcel_context_control(ctx, base, sreq, first_cdesc); 826 if (ctx->aead) 827 safexcel_aead_token(ctx, iv, first_cdesc, atoken, 828 sreq->direction, cryptlen, 829 assoclen, digestsize); 830 else 831 safexcel_skcipher_token(ctx, iv, first_cdesc, atoken, 832 cryptlen); 833 834 /* result descriptors */ 835 for_each_sg(dst, sg, sreq->nr_dst, i) { 836 bool last = (i == sreq->nr_dst - 1); 837 u32 len = sg_dma_len(sg); 838 839 /* only allow the part of the buffer we know we need */ 840 if (len > totlen_dst) 841 len = totlen_dst; 842 if (unlikely(!len)) 843 break; 844 totlen_dst -= len; 845 846 /* skip over AAD space in buffer - not written */ 847 if (assoclen) { 848 if (assoclen >= len) { 849 assoclen -= len; 850 continue; 851 } 852 rdesc = safexcel_add_rdesc(priv, ring, first, last, 853 sg_dma_address(sg) + 854 assoclen, 855 len - assoclen); 856 assoclen = 0; 857 } else { 858 rdesc = safexcel_add_rdesc(priv, ring, first, last, 859 sg_dma_address(sg), 860 len); 861 } 862 if (IS_ERR(rdesc)) { 863 /* No space left in the result descriptor ring */ 864 ret = PTR_ERR(rdesc); 865 goto rdesc_rollback; 866 } 867 if (first) { 868 first_rdesc = rdesc; 869 first = false; 870 } 871 n_rdesc++; 872 } 873 874 if (unlikely(first)) { 875 /* 876 * Special case: AEAD decrypt with only AAD data. 877 * In this case there is NO output data from the engine, 878 * but the engine still needs a result descriptor! 879 * Create a dummy one just for catching the result token. 880 */ 881 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0); 882 if (IS_ERR(rdesc)) { 883 /* No space left in the result descriptor ring */ 884 ret = PTR_ERR(rdesc); 885 goto rdesc_rollback; 886 } 887 first_rdesc = rdesc; 888 n_rdesc = 1; 889 } 890 891 safexcel_rdr_req_set(priv, ring, first_rdesc, base); 892 893 *commands = n_cdesc; 894 *results = n_rdesc; 895 return 0; 896 897 rdesc_rollback: 898 for (i = 0; i < n_rdesc; i++) 899 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); 900 cdesc_rollback: 901 for (i = 0; i < n_cdesc; i++) 902 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); 903 904 if (src == dst) { 905 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL); 906 } else { 907 dma_unmap_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); 908 dma_unmap_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE); 909 } 910 911 return ret; 912 } 913 914 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, 915 int ring, 916 struct crypto_async_request *base, 917 struct safexcel_cipher_req *sreq, 918 bool *should_complete, int *ret) 919 { 920 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 921 struct safexcel_result_desc *rdesc; 922 int ndesc = 0, enq_ret; 923 924 *ret = 0; 925 926 if (unlikely(!sreq->rdescs)) 927 return 0; 928 929 while (sreq->rdescs--) { 930 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); 931 if (IS_ERR(rdesc)) { 932 dev_err(priv->dev, 933 "cipher: invalidate: could not retrieve the result descriptor\n"); 934 *ret = PTR_ERR(rdesc); 935 break; 936 } 937 938 if (likely(!*ret)) 939 *ret = safexcel_rdesc_check_errors(priv, rdesc); 940 941 ndesc++; 942 } 943 944 safexcel_complete(priv, ring); 945 946 if (ctx->base.exit_inv) { 947 dma_pool_free(priv->context_pool, ctx->base.ctxr, 948 ctx->base.ctxr_dma); 949 950 *should_complete = true; 951 952 return ndesc; 953 } 954 955 ring = safexcel_select_ring(priv); 956 ctx->base.ring = ring; 957 958 spin_lock_bh(&priv->ring[ring].queue_lock); 959 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 960 spin_unlock_bh(&priv->ring[ring].queue_lock); 961 962 if (enq_ret != -EINPROGRESS) 963 *ret = enq_ret; 964 965 queue_work(priv->ring[ring].workqueue, 966 &priv->ring[ring].work_data.work); 967 968 *should_complete = false; 969 970 return ndesc; 971 } 972 973 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, 974 int ring, 975 struct crypto_async_request *async, 976 bool *should_complete, int *ret) 977 { 978 struct skcipher_request *req = skcipher_request_cast(async); 979 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 980 int err; 981 982 if (sreq->needs_inv) { 983 sreq->needs_inv = false; 984 err = safexcel_handle_inv_result(priv, ring, async, sreq, 985 should_complete, ret); 986 } else { 987 err = safexcel_handle_req_result(priv, ring, async, req->src, 988 req->dst, req->cryptlen, sreq, 989 should_complete, ret); 990 } 991 992 return err; 993 } 994 995 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, 996 int ring, 997 struct crypto_async_request *async, 998 bool *should_complete, int *ret) 999 { 1000 struct aead_request *req = aead_request_cast(async); 1001 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1002 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1003 int err; 1004 1005 if (sreq->needs_inv) { 1006 sreq->needs_inv = false; 1007 err = safexcel_handle_inv_result(priv, ring, async, sreq, 1008 should_complete, ret); 1009 } else { 1010 err = safexcel_handle_req_result(priv, ring, async, req->src, 1011 req->dst, 1012 req->cryptlen + crypto_aead_authsize(tfm), 1013 sreq, should_complete, ret); 1014 } 1015 1016 return err; 1017 } 1018 1019 static int safexcel_cipher_send_inv(struct crypto_async_request *base, 1020 int ring, int *commands, int *results) 1021 { 1022 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1023 struct safexcel_crypto_priv *priv = ctx->priv; 1024 int ret; 1025 1026 ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring); 1027 if (unlikely(ret)) 1028 return ret; 1029 1030 *commands = 1; 1031 *results = 1; 1032 1033 return 0; 1034 } 1035 1036 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, 1037 int *commands, int *results) 1038 { 1039 struct skcipher_request *req = skcipher_request_cast(async); 1040 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1041 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1042 struct safexcel_crypto_priv *priv = ctx->priv; 1043 int ret; 1044 1045 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1046 1047 if (sreq->needs_inv) { 1048 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1049 } else { 1050 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); 1051 u8 input_iv[AES_BLOCK_SIZE]; 1052 1053 /* 1054 * Save input IV in case of CBC decrypt mode 1055 * Will be overwritten with output IV prior to use! 1056 */ 1057 memcpy(input_iv, req->iv, crypto_skcipher_ivsize(skcipher)); 1058 1059 ret = safexcel_send_req(async, ring, sreq, req->src, 1060 req->dst, req->cryptlen, 0, 0, input_iv, 1061 commands, results); 1062 } 1063 1064 sreq->rdescs = *results; 1065 return ret; 1066 } 1067 1068 static int safexcel_aead_send(struct crypto_async_request *async, int ring, 1069 int *commands, int *results) 1070 { 1071 struct aead_request *req = aead_request_cast(async); 1072 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1073 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); 1074 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1075 struct safexcel_crypto_priv *priv = ctx->priv; 1076 int ret; 1077 1078 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv); 1079 1080 if (sreq->needs_inv) 1081 ret = safexcel_cipher_send_inv(async, ring, commands, results); 1082 else 1083 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, 1084 req->cryptlen, req->assoclen, 1085 crypto_aead_authsize(tfm), req->iv, 1086 commands, results); 1087 sreq->rdescs = *results; 1088 return ret; 1089 } 1090 1091 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm, 1092 struct crypto_async_request *base, 1093 struct safexcel_cipher_req *sreq, 1094 struct safexcel_inv_result *result) 1095 { 1096 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1097 struct safexcel_crypto_priv *priv = ctx->priv; 1098 int ring = ctx->base.ring; 1099 1100 init_completion(&result->completion); 1101 1102 ctx = crypto_tfm_ctx(base->tfm); 1103 ctx->base.exit_inv = true; 1104 sreq->needs_inv = true; 1105 1106 spin_lock_bh(&priv->ring[ring].queue_lock); 1107 crypto_enqueue_request(&priv->ring[ring].queue, base); 1108 spin_unlock_bh(&priv->ring[ring].queue_lock); 1109 1110 queue_work(priv->ring[ring].workqueue, 1111 &priv->ring[ring].work_data.work); 1112 1113 wait_for_completion(&result->completion); 1114 1115 if (result->error) { 1116 dev_warn(priv->dev, 1117 "cipher: sync: invalidate: completion error %d\n", 1118 result->error); 1119 return result->error; 1120 } 1121 1122 return 0; 1123 } 1124 1125 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm) 1126 { 1127 EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE); 1128 struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); 1129 struct safexcel_inv_result result = {}; 1130 1131 memset(req, 0, sizeof(struct skcipher_request)); 1132 1133 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1134 safexcel_inv_complete, &result); 1135 skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm)); 1136 1137 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1138 } 1139 1140 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm) 1141 { 1142 EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE); 1143 struct safexcel_cipher_req *sreq = aead_request_ctx(req); 1144 struct safexcel_inv_result result = {}; 1145 1146 memset(req, 0, sizeof(struct aead_request)); 1147 1148 aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 1149 safexcel_inv_complete, &result); 1150 aead_request_set_tfm(req, __crypto_aead_cast(tfm)); 1151 1152 return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result); 1153 } 1154 1155 static int safexcel_queue_req(struct crypto_async_request *base, 1156 struct safexcel_cipher_req *sreq, 1157 enum safexcel_cipher_direction dir) 1158 { 1159 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); 1160 struct safexcel_crypto_priv *priv = ctx->priv; 1161 int ret, ring; 1162 1163 sreq->needs_inv = false; 1164 sreq->direction = dir; 1165 1166 if (ctx->base.ctxr) { 1167 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) { 1168 sreq->needs_inv = true; 1169 ctx->base.needs_inv = false; 1170 } 1171 } else { 1172 ctx->base.ring = safexcel_select_ring(priv); 1173 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, 1174 EIP197_GFP_FLAGS(*base), 1175 &ctx->base.ctxr_dma); 1176 if (!ctx->base.ctxr) 1177 return -ENOMEM; 1178 } 1179 1180 ring = ctx->base.ring; 1181 1182 spin_lock_bh(&priv->ring[ring].queue_lock); 1183 ret = crypto_enqueue_request(&priv->ring[ring].queue, base); 1184 spin_unlock_bh(&priv->ring[ring].queue_lock); 1185 1186 queue_work(priv->ring[ring].workqueue, 1187 &priv->ring[ring].work_data.work); 1188 1189 return ret; 1190 } 1191 1192 static int safexcel_encrypt(struct skcipher_request *req) 1193 { 1194 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1195 SAFEXCEL_ENCRYPT); 1196 } 1197 1198 static int safexcel_decrypt(struct skcipher_request *req) 1199 { 1200 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 1201 SAFEXCEL_DECRYPT); 1202 } 1203 1204 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) 1205 { 1206 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1207 struct safexcel_alg_template *tmpl = 1208 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1209 alg.skcipher.base); 1210 1211 crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm), 1212 sizeof(struct safexcel_cipher_req)); 1213 1214 ctx->priv = tmpl->priv; 1215 1216 ctx->base.send = safexcel_skcipher_send; 1217 ctx->base.handle_result = safexcel_skcipher_handle_result; 1218 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1219 ctx->ctrinit = 1; 1220 return 0; 1221 } 1222 1223 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm) 1224 { 1225 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1226 1227 memzero_explicit(ctx->key, sizeof(ctx->key)); 1228 1229 /* context not allocated, skip invalidation */ 1230 if (!ctx->base.ctxr) 1231 return -ENOMEM; 1232 1233 memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data)); 1234 return 0; 1235 } 1236 1237 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) 1238 { 1239 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1240 struct safexcel_crypto_priv *priv = ctx->priv; 1241 int ret; 1242 1243 if (safexcel_cipher_cra_exit(tfm)) 1244 return; 1245 1246 if (priv->flags & EIP197_TRC_CACHE) { 1247 ret = safexcel_skcipher_exit_inv(tfm); 1248 if (ret) 1249 dev_warn(priv->dev, "skcipher: invalidation error %d\n", 1250 ret); 1251 } else { 1252 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1253 ctx->base.ctxr_dma); 1254 } 1255 } 1256 1257 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm) 1258 { 1259 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1260 struct safexcel_crypto_priv *priv = ctx->priv; 1261 int ret; 1262 1263 if (safexcel_cipher_cra_exit(tfm)) 1264 return; 1265 1266 if (priv->flags & EIP197_TRC_CACHE) { 1267 ret = safexcel_aead_exit_inv(tfm); 1268 if (ret) 1269 dev_warn(priv->dev, "aead: invalidation error %d\n", 1270 ret); 1271 } else { 1272 dma_pool_free(priv->context_pool, ctx->base.ctxr, 1273 ctx->base.ctxr_dma); 1274 } 1275 } 1276 1277 static int safexcel_skcipher_aes_ecb_cra_init(struct crypto_tfm *tfm) 1278 { 1279 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1280 1281 safexcel_skcipher_cra_init(tfm); 1282 ctx->alg = SAFEXCEL_AES; 1283 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1284 ctx->blocksz = 0; 1285 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1286 return 0; 1287 } 1288 1289 struct safexcel_alg_template safexcel_alg_ecb_aes = { 1290 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1291 .algo_mask = SAFEXCEL_ALG_AES, 1292 .alg.skcipher = { 1293 .setkey = safexcel_skcipher_aes_setkey, 1294 .encrypt = safexcel_encrypt, 1295 .decrypt = safexcel_decrypt, 1296 .min_keysize = AES_MIN_KEY_SIZE, 1297 .max_keysize = AES_MAX_KEY_SIZE, 1298 .base = { 1299 .cra_name = "ecb(aes)", 1300 .cra_driver_name = "safexcel-ecb-aes", 1301 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1302 .cra_flags = CRYPTO_ALG_ASYNC | 1303 CRYPTO_ALG_ALLOCATES_MEMORY | 1304 CRYPTO_ALG_KERN_DRIVER_ONLY, 1305 .cra_blocksize = AES_BLOCK_SIZE, 1306 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1307 .cra_alignmask = 0, 1308 .cra_init = safexcel_skcipher_aes_ecb_cra_init, 1309 .cra_exit = safexcel_skcipher_cra_exit, 1310 .cra_module = THIS_MODULE, 1311 }, 1312 }, 1313 }; 1314 1315 static int safexcel_skcipher_aes_cbc_cra_init(struct crypto_tfm *tfm) 1316 { 1317 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1318 1319 safexcel_skcipher_cra_init(tfm); 1320 ctx->alg = SAFEXCEL_AES; 1321 ctx->blocksz = AES_BLOCK_SIZE; 1322 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1323 return 0; 1324 } 1325 1326 struct safexcel_alg_template safexcel_alg_cbc_aes = { 1327 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1328 .algo_mask = SAFEXCEL_ALG_AES, 1329 .alg.skcipher = { 1330 .setkey = safexcel_skcipher_aes_setkey, 1331 .encrypt = safexcel_encrypt, 1332 .decrypt = safexcel_decrypt, 1333 .min_keysize = AES_MIN_KEY_SIZE, 1334 .max_keysize = AES_MAX_KEY_SIZE, 1335 .ivsize = AES_BLOCK_SIZE, 1336 .base = { 1337 .cra_name = "cbc(aes)", 1338 .cra_driver_name = "safexcel-cbc-aes", 1339 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1340 .cra_flags = CRYPTO_ALG_ASYNC | 1341 CRYPTO_ALG_ALLOCATES_MEMORY | 1342 CRYPTO_ALG_KERN_DRIVER_ONLY, 1343 .cra_blocksize = AES_BLOCK_SIZE, 1344 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1345 .cra_alignmask = 0, 1346 .cra_init = safexcel_skcipher_aes_cbc_cra_init, 1347 .cra_exit = safexcel_skcipher_cra_exit, 1348 .cra_module = THIS_MODULE, 1349 }, 1350 }, 1351 }; 1352 1353 static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm) 1354 { 1355 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1356 1357 safexcel_skcipher_cra_init(tfm); 1358 ctx->alg = SAFEXCEL_AES; 1359 ctx->blocksz = AES_BLOCK_SIZE; 1360 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 1361 return 0; 1362 } 1363 1364 struct safexcel_alg_template safexcel_alg_cfb_aes = { 1365 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1366 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1367 .alg.skcipher = { 1368 .setkey = safexcel_skcipher_aes_setkey, 1369 .encrypt = safexcel_encrypt, 1370 .decrypt = safexcel_decrypt, 1371 .min_keysize = AES_MIN_KEY_SIZE, 1372 .max_keysize = AES_MAX_KEY_SIZE, 1373 .ivsize = AES_BLOCK_SIZE, 1374 .base = { 1375 .cra_name = "cfb(aes)", 1376 .cra_driver_name = "safexcel-cfb-aes", 1377 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1378 .cra_flags = CRYPTO_ALG_ASYNC | 1379 CRYPTO_ALG_ALLOCATES_MEMORY | 1380 CRYPTO_ALG_KERN_DRIVER_ONLY, 1381 .cra_blocksize = 1, 1382 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1383 .cra_alignmask = 0, 1384 .cra_init = safexcel_skcipher_aes_cfb_cra_init, 1385 .cra_exit = safexcel_skcipher_cra_exit, 1386 .cra_module = THIS_MODULE, 1387 }, 1388 }, 1389 }; 1390 1391 static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm) 1392 { 1393 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1394 1395 safexcel_skcipher_cra_init(tfm); 1396 ctx->alg = SAFEXCEL_AES; 1397 ctx->blocksz = AES_BLOCK_SIZE; 1398 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 1399 return 0; 1400 } 1401 1402 struct safexcel_alg_template safexcel_alg_ofb_aes = { 1403 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1404 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, 1405 .alg.skcipher = { 1406 .setkey = safexcel_skcipher_aes_setkey, 1407 .encrypt = safexcel_encrypt, 1408 .decrypt = safexcel_decrypt, 1409 .min_keysize = AES_MIN_KEY_SIZE, 1410 .max_keysize = AES_MAX_KEY_SIZE, 1411 .ivsize = AES_BLOCK_SIZE, 1412 .base = { 1413 .cra_name = "ofb(aes)", 1414 .cra_driver_name = "safexcel-ofb-aes", 1415 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1416 .cra_flags = CRYPTO_ALG_ASYNC | 1417 CRYPTO_ALG_ALLOCATES_MEMORY | 1418 CRYPTO_ALG_KERN_DRIVER_ONLY, 1419 .cra_blocksize = 1, 1420 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1421 .cra_alignmask = 0, 1422 .cra_init = safexcel_skcipher_aes_ofb_cra_init, 1423 .cra_exit = safexcel_skcipher_cra_exit, 1424 .cra_module = THIS_MODULE, 1425 }, 1426 }, 1427 }; 1428 1429 static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, 1430 const u8 *key, unsigned int len) 1431 { 1432 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 1433 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1434 struct safexcel_crypto_priv *priv = ctx->priv; 1435 struct crypto_aes_ctx aes; 1436 int ret, i; 1437 unsigned int keylen; 1438 1439 /* last 4 bytes of key are the nonce! */ 1440 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 1441 /* exclude the nonce here */ 1442 keylen = len - CTR_RFC3686_NONCE_SIZE; 1443 ret = aes_expandkey(&aes, key, keylen); 1444 if (ret) 1445 return ret; 1446 1447 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 1448 for (i = 0; i < keylen / sizeof(u32); i++) { 1449 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 1450 ctx->base.needs_inv = true; 1451 break; 1452 } 1453 } 1454 } 1455 1456 for (i = 0; i < keylen / sizeof(u32); i++) 1457 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 1458 1459 ctx->key_len = keylen; 1460 1461 memzero_explicit(&aes, sizeof(aes)); 1462 return 0; 1463 } 1464 1465 static int safexcel_skcipher_aes_ctr_cra_init(struct crypto_tfm *tfm) 1466 { 1467 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1468 1469 safexcel_skcipher_cra_init(tfm); 1470 ctx->alg = SAFEXCEL_AES; 1471 ctx->blocksz = AES_BLOCK_SIZE; 1472 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 1473 return 0; 1474 } 1475 1476 struct safexcel_alg_template safexcel_alg_ctr_aes = { 1477 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1478 .algo_mask = SAFEXCEL_ALG_AES, 1479 .alg.skcipher = { 1480 .setkey = safexcel_skcipher_aesctr_setkey, 1481 .encrypt = safexcel_encrypt, 1482 .decrypt = safexcel_decrypt, 1483 /* Add nonce size */ 1484 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1485 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 1486 .ivsize = CTR_RFC3686_IV_SIZE, 1487 .base = { 1488 .cra_name = "rfc3686(ctr(aes))", 1489 .cra_driver_name = "safexcel-ctr-aes", 1490 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1491 .cra_flags = CRYPTO_ALG_ASYNC | 1492 CRYPTO_ALG_ALLOCATES_MEMORY | 1493 CRYPTO_ALG_KERN_DRIVER_ONLY, 1494 .cra_blocksize = 1, 1495 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1496 .cra_alignmask = 0, 1497 .cra_init = safexcel_skcipher_aes_ctr_cra_init, 1498 .cra_exit = safexcel_skcipher_cra_exit, 1499 .cra_module = THIS_MODULE, 1500 }, 1501 }, 1502 }; 1503 1504 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key, 1505 unsigned int len) 1506 { 1507 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1508 struct safexcel_crypto_priv *priv = ctx->priv; 1509 int ret; 1510 1511 ret = verify_skcipher_des_key(ctfm, key); 1512 if (ret) 1513 return ret; 1514 1515 /* if context exits and key changed, need to invalidate it */ 1516 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1517 if (memcmp(ctx->key, key, len)) 1518 ctx->base.needs_inv = true; 1519 1520 memcpy(ctx->key, key, len); 1521 ctx->key_len = len; 1522 1523 return 0; 1524 } 1525 1526 static int safexcel_skcipher_des_cbc_cra_init(struct crypto_tfm *tfm) 1527 { 1528 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1529 1530 safexcel_skcipher_cra_init(tfm); 1531 ctx->alg = SAFEXCEL_DES; 1532 ctx->blocksz = DES_BLOCK_SIZE; 1533 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1534 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1535 return 0; 1536 } 1537 1538 struct safexcel_alg_template safexcel_alg_cbc_des = { 1539 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1540 .algo_mask = SAFEXCEL_ALG_DES, 1541 .alg.skcipher = { 1542 .setkey = safexcel_des_setkey, 1543 .encrypt = safexcel_encrypt, 1544 .decrypt = safexcel_decrypt, 1545 .min_keysize = DES_KEY_SIZE, 1546 .max_keysize = DES_KEY_SIZE, 1547 .ivsize = DES_BLOCK_SIZE, 1548 .base = { 1549 .cra_name = "cbc(des)", 1550 .cra_driver_name = "safexcel-cbc-des", 1551 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1552 .cra_flags = CRYPTO_ALG_ASYNC | 1553 CRYPTO_ALG_ALLOCATES_MEMORY | 1554 CRYPTO_ALG_KERN_DRIVER_ONLY, 1555 .cra_blocksize = DES_BLOCK_SIZE, 1556 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1557 .cra_alignmask = 0, 1558 .cra_init = safexcel_skcipher_des_cbc_cra_init, 1559 .cra_exit = safexcel_skcipher_cra_exit, 1560 .cra_module = THIS_MODULE, 1561 }, 1562 }, 1563 }; 1564 1565 static int safexcel_skcipher_des_ecb_cra_init(struct crypto_tfm *tfm) 1566 { 1567 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1568 1569 safexcel_skcipher_cra_init(tfm); 1570 ctx->alg = SAFEXCEL_DES; 1571 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1572 ctx->blocksz = 0; 1573 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1574 return 0; 1575 } 1576 1577 struct safexcel_alg_template safexcel_alg_ecb_des = { 1578 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1579 .algo_mask = SAFEXCEL_ALG_DES, 1580 .alg.skcipher = { 1581 .setkey = safexcel_des_setkey, 1582 .encrypt = safexcel_encrypt, 1583 .decrypt = safexcel_decrypt, 1584 .min_keysize = DES_KEY_SIZE, 1585 .max_keysize = DES_KEY_SIZE, 1586 .base = { 1587 .cra_name = "ecb(des)", 1588 .cra_driver_name = "safexcel-ecb-des", 1589 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1590 .cra_flags = CRYPTO_ALG_ASYNC | 1591 CRYPTO_ALG_ALLOCATES_MEMORY | 1592 CRYPTO_ALG_KERN_DRIVER_ONLY, 1593 .cra_blocksize = DES_BLOCK_SIZE, 1594 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1595 .cra_alignmask = 0, 1596 .cra_init = safexcel_skcipher_des_ecb_cra_init, 1597 .cra_exit = safexcel_skcipher_cra_exit, 1598 .cra_module = THIS_MODULE, 1599 }, 1600 }, 1601 }; 1602 1603 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm, 1604 const u8 *key, unsigned int len) 1605 { 1606 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 1607 struct safexcel_crypto_priv *priv = ctx->priv; 1608 int err; 1609 1610 err = verify_skcipher_des3_key(ctfm, key); 1611 if (err) 1612 return err; 1613 1614 /* if context exits and key changed, need to invalidate it */ 1615 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 1616 if (memcmp(ctx->key, key, len)) 1617 ctx->base.needs_inv = true; 1618 1619 memcpy(ctx->key, key, len); 1620 ctx->key_len = len; 1621 1622 return 0; 1623 } 1624 1625 static int safexcel_skcipher_des3_cbc_cra_init(struct crypto_tfm *tfm) 1626 { 1627 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1628 1629 safexcel_skcipher_cra_init(tfm); 1630 ctx->alg = SAFEXCEL_3DES; 1631 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1632 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1633 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 1634 return 0; 1635 } 1636 1637 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = { 1638 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1639 .algo_mask = SAFEXCEL_ALG_DES, 1640 .alg.skcipher = { 1641 .setkey = safexcel_des3_ede_setkey, 1642 .encrypt = safexcel_encrypt, 1643 .decrypt = safexcel_decrypt, 1644 .min_keysize = DES3_EDE_KEY_SIZE, 1645 .max_keysize = DES3_EDE_KEY_SIZE, 1646 .ivsize = DES3_EDE_BLOCK_SIZE, 1647 .base = { 1648 .cra_name = "cbc(des3_ede)", 1649 .cra_driver_name = "safexcel-cbc-des3_ede", 1650 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1651 .cra_flags = CRYPTO_ALG_ASYNC | 1652 CRYPTO_ALG_ALLOCATES_MEMORY | 1653 CRYPTO_ALG_KERN_DRIVER_ONLY, 1654 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1655 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1656 .cra_alignmask = 0, 1657 .cra_init = safexcel_skcipher_des3_cbc_cra_init, 1658 .cra_exit = safexcel_skcipher_cra_exit, 1659 .cra_module = THIS_MODULE, 1660 }, 1661 }, 1662 }; 1663 1664 static int safexcel_skcipher_des3_ecb_cra_init(struct crypto_tfm *tfm) 1665 { 1666 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1667 1668 safexcel_skcipher_cra_init(tfm); 1669 ctx->alg = SAFEXCEL_3DES; 1670 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 1671 ctx->blocksz = 0; 1672 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1673 return 0; 1674 } 1675 1676 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = { 1677 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 1678 .algo_mask = SAFEXCEL_ALG_DES, 1679 .alg.skcipher = { 1680 .setkey = safexcel_des3_ede_setkey, 1681 .encrypt = safexcel_encrypt, 1682 .decrypt = safexcel_decrypt, 1683 .min_keysize = DES3_EDE_KEY_SIZE, 1684 .max_keysize = DES3_EDE_KEY_SIZE, 1685 .base = { 1686 .cra_name = "ecb(des3_ede)", 1687 .cra_driver_name = "safexcel-ecb-des3_ede", 1688 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1689 .cra_flags = CRYPTO_ALG_ASYNC | 1690 CRYPTO_ALG_ALLOCATES_MEMORY | 1691 CRYPTO_ALG_KERN_DRIVER_ONLY, 1692 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1693 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1694 .cra_alignmask = 0, 1695 .cra_init = safexcel_skcipher_des3_ecb_cra_init, 1696 .cra_exit = safexcel_skcipher_cra_exit, 1697 .cra_module = THIS_MODULE, 1698 }, 1699 }, 1700 }; 1701 1702 static int safexcel_aead_encrypt(struct aead_request *req) 1703 { 1704 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1705 1706 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 1707 } 1708 1709 static int safexcel_aead_decrypt(struct aead_request *req) 1710 { 1711 struct safexcel_cipher_req *creq = aead_request_ctx(req); 1712 1713 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 1714 } 1715 1716 static int safexcel_aead_cra_init(struct crypto_tfm *tfm) 1717 { 1718 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1719 struct safexcel_alg_template *tmpl = 1720 container_of(tfm->__crt_alg, struct safexcel_alg_template, 1721 alg.aead.base); 1722 1723 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 1724 sizeof(struct safexcel_cipher_req)); 1725 1726 ctx->priv = tmpl->priv; 1727 1728 ctx->alg = SAFEXCEL_AES; /* default */ 1729 ctx->blocksz = AES_BLOCK_SIZE; 1730 ctx->ivmask = EIP197_OPTION_4_TOKEN_IV_CMD; 1731 ctx->ctrinit = 1; 1732 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; /* default */ 1733 ctx->aead = true; 1734 ctx->base.send = safexcel_aead_send; 1735 ctx->base.handle_result = safexcel_aead_handle_result; 1736 return 0; 1737 } 1738 1739 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm) 1740 { 1741 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1742 1743 safexcel_aead_cra_init(tfm); 1744 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 1745 ctx->state_sz = SHA1_DIGEST_SIZE; 1746 return 0; 1747 } 1748 1749 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = { 1750 .type = SAFEXCEL_ALG_TYPE_AEAD, 1751 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 1752 .alg.aead = { 1753 .setkey = safexcel_aead_setkey, 1754 .encrypt = safexcel_aead_encrypt, 1755 .decrypt = safexcel_aead_decrypt, 1756 .ivsize = AES_BLOCK_SIZE, 1757 .maxauthsize = SHA1_DIGEST_SIZE, 1758 .base = { 1759 .cra_name = "authenc(hmac(sha1),cbc(aes))", 1760 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes", 1761 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1762 .cra_flags = CRYPTO_ALG_ASYNC | 1763 CRYPTO_ALG_ALLOCATES_MEMORY | 1764 CRYPTO_ALG_KERN_DRIVER_ONLY, 1765 .cra_blocksize = AES_BLOCK_SIZE, 1766 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1767 .cra_alignmask = 0, 1768 .cra_init = safexcel_aead_sha1_cra_init, 1769 .cra_exit = safexcel_aead_cra_exit, 1770 .cra_module = THIS_MODULE, 1771 }, 1772 }, 1773 }; 1774 1775 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm) 1776 { 1777 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1778 1779 safexcel_aead_cra_init(tfm); 1780 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256; 1781 ctx->state_sz = SHA256_DIGEST_SIZE; 1782 return 0; 1783 } 1784 1785 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = { 1786 .type = SAFEXCEL_ALG_TYPE_AEAD, 1787 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1788 .alg.aead = { 1789 .setkey = safexcel_aead_setkey, 1790 .encrypt = safexcel_aead_encrypt, 1791 .decrypt = safexcel_aead_decrypt, 1792 .ivsize = AES_BLOCK_SIZE, 1793 .maxauthsize = SHA256_DIGEST_SIZE, 1794 .base = { 1795 .cra_name = "authenc(hmac(sha256),cbc(aes))", 1796 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes", 1797 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1798 .cra_flags = CRYPTO_ALG_ASYNC | 1799 CRYPTO_ALG_ALLOCATES_MEMORY | 1800 CRYPTO_ALG_KERN_DRIVER_ONLY, 1801 .cra_blocksize = AES_BLOCK_SIZE, 1802 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1803 .cra_alignmask = 0, 1804 .cra_init = safexcel_aead_sha256_cra_init, 1805 .cra_exit = safexcel_aead_cra_exit, 1806 .cra_module = THIS_MODULE, 1807 }, 1808 }, 1809 }; 1810 1811 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm) 1812 { 1813 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1814 1815 safexcel_aead_cra_init(tfm); 1816 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224; 1817 ctx->state_sz = SHA256_DIGEST_SIZE; 1818 return 0; 1819 } 1820 1821 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = { 1822 .type = SAFEXCEL_ALG_TYPE_AEAD, 1823 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 1824 .alg.aead = { 1825 .setkey = safexcel_aead_setkey, 1826 .encrypt = safexcel_aead_encrypt, 1827 .decrypt = safexcel_aead_decrypt, 1828 .ivsize = AES_BLOCK_SIZE, 1829 .maxauthsize = SHA224_DIGEST_SIZE, 1830 .base = { 1831 .cra_name = "authenc(hmac(sha224),cbc(aes))", 1832 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes", 1833 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1834 .cra_flags = CRYPTO_ALG_ASYNC | 1835 CRYPTO_ALG_ALLOCATES_MEMORY | 1836 CRYPTO_ALG_KERN_DRIVER_ONLY, 1837 .cra_blocksize = AES_BLOCK_SIZE, 1838 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1839 .cra_alignmask = 0, 1840 .cra_init = safexcel_aead_sha224_cra_init, 1841 .cra_exit = safexcel_aead_cra_exit, 1842 .cra_module = THIS_MODULE, 1843 }, 1844 }, 1845 }; 1846 1847 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm) 1848 { 1849 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1850 1851 safexcel_aead_cra_init(tfm); 1852 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512; 1853 ctx->state_sz = SHA512_DIGEST_SIZE; 1854 return 0; 1855 } 1856 1857 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = { 1858 .type = SAFEXCEL_ALG_TYPE_AEAD, 1859 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1860 .alg.aead = { 1861 .setkey = safexcel_aead_setkey, 1862 .encrypt = safexcel_aead_encrypt, 1863 .decrypt = safexcel_aead_decrypt, 1864 .ivsize = AES_BLOCK_SIZE, 1865 .maxauthsize = SHA512_DIGEST_SIZE, 1866 .base = { 1867 .cra_name = "authenc(hmac(sha512),cbc(aes))", 1868 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes", 1869 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1870 .cra_flags = CRYPTO_ALG_ASYNC | 1871 CRYPTO_ALG_ALLOCATES_MEMORY | 1872 CRYPTO_ALG_KERN_DRIVER_ONLY, 1873 .cra_blocksize = AES_BLOCK_SIZE, 1874 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1875 .cra_alignmask = 0, 1876 .cra_init = safexcel_aead_sha512_cra_init, 1877 .cra_exit = safexcel_aead_cra_exit, 1878 .cra_module = THIS_MODULE, 1879 }, 1880 }, 1881 }; 1882 1883 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm) 1884 { 1885 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1886 1887 safexcel_aead_cra_init(tfm); 1888 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384; 1889 ctx->state_sz = SHA512_DIGEST_SIZE; 1890 return 0; 1891 } 1892 1893 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = { 1894 .type = SAFEXCEL_ALG_TYPE_AEAD, 1895 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 1896 .alg.aead = { 1897 .setkey = safexcel_aead_setkey, 1898 .encrypt = safexcel_aead_encrypt, 1899 .decrypt = safexcel_aead_decrypt, 1900 .ivsize = AES_BLOCK_SIZE, 1901 .maxauthsize = SHA384_DIGEST_SIZE, 1902 .base = { 1903 .cra_name = "authenc(hmac(sha384),cbc(aes))", 1904 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes", 1905 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1906 .cra_flags = CRYPTO_ALG_ASYNC | 1907 CRYPTO_ALG_ALLOCATES_MEMORY | 1908 CRYPTO_ALG_KERN_DRIVER_ONLY, 1909 .cra_blocksize = AES_BLOCK_SIZE, 1910 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1911 .cra_alignmask = 0, 1912 .cra_init = safexcel_aead_sha384_cra_init, 1913 .cra_exit = safexcel_aead_cra_exit, 1914 .cra_module = THIS_MODULE, 1915 }, 1916 }, 1917 }; 1918 1919 static int safexcel_aead_sha1_des3_cra_init(struct crypto_tfm *tfm) 1920 { 1921 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1922 1923 safexcel_aead_sha1_cra_init(tfm); 1924 ctx->alg = SAFEXCEL_3DES; /* override default */ 1925 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1926 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1927 return 0; 1928 } 1929 1930 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des3_ede = { 1931 .type = SAFEXCEL_ALG_TYPE_AEAD, 1932 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 1933 .alg.aead = { 1934 .setkey = safexcel_aead_setkey, 1935 .encrypt = safexcel_aead_encrypt, 1936 .decrypt = safexcel_aead_decrypt, 1937 .ivsize = DES3_EDE_BLOCK_SIZE, 1938 .maxauthsize = SHA1_DIGEST_SIZE, 1939 .base = { 1940 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", 1941 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des3_ede", 1942 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1943 .cra_flags = CRYPTO_ALG_ASYNC | 1944 CRYPTO_ALG_ALLOCATES_MEMORY | 1945 CRYPTO_ALG_KERN_DRIVER_ONLY, 1946 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1947 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1948 .cra_alignmask = 0, 1949 .cra_init = safexcel_aead_sha1_des3_cra_init, 1950 .cra_exit = safexcel_aead_cra_exit, 1951 .cra_module = THIS_MODULE, 1952 }, 1953 }, 1954 }; 1955 1956 static int safexcel_aead_sha256_des3_cra_init(struct crypto_tfm *tfm) 1957 { 1958 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1959 1960 safexcel_aead_sha256_cra_init(tfm); 1961 ctx->alg = SAFEXCEL_3DES; /* override default */ 1962 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 1963 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 1964 return 0; 1965 } 1966 1967 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des3_ede = { 1968 .type = SAFEXCEL_ALG_TYPE_AEAD, 1969 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 1970 .alg.aead = { 1971 .setkey = safexcel_aead_setkey, 1972 .encrypt = safexcel_aead_encrypt, 1973 .decrypt = safexcel_aead_decrypt, 1974 .ivsize = DES3_EDE_BLOCK_SIZE, 1975 .maxauthsize = SHA256_DIGEST_SIZE, 1976 .base = { 1977 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))", 1978 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des3_ede", 1979 .cra_priority = SAFEXCEL_CRA_PRIORITY, 1980 .cra_flags = CRYPTO_ALG_ASYNC | 1981 CRYPTO_ALG_ALLOCATES_MEMORY | 1982 CRYPTO_ALG_KERN_DRIVER_ONLY, 1983 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 1984 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 1985 .cra_alignmask = 0, 1986 .cra_init = safexcel_aead_sha256_des3_cra_init, 1987 .cra_exit = safexcel_aead_cra_exit, 1988 .cra_module = THIS_MODULE, 1989 }, 1990 }, 1991 }; 1992 1993 static int safexcel_aead_sha224_des3_cra_init(struct crypto_tfm *tfm) 1994 { 1995 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 1996 1997 safexcel_aead_sha224_cra_init(tfm); 1998 ctx->alg = SAFEXCEL_3DES; /* override default */ 1999 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2000 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2001 return 0; 2002 } 2003 2004 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des3_ede = { 2005 .type = SAFEXCEL_ALG_TYPE_AEAD, 2006 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2007 .alg.aead = { 2008 .setkey = safexcel_aead_setkey, 2009 .encrypt = safexcel_aead_encrypt, 2010 .decrypt = safexcel_aead_decrypt, 2011 .ivsize = DES3_EDE_BLOCK_SIZE, 2012 .maxauthsize = SHA224_DIGEST_SIZE, 2013 .base = { 2014 .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", 2015 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des3_ede", 2016 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2017 .cra_flags = CRYPTO_ALG_ASYNC | 2018 CRYPTO_ALG_ALLOCATES_MEMORY | 2019 CRYPTO_ALG_KERN_DRIVER_ONLY, 2020 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2021 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2022 .cra_alignmask = 0, 2023 .cra_init = safexcel_aead_sha224_des3_cra_init, 2024 .cra_exit = safexcel_aead_cra_exit, 2025 .cra_module = THIS_MODULE, 2026 }, 2027 }, 2028 }; 2029 2030 static int safexcel_aead_sha512_des3_cra_init(struct crypto_tfm *tfm) 2031 { 2032 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2033 2034 safexcel_aead_sha512_cra_init(tfm); 2035 ctx->alg = SAFEXCEL_3DES; /* override default */ 2036 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2037 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2038 return 0; 2039 } 2040 2041 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des3_ede = { 2042 .type = SAFEXCEL_ALG_TYPE_AEAD, 2043 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2044 .alg.aead = { 2045 .setkey = safexcel_aead_setkey, 2046 .encrypt = safexcel_aead_encrypt, 2047 .decrypt = safexcel_aead_decrypt, 2048 .ivsize = DES3_EDE_BLOCK_SIZE, 2049 .maxauthsize = SHA512_DIGEST_SIZE, 2050 .base = { 2051 .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", 2052 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des3_ede", 2053 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2054 .cra_flags = CRYPTO_ALG_ASYNC | 2055 CRYPTO_ALG_ALLOCATES_MEMORY | 2056 CRYPTO_ALG_KERN_DRIVER_ONLY, 2057 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2058 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2059 .cra_alignmask = 0, 2060 .cra_init = safexcel_aead_sha512_des3_cra_init, 2061 .cra_exit = safexcel_aead_cra_exit, 2062 .cra_module = THIS_MODULE, 2063 }, 2064 }, 2065 }; 2066 2067 static int safexcel_aead_sha384_des3_cra_init(struct crypto_tfm *tfm) 2068 { 2069 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2070 2071 safexcel_aead_sha384_cra_init(tfm); 2072 ctx->alg = SAFEXCEL_3DES; /* override default */ 2073 ctx->blocksz = DES3_EDE_BLOCK_SIZE; 2074 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2075 return 0; 2076 } 2077 2078 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des3_ede = { 2079 .type = SAFEXCEL_ALG_TYPE_AEAD, 2080 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2081 .alg.aead = { 2082 .setkey = safexcel_aead_setkey, 2083 .encrypt = safexcel_aead_encrypt, 2084 .decrypt = safexcel_aead_decrypt, 2085 .ivsize = DES3_EDE_BLOCK_SIZE, 2086 .maxauthsize = SHA384_DIGEST_SIZE, 2087 .base = { 2088 .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", 2089 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des3_ede", 2090 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2091 .cra_flags = CRYPTO_ALG_ASYNC | 2092 CRYPTO_ALG_ALLOCATES_MEMORY | 2093 CRYPTO_ALG_KERN_DRIVER_ONLY, 2094 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 2095 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2096 .cra_alignmask = 0, 2097 .cra_init = safexcel_aead_sha384_des3_cra_init, 2098 .cra_exit = safexcel_aead_cra_exit, 2099 .cra_module = THIS_MODULE, 2100 }, 2101 }, 2102 }; 2103 2104 static int safexcel_aead_sha1_des_cra_init(struct crypto_tfm *tfm) 2105 { 2106 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2107 2108 safexcel_aead_sha1_cra_init(tfm); 2109 ctx->alg = SAFEXCEL_DES; /* override default */ 2110 ctx->blocksz = DES_BLOCK_SIZE; 2111 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2112 return 0; 2113 } 2114 2115 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_des = { 2116 .type = SAFEXCEL_ALG_TYPE_AEAD, 2117 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA1, 2118 .alg.aead = { 2119 .setkey = safexcel_aead_setkey, 2120 .encrypt = safexcel_aead_encrypt, 2121 .decrypt = safexcel_aead_decrypt, 2122 .ivsize = DES_BLOCK_SIZE, 2123 .maxauthsize = SHA1_DIGEST_SIZE, 2124 .base = { 2125 .cra_name = "authenc(hmac(sha1),cbc(des))", 2126 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-des", 2127 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2128 .cra_flags = CRYPTO_ALG_ASYNC | 2129 CRYPTO_ALG_ALLOCATES_MEMORY | 2130 CRYPTO_ALG_KERN_DRIVER_ONLY, 2131 .cra_blocksize = DES_BLOCK_SIZE, 2132 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2133 .cra_alignmask = 0, 2134 .cra_init = safexcel_aead_sha1_des_cra_init, 2135 .cra_exit = safexcel_aead_cra_exit, 2136 .cra_module = THIS_MODULE, 2137 }, 2138 }, 2139 }; 2140 2141 static int safexcel_aead_sha256_des_cra_init(struct crypto_tfm *tfm) 2142 { 2143 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2144 2145 safexcel_aead_sha256_cra_init(tfm); 2146 ctx->alg = SAFEXCEL_DES; /* override default */ 2147 ctx->blocksz = DES_BLOCK_SIZE; 2148 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2149 return 0; 2150 } 2151 2152 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_des = { 2153 .type = SAFEXCEL_ALG_TYPE_AEAD, 2154 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2155 .alg.aead = { 2156 .setkey = safexcel_aead_setkey, 2157 .encrypt = safexcel_aead_encrypt, 2158 .decrypt = safexcel_aead_decrypt, 2159 .ivsize = DES_BLOCK_SIZE, 2160 .maxauthsize = SHA256_DIGEST_SIZE, 2161 .base = { 2162 .cra_name = "authenc(hmac(sha256),cbc(des))", 2163 .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-des", 2164 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2165 .cra_flags = CRYPTO_ALG_ASYNC | 2166 CRYPTO_ALG_ALLOCATES_MEMORY | 2167 CRYPTO_ALG_KERN_DRIVER_ONLY, 2168 .cra_blocksize = DES_BLOCK_SIZE, 2169 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2170 .cra_alignmask = 0, 2171 .cra_init = safexcel_aead_sha256_des_cra_init, 2172 .cra_exit = safexcel_aead_cra_exit, 2173 .cra_module = THIS_MODULE, 2174 }, 2175 }, 2176 }; 2177 2178 static int safexcel_aead_sha224_des_cra_init(struct crypto_tfm *tfm) 2179 { 2180 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2181 2182 safexcel_aead_sha224_cra_init(tfm); 2183 ctx->alg = SAFEXCEL_DES; /* override default */ 2184 ctx->blocksz = DES_BLOCK_SIZE; 2185 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2186 return 0; 2187 } 2188 2189 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_des = { 2190 .type = SAFEXCEL_ALG_TYPE_AEAD, 2191 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_256, 2192 .alg.aead = { 2193 .setkey = safexcel_aead_setkey, 2194 .encrypt = safexcel_aead_encrypt, 2195 .decrypt = safexcel_aead_decrypt, 2196 .ivsize = DES_BLOCK_SIZE, 2197 .maxauthsize = SHA224_DIGEST_SIZE, 2198 .base = { 2199 .cra_name = "authenc(hmac(sha224),cbc(des))", 2200 .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-des", 2201 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2202 .cra_flags = CRYPTO_ALG_ASYNC | 2203 CRYPTO_ALG_ALLOCATES_MEMORY | 2204 CRYPTO_ALG_KERN_DRIVER_ONLY, 2205 .cra_blocksize = DES_BLOCK_SIZE, 2206 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2207 .cra_alignmask = 0, 2208 .cra_init = safexcel_aead_sha224_des_cra_init, 2209 .cra_exit = safexcel_aead_cra_exit, 2210 .cra_module = THIS_MODULE, 2211 }, 2212 }, 2213 }; 2214 2215 static int safexcel_aead_sha512_des_cra_init(struct crypto_tfm *tfm) 2216 { 2217 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2218 2219 safexcel_aead_sha512_cra_init(tfm); 2220 ctx->alg = SAFEXCEL_DES; /* override default */ 2221 ctx->blocksz = DES_BLOCK_SIZE; 2222 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2223 return 0; 2224 } 2225 2226 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_des = { 2227 .type = SAFEXCEL_ALG_TYPE_AEAD, 2228 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2229 .alg.aead = { 2230 .setkey = safexcel_aead_setkey, 2231 .encrypt = safexcel_aead_encrypt, 2232 .decrypt = safexcel_aead_decrypt, 2233 .ivsize = DES_BLOCK_SIZE, 2234 .maxauthsize = SHA512_DIGEST_SIZE, 2235 .base = { 2236 .cra_name = "authenc(hmac(sha512),cbc(des))", 2237 .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-des", 2238 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2239 .cra_flags = CRYPTO_ALG_ASYNC | 2240 CRYPTO_ALG_ALLOCATES_MEMORY | 2241 CRYPTO_ALG_KERN_DRIVER_ONLY, 2242 .cra_blocksize = DES_BLOCK_SIZE, 2243 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2244 .cra_alignmask = 0, 2245 .cra_init = safexcel_aead_sha512_des_cra_init, 2246 .cra_exit = safexcel_aead_cra_exit, 2247 .cra_module = THIS_MODULE, 2248 }, 2249 }, 2250 }; 2251 2252 static int safexcel_aead_sha384_des_cra_init(struct crypto_tfm *tfm) 2253 { 2254 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2255 2256 safexcel_aead_sha384_cra_init(tfm); 2257 ctx->alg = SAFEXCEL_DES; /* override default */ 2258 ctx->blocksz = DES_BLOCK_SIZE; 2259 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 2260 return 0; 2261 } 2262 2263 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_des = { 2264 .type = SAFEXCEL_ALG_TYPE_AEAD, 2265 .algo_mask = SAFEXCEL_ALG_DES | SAFEXCEL_ALG_SHA2_512, 2266 .alg.aead = { 2267 .setkey = safexcel_aead_setkey, 2268 .encrypt = safexcel_aead_encrypt, 2269 .decrypt = safexcel_aead_decrypt, 2270 .ivsize = DES_BLOCK_SIZE, 2271 .maxauthsize = SHA384_DIGEST_SIZE, 2272 .base = { 2273 .cra_name = "authenc(hmac(sha384),cbc(des))", 2274 .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-des", 2275 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2276 .cra_flags = CRYPTO_ALG_ASYNC | 2277 CRYPTO_ALG_ALLOCATES_MEMORY | 2278 CRYPTO_ALG_KERN_DRIVER_ONLY, 2279 .cra_blocksize = DES_BLOCK_SIZE, 2280 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2281 .cra_alignmask = 0, 2282 .cra_init = safexcel_aead_sha384_des_cra_init, 2283 .cra_exit = safexcel_aead_cra_exit, 2284 .cra_module = THIS_MODULE, 2285 }, 2286 }, 2287 }; 2288 2289 static int safexcel_aead_sha1_ctr_cra_init(struct crypto_tfm *tfm) 2290 { 2291 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2292 2293 safexcel_aead_sha1_cra_init(tfm); 2294 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2295 return 0; 2296 } 2297 2298 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_aes = { 2299 .type = SAFEXCEL_ALG_TYPE_AEAD, 2300 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA1, 2301 .alg.aead = { 2302 .setkey = safexcel_aead_setkey, 2303 .encrypt = safexcel_aead_encrypt, 2304 .decrypt = safexcel_aead_decrypt, 2305 .ivsize = CTR_RFC3686_IV_SIZE, 2306 .maxauthsize = SHA1_DIGEST_SIZE, 2307 .base = { 2308 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))", 2309 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-aes", 2310 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2311 .cra_flags = CRYPTO_ALG_ASYNC | 2312 CRYPTO_ALG_ALLOCATES_MEMORY | 2313 CRYPTO_ALG_KERN_DRIVER_ONLY, 2314 .cra_blocksize = 1, 2315 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2316 .cra_alignmask = 0, 2317 .cra_init = safexcel_aead_sha1_ctr_cra_init, 2318 .cra_exit = safexcel_aead_cra_exit, 2319 .cra_module = THIS_MODULE, 2320 }, 2321 }, 2322 }; 2323 2324 static int safexcel_aead_sha256_ctr_cra_init(struct crypto_tfm *tfm) 2325 { 2326 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2327 2328 safexcel_aead_sha256_cra_init(tfm); 2329 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2330 return 0; 2331 } 2332 2333 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_ctr_aes = { 2334 .type = SAFEXCEL_ALG_TYPE_AEAD, 2335 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2336 .alg.aead = { 2337 .setkey = safexcel_aead_setkey, 2338 .encrypt = safexcel_aead_encrypt, 2339 .decrypt = safexcel_aead_decrypt, 2340 .ivsize = CTR_RFC3686_IV_SIZE, 2341 .maxauthsize = SHA256_DIGEST_SIZE, 2342 .base = { 2343 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 2344 .cra_driver_name = "safexcel-authenc-hmac-sha256-ctr-aes", 2345 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2346 .cra_flags = CRYPTO_ALG_ASYNC | 2347 CRYPTO_ALG_ALLOCATES_MEMORY | 2348 CRYPTO_ALG_KERN_DRIVER_ONLY, 2349 .cra_blocksize = 1, 2350 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2351 .cra_alignmask = 0, 2352 .cra_init = safexcel_aead_sha256_ctr_cra_init, 2353 .cra_exit = safexcel_aead_cra_exit, 2354 .cra_module = THIS_MODULE, 2355 }, 2356 }, 2357 }; 2358 2359 static int safexcel_aead_sha224_ctr_cra_init(struct crypto_tfm *tfm) 2360 { 2361 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2362 2363 safexcel_aead_sha224_cra_init(tfm); 2364 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2365 return 0; 2366 } 2367 2368 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_ctr_aes = { 2369 .type = SAFEXCEL_ALG_TYPE_AEAD, 2370 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_256, 2371 .alg.aead = { 2372 .setkey = safexcel_aead_setkey, 2373 .encrypt = safexcel_aead_encrypt, 2374 .decrypt = safexcel_aead_decrypt, 2375 .ivsize = CTR_RFC3686_IV_SIZE, 2376 .maxauthsize = SHA224_DIGEST_SIZE, 2377 .base = { 2378 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 2379 .cra_driver_name = "safexcel-authenc-hmac-sha224-ctr-aes", 2380 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2381 .cra_flags = CRYPTO_ALG_ASYNC | 2382 CRYPTO_ALG_ALLOCATES_MEMORY | 2383 CRYPTO_ALG_KERN_DRIVER_ONLY, 2384 .cra_blocksize = 1, 2385 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2386 .cra_alignmask = 0, 2387 .cra_init = safexcel_aead_sha224_ctr_cra_init, 2388 .cra_exit = safexcel_aead_cra_exit, 2389 .cra_module = THIS_MODULE, 2390 }, 2391 }, 2392 }; 2393 2394 static int safexcel_aead_sha512_ctr_cra_init(struct crypto_tfm *tfm) 2395 { 2396 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2397 2398 safexcel_aead_sha512_cra_init(tfm); 2399 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2400 return 0; 2401 } 2402 2403 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_ctr_aes = { 2404 .type = SAFEXCEL_ALG_TYPE_AEAD, 2405 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2406 .alg.aead = { 2407 .setkey = safexcel_aead_setkey, 2408 .encrypt = safexcel_aead_encrypt, 2409 .decrypt = safexcel_aead_decrypt, 2410 .ivsize = CTR_RFC3686_IV_SIZE, 2411 .maxauthsize = SHA512_DIGEST_SIZE, 2412 .base = { 2413 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))", 2414 .cra_driver_name = "safexcel-authenc-hmac-sha512-ctr-aes", 2415 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2416 .cra_flags = CRYPTO_ALG_ASYNC | 2417 CRYPTO_ALG_ALLOCATES_MEMORY | 2418 CRYPTO_ALG_KERN_DRIVER_ONLY, 2419 .cra_blocksize = 1, 2420 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2421 .cra_alignmask = 0, 2422 .cra_init = safexcel_aead_sha512_ctr_cra_init, 2423 .cra_exit = safexcel_aead_cra_exit, 2424 .cra_module = THIS_MODULE, 2425 }, 2426 }, 2427 }; 2428 2429 static int safexcel_aead_sha384_ctr_cra_init(struct crypto_tfm *tfm) 2430 { 2431 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2432 2433 safexcel_aead_sha384_cra_init(tfm); 2434 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; /* override default */ 2435 return 0; 2436 } 2437 2438 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_ctr_aes = { 2439 .type = SAFEXCEL_ALG_TYPE_AEAD, 2440 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_SHA2_512, 2441 .alg.aead = { 2442 .setkey = safexcel_aead_setkey, 2443 .encrypt = safexcel_aead_encrypt, 2444 .decrypt = safexcel_aead_decrypt, 2445 .ivsize = CTR_RFC3686_IV_SIZE, 2446 .maxauthsize = SHA384_DIGEST_SIZE, 2447 .base = { 2448 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))", 2449 .cra_driver_name = "safexcel-authenc-hmac-sha384-ctr-aes", 2450 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2451 .cra_flags = CRYPTO_ALG_ASYNC | 2452 CRYPTO_ALG_ALLOCATES_MEMORY | 2453 CRYPTO_ALG_KERN_DRIVER_ONLY, 2454 .cra_blocksize = 1, 2455 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2456 .cra_alignmask = 0, 2457 .cra_init = safexcel_aead_sha384_ctr_cra_init, 2458 .cra_exit = safexcel_aead_cra_exit, 2459 .cra_module = THIS_MODULE, 2460 }, 2461 }, 2462 }; 2463 2464 static int safexcel_skcipher_aesxts_setkey(struct crypto_skcipher *ctfm, 2465 const u8 *key, unsigned int len) 2466 { 2467 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 2468 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2469 struct safexcel_crypto_priv *priv = ctx->priv; 2470 struct crypto_aes_ctx aes; 2471 int ret, i; 2472 unsigned int keylen; 2473 2474 /* Check for illegal XTS keys */ 2475 ret = xts_verify_key(ctfm, key, len); 2476 if (ret) 2477 return ret; 2478 2479 /* Only half of the key data is cipher key */ 2480 keylen = (len >> 1); 2481 ret = aes_expandkey(&aes, key, keylen); 2482 if (ret) 2483 return ret; 2484 2485 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2486 for (i = 0; i < keylen / sizeof(u32); i++) { 2487 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2488 ctx->base.needs_inv = true; 2489 break; 2490 } 2491 } 2492 } 2493 2494 for (i = 0; i < keylen / sizeof(u32); i++) 2495 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2496 2497 /* The other half is the tweak key */ 2498 ret = aes_expandkey(&aes, (u8 *)(key + keylen), keylen); 2499 if (ret) 2500 return ret; 2501 2502 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2503 for (i = 0; i < keylen / sizeof(u32); i++) { 2504 if (le32_to_cpu(ctx->key[i + keylen / sizeof(u32)]) != 2505 aes.key_enc[i]) { 2506 ctx->base.needs_inv = true; 2507 break; 2508 } 2509 } 2510 } 2511 2512 for (i = 0; i < keylen / sizeof(u32); i++) 2513 ctx->key[i + keylen / sizeof(u32)] = 2514 cpu_to_le32(aes.key_enc[i]); 2515 2516 ctx->key_len = keylen << 1; 2517 2518 memzero_explicit(&aes, sizeof(aes)); 2519 return 0; 2520 } 2521 2522 static int safexcel_skcipher_aes_xts_cra_init(struct crypto_tfm *tfm) 2523 { 2524 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2525 2526 safexcel_skcipher_cra_init(tfm); 2527 ctx->alg = SAFEXCEL_AES; 2528 ctx->blocksz = AES_BLOCK_SIZE; 2529 ctx->xts = 1; 2530 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XTS; 2531 return 0; 2532 } 2533 2534 static int safexcel_encrypt_xts(struct skcipher_request *req) 2535 { 2536 if (req->cryptlen < XTS_BLOCK_SIZE) 2537 return -EINVAL; 2538 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2539 SAFEXCEL_ENCRYPT); 2540 } 2541 2542 static int safexcel_decrypt_xts(struct skcipher_request *req) 2543 { 2544 if (req->cryptlen < XTS_BLOCK_SIZE) 2545 return -EINVAL; 2546 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 2547 SAFEXCEL_DECRYPT); 2548 } 2549 2550 struct safexcel_alg_template safexcel_alg_xts_aes = { 2551 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2552 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XTS, 2553 .alg.skcipher = { 2554 .setkey = safexcel_skcipher_aesxts_setkey, 2555 .encrypt = safexcel_encrypt_xts, 2556 .decrypt = safexcel_decrypt_xts, 2557 /* XTS actually uses 2 AES keys glued together */ 2558 .min_keysize = AES_MIN_KEY_SIZE * 2, 2559 .max_keysize = AES_MAX_KEY_SIZE * 2, 2560 .ivsize = XTS_BLOCK_SIZE, 2561 .base = { 2562 .cra_name = "xts(aes)", 2563 .cra_driver_name = "safexcel-xts-aes", 2564 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2565 .cra_flags = CRYPTO_ALG_ASYNC | 2566 CRYPTO_ALG_ALLOCATES_MEMORY | 2567 CRYPTO_ALG_KERN_DRIVER_ONLY, 2568 .cra_blocksize = XTS_BLOCK_SIZE, 2569 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2570 .cra_alignmask = 0, 2571 .cra_init = safexcel_skcipher_aes_xts_cra_init, 2572 .cra_exit = safexcel_skcipher_cra_exit, 2573 .cra_module = THIS_MODULE, 2574 }, 2575 }, 2576 }; 2577 2578 static int safexcel_aead_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 2579 unsigned int len) 2580 { 2581 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2582 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2583 struct safexcel_crypto_priv *priv = ctx->priv; 2584 struct crypto_aes_ctx aes; 2585 u32 hashkey[AES_BLOCK_SIZE >> 2]; 2586 int ret, i; 2587 2588 ret = aes_expandkey(&aes, key, len); 2589 if (ret) { 2590 memzero_explicit(&aes, sizeof(aes)); 2591 return ret; 2592 } 2593 2594 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2595 for (i = 0; i < len / sizeof(u32); i++) { 2596 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2597 ctx->base.needs_inv = true; 2598 break; 2599 } 2600 } 2601 } 2602 2603 for (i = 0; i < len / sizeof(u32); i++) 2604 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2605 2606 ctx->key_len = len; 2607 2608 /* Compute hash key by encrypting zeroes with cipher key */ 2609 crypto_cipher_clear_flags(ctx->hkaes, CRYPTO_TFM_REQ_MASK); 2610 crypto_cipher_set_flags(ctx->hkaes, crypto_aead_get_flags(ctfm) & 2611 CRYPTO_TFM_REQ_MASK); 2612 ret = crypto_cipher_setkey(ctx->hkaes, key, len); 2613 if (ret) 2614 return ret; 2615 2616 memset(hashkey, 0, AES_BLOCK_SIZE); 2617 crypto_cipher_encrypt_one(ctx->hkaes, (u8 *)hashkey, (u8 *)hashkey); 2618 2619 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2620 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) { 2621 if (be32_to_cpu(ctx->ipad[i]) != hashkey[i]) { 2622 ctx->base.needs_inv = true; 2623 break; 2624 } 2625 } 2626 } 2627 2628 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++) 2629 ctx->ipad[i] = cpu_to_be32(hashkey[i]); 2630 2631 memzero_explicit(hashkey, AES_BLOCK_SIZE); 2632 memzero_explicit(&aes, sizeof(aes)); 2633 return 0; 2634 } 2635 2636 static int safexcel_aead_gcm_cra_init(struct crypto_tfm *tfm) 2637 { 2638 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2639 2640 safexcel_aead_cra_init(tfm); 2641 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_GHASH; 2642 ctx->state_sz = GHASH_BLOCK_SIZE; 2643 ctx->xcm = EIP197_XCM_MODE_GCM; 2644 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2645 2646 ctx->hkaes = crypto_alloc_cipher("aes", 0, 0); 2647 return PTR_ERR_OR_ZERO(ctx->hkaes); 2648 } 2649 2650 static void safexcel_aead_gcm_cra_exit(struct crypto_tfm *tfm) 2651 { 2652 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2653 2654 crypto_free_cipher(ctx->hkaes); 2655 safexcel_aead_cra_exit(tfm); 2656 } 2657 2658 static int safexcel_aead_gcm_setauthsize(struct crypto_aead *tfm, 2659 unsigned int authsize) 2660 { 2661 return crypto_gcm_check_authsize(authsize); 2662 } 2663 2664 struct safexcel_alg_template safexcel_alg_gcm = { 2665 .type = SAFEXCEL_ALG_TYPE_AEAD, 2666 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 2667 .alg.aead = { 2668 .setkey = safexcel_aead_gcm_setkey, 2669 .setauthsize = safexcel_aead_gcm_setauthsize, 2670 .encrypt = safexcel_aead_encrypt, 2671 .decrypt = safexcel_aead_decrypt, 2672 .ivsize = GCM_AES_IV_SIZE, 2673 .maxauthsize = GHASH_DIGEST_SIZE, 2674 .base = { 2675 .cra_name = "gcm(aes)", 2676 .cra_driver_name = "safexcel-gcm-aes", 2677 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2678 .cra_flags = CRYPTO_ALG_ASYNC | 2679 CRYPTO_ALG_ALLOCATES_MEMORY | 2680 CRYPTO_ALG_KERN_DRIVER_ONLY, 2681 .cra_blocksize = 1, 2682 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2683 .cra_alignmask = 0, 2684 .cra_init = safexcel_aead_gcm_cra_init, 2685 .cra_exit = safexcel_aead_gcm_cra_exit, 2686 .cra_module = THIS_MODULE, 2687 }, 2688 }, 2689 }; 2690 2691 static int safexcel_aead_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 2692 unsigned int len) 2693 { 2694 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 2695 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2696 struct safexcel_crypto_priv *priv = ctx->priv; 2697 struct crypto_aes_ctx aes; 2698 int ret, i; 2699 2700 ret = aes_expandkey(&aes, key, len); 2701 if (ret) { 2702 memzero_explicit(&aes, sizeof(aes)); 2703 return ret; 2704 } 2705 2706 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) { 2707 for (i = 0; i < len / sizeof(u32); i++) { 2708 if (le32_to_cpu(ctx->key[i]) != aes.key_enc[i]) { 2709 ctx->base.needs_inv = true; 2710 break; 2711 } 2712 } 2713 } 2714 2715 for (i = 0; i < len / sizeof(u32); i++) { 2716 ctx->key[i] = cpu_to_le32(aes.key_enc[i]); 2717 ctx->ipad[i + 2 * AES_BLOCK_SIZE / sizeof(u32)] = 2718 cpu_to_be32(aes.key_enc[i]); 2719 } 2720 2721 ctx->key_len = len; 2722 ctx->state_sz = 2 * AES_BLOCK_SIZE + len; 2723 2724 if (len == AES_KEYSIZE_192) 2725 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192; 2726 else if (len == AES_KEYSIZE_256) 2727 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256; 2728 else 2729 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2730 2731 memzero_explicit(&aes, sizeof(aes)); 2732 return 0; 2733 } 2734 2735 static int safexcel_aead_ccm_cra_init(struct crypto_tfm *tfm) 2736 { 2737 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2738 2739 safexcel_aead_cra_init(tfm); 2740 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128; 2741 ctx->state_sz = 3 * AES_BLOCK_SIZE; 2742 ctx->xcm = EIP197_XCM_MODE_CCM; 2743 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_XCM; /* override default */ 2744 ctx->ctrinit = 0; 2745 return 0; 2746 } 2747 2748 static int safexcel_aead_ccm_setauthsize(struct crypto_aead *tfm, 2749 unsigned int authsize) 2750 { 2751 /* Borrowed from crypto/ccm.c */ 2752 switch (authsize) { 2753 case 4: 2754 case 6: 2755 case 8: 2756 case 10: 2757 case 12: 2758 case 14: 2759 case 16: 2760 break; 2761 default: 2762 return -EINVAL; 2763 } 2764 2765 return 0; 2766 } 2767 2768 static int safexcel_ccm_encrypt(struct aead_request *req) 2769 { 2770 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2771 2772 if (req->iv[0] < 1 || req->iv[0] > 7) 2773 return -EINVAL; 2774 2775 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 2776 } 2777 2778 static int safexcel_ccm_decrypt(struct aead_request *req) 2779 { 2780 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2781 2782 if (req->iv[0] < 1 || req->iv[0] > 7) 2783 return -EINVAL; 2784 2785 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 2786 } 2787 2788 struct safexcel_alg_template safexcel_alg_ccm = { 2789 .type = SAFEXCEL_ALG_TYPE_AEAD, 2790 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 2791 .alg.aead = { 2792 .setkey = safexcel_aead_ccm_setkey, 2793 .setauthsize = safexcel_aead_ccm_setauthsize, 2794 .encrypt = safexcel_ccm_encrypt, 2795 .decrypt = safexcel_ccm_decrypt, 2796 .ivsize = AES_BLOCK_SIZE, 2797 .maxauthsize = AES_BLOCK_SIZE, 2798 .base = { 2799 .cra_name = "ccm(aes)", 2800 .cra_driver_name = "safexcel-ccm-aes", 2801 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2802 .cra_flags = CRYPTO_ALG_ASYNC | 2803 CRYPTO_ALG_ALLOCATES_MEMORY | 2804 CRYPTO_ALG_KERN_DRIVER_ONLY, 2805 .cra_blocksize = 1, 2806 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2807 .cra_alignmask = 0, 2808 .cra_init = safexcel_aead_ccm_cra_init, 2809 .cra_exit = safexcel_aead_cra_exit, 2810 .cra_module = THIS_MODULE, 2811 }, 2812 }, 2813 }; 2814 2815 static void safexcel_chacha20_setkey(struct safexcel_cipher_ctx *ctx, 2816 const u8 *key) 2817 { 2818 struct safexcel_crypto_priv *priv = ctx->priv; 2819 2820 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 2821 if (memcmp(ctx->key, key, CHACHA_KEY_SIZE)) 2822 ctx->base.needs_inv = true; 2823 2824 memcpy(ctx->key, key, CHACHA_KEY_SIZE); 2825 ctx->key_len = CHACHA_KEY_SIZE; 2826 } 2827 2828 static int safexcel_skcipher_chacha20_setkey(struct crypto_skcipher *ctfm, 2829 const u8 *key, unsigned int len) 2830 { 2831 struct safexcel_cipher_ctx *ctx = crypto_skcipher_ctx(ctfm); 2832 2833 if (len != CHACHA_KEY_SIZE) 2834 return -EINVAL; 2835 2836 safexcel_chacha20_setkey(ctx, key); 2837 2838 return 0; 2839 } 2840 2841 static int safexcel_skcipher_chacha20_cra_init(struct crypto_tfm *tfm) 2842 { 2843 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2844 2845 safexcel_skcipher_cra_init(tfm); 2846 ctx->alg = SAFEXCEL_CHACHA20; 2847 ctx->ctrinit = 0; 2848 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32; 2849 return 0; 2850 } 2851 2852 struct safexcel_alg_template safexcel_alg_chacha20 = { 2853 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 2854 .algo_mask = SAFEXCEL_ALG_CHACHA20, 2855 .alg.skcipher = { 2856 .setkey = safexcel_skcipher_chacha20_setkey, 2857 .encrypt = safexcel_encrypt, 2858 .decrypt = safexcel_decrypt, 2859 .min_keysize = CHACHA_KEY_SIZE, 2860 .max_keysize = CHACHA_KEY_SIZE, 2861 .ivsize = CHACHA_IV_SIZE, 2862 .base = { 2863 .cra_name = "chacha20", 2864 .cra_driver_name = "safexcel-chacha20", 2865 .cra_priority = SAFEXCEL_CRA_PRIORITY, 2866 .cra_flags = CRYPTO_ALG_ASYNC | 2867 CRYPTO_ALG_ALLOCATES_MEMORY | 2868 CRYPTO_ALG_KERN_DRIVER_ONLY, 2869 .cra_blocksize = 1, 2870 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 2871 .cra_alignmask = 0, 2872 .cra_init = safexcel_skcipher_chacha20_cra_init, 2873 .cra_exit = safexcel_skcipher_cra_exit, 2874 .cra_module = THIS_MODULE, 2875 }, 2876 }, 2877 }; 2878 2879 static int safexcel_aead_chachapoly_setkey(struct crypto_aead *ctfm, 2880 const u8 *key, unsigned int len) 2881 { 2882 struct safexcel_cipher_ctx *ctx = crypto_aead_ctx(ctfm); 2883 2884 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP && 2885 len > EIP197_AEAD_IPSEC_NONCE_SIZE) { 2886 /* ESP variant has nonce appended to key */ 2887 len -= EIP197_AEAD_IPSEC_NONCE_SIZE; 2888 ctx->nonce = *(u32 *)(key + len); 2889 } 2890 if (len != CHACHA_KEY_SIZE) 2891 return -EINVAL; 2892 2893 safexcel_chacha20_setkey(ctx, key); 2894 2895 return 0; 2896 } 2897 2898 static int safexcel_aead_chachapoly_setauthsize(struct crypto_aead *tfm, 2899 unsigned int authsize) 2900 { 2901 if (authsize != POLY1305_DIGEST_SIZE) 2902 return -EINVAL; 2903 return 0; 2904 } 2905 2906 static int safexcel_aead_chachapoly_crypt(struct aead_request *req, 2907 enum safexcel_cipher_direction dir) 2908 { 2909 struct safexcel_cipher_req *creq = aead_request_ctx(req); 2910 struct crypto_aead *aead = crypto_aead_reqtfm(req); 2911 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 2912 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2913 struct aead_request *subreq = aead_request_ctx(req); 2914 u32 key[CHACHA_KEY_SIZE / sizeof(u32) + 1]; 2915 int ret = 0; 2916 2917 /* 2918 * Instead of wasting time detecting umpteen silly corner cases, 2919 * just dump all "small" requests to the fallback implementation. 2920 * HW would not be faster on such small requests anyway. 2921 */ 2922 if (likely((ctx->aead != EIP197_AEAD_TYPE_IPSEC_ESP || 2923 req->assoclen >= EIP197_AEAD_IPSEC_IV_SIZE) && 2924 req->cryptlen > POLY1305_DIGEST_SIZE)) { 2925 return safexcel_queue_req(&req->base, creq, dir); 2926 } 2927 2928 /* HW cannot do full (AAD+payload) zero length, use fallback */ 2929 memcpy(key, ctx->key, CHACHA_KEY_SIZE); 2930 if (ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) { 2931 /* ESP variant has nonce appended to the key */ 2932 key[CHACHA_KEY_SIZE / sizeof(u32)] = ctx->nonce; 2933 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2934 CHACHA_KEY_SIZE + 2935 EIP197_AEAD_IPSEC_NONCE_SIZE); 2936 } else { 2937 ret = crypto_aead_setkey(ctx->fback, (u8 *)key, 2938 CHACHA_KEY_SIZE); 2939 } 2940 if (ret) { 2941 crypto_aead_clear_flags(aead, CRYPTO_TFM_REQ_MASK); 2942 crypto_aead_set_flags(aead, crypto_aead_get_flags(ctx->fback) & 2943 CRYPTO_TFM_REQ_MASK); 2944 return ret; 2945 } 2946 2947 aead_request_set_tfm(subreq, ctx->fback); 2948 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 2949 req->base.data); 2950 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 2951 req->iv); 2952 aead_request_set_ad(subreq, req->assoclen); 2953 2954 return (dir == SAFEXCEL_ENCRYPT) ? 2955 crypto_aead_encrypt(subreq) : 2956 crypto_aead_decrypt(subreq); 2957 } 2958 2959 static int safexcel_aead_chachapoly_encrypt(struct aead_request *req) 2960 { 2961 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_ENCRYPT); 2962 } 2963 2964 static int safexcel_aead_chachapoly_decrypt(struct aead_request *req) 2965 { 2966 return safexcel_aead_chachapoly_crypt(req, SAFEXCEL_DECRYPT); 2967 } 2968 2969 static int safexcel_aead_fallback_cra_init(struct crypto_tfm *tfm) 2970 { 2971 struct crypto_aead *aead = __crypto_aead_cast(tfm); 2972 struct aead_alg *alg = crypto_aead_alg(aead); 2973 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2974 2975 safexcel_aead_cra_init(tfm); 2976 2977 /* Allocate fallback implementation */ 2978 ctx->fback = crypto_alloc_aead(alg->base.cra_name, 0, 2979 CRYPTO_ALG_ASYNC | 2980 CRYPTO_ALG_NEED_FALLBACK); 2981 if (IS_ERR(ctx->fback)) 2982 return PTR_ERR(ctx->fback); 2983 2984 crypto_aead_set_reqsize(aead, max(sizeof(struct safexcel_cipher_req), 2985 sizeof(struct aead_request) + 2986 crypto_aead_reqsize(ctx->fback))); 2987 2988 return 0; 2989 } 2990 2991 static int safexcel_aead_chachapoly_cra_init(struct crypto_tfm *tfm) 2992 { 2993 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 2994 2995 safexcel_aead_fallback_cra_init(tfm); 2996 ctx->alg = SAFEXCEL_CHACHA20; 2997 ctx->mode = CONTEXT_CONTROL_CHACHA20_MODE_256_32 | 2998 CONTEXT_CONTROL_CHACHA20_MODE_CALC_OTK; 2999 ctx->ctrinit = 0; 3000 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_POLY1305; 3001 ctx->state_sz = 0; /* Precomputed by HW */ 3002 return 0; 3003 } 3004 3005 static void safexcel_aead_fallback_cra_exit(struct crypto_tfm *tfm) 3006 { 3007 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3008 3009 crypto_free_aead(ctx->fback); 3010 safexcel_aead_cra_exit(tfm); 3011 } 3012 3013 struct safexcel_alg_template safexcel_alg_chachapoly = { 3014 .type = SAFEXCEL_ALG_TYPE_AEAD, 3015 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 3016 .alg.aead = { 3017 .setkey = safexcel_aead_chachapoly_setkey, 3018 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3019 .encrypt = safexcel_aead_chachapoly_encrypt, 3020 .decrypt = safexcel_aead_chachapoly_decrypt, 3021 .ivsize = CHACHAPOLY_IV_SIZE, 3022 .maxauthsize = POLY1305_DIGEST_SIZE, 3023 .base = { 3024 .cra_name = "rfc7539(chacha20,poly1305)", 3025 .cra_driver_name = "safexcel-chacha20-poly1305", 3026 /* +1 to put it above HW chacha + SW poly */ 3027 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3028 .cra_flags = CRYPTO_ALG_ASYNC | 3029 CRYPTO_ALG_ALLOCATES_MEMORY | 3030 CRYPTO_ALG_KERN_DRIVER_ONLY | 3031 CRYPTO_ALG_NEED_FALLBACK, 3032 .cra_blocksize = 1, 3033 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3034 .cra_alignmask = 0, 3035 .cra_init = safexcel_aead_chachapoly_cra_init, 3036 .cra_exit = safexcel_aead_fallback_cra_exit, 3037 .cra_module = THIS_MODULE, 3038 }, 3039 }, 3040 }; 3041 3042 static int safexcel_aead_chachapolyesp_cra_init(struct crypto_tfm *tfm) 3043 { 3044 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3045 int ret; 3046 3047 ret = safexcel_aead_chachapoly_cra_init(tfm); 3048 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3049 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3050 return ret; 3051 } 3052 3053 struct safexcel_alg_template safexcel_alg_chachapoly_esp = { 3054 .type = SAFEXCEL_ALG_TYPE_AEAD, 3055 .algo_mask = SAFEXCEL_ALG_CHACHA20 | SAFEXCEL_ALG_POLY1305, 3056 .alg.aead = { 3057 .setkey = safexcel_aead_chachapoly_setkey, 3058 .setauthsize = safexcel_aead_chachapoly_setauthsize, 3059 .encrypt = safexcel_aead_chachapoly_encrypt, 3060 .decrypt = safexcel_aead_chachapoly_decrypt, 3061 .ivsize = CHACHAPOLY_IV_SIZE - EIP197_AEAD_IPSEC_NONCE_SIZE, 3062 .maxauthsize = POLY1305_DIGEST_SIZE, 3063 .base = { 3064 .cra_name = "rfc7539esp(chacha20,poly1305)", 3065 .cra_driver_name = "safexcel-chacha20-poly1305-esp", 3066 /* +1 to put it above HW chacha + SW poly */ 3067 .cra_priority = SAFEXCEL_CRA_PRIORITY + 1, 3068 .cra_flags = CRYPTO_ALG_ASYNC | 3069 CRYPTO_ALG_ALLOCATES_MEMORY | 3070 CRYPTO_ALG_KERN_DRIVER_ONLY | 3071 CRYPTO_ALG_NEED_FALLBACK, 3072 .cra_blocksize = 1, 3073 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3074 .cra_alignmask = 0, 3075 .cra_init = safexcel_aead_chachapolyesp_cra_init, 3076 .cra_exit = safexcel_aead_fallback_cra_exit, 3077 .cra_module = THIS_MODULE, 3078 }, 3079 }, 3080 }; 3081 3082 static int safexcel_skcipher_sm4_setkey(struct crypto_skcipher *ctfm, 3083 const u8 *key, unsigned int len) 3084 { 3085 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3086 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3087 struct safexcel_crypto_priv *priv = ctx->priv; 3088 3089 if (len != SM4_KEY_SIZE) 3090 return -EINVAL; 3091 3092 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) 3093 if (memcmp(ctx->key, key, SM4_KEY_SIZE)) 3094 ctx->base.needs_inv = true; 3095 3096 memcpy(ctx->key, key, SM4_KEY_SIZE); 3097 ctx->key_len = SM4_KEY_SIZE; 3098 3099 return 0; 3100 } 3101 3102 static int safexcel_sm4_blk_encrypt(struct skcipher_request *req) 3103 { 3104 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3105 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3106 return -EINVAL; 3107 else 3108 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3109 SAFEXCEL_ENCRYPT); 3110 } 3111 3112 static int safexcel_sm4_blk_decrypt(struct skcipher_request *req) 3113 { 3114 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3115 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3116 return -EINVAL; 3117 else 3118 return safexcel_queue_req(&req->base, skcipher_request_ctx(req), 3119 SAFEXCEL_DECRYPT); 3120 } 3121 3122 static int safexcel_skcipher_sm4_ecb_cra_init(struct crypto_tfm *tfm) 3123 { 3124 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3125 3126 safexcel_skcipher_cra_init(tfm); 3127 ctx->alg = SAFEXCEL_SM4; 3128 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_ECB; 3129 ctx->blocksz = 0; 3130 ctx->ivmask = EIP197_OPTION_2_TOKEN_IV_CMD; 3131 return 0; 3132 } 3133 3134 struct safexcel_alg_template safexcel_alg_ecb_sm4 = { 3135 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3136 .algo_mask = SAFEXCEL_ALG_SM4, 3137 .alg.skcipher = { 3138 .setkey = safexcel_skcipher_sm4_setkey, 3139 .encrypt = safexcel_sm4_blk_encrypt, 3140 .decrypt = safexcel_sm4_blk_decrypt, 3141 .min_keysize = SM4_KEY_SIZE, 3142 .max_keysize = SM4_KEY_SIZE, 3143 .base = { 3144 .cra_name = "ecb(sm4)", 3145 .cra_driver_name = "safexcel-ecb-sm4", 3146 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3147 .cra_flags = CRYPTO_ALG_ASYNC | 3148 CRYPTO_ALG_ALLOCATES_MEMORY | 3149 CRYPTO_ALG_KERN_DRIVER_ONLY, 3150 .cra_blocksize = SM4_BLOCK_SIZE, 3151 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3152 .cra_alignmask = 0, 3153 .cra_init = safexcel_skcipher_sm4_ecb_cra_init, 3154 .cra_exit = safexcel_skcipher_cra_exit, 3155 .cra_module = THIS_MODULE, 3156 }, 3157 }, 3158 }; 3159 3160 static int safexcel_skcipher_sm4_cbc_cra_init(struct crypto_tfm *tfm) 3161 { 3162 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3163 3164 safexcel_skcipher_cra_init(tfm); 3165 ctx->alg = SAFEXCEL_SM4; 3166 ctx->blocksz = SM4_BLOCK_SIZE; 3167 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CBC; 3168 return 0; 3169 } 3170 3171 struct safexcel_alg_template safexcel_alg_cbc_sm4 = { 3172 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3173 .algo_mask = SAFEXCEL_ALG_SM4, 3174 .alg.skcipher = { 3175 .setkey = safexcel_skcipher_sm4_setkey, 3176 .encrypt = safexcel_sm4_blk_encrypt, 3177 .decrypt = safexcel_sm4_blk_decrypt, 3178 .min_keysize = SM4_KEY_SIZE, 3179 .max_keysize = SM4_KEY_SIZE, 3180 .ivsize = SM4_BLOCK_SIZE, 3181 .base = { 3182 .cra_name = "cbc(sm4)", 3183 .cra_driver_name = "safexcel-cbc-sm4", 3184 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3185 .cra_flags = CRYPTO_ALG_ASYNC | 3186 CRYPTO_ALG_ALLOCATES_MEMORY | 3187 CRYPTO_ALG_KERN_DRIVER_ONLY, 3188 .cra_blocksize = SM4_BLOCK_SIZE, 3189 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3190 .cra_alignmask = 0, 3191 .cra_init = safexcel_skcipher_sm4_cbc_cra_init, 3192 .cra_exit = safexcel_skcipher_cra_exit, 3193 .cra_module = THIS_MODULE, 3194 }, 3195 }, 3196 }; 3197 3198 static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) 3199 { 3200 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3201 3202 safexcel_skcipher_cra_init(tfm); 3203 ctx->alg = SAFEXCEL_SM4; 3204 ctx->blocksz = SM4_BLOCK_SIZE; 3205 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; 3206 return 0; 3207 } 3208 3209 struct safexcel_alg_template safexcel_alg_ofb_sm4 = { 3210 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3211 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3212 .alg.skcipher = { 3213 .setkey = safexcel_skcipher_sm4_setkey, 3214 .encrypt = safexcel_encrypt, 3215 .decrypt = safexcel_decrypt, 3216 .min_keysize = SM4_KEY_SIZE, 3217 .max_keysize = SM4_KEY_SIZE, 3218 .ivsize = SM4_BLOCK_SIZE, 3219 .base = { 3220 .cra_name = "ofb(sm4)", 3221 .cra_driver_name = "safexcel-ofb-sm4", 3222 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3223 .cra_flags = CRYPTO_ALG_ASYNC | 3224 CRYPTO_ALG_ALLOCATES_MEMORY | 3225 CRYPTO_ALG_KERN_DRIVER_ONLY, 3226 .cra_blocksize = 1, 3227 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3228 .cra_alignmask = 0, 3229 .cra_init = safexcel_skcipher_sm4_ofb_cra_init, 3230 .cra_exit = safexcel_skcipher_cra_exit, 3231 .cra_module = THIS_MODULE, 3232 }, 3233 }, 3234 }; 3235 3236 static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) 3237 { 3238 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3239 3240 safexcel_skcipher_cra_init(tfm); 3241 ctx->alg = SAFEXCEL_SM4; 3242 ctx->blocksz = SM4_BLOCK_SIZE; 3243 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; 3244 return 0; 3245 } 3246 3247 struct safexcel_alg_template safexcel_alg_cfb_sm4 = { 3248 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3249 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, 3250 .alg.skcipher = { 3251 .setkey = safexcel_skcipher_sm4_setkey, 3252 .encrypt = safexcel_encrypt, 3253 .decrypt = safexcel_decrypt, 3254 .min_keysize = SM4_KEY_SIZE, 3255 .max_keysize = SM4_KEY_SIZE, 3256 .ivsize = SM4_BLOCK_SIZE, 3257 .base = { 3258 .cra_name = "cfb(sm4)", 3259 .cra_driver_name = "safexcel-cfb-sm4", 3260 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3261 .cra_flags = CRYPTO_ALG_ASYNC | 3262 CRYPTO_ALG_ALLOCATES_MEMORY | 3263 CRYPTO_ALG_KERN_DRIVER_ONLY, 3264 .cra_blocksize = 1, 3265 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3266 .cra_alignmask = 0, 3267 .cra_init = safexcel_skcipher_sm4_cfb_cra_init, 3268 .cra_exit = safexcel_skcipher_cra_exit, 3269 .cra_module = THIS_MODULE, 3270 }, 3271 }, 3272 }; 3273 3274 static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, 3275 const u8 *key, unsigned int len) 3276 { 3277 struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); 3278 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3279 3280 /* last 4 bytes of key are the nonce! */ 3281 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3282 /* exclude the nonce here */ 3283 len -= CTR_RFC3686_NONCE_SIZE; 3284 3285 return safexcel_skcipher_sm4_setkey(ctfm, key, len); 3286 } 3287 3288 static int safexcel_skcipher_sm4_ctr_cra_init(struct crypto_tfm *tfm) 3289 { 3290 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3291 3292 safexcel_skcipher_cra_init(tfm); 3293 ctx->alg = SAFEXCEL_SM4; 3294 ctx->blocksz = SM4_BLOCK_SIZE; 3295 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3296 return 0; 3297 } 3298 3299 struct safexcel_alg_template safexcel_alg_ctr_sm4 = { 3300 .type = SAFEXCEL_ALG_TYPE_SKCIPHER, 3301 .algo_mask = SAFEXCEL_ALG_SM4, 3302 .alg.skcipher = { 3303 .setkey = safexcel_skcipher_sm4ctr_setkey, 3304 .encrypt = safexcel_encrypt, 3305 .decrypt = safexcel_decrypt, 3306 /* Add nonce size */ 3307 .min_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3308 .max_keysize = SM4_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, 3309 .ivsize = CTR_RFC3686_IV_SIZE, 3310 .base = { 3311 .cra_name = "rfc3686(ctr(sm4))", 3312 .cra_driver_name = "safexcel-ctr-sm4", 3313 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3314 .cra_flags = CRYPTO_ALG_ASYNC | 3315 CRYPTO_ALG_ALLOCATES_MEMORY | 3316 CRYPTO_ALG_KERN_DRIVER_ONLY, 3317 .cra_blocksize = 1, 3318 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3319 .cra_alignmask = 0, 3320 .cra_init = safexcel_skcipher_sm4_ctr_cra_init, 3321 .cra_exit = safexcel_skcipher_cra_exit, 3322 .cra_module = THIS_MODULE, 3323 }, 3324 }, 3325 }; 3326 3327 static int safexcel_aead_sm4_blk_encrypt(struct aead_request *req) 3328 { 3329 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3330 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3331 return -EINVAL; 3332 3333 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3334 SAFEXCEL_ENCRYPT); 3335 } 3336 3337 static int safexcel_aead_sm4_blk_decrypt(struct aead_request *req) 3338 { 3339 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3340 3341 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3342 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3343 return -EINVAL; 3344 3345 return safexcel_queue_req(&req->base, aead_request_ctx(req), 3346 SAFEXCEL_DECRYPT); 3347 } 3348 3349 static int safexcel_aead_sm4cbc_sha1_cra_init(struct crypto_tfm *tfm) 3350 { 3351 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3352 3353 safexcel_aead_cra_init(tfm); 3354 ctx->alg = SAFEXCEL_SM4; 3355 ctx->blocksz = SM4_BLOCK_SIZE; 3356 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1; 3357 ctx->state_sz = SHA1_DIGEST_SIZE; 3358 return 0; 3359 } 3360 3361 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4 = { 3362 .type = SAFEXCEL_ALG_TYPE_AEAD, 3363 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3364 .alg.aead = { 3365 .setkey = safexcel_aead_setkey, 3366 .encrypt = safexcel_aead_sm4_blk_encrypt, 3367 .decrypt = safexcel_aead_sm4_blk_decrypt, 3368 .ivsize = SM4_BLOCK_SIZE, 3369 .maxauthsize = SHA1_DIGEST_SIZE, 3370 .base = { 3371 .cra_name = "authenc(hmac(sha1),cbc(sm4))", 3372 .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-sm4", 3373 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3374 .cra_flags = CRYPTO_ALG_ASYNC | 3375 CRYPTO_ALG_ALLOCATES_MEMORY | 3376 CRYPTO_ALG_KERN_DRIVER_ONLY, 3377 .cra_blocksize = SM4_BLOCK_SIZE, 3378 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3379 .cra_alignmask = 0, 3380 .cra_init = safexcel_aead_sm4cbc_sha1_cra_init, 3381 .cra_exit = safexcel_aead_cra_exit, 3382 .cra_module = THIS_MODULE, 3383 }, 3384 }, 3385 }; 3386 3387 static int safexcel_aead_fallback_setkey(struct crypto_aead *ctfm, 3388 const u8 *key, unsigned int len) 3389 { 3390 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3391 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3392 3393 /* Keep fallback cipher synchronized */ 3394 return crypto_aead_setkey(ctx->fback, (u8 *)key, len) ?: 3395 safexcel_aead_setkey(ctfm, key, len); 3396 } 3397 3398 static int safexcel_aead_fallback_setauthsize(struct crypto_aead *ctfm, 3399 unsigned int authsize) 3400 { 3401 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3402 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3403 3404 /* Keep fallback cipher synchronized */ 3405 return crypto_aead_setauthsize(ctx->fback, authsize); 3406 } 3407 3408 static int safexcel_aead_fallback_crypt(struct aead_request *req, 3409 enum safexcel_cipher_direction dir) 3410 { 3411 struct crypto_aead *aead = crypto_aead_reqtfm(req); 3412 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 3413 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3414 struct aead_request *subreq = aead_request_ctx(req); 3415 3416 aead_request_set_tfm(subreq, ctx->fback); 3417 aead_request_set_callback(subreq, req->base.flags, req->base.complete, 3418 req->base.data); 3419 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, 3420 req->iv); 3421 aead_request_set_ad(subreq, req->assoclen); 3422 3423 return (dir == SAFEXCEL_ENCRYPT) ? 3424 crypto_aead_encrypt(subreq) : 3425 crypto_aead_decrypt(subreq); 3426 } 3427 3428 static int safexcel_aead_sm4cbc_sm3_encrypt(struct aead_request *req) 3429 { 3430 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3431 3432 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3433 if (req->cryptlen & (SM4_BLOCK_SIZE - 1)) 3434 return -EINVAL; 3435 else if (req->cryptlen || req->assoclen) /* If input length > 0 only */ 3436 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3437 3438 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3439 return safexcel_aead_fallback_crypt(req, SAFEXCEL_ENCRYPT); 3440 } 3441 3442 static int safexcel_aead_sm4cbc_sm3_decrypt(struct aead_request *req) 3443 { 3444 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3445 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3446 3447 /* Workaround for HW bug: EIP96 4.3 does not report blocksize error */ 3448 if ((req->cryptlen - crypto_aead_authsize(tfm)) & (SM4_BLOCK_SIZE - 1)) 3449 return -EINVAL; 3450 else if (req->cryptlen > crypto_aead_authsize(tfm) || req->assoclen) 3451 /* If input length > 0 only */ 3452 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3453 3454 /* HW cannot do full (AAD+payload) zero length, use fallback */ 3455 return safexcel_aead_fallback_crypt(req, SAFEXCEL_DECRYPT); 3456 } 3457 3458 static int safexcel_aead_sm4cbc_sm3_cra_init(struct crypto_tfm *tfm) 3459 { 3460 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3461 3462 safexcel_aead_fallback_cra_init(tfm); 3463 ctx->alg = SAFEXCEL_SM4; 3464 ctx->blocksz = SM4_BLOCK_SIZE; 3465 ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3; 3466 ctx->state_sz = SM3_DIGEST_SIZE; 3467 return 0; 3468 } 3469 3470 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4 = { 3471 .type = SAFEXCEL_ALG_TYPE_AEAD, 3472 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3473 .alg.aead = { 3474 .setkey = safexcel_aead_fallback_setkey, 3475 .setauthsize = safexcel_aead_fallback_setauthsize, 3476 .encrypt = safexcel_aead_sm4cbc_sm3_encrypt, 3477 .decrypt = safexcel_aead_sm4cbc_sm3_decrypt, 3478 .ivsize = SM4_BLOCK_SIZE, 3479 .maxauthsize = SM3_DIGEST_SIZE, 3480 .base = { 3481 .cra_name = "authenc(hmac(sm3),cbc(sm4))", 3482 .cra_driver_name = "safexcel-authenc-hmac-sm3-cbc-sm4", 3483 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3484 .cra_flags = CRYPTO_ALG_ASYNC | 3485 CRYPTO_ALG_ALLOCATES_MEMORY | 3486 CRYPTO_ALG_KERN_DRIVER_ONLY | 3487 CRYPTO_ALG_NEED_FALLBACK, 3488 .cra_blocksize = SM4_BLOCK_SIZE, 3489 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3490 .cra_alignmask = 0, 3491 .cra_init = safexcel_aead_sm4cbc_sm3_cra_init, 3492 .cra_exit = safexcel_aead_fallback_cra_exit, 3493 .cra_module = THIS_MODULE, 3494 }, 3495 }, 3496 }; 3497 3498 static int safexcel_aead_sm4ctr_sha1_cra_init(struct crypto_tfm *tfm) 3499 { 3500 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3501 3502 safexcel_aead_sm4cbc_sha1_cra_init(tfm); 3503 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3504 return 0; 3505 } 3506 3507 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_ctr_sm4 = { 3508 .type = SAFEXCEL_ALG_TYPE_AEAD, 3509 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SHA1, 3510 .alg.aead = { 3511 .setkey = safexcel_aead_setkey, 3512 .encrypt = safexcel_aead_encrypt, 3513 .decrypt = safexcel_aead_decrypt, 3514 .ivsize = CTR_RFC3686_IV_SIZE, 3515 .maxauthsize = SHA1_DIGEST_SIZE, 3516 .base = { 3517 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(sm4)))", 3518 .cra_driver_name = "safexcel-authenc-hmac-sha1-ctr-sm4", 3519 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3520 .cra_flags = CRYPTO_ALG_ASYNC | 3521 CRYPTO_ALG_ALLOCATES_MEMORY | 3522 CRYPTO_ALG_KERN_DRIVER_ONLY, 3523 .cra_blocksize = 1, 3524 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3525 .cra_alignmask = 0, 3526 .cra_init = safexcel_aead_sm4ctr_sha1_cra_init, 3527 .cra_exit = safexcel_aead_cra_exit, 3528 .cra_module = THIS_MODULE, 3529 }, 3530 }, 3531 }; 3532 3533 static int safexcel_aead_sm4ctr_sm3_cra_init(struct crypto_tfm *tfm) 3534 { 3535 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3536 3537 safexcel_aead_sm4cbc_sm3_cra_init(tfm); 3538 ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CTR_LOAD; 3539 return 0; 3540 } 3541 3542 struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_ctr_sm4 = { 3543 .type = SAFEXCEL_ALG_TYPE_AEAD, 3544 .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_SM3, 3545 .alg.aead = { 3546 .setkey = safexcel_aead_setkey, 3547 .encrypt = safexcel_aead_encrypt, 3548 .decrypt = safexcel_aead_decrypt, 3549 .ivsize = CTR_RFC3686_IV_SIZE, 3550 .maxauthsize = SM3_DIGEST_SIZE, 3551 .base = { 3552 .cra_name = "authenc(hmac(sm3),rfc3686(ctr(sm4)))", 3553 .cra_driver_name = "safexcel-authenc-hmac-sm3-ctr-sm4", 3554 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3555 .cra_flags = CRYPTO_ALG_ASYNC | 3556 CRYPTO_ALG_ALLOCATES_MEMORY | 3557 CRYPTO_ALG_KERN_DRIVER_ONLY, 3558 .cra_blocksize = 1, 3559 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3560 .cra_alignmask = 0, 3561 .cra_init = safexcel_aead_sm4ctr_sm3_cra_init, 3562 .cra_exit = safexcel_aead_cra_exit, 3563 .cra_module = THIS_MODULE, 3564 }, 3565 }, 3566 }; 3567 3568 static int safexcel_rfc4106_gcm_setkey(struct crypto_aead *ctfm, const u8 *key, 3569 unsigned int len) 3570 { 3571 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3572 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3573 3574 /* last 4 bytes of key are the nonce! */ 3575 ctx->nonce = *(u32 *)(key + len - CTR_RFC3686_NONCE_SIZE); 3576 3577 len -= CTR_RFC3686_NONCE_SIZE; 3578 return safexcel_aead_gcm_setkey(ctfm, key, len); 3579 } 3580 3581 static int safexcel_rfc4106_gcm_setauthsize(struct crypto_aead *tfm, 3582 unsigned int authsize) 3583 { 3584 return crypto_rfc4106_check_authsize(authsize); 3585 } 3586 3587 static int safexcel_rfc4106_encrypt(struct aead_request *req) 3588 { 3589 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3590 safexcel_aead_encrypt(req); 3591 } 3592 3593 static int safexcel_rfc4106_decrypt(struct aead_request *req) 3594 { 3595 return crypto_ipsec_check_assoclen(req->assoclen) ?: 3596 safexcel_aead_decrypt(req); 3597 } 3598 3599 static int safexcel_rfc4106_gcm_cra_init(struct crypto_tfm *tfm) 3600 { 3601 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3602 int ret; 3603 3604 ret = safexcel_aead_gcm_cra_init(tfm); 3605 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3606 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3607 return ret; 3608 } 3609 3610 struct safexcel_alg_template safexcel_alg_rfc4106_gcm = { 3611 .type = SAFEXCEL_ALG_TYPE_AEAD, 3612 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3613 .alg.aead = { 3614 .setkey = safexcel_rfc4106_gcm_setkey, 3615 .setauthsize = safexcel_rfc4106_gcm_setauthsize, 3616 .encrypt = safexcel_rfc4106_encrypt, 3617 .decrypt = safexcel_rfc4106_decrypt, 3618 .ivsize = GCM_RFC4106_IV_SIZE, 3619 .maxauthsize = GHASH_DIGEST_SIZE, 3620 .base = { 3621 .cra_name = "rfc4106(gcm(aes))", 3622 .cra_driver_name = "safexcel-rfc4106-gcm-aes", 3623 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3624 .cra_flags = CRYPTO_ALG_ASYNC | 3625 CRYPTO_ALG_ALLOCATES_MEMORY | 3626 CRYPTO_ALG_KERN_DRIVER_ONLY, 3627 .cra_blocksize = 1, 3628 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3629 .cra_alignmask = 0, 3630 .cra_init = safexcel_rfc4106_gcm_cra_init, 3631 .cra_exit = safexcel_aead_gcm_cra_exit, 3632 }, 3633 }, 3634 }; 3635 3636 static int safexcel_rfc4543_gcm_setauthsize(struct crypto_aead *tfm, 3637 unsigned int authsize) 3638 { 3639 if (authsize != GHASH_DIGEST_SIZE) 3640 return -EINVAL; 3641 3642 return 0; 3643 } 3644 3645 static int safexcel_rfc4543_gcm_cra_init(struct crypto_tfm *tfm) 3646 { 3647 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3648 int ret; 3649 3650 ret = safexcel_aead_gcm_cra_init(tfm); 3651 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP_GMAC; 3652 return ret; 3653 } 3654 3655 struct safexcel_alg_template safexcel_alg_rfc4543_gcm = { 3656 .type = SAFEXCEL_ALG_TYPE_AEAD, 3657 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_GHASH, 3658 .alg.aead = { 3659 .setkey = safexcel_rfc4106_gcm_setkey, 3660 .setauthsize = safexcel_rfc4543_gcm_setauthsize, 3661 .encrypt = safexcel_rfc4106_encrypt, 3662 .decrypt = safexcel_rfc4106_decrypt, 3663 .ivsize = GCM_RFC4543_IV_SIZE, 3664 .maxauthsize = GHASH_DIGEST_SIZE, 3665 .base = { 3666 .cra_name = "rfc4543(gcm(aes))", 3667 .cra_driver_name = "safexcel-rfc4543-gcm-aes", 3668 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3669 .cra_flags = CRYPTO_ALG_ASYNC | 3670 CRYPTO_ALG_ALLOCATES_MEMORY | 3671 CRYPTO_ALG_KERN_DRIVER_ONLY, 3672 .cra_blocksize = 1, 3673 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3674 .cra_alignmask = 0, 3675 .cra_init = safexcel_rfc4543_gcm_cra_init, 3676 .cra_exit = safexcel_aead_gcm_cra_exit, 3677 }, 3678 }, 3679 }; 3680 3681 static int safexcel_rfc4309_ccm_setkey(struct crypto_aead *ctfm, const u8 *key, 3682 unsigned int len) 3683 { 3684 struct crypto_tfm *tfm = crypto_aead_tfm(ctfm); 3685 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3686 3687 /* First byte of the nonce = L = always 3 for RFC4309 (4 byte ctr) */ 3688 *(u8 *)&ctx->nonce = EIP197_AEAD_IPSEC_COUNTER_SIZE - 1; 3689 /* last 3 bytes of key are the nonce! */ 3690 memcpy((u8 *)&ctx->nonce + 1, key + len - 3691 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE, 3692 EIP197_AEAD_IPSEC_CCM_NONCE_SIZE); 3693 3694 len -= EIP197_AEAD_IPSEC_CCM_NONCE_SIZE; 3695 return safexcel_aead_ccm_setkey(ctfm, key, len); 3696 } 3697 3698 static int safexcel_rfc4309_ccm_setauthsize(struct crypto_aead *tfm, 3699 unsigned int authsize) 3700 { 3701 /* Borrowed from crypto/ccm.c */ 3702 switch (authsize) { 3703 case 8: 3704 case 12: 3705 case 16: 3706 break; 3707 default: 3708 return -EINVAL; 3709 } 3710 3711 return 0; 3712 } 3713 3714 static int safexcel_rfc4309_ccm_encrypt(struct aead_request *req) 3715 { 3716 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3717 3718 /* Borrowed from crypto/ccm.c */ 3719 if (req->assoclen != 16 && req->assoclen != 20) 3720 return -EINVAL; 3721 3722 return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT); 3723 } 3724 3725 static int safexcel_rfc4309_ccm_decrypt(struct aead_request *req) 3726 { 3727 struct safexcel_cipher_req *creq = aead_request_ctx(req); 3728 3729 /* Borrowed from crypto/ccm.c */ 3730 if (req->assoclen != 16 && req->assoclen != 20) 3731 return -EINVAL; 3732 3733 return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT); 3734 } 3735 3736 static int safexcel_rfc4309_ccm_cra_init(struct crypto_tfm *tfm) 3737 { 3738 struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); 3739 int ret; 3740 3741 ret = safexcel_aead_ccm_cra_init(tfm); 3742 ctx->aead = EIP197_AEAD_TYPE_IPSEC_ESP; 3743 ctx->aadskip = EIP197_AEAD_IPSEC_IV_SIZE; 3744 return ret; 3745 } 3746 3747 struct safexcel_alg_template safexcel_alg_rfc4309_ccm = { 3748 .type = SAFEXCEL_ALG_TYPE_AEAD, 3749 .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_CBC_MAC_ALL, 3750 .alg.aead = { 3751 .setkey = safexcel_rfc4309_ccm_setkey, 3752 .setauthsize = safexcel_rfc4309_ccm_setauthsize, 3753 .encrypt = safexcel_rfc4309_ccm_encrypt, 3754 .decrypt = safexcel_rfc4309_ccm_decrypt, 3755 .ivsize = EIP197_AEAD_IPSEC_IV_SIZE, 3756 .maxauthsize = AES_BLOCK_SIZE, 3757 .base = { 3758 .cra_name = "rfc4309(ccm(aes))", 3759 .cra_driver_name = "safexcel-rfc4309-ccm-aes", 3760 .cra_priority = SAFEXCEL_CRA_PRIORITY, 3761 .cra_flags = CRYPTO_ALG_ASYNC | 3762 CRYPTO_ALG_ALLOCATES_MEMORY | 3763 CRYPTO_ALG_KERN_DRIVER_ONLY, 3764 .cra_blocksize = 1, 3765 .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), 3766 .cra_alignmask = 0, 3767 .cra_init = safexcel_rfc4309_ccm_cra_init, 3768 .cra_exit = safexcel_aead_cra_exit, 3769 .cra_module = THIS_MODULE, 3770 }, 3771 }, 3772 }; 3773