forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1271 lines
32 KiB
1271 lines
32 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* Cryptographic API. |
|
* |
|
* Driver for EIP97 AES acceleration. |
|
* |
|
* Copyright (c) 2016 Ryder Lee <[email protected]> |
|
* |
|
* Some ideas are from atmel-aes.c drivers. |
|
*/ |
|
|
|
#include <crypto/aes.h> |
|
#include <crypto/gcm.h> |
|
#include <crypto/internal/skcipher.h> |
|
#include "mtk-platform.h" |
|
|
|
#define AES_QUEUE_SIZE 512 |
|
#define AES_BUF_ORDER 2 |
|
#define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \ |
|
& ~(AES_BLOCK_SIZE - 1)) |
|
#define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \ |
|
AES_BLOCK_SIZE * 2) |
|
#define AES_MAX_CT_SIZE 6 |
|
|
|
#define AES_CT_CTRL_HDR cpu_to_le32(0x00220000) |
|
|
|
/* AES-CBC/ECB/CTR/OFB/CFB command token */ |
|
#define AES_CMD0 cpu_to_le32(0x05000000) |
|
#define AES_CMD1 cpu_to_le32(0x2d060000) |
|
#define AES_CMD2 cpu_to_le32(0xe4a63806) |
|
/* AES-GCM command token */ |
|
#define AES_GCM_CMD0 cpu_to_le32(0x0b000000) |
|
#define AES_GCM_CMD1 cpu_to_le32(0xa0800000) |
|
#define AES_GCM_CMD2 cpu_to_le32(0x25000010) |
|
#define AES_GCM_CMD3 cpu_to_le32(0x0f020000) |
|
#define AES_GCM_CMD4 cpu_to_le32(0x21e60000) |
|
#define AES_GCM_CMD5 cpu_to_le32(0x40e60000) |
|
#define AES_GCM_CMD6 cpu_to_le32(0xd0070000) |
|
|
|
/* AES transform information word 0 fields */ |
|
#define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0) |
|
#define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0) |
|
#define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0) |
|
#define AES_TFM_GCM_IN cpu_to_le32(0xf << 0) |
|
#define AES_TFM_SIZE(x) cpu_to_le32((x) << 8) |
|
#define AES_TFM_128BITS cpu_to_le32(0xb << 16) |
|
#define AES_TFM_192BITS cpu_to_le32(0xd << 16) |
|
#define AES_TFM_256BITS cpu_to_le32(0xf << 16) |
|
#define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21) |
|
#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) |
|
/* AES transform information word 1 fields */ |
|
#define AES_TFM_ECB cpu_to_le32(0x0 << 0) |
|
#define AES_TFM_CBC cpu_to_le32(0x1 << 0) |
|
#define AES_TFM_OFB cpu_to_le32(0x4 << 0) |
|
#define AES_TFM_CFB128 cpu_to_le32(0x5 << 0) |
|
#define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */ |
|
#define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */ |
|
#define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */ |
|
#define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */ |
|
#define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10) |
|
#define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17) |
|
|
|
/* AES flags */ |
|
#define AES_FLAGS_CIPHER_MSK GENMASK(4, 0) |
|
#define AES_FLAGS_ECB BIT(0) |
|
#define AES_FLAGS_CBC BIT(1) |
|
#define AES_FLAGS_CTR BIT(2) |
|
#define AES_FLAGS_OFB BIT(3) |
|
#define AES_FLAGS_CFB128 BIT(4) |
|
#define AES_FLAGS_GCM BIT(5) |
|
#define AES_FLAGS_ENCRYPT BIT(6) |
|
#define AES_FLAGS_BUSY BIT(7) |
|
|
|
#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26)) |
|
|
|
/** |
|
* mtk_aes_info - hardware information of AES |
|
* @cmd: command token, hardware instruction |
|
* @tfm: transform state of cipher algorithm. |
|
* @state: contains keys and initial vectors. |
|
* |
|
* Memory layout of GCM buffer: |
|
* /-----------\ |
|
* | AES KEY | 128/196/256 bits |
|
* |-----------| |
|
* | HASH KEY | a string 128 zero bits encrypted using the block cipher |
|
* |-----------| |
|
* | IVs | 4 * 4 bytes |
|
* \-----------/ |
|
* |
|
* The engine requires all these info to do: |
|
* - Commands decoding and control of the engine's data path. |
|
* - Coordinating hardware data fetch and store operations. |
|
* - Result token construction and output. |
|
*/ |
|
struct mtk_aes_info { |
|
__le32 cmd[AES_MAX_CT_SIZE]; |
|
__le32 tfm[2]; |
|
__le32 state[AES_MAX_STATE_BUF_SIZE]; |
|
}; |
|
|
|
struct mtk_aes_reqctx { |
|
u64 mode; |
|
}; |
|
|
|
struct mtk_aes_base_ctx { |
|
struct mtk_cryp *cryp; |
|
u32 keylen; |
|
__le32 key[12]; |
|
__le32 keymode; |
|
|
|
mtk_aes_fn start; |
|
|
|
struct mtk_aes_info info; |
|
dma_addr_t ct_dma; |
|
dma_addr_t tfm_dma; |
|
|
|
__le32 ct_hdr; |
|
u32 ct_size; |
|
}; |
|
|
|
struct mtk_aes_ctx { |
|
struct mtk_aes_base_ctx base; |
|
}; |
|
|
|
struct mtk_aes_ctr_ctx { |
|
struct mtk_aes_base_ctx base; |
|
|
|
__be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; |
|
size_t offset; |
|
struct scatterlist src[2]; |
|
struct scatterlist dst[2]; |
|
}; |
|
|
|
struct mtk_aes_gcm_ctx { |
|
struct mtk_aes_base_ctx base; |
|
|
|
u32 authsize; |
|
size_t textlen; |
|
}; |
|
|
|
struct mtk_aes_drv { |
|
struct list_head dev_list; |
|
/* Device list lock */ |
|
spinlock_t lock; |
|
}; |
|
|
|
static struct mtk_aes_drv mtk_aes = { |
|
.dev_list = LIST_HEAD_INIT(mtk_aes.dev_list), |
|
.lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock), |
|
}; |
|
|
|
static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset) |
|
{ |
|
return readl_relaxed(cryp->base + offset); |
|
} |
|
|
|
static inline void mtk_aes_write(struct mtk_cryp *cryp, |
|
u32 offset, u32 value) |
|
{ |
|
writel_relaxed(value, cryp->base + offset); |
|
} |
|
|
|
static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx) |
|
{ |
|
struct mtk_cryp *cryp = NULL; |
|
struct mtk_cryp *tmp; |
|
|
|
spin_lock_bh(&mtk_aes.lock); |
|
if (!ctx->cryp) { |
|
list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) { |
|
cryp = tmp; |
|
break; |
|
} |
|
ctx->cryp = cryp; |
|
} else { |
|
cryp = ctx->cryp; |
|
} |
|
spin_unlock_bh(&mtk_aes.lock); |
|
|
|
return cryp; |
|
} |
|
|
|
static inline size_t mtk_aes_padlen(size_t len) |
|
{ |
|
len &= AES_BLOCK_SIZE - 1; |
|
return len ? AES_BLOCK_SIZE - len : 0; |
|
} |
|
|
|
static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len, |
|
struct mtk_aes_dma *dma) |
|
{ |
|
int nents; |
|
|
|
if (!IS_ALIGNED(len, AES_BLOCK_SIZE)) |
|
return false; |
|
|
|
for (nents = 0; sg; sg = sg_next(sg), ++nents) { |
|
if (!IS_ALIGNED(sg->offset, sizeof(u32))) |
|
return false; |
|
|
|
if (len <= sg->length) { |
|
if (!IS_ALIGNED(len, AES_BLOCK_SIZE)) |
|
return false; |
|
|
|
dma->nents = nents + 1; |
|
dma->remainder = sg->length - len; |
|
sg->length = len; |
|
return true; |
|
} |
|
|
|
if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) |
|
return false; |
|
|
|
len -= sg->length; |
|
} |
|
|
|
return false; |
|
} |
|
|
|
static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes, |
|
const struct mtk_aes_reqctx *rctx) |
|
{ |
|
/* Clear all but persistent flags and set request flags. */ |
|
aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode; |
|
} |
|
|
|
static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) |
|
{ |
|
struct scatterlist *sg = dma->sg; |
|
int nents = dma->nents; |
|
|
|
if (!dma->remainder) |
|
return; |
|
|
|
while (--nents > 0 && sg) |
|
sg = sg_next(sg); |
|
|
|
if (!sg) |
|
return; |
|
|
|
sg->length += dma->remainder; |
|
} |
|
|
|
static inline int mtk_aes_complete(struct mtk_cryp *cryp, |
|
struct mtk_aes_rec *aes, |
|
int err) |
|
{ |
|
aes->flags &= ~AES_FLAGS_BUSY; |
|
aes->areq->complete(aes->areq, err); |
|
/* Handle new request */ |
|
tasklet_schedule(&aes->queue_task); |
|
return err; |
|
} |
|
|
|
/* |
|
* Write descriptors for processing. This will configure the engine, load |
|
* the transform information and then start the packet processing. |
|
*/ |
|
static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct mtk_ring *ring = cryp->ring[aes->id]; |
|
struct mtk_desc *cmd = NULL, *res = NULL; |
|
struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg; |
|
u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len; |
|
int nents; |
|
|
|
/* Write command descriptors */ |
|
for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) { |
|
cmd = ring->cmd_next; |
|
cmd->hdr = MTK_DESC_BUF_LEN(ssg->length); |
|
cmd->buf = cpu_to_le32(sg_dma_address(ssg)); |
|
|
|
if (nents == 0) { |
|
cmd->hdr |= MTK_DESC_FIRST | |
|
MTK_DESC_CT_LEN(aes->ctx->ct_size); |
|
cmd->ct = cpu_to_le32(aes->ctx->ct_dma); |
|
cmd->ct_hdr = aes->ctx->ct_hdr; |
|
cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma); |
|
} |
|
|
|
/* Shift ring buffer and check boundary */ |
|
if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) |
|
ring->cmd_next = ring->cmd_base; |
|
} |
|
cmd->hdr |= MTK_DESC_LAST; |
|
|
|
/* Prepare result descriptors */ |
|
for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) { |
|
res = ring->res_next; |
|
res->hdr = MTK_DESC_BUF_LEN(dsg->length); |
|
res->buf = cpu_to_le32(sg_dma_address(dsg)); |
|
|
|
if (nents == 0) |
|
res->hdr |= MTK_DESC_FIRST; |
|
|
|
/* Shift ring buffer and check boundary */ |
|
if (++ring->res_next == ring->res_base + MTK_DESC_NUM) |
|
ring->res_next = ring->res_base; |
|
} |
|
res->hdr |= MTK_DESC_LAST; |
|
|
|
/* Pointer to current result descriptor */ |
|
ring->res_prev = res; |
|
|
|
/* Prepare enough space for authenticated tag */ |
|
if (aes->flags & AES_FLAGS_GCM) |
|
le32_add_cpu(&res->hdr, AES_BLOCK_SIZE); |
|
|
|
/* |
|
* Make sure that all changes to the DMA ring are done before we |
|
* start engine. |
|
*/ |
|
wmb(); |
|
/* Start DMA transfer */ |
|
mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen)); |
|
mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen)); |
|
|
|
return -EINPROGRESS; |
|
} |
|
|
|
static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = aes->ctx; |
|
|
|
dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), |
|
DMA_TO_DEVICE); |
|
|
|
if (aes->src.sg == aes->dst.sg) { |
|
dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, |
|
DMA_BIDIRECTIONAL); |
|
|
|
if (aes->src.sg != &aes->aligned_sg) |
|
mtk_aes_restore_sg(&aes->src); |
|
} else { |
|
dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents, |
|
DMA_FROM_DEVICE); |
|
|
|
if (aes->dst.sg != &aes->aligned_sg) |
|
mtk_aes_restore_sg(&aes->dst); |
|
|
|
dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, |
|
DMA_TO_DEVICE); |
|
|
|
if (aes->src.sg != &aes->aligned_sg) |
|
mtk_aes_restore_sg(&aes->src); |
|
} |
|
|
|
if (aes->dst.sg == &aes->aligned_sg) |
|
sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst), |
|
aes->buf, aes->total); |
|
} |
|
|
|
static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = aes->ctx; |
|
struct mtk_aes_info *info = &ctx->info; |
|
|
|
ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), |
|
DMA_TO_DEVICE); |
|
if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) |
|
goto exit; |
|
|
|
ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd); |
|
|
|
if (aes->src.sg == aes->dst.sg) { |
|
aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, |
|
aes->src.nents, |
|
DMA_BIDIRECTIONAL); |
|
aes->dst.sg_len = aes->src.sg_len; |
|
if (unlikely(!aes->src.sg_len)) |
|
goto sg_map_err; |
|
} else { |
|
aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, |
|
aes->src.nents, DMA_TO_DEVICE); |
|
if (unlikely(!aes->src.sg_len)) |
|
goto sg_map_err; |
|
|
|
aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg, |
|
aes->dst.nents, DMA_FROM_DEVICE); |
|
if (unlikely(!aes->dst.sg_len)) { |
|
dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, |
|
DMA_TO_DEVICE); |
|
goto sg_map_err; |
|
} |
|
} |
|
|
|
return mtk_aes_xmit(cryp, aes); |
|
|
|
sg_map_err: |
|
dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE); |
|
exit: |
|
return mtk_aes_complete(cryp, aes, -EINVAL); |
|
} |
|
|
|
/* Initialize transform information of CBC/ECB/CTR/OFB/CFB mode */ |
|
static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, |
|
size_t len) |
|
{ |
|
struct skcipher_request *req = skcipher_request_cast(aes->areq); |
|
struct mtk_aes_base_ctx *ctx = aes->ctx; |
|
struct mtk_aes_info *info = &ctx->info; |
|
u32 cnt = 0; |
|
|
|
ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); |
|
info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len); |
|
info->cmd[cnt++] = AES_CMD1; |
|
|
|
info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode; |
|
if (aes->flags & AES_FLAGS_ENCRYPT) |
|
info->tfm[0] |= AES_TFM_BASIC_OUT; |
|
else |
|
info->tfm[0] |= AES_TFM_BASIC_IN; |
|
|
|
switch (aes->flags & AES_FLAGS_CIPHER_MSK) { |
|
case AES_FLAGS_CBC: |
|
info->tfm[1] = AES_TFM_CBC; |
|
break; |
|
case AES_FLAGS_ECB: |
|
info->tfm[1] = AES_TFM_ECB; |
|
goto ecb; |
|
case AES_FLAGS_CTR: |
|
info->tfm[1] = AES_TFM_CTR_LOAD; |
|
goto ctr; |
|
case AES_FLAGS_OFB: |
|
info->tfm[1] = AES_TFM_OFB; |
|
break; |
|
case AES_FLAGS_CFB128: |
|
info->tfm[1] = AES_TFM_CFB128; |
|
break; |
|
default: |
|
/* Should not happen... */ |
|
return; |
|
} |
|
|
|
memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE); |
|
ctr: |
|
le32_add_cpu(&info->tfm[0], |
|
le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)))); |
|
info->tfm[1] |= AES_TFM_FULL_IV; |
|
info->cmd[cnt++] = AES_CMD2; |
|
ecb: |
|
ctx->ct_size = cnt; |
|
} |
|
|
|
static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, |
|
struct scatterlist *src, struct scatterlist *dst, |
|
size_t len) |
|
{ |
|
size_t padlen = 0; |
|
bool src_aligned, dst_aligned; |
|
|
|
aes->total = len; |
|
aes->src.sg = src; |
|
aes->dst.sg = dst; |
|
aes->real_dst = dst; |
|
|
|
src_aligned = mtk_aes_check_aligned(src, len, &aes->src); |
|
if (src == dst) |
|
dst_aligned = src_aligned; |
|
else |
|
dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst); |
|
|
|
if (!src_aligned || !dst_aligned) { |
|
padlen = mtk_aes_padlen(len); |
|
|
|
if (len + padlen > AES_BUF_SIZE) |
|
return mtk_aes_complete(cryp, aes, -ENOMEM); |
|
|
|
if (!src_aligned) { |
|
sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); |
|
aes->src.sg = &aes->aligned_sg; |
|
aes->src.nents = 1; |
|
aes->src.remainder = 0; |
|
} |
|
|
|
if (!dst_aligned) { |
|
aes->dst.sg = &aes->aligned_sg; |
|
aes->dst.nents = 1; |
|
aes->dst.remainder = 0; |
|
} |
|
|
|
sg_init_table(&aes->aligned_sg, 1); |
|
sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen); |
|
} |
|
|
|
mtk_aes_info_init(cryp, aes, len + padlen); |
|
|
|
return mtk_aes_map(cryp, aes); |
|
} |
|
|
|
static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id, |
|
struct crypto_async_request *new_areq) |
|
{ |
|
struct mtk_aes_rec *aes = cryp->aes[id]; |
|
struct crypto_async_request *areq, *backlog; |
|
struct mtk_aes_base_ctx *ctx; |
|
unsigned long flags; |
|
int ret = 0; |
|
|
|
spin_lock_irqsave(&aes->lock, flags); |
|
if (new_areq) |
|
ret = crypto_enqueue_request(&aes->queue, new_areq); |
|
if (aes->flags & AES_FLAGS_BUSY) { |
|
spin_unlock_irqrestore(&aes->lock, flags); |
|
return ret; |
|
} |
|
backlog = crypto_get_backlog(&aes->queue); |
|
areq = crypto_dequeue_request(&aes->queue); |
|
if (areq) |
|
aes->flags |= AES_FLAGS_BUSY; |
|
spin_unlock_irqrestore(&aes->lock, flags); |
|
|
|
if (!areq) |
|
return ret; |
|
|
|
if (backlog) |
|
backlog->complete(backlog, -EINPROGRESS); |
|
|
|
ctx = crypto_tfm_ctx(areq->tfm); |
|
/* Write key into state buffer */ |
|
memcpy(ctx->info.state, ctx->key, sizeof(ctx->key)); |
|
|
|
aes->areq = areq; |
|
aes->ctx = ctx; |
|
|
|
return ctx->start(cryp, aes); |
|
} |
|
|
|
static int mtk_aes_transfer_complete(struct mtk_cryp *cryp, |
|
struct mtk_aes_rec *aes) |
|
{ |
|
return mtk_aes_complete(cryp, aes, 0); |
|
} |
|
|
|
static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct skcipher_request *req = skcipher_request_cast(aes->areq); |
|
struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req); |
|
|
|
mtk_aes_set_mode(aes, rctx); |
|
aes->resume = mtk_aes_transfer_complete; |
|
|
|
return mtk_aes_dma(cryp, aes, req->src, req->dst, req->cryptlen); |
|
} |
|
|
|
static inline struct mtk_aes_ctr_ctx * |
|
mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx) |
|
{ |
|
return container_of(ctx, struct mtk_aes_ctr_ctx, base); |
|
} |
|
|
|
static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = aes->ctx; |
|
struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx); |
|
struct skcipher_request *req = skcipher_request_cast(aes->areq); |
|
struct scatterlist *src, *dst; |
|
u32 start, end, ctr, blocks; |
|
size_t datalen; |
|
bool fragmented = false; |
|
|
|
/* Check for transfer completion. */ |
|
cctx->offset += aes->total; |
|
if (cctx->offset >= req->cryptlen) |
|
return mtk_aes_transfer_complete(cryp, aes); |
|
|
|
/* Compute data length. */ |
|
datalen = req->cryptlen - cctx->offset; |
|
blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); |
|
ctr = be32_to_cpu(cctx->iv[3]); |
|
|
|
/* Check 32bit counter overflow. */ |
|
start = ctr; |
|
end = start + blocks - 1; |
|
if (end < start) { |
|
ctr = 0xffffffff; |
|
datalen = AES_BLOCK_SIZE * -start; |
|
fragmented = true; |
|
} |
|
|
|
/* Jump to offset. */ |
|
src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset); |
|
dst = ((req->src == req->dst) ? src : |
|
scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); |
|
|
|
/* Write IVs into transform state buffer. */ |
|
memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE); |
|
|
|
if (unlikely(fragmented)) { |
|
/* |
|
* Increment the counter manually to cope with the hardware |
|
* counter overflow. |
|
*/ |
|
cctx->iv[3] = cpu_to_be32(ctr); |
|
crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE); |
|
} |
|
|
|
return mtk_aes_dma(cryp, aes, src, dst, datalen); |
|
} |
|
|
|
static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx); |
|
struct skcipher_request *req = skcipher_request_cast(aes->areq); |
|
struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req); |
|
|
|
mtk_aes_set_mode(aes, rctx); |
|
|
|
memcpy(cctx->iv, req->iv, AES_BLOCK_SIZE); |
|
cctx->offset = 0; |
|
aes->total = 0; |
|
aes->resume = mtk_aes_ctr_transfer; |
|
|
|
return mtk_aes_ctr_transfer(cryp, aes); |
|
} |
|
|
|
/* Check and set the AES key to transform state buffer */ |
|
static int mtk_aes_setkey(struct crypto_skcipher *tfm, |
|
const u8 *key, u32 keylen) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); |
|
|
|
switch (keylen) { |
|
case AES_KEYSIZE_128: |
|
ctx->keymode = AES_TFM_128BITS; |
|
break; |
|
case AES_KEYSIZE_192: |
|
ctx->keymode = AES_TFM_192BITS; |
|
break; |
|
case AES_KEYSIZE_256: |
|
ctx->keymode = AES_TFM_256BITS; |
|
break; |
|
|
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
ctx->keylen = SIZE_IN_WORDS(keylen); |
|
memcpy(ctx->key, key, keylen); |
|
|
|
return 0; |
|
} |
|
|
|
static int mtk_aes_crypt(struct skcipher_request *req, u64 mode) |
|
{ |
|
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); |
|
struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); |
|
struct mtk_aes_reqctx *rctx; |
|
struct mtk_cryp *cryp; |
|
|
|
cryp = mtk_aes_find_dev(ctx); |
|
if (!cryp) |
|
return -ENODEV; |
|
|
|
rctx = skcipher_request_ctx(req); |
|
rctx->mode = mode; |
|
|
|
return mtk_aes_handle_queue(cryp, !(mode & AES_FLAGS_ENCRYPT), |
|
&req->base); |
|
} |
|
|
|
static int mtk_aes_ecb_encrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB); |
|
} |
|
|
|
static int mtk_aes_ecb_decrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_ECB); |
|
} |
|
|
|
static int mtk_aes_cbc_encrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC); |
|
} |
|
|
|
static int mtk_aes_cbc_decrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_CBC); |
|
} |
|
|
|
static int mtk_aes_ctr_encrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); |
|
} |
|
|
|
static int mtk_aes_ctr_decrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_CTR); |
|
} |
|
|
|
static int mtk_aes_ofb_encrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_OFB); |
|
} |
|
|
|
static int mtk_aes_ofb_decrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_OFB); |
|
} |
|
|
|
static int mtk_aes_cfb_encrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CFB128); |
|
} |
|
|
|
static int mtk_aes_cfb_decrypt(struct skcipher_request *req) |
|
{ |
|
return mtk_aes_crypt(req, AES_FLAGS_CFB128); |
|
} |
|
|
|
static int mtk_aes_init_tfm(struct crypto_skcipher *tfm) |
|
{ |
|
struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm); |
|
|
|
crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx)); |
|
ctx->base.start = mtk_aes_start; |
|
return 0; |
|
} |
|
|
|
static int mtk_aes_ctr_init_tfm(struct crypto_skcipher *tfm) |
|
{ |
|
struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm); |
|
|
|
crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx)); |
|
ctx->base.start = mtk_aes_ctr_start; |
|
return 0; |
|
} |
|
|
|
static struct skcipher_alg aes_algs[] = { |
|
{ |
|
.base.cra_name = "cbc(aes)", |
|
.base.cra_driver_name = "cbc-aes-mtk", |
|
.base.cra_priority = 400, |
|
.base.cra_flags = CRYPTO_ALG_ASYNC, |
|
.base.cra_blocksize = AES_BLOCK_SIZE, |
|
.base.cra_ctxsize = sizeof(struct mtk_aes_ctx), |
|
.base.cra_alignmask = 0xf, |
|
.base.cra_module = THIS_MODULE, |
|
|
|
.min_keysize = AES_MIN_KEY_SIZE, |
|
.max_keysize = AES_MAX_KEY_SIZE, |
|
.setkey = mtk_aes_setkey, |
|
.encrypt = mtk_aes_cbc_encrypt, |
|
.decrypt = mtk_aes_cbc_decrypt, |
|
.ivsize = AES_BLOCK_SIZE, |
|
.init = mtk_aes_init_tfm, |
|
}, |
|
{ |
|
.base.cra_name = "ecb(aes)", |
|
.base.cra_driver_name = "ecb-aes-mtk", |
|
.base.cra_priority = 400, |
|
.base.cra_flags = CRYPTO_ALG_ASYNC, |
|
.base.cra_blocksize = AES_BLOCK_SIZE, |
|
.base.cra_ctxsize = sizeof(struct mtk_aes_ctx), |
|
.base.cra_alignmask = 0xf, |
|
.base.cra_module = THIS_MODULE, |
|
|
|
.min_keysize = AES_MIN_KEY_SIZE, |
|
.max_keysize = AES_MAX_KEY_SIZE, |
|
.setkey = mtk_aes_setkey, |
|
.encrypt = mtk_aes_ecb_encrypt, |
|
.decrypt = mtk_aes_ecb_decrypt, |
|
.init = mtk_aes_init_tfm, |
|
}, |
|
{ |
|
.base.cra_name = "ctr(aes)", |
|
.base.cra_driver_name = "ctr-aes-mtk", |
|
.base.cra_priority = 400, |
|
.base.cra_flags = CRYPTO_ALG_ASYNC, |
|
.base.cra_blocksize = 1, |
|
.base.cra_ctxsize = sizeof(struct mtk_aes_ctx), |
|
.base.cra_alignmask = 0xf, |
|
.base.cra_module = THIS_MODULE, |
|
|
|
.min_keysize = AES_MIN_KEY_SIZE, |
|
.max_keysize = AES_MAX_KEY_SIZE, |
|
.ivsize = AES_BLOCK_SIZE, |
|
.setkey = mtk_aes_setkey, |
|
.encrypt = mtk_aes_ctr_encrypt, |
|
.decrypt = mtk_aes_ctr_decrypt, |
|
.init = mtk_aes_ctr_init_tfm, |
|
}, |
|
{ |
|
.base.cra_name = "ofb(aes)", |
|
.base.cra_driver_name = "ofb-aes-mtk", |
|
.base.cra_priority = 400, |
|
.base.cra_flags = CRYPTO_ALG_ASYNC, |
|
.base.cra_blocksize = AES_BLOCK_SIZE, |
|
.base.cra_ctxsize = sizeof(struct mtk_aes_ctx), |
|
.base.cra_alignmask = 0xf, |
|
.base.cra_module = THIS_MODULE, |
|
|
|
.min_keysize = AES_MIN_KEY_SIZE, |
|
.max_keysize = AES_MAX_KEY_SIZE, |
|
.ivsize = AES_BLOCK_SIZE, |
|
.setkey = mtk_aes_setkey, |
|
.encrypt = mtk_aes_ofb_encrypt, |
|
.decrypt = mtk_aes_ofb_decrypt, |
|
}, |
|
{ |
|
.base.cra_name = "cfb(aes)", |
|
.base.cra_driver_name = "cfb-aes-mtk", |
|
.base.cra_priority = 400, |
|
.base.cra_flags = CRYPTO_ALG_ASYNC, |
|
.base.cra_blocksize = 1, |
|
.base.cra_ctxsize = sizeof(struct mtk_aes_ctx), |
|
.base.cra_alignmask = 0xf, |
|
.base.cra_module = THIS_MODULE, |
|
|
|
.min_keysize = AES_MIN_KEY_SIZE, |
|
.max_keysize = AES_MAX_KEY_SIZE, |
|
.ivsize = AES_BLOCK_SIZE, |
|
.setkey = mtk_aes_setkey, |
|
.encrypt = mtk_aes_cfb_encrypt, |
|
.decrypt = mtk_aes_cfb_decrypt, |
|
}, |
|
}; |
|
|
|
static inline struct mtk_aes_gcm_ctx * |
|
mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) |
|
{ |
|
return container_of(ctx, struct mtk_aes_gcm_ctx, base); |
|
} |
|
|
|
/* |
|
* Engine will verify and compare tag automatically, so we just need |
|
* to check returned status which stored in the result descriptor. |
|
*/ |
|
static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, |
|
struct mtk_aes_rec *aes) |
|
{ |
|
__le32 status = cryp->ring[aes->id]->res_prev->ct; |
|
|
|
return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? |
|
-EBADMSG : 0); |
|
} |
|
|
|
/* Initialize transform information of GCM mode */ |
|
static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, |
|
struct mtk_aes_rec *aes, |
|
size_t len) |
|
{ |
|
struct aead_request *req = aead_request_cast(aes->areq); |
|
struct mtk_aes_base_ctx *ctx = aes->ctx; |
|
struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); |
|
struct mtk_aes_info *info = &ctx->info; |
|
u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); |
|
u32 cnt = 0; |
|
|
|
ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); |
|
|
|
info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); |
|
info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); |
|
info->cmd[cnt++] = AES_GCM_CMD2; |
|
info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); |
|
|
|
if (aes->flags & AES_FLAGS_ENCRYPT) { |
|
info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); |
|
info->tfm[0] = AES_TFM_GCM_OUT; |
|
} else { |
|
info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); |
|
info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); |
|
info->tfm[0] = AES_TFM_GCM_IN; |
|
} |
|
ctx->ct_size = cnt; |
|
|
|
info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE( |
|
ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) | |
|
ctx->keymode; |
|
info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | |
|
AES_TFM_ENC_HASH; |
|
|
|
memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE), |
|
req->iv, ivsize); |
|
} |
|
|
|
static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, |
|
struct scatterlist *src, struct scatterlist *dst, |
|
size_t len) |
|
{ |
|
bool src_aligned, dst_aligned; |
|
|
|
aes->src.sg = src; |
|
aes->dst.sg = dst; |
|
aes->real_dst = dst; |
|
|
|
src_aligned = mtk_aes_check_aligned(src, len, &aes->src); |
|
if (src == dst) |
|
dst_aligned = src_aligned; |
|
else |
|
dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst); |
|
|
|
if (!src_aligned || !dst_aligned) { |
|
if (aes->total > AES_BUF_SIZE) |
|
return mtk_aes_complete(cryp, aes, -ENOMEM); |
|
|
|
if (!src_aligned) { |
|
sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); |
|
aes->src.sg = &aes->aligned_sg; |
|
aes->src.nents = 1; |
|
aes->src.remainder = 0; |
|
} |
|
|
|
if (!dst_aligned) { |
|
aes->dst.sg = &aes->aligned_sg; |
|
aes->dst.nents = 1; |
|
aes->dst.remainder = 0; |
|
} |
|
|
|
sg_init_table(&aes->aligned_sg, 1); |
|
sg_set_buf(&aes->aligned_sg, aes->buf, aes->total); |
|
} |
|
|
|
mtk_aes_gcm_info_init(cryp, aes, len); |
|
|
|
return mtk_aes_map(cryp, aes); |
|
} |
|
|
|
/* Todo: GMAC */ |
|
static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) |
|
{ |
|
struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx); |
|
struct aead_request *req = aead_request_cast(aes->areq); |
|
struct mtk_aes_reqctx *rctx = aead_request_ctx(req); |
|
u32 len = req->assoclen + req->cryptlen; |
|
|
|
mtk_aes_set_mode(aes, rctx); |
|
|
|
if (aes->flags & AES_FLAGS_ENCRYPT) { |
|
u32 tag[4]; |
|
|
|
aes->resume = mtk_aes_transfer_complete; |
|
/* Compute total process length. */ |
|
aes->total = len + gctx->authsize; |
|
/* Hardware will append authenticated tag to output buffer */ |
|
scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1); |
|
} else { |
|
aes->resume = mtk_aes_gcm_tag_verify; |
|
aes->total = len; |
|
} |
|
|
|
return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len); |
|
} |
|
|
|
static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); |
|
struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); |
|
struct mtk_aes_reqctx *rctx = aead_request_ctx(req); |
|
struct mtk_cryp *cryp; |
|
bool enc = !!(mode & AES_FLAGS_ENCRYPT); |
|
|
|
cryp = mtk_aes_find_dev(ctx); |
|
if (!cryp) |
|
return -ENODEV; |
|
|
|
/* Compute text length. */ |
|
gctx->textlen = req->cryptlen - (enc ? 0 : gctx->authsize); |
|
|
|
/* Empty messages are not supported yet */ |
|
if (!gctx->textlen && !req->assoclen) |
|
return -EINVAL; |
|
|
|
rctx->mode = AES_FLAGS_GCM | mode; |
|
|
|
return mtk_aes_handle_queue(cryp, enc, &req->base); |
|
} |
|
|
|
/* |
|
* Because of the hardware limitation, we need to pre-calculate key(H) |
|
* for the GHASH operation. The result of the encryption operation |
|
* need to be stored in the transform state buffer. |
|
*/ |
|
static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, |
|
u32 keylen) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); |
|
union { |
|
u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)]; |
|
u8 x8[AES_BLOCK_SIZE]; |
|
} hash = {}; |
|
struct crypto_aes_ctx aes_ctx; |
|
int err; |
|
int i; |
|
|
|
switch (keylen) { |
|
case AES_KEYSIZE_128: |
|
ctx->keymode = AES_TFM_128BITS; |
|
break; |
|
case AES_KEYSIZE_192: |
|
ctx->keymode = AES_TFM_192BITS; |
|
break; |
|
case AES_KEYSIZE_256: |
|
ctx->keymode = AES_TFM_256BITS; |
|
break; |
|
|
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
ctx->keylen = SIZE_IN_WORDS(keylen); |
|
|
|
err = aes_expandkey(&aes_ctx, key, keylen); |
|
if (err) |
|
return err; |
|
|
|
aes_encrypt(&aes_ctx, hash.x8, hash.x8); |
|
memzero_explicit(&aes_ctx, sizeof(aes_ctx)); |
|
|
|
memcpy(ctx->key, key, keylen); |
|
|
|
/* Why do we need to do this? */ |
|
for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) |
|
hash.x32[i] = swab32(hash.x32[i]); |
|
|
|
memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE); |
|
|
|
return 0; |
|
} |
|
|
|
static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead, |
|
u32 authsize) |
|
{ |
|
struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); |
|
struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); |
|
|
|
/* Same as crypto_gcm_authsize() from crypto/gcm.c */ |
|
switch (authsize) { |
|
case 8: |
|
case 12: |
|
case 16: |
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
gctx->authsize = authsize; |
|
return 0; |
|
} |
|
|
|
static int mtk_aes_gcm_encrypt(struct aead_request *req) |
|
{ |
|
return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); |
|
} |
|
|
|
static int mtk_aes_gcm_decrypt(struct aead_request *req) |
|
{ |
|
return mtk_aes_gcm_crypt(req, 0); |
|
} |
|
|
|
static int mtk_aes_gcm_init(struct crypto_aead *aead) |
|
{ |
|
struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); |
|
|
|
crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx)); |
|
ctx->base.start = mtk_aes_gcm_start; |
|
return 0; |
|
} |
|
|
|
static struct aead_alg aes_gcm_alg = { |
|
.setkey = mtk_aes_gcm_setkey, |
|
.setauthsize = mtk_aes_gcm_setauthsize, |
|
.encrypt = mtk_aes_gcm_encrypt, |
|
.decrypt = mtk_aes_gcm_decrypt, |
|
.init = mtk_aes_gcm_init, |
|
.ivsize = GCM_AES_IV_SIZE, |
|
.maxauthsize = AES_BLOCK_SIZE, |
|
|
|
.base = { |
|
.cra_name = "gcm(aes)", |
|
.cra_driver_name = "gcm-aes-mtk", |
|
.cra_priority = 400, |
|
.cra_flags = CRYPTO_ALG_ASYNC, |
|
.cra_blocksize = 1, |
|
.cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx), |
|
.cra_alignmask = 0xf, |
|
.cra_module = THIS_MODULE, |
|
}, |
|
}; |
|
|
|
static void mtk_aes_queue_task(unsigned long data) |
|
{ |
|
struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; |
|
|
|
mtk_aes_handle_queue(aes->cryp, aes->id, NULL); |
|
} |
|
|
|
static void mtk_aes_done_task(unsigned long data) |
|
{ |
|
struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; |
|
struct mtk_cryp *cryp = aes->cryp; |
|
|
|
mtk_aes_unmap(cryp, aes); |
|
aes->resume(cryp, aes); |
|
} |
|
|
|
static irqreturn_t mtk_aes_irq(int irq, void *dev_id) |
|
{ |
|
struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id; |
|
struct mtk_cryp *cryp = aes->cryp; |
|
u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id)); |
|
|
|
mtk_aes_write(cryp, RDR_STAT(aes->id), val); |
|
|
|
if (likely(AES_FLAGS_BUSY & aes->flags)) { |
|
mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST); |
|
mtk_aes_write(cryp, RDR_THRESH(aes->id), |
|
MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); |
|
|
|
tasklet_schedule(&aes->done_task); |
|
} else { |
|
dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); |
|
} |
|
return IRQ_HANDLED; |
|
} |
|
|
|
/* |
|
* The purpose of creating encryption and decryption records is |
|
* to process outbound/inbound data in parallel, it can improve |
|
* performance in most use cases, such as IPSec VPN, especially |
|
* under heavy network traffic. |
|
*/ |
|
static int mtk_aes_record_init(struct mtk_cryp *cryp) |
|
{ |
|
struct mtk_aes_rec **aes = cryp->aes; |
|
int i, err = -ENOMEM; |
|
|
|
for (i = 0; i < MTK_REC_NUM; i++) { |
|
aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL); |
|
if (!aes[i]) |
|
goto err_cleanup; |
|
|
|
aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL, |
|
AES_BUF_ORDER); |
|
if (!aes[i]->buf) |
|
goto err_cleanup; |
|
|
|
aes[i]->cryp = cryp; |
|
|
|
spin_lock_init(&aes[i]->lock); |
|
crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE); |
|
|
|
tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task, |
|
(unsigned long)aes[i]); |
|
tasklet_init(&aes[i]->done_task, mtk_aes_done_task, |
|
(unsigned long)aes[i]); |
|
} |
|
|
|
/* Link to ring0 and ring1 respectively */ |
|
aes[0]->id = MTK_RING0; |
|
aes[1]->id = MTK_RING1; |
|
|
|
return 0; |
|
|
|
err_cleanup: |
|
for (; i--; ) { |
|
free_page((unsigned long)aes[i]->buf); |
|
kfree(aes[i]); |
|
} |
|
|
|
return err; |
|
} |
|
|
|
static void mtk_aes_record_free(struct mtk_cryp *cryp) |
|
{ |
|
int i; |
|
|
|
for (i = 0; i < MTK_REC_NUM; i++) { |
|
tasklet_kill(&cryp->aes[i]->done_task); |
|
tasklet_kill(&cryp->aes[i]->queue_task); |
|
|
|
free_page((unsigned long)cryp->aes[i]->buf); |
|
kfree(cryp->aes[i]); |
|
} |
|
} |
|
|
|
static void mtk_aes_unregister_algs(void) |
|
{ |
|
int i; |
|
|
|
crypto_unregister_aead(&aes_gcm_alg); |
|
|
|
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) |
|
crypto_unregister_skcipher(&aes_algs[i]); |
|
} |
|
|
|
static int mtk_aes_register_algs(void) |
|
{ |
|
int err, i; |
|
|
|
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { |
|
err = crypto_register_skcipher(&aes_algs[i]); |
|
if (err) |
|
goto err_aes_algs; |
|
} |
|
|
|
err = crypto_register_aead(&aes_gcm_alg); |
|
if (err) |
|
goto err_aes_algs; |
|
|
|
return 0; |
|
|
|
err_aes_algs: |
|
for (; i--; ) |
|
crypto_unregister_skcipher(&aes_algs[i]); |
|
|
|
return err; |
|
} |
|
|
|
int mtk_cipher_alg_register(struct mtk_cryp *cryp) |
|
{ |
|
int ret; |
|
|
|
INIT_LIST_HEAD(&cryp->aes_list); |
|
|
|
/* Initialize two cipher records */ |
|
ret = mtk_aes_record_init(cryp); |
|
if (ret) |
|
goto err_record; |
|
|
|
ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq, |
|
0, "mtk-aes", cryp->aes[0]); |
|
if (ret) { |
|
dev_err(cryp->dev, "unable to request AES irq.\n"); |
|
goto err_res; |
|
} |
|
|
|
ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq, |
|
0, "mtk-aes", cryp->aes[1]); |
|
if (ret) { |
|
dev_err(cryp->dev, "unable to request AES irq.\n"); |
|
goto err_res; |
|
} |
|
|
|
/* Enable ring0 and ring1 interrupt */ |
|
mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0); |
|
mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1); |
|
|
|
spin_lock(&mtk_aes.lock); |
|
list_add_tail(&cryp->aes_list, &mtk_aes.dev_list); |
|
spin_unlock(&mtk_aes.lock); |
|
|
|
ret = mtk_aes_register_algs(); |
|
if (ret) |
|
goto err_algs; |
|
|
|
return 0; |
|
|
|
err_algs: |
|
spin_lock(&mtk_aes.lock); |
|
list_del(&cryp->aes_list); |
|
spin_unlock(&mtk_aes.lock); |
|
err_res: |
|
mtk_aes_record_free(cryp); |
|
err_record: |
|
|
|
dev_err(cryp->dev, "mtk-aes initialization failed.\n"); |
|
return ret; |
|
} |
|
|
|
void mtk_cipher_alg_release(struct mtk_cryp *cryp) |
|
{ |
|
spin_lock(&mtk_aes.lock); |
|
list_del(&cryp->aes_list); |
|
spin_unlock(&mtk_aes.lock); |
|
|
|
mtk_aes_unregister_algs(); |
|
mtk_aes_record_free(cryp); |
|
}
|
|
|