forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
562 lines
15 KiB
562 lines
15 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* Cryptographic API. |
|
* |
|
* Support for VIA PadLock hardware crypto engine. |
|
* |
|
* Copyright (c) 2006 Michal Ludvig <[email protected]> |
|
*/ |
|
|
|
#include <crypto/internal/hash.h> |
|
#include <crypto/padlock.h> |
|
#include <crypto/sha1.h> |
|
#include <crypto/sha2.h> |
|
#include <linux/err.h> |
|
#include <linux/module.h> |
|
#include <linux/init.h> |
|
#include <linux/errno.h> |
|
#include <linux/interrupt.h> |
|
#include <linux/kernel.h> |
|
#include <linux/scatterlist.h> |
|
#include <asm/cpu_device_id.h> |
|
#include <asm/fpu/api.h> |
|
|
|
struct padlock_sha_desc { |
|
struct shash_desc fallback; |
|
}; |
|
|
|
struct padlock_sha_ctx { |
|
struct crypto_shash *fallback; |
|
}; |
|
|
|
static int padlock_sha_init(struct shash_desc *desc) |
|
{ |
|
struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
|
struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); |
|
|
|
dctx->fallback.tfm = ctx->fallback; |
|
return crypto_shash_init(&dctx->fallback); |
|
} |
|
|
|
static int padlock_sha_update(struct shash_desc *desc, |
|
const u8 *data, unsigned int length) |
|
{ |
|
struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
|
|
|
return crypto_shash_update(&dctx->fallback, data, length); |
|
} |
|
|
|
static int padlock_sha_export(struct shash_desc *desc, void *out) |
|
{ |
|
struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
|
|
|
return crypto_shash_export(&dctx->fallback, out); |
|
} |
|
|
|
static int padlock_sha_import(struct shash_desc *desc, const void *in) |
|
{ |
|
struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
|
struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); |
|
|
|
dctx->fallback.tfm = ctx->fallback; |
|
return crypto_shash_import(&dctx->fallback, in); |
|
} |
|
|
|
static inline void padlock_output_block(uint32_t *src, |
|
uint32_t *dst, size_t count) |
|
{ |
|
while (count--) |
|
*dst++ = swab32(*src++); |
|
} |
|
|
|
static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, |
|
unsigned int count, u8 *out) |
|
{ |
|
/* We can't store directly to *out as it may be unaligned. */ |
|
/* BTW Don't reduce the buffer size below 128 Bytes! |
|
* PadLock microcode needs it that big. */ |
|
char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
|
((aligned(STACK_ALIGN))); |
|
char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
|
struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
|
struct sha1_state state; |
|
unsigned int space; |
|
unsigned int leftover; |
|
int err; |
|
|
|
err = crypto_shash_export(&dctx->fallback, &state); |
|
if (err) |
|
goto out; |
|
|
|
if (state.count + count > ULONG_MAX) |
|
return crypto_shash_finup(&dctx->fallback, in, count, out); |
|
|
|
leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1; |
|
space = SHA1_BLOCK_SIZE - leftover; |
|
if (space) { |
|
if (count > space) { |
|
err = crypto_shash_update(&dctx->fallback, in, space) ?: |
|
crypto_shash_export(&dctx->fallback, &state); |
|
if (err) |
|
goto out; |
|
count -= space; |
|
in += space; |
|
} else { |
|
memcpy(state.buffer + leftover, in, count); |
|
in = state.buffer; |
|
count += leftover; |
|
state.count &= ~(SHA1_BLOCK_SIZE - 1); |
|
} |
|
} |
|
|
|
memcpy(result, &state.state, SHA1_DIGEST_SIZE); |
|
|
|
asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ |
|
: \ |
|
: "c"((unsigned long)state.count + count), \ |
|
"a"((unsigned long)state.count), \ |
|
"S"(in), "D"(result)); |
|
|
|
padlock_output_block((uint32_t *)result, (uint32_t *)out, 5); |
|
|
|
out: |
|
return err; |
|
} |
|
|
|
static int padlock_sha1_final(struct shash_desc *desc, u8 *out) |
|
{ |
|
u8 buf[4]; |
|
|
|
return padlock_sha1_finup(desc, buf, 0, out); |
|
} |
|
|
|
static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, |
|
unsigned int count, u8 *out) |
|
{ |
|
/* We can't store directly to *out as it may be unaligned. */ |
|
/* BTW Don't reduce the buffer size below 128 Bytes! |
|
* PadLock microcode needs it that big. */ |
|
char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
|
((aligned(STACK_ALIGN))); |
|
char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
|
struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
|
struct sha256_state state; |
|
unsigned int space; |
|
unsigned int leftover; |
|
int err; |
|
|
|
err = crypto_shash_export(&dctx->fallback, &state); |
|
if (err) |
|
goto out; |
|
|
|
if (state.count + count > ULONG_MAX) |
|
return crypto_shash_finup(&dctx->fallback, in, count, out); |
|
|
|
leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1; |
|
space = SHA256_BLOCK_SIZE - leftover; |
|
if (space) { |
|
if (count > space) { |
|
err = crypto_shash_update(&dctx->fallback, in, space) ?: |
|
crypto_shash_export(&dctx->fallback, &state); |
|
if (err) |
|
goto out; |
|
count -= space; |
|
in += space; |
|
} else { |
|
memcpy(state.buf + leftover, in, count); |
|
in = state.buf; |
|
count += leftover; |
|
state.count &= ~(SHA1_BLOCK_SIZE - 1); |
|
} |
|
} |
|
|
|
memcpy(result, &state.state, SHA256_DIGEST_SIZE); |
|
|
|
asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ |
|
: \ |
|
: "c"((unsigned long)state.count + count), \ |
|
"a"((unsigned long)state.count), \ |
|
"S"(in), "D"(result)); |
|
|
|
padlock_output_block((uint32_t *)result, (uint32_t *)out, 8); |
|
|
|
out: |
|
return err; |
|
} |
|
|
|
static int padlock_sha256_final(struct shash_desc *desc, u8 *out) |
|
{ |
|
u8 buf[4]; |
|
|
|
return padlock_sha256_finup(desc, buf, 0, out); |
|
} |
|
|
|
static int padlock_init_tfm(struct crypto_shash *hash) |
|
{ |
|
const char *fallback_driver_name = crypto_shash_alg_name(hash); |
|
struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); |
|
struct crypto_shash *fallback_tfm; |
|
|
|
/* Allocate a fallback and abort if it failed. */ |
|
fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, |
|
CRYPTO_ALG_NEED_FALLBACK); |
|
if (IS_ERR(fallback_tfm)) { |
|
printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", |
|
fallback_driver_name); |
|
return PTR_ERR(fallback_tfm); |
|
} |
|
|
|
ctx->fallback = fallback_tfm; |
|
hash->descsize += crypto_shash_descsize(fallback_tfm); |
|
return 0; |
|
} |
|
|
|
static void padlock_exit_tfm(struct crypto_shash *hash) |
|
{ |
|
struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash); |
|
|
|
crypto_free_shash(ctx->fallback); |
|
} |
|
|
|
static struct shash_alg sha1_alg = { |
|
.digestsize = SHA1_DIGEST_SIZE, |
|
.init = padlock_sha_init, |
|
.update = padlock_sha_update, |
|
.finup = padlock_sha1_finup, |
|
.final = padlock_sha1_final, |
|
.export = padlock_sha_export, |
|
.import = padlock_sha_import, |
|
.init_tfm = padlock_init_tfm, |
|
.exit_tfm = padlock_exit_tfm, |
|
.descsize = sizeof(struct padlock_sha_desc), |
|
.statesize = sizeof(struct sha1_state), |
|
.base = { |
|
.cra_name = "sha1", |
|
.cra_driver_name = "sha1-padlock", |
|
.cra_priority = PADLOCK_CRA_PRIORITY, |
|
.cra_flags = CRYPTO_ALG_NEED_FALLBACK, |
|
.cra_blocksize = SHA1_BLOCK_SIZE, |
|
.cra_ctxsize = sizeof(struct padlock_sha_ctx), |
|
.cra_module = THIS_MODULE, |
|
} |
|
}; |
|
|
|
static struct shash_alg sha256_alg = { |
|
.digestsize = SHA256_DIGEST_SIZE, |
|
.init = padlock_sha_init, |
|
.update = padlock_sha_update, |
|
.finup = padlock_sha256_finup, |
|
.final = padlock_sha256_final, |
|
.export = padlock_sha_export, |
|
.import = padlock_sha_import, |
|
.init_tfm = padlock_init_tfm, |
|
.exit_tfm = padlock_exit_tfm, |
|
.descsize = sizeof(struct padlock_sha_desc), |
|
.statesize = sizeof(struct sha256_state), |
|
.base = { |
|
.cra_name = "sha256", |
|
.cra_driver_name = "sha256-padlock", |
|
.cra_priority = PADLOCK_CRA_PRIORITY, |
|
.cra_flags = CRYPTO_ALG_NEED_FALLBACK, |
|
.cra_blocksize = SHA256_BLOCK_SIZE, |
|
.cra_ctxsize = sizeof(struct padlock_sha_ctx), |
|
.cra_module = THIS_MODULE, |
|
} |
|
}; |
|
|
|
/* Add two shash_alg instance for hardware-implemented * |
|
* multiple-parts hash supported by VIA Nano Processor.*/ |
|
static int padlock_sha1_init_nano(struct shash_desc *desc) |
|
{ |
|
struct sha1_state *sctx = shash_desc_ctx(desc); |
|
|
|
*sctx = (struct sha1_state){ |
|
.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, |
|
}; |
|
|
|
return 0; |
|
} |
|
|
|
static int padlock_sha1_update_nano(struct shash_desc *desc, |
|
const u8 *data, unsigned int len) |
|
{ |
|
struct sha1_state *sctx = shash_desc_ctx(desc); |
|
unsigned int partial, done; |
|
const u8 *src; |
|
/*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ |
|
u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
|
((aligned(STACK_ALIGN))); |
|
u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
|
|
|
partial = sctx->count & 0x3f; |
|
sctx->count += len; |
|
done = 0; |
|
src = data; |
|
memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE); |
|
|
|
if ((partial + len) >= SHA1_BLOCK_SIZE) { |
|
|
|
/* Append the bytes in state's buffer to a block to handle */ |
|
if (partial) { |
|
done = -partial; |
|
memcpy(sctx->buffer + partial, data, |
|
done + SHA1_BLOCK_SIZE); |
|
src = sctx->buffer; |
|
asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" |
|
: "+S"(src), "+D"(dst) \ |
|
: "a"((long)-1), "c"((unsigned long)1)); |
|
done += SHA1_BLOCK_SIZE; |
|
src = data + done; |
|
} |
|
|
|
/* Process the left bytes from the input data */ |
|
if (len - done >= SHA1_BLOCK_SIZE) { |
|
asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" |
|
: "+S"(src), "+D"(dst) |
|
: "a"((long)-1), |
|
"c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE))); |
|
done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE); |
|
src = data + done; |
|
} |
|
partial = 0; |
|
} |
|
memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE); |
|
memcpy(sctx->buffer + partial, src, len - done); |
|
|
|
return 0; |
|
} |
|
|
|
static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out) |
|
{ |
|
struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc); |
|
unsigned int partial, padlen; |
|
__be64 bits; |
|
static const u8 padding[64] = { 0x80, }; |
|
|
|
bits = cpu_to_be64(state->count << 3); |
|
|
|
/* Pad out to 56 mod 64 */ |
|
partial = state->count & 0x3f; |
|
padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); |
|
padlock_sha1_update_nano(desc, padding, padlen); |
|
|
|
/* Append length field bytes */ |
|
padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits)); |
|
|
|
/* Swap to output */ |
|
padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5); |
|
|
|
return 0; |
|
} |
|
|
|
static int padlock_sha256_init_nano(struct shash_desc *desc) |
|
{ |
|
struct sha256_state *sctx = shash_desc_ctx(desc); |
|
|
|
*sctx = (struct sha256_state){ |
|
.state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \ |
|
SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7}, |
|
}; |
|
|
|
return 0; |
|
} |
|
|
|
static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data, |
|
unsigned int len) |
|
{ |
|
struct sha256_state *sctx = shash_desc_ctx(desc); |
|
unsigned int partial, done; |
|
const u8 *src; |
|
/*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ |
|
u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
|
((aligned(STACK_ALIGN))); |
|
u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
|
|
|
partial = sctx->count & 0x3f; |
|
sctx->count += len; |
|
done = 0; |
|
src = data; |
|
memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE); |
|
|
|
if ((partial + len) >= SHA256_BLOCK_SIZE) { |
|
|
|
/* Append the bytes in state's buffer to a block to handle */ |
|
if (partial) { |
|
done = -partial; |
|
memcpy(sctx->buf + partial, data, |
|
done + SHA256_BLOCK_SIZE); |
|
src = sctx->buf; |
|
asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" |
|
: "+S"(src), "+D"(dst) |
|
: "a"((long)-1), "c"((unsigned long)1)); |
|
done += SHA256_BLOCK_SIZE; |
|
src = data + done; |
|
} |
|
|
|
/* Process the left bytes from input data*/ |
|
if (len - done >= SHA256_BLOCK_SIZE) { |
|
asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" |
|
: "+S"(src), "+D"(dst) |
|
: "a"((long)-1), |
|
"c"((unsigned long)((len - done) / 64))); |
|
done += ((len - done) - (len - done) % 64); |
|
src = data + done; |
|
} |
|
partial = 0; |
|
} |
|
memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE); |
|
memcpy(sctx->buf + partial, src, len - done); |
|
|
|
return 0; |
|
} |
|
|
|
static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out) |
|
{ |
|
struct sha256_state *state = |
|
(struct sha256_state *)shash_desc_ctx(desc); |
|
unsigned int partial, padlen; |
|
__be64 bits; |
|
static const u8 padding[64] = { 0x80, }; |
|
|
|
bits = cpu_to_be64(state->count << 3); |
|
|
|
/* Pad out to 56 mod 64 */ |
|
partial = state->count & 0x3f; |
|
padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); |
|
padlock_sha256_update_nano(desc, padding, padlen); |
|
|
|
/* Append length field bytes */ |
|
padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits)); |
|
|
|
/* Swap to output */ |
|
padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8); |
|
|
|
return 0; |
|
} |
|
|
|
static int padlock_sha_export_nano(struct shash_desc *desc, |
|
void *out) |
|
{ |
|
int statesize = crypto_shash_statesize(desc->tfm); |
|
void *sctx = shash_desc_ctx(desc); |
|
|
|
memcpy(out, sctx, statesize); |
|
return 0; |
|
} |
|
|
|
static int padlock_sha_import_nano(struct shash_desc *desc, |
|
const void *in) |
|
{ |
|
int statesize = crypto_shash_statesize(desc->tfm); |
|
void *sctx = shash_desc_ctx(desc); |
|
|
|
memcpy(sctx, in, statesize); |
|
return 0; |
|
} |
|
|
|
static struct shash_alg sha1_alg_nano = { |
|
.digestsize = SHA1_DIGEST_SIZE, |
|
.init = padlock_sha1_init_nano, |
|
.update = padlock_sha1_update_nano, |
|
.final = padlock_sha1_final_nano, |
|
.export = padlock_sha_export_nano, |
|
.import = padlock_sha_import_nano, |
|
.descsize = sizeof(struct sha1_state), |
|
.statesize = sizeof(struct sha1_state), |
|
.base = { |
|
.cra_name = "sha1", |
|
.cra_driver_name = "sha1-padlock-nano", |
|
.cra_priority = PADLOCK_CRA_PRIORITY, |
|
.cra_blocksize = SHA1_BLOCK_SIZE, |
|
.cra_module = THIS_MODULE, |
|
} |
|
}; |
|
|
|
static struct shash_alg sha256_alg_nano = { |
|
.digestsize = SHA256_DIGEST_SIZE, |
|
.init = padlock_sha256_init_nano, |
|
.update = padlock_sha256_update_nano, |
|
.final = padlock_sha256_final_nano, |
|
.export = padlock_sha_export_nano, |
|
.import = padlock_sha_import_nano, |
|
.descsize = sizeof(struct sha256_state), |
|
.statesize = sizeof(struct sha256_state), |
|
.base = { |
|
.cra_name = "sha256", |
|
.cra_driver_name = "sha256-padlock-nano", |
|
.cra_priority = PADLOCK_CRA_PRIORITY, |
|
.cra_blocksize = SHA256_BLOCK_SIZE, |
|
.cra_module = THIS_MODULE, |
|
} |
|
}; |
|
|
|
static const struct x86_cpu_id padlock_sha_ids[] = { |
|
X86_MATCH_FEATURE(X86_FEATURE_PHE, NULL), |
|
{} |
|
}; |
|
MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids); |
|
|
|
static int __init padlock_init(void) |
|
{ |
|
int rc = -ENODEV; |
|
struct cpuinfo_x86 *c = &cpu_data(0); |
|
struct shash_alg *sha1; |
|
struct shash_alg *sha256; |
|
|
|
if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN)) |
|
return -ENODEV; |
|
|
|
/* Register the newly added algorithm module if on * |
|
* VIA Nano processor, or else just do as before */ |
|
if (c->x86_model < 0x0f) { |
|
sha1 = &sha1_alg; |
|
sha256 = &sha256_alg; |
|
} else { |
|
sha1 = &sha1_alg_nano; |
|
sha256 = &sha256_alg_nano; |
|
} |
|
|
|
rc = crypto_register_shash(sha1); |
|
if (rc) |
|
goto out; |
|
|
|
rc = crypto_register_shash(sha256); |
|
if (rc) |
|
goto out_unreg1; |
|
|
|
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n"); |
|
|
|
return 0; |
|
|
|
out_unreg1: |
|
crypto_unregister_shash(sha1); |
|
|
|
out: |
|
printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n"); |
|
return rc; |
|
} |
|
|
|
static void __exit padlock_fini(void) |
|
{ |
|
struct cpuinfo_x86 *c = &cpu_data(0); |
|
|
|
if (c->x86_model >= 0x0f) { |
|
crypto_unregister_shash(&sha1_alg_nano); |
|
crypto_unregister_shash(&sha256_alg_nano); |
|
} else { |
|
crypto_unregister_shash(&sha1_alg); |
|
crypto_unregister_shash(&sha256_alg); |
|
} |
|
} |
|
|
|
module_init(padlock_init); |
|
module_exit(padlock_fini); |
|
|
|
MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support."); |
|
MODULE_LICENSE("GPL"); |
|
MODULE_AUTHOR("Michal Ludvig"); |
|
|
|
MODULE_ALIAS_CRYPTO("sha1-all"); |
|
MODULE_ALIAS_CRYPTO("sha256-all"); |
|
MODULE_ALIAS_CRYPTO("sha1-padlock"); |
|
MODULE_ALIAS_CRYPTO("sha256-padlock");
|
|
|