summaryrefslogtreecommitdiff
path: root/rsa.c
diff options
context:
space:
mode:
Diffstat (limited to 'rsa.c')
-rw-r--r--rsa.c113
1 files changed, 75 insertions, 38 deletions
diff --git a/rsa.c b/rsa.c
index d0b5bbf5..5ecacef9 100644
--- a/rsa.c
+++ b/rsa.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa.c,v 1.31 2014/02/02 03:44:31 djm Exp $ */
+/* $OpenBSD: rsa.c,v 1.32 2014/06/24 01:13:21 djm Exp $ */
/*
* Author: Tatu Ylonen <ylo@cs.hut.fi>
* Copyright (c) 1995 Tatu Ylonen <ylo@cs.hut.fi>, Espoo, Finland
@@ -67,85 +67,122 @@
#include <stdarg.h>
#include <string.h>
-#include "xmalloc.h"
#include "rsa.h"
#include "log.h"
+#include "ssherr.h"
-void
+int
rsa_public_encrypt(BIGNUM *out, BIGNUM *in, RSA *key)
{
- u_char *inbuf, *outbuf;
- int len, ilen, olen;
+ u_char *inbuf = NULL, *outbuf = NULL;
+ int len, ilen, olen, r = SSH_ERR_INTERNAL_ERROR;
if (BN_num_bits(key->e) < 2 || !BN_is_odd(key->e))
- fatal("rsa_public_encrypt() exponent too small or not odd");
+ return SSH_ERR_INVALID_ARGUMENT;
olen = BN_num_bytes(key->n);
- outbuf = xmalloc(olen);
+ if ((outbuf = malloc(olen)) == NULL) {
+ r = SSH_ERR_ALLOC_FAIL;
+ goto out;
+ }
ilen = BN_num_bytes(in);
- inbuf = xmalloc(ilen);
+ if ((inbuf = malloc(ilen)) == NULL) {
+ r = SSH_ERR_ALLOC_FAIL;
+ goto out;
+ }
BN_bn2bin(in, inbuf);
if ((len = RSA_public_encrypt(ilen, inbuf, outbuf, key,
- RSA_PKCS1_PADDING)) <= 0)
- fatal("rsa_public_encrypt() failed");
+ RSA_PKCS1_PADDING)) <= 0) {
+ r = SSH_ERR_LIBCRYPTO_ERROR;
+ goto out;
+ }
- if (BN_bin2bn(outbuf, len, out) == NULL)
- fatal("rsa_public_encrypt: BN_bin2bn failed");
+ if (BN_bin2bn(outbuf, len, out) == NULL) {
+ r = SSH_ERR_LIBCRYPTO_ERROR;
+ goto out;
+ }
+ r = 0;
- explicit_bzero(outbuf, olen);
- explicit_bzero(inbuf, ilen);
- free(outbuf);
- free(inbuf);
+ out:
+ if (outbuf != NULL) {
+ explicit_bzero(outbuf, olen);
+ free(outbuf);
+ }
+ if (inbuf != NULL) {
+ explicit_bzero(inbuf, ilen);
+ free(inbuf);
+ }
+ return r;
}
int
rsa_private_decrypt(BIGNUM *out, BIGNUM *in, RSA *key)
{
- u_char *inbuf, *outbuf;
- int len, ilen, olen;
+ u_char *inbuf = NULL, *outbuf = NULL;
+ int len, ilen, olen, r = SSH_ERR_INTERNAL_ERROR;
olen = BN_num_bytes(key->n);
- outbuf = xmalloc(olen);
+ if ((outbuf = malloc(olen)) == NULL) {
+ r = SSH_ERR_ALLOC_FAIL;
+ goto out;
+ }
ilen = BN_num_bytes(in);
- inbuf = xmalloc(ilen);
+ if ((inbuf = malloc(ilen)) == NULL) {
+ r = SSH_ERR_ALLOC_FAIL;
+ goto out;
+ }
BN_bn2bin(in, inbuf);
if ((len = RSA_private_decrypt(ilen, inbuf, outbuf, key,
RSA_PKCS1_PADDING)) <= 0) {
- error("rsa_private_decrypt() failed");
- } else {
- if (BN_bin2bn(outbuf, len, out) == NULL)
- fatal("rsa_private_decrypt: BN_bin2bn failed");
+ r = SSH_ERR_LIBCRYPTO_ERROR;
+ goto out;
+ } else if (BN_bin2bn(outbuf, len, out) == NULL) {
+ r = SSH_ERR_LIBCRYPTO_ERROR;
+ goto out;
+ }
+ r = 0;
+ out:
+ if (outbuf != NULL) {
+ explicit_bzero(outbuf, olen);
+ free(outbuf);
+ }
+ if (inbuf != NULL) {
+ explicit_bzero(inbuf, ilen);
+ free(inbuf);
}
- explicit_bzero(outbuf, olen);
- explicit_bzero(inbuf, ilen);
- free(outbuf);
- free(inbuf);
- return len;
+ return r;
}
/* calculate p-1 and q-1 */
-void
+int
rsa_generate_additional_parameters(RSA *rsa)
{
- BIGNUM *aux;
- BN_CTX *ctx;
+ BIGNUM *aux = NULL;
+ BN_CTX *ctx = NULL;
+ int r;
- if ((aux = BN_new()) == NULL)
- fatal("rsa_generate_additional_parameters: BN_new failed");
if ((ctx = BN_CTX_new()) == NULL)
- fatal("rsa_generate_additional_parameters: BN_CTX_new failed");
+ return SSH_ERR_ALLOC_FAIL;
+ if ((aux = BN_new()) == NULL) {
+ r = SSH_ERR_ALLOC_FAIL;
+ goto out;
+ }
if ((BN_sub(aux, rsa->q, BN_value_one()) == 0) ||
(BN_mod(rsa->dmq1, rsa->d, aux, ctx) == 0) ||
(BN_sub(aux, rsa->p, BN_value_one()) == 0) ||
- (BN_mod(rsa->dmp1, rsa->d, aux, ctx) == 0))
- fatal("rsa_generate_additional_parameters: BN_sub/mod failed");
-
+ (BN_mod(rsa->dmp1, rsa->d, aux, ctx) == 0)) {
+ r = SSH_ERR_LIBCRYPTO_ERROR;
+ goto out;
+ }
+ r = 0;
+ out:
BN_clear_free(aux);
BN_CTX_free(ctx);
+ return r;
}