Migrating from OpenSSL 1.1 to 3.0: How to generate DH key pair with 32-byte prime using EVP_PKEY-DH interface?

Question

I'm migrating my Diffie-Hellman (DH) implementation from OpenSSL 1.1 to OpenSSL 3.0. Previously, I used APIs like DH_set0_pqg, but these are now deprecated. I am switching to the EVP_PKEY-DH interface as recommended.

My application uses a 32-byte prime for DH, I know 32-byte keys are not recommended for use, but they cannot be changed due to compatibility requirements across many systems. When I try to generate a DH key pair with this 32-byte prime using the new EVP interface, I get the following error:

0002EF4E7B7F0000:error:0280007E:Diffie-Hellman routines:(unknown function):modulus too small:crypto/dh/dh_key.c:283:

Is there any way to generate a DH key pair with a 32-byte prime in OpenSSL 3.0? Or is there a workaround for this limitation?

Below is a minimal sample code demonstrating the issue:

#include <stdio.h>
#include <string.h>
#include <openssl/evp.h>
#include <openssl/core_names.h>
#include <openssl/err.h>
#include <openssl/param_build.h>

#define MY_ERR_ERR -1
#define MY_ERR_OK 0
typedef struct my_evp_st {
    EVP_PKEY *params;
    EVP_PKEY *pkey;
} MY_EVP_DH;

void my_evp_free(MY_EVP_DH *dh) {
    if (dh) {
        if (dh->params)
            EVP_PKEY_free(dh->params);
        dh->params = NULL;
        if (dh->pkey)
            EVP_PKEY_free(dh->pkey);
        dh->pkey = NULL;
        OPENSSL_free(dh);
        dh = NULL;
    }
}

MY_EVP_DH *my_evp_new(void) {
    MY_EVP_DH *dh = OPENSSL_zalloc(sizeof(MY_EVP_DH));
    if (!dh)
        return NULL;
    return dh;
}

int my_evp_set_pqg(MY_EVP_DH *dh, BIGNUM *p, BIGNUM *q, BIGNUM *g) {
    int ret = MY_ERR_ERR;
    OSSL_PARAM_BLD *bld = NULL;
    EVP_PKEY_CTX *paramgen_ctx = NULL;
    OSSL_PARAM *key_params = NULL;
    do {
        paramgen_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL);
        if (!paramgen_ctx) {
            fprintf(stderr, "Error: EVP_PKEY_CTX_new_from_name failed\n");
            break;
        }
        if (EVP_PKEY_fromdata_init(paramgen_ctx) <= 0) {
            fprintf(stderr, "EVP_PKEY_fromdata_init failed\n");
            break;
        }
        bld = OSSL_PARAM_BLD_new();
        if (!bld) {
            fprintf(stderr, "Error: OSSL_PARAM_BLD_new failed\n");
            break;
        }
        if (p) {
            if (!OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_FFC_P, p)) {
                fprintf(stderr, "Error: OSSL_PARAM_BLD_push_BN (P) failed\n");
                break;
            }
        }
        if (g) {
            if (!OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_FFC_G, g)) {
                fprintf(stderr, "Error: OSSL_PARAM_BLD_push_BN (G) failed\n");
                break;
            }
        }
        if (q) {
            if (!OSSL_PARAM_BLD_push_BN(bld, OSSL_PKEY_PARAM_FFC_Q, q)) {
                fprintf(stderr, "Error: OSSL_PARAM_BLD_push_BN (Q) failed\n");
                break;
            }
        }
        key_params = OSSL_PARAM_BLD_to_param(bld);
        if (!key_params) {
            fprintf(stderr, "Error: OSSL_PARAM_BLD_to_param failed\n");
            break;
        }
        if (EVP_PKEY_fromdata(paramgen_ctx, &dh->params, EVP_PKEY_KEY_PARAMETERS, key_params) <= 0) {
            fprintf(stderr, "EVP_PKEY_fromdata failed\n");
            break;
        }
        ret = MY_ERR_OK;
    } while(0);

    if(paramgen_ctx)
        EVP_PKEY_CTX_free(paramgen_ctx);
    if(key_params)
        OSSL_PARAM_free(key_params);
    if(bld)
        OSSL_PARAM_BLD_free(bld);
    return ret;
}

int my_evp_generate_key(MY_EVP_DH *dh) {
    EVP_PKEY_CTX *ctx = NULL;
    int ret = MY_ERR_ERR;
    if (!dh || !dh->params)
        return MY_ERR_ERR;
    ctx = EVP_PKEY_CTX_new_from_pkey(NULL, dh->params, NULL);
    if (!ctx) {
        fprintf(stderr, "EVP_PKEY_CTX_new: cannot create DH context\n");
        return MY_ERR_ERR;
    }
    if (EVP_PKEY_keygen_init(ctx) != 1) {
        fprintf(stderr, "EVP_PKEY_keygen_init: cannot init DH keygen\n");
        EVP_PKEY_CTX_free(ctx);
        return MY_ERR_ERR;
    }
    if (EVP_PKEY_keygen(ctx, &dh->pkey) != 1) {
        fprintf(stderr, "EVP_PKEY_keygen: cannot generate DH key\n");
        EVP_PKEY_CTX_free(ctx);
        return MY_ERR_ERR;
    }
    EVP_PKEY_CTX_free(ctx);
    ret = MY_ERR_OK;
    return ret;
}

static const unsigned char prime_bytes[32] = {
/* 32 byte prime */
};

static const unsigned char generator_bytes[32] = {
   /* 32 byte Generator */
};

int main () {
    MY_EVP_DH *dh_bob = NULL;
    dh_bob = my_evp_new();
    if (!dh_bob) {
        fprintf(stderr, "Failed to allocate DH for Bob\n");
        return 1;
    }
    BIGNUM *p = NULL;
    BIGNUM *g = NULL;
    p = BN_bin2bn(prime_bytes, sizeof(prime_bytes), NULL);
    g = BN_bin2bn(generator_bytes, sizeof(generator_bytes), NULL);
    if (BN_is_zero(p) || BN_is_zero(g)) {
        fprintf(stderr, "Invalid parameters: p and g must be non-zero\n");
        ERR_print_errors_fp(stderr);
        BN_free(p);
        BN_free(g);
        my_evp_free(dh_bob);
        return 1;
    }

    if (my_evp_set_pqg(dh_bob, (BIGNUM *)p, NULL, (BIGNUM *)g) != MY_ERR_OK) {
        fprintf(stderr, "Failed to set p and g for Bob\n");
        ERR_print_errors_fp(stderr);
        my_evp_free(dh_bob);
        return 1;
    }
    if (my_evp_generate_key(dh_bob) != MY_ERR_OK) {
        fprintf(stderr, "Failed to generate DH key for Bob\n");
        ERR_print_errors_fp(stderr);
        my_evp_free(dh_bob);
        return 1;
    }

    my_evp_free(dh_bob);
    return 0;
}

Andrew Henle · Accepted Answer · 2025-10-29 20:49:46Z

1

In OpenSSL v3.0.0, the minimum DH modulus size is hardcoded at 512 bits:

#define DH_MIN_MODULUS_BITS     512

And the error is generated here:

if (BN_num_bits(dh->params.p) < DH_MIN_MODULUS_BITS) {
    ERR_raise(ERR_LIB_DH, DH_R_MODULUS_TOO_SMALL);
    return 0;
}

Your only option for creating a key with a 256-bit modulus (32 bytes) with OpenSSL 3.0.0 appears to be modifying OpenSSL and recompiling, putting you into the OpenSSL-distro-maintenance business.

Note that merely generating the key isn't your only issue - it's likely to cause failures in many, many ways.

answered Oct 29 at 20:49

Andrew Henle

35k3 gold badges29 silver badges62 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

Migrating from OpenSSL 1.1 to 3.0: How to generate DH key pair with 32-byte prime using EVP_PKEY-DH interface?

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related