OpenSSL RSA: Unable to encrypt/decrypt messages lo

I'm working on a simple program that uses OpenSSL to do basic RSA encryption and decryption. It is working fine for small messages (<16 bytes), but fails for anything over that. I understand that a limitation of public key cryptography is that you cannot encrypt anything longer than the key size. In my case, I'm using a 1024bit key, so I should have 128bytes to work with (maybe slightly less due to padding), correct? If so, that's not what I'm experiencing.

Here's the output from my program with 15 bytes:

Generating RSA keypair...done.
Message to encrypt: 0123456789ABCDE
16 bytes encrypted
Decrypted message: 0123456789ABCDE

And with 16 bytes:

Generating RSA keypair...done.
Message to encrypt: 0123456789ABCDEF
16 bytes encrypted
140153837057696:error:06065064:digital envelope routines:EVP_DecryptFinal_ex:bad     decrypt:evp_enc.c:467:
Decrypted message: (null)

It seems that no matter what, only a total of 16 bytes are encrypted.

My encryption function (updated with fix):

unsigned char* rsa_seal(EVP_PKEY *pub_key, unsigned char *msg, size_t **enc_msg_len, unsigned char **sym_key, int *sym_key_len, unsigned char **iv) {
    size_t msg_len = strlen((char*)msg);
    unsigned char *encrypt = malloc(EVP_PKEY_size(pub_key));

    EVP_CIPHER_CTX *ctx = malloc(sizeof(EVP_CIPHER_CTX));
    EVP_CIPHER_CTX_init(ctx);

    *sym_key = malloc(EVP_PKEY_size(pub_key));
    *iv = malloc(EVP_MAX_IV_LENGTH);

    **enc_msg_len = 0;

    if(!EVP_SealInit(ctx, EVP_aes_128_cbc(), sym_key, sym_key_len, *iv, &pub_key, 1)) {
        ERR_print_errors_fp(stderr);
        encrypt = NULL;
        goto return_free;
    }

    if(!EVP_SealUpdate(ctx, encrypt, (int*)*enc_msg_len, msg, (int)msg_len)) {
        ERR_print_errors_fp(stderr);
        encrypt = NULL;
        goto return_free;
    }

    if(!EVP_SealFinal(ctx, encrypt, (int*)*enc_msg_len)) {
        ERR_print_errors_fp(stderr);
        encrypt = NULL;
        goto return_free;
    }

    return_free:
    EVP_CIPHER_CTX_cleanup(ctx);
    free(ctx);
    ctx = NULL;

    return encrypt;
}

The corresponding decryption function (updated with fix):

char* rsa_open(EVP_PKEY *pri_key, unsigned char *enc_msg, size_t *enc_msg_len, unsigned char *sym_key, int sym_key_len, unsigned char *iv) {
    size_t dec_len = 0;
    unsigned char *decrypt = malloc((*enc_msg_len) + EVP_MAX_IV_LENGTH);
    if(decrypt == NULL) return NULL;

    EVP_CIPHER_CTX *ctx = malloc(sizeof(EVP_CIPHER_CTX));
    EVP_CIPHER_CTX_init(ctx);

    if(!EVP_OpenInit(ctx, EVP_aes_128_cbc(), sym_key, sym_key_len, iv, pri_key)) {
        ERR_print_errors_fp(stderr);
        decrypt = NULL;
        goto return_free;
    }

    if(!EVP_OpenUpdate(ctx, decrypt, (int*)&dec_len, enc_msg, (int)*enc_msg_len)) {
        ERR_print_errors_fp(stderr);
        decrypt = NULL;
        goto return_free;
    }

    if(!EVP_OpenFinal(ctx, decrypt, (int*)&dec_len)) {
        ERR_print_errors_fp(stderr);
        decrypt = NULL;
        goto return_free;
    }

    decrypt[dec_len] = '\0';

    return_free:
    EVP_CIPHER_CTX_cleanup(ctx);
    free(ctx);
    ctx = NULL;

    return (char*)decrypt;
}

The key generation function:

int rsa_init(EVP_PKEY **rsa_keypair) {
    EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);

    if(!EVP_PKEY_keygen_init(ctx)) {
        ERR_print_errors_fp(stderr);
        return -1;
    }

    if(!EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, KEY_LENGTH)) {
        ERR_print_errors_fp(stderr);
        return -1;
    }

    if(!EVP_PKEY_keygen(ctx, rsa_keypair)) {
        ERR_print_errors_fp(stderr);
        return -1;
    }

    EVP_PKEY_CTX_free(ctx);

    return 0;
}

And finally, my main:

int main() {
    EVP_PKEY      *rsa_keypair = NULL; // RSA keypair 
    char          msg[BUFFER];         // Message to encrypt
    unsigned char *encrypt = NULL;     // Encrypted message
    char          *decrypt = NULL;     // Decrypted message

    // Generate key pair
    printf("Generating RSA keypair...");
    if(rsa_init(&rsa_keypair) == -1) {
        fprintf(stderr, "\nError generating RSA keypair.\n");
        exit(1);
    }
    printf("done.\n");

    // Get the message to encrypt
    printf("Message to encrypt: ");
    fgets(msg, BUFFER-1, stdin);
    msg[strlen(msg)-1] = '\0';

    // Load error strings in anticipation of error
    ERR_load_crypto_strings();

    // Encrypt the message
    size_t *encrypt_len = malloc(sizeof(size_t));
    unsigned char *sym_key = NULL;
    unsigned char *iv = NULL;
    int sym_key_len;
    encrypt = rsa_seal(rsa_keypair, (unsigned char*)msg, &encrypt_len, &sym_key, &sym_key_len, &iv);
    printf("%d bytes encrypted\n", (int)*encrypt_len);

    // Decrypt it
    decrypt = rsa_open(rsa_keypair, (unsigned char*)encrypt, (size_t*)encrypt_len, sym_key, sym_key_len, iv);
    printf("Decrypted message: %s\n", decrypt);

    free(encrypt);
    free(decrypt);
    free(encrypt_len);
    free(sym_key);
    free(iv);
    EVP_PKEY_free(rsa_keypair);

    return 0;
}

Any help is greatly appreciated! Thank you.

EDIT: As pointed out by math below, it seems that the answer to my mistake was hiding in the OpenSSL here: https://www.openssl.org/docs/crypto/EVP_EncryptInit.html#

This is because you do not properly handle out and outl parameters in EVP_SealUpdate(), EVP_SealFinal(), EVP_OpenUpdate() and EVP_OpenFinal().

Each EVP_XxxxUpdate() and EVP_XxxxFinal() call will contribute to the output buffer. So, you are required to keep track of the seal/open process by summing each outl returned and providing the expected buffer each time (start of buffer + already handled bytes).

unsigned char* rsa_seal(...)
{
  ...
  **enc_msg_len = 0;

  EVP_SealUpdate(ctx, encrypt + **enc_msg_len, &outl, msg, (int)msg_len);
  **enc_msg_len += outl;

  EVP_SealFinal(ctx, encrypt + **enc_msg_len, &outl);
  **enc_msg_len += outl;
  ...
}

char* rsa_open(...)
{
  ...
  dec_len = 0;

  EVP_OpenUpdate(ctx, decrypt + dec_len, &outl, enc_msg, (int)*enc_msg_len);
  dec_len += outl;

  EVP_OpenFinal(ctx, decrypt + dec_len, &outl);
  dec_len += outl;
  ...
}

The program was working with 15-bytes buffer because in that case, the EVP_XxxxUpdate() call is returning 0 in outl (not enough data to seal/open a block), hiding the problem in your code logic.

Note: The data is not directly encrypted using the RSA key but using a generated symetric key (AES-128 in your case). This is why the block size is 16 bytes.