CryptEncrypt does not encrypt whole text

Question

I am trying to encrypt a text message with wincrypt. My code is however unpredictable. It doesn't encrypt/decrypt the whole plaintext but only a part of it. If i change the length of password (for example to "password123") it encrypts/decrypts a different amount of characters. Here is my code.

#include <windows.h>
#include <wincrypt.h>
#include <stdio.h>


int main()
{
 const char* passw = "password12";
 const char* toencrypt = "consectetur adipiscing elit. In tellus nisl,   sodales non arcu quis, sagittis maximus orci cras amet.";

 HCRYPTPROV hProv;
 HCRYPTHASH hHash;
 HCRYPTKEY hKey;
 DWORD todwSize = (DWORD)strlen(toencrypt);
 PBYTE pBuffer;

 CryptAcquireContext(&hProv, NULL, MS_DEF_PROV, PROV_RSA_FULL, CRYPT_DELETEKEYSET);
 CryptAcquireContext(&hProv, NULL, MS_DEF_PROV, PROV_RSA_FULL, CRYPT_NEWKEYSET);

 CryptCreateHash(hProv, CALG_MD5, 0, 0, &hHash);
 CryptHashData(hHash, (BYTE*)passw, strlen(passw), 0);
 CryptDeriveKey(hProv, CALG_RC4, hHash, 0, &hKey);

 //--------------------------------------------------------------------

 CryptEncrypt(hKey, 0, TRUE, 0, NULL, &todwSize, todwSize);

 pBuffer = (BYTE *)malloc(todwSize);
 strcpy((char*)pBuffer, toencrypt);

 CryptEncrypt(hKey, 0, TRUE, 0, pBuffer, &todwSize, todwSize);
 PBYTE pBreturn = pBuffer;
 const char* message =  (const char*)pBreturn;
 printf("%s", message); 

 //--------------------------------------------------------------------

 DWORD dwSize = (DWORD)strlen(message);
 PBYTE depBuffer;


 depBuffer = (BYTE *)malloc(1460);
 strcpy((char*)depBuffer, message);



 CryptDecrypt(hKey, 0, TRUE, 0, depBuffer, &dwSize);



 CryptDestroyKey(hKey);
 CryptDestroyHash(hHash);

 CryptReleaseContext(hProv, 0);
 if(GetLastError() != 0)
 {
   printf("%d", GetLastError());
 }
 PBYTE depBreturn = depBuffer;
 printf("%s", (const char*)depBreturn);
 printf("\n%d", strlen(message)); 
 return 0;
}

Thanks in advance for help!

Answer 1

think your code encrypt ok, but fail decrypt whole message because you use wrong length in call CryptDecrypt

your main error is DWORD dwSize = (DWORD)strlen(message); and strcpy((char*)depBuffer, message); but message is encrypted buffer, not 0 terminate ansi string. so you can not use strlen or strcpy on encrypted data - you have the encrypted data length returned by CryptEncrypt : todwSize - so you and must use with memcpy if you need copy encrypted buffer and pass todwSize as is to CryptDecrypt

also how noted Harry Johnston you incorrect use plain data/buffer size in call CryptEncrypt.

must be CryptEncrypt(hKey, 0, TRUE, 0, NULL, &(needSize = todwSize), 0) on first call (last parameter to CryptEncrypt dwBufLen must be 0 because you use NULL as buffer, and need use another variable DWORD needSize for get size of encrypted buffer and not overwrite size of plain text buffer (todwSize) then allocated needSize buffer, copy to it todSize and call CryptEncrypt(hKey, 0, TRUE, 0, pBuffer, &todwSize, needSize). but however for RC4 the encrypted and plain text size always equal. so needSize == todSize always in case RC4

also you need call CryptAcquireContext only once with flag CRYPT_VERIFYCONTEXT in your case. and always need check function result. so test code can be like this

int main()
{
    const char* passw = "password12";
    const char* toencrypt = "consectetur adipiscing elit. In tellus nisl,   sodales non arcu quis, sagittis maximus orci cras amet.";

    HCRYPTPROV hProv;
    HCRYPTHASH hHash;
    HCRYPTKEY hKey;
    DWORD todwSize = (DWORD)strlen(toencrypt), needSize;
    PBYTE pBuffer;

    if (CryptAcquireContext(&hProv, NULL, MS_DEF_PROV, PROV_RSA_FULL, CRYPT_VERIFYCONTEXT))
    {
        if (CryptCreateHash(hProv, CALG_MD5, 0, 0, &hHash))
        {
            if (CryptHashData(hHash, (BYTE*)passw, (DWORD)strlen(passw), 0) &&
                CryptDeriveKey(hProv, CALG_RC4, hHash, 0, &hKey))
            {
                if (CryptEncrypt(hKey, 0, TRUE, 0, NULL, &(needSize = todwSize), 0))
                {
                    memcpy(pBuffer = (BYTE *)_alloca(needSize), toencrypt, todwSize);

                    if (CryptEncrypt(hKey, 0, TRUE, 0, pBuffer, &todwSize, needSize))
                    {
                        if (CryptDecrypt(hKey, 0, TRUE, 0, pBuffer, &todwSize))
                        {
                            if (memcmp(pBuffer, toencrypt, strlen(toencrypt)))
                            {
                                __debugbreak();
                            }
                        }
                    }
                }
                CryptDestroyKey(hKey);
            }
            CryptDestroyHash(hHash);
        }
        CryptReleaseContext(hProv, 0);
    }

    return 0;
}