Open File with Non ASCII Characters

Question

I am trying to compute SHA-256 of file. I have the following code that gives correct value of Checksum when the path is valid ie. It is ASCII. I have the following code:

#include <openssl\evp.h>
#include <sys\stat.h>
#include <iostream>
#include <string>
#include <fstream>
#include <cstdio>
const int MAX_BUFFER_SIZE = 1024;
std::string FileChecksum(std::string, std::string);

long long int GetFileSize(std::string filename)
{
    struct _stat64 stat_buf;
    int rc = _stat64(filename.c_str(), &stat_buf);
    return rc == 0 ? stat_buf.st_size : -1;
}

std::string fname = "D:\\Private\\Test\\asdf.txt"; // Need to support this D:\\Private\\Test\\सर्वज्ञ पन्त.txt

int main()
{
    std::string checksum = FileChecksum(fname , "sha256");
    std::cout << checksum << std::endl;
    return 0;
}


static std::string FileChecksum(std::string file_path, std::string algorithm="sha256")
{
    EVP_MD_CTX *mdctx;
    const EVP_MD *md;
    unsigned char md_value[EVP_MAX_MD_SIZE];
    int i;
    unsigned int md_len;

    OpenSSL_add_all_digests();
    md = EVP_get_digestbyname(algorithm.c_str());

    if(!md) {
        printf("Unknown message digest %s\n",algorithm);
        return "";
    }

    mdctx = EVP_MD_CTX_create();
    std::ifstream readfile(file_path,std::ifstream::binary);
    if(!readfile.is_open())
    {
        std::cout << "COuldnot open file\n";
        return "";
    }
    readfile.seekg(0, std::ios::end);
    long long filelen = readfile.tellg();
    std::cout << "LEN IS " << filelen << std::endl;
    readfile.seekg(0, std::ios::beg);
    if(filelen == -1)
    {
        std::cout << "Return Null \n";
        return "";
    }

    EVP_DigestInit_ex(mdctx, md, NULL);
    long long temp_fil = filelen;
    while(!readfile.eof() && readfile.is_open() && temp_fil>0)
    {

        int bufferS = (temp_fil < MAX_BUFFER_SIZE) ? temp_fil : MAX_BUFFER_SIZE;
        char *buffer = new char[bufferS+1];
        buffer[bufferS] = 0;
        readfile.read(buffer, bufferS);
        EVP_DigestUpdate(mdctx, buffer, bufferS);
        temp_fil -= bufferS;
        delete[] buffer;
    }
    EVP_DigestFinal_ex(mdctx, md_value, &md_len);
    EVP_MD_CTX_destroy(mdctx);
    char str[128] = { 0 };
    char *ptr = str;
    std::string ret;
    for(i = 0; i < md_len; i++)
    {
    //_snprintf(checksum_msg+cx,md_len-cx,"%02x",md_value[i]);
        sprintf(ptr,"%02x", md_value[i]);
        ptr += 2;
    }

    ret = str;
    /* Call this once before exit. */
    EVP_cleanup();
    return ret;
}

The code will give correct checksum of files with valid name. But once non-ascii character files are given, the program fails. I used std::wstring and it seems to fix the issue but the site here discourages to use std::wstring by saying Do not use wchar_t or std::wstring in any place other than adjacent point to APIs accepting UTF-16. If I were to follow this, how do I make this code work for all types of path. I am using VS2010.

Answer 1

wchar_t is not portable across multiple platforms, as it is 2 bytes (UTF-16) on some platforms (Windows) but is 4 bytes (UTF-32) on other platforms (Linux, etc). That is what the site is warning you about.

In your particular case, you are only focusing on Windows, so std::wstring is perfectly fine to use, since it uses UTF-16, which is the same encoding that the Win32 API uses all over the place. What you are looking for is Microsoft's _wstat64() function, and Microsoft's non-standard std::ifstream constructor that accepts a wchar_t* filename:

long long int GetFileSize(std::wstring filename)
{
    struct _stat64 stat_buf;
    int rc = _wstat64(filename.c_str(), &stat_buf);
    return rc == 0 ? stat_buf.st_size : -1;
}

std::wstring file_path = L"D:\\Private\\Test\\सर्वज्ञ पन्त.txt";

...

static std::string FileChecksum(std::wstring file_path, std::string algorithm="sha256")
{
    ...
    std::ifstream readfile(file_path.c_str(), std::ifstream::binary);
    ...
}

That being said, your FileChecksum() function is more complicated then it needs to be, it is not cleaning up correctly if an error occurs, it is not validating that std::ifstream::read() is actually reading as many bytes as you requested (it could read less), and it is misusing std::ifstream::eof().

Try something more like this instead:

#include <openssl\evp.h>
#include <sys\stat.h>
#include <iostream>
#include <string>
#include <fstream>
#include <sstream>
#include <iomanip>

const int MAX_BUFFER_SIZE = 1024;
std::string FileChecksum(std::wstring file_path, std::string algorithm = "sha256");

std::wstring fname = L"D:\\Private\\Test\\सर्वज्ञ पन्त.txt";

int main()
{
    std::string checksum = FileChecksum(fname, "sha256");
    std::cout << checksum << std::endl;
    return 0;
}

std::string FileChecksum(std::wstring file_path, std::string algorithm)
{
    EVP_MD_CTX *mdctx = NULL;
    const EVP_MD *md;
    unsigned char md_value[EVP_MAX_MD_SIZE];
    char buffer[MAX_BUFFER_SIZE];
    unsigned int md_len;
    std::ostringstream oss;
    std::string ret;

    std::ifstream readfile(file_path.c_str(), std::ifstream::binary);
    if (readfile.fail())
    {
        std::cout << "Could not open file\n";
        goto finished;
    }

    OpenSSL_add_all_digests();

    md = EVP_get_digestbyname(algorithm.c_str());    
    if (!md) {
        std::cout << "Unknown message digest " << algorithm << "\n";
        goto cleanup;
    }

    mdctx = EVP_MD_CTX_create();
    if (!mdctx) {
        std::cout << "Could not create context for message digest " << algorithm << "\n";
        goto cleanup;
    }

    EVP_DigestInit_ex(mdctx, md, NULL);

    do
    {
        readfile.read(buffer, sizeof(buffer));
        if ((readfile.fail()) && (!readfile.eof()))
        {
            std::cout << "Could not read from file\n";
            goto cleanup;
        }

        EVP_DigestUpdate(mdctx, buffer, readfile.gcount());
    }
    while (!readfile.eof());

    EVP_DigestFinal_ex(mdctx, md_value, &md_len);

    for(unsigned int i = 0; i < md_len; i++)
        oss << std::hex << std::setw(2) << std::setfill('0') << (int) md_value[i];
    ret = oss.str();

cleanup:
    if (mdctx) EVP_MD_CTX_destroy(mdctx);
    EVP_cleanup();

finished:
    return ret;
}