openssl -decrypt by Java

Question

all! I am trying to resolve the issue.

I have bat-file with command:

openssl smime -decrypt -binary -inform DER -recip [path to certificate] -inkey  [path to private key] <[path to encoded file] >[path to decoded file]

I have implement it on Java.

So I need decode file with RSA private key.

First, I tried this way:

package javatest;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.security.PrivateKey;
import java.security.Security;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openssl.PEMKeyPair;
import org.bouncycastle.openssl.PEMParser;
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;

/**
 * Test decrypt
 *
 * @author a.chernyy
 */
public class JavaTest {

    /**
     * String to hold name of the encryption algorithm.
     */
    public static final String ALGORITHM = "RSA";

    /**
     * String to hold the path to the keys' dir.
     */
    public static final String KEYS_DIR = "D:" + File.separator + File.separator + "keystore" + File.separator;

    /**
     * String to hold the name of the private key file.
     */
    public static final String PRIVATE_KEY_FILE = KEYS_DIR + "priv.key";

    /**
     * String to hold name of the public key file.
     */
    public static final String CERT_FILE = KEYS_DIR + "cert.cer";

    /**
     * String to hold name of the encrypted file.
     */
    public static final String ENCRYPTED_FILE = "D:" + File.separator + "Temp" + File.separator + "encrypded.xml";

    /**
     * String to hold name of the decrypted file.
     */
    public static final String DECRYPTED_FILE = "D:" + File.separator + "Temp" + File.separator + "decrypted.xml";

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        try {
            //get private key
            File keyFl = new File(PRIVATE_KEY_FILE);
            Security.addProvider(new BouncyCastleProvider());
            PEMParser pemParser = new PEMParser(new InputStreamReader(new FileInputStream(keyFl)));
            PEMKeyPair pemKeyPair = (PEMKeyPair) pemParser.readObject();
            JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider("BC");
            PrivateKey key = converter.getPrivateKey(pemKeyPair.getPrivateKeyInfo());

            //decrypt file           
            Cipher cipher = Cipher.getInstance(ALGORITHM);
            cipher.init(Cipher.DECRYPT_MODE, key);
            InputStream is = new FileInputStream(ENCRYPTED_FILE);
            OutputStream out = new FileOutputStream(DECRYPTED_FILE);
            CipherInputStream cis = new CipherInputStream(is, cipher);
            byte[] buffer = new byte[1024];
            int r;
            while ((r = cis.read(buffer)) > 0) {
                out.write(buffer, 0, r);
            }
            cis.close();
            is.close();
            out.close();
        } catch (Exception e) {
            System.out.println("It's a pity...");
            System.err.println(e.getMessage());
        }

        System.out.println("THE END");
    }

}

But I get error:

javax.crypto.IllegalBlockSizeException: Data must not be longer than 128 bytes

Also I tried use this solution. I implemented it that way:

package javatest;

//some imports...

/**
 * Test decrypt main class
 *
 * @author a.chernyy
 */
public class JavaTest {

    /**
     * String to hold name of the encryption algorithm.
     */
    public static final String ALGORITHM = "RSA";

    /**
     * String to hold the path to the keys' dir.
     */
    public static final String KEYS_DIR = "D:" + File.separator + File.separator + "keystore" + File.separator;

    /**
     * String to hold the name of the private key file.
     */
    public static final String PRIVATE_KEY_FILE = KEYS_DIR + "priv.key";

    /**
     * String to hold name of the public key file.
     */
    public static final String CERT_FILE = KEYS_DIR + "cert.cer";

    /**
     * String to hold name of the encrypted file.
     */
    public static final String ENCRYPTED_FILE = "D:" + File.separator + "Temp" + File.separator + "encrypded.xml";

    /**
     * String to hold name of the decrypted file.
     */
    public static final String DECRYPTED_FILE = "D:" + File.separator + "Temp" + File.separator + "decrypted.xml";

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        try {
            RSA rsa = RSA.getInstance(PRIVATE_KEY_FILE);
            rsa.decrypt(ENCRYPTED_FILE, DECRYPTED_FILE);
        } catch (Exception e) {
            System.out.println("It's a pity...");
            System.err.println(e.getMessage());
        }

        System.out.println("THE END");
    }

}

And:

package javatest;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.Security;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openssl.PEMKeyPair;
import org.bouncycastle.openssl.PEMParser;
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;

/**
 * Decrypt class
 *
 * @link http://coding.westreicher.org/?p=23
 * @author a.chernyy
 */
public class RSA {

    /**
     * Singleton class object RSA
     */
    private static volatile RSA instance;

    /**
     * Private key
     */
    private final PrivateKey privateKey;

    /**
     * Cipher
     */
    private final Cipher cipher;

    /**
     * Constructor
     *
     * @throws NoSuchAlgorithmException
     * @throws NoSuchPaddingException
     */
    private RSA(String privateKeyPath) throws NoSuchAlgorithmException, NoSuchPaddingException, FileNotFoundException, IOException {
        //create cipher
        this.cipher = Cipher.getInstance("RSA");

        //get private key
        File keyFl = new File(privateKeyPath);
        Security.addProvider(new BouncyCastleProvider());
        PEMParser pemParser = new PEMParser(new InputStreamReader(new FileInputStream(keyFl)));
        PEMKeyPair pemKeyPair = (PEMKeyPair) pemParser.readObject();
        JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider("BC");
        this.privateKey = converter.getPrivateKey(pemKeyPair.getPrivateKeyInfo());
    }

    //Static methods
    /**
     * Static method getInstance return single refer on object RSA. If object
     * not exists, it will be created
     *
     * @param privateKeyPath
     * @throws NoSuchAlgorithmException
     * @throws NoSuchPaddingException
     * @throws java.io.FileNotFoundException
     * @return RSA
     */
    public static RSA getInstance(String privateKeyPath) throws NoSuchAlgorithmException, NoSuchPaddingException, FileNotFoundException, IOException {
        if (instance == null) {
            synchronized (RSA.class) {
                if (instance == null) {
                    instance = new RSA(privateKeyPath);
                }
            }
        }
        return instance;
    }

    /**
     * Block chipher
     *
     * @param bytes
     * @param mode
     * @return
     * @throws IllegalBlockSizeException
     * @throws BadPaddingException
     */
    private byte[] blockCipher(byte[] bytes, int mode) throws IllegalBlockSizeException, BadPaddingException {
        // string initialize 2 buffers.
        // scrambled will hold intermediate results
        byte[] scrambled = new byte[0];

        // toReturn will hold the total result
        byte[] toReturn = new byte[0];
        // if we encrypt we use 100 byte long blocks. Decryption requires 128 byte long blocks (because of RSA)
        int length = (mode == Cipher.ENCRYPT_MODE) ? 100 : 128;

        // another buffer. this one will hold the bytes that have to be modified in this step
        byte[] buffer = new byte[length];

        for (int i = 0; i < bytes.length; i++) {

            // if we filled our buffer array we have our block ready for de- or encryption
            if ((i > 0) && (i % length == 0)) {
                //execute the operation
                scrambled = cipher.doFinal(buffer);
                // add the result to our total result.
                toReturn = append(toReturn, scrambled);
                // here we calculate the length of the next buffer required
                int newlength = length;

                // if newlength would be longer than remaining bytes in the bytes array we shorten it.
                if (i + length > bytes.length) {
                    newlength = bytes.length - i;
                }
                // clean the buffer array
                buffer = new byte[newlength];
            }
            // copy byte into our buffer.
            buffer[i % length] = bytes[i];
        }

        // this step is needed if we had a trailing buffer. should only happen when encrypting.
        // example: we encrypt 110 bytes. 100 bytes per run means we "forgot" the last 10 bytes. they are in the buffer array
        scrambled = cipher.doFinal(buffer);

        // final step before we can return the modified data.
        toReturn = append(toReturn, scrambled);

        return toReturn;
    }

    /**
     * Concatinate bytes
     *
     * @param prefix
     * @param suffix
     * @return
     */
    private byte[] append(byte[] prefix, byte[] suffix) {
        byte[] toReturn = new byte[prefix.length + suffix.length];
        for (int i = 0; i < prefix.length; i++) {
            toReturn[i] = prefix[i];
        }
        for (int i = 0; i < suffix.length; i++) {
            toReturn[i + prefix.length] = suffix[i];
        }
        return toReturn;
    }

    public void decrypt(String filePath, String fileDecryptPath) throws Exception {
        //Convert file into bytes
        this.cipher.init(Cipher.DECRYPT_MODE, this.privateKey);
        File encryptedFile = new File(filePath);
        FileInputStream isEncryptedFile = new FileInputStream(encryptedFile);
        byte encryptedFileData[] = new byte[(int) encryptedFile.length()];
        isEncryptedFile.read(encryptedFileData);
        byte[] bts = encryptedFileData;

        //decrypt
        byte[] decrypted = blockCipher(bts, Cipher.DECRYPT_MODE);

        //Push decrypted data into file
        //...
    }
}

But I got the error:

Decryption error

Can anyone suggest a solution?

Answer 1

S/MIME is based on a standard called PKCS #7, the Cryptographic Message Syntax. An S/MIME–encrypted message is not simply the output of an encryption operation; it's a package of meta-data and (optionally) the encrypted content itself. It has to be parsed to find the cipher text, the encrypted content encryption key(s), the algorithm and parameters applied to the content encryption key, the algorithm and parameters applied to the content itself, etc.

A CMS enveloped-data structure contains information for one or more recipients. Each bundle of recipient information contains an identifier for the recipient and a bit of cipher text encrypted with that recipient's public key. If you have the private key corresponding to one of those recipients, you can decrypt the bit of cipher text, which turns out to be another key for a symmetric cipher. With that, and more meta-data about the symmetric cipher mode and parameters, you can decrypt the actual message itself.

To decrypt it, you'll need an S/MIME or CMS library. (Library recommendations are off-topic, but you can look at BouncyCastle.)

Answer 2

I've found the solution for my question! Thank you all for your advises.

I've found the source. Thank you very much to author of this source.

I implemented it that way:

package javatest;

import org.bouncycastle.cms.*;
import org.bouncycastle.cms.jcajce.JceCMSContentEncryptorBuilder;
import org.bouncycastle.cms.jcajce.JceKeyTransEnvelopedRecipient;
import org.bouncycastle.cms.jcajce.JceKeyTransRecipientInfoGenerator;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.operator.OutputEncryptor;

import java.io.*;
import java.nio.file.Files;
import java.security.*;
import java.security.cert.CertificateEncodingException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Collection;
import java.util.Enumeration;
import java.util.Iterator;

/**
 *
 * @author kagkarlsson
 * @link https://github.com/kagkarlsson/cms-decrypt-example/blob/master/src/main/java/no/posten/dpost/EncryptAndDecrypt.java
 */
public class EncryptAndDecrypt {
    public static void decrypt(PrivateKey privateKey, File encrypted, File decryptedDestination) throws IOException, CMSException {
        byte[] encryptedData = Files.readAllBytes(encrypted.toPath());

        CMSEnvelopedDataParser parser = new CMSEnvelopedDataParser(encryptedData);

        RecipientInformation recInfo = getSingleRecipient(parser);
        Recipient recipient = new JceKeyTransEnvelopedRecipient(privateKey);

        try (InputStream decryptedStream = recInfo.getContentStream(recipient).getContentStream()) {
            Files.copy(decryptedStream, decryptedDestination.toPath());
        }

        System.out.println(String.format("Decrypted '%s' to '%s'", encrypted.getAbsolutePath(), decryptedDestination.getAbsolutePath()));
    }

    public static void encrypt(X509Certificate cert, File source, File destination) throws CertificateEncodingException, CMSException, IOException {
        CMSEnvelopedDataStreamGenerator gen = new CMSEnvelopedDataStreamGenerator();
        gen.addRecipientInfoGenerator(new JceKeyTransRecipientInfoGenerator(cert));
        OutputEncryptor encryptor = new JceCMSContentEncryptorBuilder(CMSAlgorithm.AES256_CBC).setProvider(BouncyCastleProvider.PROVIDER_NAME).build();

        try (FileOutputStream fileStream = new FileOutputStream(destination);
                OutputStream encryptingStream = gen.open(fileStream, encryptor)) {

            byte[] unencryptedContent = Files.readAllBytes(source.toPath());
            encryptingStream.write(unencryptedContent);
        }

        System.out.println(String.format("Encrypted '%s' to '%s'", source.getAbsolutePath(), destination.getAbsolutePath()));
    }

    public static X509Certificate getX509Certificate(File certificate) throws IOException, CertificateException {
        try (InputStream inStream = new FileInputStream(certificate)) {
            CertificateFactory cf = CertificateFactory.getInstance("X.509");
            return (X509Certificate) cf.generateCertificate(inStream);
        }
    }

    public static PrivateKey getPrivateKey(File file, String password) throws Exception {
        KeyStore ks = KeyStore.getInstance("PKCS12");
        try (FileInputStream fis = new FileInputStream(file)) {
            ks.load(fis, password.toCharArray());
        }

        Enumeration<String> aliases = ks.aliases();
        String alias = aliases.nextElement();
        return (PrivateKey) ks.getKey(alias, password.toCharArray());
    }

    private static RecipientInformation getSingleRecipient(CMSEnvelopedDataParser parser) {
        Collection recInfos = parser.getRecipientInfos().getRecipients();
        Iterator recipientIterator = recInfos.iterator();
        if (!recipientIterator.hasNext()) {
            throw new RuntimeException("Could not find recipient");
        }
        return (RecipientInformation) recipientIterator.next();
    }
}

and

package javatest;

import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.security.PrivateKey;
import java.security.Security;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.openssl.PEMKeyPair;
import org.bouncycastle.openssl.PEMParser;
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;

/**
 * Test decrypt file
 *
 * @author a.chernyy
 */
public class JavaTest {

    /**
     * String to hold name of the encryption algorithm.
     */
    public static final String ALGORITHM = "RSA";

    /**
     * String to hold the path to the keys' dir.
     */
    public static final String KEYS_DIR = "D:" + File.separator + "keystore" + File.separator;

    /**
     * String to hold the name of the private key file.
     */
    public static final String PRIVATE_KEY_FILE = KEYS_DIR + "priv.key";

    /**
     * String to hold name of the public key file.
     */
    public static final String CERT_FILE = KEYS_DIR + "cert.cer";

    /**
     * String to hold name of the encrypted file.
     */
    public static final String ENCRYPTED_FILE = "D:" + File.separator + "Temp" + File.separator + "encrypted.xml";

    /**
     * String to hold name of the decrypted file.
     */
    public static final String DECRYPTED_FILE = "D:" + File.separator + "Temp" + File.separator + "decrypted.xml";

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        try {
            //get private key
            File keyFl = new File(PRIVATE_KEY_FILE);
            Security.addProvider(new BouncyCastleProvider());
            PEMParser pemParser = new PEMParser(new InputStreamReader(new FileInputStream(keyFl)));
            PEMKeyPair pemKeyPair = (PEMKeyPair) pemParser.readObject();
            JcaPEMKeyConverter converter = new JcaPEMKeyConverter().setProvider("BC");
            PrivateKey privateKey = converter.getPrivateKey(pemKeyPair.getPrivateKeyInfo());
            
            //Decrypt file
            File encryptedFile = new File(ENCRYPTED_FILE);
            File decryptedFile = new File(DECRYPTED_FILE);
            Files.deleteIfExists(decryptedFile.toPath());
            Security.addProvider(new BouncyCastleProvider());
            EncryptAndDecrypt.decrypt(privateKey, encryptedFile, decryptedFile);
        } catch (Exception e) {
            System.out.println("It's a pity...");
            System.err.println(e.getMessage());
        }
        System.out.println("THE END");
    }

}

So, it works! :)