CWE-780: Use of RSA Without OAEP - Java

Overview

Using RSA encryption without OAEP (Optimal Asymmetric Encryption Padding) enables padding oracle attacks, chosen ciphertext attacks, and message malleability. In Java, this commonly occurs when using Cipher.getInstance("RSA") without specifying the padding mode, which defaults to the insecure PKCS#1 v1.5 padding.

Primary Defence: Always use Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding") or stronger variants instead of the default "RSA" transformation, specify OAEP parameters explicitly with OAEPParameterSpec for SHA-256 or SHA-384, never rely on default cipher transformations, and consider using hybrid encryption (RSA for key exchange + AES for data) to prevent padding oracle attacks and ensure cryptographic security.

Remediation Strategy

PRIORITY 1: Use RSA with OAEP Padding (PRIMARY FIX)

Always specify the complete cipher transformation string including OAEP padding.

Basic RSA-OAEP Encryption/Decryption

import javax.crypto.Cipher;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.SecureRandom;
import java.security.spec.OAEPParameterSpec;
import java.security.spec.MGF1ParameterSpec;
import javax.crypto.spec.PSource;

public class SecureRSAExample {

    public static KeyPair generateKeyPair() throws Exception {
        KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA");
        keyGen.initialize(2048, new SecureRandom());  // Minimum 2048 bits
        return keyGen.generateKeyPair();
    }

    // SECURE - RSA with OAEP and SHA-256
    public static byte[] encryptOAEP(byte[] plaintext, PublicKey publicKey) 
            throws Exception {
        // Method 1: Using transformation string (recommended)
        Cipher cipher = Cipher.getInstance(
            "RSA/ECB/OAEPWithSHA-256AndMGF1Padding"
        );
        cipher.init(Cipher.ENCRYPT_MODE, publicKey);
        return cipher.doFinal(plaintext);
    }

    public static byte[] decryptOAEP(byte[] ciphertext, PrivateKey privateKey) 
            throws Exception {
        Cipher cipher = Cipher.getInstance(
            "RSA/ECB/OAEPWithSHA-256AndMGF1Padding"
        );
        cipher.init(Cipher.DECRYPT_MODE, privateKey);
        return cipher.doFinal(ciphertext);
    }

    // Alternative: Using OAEPParameterSpec for more control
    public static byte[] encryptOAEPWithParams(byte[] plaintext, PublicKey publicKey) 
            throws Exception {
        OAEPParameterSpec oaepParams = new OAEPParameterSpec(
            "SHA-256",           // Hash function
            "MGF1",              // Mask generation function
            MGF1ParameterSpec.SHA256,  // MGF parameters
            PSource.PSpecified.DEFAULT  // Source of label (usually empty)
        );

        Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPPadding");
        cipher.init(Cipher.ENCRYPT_MODE, publicKey, oaepParams);
        return cipher.doFinal(plaintext);
    }

    public static byte[] decryptOAEPWithParams(byte[] ciphertext, PrivateKey privateKey) 
            throws Exception {
        OAEPParameterSpec oaepParams = new OAEPParameterSpec(
            "SHA-256",
            "MGF1",
            MGF1ParameterSpec.SHA256,
            PSource.PSpecified.DEFAULT
        );

        Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPPadding");
        cipher.init(Cipher.DECRYPT_MODE, privateKey, oaepParams);
        return cipher.doFinal(ciphertext);
    }

    public static void main(String[] args) throws Exception {
        KeyPair keyPair = generateKeyPair();

        String message = "Sensitive data to encrypt";
        byte[] plaintext = message.getBytes("UTF-8");

        // Encrypt
        byte[] ciphertext = encryptOAEP(plaintext, keyPair.getPublic());

        // Decrypt
        byte[] decrypted = decryptOAEP(ciphertext, keyPair.getPrivate());

        System.out.println("Original: " + message);
        System.out.println("Decrypted: " + new String(decrypted, "UTF-8"));
    }
}

PRIORITY 2: Hybrid Encryption for Large Data

RSA is limited to small data (~190 bytes for 2048-bit RSA with SHA-256). Use hybrid encryption for larger data.

import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.security.*;
import java.util.Arrays;

public class HybridEncryption {

    private static final int AES_KEY_SIZE = 256;
    private static final int GCM_IV_SIZE = 12;
    private static final int GCM_TAG_SIZE = 128;

    public static class EncryptedData {
        public byte[] encryptedKey;
        public byte[] iv;
        public byte[] ciphertext;

        public EncryptedData(byte[] encryptedKey, byte[] iv, byte[] ciphertext) {
            this.encryptedKey = encryptedKey;
            this.iv = iv;
            this.ciphertext = ciphertext;
        }
    }

    public static EncryptedData hybridEncrypt(byte[] plaintext, PublicKey rsaPublicKey) 
            throws Exception {
        // Step 1: Generate random AES key
        KeyGenerator keyGen = KeyGenerator.getInstance("AES");
        keyGen.init(AES_KEY_SIZE, new SecureRandom());
        SecretKey aesKey = keyGen.generateKey();

        // Step 2: Encrypt data with AES-GCM
        byte[] iv = new byte[GCM_IV_SIZE];
        new SecureRandom().nextBytes(iv);

        Cipher aesCipher = Cipher.getInstance("AES/GCM/NoPadding");
        GCMParameterSpec gcmSpec = new GCMParameterSpec(GCM_TAG_SIZE, iv);
        aesCipher.init(Cipher.ENCRYPT_MODE, aesKey, gcmSpec);
        byte[] ciphertext = aesCipher.doFinal(plaintext);

        // Step 3: Encrypt AES key with RSA-OAEP
        Cipher rsaCipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
        rsaCipher.init(Cipher.ENCRYPT_MODE, rsaPublicKey);
        byte[] encryptedKey = rsaCipher.doFinal(aesKey.getEncoded());

        return new EncryptedData(encryptedKey, iv, ciphertext);
    }

    public static byte[] hybridDecrypt(EncryptedData encrypted, PrivateKey rsaPrivateKey) 
            throws Exception {
        // Step 1: Decrypt AES key with RSA-OAEP
        Cipher rsaCipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
        rsaCipher.init(Cipher.DECRYPT_MODE, rsaPrivateKey);
        byte[] aesKeyBytes = rsaCipher.doFinal(encrypted.encryptedKey);
        SecretKey aesKey = new SecretKeySpec(aesKeyBytes, "AES");

        // Step 2: Decrypt data with AES-GCM
        Cipher aesCipher = Cipher.getInstance("AES/GCM/NoPadding");
        GCMParameterSpec gcmSpec = new GCMParameterSpec(GCM_TAG_SIZE, encrypted.iv);
        aesCipher.init(Cipher.DECRYPT_MODE, aesKey, gcmSpec);
        return aesCipher.doFinal(encrypted.ciphertext);
    }

    public static void main(String[] args) throws Exception {
        KeyPair keyPair = SecureRSAExample.generateKeyPair();

        // Large data (can be megabytes)
        byte[] largeData = new byte[1000000];
        new SecureRandom().nextBytes(largeData);

        // Encrypt
        EncryptedData encrypted = hybridEncrypt(largeData, keyPair.getPublic());

        // Decrypt
        byte[] decrypted = hybridDecrypt(encrypted, keyPair.getPrivate());

        System.out.println("Match: " + Arrays.equals(largeData, decrypted));
    }
}

PRIORITY 3: RSA Signatures with PSS (Not OAEP)

For signatures (not encryption), use PSS padding instead of OAEP.

import java.security.*;
import java.security.spec.MGF1ParameterSpec;
import java.security.spec.PSSParameterSpec;

public class RSASignatureExample {

    public static byte[] signWithPSS(byte[] message, PrivateKey privateKey) 
            throws Exception {
        // Configure PSS parameters
        PSSParameterSpec pssSpec = new PSSParameterSpec(
            "SHA-256",                    // Hash algorithm
            "MGF1",                       // Mask generation function
            MGF1ParameterSpec.SHA256,     // MGF parameters
            32,                           // Salt length (hash output length)
            1                             // Trailer field
        );

        Signature signature = Signature.getInstance("RSASSA-PSS");
        signature.setParameter(pssSpec);
        signature.initSign(privateKey);
        signature.update(message);
        return signature.sign();
    }

    public static boolean verifyWithPSS(byte[] message, byte[] signatureBytes, 
                                       PublicKey publicKey) throws Exception {
        PSSParameterSpec pssSpec = new PSSParameterSpec(
            "SHA-256",
            "MGF1",
            MGF1ParameterSpec.SHA256,
            32,
            1
        );

        Signature signature = Signature.getInstance("RSASSA-PSS");
        signature.setParameter(pssSpec);
        signature.initVerify(publicKey);
        signature.update(message);
        return signature.verify(signatureBytes);
    }
}

Common Vulnerable Patterns

Default RSA Transformation (Uses PKCS#1 v1.5)

// VULNERABLE - Defaults to "RSA/ECB/PKCS1Padding" (weak!)
import javax.crypto.Cipher;
import java.security.PublicKey;

Cipher cipher = Cipher.getInstance("RSA");
cipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] encrypted = cipher.doFinal(plaintext);
// This uses PKCS#1 v1.5, vulnerable to Bleichenbacher's attack!

Why this is vulnerable:

• Cipher.getInstance("RSA") defaults to RSA/ECB/PKCS1Padding (PKCS#1 v1.5), not OAEP.
• PKCS#1 v1.5 padding is susceptible to padding oracle and chosen-ciphertext attacks.
• The Bleichenbacher attack (1998) breaks PKCS#1 v1.5 when decryption success/failure can be observed.
• An attacker can adaptively probe decryption to recover plaintext without the private key.
• Secure use requires explicitly specifying OAEP in the transformation string.

Explicitly Using PKCS1Padding

// VULNERABLE - Explicitly specifying weak padding
Cipher cipher = Cipher.getInstance("RSA/ECB/PKCS1Padding");
cipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] encrypted = cipher.doFinal(plaintext);
// PKCS#1 v1.5 padding enables Bleichenbacher's attack

Why this is vulnerable:

• RSA/ECB/PKCS1Padding is PKCS#1 v1.5, which is vulnerable to Bleichenbacher-style padding oracles.
• Attackers can adaptively query decryption and learn plaintext from success/failure signals.
• The padding structure is deterministic enough to enable chosen-ciphertext recovery.
• PKCS#1 v1.5 also allows malleability and timing side-channel leakage.

Using Deprecated SHA-1 with OAEP

// VULNERABLE - SHA-1 is deprecated
Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-1AndMGF1Padding");
cipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] encrypted = cipher.doFinal(plaintext);
// SHA-1 is cryptographically broken, use SHA-256 or better

Why this is vulnerable:

• OAEP is correct, but SHA-1 is broken (practical collisions since 2017).
• OAEP assumes a strong hash; SHA-1’s weaknesses undermine that proof.
• Collision attacks can enable crafted inputs that weaken OAEP’s masking behavior.
• Modern standards prohibit SHA-1 for new systems; use SHA-256/384/512.

Encrypting Large Data Directly

// VULNERABLE - Will throw exception or fail
// Max plaintext for 2048-bit RSA with SHA-256 OAEP: ~190 bytes
byte[] largeData = new byte[1000];
Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
cipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] encrypted = cipher.doFinal(largeData);  // javax.crypto.IllegalBlockSizeException!

Why this is vulnerable:

• RSA can only encrypt data smaller than the modulus minus padding overhead.
• For 2048-bit RSA with SHA-256 OAEP, max plaintext is ~190 bytes.
• Larger inputs throw IllegalBlockSizeException or risk truncation in some implementations.
• Even with bigger keys, RSA is too slow for bulk data.
• The safe pattern is hybrid encryption (AES-GCM for data + RSA-OAEP for the key).

Secure Patterns

RSA with OAEP and SHA-256 (Recommended)

// SECURE - RSA with OAEP padding and SHA-256
import javax.crypto.Cipher;
import java.security.PublicKey;
import java.security.PrivateKey;

// Encryption
Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
cipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] ciphertext = cipher.doFinal(plaintext);

// Decryption
cipher.init(Cipher.DECRYPT_MODE, privateKey);
byte[] decrypted = cipher.doFinal(ciphertext);

Why this works:

• OAEP makes RSA probabilistic, blocking ciphertext replay and oracle-style probes.
• SHA-256 provides strong collision resistance for OAEP’s security proof.
• MGF1 spreads randomness across the message to defeat chosen-ciphertext attacks.
• OAEP with SHA-256 meets modern standards and avoids PKCS#1 v1.5 weaknesses.

RSA-OAEP with Explicit Parameters

// SECURE - Using OAEPParameterSpec for full control
import javax.crypto.Cipher;
import java.security.spec.OAEPParameterSpec;
import java.security.spec.MGF1ParameterSpec;
import javax.crypto.spec.PSource;

OAEPParameterSpec oaepParams = new OAEPParameterSpec(
    "SHA-256",                      // Hash function
    "MGF1",                          // Mask generation function
    MGF1ParameterSpec.SHA256,        // MGF parameters
    PSource.PSpecified.DEFAULT       // Source of label (usually empty)
);

Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPPadding");
cipher.init(Cipher.ENCRYPT_MODE, publicKey, oaepParams);
byte[] ciphertext = cipher.doFinal(plaintext);

Why this works:

• Explicit OAEPParameterSpec avoids provider defaults that might select weak hashes.
• Matching SHA-256 for both OAEP and MGF1 keeps padding strength consistent.
• The empty label is standard and interoperable across implementations.
• Explicit parameters reduce surprises during provider upgrades or migrations.

Hybrid Encryption for Large Data

// SECURE - Hybrid encryption combining AES-GCM with RSA-OAEP
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.GCMParameterSpec;
import java.security.SecureRandom;

// Step 1: Generate random AES key
KeyGenerator keyGen = KeyGenerator.getInstance("AES");
keyGen.init(256, new SecureRandom());
SecretKey aesKey = keyGen.generateKey();

// Step 2: Encrypt large data with AES-GCM
byte[] iv = new byte[12];
new SecureRandom().nextBytes(iv);
Cipher aesCipher = Cipher.getInstance("AES/GCM/NoPadding");
aesCipher.init(Cipher.ENCRYPT_MODE, aesKey, new GCMParameterSpec(128, iv));
byte[] encryptedData = aesCipher.doFinal(largeData);

// Step 3: Encrypt AES key with RSA-OAEP
Cipher rsaCipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
rsaCipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] encryptedKey = rsaCipher.doFinal(aesKey.getEncoded());

// Send: encryptedKey, iv, encryptedData

Why this works:

• RSA’s size limits make direct encryption impractical for large data.
• AES-GCM efficiently encrypts bulk data and provides integrity checks.
• RSA-OAEP securely wraps the ephemeral AES key.
• The split (encrypted key + IV + ciphertext) matches standard TLS-style designs.

RSA-OAEP with SHA-512 (Maximum Security)

// SECURE - Using SHA-512 for maximum security
Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-512AndMGF1Padding");
cipher.init(Cipher.ENCRYPT_MODE, publicKey);
byte[] ciphertext = cipher.doFinal(plaintext);

Why this works:

• SHA-512 provides a higher security margin than SHA-256 for long-term secrecy.
• OAEP construction stays the same, but the hash primitive is stronger.
• Useful for high-assurance or long-retention data requirements.
• Trade-off: slightly larger ciphertext overhead.

Complete Working Example with Key Storage

import javax.crypto.Cipher;
import java.io.*;
import java.security.*;
import java.security.spec.*;
import java.util.Base64;

public class RSAKeyManagement {

    // Generate and save key pair
    public static void generateAndSaveKeyPair(String publicKeyFile, String privateKeyFile) 
            throws Exception {
        KeyPairGenerator keyGen = KeyPairGenerator.getInstance("RSA");
        keyGen.initialize(2048, new SecureRandom());
        KeyPair keyPair = keyGen.generateKeyPair();

        // Save public key
        X509EncodedKeySpec x509Spec = new X509EncodedKeySpec(
            keyPair.getPublic().getEncoded()
        );
        try (FileOutputStream fos = new FileOutputStream(publicKeyFile)) {
            fos.write(x509Spec.getEncoded());
        }

        // Save private key
        PKCS8EncodedKeySpec pkcs8Spec = new PKCS8EncodedKeySpec(
            keyPair.getPrivate().getEncoded()
        );
        try (FileOutputStream fos = new FileOutputStream(privateKeyFile)) {
            fos.write(pkcs8Spec.getEncoded());
        }
    }

    // Load public key
    public static PublicKey loadPublicKey(String filename) throws Exception {
        byte[] keyBytes = readFile(filename);
        X509EncodedKeySpec spec = new X509EncodedKeySpec(keyBytes);
        KeyFactory keyFactory = KeyFactory.getInstance("RSA");
        return keyFactory.generatePublic(spec);
    }

    // Load private key
    public static PrivateKey loadPrivateKey(String filename) throws Exception {
        byte[] keyBytes = readFile(filename);
        PKCS8EncodedKeySpec spec = new PKCS8EncodedKeySpec(keyBytes);
        KeyFactory keyFactory = KeyFactory.getInstance("RSA");
        return keyFactory.generatePrivate(spec);
    }

    private static byte[] readFile(String filename) throws IOException {
        try (FileInputStream fis = new FileInputStream(filename);
             ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
            byte[] buffer = new byte[1024];
            int bytesRead;
            while ((bytesRead = fis.read(buffer)) != -1) {
                baos.write(buffer, 0, bytesRead);
            }
            return baos.toByteArray();
        }
    }

    public static void main(String[] args) throws Exception {
        // Generate keys
        generateAndSaveKeyPair("public.key", "private.key");

        // Load keys
        PublicKey publicKey = loadPublicKey("public.key");
        PrivateKey privateKey = loadPrivateKey("private.key");

        // Encrypt
        String message = "Confidential data";
        byte[] plaintext = message.getBytes("UTF-8");

        Cipher cipher = Cipher.getInstance("RSA/ECB/OAEPWithSHA-256AndMGF1Padding");
        cipher.init(Cipher.ENCRYPT_MODE, publicKey);
        byte[] ciphertext = cipher.doFinal(plaintext);

        // Decrypt
        cipher.init(Cipher.DECRYPT_MODE, privateKey);
        byte[] decrypted = cipher.doFinal(ciphertext);

        System.out.println("Original: " + message);
        System.out.println("Decrypted: " + new String(decrypted, "UTF-8"));
    }
}

Migration from PKCS#1 v1.5 to OAEP

Verification with Encryption/Decryption Roundtrip

// Manual test to verify OAEP works correctly:
KeyPair keyPair = SecureRSAExample.generateKeyPair();

String message = "Test message";
byte[] plaintext = message.getBytes("UTF-8");

// Encrypt with OAEP
byte[] ciphertext = SecureRSAExample.encryptOAEP(plaintext, keyPair.getPublic());

// Decrypt with OAEP  
byte[] decrypted = SecureRSAExample.decryptOAEP(ciphertext, keyPair.getPrivate());

// Verify decryption works
if (message.equals(new String(decrypted, "UTF-8"))) {
    System.out.println("✓ OAEP encryption/decryption working");
} else {
    System.err.println("ERROR: Decryption failed!");
}

// Verify probabilistic behavior (same plaintext -> different ciphertext)
byte[] ciphertext2 = SecureRSAExample.encryptOAEP(plaintext, keyPair.getPublic());
if (!java.util.Arrays.equals(ciphertext, ciphertext2)) {
    System.out.println("✓ OAEP is probabilistic (ciphertexts differ)");
} else {
    System.err.println("ERROR: Encryption is deterministic (not using OAEP properly)!");
}

Additional Resources

• CWE-780: Use of RSA Without OAEP
• Java Cryptography Architecture (JCA) Reference Guide
• OWASP Cryptographic Storage Cheat Sheet
• RFC 8017: PKCS #1 v2.2