update documentation and added null checks

crypto/fipsmodule/rsa/internal.h

@@ -81,16 +81,20 @@ struct rsa_meth_st {
     int (*sign)(int type, const uint8_t *m, unsigned int m_length,
                 uint8_t *sigret, unsigned int *siglen, const RSA *rsa);
 
-    // Set via |RSA_meth_set_priv_enc|
+    // Set via |RSA_meth_set_priv_enc|. |sign_raw| is equivalent to the
+    // |priv_enc| field of OpenSSL's |RSA_METHOD| struct.
     int (*sign_raw)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
                     int padding);
-    // Set via |RSA_meth_set_pub_dec|
+    // Set via |RSA_meth_set_pub_dec|. |verify_raw| is equivalent to the
+    // |pub_dec| field of OpenSSL's |RSA_METHOD| struct.
     int (*verify_raw)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
                       int padding);
-    // Set via |RSA_meth_set_priv_dec|
+    // Set via |RSA_meth_set_priv_dec|. |decrypt| is equivalent to the
+    // |priv_dec| field of OpenSSL's |RSA_METHOD| struct.
     int (*decrypt)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
                    int padding);
-    // Set via |RSA_meth_set_pub_enc|
+    // Set via |RSA_meth_set_pub_enc|. |encrypt| is equivalent to the
+    // |pub_enc| field of OpenSSL's |RSA_METHOD| struct.
     int (*encrypt)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
                    int padding);
 

crypto/fipsmodule/rsa/rsa.c

@@ -572,7 +572,7 @@ static int rsa_sign_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
                                      size_t max_out, const uint8_t *in,
                                      size_t in_len, int padding) {
   SET_DIT_AUTO_DISABLE;
-  if (rsa->meth->sign_raw) {
+  if (rsa->meth && rsa->meth->sign_raw) {
     // In OpenSSL, the RSA_METHOD |sign_raw| or |priv_enc| operation does
     // not directly take and initialize an |out_len| parameter. Instead, it
     // returns the size of the encrypted data or a negative number for error.
@@ -602,7 +602,8 @@ int RSA_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
 
 unsigned RSA_size(const RSA *rsa) {
   SET_DIT_AUTO_DISABLE;
-  size_t ret = rsa->meth->size ? rsa->meth->size(rsa) : rsa_default_size(rsa);
+  size_t ret = (rsa->meth && rsa->meth->size) ?
+          rsa->meth->size(rsa) : rsa_default_size(rsa);
   // RSA modulus sizes are bounded by |BIGNUM|, which must fit in |unsigned|.
   //
   // TODO(https://crbug.com/boringssl/516): Should we make this return |size_t|?
@@ -797,7 +798,7 @@ int RSA_add_pkcs1_prefix(uint8_t **out_msg, size_t *out_msg_len,
 int rsa_sign_no_self_test(int hash_nid, const uint8_t *digest,
                           size_t digest_len, uint8_t *out, unsigned *out_len,
                           RSA *rsa) {
-  if (rsa->meth->sign) {
+  if (rsa->meth && rsa->meth->sign) {
     if (!rsa_check_digest_size(hash_nid, digest_len)) {
       return 0;
     }
@@ -994,7 +995,7 @@ int RSA_verify_pss_mgf1(RSA *rsa, const uint8_t *digest, size_t digest_len,
 
 int rsa_private_transform_no_self_test(RSA *rsa, uint8_t *out,
                                        const uint8_t *in, size_t len) {
-  if (rsa->meth->private_transform) {
+  if (rsa->meth && rsa->meth->private_transform) {
     return rsa->meth->private_transform(rsa, out, in, len);
   }
 
@@ -1010,17 +1011,32 @@ int rsa_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
 
 int RSA_flags(const RSA *rsa) {
   SET_DIT_AUTO_DISABLE;
-  return rsa->flags;
+  if (rsa) {
+    return rsa->flags;
+  }
+
+  OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+  return 0;
 }
 
 void RSA_set_flags(RSA *rsa, int flags) {
   SET_DIT_AUTO_DISABLE;
+  if (rsa == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return;
+  }
+
   rsa->flags |= flags;
 }
 
 int RSA_test_flags(const RSA *rsa, int flags) {
   SET_DIT_AUTO_DISABLE;
-  return rsa->flags & flags;
+  if (rsa) {
+    return rsa->flags & flags;
+  }
+
+  OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+  return 0;
 }
 
 int RSA_blinding_on(RSA *rsa, BN_CTX *ctx) {

crypto/fipsmodule/rsa/rsa_impl.c

@@ -412,7 +412,7 @@ static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx);
 int rsa_verify_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
                                 size_t max_out, const uint8_t *in,
                                 size_t in_len, int padding) {
-  if(rsa->meth->verify_raw) {
+  if(rsa->meth && rsa->meth->verify_raw) {
     // In OpenSSL, the RSA_METHOD |verify_raw| or |pub_dec| operation does
     // not directly take and initialize an |out_len| parameter. Instead, it
     // returns the size of the recovered plaintext or negative number for error.

crypto/rsa_extra/rsa_crypt.c

@@ -385,7 +385,7 @@ int RSA_private_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
 
 int RSA_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
                 const uint8_t *in, size_t in_len, int padding) {
-  if(rsa->meth->encrypt) {
+  if(rsa->meth && rsa->meth->encrypt) {
     // In OpenSSL, the RSA_METHOD |encrypt| or |pub_enc| operation does not
     // directly take and initialize an |out_len| parameter. Instead, it returns
     // the number of bytes written to |out| or a negative number for error.
@@ -557,7 +557,7 @@ static int rsa_default_decrypt(RSA *rsa, size_t *out_len, uint8_t *out,
 
 int RSA_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
                 const uint8_t *in, size_t in_len, int padding) {
-  if (rsa->meth->decrypt) {
+  if (rsa->meth && rsa->meth->decrypt) {
     // In OpenSSL, the RSA_METHOD |decrypt| operation does not directly take
     // and initialize an |out_len| parameter. Instead, it returns the number
     // of bytes written to |out| or a negative number for error. Our wrapping

include/openssl/rsa.h

@@ -240,37 +240,52 @@ OPENSSL_EXPORT void RSA_meth_free(RSA_METHOD *meth);
 // The following functions set the corresponding fields on |meth|. Returns one
 // on success and zero on failure.
 
-// RSA_meth_set_init sets |init| on |meth|.
+// RSA_meth_set_init sets |init| on |meth|. |init| should return one on
+// success and zero on failure.
 OPENSSL_EXPORT int RSA_meth_set_init(RSA_METHOD *meth, int (*init) (RSA *rsa));
 
 // RSA_meth_set_finish sets |finish| on |meth|. The |finish| function
-// is called in |RSA_free| before freeing the key.
+// is called in |RSA_free| before freeing the key. |finish| should return
+// one on success and zero on failure.
 OPENSSL_EXPORT int RSA_meth_set_finish(RSA_METHOD *meth,
                                        int (*finish) (RSA *rsa));
 
 // RSA_meth_set_priv_dec sets |priv_dec| on |meth|. The |priv_dec| function
 // should return the number of bytes written to the object |to| or -1 for error.
+// |priv_dec| should decrypt |max_out| bytes at |from| using the private key
+// |rsa| and store the plaintext in |to|. |priv_dec| should return the size of
+// the recovered plaintext or a negative number on error.
 OPENSSL_EXPORT int RSA_meth_set_priv_dec(RSA_METHOD *meth,
                           int (*priv_dec) (int max_out, const uint8_t *from,
                                            uint8_t *to, RSA *rsa,
                                            int padding));
 
 // RSA_meth_set_priv_enc sets |priv_enc| on |meth|. The |priv_enc| function
 // should return the number of bytes written to the object |to| or -1 for error.
+// |priv_enc| should sign |max_out| bytes at |from| using the private key |rsa|
+// and store the signature in |to|. |priv_enc| should return the size of the
+// signature or a negative number for error.
 OPENSSL_EXPORT int RSA_meth_set_priv_enc(RSA_METHOD *meth,
                           int (*priv_enc) (int max_out, const uint8_t *from,
                                            uint8_t *to, RSA *rsa,
                                            int padding));
 
 // RSA_meth_set_pub_dec sets |pub_dec| on |meth|. The |pub_dec| function
 // should return the number of bytes written to the object |to| or -1 for error.
+// |pub_dec| should recover the |max_out| bytes of the message digest at |from|
+// using the signer's public key |rsa| and store it in |to|. |pub_dec| should
+// return the size of the recovered message digest or a negative number on
+// error.
 OPENSSL_EXPORT int RSA_meth_set_pub_dec(RSA_METHOD *meth,
                          int (*pub_dec) (int max_out, const uint8_t *from,
                                          uint8_t *to, RSA *rsa,
                                          int padding));
 
 // RSA_meth_set_pub_enc sets |pub_enc| on |meth|. The |pub_enc| function
 // should return the number of bytes written to the object |to| or -1 for error.
+// |pub_enc| should encrypt |max_out| bytes at |from| using the public key |rsa|
+// and stores the ciphertext in |to|. |pub_enc| should return the size of the
+// encrypted data or a negative number on error.
 OPENSSL_EXPORT int RSA_meth_set_pub_enc(RSA_METHOD *meth,
                          int (*pub_enc) (int max_out, const uint8_t *from,
                                          uint8_t *to, RSA *rsa,