Refactor RSA_METHOD and expand API (#1790)

### Issues
`CryptoAlg-2482
`
### Description of changes: 
This PR is the last in a series of three (prev PRs:
[1st](#1776),
[2nd](#1785)) to refactor and expand
ENGINE, EC_KEY_METHOD and RSA_METHOD. This PR:
1. Expands the RSA_METHOD struct and changes the function signatures for
encrypt, decrypt, sign_raw, and verify_raw to match OpenSSL 1.1.1.
2. Makes the struct definition for RSA_METHOD internal. This means
static allocation of this struct is no longer possible.
3. Adds various functions to get/set, create, free, and manipulate
RSA_METHOD

### Callout: 
This PR includes the commits for the other 2 PRs. They should be
reviewed and merged before this one.

### Testing:
Tested with various fields initialized in RSA_METHOD to ensure that
custom functionality is correctly invoked.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

crypto/engine/engine.c

@@ -50,7 +50,10 @@ int ENGINE_set_RSA(ENGINE *engine, const RSA_METHOD *method) {
 }
 
 const RSA_METHOD *ENGINE_get_RSA(const ENGINE *engine) {
-  return engine->rsa_method;
+  if(engine) {
+    return engine->rsa_method;;
+  }
+  return NULL;
 }
 
 int ENGINE_set_EC(ENGINE *engine, const EC_KEY_METHOD *method) {
@@ -64,7 +67,10 @@ int ENGINE_set_EC(ENGINE *engine, const EC_KEY_METHOD *method) {
 }
 
 const EC_KEY_METHOD *ENGINE_get_EC(const ENGINE *engine) {
-  return engine->eckey_method;
+  if(engine) {
+    return engine->eckey_method;
+  }
+  return NULL;
 }
 
 OPENSSL_DECLARE_ERROR_REASON(ENGINE, OPERATION_NOT_SUPPORTED)

crypto/fipsmodule/ec/ec_test.cc

@@ -2525,3 +2525,29 @@ TEST(ECTest, ECKEYMETHOD) {
   EC_KEY_METHOD_set_flags(ec_method, ECDSA_FLAG_OPAQUE);
   ASSERT_TRUE(EC_KEY_is_opaque(ec));
 }
+
+TEST(ECTest, ECEngine) {
+  ENGINE *engine = ENGINE_new();
+  ASSERT_TRUE(engine);
+  ASSERT_FALSE(ENGINE_get_EC(engine));
+
+  EC_KEY_METHOD *eng_funcs = EC_KEY_METHOD_new(NULL);
+  ASSERT_TRUE(eng_funcs);
+  EC_KEY_METHOD_set_sign(eng_funcs, NULL, NULL, ecdsa_sign_sig);
+
+  ASSERT_TRUE(ENGINE_set_EC(engine, eng_funcs));
+  ASSERT_TRUE(ENGINE_get_EC(engine));
+
+  EC_KEY *key = EC_KEY_new_method(engine);
+  ASSERT_TRUE(key);
+
+  // Call custom Engine implementation
+  ECDSA_do_sign(NULL, 0, key);
+  ASSERT_STREQ(static_cast<const char*>(EC_KEY_get_ex_data(key, 1))
+  , "ecdsa_sign_sig");
+
+  EC_KEY_free(key);
+  ENGINE_free(engine);
+  EC_KEY_METHOD_free(eng_funcs);
+}
+

crypto/fipsmodule/ec/internal.h

@@ -763,14 +763,16 @@ struct ec_key_method_st {
     // AWS-LC doesn't support custom values for EC_KEY operations
     // as of now. |k_inv| and |r| must be NULL parameters.
     // The |type| parameter is ignored in OpenSSL, we pass in zero for it.
-    // |sign| is invoked in |ECDSA_sign|.
+    // The default behavior for |sign| is implemented in |ECDSA_sign|. If custom
+    // functionality is provided, |sign| will be invoked within |ECDSA_sign|.
     int (*sign)(int type, const uint8_t *digest, int digest_len,
                 uint8_t *sig, unsigned int *siglen, const BIGNUM *k_inv,
                 const BIGNUM *r, EC_KEY *eckey);
 
     // AWS-LC doesn't support custom values for EC_KEY operations
-    // as of now. |k_inv| and |r| must be NULL parameters. |sign_sig| is
-    // invoked in |ECDSA_do_sign|.
+    // as of now. |k_inv| and |r| must be NULL parameters. The default behavior
+    // for |sign_sig| is implemented in |ECDSA_do_sign|. If custom functionality
+    // is provided, |sign_sig| will be invoked within |ECDSA_do_sign|.
     ECDSA_SIG *(*sign_sig)(const uint8_t *digest, int digest_len,
                            const BIGNUM *in_kinv, const BIGNUM *in_r,
                            EC_KEY *eckey);

crypto/fipsmodule/rsa/internal.h

@@ -69,6 +69,71 @@ extern "C" {
 
 typedef struct bn_blinding_st BN_BLINDING;
 
+struct rsa_meth_st {
+    void *app_data;
+
+    int (*init)(RSA *rsa);
+    int (*finish)(RSA *rsa);
+
+    // size returns the size of the RSA modulus in bytes.
+    size_t (*size)(const RSA *rsa);
+
+    // Set via |RSA_meth_set_sign|. The default behavior for |sign| is
+    // implemented in |RSA_sign|. If custom functionality is provided, |sign|
+    // will be invoked within |RSA_sign|.
+    int (*sign)(int type, const uint8_t *m, unsigned int m_length,
+                uint8_t *sigret, unsigned int *siglen, const RSA *rsa);
+
+    // Set via |RSA_meth_set_priv_enc|. |sign_raw| is equivalent to the
+    // |priv_enc| field of OpenSSL's |RSA_METHOD| struct. The default behavior
+    // for |sign_raw| is implemented in |RSA_sign_raw|. If custom
+    // functionality is provided, |sign_raw| will be invoked within
+    // |RSA_sign_raw|.
+    int (*sign_raw)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
+                    int padding);
+
+    // Set via |RSA_meth_set_pub_dec|. |verify_raw| is equivalent to the
+    // |pub_dec| field of OpenSSL's |RSA_METHOD| struct. The default behavior
+    // for |verify_raw| is implemented in |RSA_verify_raw|. If custom
+    // functionality is provided, |verify_raw| will be invoked within
+    // |RSA_verify_raw|.
+    int (*verify_raw)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
+                      int padding);
+
+    // Set via |RSA_meth_set_priv_dec|. |decrypt| is equivalent to the
+    // |priv_dec| field of OpenSSL's |RSA_METHOD| struct. The default behavior
+    // for |decrypt| is implemented in |RSA_decrypt|. If custom
+    // functionality is provided, |decrypt| will be invoked within
+    // |RSA_decrypt|.
+    int (*decrypt)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
+                   int padding);
+
+    // Set via |RSA_meth_set_pub_enc|. |encrypt| is equivalent to the
+    // |pub_enc| field of OpenSSL's |RSA_METHOD| struct. The default behavior
+    // for |encrypt| is implemented in |RSA_encrypt|. If custom
+    // functionality is provided, |encrypt| will be invoked within
+    // |RSA_encrypt|.
+    int (*encrypt)(int max_out, const uint8_t *in, uint8_t *out, RSA *rsa,
+                   int padding);
+
+    // private_transform takes a big-endian integer from |in|, calculates the
+    // d'th power of it, modulo the RSA modulus and writes the result as a
+    // big-endian integer to |out|. Both |in| and |out| are |len| bytes long and
+    // |len| is always equal to |RSA_size(rsa)|. If the result of the transform
+    // can be represented in fewer than |len| bytes, then |out| must be zero
+    // padded on the left.
+    //
+    // It returns one on success and zero otherwise.
+    //
+    // RSA decrypt and sign operations will call this, thus an ENGINE might wish
+    // to override it in order to avoid having to implement the padding
+    // functionality demanded by those, higher level, operations.
+    int (*private_transform)(RSA *rsa, uint8_t *out, const uint8_t *in,
+                             size_t len);
+
+    int flags;
+};
+
 struct rsa_st {
   const RSA_METHOD *meth;
 
@@ -128,10 +193,6 @@ struct rsa_st {
 
 // Default implementations of RSA operations.
 
-// RSA_default_method returns a zero initialized |RSA_METHOD| object. The
-// wrapper functions will select the appropriate |rsa_default_*| implementation.
-const RSA_METHOD *RSA_default_method(void);
-
 size_t rsa_default_size(const RSA *rsa);
 int rsa_default_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out,
                          size_t max_out, const uint8_t *in, size_t in_len,

crypto/fipsmodule/rsa/rsa.c

@@ -221,7 +221,7 @@ RSA *RSA_new_method(const ENGINE *engine) {
   }
 
   if (rsa->meth == NULL) {
-    rsa->meth = (RSA_METHOD *) RSA_default_method();
+    rsa->meth = (RSA_METHOD *) RSA_get_default_method();
   }
 
   rsa->references = 1;
@@ -448,12 +448,159 @@ int RSA_set0_crt_params(RSA *rsa, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp) {
   return 1;
 }
 
+RSA_METHOD *RSA_meth_new(const char *name, int flags) {
+  RSA_METHOD *meth = OPENSSL_zalloc(sizeof(*meth));
+
+  if (meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
+    return NULL;
+  }
+
+  if (flags == RSA_FLAG_OPAQUE) {
+    meth->flags = flags;
+  }
+  return meth;
+}
+
+int RSA_set_method(RSA *rsa, const RSA_METHOD *meth) {
+  if(rsa == NULL || meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  rsa->meth = meth;
+  return 1;
+}
+
+const RSA_METHOD *RSA_get_method(const RSA *rsa) {
+  if(rsa == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return NULL;
+  }
+
+  return rsa->meth;
+}
+
+void RSA_meth_free(RSA_METHOD *meth)
+{
+  if (meth != NULL) {
+    OPENSSL_free(meth);
+  }
+}
+
+int RSA_meth_set_init(RSA_METHOD *meth, int (*init) (RSA *rsa)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->init = init;
+  return 1;
+}
+
+int RSA_meth_set_finish(RSA_METHOD *meth, int (*finish) (RSA *rsa)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->finish = finish;
+  return 1;
+}
+
+int RSA_meth_set_priv_dec(RSA_METHOD *meth,
+                          int (*priv_dec) (int max_out, const uint8_t *from,
+                                           uint8_t *to, RSA *rsa,
+                                           int padding)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->decrypt = priv_dec;
+  return 1;
+}
+
+int RSA_meth_set_priv_enc(RSA_METHOD *meth,
+                          int (*priv_enc) (int max_out, const uint8_t *from,
+                                           uint8_t *to, RSA *rsa,
+                                           int padding)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->sign_raw = priv_enc;
+  return 1;
+}
+
+int RSA_meth_set_pub_dec(RSA_METHOD *meth,
+                         int (*pub_dec) (int max_out, const uint8_t *from,
+                                         uint8_t *to, RSA *rsa,
+                                         int padding)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->verify_raw = pub_dec;
+  return 1;
+}
+
+int RSA_meth_set_pub_enc(RSA_METHOD *meth,
+                         int (*pub_enc) (int max_out, const uint8_t *from,
+                                         uint8_t *to, RSA *rsa,
+                                         int padding)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->encrypt = pub_enc;
+  return 1;
+}
+
+int RSA_meth_set0_app_data(RSA_METHOD *meth, void *app_data) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->app_data = app_data;
+  return 1;
+}
+
+int RSA_meth_set_sign(RSA_METHOD *meth, int (*sign) (int type,
+        const unsigned char *m, unsigned int m_length, unsigned char *sigret,
+        unsigned int *siglen, const RSA *rsa)) {
+  if(meth == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
+  meth->sign = sign;
+  return 1;
+}
+
 static int rsa_sign_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
                                      size_t max_out, const uint8_t *in,
                                      size_t in_len, int padding) {
   SET_DIT_AUTO_DISABLE;
-  if (rsa->meth->sign_raw) {
-    return rsa->meth->sign_raw(rsa, out_len, out, max_out, in, in_len, padding);
+  if (rsa->meth && rsa->meth->sign_raw) {
+    // In OpenSSL, the RSA_METHOD |sign_raw| or |priv_enc| operation does
+    // not directly take and initialize an |out_len| parameter. Instead, it
+    // returns the size of the encrypted data or a negative number for error.
+    // Our wrapping functions like |RSA_sign_raw| diverge from this paradigm
+    // and expect an |out_len| parameter. To remain compatible with this new
+    // paradigm and OpenSSL, we initialize |out_len| based on the return value
+    // here.
+    int ret = rsa->meth->sign_raw((int)max_out, in, out, rsa, padding);
+    if(ret < 0) {
+      *out_len = 0;
+      return 0;
+    }
+    *out_len = ret;
+    return 1;
   }
 
   return rsa_default_sign_raw(rsa, out_len, out, max_out, in, in_len, padding);
@@ -470,7 +617,8 @@ int RSA_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
 
 unsigned RSA_size(const RSA *rsa) {
   SET_DIT_AUTO_DISABLE;
-  size_t ret = rsa->meth->size ? rsa->meth->size(rsa) : rsa_default_size(rsa);
+  size_t ret = (rsa->meth && rsa->meth->size) ?
+          rsa->meth->size(rsa) : rsa_default_size(rsa);
   // RSA modulus sizes are bounded by |BIGNUM|, which must fit in |unsigned|.
   //
   // TODO(https://crbug.com/boringssl/516): Should we make this return |size_t|?
@@ -693,7 +841,7 @@ int RSA_add_pkcs1_prefix(uint8_t **out_msg, size_t *out_msg_len,
 int rsa_sign_no_self_test(int hash_nid, const uint8_t *digest,
                           size_t digest_len, uint8_t *out, unsigned *out_len,
                           RSA *rsa) {
-  if (rsa->meth->sign) {
+  if (rsa->meth && rsa->meth->sign) {
     if (!rsa_check_digest_size(hash_nid, digest_len)) {
       return 0;
     }
@@ -890,7 +1038,7 @@ int RSA_verify_pss_mgf1(RSA *rsa, const uint8_t *digest, size_t digest_len,
 
 int rsa_private_transform_no_self_test(RSA *rsa, uint8_t *out,
                                        const uint8_t *in, size_t len) {
-  if (rsa->meth->private_transform) {
+  if (rsa->meth && rsa->meth->private_transform) {
     return rsa->meth->private_transform(rsa, out, in, len);
   }
 
@@ -906,12 +1054,32 @@ int rsa_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
 
 int RSA_flags(const RSA *rsa) {
   SET_DIT_AUTO_DISABLE;
+  if (rsa == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return 0;
+  }
+
   return rsa->flags;
 }
 
+void RSA_set_flags(RSA *rsa, int flags) {
+  SET_DIT_AUTO_DISABLE;
+  if (rsa == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+    return;
+  }
+
+  rsa->flags |= flags;
+}
+
 int RSA_test_flags(const RSA *rsa, int flags) {
   SET_DIT_AUTO_DISABLE;
-  return rsa->flags & flags;
+  if (rsa) {
+    return rsa->flags & flags;
+  }
+
+  OPENSSL_PUT_ERROR(RSA, ERR_R_PASSED_NULL_PARAMETER);
+  return 0;
 }
 
 int RSA_blinding_on(RSA *rsa, BN_CTX *ctx) {