diff --git a/errors.toml b/errors.toml index 4d552075c13..9c4ab2e8d9e 100644 --- a/errors.toml +++ b/errors.toml @@ -1,6 +1,11 @@ # AUTOGENERATED BY github.com/pingcap/tiup/components/errdoc/errdoc-gen # YOU CAN CHANGE THE 'description'/'workaround' FIELDS IF THEM ARE IMPROPER. +["PD:ErrEncryptionKMS"] +error = ''' +KMS error +''' + ["PD:apiutil:ErrRedirect"] error = ''' redirect failed diff --git a/go.mod b/go.mod index ce61b31aa46..9345af9447f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.1 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 + github.com/aws/aws-sdk-go v1.35.3 github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/coreos/go-semver v0.3.0 diff --git a/go.sum b/go.sum index 661bcb25e9d..14f5ab43300 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,8 @@ github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/EventBus v0.0.0-20180315140547-d46933a94f05/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= +github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54= +github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -421,6 +423,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -969,6 +975,7 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= diff --git a/pkg/encryption/kms.go b/pkg/encryption/kms.go index eb613cceb14..7251e933523 100644 --- a/pkg/encryption/kms.go +++ b/pkg/encryption/kms.go @@ -13,7 +13,124 @@ package encryption +import ( + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/pingcap/kvproto/pkg/encryptionpb" + "github.com/tikv/pd/pkg/errs" +) + const ( // We only support AWS KMS right now. kmsVendorAWS = "AWS" + + // K8S IAM related environment variables. + envAwsRoleArn = "AWS_ROLE_ARN" + envAwsWebIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE" + envAwsRoleSessionName = "AWS_ROLE_SESSION_NAME" ) + +func newMasterKeyFromKMS( + config *encryptionpb.MasterKeyKms, + ciphertextKey []byte, +) (masterKey *MasterKey, err error) { + if config == nil { + return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("missing master key file config") + } + if config.Vendor != kmsVendorAWS { + return nil, errs.ErrEncryptionKMS.GenWithStack("unsupported KMS vendor: %s", config.Vendor) + } + credentials, err := newAwsCredentials() + if err != nil { + return nil, err + } + session, err := session.NewSession(&aws.Config{ + Credentials: credentials, + Region: &config.Region, + Endpoint: &config.Endpoint, + }) + if err != nil { + return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack( + "fail to create AWS session to access KMS CMK") + } + client := kms.New(session) + if len(ciphertextKey) == 0 { + numberOfBytes := int64(masterKeyLength) + // Create a new data key. + output, err := client.GenerateDataKey(&kms.GenerateDataKeyInput{ + KeyId: &config.KeyId, + NumberOfBytes: &numberOfBytes, + }) + if err != nil { + return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack( + "fail to generate data key from AWS KMS") + } + if len(output.Plaintext) != masterKeyLength { + return nil, errs.ErrEncryptionKMS.GenWithStack( + "unexpected data key length generated from AWS KMS, expectd %d vs actual %d", + masterKeyLength, len(output.Plaintext)) + } + masterKey = &MasterKey{ + key: output.Plaintext, + ciphertextKey: output.CiphertextBlob, + } + } else { + // Decrypt existing data key. + output, err := client.Decrypt(&kms.DecryptInput{ + KeyId: &config.KeyId, + CiphertextBlob: ciphertextKey, + }) + if err != nil { + return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack( + "fail to decrypt data key from AWS KMS") + } + if len(output.Plaintext) != masterKeyLength { + return nil, errs.ErrEncryptionKMS.GenWithStack( + "unexpected data key length decrypted from AWS KMS, expected %d vs actual %d", + masterKeyLength, len(output.Plaintext)) + } + masterKey = &MasterKey{ + key: output.Plaintext, + ciphertextKey: ciphertextKey, + } + } + return +} + +func newAwsCredentials() (*credentials.Credentials, error) { + var providers []credentials.Provider + + // Credentials from K8S IAM role. + roleArn := os.Getenv(envAwsRoleArn) + tokenFile := os.Getenv(envAwsWebIdentityTokenFile) + sessionName := os.Getenv(envAwsRoleSessionName) + // Session name is optional. + if roleArn != "" && tokenFile != "" { + session, err := session.NewSession() + if err != nil { + return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack( + "fail to create AWS session to create a WebIdentityRoleProvider") + } + webIdentityProvider := stscreds.NewWebIdentityRoleProvider( + sts.New(session), roleArn, sessionName, tokenFile) + providers = append(providers, webIdentityProvider) + } + + // Credentials from AWS environment variables. + providers = append(providers, &credentials.EnvProvider{}) + + // Credentials from default AWS credentials file. + providers = append(providers, &credentials.SharedCredentialsProvider{ + Filename: "", + Profile: "", + }) + + credentials := credentials.NewChainCredentials(providers) + return credentials, nil +} diff --git a/pkg/encryption/master_key.go b/pkg/encryption/master_key.go index f46e7c71ec8..e8e340ac0cb 100644 --- a/pkg/encryption/master_key.go +++ b/pkg/encryption/master_key.go @@ -18,7 +18,6 @@ import ( "io/ioutil" "strings" - "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/encryptionpb" "github.com/tikv/pd/pkg/errs" ) @@ -33,11 +32,13 @@ type MasterKey struct { // Encryption key in plaintext. If it is nil, encryption is no-op. // Never output it to info log or persist it on disk. key []byte + // Key in ciphertext form. Used by KMS key type. + ciphertextKey []byte } // NewMasterKey obtains a master key from backend specified by given config. // The config may be altered to fill in metadata generated when initializing the master key. -func NewMasterKey(config *encryptionpb.MasterKey) (*MasterKey, error) { +func NewMasterKey(config *encryptionpb.MasterKey, ciphertextKey []byte) (*MasterKey, error) { if config == nil { return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("master key config is empty") } @@ -47,15 +48,21 @@ func NewMasterKey(config *encryptionpb.MasterKey) (*MasterKey, error) { }, nil } if file := config.GetFile(); file != nil { - key, err := newMasterKeyFromFile(file) - if err != nil { - return nil, err - } - return &MasterKey{ - key: key, - }, nil + return newMasterKeyFromFile(file) + } + if kms := config.GetKms(); kms != nil { + return newMasterKeyFromKMS(kms, ciphertextKey) + } + return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("unrecognized master key type") +} + +// NewCustomMasterKeyForTest construct a master key instance from raw key and ciphertext key bytes. +// Used for test only. +func NewCustomMasterKeyForTest(key []byte, ciphertextKey []byte) *MasterKey { + return &MasterKey{ + key: key, + ciphertextKey: ciphertextKey, } - return nil, errors.New("unrecognized master key type") } // Encrypt encrypts given plaintext using the master key. @@ -84,10 +91,16 @@ func (k *MasterKey) IsPlaintext() bool { return k.key == nil } +// CiphertextKey returns the key in encrypted form. +// KMS key type recover the key by decrypting the ciphertextKey from KMS. +func (k *MasterKey) CiphertextKey() []byte { + return k.ciphertextKey +} + // newMasterKeyFromFile reads a hex-string from file specified in the config, and construct a // MasterKey object. The key must be of 256 bits (32 bytes). The file can contain leading and // tailing spaces. -func newMasterKeyFromFile(config *encryptionpb.MasterKeyFile) ([]byte, error) { +func newMasterKeyFromFile(config *encryptionpb.MasterKeyFile) (*MasterKey, error) { if config == nil { return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("missing master key file config") } @@ -110,5 +123,5 @@ func newMasterKeyFromFile(config *encryptionpb.MasterKeyFile) ([]byte, error) { "unexpected key length from master key file, expected %d vs actual %d", masterKeyLength, len(key)) } - return key, nil + return &MasterKey{key: key}, nil } diff --git a/pkg/encryption/master_key_test.go b/pkg/encryption/master_key_test.go index 02d8720e018..908308791b6 100644 --- a/pkg/encryption/master_key_test.go +++ b/pkg/encryption/master_key_test.go @@ -36,7 +36,7 @@ func (s *testMasterKeySuite) TestPlaintextMasterKey(c *C) { Plaintext: &encryptionpb.MasterKeyPlaintext{}, }, } - masterKey, err := NewMasterKey(config) + masterKey, err := NewMasterKey(config, nil) c.Assert(err, IsNil) c.Assert(masterKey, Not(IsNil)) c.Assert(len(masterKey.key), Equals, 0) @@ -91,7 +91,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyMissingPath(c *C) { }, }, } - _, err := NewMasterKey(config) + _, err := NewMasterKey(config, nil) c.Assert(err, Not(IsNil)) } @@ -106,7 +106,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyMissingFile(c *C) { }, }, } - _, err = NewMasterKey(config) + _, err = NewMasterKey(config, nil) c.Assert(err, Not(IsNil)) } @@ -122,7 +122,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyNotHexString(c *C) { }, }, } - _, err = NewMasterKey(config) + _, err = NewMasterKey(config, nil) c.Assert(err, Not(IsNil)) } @@ -138,7 +138,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyLengthMismatch(c *C) { }, }, } - _, err = NewMasterKey(config) + _, err = NewMasterKey(config, nil) c.Assert(err, Not(IsNil)) } @@ -155,7 +155,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKey(c *C) { }, }, } - masterKey, err := NewMasterKey(config) + masterKey, err := NewMasterKey(config, nil) c.Assert(err, IsNil) c.Assert(hex.EncodeToString(masterKey.key), Equals, key) } diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index ba43af1093e..55e2689400b 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -285,4 +285,5 @@ var ( ErrEncryptionLoadKeys = errors.Normalize("load data keys error", errors.RFCCodeText("PD:encryption:ErrEncryptionLoadKeys")) ErrEncryptionRotateDataKey = errors.Normalize("failed to rotate data key", errors.RFCCodeText("PD:encryption:ErrEncryptionRotateDataKey")) ErrEncryptionSaveDataKeys = errors.Normalize("failed to save data keys", errors.RFCCodeText("PD:encryption:ErrEncryptionSaveDataKeys")) + ErrEncryptionKMS = errors.Normalize("KMS error", errors.RFCCodeText("PD:ErrEncryptionKMS")) ) diff --git a/server/encryptionkm/key_manager.go b/server/encryptionkm/key_manager.go index bf3a68788af..fbce747e7ed 100644 --- a/server/encryptionkm/key_manager.go +++ b/server/encryptionkm/key_manager.go @@ -76,9 +76,10 @@ func saveKeys( leadership *election.Leadership, masterKeyMeta *encryptionpb.MasterKey, keys *encryptionpb.KeyDictionary, -) error { + helper keyManagerHelper, +) (err error) { // Get master key. - masterKey, err := encryption.NewMasterKey(masterKeyMeta) + masterKey, err := helper.newMasterKey(masterKeyMeta, nil) if err != nil { return err } @@ -98,9 +99,10 @@ func saveKeys( return err } content := &encryptionpb.EncryptedContent{ - Content: ciphertextContent, - MasterKey: masterKeyMeta, - Iv: iv, + Content: ciphertextContent, + MasterKey: masterKeyMeta, + Iv: iv, + CiphertextKey: masterKey.CiphertextKey(), } value, err := proto.Marshal(content) if err != nil { @@ -124,7 +126,10 @@ func saveKeys( } // extractKeysFromKV unpack encrypted keys from etcd KV. -func extractKeysFromKV(kv *mvccpb.KeyValue) (*encryptionpb.KeyDictionary, error) { +func extractKeysFromKV( + kv *mvccpb.KeyValue, + helper keyManagerHelper, +) (*encryptionpb.KeyDictionary, error) { content := &encryptionpb.EncryptedContent{} err := content.Unmarshal(kv.Value) if err != nil { @@ -136,7 +141,7 @@ func extractKeysFromKV(kv *mvccpb.KeyValue) (*encryptionpb.KeyDictionary, error) return nil, errs.ErrEncryptionLoadKeys.GenWithStack( "no master key config found with encryption keys") } - masterKey, err := encryption.NewMasterKey(masterKeyConfig) + masterKey, err := helper.newMasterKey(masterKeyConfig, content.CiphertextKey) if err != nil { return nil, err } @@ -272,7 +277,7 @@ func (m *KeyManager) loadKeysFromKVImpl( if kv.ModRevision <= m.mu.keysRevision { return m.getKeys(), nil } - keys, err := extractKeysFromKV(kv) + keys, err := extractKeysFromKV(kv, m.helper) if err != nil { return nil, err } @@ -396,7 +401,7 @@ func (m *KeyManager) rotateKeyIfNeeded(forceUpdate bool) error { return nil } // Store updated keys in etcd. - err = saveKeys(m.mu.leadership, m.masterKeyMeta, keys) + err = saveKeys(m.mu.leadership, m.masterKeyMeta, keys, m.helper) if err != nil { m.helper.eventSaveKeysFailure() log.Error("failed to save keys", zap.Error(err)) @@ -488,6 +493,7 @@ func (m *KeyManager) SetLeadership(leadership *election.Leadership) error { type keyManagerHelper struct { now func() time.Time tick func(ticker *time.Ticker) <-chan time.Time + newMasterKey func(*encryptionpb.MasterKey, []byte) (*encryption.MasterKey, error) eventAfterReloadByWatcher func() eventAfterTicker func() eventAfterLeaderCheckSuccess func() @@ -498,6 +504,7 @@ func defaultKeyManagerHelper() keyManagerHelper { return keyManagerHelper{ now: func() time.Time { return time.Now() }, tick: func(ticker *time.Ticker) <-chan time.Time { return ticker.C }, + newMasterKey: encryption.NewMasterKey, eventAfterReloadByWatcher: func() {}, eventAfterTicker: func() {}, eventAfterLeaderCheckSuccess: func() {}, diff --git a/server/encryptionkm/key_manager_test.go b/server/encryptionkm/key_manager_test.go index 2cb047c67a5..9425220f141 100644 --- a/server/encryptionkm/key_manager_test.go +++ b/server/encryptionkm/key_manager_test.go @@ -14,6 +14,7 @@ package encryptionkm import ( + "bytes" "context" "encoding/hex" "fmt" @@ -45,9 +46,10 @@ type testKeyManagerSuite struct{} var _ = Suite(&testKeyManagerSuite{}) const ( - testMasterKey = "8fd7e3e917c170d92f3e51a981dd7bc8fba11f3df7d8df994842f6e86f69b530" - testMasterKey2 = "8fd7e3e917c170d92f3e51a981dd7bc8fba11f3df7d8df994842f6e86f69b531" - testDataKey = "be798242dde0c40d9a65cdbc36c1c9ac" + testMasterKey = "8fd7e3e917c170d92f3e51a981dd7bc8fba11f3df7d8df994842f6e86f69b530" + testMasterKey2 = "8fd7e3e917c170d92f3e51a981dd7bc8fba11f3df7d8df994842f6e86f69b531" + testCiphertextKey = "8fd7e3e917c170d92f3e51a981dd7bc8fba11f3df7d8df994842f6e86f69b532" + testDataKey = "be798242dde0c40d9a65cdbc36c1c9ac" ) func getTestDataKey() []byte { @@ -114,11 +116,12 @@ func newTestLeader(c *C, client *clientv3.Client) *election.Leadership { return leader } -func checkMasterKeyMeta(c *C, value []byte, meta *encryptionpb.MasterKey) { +func checkMasterKeyMeta(c *C, value []byte, meta *encryptionpb.MasterKey, ciphertextKey []byte) { content := &encryptionpb.EncryptedContent{} err := content.Unmarshal(value) c.Assert(err, IsNil) c.Assert(proto.Equal(content.MasterKey, meta), IsTrue) + c.Assert(bytes.Equal(content.CiphertextKey, ciphertextKey), IsTrue) } func (s *testKeyManagerSuite) TestNewKeyManagerBasic(c *C) { @@ -212,7 +215,7 @@ func (s *testKeyManagerSuite) TestNewKeyManagerLoadKeys(c *C) { }, }, } - err = saveKeys(leadership, masterKeyMeta, keys) + err = saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Create the key manager. m, err := NewKeyManager(client, config) @@ -225,7 +228,7 @@ func (s *testKeyManagerSuite) TestNewKeyManagerLoadKeys(c *C) { // Check etcd KV. resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, keys), IsTrue) } @@ -305,7 +308,7 @@ func (s *testKeyManagerSuite) TestGetKey(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Use default config. config := &encryption.Config{} @@ -360,7 +363,7 @@ func (s *testKeyManagerSuite) TestLoadKeyEmpty(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Use default config. config := &encryption.Config{} @@ -424,7 +427,7 @@ func (s *testKeyManagerSuite) TestWatcher(c *C) { }, }, } - err = saveKeys(leadership, masterKeyMeta, keys) + err = saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) <-reloadEvent key, err := m.GetKey(123) @@ -450,7 +453,7 @@ func (s *testKeyManagerSuite) TestWatcher(c *C) { }, }, } - err = saveKeys(leadership, masterKeyMeta, keys) + err = saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) <-reloadEvent key, err = m.GetKey(123) @@ -533,7 +536,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithEncryptionEnabling(c *C) { c.Assert(proto.Equal(loadedKeys.Keys[currentKeyID], currentKey), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(loadedKeys, storedKeys), IsTrue) } @@ -576,7 +579,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithEncryptionMethodChanged(c *C) }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Config with different encrption method. config := &encryption.Config{ @@ -610,7 +613,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithEncryptionMethodChanged(c *C) c.Assert(proto.Equal(loadedKeys.Keys[123], keys.Keys[123]), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(loadedKeys, storedKeys), IsTrue) } @@ -653,7 +656,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithCurrentKeyExposed(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Config with different encrption method. config := &encryption.Config{ @@ -688,7 +691,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithCurrentKeyExposed(c *C) { c.Assert(proto.Equal(loadedKeys.Keys[123], keys.Keys[123]), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(loadedKeys, storedKeys), IsTrue) } @@ -731,7 +734,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithCurrentKeyExpired(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Config with 100s rotation period. rotationPeriod, err := time.ParseDuration("100s") @@ -770,7 +773,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithCurrentKeyExpired(c *C) { c.Assert(proto.Equal(loadedKeys.Keys[123], keys.Keys[123]), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(loadedKeys, storedKeys), IsTrue) } @@ -815,7 +818,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithMasterKeyChanged(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Config with a different master key. config := &encryption.Config{ @@ -842,12 +845,95 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithMasterKeyChanged(c *C) { c.Assert(proto.Equal(m.keys.Load().(*encryptionpb.KeyDictionary), keys), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, keys), IsTrue) meta, err := config.GetMasterKeyMeta() c.Assert(err, IsNil) - checkMasterKeyMeta(c, resp.Kvs[0].Value, meta) + checkMasterKeyMeta(c, resp.Kvs[0].Value, meta, nil) +} + +func (s *testKeyManagerSuite) TestSetLeadershipMasterKeyWithCiphertextKey(c *C) { + // Initialize. + client, cleanupEtcd := newTestEtcd(c) + defer cleanupEtcd() + keyFile, cleanupKeyFile := newTestKeyFile(c) + defer cleanupKeyFile() + leadership := newTestLeader(c, client) + // Setup helper + helper := defaultKeyManagerHelper() + // Mock time + helper.now = func() time.Time { return time.Unix(int64(1601679533), 0) } + // mock NewMasterKey + newMasterKeyCalled := 0 + outputMasterKey, _ := hex.DecodeString(testMasterKey) + outputCiphertextKey, _ := hex.DecodeString(testCiphertextKey) + helper.newMasterKey = func( + meta *encryptionpb.MasterKey, + ciphertext []byte, + ) (*encryption.MasterKey, error) { + if newMasterKeyCalled < 2 { + // initial load and save. no ciphertextKey + c.Assert(ciphertext, IsNil) + } else if newMasterKeyCalled == 2 { + // called by loadKeys after saveKeys + c.Assert(bytes.Equal(ciphertext, outputCiphertextKey), IsTrue) + } + newMasterKeyCalled += 1 + return encryption.NewCustomMasterKeyForTest(outputMasterKey, outputCiphertextKey), nil + } + // Update keys in etcd + masterKeyMeta := &encryptionpb.MasterKey{ + Backend: &encryptionpb.MasterKey_File{ + File: &encryptionpb.MasterKeyFile{ + Path: keyFile, + }, + }, + } + keys := &encryptionpb.KeyDictionary{ + CurrentKeyId: 123, + Keys: map[uint64]*encryptionpb.DataKey{ + 123: { + Key: getTestDataKey(), + Method: encryptionpb.EncryptionMethod_AES128_CTR, + CreationTime: uint64(1601679533), + WasExposed: false, + }, + }, + } + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) + c.Assert(err, IsNil) + // Config with a different master key. + config := &encryption.Config{ + DataEncryptionMethod: "aes128-ctr", + MasterKey: encryption.MasterKeyConfig{ + Type: "file", + MasterKeyFileConfig: encryption.MasterKeyFileConfig{ + FilePath: keyFile, + }, + }, + } + err = config.Adjust() + c.Assert(err, IsNil) + // Create the key manager. + m, err := newKeyManagerImpl(client, config, helper) + c.Assert(err, IsNil) + c.Assert(proto.Equal(m.keys.Load().(*encryptionpb.KeyDictionary), keys), IsTrue) + // Set leadership + err = m.SetLeadership(leadership) + c.Assert(err, IsNil) + c.Assert(newMasterKeyCalled, Equals, 3) + // Check if keys are the same + c.Assert(proto.Equal(m.keys.Load().(*encryptionpb.KeyDictionary), keys), IsTrue) + resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) + c.Assert(err, IsNil) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) + c.Assert(err, IsNil) + c.Assert(proto.Equal(storedKeys, keys), IsTrue) + meta, err := config.GetMasterKeyMeta() + c.Assert(err, IsNil) + // Check ciphertext key is stored with keys. + checkMasterKeyMeta(c, resp.Kvs[0].Value, meta, outputCiphertextKey) } func (s *testKeyManagerSuite) TestSetLeadershipWithEncryptionDisabling(c *C) { @@ -886,7 +972,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithEncryptionDisabling(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Use default config. config := &encryption.Config{} @@ -908,7 +994,7 @@ func (s *testKeyManagerSuite) TestSetLeadershipWithEncryptionDisabling(c *C) { c.Assert(proto.Equal(m.keys.Load().(*encryptionpb.KeyDictionary), expectedKeys), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, expectedKeys), IsTrue) } @@ -960,7 +1046,7 @@ func (s *testKeyManagerSuite) TestKeyRotation(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Config with 100s rotation period. rotationPeriod, err := time.ParseDuration("100s") @@ -989,7 +1075,7 @@ func (s *testKeyManagerSuite) TestKeyRotation(c *C) { c.Assert(proto.Equal(m.keys.Load().(*encryptionpb.KeyDictionary), keys), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, keys), IsTrue) // Advance time and trigger ticker @@ -1011,7 +1097,7 @@ func (s *testKeyManagerSuite) TestKeyRotation(c *C) { c.Assert(proto.Equal(loadedKeys.Keys[currentKeyID], currentKey), IsTrue) resp, err = etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err = extractKeysFromKV(resp.Kvs[0]) + storedKeys, err = extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, loadedKeys), IsTrue) } @@ -1073,7 +1159,7 @@ func (s *testKeyManagerSuite) TestKeyRotationConflict(c *C) { }, }, } - err := saveKeys(leadership, masterKeyMeta, keys) + err := saveKeys(leadership, masterKeyMeta, keys, defaultKeyManagerHelper()) c.Assert(err, IsNil) // Config with 100s rotation period. rotationPeriod, err := time.ParseDuration("100s") @@ -1102,7 +1188,7 @@ func (s *testKeyManagerSuite) TestKeyRotationConflict(c *C) { c.Assert(proto.Equal(m.keys.Load().(*encryptionpb.KeyDictionary), keys), IsTrue) resp, err := etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err := extractKeysFromKV(resp.Kvs[0]) + storedKeys, err := extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, keys), IsTrue) // Invalidate leader after leader check. @@ -1116,7 +1202,7 @@ func (s *testKeyManagerSuite) TestKeyRotationConflict(c *C) { // Check keys is unchanged. resp, err = etcdutil.EtcdKVGet(client, EncryptionKeysPath) c.Assert(err, IsNil) - storedKeys, err = extractKeysFromKV(resp.Kvs[0]) + storedKeys, err = extractKeysFromKV(resp.Kvs[0], defaultKeyManagerHelper()) c.Assert(err, IsNil) c.Assert(proto.Equal(storedKeys, keys), IsTrue) }