security: Encrypt region boundary keys, Part 4 - KMS (#3141)

* Encrypt region boundary keys, Part 2 - server changes Signed-off-by: Yi Wu <yiwu@pingcap.com> * update errno and config template Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix typo Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix tests Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix tests Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix tests Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix tests Signed-off-by: Yi Wu <yiwu@pingcap.com> * address comment in #2931 Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix loadRegion Signed-off-by: Yi Wu <yiwu@pingcap.com> * rename encryption_key_manager package Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix lint Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix lint Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix comments Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix comment Signed-off-by: Yi Wu <yiwu@pingcap.com> * use option pattern Signed-off-by: Yi Wu <yiwu@pingcap.com> * move loadRegion and saveRegion Signed-off-by: Yi Wu <yiwu@pingcap.com> * revert changes Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix doc Signed-off-by: Yi Wu <yiwu@pingcap.com> * address comment Signed-off-by: Yi Wu <yiwu@pingcap.com> * key manager Signed-off-by: Yi Wu <yiwu@pingcap.com> * add test and refactor Signed-off-by: Yi Wu <yiwu@pingcap.com> * make EncryptionKeysPath a const Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix region_crypter key manager nil check Signed-off-by: Yi Wu <yiwu@pingcap.com> * save conflict test Signed-off-by: Yi Wu <yiwu@pingcap.com> * sanity check keys revision Signed-off-by: Yi Wu <yiwu@pingcap.com> * kms Signed-off-by: Yi Wu <yiwu@pingcap.com> * test set ciphertextKey Signed-off-by: Yi Wu <yiwu@pingcap.com> * test set ciphertextKey Signed-off-by: Yi Wu <yiwu@pingcap.com> * clone region only when needed Signed-off-by: Yi Wu <yiwu@pingcap.com> * add test for config Signed-off-by: Yi Wu <yiwu@pingcap.com> * update errors Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix lint Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix lint Signed-off-by: Yi Wu <yiwu@pingcap.com> * make errdoc Signed-off-by: Yi Wu <yiwu@pingcap.com> * address comment Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix error type Signed-off-by: Yi Wu <yiwu@pingcap.com> * fix test Signed-off-by: Yi Wu <yiwu@pingcap.com>
tikv · Nov 13, 2020 · 3c62563 · 3c62563
1 parent 7ca7645
commit 3c62563
Show file tree

Hide file tree

Showing 9 changed files with 292 additions and 55 deletions.
diff --git a/errors.toml b/errors.toml
@@ -1,6 +1,11 @@
 # AUTOGENERATED BY github.com/pingcap/tiup/components/errdoc/errdoc-gen
 # YOU CAN CHANGE THE 'description'/'workaround' FIELDS IF THEM ARE IMPROPER.
 
+["PD:ErrEncryptionKMS"]
+error = '''
+KMS error
+'''
+
 ["PD:apiutil:ErrRedirect"]
 error = '''
 redirect failed

diff --git a/go.mod b/go.mod
@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
+	github.com/aws/aws-sdk-go v1.35.3
 	github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
 	github.com/coreos/go-semver v0.3.0

diff --git a/go.sum b/go.sum
@@ -85,6 +85,8 @@ github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS
 github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/asaskevich/EventBus v0.0.0-20180315140547-d46933a94f05/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII=
+github.com/aws/aws-sdk-go v1.35.3 h1:r0puXncSaAfRt7Btml2swUo74Kao+vKhO3VLjwDjK54=
+github.com/aws/aws-sdk-go v1.35.3/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -421,6 +423,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
@@ -969,6 +975,7 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=

diff --git a/pkg/encryption/kms.go b/pkg/encryption/kms.go
@@ -13,7 +13,124 @@
 
 package encryption
 
+import (
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/kms"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/pingcap/kvproto/pkg/encryptionpb"
+	"github.com/tikv/pd/pkg/errs"
+)
+
 const (
 	// We only support AWS KMS right now.
 	kmsVendorAWS = "AWS"
+
+	// K8S IAM related environment variables.
+	envAwsRoleArn              = "AWS_ROLE_ARN"
+	envAwsWebIdentityTokenFile = "AWS_WEB_IDENTITY_TOKEN_FILE"
+	envAwsRoleSessionName      = "AWS_ROLE_SESSION_NAME"
 )
+
+func newMasterKeyFromKMS(
+	config *encryptionpb.MasterKeyKms,
+	ciphertextKey []byte,
+) (masterKey *MasterKey, err error) {
+	if config == nil {
+		return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("missing master key file config")
+	}
+	if config.Vendor != kmsVendorAWS {
+		return nil, errs.ErrEncryptionKMS.GenWithStack("unsupported KMS vendor: %s", config.Vendor)
+	}
+	credentials, err := newAwsCredentials()
+	if err != nil {
+		return nil, err
+	}
+	session, err := session.NewSession(&aws.Config{
+		Credentials: credentials,
+		Region:      &config.Region,
+		Endpoint:    &config.Endpoint,
+	})
+	if err != nil {
+		return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack(
+			"fail to create AWS session to access KMS CMK")
+	}
+	client := kms.New(session)
+	if len(ciphertextKey) == 0 {
+		numberOfBytes := int64(masterKeyLength)
+		// Create a new data key.
+		output, err := client.GenerateDataKey(&kms.GenerateDataKeyInput{
+			KeyId:         &config.KeyId,
+			NumberOfBytes: &numberOfBytes,
+		})
+		if err != nil {
+			return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack(
+				"fail to generate data key from AWS KMS")
+		}
+		if len(output.Plaintext) != masterKeyLength {
+			return nil, errs.ErrEncryptionKMS.GenWithStack(
+				"unexpected data key length generated from AWS KMS, expectd %d vs actual %d",
+				masterKeyLength, len(output.Plaintext))
+		}
+		masterKey = &MasterKey{
+			key:           output.Plaintext,
+			ciphertextKey: output.CiphertextBlob,
+		}
+	} else {
+		// Decrypt existing data key.
+		output, err := client.Decrypt(&kms.DecryptInput{
+			KeyId:          &config.KeyId,
+			CiphertextBlob: ciphertextKey,
+		})
+		if err != nil {
+			return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack(
+				"fail to decrypt data key from AWS KMS")
+		}
+		if len(output.Plaintext) != masterKeyLength {
+			return nil, errs.ErrEncryptionKMS.GenWithStack(
+				"unexpected data key length decrypted from AWS KMS, expected %d vs actual %d",
+				masterKeyLength, len(output.Plaintext))
+		}
+		masterKey = &MasterKey{
+			key:           output.Plaintext,
+			ciphertextKey: ciphertextKey,
+		}
+	}
+	return
+}
+
+func newAwsCredentials() (*credentials.Credentials, error) {
+	var providers []credentials.Provider
+
+	// Credentials from K8S IAM role.
+	roleArn := os.Getenv(envAwsRoleArn)
+	tokenFile := os.Getenv(envAwsWebIdentityTokenFile)
+	sessionName := os.Getenv(envAwsRoleSessionName)
+	// Session name is optional.
+	if roleArn != "" && tokenFile != "" {
+		session, err := session.NewSession()
+		if err != nil {
+			return nil, errs.ErrEncryptionKMS.Wrap(err).GenWithStack(
+				"fail to create AWS session to create a WebIdentityRoleProvider")
+		}
+		webIdentityProvider := stscreds.NewWebIdentityRoleProvider(
+			sts.New(session), roleArn, sessionName, tokenFile)
+		providers = append(providers, webIdentityProvider)
+	}
+
+	// Credentials from AWS environment variables.
+	providers = append(providers, &credentials.EnvProvider{})
+
+	// Credentials from default AWS credentials file.
+	providers = append(providers, &credentials.SharedCredentialsProvider{
+		Filename: "",
+		Profile:  "",
+	})
+
+	credentials := credentials.NewChainCredentials(providers)
+	return credentials, nil
+}
diff --git a/pkg/encryption/master_key.go b/pkg/encryption/master_key.go
@@ -18,7 +18,6 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/encryptionpb"
 	"github.com/tikv/pd/pkg/errs"
 )
@@ -33,11 +32,13 @@ type MasterKey struct {
 	// Encryption key in plaintext. If it is nil, encryption is no-op.
 	// Never output it to info log or persist it on disk.
 	key []byte
+	// Key in ciphertext form. Used by KMS key type.
+	ciphertextKey []byte
 }
 
 // NewMasterKey obtains a master key from backend specified by given config.
 // The config may be altered to fill in metadata generated when initializing the master key.
-func NewMasterKey(config *encryptionpb.MasterKey) (*MasterKey, error) {
+func NewMasterKey(config *encryptionpb.MasterKey, ciphertextKey []byte) (*MasterKey, error) {
 	if config == nil {
 		return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("master key config is empty")
 	}
@@ -47,15 +48,21 @@ func NewMasterKey(config *encryptionpb.MasterKey) (*MasterKey, error) {
 		}, nil
 	}
 	if file := config.GetFile(); file != nil {
-		key, err := newMasterKeyFromFile(file)
-		if err != nil {
-			return nil, err
-		}
-		return &MasterKey{
-			key: key,
-		}, nil
+		return newMasterKeyFromFile(file)
+	}
+	if kms := config.GetKms(); kms != nil {
+		return newMasterKeyFromKMS(kms, ciphertextKey)
+	}
+	return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("unrecognized master key type")
+}
+
+// NewCustomMasterKeyForTest construct a master key instance from raw key and ciphertext key bytes.
+// Used for test only.
+func NewCustomMasterKeyForTest(key []byte, ciphertextKey []byte) *MasterKey {
+	return &MasterKey{
+		key:           key,
+		ciphertextKey: ciphertextKey,
 	}
-	return nil, errors.New("unrecognized master key type")
 }
 
 // Encrypt encrypts given plaintext using the master key.
@@ -84,10 +91,16 @@ func (k *MasterKey) IsPlaintext() bool {
 	return k.key == nil
 }
 
+// CiphertextKey returns the key in encrypted form.
+// KMS key type recover the key by decrypting the ciphertextKey from KMS.
+func (k *MasterKey) CiphertextKey() []byte {
+	return k.ciphertextKey
+}
+
 // newMasterKeyFromFile reads a hex-string from file specified in the config, and construct a
 // MasterKey object. The key must be of 256 bits (32 bytes). The file can contain leading and
 // tailing spaces.
-func newMasterKeyFromFile(config *encryptionpb.MasterKeyFile) ([]byte, error) {
+func newMasterKeyFromFile(config *encryptionpb.MasterKeyFile) (*MasterKey, error) {
 	if config == nil {
 		return nil, errs.ErrEncryptionNewMasterKey.GenWithStack("missing master key file config")
 	}
@@ -110,5 +123,5 @@ func newMasterKeyFromFile(config *encryptionpb.MasterKeyFile) ([]byte, error) {
 			"unexpected key length from master key file, expected %d vs actual %d",
 			masterKeyLength, len(key))
 	}
-	return key, nil
+	return &MasterKey{key: key}, nil
 }
diff --git a/pkg/encryption/master_key_test.go b/pkg/encryption/master_key_test.go
@@ -36,7 +36,7 @@ func (s *testMasterKeySuite) TestPlaintextMasterKey(c *C) {
 			Plaintext: &encryptionpb.MasterKeyPlaintext{},
 		},
 	}
-	masterKey, err := NewMasterKey(config)
+	masterKey, err := NewMasterKey(config, nil)
 	c.Assert(err, IsNil)
 	c.Assert(masterKey, Not(IsNil))
 	c.Assert(len(masterKey.key), Equals, 0)
@@ -91,7 +91,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyMissingPath(c *C) {
 			},
 		},
 	}
-	_, err := NewMasterKey(config)
+	_, err := NewMasterKey(config, nil)
 	c.Assert(err, Not(IsNil))
 }
 
@@ -106,7 +106,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyMissingFile(c *C) {
 			},
 		},
 	}
-	_, err = NewMasterKey(config)
+	_, err = NewMasterKey(config, nil)
 	c.Assert(err, Not(IsNil))
 }
 
@@ -122,7 +122,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyNotHexString(c *C) {
 			},
 		},
 	}
-	_, err = NewMasterKey(config)
+	_, err = NewMasterKey(config, nil)
 	c.Assert(err, Not(IsNil))
 }
 
@@ -138,7 +138,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKeyLengthMismatch(c *C) {
 			},
 		},
 	}
-	_, err = NewMasterKey(config)
+	_, err = NewMasterKey(config, nil)
 	c.Assert(err, Not(IsNil))
 }
 
@@ -155,7 +155,7 @@ func (s *testMasterKeySuite) TestNewFileMasterKey(c *C) {
 			},
 		},
 	}
-	masterKey, err := NewMasterKey(config)
+	masterKey, err := NewMasterKey(config, nil)
 	c.Assert(err, IsNil)
 	c.Assert(hex.EncodeToString(masterKey.key), Equals, key)
 }
diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go
@@ -285,4 +285,5 @@ var (
 	ErrEncryptionLoadKeys           = errors.Normalize("load data keys error", errors.RFCCodeText("PD:encryption:ErrEncryptionLoadKeys"))
 	ErrEncryptionRotateDataKey      = errors.Normalize("failed to rotate data key", errors.RFCCodeText("PD:encryption:ErrEncryptionRotateDataKey"))
 	ErrEncryptionSaveDataKeys       = errors.Normalize("failed to save data keys", errors.RFCCodeText("PD:encryption:ErrEncryptionSaveDataKeys"))
+	ErrEncryptionKMS                = errors.Normalize("KMS error", errors.RFCCodeText("PD:ErrEncryptionKMS"))
 )
diff --git a/server/encryptionkm/key_manager.go b/server/encryptionkm/key_manager.go
@@ -76,9 +76,10 @@ func saveKeys(
 	leadership *election.Leadership,
 	masterKeyMeta *encryptionpb.MasterKey,
 	keys *encryptionpb.KeyDictionary,
-) error {
+	helper keyManagerHelper,
+) (err error) {
 	// Get master key.
-	masterKey, err := encryption.NewMasterKey(masterKeyMeta)
+	masterKey, err := helper.newMasterKey(masterKeyMeta, nil)
 	if err != nil {
 		return err
 	}
@@ -98,9 +99,10 @@ func saveKeys(
 		return err
 	}
 	content := &encryptionpb.EncryptedContent{
-		Content:   ciphertextContent,
-		MasterKey: masterKeyMeta,
-		Iv:        iv,
+		Content:       ciphertextContent,
+		MasterKey:     masterKeyMeta,
+		Iv:            iv,
+		CiphertextKey: masterKey.CiphertextKey(),
 	}
 	value, err := proto.Marshal(content)
 	if err != nil {
@@ -124,7 +126,10 @@ func saveKeys(
 }
 
 // extractKeysFromKV unpack encrypted keys from etcd KV.
-func extractKeysFromKV(kv *mvccpb.KeyValue) (*encryptionpb.KeyDictionary, error) {
+func extractKeysFromKV(
+	kv *mvccpb.KeyValue,
+	helper keyManagerHelper,
+) (*encryptionpb.KeyDictionary, error) {
 	content := &encryptionpb.EncryptedContent{}
 	err := content.Unmarshal(kv.Value)
 	if err != nil {
@@ -136,7 +141,7 @@ func extractKeysFromKV(kv *mvccpb.KeyValue) (*encryptionpb.KeyDictionary, error)
 		return nil, errs.ErrEncryptionLoadKeys.GenWithStack(
 			"no master key config found with encryption keys")
 	}
-	masterKey, err := encryption.NewMasterKey(masterKeyConfig)
+	masterKey, err := helper.newMasterKey(masterKeyConfig, content.CiphertextKey)
 	if err != nil {
 		return nil, err
 	}
@@ -272,7 +277,7 @@ func (m *KeyManager) loadKeysFromKVImpl(
 	if kv.ModRevision <= m.mu.keysRevision {
 		return m.getKeys(), nil
 	}
-	keys, err := extractKeysFromKV(kv)
+	keys, err := extractKeysFromKV(kv, m.helper)
 	if err != nil {
 		return nil, err
 	}
@@ -396,7 +401,7 @@ func (m *KeyManager) rotateKeyIfNeeded(forceUpdate bool) error {
 		return nil
 	}
 	// Store updated keys in etcd.
-	err = saveKeys(m.mu.leadership, m.masterKeyMeta, keys)
+	err = saveKeys(m.mu.leadership, m.masterKeyMeta, keys, m.helper)
 	if err != nil {
 		m.helper.eventSaveKeysFailure()
 		log.Error("failed to save keys", zap.Error(err))
@@ -488,6 +493,7 @@ func (m *KeyManager) SetLeadership(leadership *election.Leadership) error {
 type keyManagerHelper struct {
 	now                          func() time.Time
 	tick                         func(ticker *time.Ticker) <-chan time.Time
+	newMasterKey                 func(*encryptionpb.MasterKey, []byte) (*encryption.MasterKey, error)
 	eventAfterReloadByWatcher    func()
 	eventAfterTicker             func()
 	eventAfterLeaderCheckSuccess func()
@@ -498,6 +504,7 @@ func defaultKeyManagerHelper() keyManagerHelper {
 	return keyManagerHelper{
 		now:                          func() time.Time { return time.Now() },
 		tick:                         func(ticker *time.Ticker) <-chan time.Time { return ticker.C },
+		newMasterKey:                 encryption.NewMasterKey,
 		eventAfterReloadByWatcher:    func() {},
 		eventAfterTicker:             func() {},
 		eventAfterLeaderCheckSuccess: func() {},