Skip to content

Commit

Permalink
Merge pull request open-horizon#4053 from LiilyZhang/zhangl/serviceDef
Browse files Browse the repository at this point in the history 
Issue open-horizon#4052 - MMSinCluster: Add mmsPVC in the service def…
  • Loading branch information
@LiilyZhang
LiilyZhang authored May 7, 2024
2 parents 80dd0db + 0533cc0 commit 4a994a5
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 18 deletions.
20 changes: 20 additions & 0 deletions cli/kube_deployment/kube_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,22 @@ func (p *KubeDeploymentConfigPlugin) Sign(dep map[string]interface{}, privKey *r
md["namespace"] = namespaceInOperator
dep["metadata"] = md

if _, ok := dep["mmsPVC"]; ok {
// mmsPVC field is defined
mmsPVCConfig := dep["mmsPVC"].(map[string]interface{})
enableVal, ok := mmsPVCConfig["enable"]
if !ok || !enableVal.(bool) {
msgPrinter.Printf("Warning: mmsPVC is not enabled for this cluster service")
// remove the "mmsPVC" section
delete(dep, "mmsPVC")
}

if pvcSizeVal, ok := mmsPVCConfig["pvcSize"]; ok {
pvcSize := int64(pvcSizeVal.(float64))
msgPrinter.Printf("pvcSize: %v\n", pvcSize)
}
}

// Stringify and sign the deployment string.
deployment, err := json.Marshal(dep)
if err != nil {
Expand Down Expand Up @@ -109,6 +125,10 @@ func (p *KubeDeploymentConfigPlugin) DefaultConfig(imageInfo interface{}) interf
func (p *KubeDeploymentConfigPlugin) DefaultClusterConfig() interface{} {
return map[string]interface{}{
"operatorYamlArchive": "",
"mmsPVC": map[string]interface{}{
"enable": false,
"pvcSize": 0,
},
}
}

Expand Down
3 changes: 2 additions & 1 deletion common/deploymentconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ type ClusterDeploymentConfig struct {
Metadata map[string]interface{} `json:"metadata,omitempty"`
OperatorYamlArchive string `json:"operatorYamlArchive"`
Secrets map[string]containermessage.Secret `json:"secrets"`
MMSPVC map[string]interface{} `json:"mmspvc,omitempty"`
}

// Take the deployment field, which we have told the json unmarshaller was unknown type (so we can handle both escaped string and struct)
Expand Down Expand Up @@ -221,6 +222,6 @@ func GetClusterDeploymentMetadata(clusterDeployment interface{}, inspectOperator
}

func GetKubeOperatorNamespace(tar string) (string, error) {
_, namespace, err := kube_operator.ProcessDeployment(tar, nil, map[string]string{}, "", "", map[string]string{}, "", 0)
_, namespace, err := kube_operator.ProcessDeployment(tar, nil, nil, map[string]string{}, "", "", map[string]string{}, "", 0)
return namespace, err
}
32 changes: 25 additions & 7 deletions kube_operator/api_objects.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ type APIObjectInterface interface {
// Sort a slice of k8s api objects by kind of object
// Returns a map of object type names to api object interfaces types, the namespace to be used for the operator, and an error if one occurs
// Also verifies that all objects are named so they can be found and uninstalled
func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstructured.Unstructured, metadata map[string]interface{}, envVarMap map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agreementId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) {
func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstructured.Unstructured, metadata map[string]interface{}, mmsPVCConfig map[string]interface{}, envVarMap map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agreementId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) {
namespace := ""

// get the namespace from metadata
Expand Down Expand Up @@ -123,7 +123,7 @@ func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstr
return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: multiple namespaces specified in operator: %s and %s", namespace, typedDeployment.ObjectMeta.Namespace)))
}
}
newDeployment := DeploymentAppsV1{DeploymentObject: typedDeployment, EnvVarMap: envVarMap, FssAuthFilePath: fssAuthFilePath, FssCertFilePath: fssCertFilePath, ServiceSecrets: secretsMap, AgreementId: agreementId}
newDeployment := DeploymentAppsV1{DeploymentObject: typedDeployment, EnvVarMap: envVarMap, FssAuthFilePath: fssAuthFilePath, FssCertFilePath: fssCertFilePath, MMSPVCConfig: mmsPVCConfig, ServiceSecrets: secretsMap, AgreementId: agreementId}
if newDeployment.Name() != "" {
glog.V(4).Infof(kwlog(fmt.Sprintf("Found kubernetes deployment object %s.", newDeployment.Name())))
objMap[K8S_DEPLOYMENT_TYPE] = append(objMap[K8S_DEPLOYMENT_TYPE], newDeployment)
Expand Down Expand Up @@ -631,6 +631,7 @@ type DeploymentAppsV1 struct {
EnvVarMap map[string]string
FssAuthFilePath string
FssCertFilePath string
MMSPVCConfig map[string]interface{}
ServiceSecrets map[string]string
AgreementId string
}
Expand Down Expand Up @@ -676,12 +677,17 @@ func (d DeploymentAppsV1) Install(c KubeClient, namespace string) error {
}

// create MMS pvc
pvcName, err := c.CreateMMSPVC(d.EnvVarMap, d.AgreementId, namespace)
if err != nil && errors.IsAlreadyExists(err) {
c.DeleteMMSPVC(d.AgreementId, namespace)
pvcName, _ = c.CreateMMSPVC(d.EnvVarMap, d.AgreementId, namespace)
if !IsMMSPVCEnabled(d.MMSPVCConfig) {
glog.V(3).Infof(kwlog(fmt.Sprintf("MMSPVCConfig is not enabled %v, skip creating the MMS PVC", d.MMSPVCConfig)))
} else {
glog.V(3).Infof(kwlog(fmt.Sprintf("MMSPVCConfig is enabled %v, creating the MMS PVC", d.MMSPVCConfig)))
pvcName, err := c.CreateMMSPVC(d.EnvVarMap, d.MMSPVCConfig, d.AgreementId, namespace)
if err != nil && errors.IsAlreadyExists(err) {
c.DeleteMMSPVC(d.AgreementId, namespace)
pvcName, _ = c.CreateMMSPVC(d.EnvVarMap, d.MMSPVCConfig, d.AgreementId, namespace)
}
glog.V(3).Infof(kwlog(fmt.Sprintf("MMS pvc %v is created under namespace: %v", pvcName, namespace)))
}
glog.V(3).Infof(kwlog(fmt.Sprintf("MMS pvc %v is created under namespace: %v", pvcName, namespace)))

// Let the operator know about the config map
dWithEnv := addConfigMapVarToDeploymentObject(*d.DeploymentObject, mapName)
Expand Down Expand Up @@ -1392,3 +1398,15 @@ func removeRoleBindingSubjects(allSubjects []rbacv1.Subject, subjectsToRemove []
}
return allSubjects
}

func IsMMSPVCEnabled(MMSPVCConfig map[string]interface{}) bool {
if MMSPVCConfig == nil {
return false
}

enableVal, ok := MMSPVCConfig["enable"]
if !ok || !enableVal.(bool) {
return false
}
return true
}
27 changes: 18 additions & 9 deletions kube_operator/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"k8s.io/client-go/kubernetes/scheme"
"os"
"reflect"
"strconv"
"strings"
)

Expand Down Expand Up @@ -147,9 +148,9 @@ func NewDynamicKubeClient() (dynamic.Interface, error) {
}

// Install creates the objects specified in the operator deployment in the cluster and creates the custom resource to start the operator
func (c KubeClient) Install(tar string, metadata map[string]interface{}, envVars map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agId string, reqNamespace string, crInstallTimeout int64) error {
func (c KubeClient) Install(tar string, metadata map[string]interface{}, mmsPVCConfig map[string]interface{}, envVars map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agId string, reqNamespace string, crInstallTimeout int64) error {

apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, envVars, fssAuthFilePath, fssCertFilePath, secretsMap, agId, crInstallTimeout)
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, mmsPVCConfig, envVars, fssAuthFilePath, fssCertFilePath, secretsMap, agId, crInstallTimeout)
if err != nil {
return err
}
Expand Down Expand Up @@ -198,7 +199,7 @@ func (c KubeClient) Install(tar string, metadata map[string]interface{}, envVars
// Install creates the objects specified in the operator deployment in the cluster and creates the custom resource to start the operator
func (c KubeClient) Uninstall(tar string, metadata map[string]interface{}, agId string, reqNamespace string) error {

apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, "", "", map[string]string{}, agId, 0)
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, nil, map[string]string{}, "", "", map[string]string{}, agId, 0)
if err != nil {
return err
}
Expand Down Expand Up @@ -237,7 +238,7 @@ func (c KubeClient) Uninstall(tar string, metadata map[string]interface{}, agId
return nil
}
func (c KubeClient) OperatorStatus(tar string, metadata map[string]interface{}, agId string, reqNamespace string) (interface{}, error) {
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, "", "", map[string]string{}, agId, 0)
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, nil, map[string]string{}, "", "", map[string]string{}, agId, 0)
if err != nil {
return nil, err
}
Expand All @@ -255,7 +256,7 @@ func (c KubeClient) OperatorStatus(tar string, metadata map[string]interface{},
}

func (c KubeClient) Status(tar string, metadata map[string]interface{}, agId string, reqNamespace string) ([]ContainerStatus, error) {
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, "", "", map[string]string{}, agId, 0)
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, nil, map[string]string{}, "", "", map[string]string{}, agId, 0)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -311,7 +312,7 @@ func (c KubeClient) Update(tar string, metadata map[string]interface{}, agId str
}

// Current implementaion only updatedSecrets will be passed into this function
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, updatedEnv, "", "", updatedSecretsMap, agId, 0)
apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, nil, updatedEnv, "", "", updatedSecretsMap, agId, 0)
if err != nil {
return err
}
Expand All @@ -332,7 +333,7 @@ func (c KubeClient) Update(tar string, metadata map[string]interface{}, agId str
}

// processDeployment takes the deployment string and converts it to a map with the k8s objects, the namespace to be used, and an error if one occurs
func ProcessDeployment(tar string, metadata map[string]interface{}, envVars map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) {
func ProcessDeployment(tar string, metadata map[string]interface{}, mmsPVCConfig map[string]interface{}, envVars map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) {
// Read the yaml files from the commpressed tar files
yamls, err := getYamlFromTarGz(tar)
if err != nil {
Expand All @@ -355,7 +356,7 @@ func ProcessDeployment(tar string, metadata map[string]interface{}, envVars map[
}

// Sort the k8s api objects by kind
return sortAPIObjects(k8sObjs, customResourceKindMap, metadata, envVars, fssAuthFilePath, fssCertFilePath, secretsMap, agId, crInstallTimeout)
return sortAPIObjects(k8sObjs, customResourceKindMap, metadata, mmsPVCConfig, envVars, fssAuthFilePath, fssCertFilePath, secretsMap, agId, crInstallTimeout)
}

// CreateConfigMap will create a config map with the provided environment variable map
Expand Down Expand Up @@ -479,8 +480,9 @@ func (c KubeClient) DeleteK8SSecrets(agId string, namespace string) error {
return nil
}

func (c KubeClient) CreateMMSPVC(envVars map[string]string, agId string, namespace string) (string, error) {
func (c KubeClient) CreateMMSPVC(envVars map[string]string, mmsPVCConfig map[string]interface{}, agId string, namespace string) (string, error) {
storageClass, accessModes, _ := cutil.GetAgentPVCInfo()

if scInUserinput, ok := envVars[STORAGE_CLASS_USERINPUT_NAME]; ok {
storageClass = scInUserinput
}
Expand All @@ -492,6 +494,13 @@ func (c KubeClient) CreateMMSPVC(envVars map[string]string, agId string, namespa
}

pvcSizeInString := DEFAULT_PVC_SIZE_IN_STRING
if size, ok := mmsPVCConfig["pvcSize"]; ok {
sizeInServiceDef := int64(size.(float64))
if sizeInServiceDef > 0 {
pvcSizeInString = strconv.FormatInt(sizeInServiceDef, 10)
}
}

if pvcSizeInUserInput, ok := envVars[PVC_SIZE_USERINPUT_NAME]; ok {
pvcSizeInString = pvcSizeInUserInput
}
Expand Down
2 changes: 1 addition & 1 deletion kube_operator/kubeworker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func (w *KubeWorker) processKubeOperator(lc *events.AgreementLaunchContext, kd *

fssAuthFilePath := path.Join(w.GetAuthenticationManager().GetCredentialPath(lc.AgreementId), config.HZN_FSS_AUTH_FILE) // /var/horizon/ess-auth/<agreementId>/auth.json
fssCertFilePath := path.Join(w.config.GetESSSSLClientCertPath(), config.HZN_FSS_CERT_FILE) // /var/horizon/ess-auth/SSL/cert/cert.pem
err = client.Install(kd.OperatorYamlArchive, kd.Metadata, *(lc.EnvironmentAdditions), fssAuthFilePath, fssCertFilePath, secretsMap, lc.AgreementId, lc.Configure.ClusterNamespace, crInstallTimeout)
err = client.Install(kd.OperatorYamlArchive, kd.Metadata, kd.MMSPVC, *(lc.EnvironmentAdditions), fssAuthFilePath, fssCertFilePath, secretsMap, lc.AgreementId, lc.Configure.ClusterNamespace, crInstallTimeout)
if err != nil {
return err
}
Expand Down
1 change: 1 addition & 0 deletions persistence/kube_deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type KubeDeploymentConfig struct {
Metadata map[string]interface{} `json:"metadata,omitempty"`
OperatorYamlArchive string `json:"operatorYamlArchive"`
Secrets map[string]interface{} `json:"secrets,omitempty"`
MMSPVC map[string]interface{} `json:"mmspvc,omitempty"`
}

func (k *KubeDeploymentConfig) ToString() string {
Expand Down

0 comments on commit 4a994a5

Please sign in to comment.