From 8fa188fa31dc4b0ebb835fe1b256b7feac644b9d Mon Sep 17 00:00:00 2001 From: Sooyoung Kim Date: Mon, 26 Feb 2024 06:12:37 +0000 Subject: [PATCH] Update alibaba api calls as SDK and support for accessing internet from cluster fix #902, #1078 --- .../drivers/alibaba/AlibabaDriver.go | 84 + .../alibaba/connect/AlibabaCloudConnection.go | 10 +- .../alibaba/resources/ClusterHandler.go | 2276 +++++++++++++---- .../alibaba/utils/alibaba/alibaba_api.go | 389 --- .../drivers/alibaba/utils/alibaba_api_test.go | 547 ---- go.mod | 48 +- go.sum | 152 +- 7 files changed, 2021 insertions(+), 1485 deletions(-) delete mode 100644 cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba/alibaba_api.go delete mode 100644 cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba_api_test.go diff --git a/cloud-control-manager/cloud-driver/drivers/alibaba/AlibabaDriver.go b/cloud-control-manager/cloud-driver/drivers/alibaba/AlibabaDriver.go index a77789269..000123373 100644 --- a/cloud-control-manager/cloud-driver/drivers/alibaba/AlibabaDriver.go +++ b/cloud-control-manager/cloud-driver/drivers/alibaba/AlibabaDriver.go @@ -13,6 +13,12 @@ package alibaba import ( "fmt" + cs2015 "github.com/alibabacloud-go/cs-20151215/v4/client" + openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client" + ecs2014 "github.com/alibabacloud-go/ecs-20140526/v4/client" + "github.com/alibabacloud-go/tea/tea" + vpc2016 "github.com/alibabacloud-go/vpc-20160428/v6/client" + "github.com/aliyun/alibaba-cloud-sdk-go/services/bssopenapi" "github.com/aliyun/alibaba-cloud-sdk-go/services/slb" @@ -83,6 +89,21 @@ func (driver *AlibabaDriver) ConnectCloud(connectionInfo idrv.ConnectionInfo) (i return nil, err } + Vpc2016Client, err := getVpc2016Client(connectionInfo) + if err != nil { + return nil, err + } + + Cs2015Client, err := getCs2015Client(connectionInfo) + if err != nil { + return nil, err + } + + Ecs2014Client, err := getEcs2014Client(connectionInfo) + if err != nil { + return nil, err + } + iConn := alicon.AlibabaCloudConnection{ CredentialInfo: connectionInfo.CredentialInfo, Region: connectionInfo.RegionInfo, @@ -100,6 +121,9 @@ func (driver *AlibabaDriver) ConnectCloud(connectionInfo idrv.ConnectionInfo) (i DiskClient: ECSClient, MyImageClient: ECSClient, RegionZoneClient: ECSClient, + Vpc2016Client: Vpc2016Client, + Cs2015Client: Cs2015Client, + Ecs2014Client: Ecs2014Client, BssClient: BssClient, } return &iConn, nil @@ -280,3 +304,63 @@ func getBssClient(connectionInfo idrv.ConnectionInfo) (*bssopenapi.Client, error return bssClient, nil } + +func getVpc2016Client(connectionInfo idrv.ConnectionInfo) (*vpc2016.Client, error) { + + // Region Info + fmt.Println("AlibabaDriver : getVpc2016Client() - Region : [" + connectionInfo.RegionInfo.Region + "]") + + config := &openapi.Config{ + AccessKeyId: tea.String(connectionInfo.CredentialInfo.ClientId), + AccessKeySecret: tea.String(connectionInfo.CredentialInfo.ClientSecret), + RegionId: tea.String(connectionInfo.RegionInfo.Region), + } + + vpcClient, err := vpc2016.NewClient(config) + if err != nil { + fmt.Println("Could not create alibaba's vpc service client", err) + return nil, err + } + + return vpcClient, nil +} + +func getCs2015Client(connectionInfo idrv.ConnectionInfo) (*cs2015.Client, error) { + + // Region Info + fmt.Println("AlibabaDriver : getCs2015Client() - Region : [" + connectionInfo.RegionInfo.Region + "]") + + config := &openapi.Config{ + AccessKeyId: tea.String(connectionInfo.CredentialInfo.ClientId), + AccessKeySecret: tea.String(connectionInfo.CredentialInfo.ClientSecret), + RegionId: tea.String(connectionInfo.RegionInfo.Region), + } + + csClient, err := cs2015.NewClient(config) + if err != nil { + fmt.Println("Could not create alibaba's cs service client", err) + return nil, err + } + + return csClient, nil +} + +func getEcs2014Client(connectionInfo idrv.ConnectionInfo) (*ecs2014.Client, error) { + + // Region Info + fmt.Println("AlibabaDriver : getEcs2014Client() - Region : [" + connectionInfo.RegionInfo.Region + "]") + + config := &openapi.Config{ + AccessKeyId: tea.String(connectionInfo.CredentialInfo.ClientId), + AccessKeySecret: tea.String(connectionInfo.CredentialInfo.ClientSecret), + RegionId: tea.String(connectionInfo.RegionInfo.Region), + } + + ecsClient, err := ecs2014.NewClient(config) + if err != nil { + fmt.Println("Could not create alibaba's ecs service client", err) + return nil, err + } + + return ecsClient, nil +} diff --git a/cloud-control-manager/cloud-driver/drivers/alibaba/connect/AlibabaCloudConnection.go b/cloud-control-manager/cloud-driver/drivers/alibaba/connect/AlibabaCloudConnection.go index 58dbbce31..e7fdff749 100644 --- a/cloud-control-manager/cloud-driver/drivers/alibaba/connect/AlibabaCloudConnection.go +++ b/cloud-control-manager/cloud-driver/drivers/alibaba/connect/AlibabaCloudConnection.go @@ -11,6 +11,9 @@ package connect import ( + cs2015 "github.com/alibabacloud-go/cs-20151215/v4/client" + ecs2014 "github.com/alibabacloud-go/ecs-20140526/v4/client" + vpc2016 "github.com/alibabacloud-go/vpc-20160428/v6/client" bssopenapi "github.com/aliyun/alibaba-cloud-sdk-go/services/bssopenapi" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/aliyun/alibaba-cloud-sdk-go/services/slb" @@ -50,7 +53,10 @@ type AlibabaCloudConnection struct { MyImageClient *ecs.Client RegionZoneClient *ecs.Client - BssClient *bssopenapi.Client + Vpc2016Client *vpc2016.Client + Cs2015Client *cs2015.Client + Ecs2014Client *ecs2014.Client + BssClient *bssopenapi.Client } func (cloudConn *AlibabaCloudConnection) CreateRegionZoneHandler() (irs.RegionZoneHandler, error) { @@ -143,7 +149,7 @@ func (cloudConn *AlibabaCloudConnection) CreateClusterHandler() (irs.ClusterHand // temp // getEnv & Setting - clusterHandler := alirs.AlibabaClusterHandler{RegionInfo: cloudConn.Region, CredentialInfo: cloudConn.CredentialInfo} + clusterHandler := alirs.AlibabaClusterHandler{RegionInfo: cloudConn.Region, CredentialInfo: cloudConn.CredentialInfo, VpcClient: cloudConn.Vpc2016Client, CsClient: cloudConn.Cs2015Client, EcsClient: cloudConn.Ecs2014Client} return &clusterHandler, nil diff --git a/cloud-control-manager/cloud-driver/drivers/alibaba/resources/ClusterHandler.go b/cloud-control-manager/cloud-driver/drivers/alibaba/resources/ClusterHandler.go index 6747ee7a3..b6b448bb0 100644 --- a/cloud-control-manager/cloud-driver/drivers/alibaba/resources/ClusterHandler.go +++ b/cloud-control-manager/cloud-driver/drivers/alibaba/resources/ClusterHandler.go @@ -13,16 +13,21 @@ package resources import ( "encoding/json" "fmt" - "runtime/debug" "strconv" "strings" "time" call "github.com/cloud-barista/cb-spider/cloud-control-manager/cloud-driver/call-log" - "github.com/cloud-barista/cb-spider/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba" idrv "github.com/cloud-barista/cb-spider/cloud-control-manager/cloud-driver/interfaces" irs "github.com/cloud-barista/cb-spider/cloud-control-manager/cloud-driver/interfaces/resources" "github.com/jeremywohl/flatten" + + cs2015 "github.com/alibabacloud-go/cs-20151215/v4/client" + ecs2014 "github.com/alibabacloud-go/ecs-20140526/v4/client" + "github.com/alibabacloud-go/tea/tea" + vpc2016 "github.com/alibabacloud-go/vpc-20160428/v6/client" + + "github.com/Masterminds/semver/v3" ) // calllogger @@ -36,705 +41,1972 @@ import ( // }) // } +const ( + defaultClusterType = "ManagedKubernetes" + defaultClusterSpec = "ack.pro.small" + defaultClusterRuntimeName = "containerd" + + tagKeyAckAliyunCom = "ack.aliyun.com" + tagKeyCbSpiderPmksCluster = "CB-SPIDER:PMKS:CLUSTER" + tagValueOwned = "owned" + + clusterStateInitial = "initial" + clusterStateFailed = "failed" + clusterStateRunning = "running" + clusterStateUpdating = "updating" + clusterStateUpdatingFailed = "updating_failed" + clusterStateScaling = "scaling" + clusterStateWaiting = "waiting" + clusterStateDisconnected = "disconnected" + clusterStateStopped = "stopped" + clusterStateDeleting = "deleting" + clusterStateDeleted = "deleted" + clusterStateDeletedFailed = "deleted_failed" + + nodepoolStatusActive = "active" + nodepoolStatusScaling = "scaling" + nodepoolStatusRemoving = "removing" + nodepoolStatusDeleting = "deleting" + nodepoolStatusUpdating = "updating" +) + type AlibabaClusterHandler struct { RegionInfo idrv.RegionInfo CredentialInfo idrv.CredentialInfo + VpcClient *vpc2016.Client + CsClient *cs2015.Client + EcsClient *ecs2014.Client } -func (clusterHandler *AlibabaClusterHandler) CreateCluster(clusterReqInfo irs.ClusterInfo) (irs.ClusterInfo, error) { - cblogger.Info("Alibaba Cloud Driver: called CreateCluster()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, "CreateCluster()", "CreateCluster()") - +func (ach *AlibabaClusterHandler) CreateCluster(clusterReqInfo irs.ClusterInfo) (irs.ClusterInfo, error) { + cblogger.Debug("Alibaba Cloud Driver: called CreateCluster()") + emptyClusterInfo := irs.ClusterInfo{} + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, "CreateCluster()", "CreateCluster()") start := call.Start() - // 클러스터 생성 요청을 JSON 요청으로 변환 - payload, err := getClusterInfoJSON(clusterHandler, clusterReqInfo) + + cblogger.Info(fmt.Sprintf("Create Cluster")) + + // + // Validation + // + err := validateAtCreateCluster(clusterReqInfo) if err != nil { - err := fmt.Errorf("Failed to Get ClusterInfo JSON : %v", err) + err = fmt.Errorf("Failed to Create Cluster: %v", err) cblogger.Error(err) - return irs.ClusterInfo{}, err + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err } - response_json_str, err := alibaba.CreateCluster(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, payload) - callLogInfo.ElapsedTime = call.Elapsed(start) + + regionId := ach.RegionInfo.Region + vpcId := clusterReqInfo.Network.VpcIID.SystemId + + // + // Get all VSwitches in VPC + // + vswitches, err := aliDescribeVSwitches(ach.VpcClient, regionId, vpcId) if err != nil { - err := fmt.Errorf("Failed to Create Cluster : %v", err) + err = fmt.Errorf("Failed to Create Cluster: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) - return irs.ClusterInfo{}, err - } - calllogger.Info(call.String(callLogInfo)) - - // NodeGroup 생성 정보가 있는경우 생성을 시도한다. - // 현재는 생성 시도를 안한다. 생성하기로 결정되면 아래 주석을 풀어서 사용한다. - // 이유: - // - Cluster 생성이 완료되어야 NodeGroup 생성이 가능하다. - // - Cluster 생성이 완료되려면 최소 10분 이상 걸린다. - // - 성공할때까지 대기한 후에 생성을 시도해야 한다. - // for _, node_group := range clusterReqInfo.NodeGroupList { - // node_group_info, err := clusterHandler.AddNodeGroup(clusterReqInfo.IId, node_group) - // if err != nil { - // cblogger.Error(err) - // return irs.ClusterInfo{}, err - // } - // } - var response_json_obj map[string]interface{} - json.Unmarshal([]byte(response_json_str), &response_json_obj) - cluster_id := response_json_obj["cluster_id"].(string) - cluster_info, err := getClusterInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, cluster_id) - if err != nil { - err := fmt.Errorf("Failed to Get Cluster Info : %v", err) + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err + } + + if len(vswitches) == 0 { + err = fmt.Errorf("No VSwitch in VPC(ID=%s)", vpcId) + err = fmt.Errorf("Failed to Create Cluster: %v", err) cblogger.Error(err) - return irs.ClusterInfo{}, err + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err } - return *cluster_info, nil -} + var vswitchIds []string + for _, vs := range vswitches { + vswitchIds = append(vswitchIds, tea.StringValue(vs.VSwitchId)) + } + cblogger.Debug(fmt.Sprintf("VSwiches in VPC(%s): %v", vpcId, vswitchIds)) + + cidrList, err := ach.getAvailableCidrList() + if err != nil { + err = fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err + } + + if len(cidrList) < 2 { + err = fmt.Errorf("insufficient CIDRs") + err = fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err + + } + cblogger.Debug(fmt.Sprintf("Available CIDR: ContainerCidr(%s), ServiceCidr(%s)", cidrList[0], cidrList[1])) + + // + // Create a Cluster + // + clusterName := clusterReqInfo.IId.NameId + k8sVersion := clusterReqInfo.Version + clusterType := defaultClusterType + clusterSpec := defaultClusterSpec + containerCidr := cidrList[0] + serviceCidr := cidrList[1] + secGroupId := clusterReqInfo.Network.SecurityGroupIIDs[0].SystemId + snatEntry := true + epPublicAccess := true + + runtimeName, runtimeVersion, err := getLatestRuntime(ach.CsClient, regionId, clusterType, k8sVersion) + if err != nil { + err := fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err + } + cblogger.Debug(fmt.Sprintf("Selected Runtime (Name=%s, Version=%s)", runtimeName, runtimeVersion)) + + nodepools := getNodepoolsFromNodeGroupList(clusterReqInfo.NodeGroupList, runtimeName, runtimeVersion, vswitchIds) + + clusterId, err := aliCreateCluster(ach.CsClient, clusterName, regionId, clusterType, clusterSpec, k8sVersion, runtimeName, runtimeVersion, vpcId, containerCidr, serviceCidr, secGroupId, snatEntry, epPublicAccess, vswitchIds, tagKeyCbSpiderPmksCluster, tagValueOwned, nodepools) + if err != nil { + err := fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err + } + cblogger.Debug(fmt.Sprintf("Request to Create Cluster is In Progress.")) + + var createErr error = nil + defer func() { + if createErr != nil { + cleanCluster(ach.CsClient, tea.StringValue(clusterId)) + cblogger.Info(fmt.Sprintf("Cluster(%s) will be Deleted.", clusterName)) + } + }() + + // + // Get ClusterInfo + // + clusterInfo, err := ach.getClusterInfo(regionId, tea.StringValue(clusterId)) + if err != nil { + createErr = fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(createErr) + LoggingError(hiscallInfo, createErr) + return emptyClusterInfo, createErr + } -func (clusterHandler *AlibabaClusterHandler) ListCluster() ([]*irs.ClusterInfo, error) { - cblogger.Info("Alibaba Cloud Driver: called ListCluster()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, "ListCluster()", "ListCluster()") + LoggingInfo(hiscallInfo, start) + + cblogger.Info(fmt.Sprintf("Creating Cluster(Name=%s, ID=%s).", clusterInfo.IId.NameId, clusterInfo.IId.SystemId)) + + return *clusterInfo, nil +} +func (ach *AlibabaClusterHandler) ListCluster() ([]*irs.ClusterInfo, error) { + cblogger.Debug("Alibaba Cloud Driver: called ListCluster()") + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, "ListCluster()", "ListCluster()") start := call.Start() - clusters_json_str, err := alibaba.GetClusters(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region) - callLogInfo.ElapsedTime = call.Elapsed(start) + + cblogger.Info(fmt.Sprintf("Get Cluster List")) + + // + // Get Cluster List + // + regionId := ach.RegionInfo.Region + clusters, err := aliDescribeClustersV1(ach.CsClient, regionId) if err != nil { - err := fmt.Errorf("Failed to Get Clusters : %v", err) + err := fmt.Errorf("Failed to List Cluster: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) + LoggingError(hiscallInfo, err) return nil, err } - calllogger.Info(call.String(callLogInfo)) - var clusters_json_obj map[string]interface{} - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - clusters := clusters_json_obj["clusters"].([]interface{}) - cluster_info_list := make([]*irs.ClusterInfo, len(clusters)) - for i, cluster := range clusters { - println(i, cluster) - cluster_id := cluster.(map[string]interface{})["cluster_id"].(string) - cluster_info_list[i], err = getClusterInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, cluster_id) + // + // Get ClusterInfo List + // + var clusterInfoList []*irs.ClusterInfo + for _, cluster := range clusters { + clusterInfo, err := ach.getClusterInfo(regionId, tea.StringValue(cluster.ClusterId)) if err != nil { - err := fmt.Errorf("Failed to Get ClusterInfo : %v", err) + err := fmt.Errorf("Failed to List Cluster: %v", err) cblogger.Error(err) + LoggingError(hiscallInfo, err) return nil, err } + + clusterInfoList = append(clusterInfoList, clusterInfo) } - return cluster_info_list, nil -} + LoggingInfo(hiscallInfo, start) -func (clusterHandler *AlibabaClusterHandler) GetCluster(clusterIID irs.IID) (irs.ClusterInfo, error) { - cblogger.Info("Alibaba Cloud Driver: called GetCluster()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "GetCluster()") + return clusterInfoList, nil +} +func (ach *AlibabaClusterHandler) GetCluster(clusterIID irs.IID) (irs.ClusterInfo, error) { + cblogger.Debug("Alibaba Cloud Driver: called GetCluster()") + emptyClusterInfo := irs.ClusterInfo{} + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "GetCluster()") start := call.Start() - cluster_info, err := getClusterInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId) - callLogInfo.ElapsedTime = call.Elapsed(start) + + // + // Get ClusterInfo + // + regionId := ach.RegionInfo.Region + clusterInfo, err := ach.getClusterInfo(regionId, clusterIID.SystemId) if err != nil { - err := fmt.Errorf("Failed to Get ClusterInfo : %v", err) + err := fmt.Errorf("Failed to Get Cluster: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) - return irs.ClusterInfo{}, err + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err } - calllogger.Info(call.String(callLogInfo)) - return *cluster_info, nil -} + LoggingInfo(hiscallInfo, start) -func (clusterHandler *AlibabaClusterHandler) DeleteCluster(clusterIID irs.IID) (bool, error) { - cblogger.Info("Alibaba Cloud Driver: called DeleteCluster()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "DeleteCluster()") + cblogger.Info(fmt.Sprintf("Get Cluster(Name=%s, ID=%s)", clusterInfo.IId.NameId, clusterInfo.IId.SystemId)) + return *clusterInfo, nil +} + +func (ach *AlibabaClusterHandler) DeleteCluster(clusterIID irs.IID) (bool, error) { + cblogger.Debug("Alibaba Cloud Driver: called DeleteCluster()") + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "DeleteCluster()") start := call.Start() - res, err := alibaba.DeleteCluster(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId) - callLogInfo.ElapsedTime = call.Elapsed(start) + + // + // Get Cluster Detailed Information + // + cluster, err := aliDescribeClusterDetail(ach.CsClient, clusterIID.SystemId) + if err != nil { + err := fmt.Errorf("Failed to Delete Cluster: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return false, err + } + + // + // Check if there is a nat gateway automatically created with the cluster + // + cblogger.Debug(fmt.Sprintf("Check if NAT Gatway Automatically Created with Cluster(%s)", clusterIID.NameId)) + + regionId := ach.RegionInfo.Region + vpcId := tea.StringValue(cluster.VpcId) + tagKey := tagKeyAckAliyunCom + tagValue := clusterIID.SystemId + ngwsRetaining, err := getInternetNatGatewaysWithTagInVpc(ach.VpcClient, regionId, vpcId, tagKey, tagValue) + if err != nil { + err = fmt.Errorf("Failed to Delete Cluster: %v", err) + cblogger.Error(err) + hiscallInfo.ErrorMSG = err.Error() + calllogger.Error(call.String(hiscallInfo)) + return false, err + } + + var retainResources []string + if len(ngwsRetaining) > 0 { + retainResources = append(retainResources, tea.StringValue(ngwsRetaining[0].NatGatewayId)) + } + if len(retainResources) > 0 { + cblogger.Debug(fmt.Sprintf("The NAT Gateway(%v) is retained.", retainResources)) + } + + // + // Delete a Cluster without NAT Gateway + // + _, err = aliDeleteCluster(ach.CsClient, tea.StringValue(cluster.ClusterId), retainResources) + if err != nil { + err := fmt.Errorf("Failed to Delete Cluster: %v", err) + cblogger.Error(err) + hiscallInfo.ErrorMSG = err.Error() + calllogger.Error(call.String(hiscallInfo)) + return false, err + } + cblogger.Debug(fmt.Sprintf("Request to Delete Cluster is In Progress.")) + + // + // Cleanup NAT Gateway if there is no more cluster created by CB-SPIDER + // + cblogger.Debug(fmt.Sprintf("Check if Cluster Created By CB-SPIDER Exists.")) + + exist, err := existNotDeletedClusterWithTagInVpc(ach.CsClient, tea.StringValue(cluster.RegionId), tea.StringValue(cluster.VpcId), tagKeyCbSpiderPmksCluster, tagValueOwned) if err != nil { - err := fmt.Errorf("Failed to Delete Cluster : %v", err) + err := fmt.Errorf("Failed to Delete Cluster: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) + LoggingError(hiscallInfo, err) return false, err } - cblogger.Info(res) - calllogger.Info(call.String(callLogInfo)) + if exist == false { + cblogger.Debug(fmt.Sprintf("No More Cluster Created By CB-SPIDER.")) + + tagKey = tagKeyCbSpiderPmksCluster + tagValue = tagValueOwned + ngwsRetained, err := getInternetNatGatewaysWithTagInVpc(ach.VpcClient, regionId, vpcId, tagKey, tagValue) + if err != nil { + err = fmt.Errorf("Failed to Delete Cluster: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return false, err + } + + for _, ngw := range ngwsRetained { + aliDeleteNatGateway(ach.VpcClient, tea.StringValue(cluster.RegionId), tea.StringValue(ngw.NatGatewayId)) + cblogger.Info(fmt.Sprintf("Internet NAT Gateway(%s) will be deleted", tea.StringValue(ngw.NatGatewayId))) + } + } else { + cblogger.Debug(fmt.Sprintf("Cluster Created By CB-SPIDER Exists.")) + } + + LoggingInfo(hiscallInfo, start) + + cblogger.Info(fmt.Sprintf("Deleting Cluster(Name=%s, ID=%s).", clusterIID.NameId, clusterIID.SystemId)) return true, nil } -func (clusterHandler *AlibabaClusterHandler) AddNodeGroup(clusterIID irs.IID, nodeGroupReqInfo irs.NodeGroupInfo) (irs.NodeGroupInfo, error) { - cblogger.Info("Alibaba Cloud Driver: called AddNodeGroup()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "AddNodeGroup()") - +func (ach *AlibabaClusterHandler) AddNodeGroup(clusterIID irs.IID, nodeGroupReqInfo irs.NodeGroupInfo) (irs.NodeGroupInfo, error) { + cblogger.Debug("Alibaba Cloud Driver: called AddNodeGroup()") + emptyNodeGroupInfo := irs.NodeGroupInfo{} + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "AddNodeGroup()") start := call.Start() - // 노드 그룹 생성 요청을 JSON 요청으로 변환 - payload, err := getNodeGroupJSONString(clusterHandler, clusterIID, nodeGroupReqInfo) + + cblogger.Info(fmt.Sprintf("Add NodeGroup")) + + // + // Validation + // + err := validateAtAddNodeGroup(clusterIID, nodeGroupReqInfo) if err != nil { - err := fmt.Errorf("Failed to Get NodeGroup JSON String : %v", err) + err = fmt.Errorf("Failed to Add Node GRoup: %v", err) cblogger.Error(err) - return irs.NodeGroupInfo{}, err + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err } - result_json_str, err := alibaba.CreateNodeGroup(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, payload) - callLogInfo.ElapsedTime = call.Elapsed(start) + // + // Get Cluster Detailed Information + // + regionId := ach.RegionInfo.Region + clusterId := clusterIID.SystemId + + cluster, err := aliDescribeClusterDetail(ach.CsClient, clusterId) if err != nil { - err := fmt.Errorf("Failed to Create NodeGroup : %v", err) + err = fmt.Errorf("Failed to Add NodeGroup: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) - return irs.NodeGroupInfo{}, err + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err } - calllogger.Info(call.String(callLogInfo)) - var result_json_obj map[string]interface{} - json.Unmarshal([]byte(result_json_str), &result_json_obj) - nodepool_id := result_json_obj["nodepool_id"].(string) - node_group_info, err := getNodeGroupInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, nodepool_id) + // + // Get all VSwitches in VPC + // + vswitches, err := aliDescribeVSwitches(ach.VpcClient, regionId, tea.StringValue(cluster.VpcId)) if err != nil { - err := fmt.Errorf("Failed to Get NodeGroupInfo : %v", err) + err = fmt.Errorf("Failed to Add NodeGroup: %v", err) cblogger.Error(err) - return irs.NodeGroupInfo{}, err - } - - return *node_group_info, nil -} - -// func (clusterHandler *AlibabaClusterHandler) ListNodeGroup(clusterIID irs.IID) ([]*irs.NodeGroupInfo, error) { -// cblogger.Info("Alibaba Cloud Driver: called ListNodeGroup()") -// callLogInfo := getCallLogScheme(clusterHandler.RegionInfo.Region, call.CLUSTER, clusterIID.NameId, "ListNodeGroup()") - -// start := call.Start() -// node_group_info_list := []*irs.NodeGroupInfo{} -// node_groups_json_str, err := alibaba.ListNodeGroup(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId) -// callLogInfo.ElapsedTime = call.Elapsed(start) -// if err != nil { -// err := fmt.Errorf("Failed to List NodeGroup : %v", err) -// cblogger.Error(err) -// callLogInfo.ErrorMSG = err.Error() -// calllogger.Error(call.String(callLogInfo)) -// return node_group_info_list, err -// } -// calllogger.Info(call.String(callLogInfo)) - -// var node_groups_json_obj map[string]interface{} -// json.Unmarshal([]byte(node_groups_json_str), &node_groups_json_obj) -// node_groups := node_groups_json_obj["nodepools"].([]interface{}) -// for _, node_group := range node_groups { -// node_group_id := node_group.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) -// node_group_info, err := getNodeGroupInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, node_group_id) -// if err != nil { -// err := fmt.Errorf("Failed to Get NodeGroupInfo : %v", err) -// cblogger.Error(err) -// return nil, err -// } -// node_group_info_list = append(node_group_info_list, node_group_info) -// } - -// return node_group_info_list, nil -// } + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err + } -// func (clusterHandler *AlibabaClusterHandler) GetNodeGroup(clusterIID irs.IID, nodeGroupIID irs.IID) (irs.NodeGroupInfo, error) { -// cblogger.Info("Alibaba Cloud Driver: called GetNodeGroup()") -// callLogInfo := getCallLogScheme(clusterHandler.RegionInfo.Region, call.CLUSTER, clusterIID.NameId, "GetNodeGroup()") - -// start := call.Start() -// temp, err := getNodeGroupInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, nodeGroupIID.SystemId) -// callLogInfo.ElapsedTime = call.Elapsed(start) -// if err != nil { -// err := fmt.Errorf("Failed to Get NodeGroupInfo : %v", err) -// cblogger.Error(err) -// callLogInfo.ErrorMSG = err.Error() -// calllogger.Error(call.String(callLogInfo)) -// return irs.NodeGroupInfo{}, err -// } -// calllogger.Info(call.String(callLogInfo)) - -// return *temp, nil -// } + if len(vswitches) == 0 { + err = fmt.Errorf("No VSwitch in VPC(ID=%s)", tea.StringValue(cluster.VpcId)) + err = fmt.Errorf("Failed to Add NodeGroup: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err + } + + var vswitchIds []string + for _, vs := range vswitches { + vswitchIds = append(vswitchIds, tea.StringValue(vs.VSwitchId)) + } + cblogger.Debug(fmt.Sprintf("VSwiches in VPC(%s): %v", tea.StringValue(cluster.VpcId), vswitchIds)) + + // + // Create Node Groups + // + name := nodeGroupReqInfo.IId.NameId + autoScalingEnable := nodeGroupReqInfo.OnAutoScaling + maxInstances := int64(nodeGroupReqInfo.MaxNodeSize) + minInstances := int64(nodeGroupReqInfo.MinNodeSize) + instanceTypes := []string{nodeGroupReqInfo.VMSpecName} + systemDiskCategory := nodeGroupReqInfo.RootDiskType + systemDiskSize, _ := strconv.ParseInt(nodeGroupReqInfo.RootDiskSize, 10, 64) + keyPair := nodeGroupReqInfo.KeyPairIID.NameId + desiredSize := int64(nodeGroupReqInfo.DesiredNodeSize) + + nodepoolId, err := aliCreateClusterNodePool(ach.CsClient, clusterId, name, + autoScalingEnable, maxInstances, minInstances, vswitchIds, instanceTypes, systemDiskCategory, systemDiskSize, keyPair, desiredSize) + if err != nil { + err = fmt.Errorf("Failed to Add NodeGroup: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err + } + cblogger.Debug(fmt.Sprintf("Request to Create NodePool is In Progress.")) + + // + // Get NodeGroupInfo + // + nodeGroupInfo, err := ach.getNodeGroupInfo(clusterId, tea.StringValue(nodepoolId)) + if err != nil { + err = fmt.Errorf("Failed to Add NodeGroup: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err + } + + LoggingInfo(hiscallInfo, start) + + cblogger.Info(fmt.Sprintf("Adding NodeGroup(Name=%s, ID=%s) to Cluster(%s).", nodeGroupInfo.IId.NameId, nodeGroupInfo.IId.SystemId, clusterId)) -func (clusterHandler *AlibabaClusterHandler) SetNodeGroupAutoScaling(clusterIID irs.IID, nodeGroupIID irs.IID, on bool) (bool, error) { - cblogger.Info("Alibaba Cloud Driver: called SetNodeGroupAutoScaling()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "SetNodeGroupAutoScaling()") + return *nodeGroupInfo, nil +} +func (ach *AlibabaClusterHandler) SetNodeGroupAutoScaling(clusterIID irs.IID, nodeGroupIID irs.IID, on bool) (bool, error) { + cblogger.Debug("Alibaba Cloud Driver: called SetNodeGroupAutoScaling()") + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "SetNodeGroupAutoScaling()") start := call.Start() - temp := `{"auto_scaling":{"enable":%t}}` - body := fmt.Sprintf(temp, on) - res, err := alibaba.ModifyNodeGroup(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, nodeGroupIID.SystemId, body) - callLogInfo.ElapsedTime = call.Elapsed(start) + + cblogger.Info(fmt.Sprintf("Set NodeGroup AutoScaling")) + + // + // Set NodeGroup AutoScaling + // + clusterId := clusterIID.SystemId + ngId := nodeGroupIID.SystemId + _, err := aliModifyClusterNodePoolAutoScalingEnable(ach.CsClient, clusterId, ngId, on) if err != nil { - err := fmt.Errorf("Failed to Modify NodeGroup : %v", err) + err := fmt.Errorf("Failed to Set NodeGroup AutoScaling: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) + LoggingError(hiscallInfo, err) return false, err } - cblogger.Info(res) - calllogger.Info(call.String(callLogInfo)) + + LoggingInfo(hiscallInfo, start) + + cblogger.Info(fmt.Sprintf("Modifying AutoScaling of NodeGroup(Name=%s, ID=%s) in Cluster(%s).", nodeGroupIID.NameId, nodeGroupIID.SystemId, clusterIID.NameId)) return true, nil } -func (clusterHandler *AlibabaClusterHandler) ChangeNodeGroupScaling(clusterIID irs.IID, nodeGroupIID irs.IID, desiredNodeSize int, minNodeSize int, maxNodeSize int) (irs.NodeGroupInfo, error) { - cblogger.Info("Alibaba Cloud Driver: called ChangeNodeGroupScaling()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "ChangeNodeGroupScaling()") - +func (ach *AlibabaClusterHandler) ChangeNodeGroupScaling(clusterIID irs.IID, nodeGroupIID irs.IID, desiredNodeSize int, minNodeSize int, maxNodeSize int) (irs.NodeGroupInfo, error) { + cblogger.Debug("Alibaba Cloud Driver: called ChangeNodeGroupScaling()") + emptyNodeGroupInfo := irs.NodeGroupInfo{} + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "ChangeNodeGroupScaling()") start := call.Start() - // desired_size is not supported in alibaba with auto scaling mode - temp := `{"auto_scaling":{"max_instances":%d,"min_instances":%d}}` - body := fmt.Sprintf(temp, maxNodeSize, minNodeSize) - res, err := alibaba.ModifyNodeGroup(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, nodeGroupIID.SystemId, body) - callLogInfo.ElapsedTime = call.Elapsed(start) + + cblogger.Info(fmt.Sprintf("Change NodeGroup Scaling")) + + // + // Validation + // + err := validateAtChangeNodeGroupScaling(clusterIID, nodeGroupIID, minNodeSize, maxNodeSize) if err != nil { - err := fmt.Errorf("Failed to Modify NodeGroup : %v", err) + err = fmt.Errorf("Failed to Change Node Group Scaling: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) - return irs.NodeGroupInfo{}, err + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err } - cblogger.Info(res) - calllogger.Info(call.String(callLogInfo)) - node_group_info, err := getNodeGroupInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, nodeGroupIID.SystemId) + // + // Change NodeGroup's Scaling Size + // + clusterId := clusterIID.SystemId + ngId := nodeGroupIID.SystemId + autoScalingEnable := true + + // CAUTION: desiredNodeSize cannot be applied in alibaba with auto scaling mode + _, err = aliModifyClusterNodePoolScalingSize(ach.CsClient, clusterId, ngId, autoScalingEnable, int64(maxNodeSize), int64(minNodeSize), int64(desiredNodeSize)) if err != nil { - err := fmt.Errorf("Failed to Get NodeGroupInfo : %v", err) + err = fmt.Errorf("Failed to Change NodeGroup Scaling: %v", err) cblogger.Error(err) - return irs.NodeGroupInfo{}, err + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err } - return *node_group_info, nil -} + // + // Get NodeGroupInfo + // + nodeGroupInfo, err := ach.getNodeGroupInfo(clusterId, ngId) + if err != nil { + err = fmt.Errorf("Failed to Change NodeGroup Scaling: %v", err) + cblogger.Error(err) + LoggingError(hiscallInfo, err) + return emptyNodeGroupInfo, err + } + + LoggingInfo(hiscallInfo, start) -func (clusterHandler *AlibabaClusterHandler) RemoveNodeGroup(clusterIID irs.IID, nodeGroupIID irs.IID) (bool, error) { - cblogger.Info("Alibaba Cloud Driver: called RemoveNodeGroup()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "RemoveNodeGroup()") + cblogger.Info(fmt.Sprintf("Modifying Scaling of NodeGroup(Name=%s, ID=%s) in Cluster(%s).", nodeGroupInfo.IId.NameId, nodeGroupInfo.IId.SystemId, clusterIID.NameId)) + + return *nodeGroupInfo, nil +} +func (ach *AlibabaClusterHandler) RemoveNodeGroup(clusterIID irs.IID, nodeGroupIID irs.IID) (bool, error) { + cblogger.Debug("Alibaba Cloud Driver: called RemoveNodeGroup()") + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "RemoveNodeGroup()") start := call.Start() - res, err := alibaba.DeleteNodeGroup(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, nodeGroupIID.SystemId) - callLogInfo.ElapsedTime = call.Elapsed(start) + + cblogger.Info(fmt.Sprintf("Remove NodeGroup")) + + // + // Remove NodeGroup + // + clusterId := clusterIID.SystemId + ngId := nodeGroupIID.SystemId + + _, err := aliDeleteClusterNodepool(ach.CsClient, clusterId, ngId, true) + hiscallInfo.ElapsedTime = call.Elapsed(start) if err != nil { - err := fmt.Errorf("Failed to Delete NodeGroup : %v", err) + err := fmt.Errorf("Failed to Remove NodeGroup: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) + LoggingError(hiscallInfo, err) return false, err } - cblogger.Info(res) - calllogger.Info(call.String(callLogInfo)) + + LoggingInfo(hiscallInfo, start) + + cblogger.Info(fmt.Sprintf("Removing NodeGroup(Name=%s, ID=%s) to Cluster(%s).", nodeGroupIID.NameId, nodeGroupIID.SystemId, clusterIID.NameId)) return true, nil } -func (clusterHandler *AlibabaClusterHandler) UpgradeCluster(clusterIID irs.IID, newVersion string) (irs.ClusterInfo, error) { - cblogger.Info("Alibaba Cloud Driver: called UpgradeCluster()") - callLogInfo := GetCallLogScheme(clusterHandler.RegionInfo, call.CLUSTER, clusterIID.NameId, "UpgradeCluster()") - +func (ach *AlibabaClusterHandler) UpgradeCluster(clusterIID irs.IID, newVersion string) (irs.ClusterInfo, error) { + cblogger.Debug("Alibaba Cloud Driver: called UpgradeCluster()") + emptyClusterInfo := irs.ClusterInfo{} + hiscallInfo := GetCallLogScheme(ach.RegionInfo, call.CLUSTER, clusterIID.NameId, "UpgradeCluster()") start := call.Start() - temp := `{"next_version" : "%s"}` - body := fmt.Sprintf(temp, newVersion) - res, err := alibaba.UpgradeCluster(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId, body) - callLogInfo.ElapsedTime = call.Elapsed(start) + + cblogger.Info(fmt.Sprintf("Upgrade Cluster")) + + // + // Upgrade Cluster + // + regionId := ach.RegionInfo.Region + clusterId := clusterIID.SystemId + + _, err := aliUpgradeCluster(ach.CsClient, clusterId, newVersion) if err != nil { - err := fmt.Errorf("Failed to Upgrade Cluster : %v", err) + err := fmt.Errorf("Failed to Upgrade Cluster: %v", err) cblogger.Error(err) - callLogInfo.ErrorMSG = err.Error() - calllogger.Error(call.String(callLogInfo)) - return irs.ClusterInfo{}, err + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err } - cblogger.Info(res) - calllogger.Info(call.String(callLogInfo)) - clusterInfo, err := getClusterInfo(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterIID.SystemId) + // + // Get ClusterInfo + // + clusterInfo, err := ach.getClusterInfo(regionId, clusterId) if err != nil { - err := fmt.Errorf("Failed to Get ClusterInfo : %v", err) + err = fmt.Errorf("Failed to Upgrade Cluster: %v", err) cblogger.Error(err) - return irs.ClusterInfo{}, err + LoggingError(hiscallInfo, err) + return emptyClusterInfo, err } - return *clusterInfo, nil -} + LoggingInfo(hiscallInfo, start) -func getClusterInfo(access_key string, access_secret string, region_id string, cluster_id string) (clusterInfo *irs.ClusterInfo, err error) { + cblogger.Info(fmt.Sprintf("Upgrading Cluster(Name=%s, ID=%s)", clusterInfo.IId.NameId, clusterInfo.IId.SystemId)) - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Failed to Process getClusterInfo() : %v\n\n%v", r, string(debug.Stack())) - cblogger.Error(err) - } - }() + return *clusterInfo, nil +} - cluster_json_str, err := alibaba.GetCluster(access_key, access_secret, region_id, cluster_id) +func (ach *AlibabaClusterHandler) getClusterInfoListWithoutNodeGroupList(regionId string) ([]*irs.ClusterInfo, error) { + clusters, err := aliDescribeClustersV1(ach.CsClient, regionId) if err != nil { + err = fmt.Errorf("failed to get ClusterInfoList: %v", err) return nil, err } - var cluster_json_obj map[string]interface{} - json.Unmarshal([]byte(cluster_json_str), &cluster_json_obj) + var clusterInfoList []*irs.ClusterInfo + for _, cluster := range clusters { + clusterInfo, err := ach.getClusterInfoWithoutNodeGroupList(regionId, tea.StringValue(cluster.ClusterId)) + if err != nil { + err = fmt.Errorf("failed to get ClusterInfoList: %v", err) + return nil, err + } - // https://www.alibabacloud.com/help/doc-detail/86987.html - // Initializing Creating the cloud resources that are used by the cluster. - // Creation Failed Failed to create the cloud resources that are used by the cluster. - // Running The cloud resources used by the cluster are created. - // Updating Updating the metadata of the cluster. - // Scaling Adding nodes to the cluster. - // Removing Removing nodes from the cluster. - // Upgrading Upgrading the cluster. - // Draining Evicting pods from a node to other nodes. After all pods are evicted from the node, the node becomes unschudulable. - // Deleting Deleting the cluster. - // Deletion Failed Failed to delete the cluster. - // Deleted (invisible to users) The cluster is deleted." - health_status := cluster_json_obj["state"].(string) - cluster_status := irs.ClusterInactive - if strings.EqualFold(health_status, "Initializing") || strings.EqualFold(health_status, "initial") { - cluster_status = irs.ClusterCreating - } else if strings.EqualFold(health_status, "Updating") { - cluster_status = irs.ClusterUpdating - } else if strings.EqualFold(health_status, "Creation Failed") { - cluster_status = irs.ClusterInactive - } else if strings.EqualFold(health_status, "Deleting") { - cluster_status = irs.ClusterDeleting - } else if strings.EqualFold(health_status, "Running") { - cluster_status = irs.ClusterActive + clusterInfoList = append(clusterInfoList, clusterInfo) } - created_at := cluster_json_obj["created"].(string) // 2022-09-08T09:02:16+08:00, - datetime, err := time.Parse(time.RFC3339, created_at) + return clusterInfoList, nil +} + +func (ach *AlibabaClusterHandler) getClusterInfoWithoutNodeGroupList(regionId, clusterId string) (*irs.ClusterInfo, error) { + // + // Fill clusterInfo + // + cluster, err := aliDescribeClusterDetail(ach.CsClient, clusterId) if err != nil { - err := fmt.Errorf("Failed to Parse Created Time : %v", err) - cblogger.Error(err) - panic(err) + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err + } + + clusterStatus := irs.ClusterInactive + if strings.EqualFold(tea.StringValue(cluster.State), clusterStateInitial) { + clusterStatus = irs.ClusterCreating + } else if strings.EqualFold(tea.StringValue(cluster.State), clusterStateUpdating) { + clusterStatus = irs.ClusterUpdating + } else if strings.EqualFold(tea.StringValue(cluster.State), clusterStateFailed) { + clusterStatus = irs.ClusterInactive + } else if strings.EqualFold(tea.StringValue(cluster.State), clusterStateDeleting) { + clusterStatus = irs.ClusterDeleting + } else if strings.EqualFold(tea.StringValue(cluster.State), clusterStateRunning) { + clusterStatus = irs.ClusterActive } - //"{\"api_server_endpoint\":\"https://47.74.22.109:6443\",\"dashboard_endpoint\":\"\",\"intranet_api_server_endpoint\":\"https://10.0.11.77:6443\"}" + createdTime, err := time.Parse(time.RFC3339, tea.StringValue(cluster.Created)) // 2022-09-08T09:02:16+08:00, + + // "{\"api_server_endpoint\":\"https://111.222.333.444:6443\",\"intranet_api_server_endpoint\":\"https://10.2.1.1:6443\"}" // if api_server_endpoint is not exist, it'll throw error - master_url_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(cluster_json_obj["master_url"].(string)), &master_url_json_obj) - end_point := "Endpoint is not ready yet!" - if master_url_json_obj["api_server_endpoint"] != nil { - end_point = master_url_json_obj["api_server_endpoint"].(string) + endpoint := "Endpoint is not ready yet!" + if cluster.MasterUrl != nil && !strings.EqualFold(tea.StringValue(cluster.MasterUrl), "") { + jsonMasterUrl := make(map[string]interface{}) + err = json.Unmarshal([]byte(tea.StringValue(cluster.MasterUrl)), &jsonMasterUrl) + if err != nil { + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err + } + + if jsonMasterUrl["api_server_endpoint"] != nil { + endpoint = jsonMasterUrl["api_server_endpoint"].(string) + } } - // get kubeconfig - kube_config := "Kubeconfig is not ready yet!" - cluster_kube_config_json_str, err := alibaba.GetClusterKubeConfig(access_key, access_secret, region_id, cluster_id) + kubeconfig := "Kubeconfig is not ready yet!" + userKubeconfig, err := aliDescribeClusterUserKubeconfig(ach.CsClient, clusterId) if err != nil { cblogger.Info("Kubeconfig is not ready yet!") - //return nil, err } else { - cluster_kube_config_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(cluster_kube_config_json_str), &cluster_kube_config_json_obj) - // println(cluster_kube_config_json_obj["config"].(string)) - kube_config = strings.TrimSpace(cluster_kube_config_json_obj["config"].(string)) + kubeconfig = strings.TrimSpace(tea.StringValue(userKubeconfig.Config)) + } + + vpcAttr, err := aliDescribeVpcAttribute(ach.VpcClient, regionId, tea.StringValue(cluster.VpcId)) + if err != nil { + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err + } + + vswitchAttr, err := aliDescribeVSwitchAttributes(ach.VpcClient, regionId, tea.StringValue(cluster.VswitchId)) + if err != nil { + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err + } + + secGroupAttr, err := aliDescribeSecurityGroupAttribute(ach.EcsClient, regionId, tea.StringValue(cluster.SecurityGroupId)) + if err != nil { + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err } - clusterInfo = &irs.ClusterInfo{ + clusterInfo := &irs.ClusterInfo{ IId: irs.IID{ - NameId: cluster_json_obj["name"].(string), - SystemId: cluster_json_obj["cluster_id"].(string), + NameId: tea.StringValue(cluster.Name), + SystemId: tea.StringValue(cluster.ClusterId), }, - Version: cluster_json_obj["current_version"].(string), + Version: tea.StringValue(cluster.CurrentVersion), Network: irs.NetworkInfo{ VpcIID: irs.IID{ - NameId: "", - SystemId: cluster_json_obj["vpc_id"].(string), + NameId: tea.StringValue(vpcAttr.VpcName), + SystemId: tea.StringValue(cluster.VpcId), }, SubnetIIDs: []irs.IID{ { - NameId: "", - SystemId: cluster_json_obj["vswitch_id"].(string), + NameId: tea.StringValue(vswitchAttr.VSwitchName), + SystemId: tea.StringValue(cluster.VswitchId), }, }, SecurityGroupIIDs: []irs.IID{ { - NameId: "", - SystemId: cluster_json_obj["security_group_id"].(string), + NameId: tea.StringValue(secGroupAttr.SecurityGroupName), + SystemId: tea.StringValue(cluster.SecurityGroupId), }, }, }, - Status: cluster_status, - CreatedTime: datetime, + Status: clusterStatus, + CreatedTime: createdTime, AccessInfo: irs.AccessInfo{ - Endpoint: end_point, - Kubeconfig: kube_config, + Endpoint: endpoint, + Kubeconfig: kubeconfig, }, - // KeyValueList: []irs.KeyValue{}, // flatten data 입력하기 + //KeyValueList: []irs.KeyValue{}, } - // k,v 추출 & 추가 - flat, err := flatten.Flatten(cluster_json_obj, "", flatten.DotStyle) + // + // Fill clusterInfo.KeyValueList + // + jsonCluster, err := json.Marshal(cluster) if err != nil { - err := fmt.Errorf("Failed to Flatten ClusterInfo : %v", err) - cblogger.Error(err) + err = fmt.Errorf("failed to marshal cluster: %v", err) + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err + } + + var mapCluster map[string]interface{} + err = json.Unmarshal(jsonCluster, &mapCluster) + if err != nil { + err = fmt.Errorf("failed to unmarshal cluster: %v", err) + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err + } + + flat, err := flatten.Flatten(mapCluster, "", flatten.DotStyle) + if err != nil { + err = fmt.Errorf("failed to flatten cluster: %v", err) + err = fmt.Errorf("failed to get ClusterInfo: %v", err) return nil, err } delete(flat, "meta_data") for k, v := range flat { - temp := fmt.Sprintf("%v", v) - clusterInfo.KeyValueList = append(clusterInfo.KeyValueList, irs.KeyValue{Key: k, Value: temp}) + clusterInfo.KeyValueList = append(clusterInfo.KeyValueList, irs.KeyValue{Key: k, Value: fmt.Sprintf("%v", v)}) + } + + return clusterInfo, nil +} + +func (ach *AlibabaClusterHandler) getClusterInfo(regionId, clusterId string) (*irs.ClusterInfo, error) { + // + // Fill clusterInfo + // + clusterInfo, err := ach.getClusterInfoWithoutNodeGroupList(regionId, clusterId) + if err != nil { + err = fmt.Errorf("failed to get ClusterInfo: %v", err) + return nil, err } - // NodeGroups - node_groups_json_str, err := alibaba.ListNodeGroup(access_key, access_secret, region_id, cluster_id) + // + // Fill clusterInfo.NodeGroupList + // + nodepools, err := aliDescribeClusterNodePools(ach.CsClient, clusterId) if err != nil { - err := fmt.Errorf("Failed to List NodeGroup : %v", err) - cblogger.Error(err) + err = fmt.Errorf("failed to get ClusterInfo: %v", err) return nil, err } - var node_groups_json_obj map[string]interface{} - json.Unmarshal([]byte(node_groups_json_str), &node_groups_json_obj) - node_groups := node_groups_json_obj["nodepools"].([]interface{}) - for _, node_group := range node_groups { - node_group_id := node_group.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) - node_group_info, err := getNodeGroupInfo(access_key, access_secret, region_id, cluster_id, node_group_id) + for _, np := range nodepools { + ngId := tea.StringValue(np.NodepoolInfo.NodepoolId) + nodeGroupInfo, err := ach.getNodeGroupInfo(clusterId, ngId) if err != nil { - err := fmt.Errorf("Failed to Get NodeGroupInfo : %v", err) - cblogger.Error(err) + err = fmt.Errorf("failed to get ClusterInfo: %v", err) return nil, err } - clusterInfo.NodeGroupList = append(clusterInfo.NodeGroupList, *node_group_info) + + clusterInfo.NodeGroupList = append(clusterInfo.NodeGroupList, *nodeGroupInfo) } - return clusterInfo, err + return clusterInfo, nil } -func getNodeGroupInfo(access_key, access_secret, region_id, cluster_id, node_group_id string) (nodeGroupInfo *irs.NodeGroupInfo, err error) { +func (ach *AlibabaClusterHandler) getNodeGroupInfo(clusterId, nodeGroupId string) (*irs.NodeGroupInfo, error) { + // + // Fill nodeGroupInfo + // + nodepool, err := aliDescribeClusterNodePoolDetail(ach.CsClient, clusterId, nodeGroupId) + if err != nil { + err = fmt.Errorf("failed to get NodeGroupInfo: %v", err) + return nil, err + } + if nodepool.Status == nil || + nodepool.ScalingGroup == nil || + nodepool.AutoScaling == nil || + nodepool.NodepoolInfo == nil { + err = fmt.Errorf("invalid nodepool's information") + err = fmt.Errorf("failed to get NodeGroupInfo: %v", err) + return nil, err + } - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Failed to Process getNodeGroupInfo() : %v\n\n%v", r, string(debug.Stack())) - cblogger.Error(err) - } - }() + ngiStatus := irs.NodeGroupInactive + if strings.EqualFold(tea.StringValue(nodepool.Status.State), nodepoolStatusActive) { + ngiStatus = irs.NodeGroupActive + } else if strings.EqualFold(tea.StringValue(nodepool.Status.State), nodepoolStatusScaling) { + ngiStatus = irs.NodeGroupUpdating + } else if strings.EqualFold(tea.StringValue(nodepool.Status.State), nodepoolStatusRemoving) { + ngiStatus = irs.NodeGroupUpdating // removing is a kind of updating? + } else if strings.EqualFold(tea.StringValue(nodepool.Status.State), nodepoolStatusDeleting) { + ngiStatus = irs.NodeGroupDeleting + } else if strings.EqualFold(tea.StringValue(nodepool.Status.State), nodepoolStatusUpdating) { + ngiStatus = irs.NodeGroupUpdating + } - node_group_json_str, err := alibaba.GetNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id) - if err != nil { - err := fmt.Errorf("Failed to Get NodeGroup : %v", err) - cblogger.Error(err) + if len(nodepool.ScalingGroup.InstanceTypes) == 0 { + err = fmt.Errorf("failed to get NodeGroupInfo: invalid InstanceTypes") return nil, err } - var node_group_json_obj map[string]interface{} - json.Unmarshal([]byte(node_group_json_str), &node_group_json_obj) - - // nodegroup.state - // https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/query-the-details-of-a-node-pool - // active: The node pool is active. - // scaling: The node pool is being scaled. - // removing: Nodes are being removed from the node pool. - // deleting: The node pool is being deleted. - // updating: The node pool is being updated. - health_status := node_group_json_obj["status"].(map[string]interface{})["state"].(string) - status := irs.NodeGroupActive - if strings.EqualFold(health_status, "active") { - status = irs.NodeGroupActive - } else if strings.EqualFold(health_status, "scaling") { - status = irs.NodeGroupUpdating - } else if strings.EqualFold(health_status, "removing") { - status = irs.NodeGroupUpdating // removing is a kind of updating? - } else if strings.EqualFold(health_status, "deleting") { - status = irs.NodeGroupDeleting - } else if strings.EqualFold(health_status, "updating") { - status = irs.NodeGroupUpdating - } - - nodeGroupInfo = &irs.NodeGroupInfo{ + nodeGroupInfo := &irs.NodeGroupInfo{ IId: irs.IID{ - NameId: node_group_json_obj["nodepool_info"].(map[string]interface{})["name"].(string), - SystemId: node_group_json_obj["nodepool_info"].(map[string]interface{})["nodepool_id"].(string), + NameId: tea.StringValue(nodepool.NodepoolInfo.Name), + SystemId: tea.StringValue(nodepool.NodepoolInfo.NodepoolId), }, ImageIID: irs.IID{ - NameId: node_group_json_obj["scaling_group"].(map[string]interface{})["image_type"].(string), - SystemId: node_group_json_obj["scaling_group"].(map[string]interface{})["image_id"].(string), + NameId: tea.StringValue(nodepool.ScalingGroup.ImageType), + SystemId: tea.StringValue(nodepool.ScalingGroup.ImageId), }, - VMSpecName: node_group_json_obj["scaling_group"].(map[string]interface{})["instance_types"].([]interface{})[0].(string), - RootDiskType: node_group_json_obj["scaling_group"].(map[string]interface{})["system_disk_category"].(string), - RootDiskSize: strconv.Itoa(int(node_group_json_obj["scaling_group"].(map[string]interface{})["system_disk_size"].(float64))), + VMSpecName: tea.StringValue(nodepool.ScalingGroup.InstanceTypes[0]), + RootDiskType: tea.StringValue(nodepool.ScalingGroup.SystemDiskCategory), + RootDiskSize: strconv.FormatInt(tea.Int64Value(nodepool.ScalingGroup.SystemDiskSize), 10), KeyPairIID: irs.IID{ - NameId: node_group_json_obj["scaling_group"].(map[string]interface{})["key_pair"].(string), - SystemId: node_group_json_obj["scaling_group"].(map[string]interface{})["key_pair"].(string), // key-pair id is not exist. so use name. + NameId: tea.StringValue(nodepool.ScalingGroup.KeyPair), + SystemId: tea.StringValue(nodepool.ScalingGroup.KeyPair), }, - Status: status, - OnAutoScaling: node_group_json_obj["auto_scaling"].(map[string]interface{})["enable"].(bool), - MinNodeSize: int(node_group_json_obj["auto_scaling"].(map[string]interface{})["min_instances"].(float64)), - MaxNodeSize: int(node_group_json_obj["auto_scaling"].(map[string]interface{})["max_instances"].(float64)), - DesiredNodeSize: -1, // Parameter desired_size/count setting or modification is not supported for autoscaling-enabled nodepool + Status: ngiStatus, + OnAutoScaling: tea.BoolValue(nodepool.AutoScaling.Enable), + MinNodeSize: int(tea.Int64Value(nodepool.AutoScaling.MinInstances)), + MaxNodeSize: int(tea.Int64Value(nodepool.AutoScaling.MaxInstances)), + DesiredNodeSize: int(tea.Int64Value(nodepool.ScalingGroup.DesiredSize)), + } + + // + // Fill nodeGroupInfo.Nodes + // + nodes, err := aliDescribeClusterNodes(ach.CsClient, clusterId, tea.StringValue(nodepool.NodepoolInfo.NodepoolId)) + if err != nil { + err = fmt.Errorf("failed to get NodeGroupInfo: %v", err) + return nil, err + } - Nodes: []irs.IID{}, - KeyValueList: []irs.KeyValue{}, + for _, n := range nodes { + nId := tea.StringValue(n.InstanceId) + if nId != "" { + node := irs.IID{ + NameId: tea.StringValue(n.InstanceName), + SystemId: tea.StringValue(n.InstanceId), + } + nodeGroupInfo.Nodes = append(nodeGroupInfo.Nodes, node) + } } - // k,v 추출 & 추가 - flat, err := flatten.Flatten(node_group_json_obj, "", flatten.DotStyle) + // + // Fill nodeGroupInfo.KeyValueList + // + jsonNodepool, err := json.Marshal(nodepool) if err != nil { - err := fmt.Errorf("Failed to Flatten NodeGroup : %v", err) - cblogger.Error(err) + err = fmt.Errorf("failed to marshal nodepool: %v", err) + err = fmt.Errorf("failed to get NodeGroupInfo: %v", err) return nil, err } - delete(flat, "meta_data") - for k, v := range flat { - temp := fmt.Sprintf("%v", v) - nodeGroupInfo.KeyValueList = append(nodeGroupInfo.KeyValueList, irs.KeyValue{Key: k, Value: temp}) + + var mapNodepool map[string]interface{} + err = json.Unmarshal(jsonNodepool, &mapNodepool) + if err != nil { + err = fmt.Errorf("failed to unmarshal nodepool: %v", err) + err = fmt.Errorf("failed to get NodeGroupInfo: %v", err) + return nil, err } - nodes_json_str, err := alibaba.DescribeClusterNodes(access_key, access_secret, region_id, cluster_id, node_group_id) + flat, err := flatten.Flatten(mapNodepool, "", flatten.DotStyle) if err != nil { - err := fmt.Errorf("Failed to Get Nodes : %v", err) - cblogger.Error(err) + err = fmt.Errorf("failed to flatten nodepool: %v", err) + err = fmt.Errorf("failed to get NodeGroupInfo: %v", err) return nil, err } - var nodes_json_obj map[string]interface{} - json.Unmarshal([]byte(nodes_json_str), &nodes_json_obj) - nodes := nodes_json_obj["nodes"].([]interface{}) - for _, node := range nodes { - node_id := node.(map[string]interface{})["instance_id"].(string) - if node_id != "" { - nodeGroupInfo.Nodes = append(nodeGroupInfo.Nodes, irs.IID{NameId: "", SystemId: node_id}) - } + delete(flat, "meta_data") + for k, v := range flat { + nodeGroupInfo.KeyValueList = append(nodeGroupInfo.KeyValueList, irs.KeyValue{Key: k, Value: fmt.Sprintf("%v", v)}) } return nodeGroupInfo, err } -func getClusterInfoJSON(clusterHandler *AlibabaClusterHandler, clusterInfo irs.ClusterInfo) (clusterInfoJSON string, err error) { - - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Failed to Process getClusterInfoJSON() : %v\n\n%v", r, string(debug.Stack())) - cblogger.Error(err) - } - }() - - // get vswitch_id - master_vswitch_id := "" - res, err := alibaba.DescribeVSwitches(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterInfo.Network.VpcIID.SystemId) - if err != nil { - err := fmt.Errorf("Failed to Describe VSwitches : %v", err) - cblogger.Error(err) - return "", err - } - for _, v := range res.VSwitches.VSwitch { - master_vswitch_id = v.VSwitchId - break +func aliDescribeNatGatewaysWithTagInVpc(vpcClient *vpc2016.Client, regionId, vpcId, networkType, tagKey, tagValue string) ([]*vpc2016.DescribeNatGatewaysResponseBodyNatGatewaysNatGateway, error) { + tags := []*vpc2016.DescribeNatGatewaysRequestTag{ + &vpc2016.DescribeNatGatewaysRequestTag{ + Key: tea.String(tagKey), + Value: tea.String(tagValue), + }, } - // get cidr list - //cidr: Valid values: 10.0.0.0/16-24, 172.16-31.0.0/16-24, and 192.168.0.0/16-24. - m_cidr := make(map[string]bool) - for i := 16; i < 32; i++ { - m_cidr[fmt.Sprintf("172.%v.0.0/16", i)] = true + describeNatGatewaysRequest := &vpc2016.DescribeNatGatewaysRequest{ + RegionId: tea.String(regionId), + VpcId: tea.String(vpcId), + NetworkType: tea.String(networkType), + Tag: tags, } - clusters, err := clusterHandler.ListCluster() + describeNatGatewaysResponse, err := vpcClient.DescribeNatGateways(describeNatGatewaysRequest) if err != nil { - err := fmt.Errorf("Failed to List Cluster : %v", err) - cblogger.Error(err) - return "", err - } - for _, cluster := range clusters { - for _, v := range cluster.KeyValueList { - if v.Key == "parameters.ServiceCIDR" || v.Key == "subnet_cidr" { - delete(m_cidr, v.Value) - } - } - } - cidr_list := []string{} - for k := range m_cidr { - cidr_list = append(cidr_list, k) + return make([]*vpc2016.DescribeNatGatewaysResponseBodyNatGatewaysNatGateway, 0), err } - // create request json - temp := `{ - "name": "%s", - "region_id": "%s", - "cluster_type": "ManagedKubernetes", - "cluster_spec": "ack.pro.small", - "kubernetes_version": "%s", - "vpcid": "%s", - "container_cidr": "%s", - "service_cidr": "%s", - "num_of_nodes": 0, - "master_vswitch_ids": ["%s"], - "security_group_id": "%s", - "endpoint_public_access": true - }` + return describeNatGatewaysResponse.Body.NatGateways.NatGateway, nil +} - clusterInfoJSON = fmt.Sprintf(temp, clusterInfo.IId.NameId, clusterHandler.RegionInfo.Region, clusterInfo.Version, clusterInfo.Network.VpcIID.SystemId, cidr_list[0], cidr_list[1], master_vswitch_id, clusterInfo.Network.SecurityGroupIIDs[0].SystemId) +func getInternetNatGatewaysWithTagInVpc(vpcClient *vpc2016.Client, regionId, vpcId, tagKey, tagValue string) ([]*vpc2016.DescribeNatGatewaysResponseBodyNatGatewaysNatGateway, error) { + natGatewayList, err := aliDescribeNatGatewaysWithTagInVpc(vpcClient, regionId, vpcId, "internet", tagKey, tagValue) + if err != nil { + return nil, err + } - return clusterInfoJSON, err + return natGatewayList, nil } -func getNodeGroupJSONString(clusterHandler *AlibabaClusterHandler, clusterIID irs.IID, nodeGroupReqInfo irs.NodeGroupInfo) (payload string, err error) { - - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("Failed to Process getNodeGroupJSONString() : %v\n\n%v", r, string(debug.Stack())) - cblogger.Error(err) - } - }() +func aliTagNatGateway(vpcClient *vpc2016.Client, regionId, natGatewayId, tagKey, tagValue string) error { + tags := []*vpc2016.TagResourcesRequestTag{ + &vpc2016.TagResourcesRequestTag{ + Key: tea.String(tagKey), + Value: tea.String(tagValue), + }, + } - name := nodeGroupReqInfo.IId.NameId - // image_id := nodeGroupReqInfo.ImageIID.SystemId // 옵션은 있으나, 설정해도 반영 안됨 - enable := nodeGroupReqInfo.OnAutoScaling - max_instances := nodeGroupReqInfo.MaxNodeSize - min_instances := nodeGroupReqInfo.MinNodeSize - //desired_instances := nodeGroupReqInfo.DesiredNodeSize - instance_type := nodeGroupReqInfo.VMSpecName - key_pair := nodeGroupReqInfo.KeyPairIID.NameId - system_disk_category := nodeGroupReqInfo.RootDiskType - system_disk_size, _ := strconv.ParseInt(nodeGroupReqInfo.RootDiskSize, 10, 32) - - // get vswitch_id - clusterInfo, err := clusterHandler.GetCluster(clusterIID) - if err != nil { - err := fmt.Errorf("Failed to Get Cluster : %v", err) - cblogger.Error(err) - return "", err + tagResourcesRequest := &vpc2016.TagResourcesRequest{ + RegionId: tea.String(regionId), + ResourceId: tea.StringSlice([]string{natGatewayId}), + ResourceType: tea.String("NATGATEWAY"), + Tag: tags, } - vswitch_id := "" // get vswitch_id, get from cluster info - res, err := alibaba.DescribeVSwitches(clusterHandler.CredentialInfo.ClientId, clusterHandler.CredentialInfo.ClientSecret, clusterHandler.RegionInfo.Region, clusterInfo.Network.VpcIID.SystemId) + //spew.Dump(tagResourcesRequest) + _, err := vpcClient.TagResources(tagResourcesRequest) if err != nil { - err := fmt.Errorf("Failed to Describe VSwitches : %v", err) - cblogger.Error(err) - return "", err + return err + } + //spew.Dump(tagResourcesResponse.Body) + + return nil +} + +func aliDeleteNatGateway(vpcClient *vpc2016.Client, regionId, natGatewayId string) error { + deleteNatGatewayRequest := &vpc2016.DeleteNatGatewayRequest{ + RegionId: tea.String(regionId), + Force: tea.Bool(true), // When deleting the NAT Gateway, the Snat Entry will be deleted + NatGatewayId: tea.String(natGatewayId), } - for _, v := range res.VSwitches.VSwitch { - vswitch_id = v.VSwitchId - break + //spew.Dump(deleteNatGatewayRequest) + _, err := vpcClient.DeleteNatGateway(deleteNatGatewayRequest) + if err != nil { + return err } + //spew.Dump(deleteNatGatewayResponse) - temp := `{ - "nodepool_info": { - "name": "%s" - }, - "auto_scaling": { - "enable": %t, - "max_instances": %d, - "min_instances": %d - }, - "scaling_group": { - "instance_types": ["%s"], - "key_pair": "%s", - "system_disk_category": "%s", - "system_disk_size": %d, - "vswitch_ids": ["%s"] - }, - "management": { - "enable":true - } - }` + return nil +} - payload = fmt.Sprintf(temp, name, enable, max_instances, min_instances, instance_type, key_pair, system_disk_category, system_disk_size, vswitch_id) +func aliDescribeClustersV1(csClient *cs2015.Client, regionId string) ([]*cs2015.DescribeClustersV1ResponseBodyClusters, error) { + describeClustersV1Request := &cs2015.DescribeClustersV1Request{ + ClusterType: tea.String("ManagedKubernetes"), + RegionId: tea.String(regionId), + } + //spew.Dump(describeClustersV1Request) + describeClustersV1Response, err := csClient.DescribeClustersV1(describeClustersV1Request) + if err != nil { + return make([]*cs2015.DescribeClustersV1ResponseBodyClusters, 0), err + } + //spew.Dump(describeClustersV1Response.Body) - return payload, err + return describeClustersV1Response.Body.Clusters, nil } -// func getCallLogScheme(region string, resourceType call.RES_TYPE, resourceName string, apiName string) call.CLOUDLOGSCHEMA { -// cblogger.Info(fmt.Sprintf("Call %s %s", call.ALIBABA, apiName)) -// return call.CLOUDLOGSCHEMA{ -// CloudOS: call.ALIBABA, -// RegionZone: region, -// ResourceType: resourceType, -// ResourceName: resourceName, -// CloudOSAPI: apiName, -// } -// } +func aliDescribeClusterDetail(csClient *cs2015.Client, clusterId string) (*cs2015.DescribeClusterDetailResponseBody, error) { + describeClusterDetailResponse, err := csClient.DescribeClusterDetail(tea.String(clusterId)) + if err != nil { + return nil, err + } + //spew.Dump(describeClusterDetailResponse.Body) + + return describeClusterDetailResponse.Body, nil +} + +func existNotDeletedClusterWithTagInVpc(csClient *cs2015.Client, regionId, vpcId, tagKey, tagValue string) (bool, error) { + clusterList, err := aliDescribeClustersV1(csClient, regionId) + if err != nil { + return false, err + } + + var clusterListWithTagInVpc []*cs2015.DescribeClustersV1ResponseBodyClusters + for _, cluster := range clusterList { + if strings.EqualFold(*cluster.State, "deleting") || + strings.EqualFold(*cluster.State, "deleted") { + continue + } + if strings.EqualFold(*cluster.VpcId, vpcId) { + for _, tag := range cluster.Tags { + if strings.EqualFold(*tag.Key, tagKey) && + strings.EqualFold(*tag.Value, tagValue) { + clusterListWithTagInVpc = append(clusterListWithTagInVpc, cluster) + break + } + } + } + } + + if len(clusterListWithTagInVpc) > 0 { + return true, nil + } else { + return false, nil + } +} + +func aliDescribeVSwitches(vpcClient *vpc2016.Client, regionId, vpcId string) ([]*vpc2016.DescribeVSwitchesResponseBodyVSwitchesVSwitch, error) { + describeVSwitchesRequest := &vpc2016.DescribeVSwitchesRequest{ + RegionId: tea.String(regionId), + VpcId: tea.String(vpcId), + } + //spew.Dump(describeVSwitchesRequest) + describeVSwitchesResponse, err := vpcClient.DescribeVSwitches(describeVSwitchesRequest) + if err != nil { + return make([]*vpc2016.DescribeVSwitchesResponseBodyVSwitchesVSwitch, 0), err + } + //spew.Dump(describeVSwitchesResponse.Body.VSwitches.VSwitch) + + return describeVSwitchesResponse.Body.VSwitches.VSwitch, nil +} + +func getLatestRuntime(csClient *cs2015.Client, regionId, clusterType, k8sVersion string) (string, string, error) { + metadata, err := aliDescribeKubernetesVersionMetadata(csClient, regionId, clusterType, k8sVersion) + if err != nil { + err = fmt.Errorf("failed to get latest runtime name and version: %v", err) + return "", "", err + } + if len(metadata) == 0 { + err = fmt.Errorf("failed to get kubernetes version metadata") + return "", "", err + } + + runtimeName := defaultClusterRuntimeName + invalidVersion, _ := semver.NewVersion("0.0.0") + latestVersion := invalidVersion + for _, rt := range metadata[0].Runtimes { + if strings.EqualFold(tea.StringValue(rt.Name), runtimeName) { + rtVersion, err := semver.NewVersion(tea.StringValue(rt.Version)) + if err != nil { + continue + } + if latestVersion.LessThan(rtVersion) { + latestVersion = rtVersion + } + } + } + + if latestVersion.Equal(invalidVersion) { + err = fmt.Errorf("failed to get valid runtime version") + return "", "", err + } + runtimeVersion := latestVersion.String() + + return runtimeName, runtimeVersion, nil +} + +func getNodepoolsFromNodeGroupList(nodeGroupInfoList []irs.NodeGroupInfo, runtimeName, runtimeVersion string, vswitchIds []string) []*cs2015.Nodepool { + var nodepools []*cs2015.Nodepool + for _, ngInfo := range nodeGroupInfoList { + name := ngInfo.IId.NameId + autoScalingEnable := ngInfo.OnAutoScaling + maxInstances := ngInfo.MaxNodeSize + minInstances := ngInfo.MinNodeSize + instanceTypes := []string{ngInfo.VMSpecName} + systemDiskCategory := ngInfo.RootDiskType + systemDiskSize, _ := strconv.ParseInt(ngInfo.RootDiskSize, 10, 64) + keyPair := ngInfo.KeyPairIID.NameId + + nodepool := cs2015.Nodepool{ + NodepoolInfo: &cs2015.NodepoolNodepoolInfo{ + Name: tea.String(name), + }, + AutoScaling: &cs2015.NodepoolAutoScaling{ + Enable: tea.Bool(autoScalingEnable), + MaxInstances: tea.Int64(int64(maxInstances)), + MinInstances: tea.Int64(int64(minInstances)), + }, + KubernetesConfig: &cs2015.NodepoolKubernetesConfig{ + Runtime: tea.String(runtimeName), + RuntimeVersion: tea.String(runtimeVersion), + }, + ScalingGroup: &cs2015.NodepoolScalingGroup{ + VswitchIds: tea.StringSlice(vswitchIds), + InstanceTypes: tea.StringSlice(instanceTypes), + SystemDiskCategory: tea.String(systemDiskCategory), + SystemDiskSize: tea.Int64(systemDiskSize), + KeyPair: tea.String(keyPair), + //DesiredSize: tea.Int64(desiredSize), + }, + Management: &cs2015.NodepoolManagement{ + Enable: tea.Bool(true), + }, + } + + // CAUTION: if DesiredSize is set when AutoScaling is enabled, Alibaba reject the request + if autoScalingEnable == false { + nodepool.ScalingGroup.DesiredSize = tea.Int64(int64(ngInfo.DesiredNodeSize)) + } + + nodepools = append(nodepools, &nodepool) + } + + return nodepools +} + +func aliCreateCluster(csClient *cs2015.Client, name, regionId, clusterType, clusterSpec, k8sVersion, runtimeName, runtimeVersion, vpcId, containerCidr, serviceCidr, secGroupId string, snatEntry, endpointPublicAccess bool, masterVswitchIds []string, tagKey, tagValue string, nodepools []*cs2015.Nodepool) (*string, error) { + tags := []*cs2015.Tag{ + &cs2015.Tag{ + Key: tea.String(tagKey), + Value: tea.String(tagValue), + }, + } + + createClusterRequest := &cs2015.CreateClusterRequest{ + Name: tea.String(name), + RegionId: tea.String(regionId), + ClusterType: tea.String(clusterType), + ClusterSpec: tea.String(clusterSpec), + KubernetesVersion: tea.String(k8sVersion), + Runtime: &cs2015.Runtime{ + Name: tea.String(runtimeName), + Version: tea.String(runtimeVersion), + }, + Vpcid: tea.String(vpcId), + ContainerCidr: tea.String(containerCidr), + ServiceCidr: tea.String(serviceCidr), + MasterVswitchIds: tea.StringSlice(masterVswitchIds), + SecurityGroupId: tea.String(secGroupId), + SnatEntry: tea.Bool(snatEntry), + EndpointPublicAccess: tea.Bool(endpointPublicAccess), + Tags: tags, + //Nodepools: nodepools, + } + if len(nodepools) > 0 { + createClusterRequest.Nodepools = nodepools + } + //spew.Dump(createClusterRequest) + createClusterResponse, err := csClient.CreateCluster(createClusterRequest) + if err != nil { + return nil, err + } + //spew.Dump(createClusterResponse.Body) + + return createClusterResponse.Body.ClusterId, nil +} + +func waitUntilClusterIsState(csClient *cs2015.Client, clusterId, state string) error { + apiCallCount := 0 + maxAPICallCount := 20 + + var waitingErr error + for { + cluster, err := aliDescribeClusterDetail(csClient, clusterId) + if err != nil { + maxAPICallCount = maxAPICallCount / 2 + } + if strings.EqualFold(tea.StringValue(cluster.State), state) { + return nil + } + apiCallCount++ + if apiCallCount >= maxAPICallCount { + waitingErr = fmt.Errorf("failed to get cluster: The maximum number of verification requests has been exceeded while waiting for availability of that resource") + break + } + time.Sleep(5 * time.Second) + cblogger.Info("Wait until cluster's state is ", state) + } + + return waitingErr +} + +func aliDescribeVpcAttribute(vpcClient *vpc2016.Client, regionId, vpcId string) (*vpc2016.DescribeVpcAttributeResponseBody, error) { + describeVpcAttributeRequest := &vpc2016.DescribeVpcAttributeRequest{ + RegionId: tea.String(regionId), + VpcId: tea.String(vpcId), + } + //spew.Dump(describeVpcAttributeRequest) + describeVpcAttributeResponse, err := vpcClient.DescribeVpcAttribute(describeVpcAttributeRequest) + if err != nil { + return nil, err + } + //spew.Dump(describeVpcAttributeResponse.Body) + + return describeVpcAttributeResponse.Body, nil +} + +func aliDescribeVSwitchAttributes(vpcClient *vpc2016.Client, regionId, vswitchId string) (*vpc2016.DescribeVSwitchAttributesResponseBody, error) { + describeVSwitchAttributesRequest := &vpc2016.DescribeVSwitchAttributesRequest{ + RegionId: tea.String(regionId), + VSwitchId: tea.String(vswitchId), + } + //spew.Dump(describeVSwitchAttributesRequest) + describeVSwitchAttributesResponse, err := vpcClient.DescribeVSwitchAttributes(describeVSwitchAttributesRequest) + if err != nil { + return nil, err + } + //spew.Dump(describeVSwitchAttributesResponse.Body) + + return describeVSwitchAttributesResponse.Body, nil +} + +func aliDescribeSecurityGroupAttribute(ecsClient *ecs2014.Client, regionId, securityGroupId string) (*ecs2014.DescribeSecurityGroupAttributeResponseBody, error) { + describeSecurityGroupAttributeRequest := &ecs2014.DescribeSecurityGroupAttributeRequest{ + RegionId: tea.String(regionId), + SecurityGroupId: tea.String(securityGroupId), + } + //spew.Dump(describeSecurityGroupAttributeRequest) + describeSecurityGroupAttributeResponse, err := ecsClient.DescribeSecurityGroupAttribute(describeSecurityGroupAttributeRequest) + if err != nil { + return nil, err + } + //spew.Dump(describeSecurityGroupAttributeResponse.Body) + + return describeSecurityGroupAttributeResponse.Body, nil +} + +func aliDeleteCluster(csClient *cs2015.Client, clusterId string, retainResources []string) (*cs2015.DeleteClusterResponseBody, error) { + deleteClusterRequest := &cs2015.DeleteClusterRequest{ + RetainResources: tea.StringSlice(retainResources), + } + //spew.Dump(deleteClusterRequest) + deleteClusterResponse, err := csClient.DeleteCluster(tea.String(clusterId), deleteClusterRequest) + if err != nil { + return nil, err + } + //spew.Dump(deleteClusterResponse.Body) + + return deleteClusterResponse.Body, nil +} + +func aliDescribeKubernetesVersionMetadata(csClient *cs2015.Client, regionId, clusterType, k8sVersion string) ([]*cs2015.DescribeKubernetesVersionMetadataResponseBody, error) { + describeKubernetesVersionMetadataRequest := &cs2015.DescribeKubernetesVersionMetadataRequest{ + Region: tea.String(regionId), + ClusterType: tea.String(clusterType), + KubernetesVersion: tea.String(k8sVersion), + } + //spew.Dump(describeKubernetesVersionMetadataRequest) + describeKubernetesVersionMetadataResponse, err := csClient.DescribeKubernetesVersionMetadata(describeKubernetesVersionMetadataRequest) + if err != nil { + return nil, err + } + //spew.Dump(describeKubernetesVersionMetadataResponse.Body) + + return describeKubernetesVersionMetadataResponse.Body, nil +} + +func aliCreateClusterNodePool(csClient *cs2015.Client, clusterId, name string, autoScalingEnable bool, maxInstances, minInstances int64, vswitchIds, instanceTypes []string, systemDiskCategory string, systemDiskSize int64, keyPair string, desiredSize int64) (*string, error) { + createClusterNodePoolRequest := &cs2015.CreateClusterNodePoolRequest{ + NodepoolInfo: &cs2015.CreateClusterNodePoolRequestNodepoolInfo{ + Name: tea.String(name), + }, + AutoScaling: &cs2015.CreateClusterNodePoolRequestAutoScaling{ + Enable: tea.Bool(autoScalingEnable), + MaxInstances: tea.Int64(maxInstances), + MinInstances: tea.Int64(minInstances), + }, + ScalingGroup: &cs2015.CreateClusterNodePoolRequestScalingGroup{ + VswitchIds: tea.StringSlice(vswitchIds), + InstanceTypes: tea.StringSlice(instanceTypes), + SystemDiskCategory: tea.String(systemDiskCategory), + SystemDiskSize: tea.Int64(systemDiskSize), + KeyPair: tea.String(keyPair), + //DesiredSize: tea.Int64(desiredSize), + }, + Management: &cs2015.CreateClusterNodePoolRequestManagement{ + Enable: tea.Bool(true), + }, + } + + // CAUTION: if DesiredSize is set when AutoScaling is enabled, Alibaba reject the request + if autoScalingEnable == false { + createClusterNodePoolRequest.ScalingGroup.DesiredSize = tea.Int64(desiredSize) + } + + //spew.Dump(createClusterNodePoolRequest) + createClusterNodePoolResponse, err := csClient.CreateClusterNodePool(tea.String(clusterId), createClusterNodePoolRequest) + if err != nil { + return nil, err + } + //spew.Dump(createClusterNodePoolResponse.Body) + + return createClusterNodePoolResponse.Body.NodepoolId, nil +} + +func aliDeleteClusterNodepool(csClient *cs2015.Client, clusterId, nodepoolId string, force bool) (*cs2015.DeleteClusterNodepoolResponseBody, error) { + deleteClusterNodepoolRequest := &cs2015.DeleteClusterNodepoolRequest{ + Force: tea.Bool(force), + } + //spew.Dump(deleteClusterNodepoolRequest) + deleteClusterNodepoolResponse, err := csClient.DeleteClusterNodepool(tea.String(clusterId), tea.String(nodepoolId), deleteClusterNodepoolRequest) + if err != nil { + return nil, err + } + //spew.Dump(deleteClusterNodepoolResponse.Body) + + return deleteClusterNodepoolResponse.Body, nil +} + +func aliDescribeClusterUserKubeconfig(csClient *cs2015.Client, clusterId string) (*cs2015.DescribeClusterUserKubeconfigResponseBody, error) { + describeClusterUserKubeconfigRequest := &cs2015.DescribeClusterUserKubeconfigRequest{} + describeClusterUserKubeconfigResponse, err := csClient.DescribeClusterUserKubeconfig(tea.String(clusterId), describeClusterUserKubeconfigRequest) + if err != nil { + return nil, err + } + + return describeClusterUserKubeconfigResponse.Body, nil +} + +func aliDescribeClusterNodePools(csClient *cs2015.Client, clusterId string) ([]*cs2015.DescribeClusterNodePoolsResponseBodyNodepools, error) { + describeClusterNodePoolsResponse, err := csClient.DescribeClusterNodePools(tea.String(clusterId)) + if err != nil { + return make([]*cs2015.DescribeClusterNodePoolsResponseBodyNodepools, 0), err + } + //spew.Dump(describeClusterNodePoolsResponse.Body) + + return describeClusterNodePoolsResponse.Body.Nodepools, nil +} + +func aliDescribeClusterNodePoolDetail(csClient *cs2015.Client, clusterId, nodepoolId string) (*cs2015.DescribeClusterNodePoolDetailResponseBody, error) { + describeClusterNodePoolDetailResponse, err := csClient.DescribeClusterNodePoolDetail(tea.String(clusterId), tea.String(nodepoolId)) + if err != nil { + return nil, err + } + //spew.Dump(describeClusterNodePoolDetailResponse.Body) + + return describeClusterNodePoolDetailResponse.Body, nil +} + +func aliDescribeClusterNodes(csClient *cs2015.Client, clusterId, nodepoolId string) ([]*cs2015.DescribeClusterNodesResponseBodyNodes, error) { + describeClusterNodesRequest := &cs2015.DescribeClusterNodesRequest{ + NodepoolId: tea.String(nodepoolId), + } + //spew.Dump(describeClusterNodesRequest) + describeClusterNodesResponse, err := csClient.DescribeClusterNodes(tea.String(clusterId), describeClusterNodesRequest) + if err != nil { + return nil, err + } + //spew.Dump(describeClusterNodesResponse.Body) + + return describeClusterNodesResponse.Body.Nodes, nil +} + +func aliModifyClusterNodePoolAutoScalingEnable(csClient *cs2015.Client, clusterId, nodepoolId string, enable bool) (*cs2015.ModifyClusterNodePoolResponseBody, error) { + modifyClusterNodePoolRequest := &cs2015.ModifyClusterNodePoolRequest{ + AutoScaling: &cs2015.ModifyClusterNodePoolRequestAutoScaling{ + Enable: tea.Bool(enable), + }, + } + modifyClusterNodePoolResponse, err := csClient.ModifyClusterNodePool(tea.String(clusterId), tea.String(nodepoolId), modifyClusterNodePoolRequest) + if err != nil { + return nil, err + } + //spew.Dump(modifyClusterNodePoolResponse.Body) + + return modifyClusterNodePoolResponse.Body, nil +} + +func aliModifyClusterNodePoolScalingSize(csClient *cs2015.Client, clusterId, nodepoolId string, autoScalingEnable bool, maxInstances, minInstances, desiredSize int64) (*cs2015.ModifyClusterNodePoolResponseBody, error) { + modifyClusterNodePoolRequest := &cs2015.ModifyClusterNodePoolRequest{ + AutoScaling: &cs2015.ModifyClusterNodePoolRequestAutoScaling{ + Enable: tea.Bool(autoScalingEnable), + MaxInstances: tea.Int64(maxInstances), + MinInstances: tea.Int64(minInstances), + }, + } + + // CAUTION: if DesiredSize is set when AutoScaling is enabled, Alibaba reject the request + if autoScalingEnable == false { + modifyClusterNodePoolRequest.ScalingGroup.DesiredSize = tea.Int64(desiredSize) + } + //spew.Dump(modifyClusterNodePoolRequest) + + modifyClusterNodePoolResponse, err := csClient.ModifyClusterNodePool(tea.String(clusterId), tea.String(nodepoolId), modifyClusterNodePoolRequest) + //spew.Dump("aliModifyClusterNodePoolScalingSize") + if err != nil { + return nil, err + } + //spew.Dump(modifyClusterNodePoolResponse.Body) + + return modifyClusterNodePoolResponse.Body, nil +} + +func aliUpgradeCluster(csClient *cs2015.Client, clusterId, nextVersion string) (*cs2015.UpgradeClusterResponseBody, error) { + upgradeClusterRequest := &cs2015.UpgradeClusterRequest{ + NextVersion: tea.String(nextVersion), + } + //spew.Dump(upgradeClusterRequest) + upgradeClusterResponse, err := csClient.UpgradeCluster(tea.String(clusterId), upgradeClusterRequest) + if err != nil { + return nil, err + } + //spew.Dump(upgradeClusterResponse.Body) + + return upgradeClusterResponse.Body, nil +} + +func cleanCluster(csClient *cs2015.Client, clusterId string) error { + _, err := aliDeleteCluster(csClient, clusterId, []string{}) + if err != nil { + return err + } + + return nil +} + +func validateAtCreateCluster(clusterInfo irs.ClusterInfo) error { + if clusterInfo.IId.NameId == "" { + return fmt.Errorf("Cluster name is required") + } + if clusterInfo.Network.VpcIID.SystemId == "" && clusterInfo.Network.VpcIID.NameId == "" { + return fmt.Errorf("Cannot identify VPC(IID=%s)", clusterInfo.Network.VpcIID) + } + if len(clusterInfo.Network.SubnetIIDs) < 1 { + return fmt.Errorf("At least one Subnet must be specified") + } + // CAUTION: Currently CB-Spider's Alibaba PMKS Drivers does not support to create a cluster with nodegroups + if len(clusterInfo.NodeGroupList) > 0 { + return fmt.Errorf("Node Group cannot be specified") + } + /* + if clusterInfo.Version == "" || clusterInfo.Version == "default" { + clusterInfo.Version = "1.24.8" + } + */ + return nil +} + +func validateAtAddNodeGroup(clusterIID irs.IID, nodeGroupInfo irs.NodeGroupInfo) error { + if clusterIID.SystemId == "" && clusterIID.NameId == "" { + return fmt.Errorf("Invalid Cluster IID") + } + if nodeGroupInfo.IId.NameId == "" { + return fmt.Errorf("Node Group name is required") + } + if nodeGroupInfo.MaxNodeSize < 1 { + return fmt.Errorf("MaxNodeSize cannot be smaller than 1") + } + if nodeGroupInfo.MinNodeSize < 1 { + return fmt.Errorf("MaxNodeSize cannot be smaller than 1") + } + if nodeGroupInfo.DesiredNodeSize < 1 { + return fmt.Errorf("DesiredNodeSize cannot be smaller than 1") + } + if nodeGroupInfo.VMSpecName == "" { + return fmt.Errorf("VM Spec Name is required") + } + + return nil +} + +func validateAtChangeNodeGroupScaling(clusterIID irs.IID, nodeGroupIID irs.IID, minNodeSize int, maxNodeSize int) error { + if clusterIID.SystemId == "" && clusterIID.NameId == "" { + return fmt.Errorf("Invalid Cluster IID") + } + if nodeGroupIID.SystemId == "" && nodeGroupIID.NameId == "" { + return fmt.Errorf("Invalid Node Group IID") + } + if minNodeSize < 1 { + return fmt.Errorf("MaxNodeSize cannot be smaller than 1") + } + if maxNodeSize < 1 { + return fmt.Errorf("MaxNodeSize cannot be smaller than 1") + } + + return nil +} + +func (ach *AlibabaClusterHandler) getAvailableCidrList() ([]string, error) { + // + // Valid CIDR: 10.0.0.0/16-24, 172.16-31.0.0/16-24, and 192.168.0.0/16-24. + // + mapCidr := make(map[string]bool) + for i := 16; i < 32; i++ { + mapCidr[fmt.Sprintf("172.%v.0.0/16", i)] = true + } + + clusterInfoList, err := ach.getClusterInfoListWithoutNodeGroupList(ach.RegionInfo.Region) + if err != nil { + err = fmt.Errorf("failed to get available CIDR list: %v", err) + return []string{}, err + } + + for _, clusterInfo := range clusterInfoList { + for _, v := range clusterInfo.KeyValueList { + if v.Key == "parameters.ServiceCIDR" || v.Key == "subnet_cidr" { + delete(mapCidr, v.Value) + } + } + } + + cidrList := []string{} + for k := range mapCidr { + cidrList = append(cidrList, k) + } + + return cidrList, nil +} + +/* +func waitUntilNodepoolIsState(csClient *cs2015.Client, clusterId, nodepoolId, state string) error { + apiCallCount := 0 + maxAPICallCount := 20 + + var waitingErr error + for { + nodepool, err := aliDescribeClusterNodePoolDetail(csClient, clusterId, nodepoolId) + if err != nil { + maxAPICallCount = maxAPICallCount / 2 + } + if nodepool.Status != nil && strings.EqualFold(tea.StringValue(nodepool.Status.State), state) { + return nil + } + apiCallCount++ + if apiCallCount >= maxAPICallCount { + waitingErr = fmt.Errorf("failed to get nodepool: The maximum number of verification requests has been exceeded while waiting for availability of that resource") + break + } + time.Sleep(5 * time.Second) + } + + return waitingErr +} + +// Check whether Internet NAT Gateway is avaiable or not +func isExistInternetNatGatewayInVpc(vpcClient *vpc2016.Client, regionId, vpcId, vSwitchId string) (bool, error) { + natGatewayList, err := aliDescribeNatGateways(vpcClient, regionId, vpcId, vSwitchId, "internet") + if err != nil { + return false, err + } + + if len(natGatewayList) > 0 { + return true, nil + } + + return false, nil +} + +func aliAllocateEipAddress(vpcClient *vpc2016.Client, regionId, vpcId string) (*string, *string, error) { + description := delimiterVpcId + vpcId + + allocateEipAddressRequest := &vpc2016.AllocateEipAddressRequest{ + RegionId: tea.String(regionId), + Description: tea.String(description), + } + allocateEipAddressResponse, err := vpcClient.AllocateEipAddress(allocateEipAddressRequest) + if err != nil { + return nil, nil, err + } + + return allocateEipAddressResponse.Body.EipAddress, allocateEipAddressResponse.Body.AllocationId, nil +} + +func aliDescribeEipAddressWithIdAndNat(vpcClient *vpc2016.Client, regionId, eipId, natGatewayId string) (eipAddress *vpc2016.DescribeEipAddressesResponseBodyEipAddressesEipAddress, err error) { + describeEipAddressesRequest := &vpc2016.DescribeEipAddressesRequest{ + RegionId: tea.String(regionId), + AllocationId: tea.String(eipId), + AssociatedInstanceType: tea.String("Nat"), + AssociatedInstanceId: tea.String(natGatewayId), + } + //spew.Dump(describeEipAddressesRequest) + describeEipAddressesResponse, err := vpcClient.DescribeEipAddresses(describeEipAddressesRequest) + if err != nil { + return nil, err + } + //spew.Dump(describeEipAddressesResponse.Body) + + eipCount := len(describeEipAddressesResponse.Body.EipAddresses.EipAddress) + if eipCount == 1 { + eipAddress = describeEipAddressesResponse.Body.EipAddresses.EipAddress[0] + err = nil + } else if eipCount == 0 { + eipAddress = nil + err = fmt.Errorf("no eip address(ID=%s)", eipId) + } else { + eipAddress = nil + err = fmt.Errorf("more than one eip address(ID=%s)", eipId) + } + + return eipAddress, err +} + +func aliDescribeEipAddressesWithNat(vpcClient *vpc2016.Client, regionId, natGatewayId string) ([]*vpc2016.DescribeEipAddressesResponseBodyEipAddressesEipAddress, error) { + describeEipAddressesRequest := &vpc2016.DescribeEipAddressesRequest{ + RegionId: tea.String(regionId), + AssociatedInstanceType: tea.String("Nat"), + AssociatedInstanceId: tea.String(natGatewayId), + } + //spew.Dump(describeEipAddressesRequest) + describeEipAddressesResponse, err := vpcClient.DescribeEipAddresses(describeEipAddressesRequest) + if err != nil { + return make([]*vpc2016.DescribeEipAddressesResponseBodyEipAddressesEipAddress, 0), err + } + //spew.Dump(describeEipAddressesResponse.Body) + + return describeEipAddressesResponse.Body.EipAddresses.EipAddress, nil +} + +func aliReleaseEipAddress(vpcClient *vpc2016.Client, regionId, eipId string) error { + releaseEipAddressRequest := &vpc2016.ReleaseEipAddressRequest{ + RegionId: tea.String(regionId), + AllocationId: tea.String(eipId), + } + //spew.Dump(releaseEipAddressRequest) + _, err := vpcClient.ReleaseEipAddress(releaseEipAddressRequest) + if err != nil { + return err + } + //spew.Dump(releaseEipAddressResponse.Body) + + return nil +} + +func aliCreateNatGateway(vpcClient *vpc2016.Client, regionId, vpcId, vSwitchId string) (*string, []*string, error) { + description := delimiterVpcId + vpcId + + createNatGatewayRequest := &vpc2016.CreateNatGatewayRequest{ + RegionId: tea.String(regionId), + VpcId: tea.String(vpcId), + VSwitchId: tea.String(vSwitchId), + NatType: tea.String("Enhanced"), + NetworkType: tea.String("internet"), + Description: tea.String(description), + EipBindMode: tea.String("NAT"), + } + //spew.Dump(createNatGatewayRequest) + createNatGatewayResponse, err := vpcClient.CreateNatGateway(createNatGatewayRequest) + if err != nil { + return nil, make([]*string, 0), err + } + //spew.Dump(createNatGatewayResponse.Body) + + return createNatGatewayResponse.Body.NatGatewayId, createNatGatewayResponse.Body.SnatTableIds.SnatTableId, nil +} + +func aliGetNatGatewayAttribute(vpcClient *vpc2016.Client, regionId, natGatewayId string) (*vpc2016.GetNatGatewayAttributeResponseBody, error) { + getNatGatewayAttributeRequest := &vpc2016.GetNatGatewayAttributeRequest{ + RegionId: tea.String(regionId), + NatGatewayId: tea.String(natGatewayId), + } + //spew.Dump(getNatGatewayAttributeRequest) + getNatGatewayAttributeResponse, err := vpcClient.GetNatGatewayAttribute(getNatGatewayAttributeRequest) + if err != nil { + return nil, err + } + //spew.Dump(getNatGatewayAttributeResponse.Body) + + return getNatGatewayAttributeResponse.Body, nil +} + +func waitUntilNatGatewayIsAvailable(vpcClient *vpc2016.Client, regionId, natGatewayId string) error { + apiCallCount := 0 + maxAPICallCount := 20 + + var waitingErr error + for { + ngw, err := aliGetNatGatewayAttribute(vpcClient, regionId, natGatewayId) + if err != nil { + maxAPICallCount = maxAPICallCount / 2 + } + if ngw != nil && strings.EqualFold(*ngw.Status, "Available") { + return nil + } + apiCallCount++ + if apiCallCount >= maxAPICallCount { + waitingErr = fmt.Errorf("failed to get NAT Gateway: The maximum number of verification requests has been exceeded while waiting for the creation of that resource") + break + } + time.Sleep(10 * time.Second) + } + + return waitingErr +} + +func aliCreateSnatEntryForVpc(vpcClient *vpc2016.Client, regionId, snatTableId, snatIp, srcCidr string) error { + createSnatEntryRequest := &vpc2016.CreateSnatEntryRequest{ + RegionId: tea.String(regionId), + SnatIp: tea.String(snatIp), + SnatTableId: tea.String(snatTableId), + SourceCIDR: tea.String(srcCidr), + } + //spew.Dump(createSnatEntryRequest) + _, err := vpcClient.CreateSnatEntry(createSnatEntryRequest) + if err != nil { + return err + } + //spew.Dump(createSnatEntryResponse.Body) + + return nil +} + +func waitUntilEipIsStatus(vpcClient *vpc2016.Client, regionId, eipId, natGatewayId, status string) error { + apiCallCount := 0 + maxAPICallCount := 20 + + var waitingErr error + for { + eipAddress, err := aliDescribeEipAddressWithIdAndNat(vpcClient, regionId, eipId, natGatewayId) + if err != nil { + maxAPICallCount = maxAPICallCount / 2 + } + if eipAddress != nil && strings.EqualFold(*eipAddress.Status, status) { + return nil + } + apiCallCount++ + if apiCallCount >= maxAPICallCount { + waitingErr = fmt.Errorf("failed to get eip address: The maximum number of verification requests has been exceeded while waiting for availability of that resource") + break + } + time.Sleep(5 * time.Second) + } + + return waitingErr +} + +func aliAssociateEipAddressToNatGateway(vpcClient *vpc2016.Client, regionId, eipId, natGatewayId, vpcId string) error { + associateEipAddressRequest := &vpc2016.AssociateEipAddressRequest{ + RegionId: tea.String(regionId), + AllocationId: tea.String(eipId), + InstanceId: tea.String(natGatewayId), + InstanceType: tea.String("Nat"), + VpcId: tea.String(vpcId), + } + //spew.Dump(associateEipAddressRequest) + _, err := vpcClient.AssociateEipAddress(associateEipAddressRequest) + if err != nil { + return err + } + //spew.Dump(associateEipAddressResponse.Body) + + return nil +} + +func aliUnassociateEipAddressFromNatGateway(vpcClient *vpc2016.Client, regionId, eipId, natGatewayId string) error { + unassociateEipAddressRequest := &vpc2016.UnassociateEipAddressRequest{ + RegionId: tea.String(regionId), + AllocationId: tea.String(eipId), + InstanceId: tea.String(natGatewayId), + InstanceType: tea.String("Nat"), + } + //spew.Dump(unassociateEipAddressRequest) + _, err := vpcClient.UnassociateEipAddress(unassociateEipAddressRequest) + if err != nil { + return err + } + //spew.Dump(unassociateEipAddressResponse.Body) + + return nil +} + +func createNatGatewayWithEip(vpcClient *vpc2016.Client, regionId, vpcId, vSwitchId string) (waitErr error) { + var err error + + vpcAttribute, vpcErr := aliDescribeVpcAttribute(vpcClient, regionId, vpcId) + if vpcErr != nil { + vpcErr = fmt.Errorf("failed to get VPC Attribute: %v", vpcErr) + return vpcErr + } + + cblogger.Debug("Request to allocate EIP") + eipAddress, eipId, allocateErr := aliAllocateEipAddress(vpcClient, regionId, vpcId) + if allocateErr != nil { + cblogger.Debug("Failed to allocate EIP: ", allocateErr) + allocateErr = fmt.Errorf("failed to allocate an EIP: %v", allocateErr) + return allocateErr + } + cblogger.Debug("Successfully allocated EIP: IP=", *eipAddress) + + cblogger.Debug("Request to create NAT Gateway") + natGatewayId, snatTableIds, createNatGatewayErr := aliCreateNatGateway(vpcClient, regionId, vpcId, vSwitchId) + if createNatGatewayErr != nil || len(snatTableIds) == 0 { + cblogger.Debug("Failed to create NAT Gateway: ", createNatGatewayErr) + err = aliReleaseEipAddress(vpcClient, regionId, *eipId) + if err != nil { + createNatGatewayErr = fmt.Errorf("failed to release EIP(ID=%s), it should be manually released: %v: %v", eipId, err, createNatGatewayErr) + } + + createNatGatewayErr = fmt.Errorf("failed to create NAT Gateway: %v", createNatGatewayErr) + return createNatGatewayErr + } + cblogger.Debug("Successfully created NAT Gateway: ID=", *natGatewayId) + + cblogger.Debug("Wait until NAT Gateway is Available: ID=", *natGatewayId) + waitErr = waitUntilNatGatewayIsAvailable(vpcClient, regionId, *natGatewayId) + if waitErr != nil { + cblogger.Debug("Failed to wait until NAT Gateway is Available: ID=", *natGatewayId, ": ", waitErr) + + err = aliDeleteNatGateway(vpcClient, regionId, *natGatewayId) + if err != nil { + waitErr = fmt.Errorf("failed to delete NAT Gateway(ID=%s), it should be manually deleted: %v: %v", natGatewayId, err, waitErr) + } + + err = aliReleaseEipAddress(vpcClient, regionId, *eipId) + if err != nil { + waitErr = fmt.Errorf("failed to release EIP(ID=%s), it should be manually released: %v: %v", eipId, err, waitErr) + } + + waitErr = fmt.Errorf("failed to wait until NAT Gateway is available: %v", waitErr) + return waitErr + } + + cblogger.Debug("Request to associate EIP to NAT Gateway: IP=", *eipAddress, " NAT Gateway ID=", *natGatewayId) + associateErr := aliAssociateEipAddressToNatGateway(vpcClient, regionId, *eipId, *natGatewayId, vpcId) + if associateErr != nil { + cblogger.Debug("Failed to associate EIP to NAT Gateway: IP=", *eipAddress, " NAT Gateway ID=", *natGatewayId, ": ", associateErr) + + err = aliDeleteNatGateway(vpcClient, regionId, *natGatewayId) + if err != nil { + associateErr = fmt.Errorf("failed to delete NAT Gateway(ID=%s), it should be manually deleted: %v: %v", natGatewayId, err, associateErr) + } + + err = aliReleaseEipAddress(vpcClient, regionId, *eipId) + if err != nil { + associateErr = fmt.Errorf("failed to release EIP(ID=%s), it should be manually released: %v: %v", eipId, err, associateErr) + } + + associateErr = fmt.Errorf("failed to associate EIP(ID=%s): %v", natGatewayId, associateErr) + return associateErr + } + cblogger.Debug("Successfully associated EIP to NAT Gateway: IP=", *eipAddress, " NAT Gateway ID=", *natGatewayId) + + cblogger.Debug("Wait until EIP is InUse: IP=", *eipAddress) + waitErr = waitUntilEipIsStatus(vpcClient, regionId, *eipId, *natGatewayId, "InUse") + if waitErr != nil { + cblogger.Debug("Failed to wait until EIP is InUse: IP=", *eipAddress, ": ", waitErr) + + err = aliDeleteNatGateway(vpcClient, regionId, *natGatewayId) + if err != nil { + waitErr = fmt.Errorf("failed to delete NAT Gateway(ID=%s), it should be manually deleted: %v: %v", natGatewayId, err, waitErr) + } + + err = aliReleaseEipAddress(vpcClient, regionId, *eipId) + if err != nil { + waitErr = fmt.Errorf("failed to release EIP(ID=%s), it should be manually released: %v: %v", eipId, err, waitErr) + } + + waitErr = fmt.Errorf("failed to wait until NAT Gateway is available: %v", waitErr) + return waitErr + } + + cblogger.Debug("Request to create SNAT Entry in NAT Gateway for VPC: NAT Gateway ID=", *natGatewayId, ", VPC ID=", vpcId) + createSnatEntryErr := aliCreateSnatEntryForVpc(vpcClient, regionId, *snatTableIds[0], *eipAddress, *vpcAttribute.CidrBlock) + if createSnatEntryErr != nil { + cblogger.Debug("Failed to create SNAT Entry in NAT Gateway for VPC: NAT Gateway ID=", *natGatewayId, ", VPC ID=", vpcId, ": ", createSnatEntryErr) + + err = aliDeleteNatGateway(vpcClient, regionId, *natGatewayId) + if err != nil { + createSnatEntryErr = fmt.Errorf("failed to delete NAT Gateway(ID=%s), it should be manually deleted: %v: %v", natGatewayId, err, createSnatEntryErr) + } + + err = aliReleaseEipAddress(vpcClient, regionId, *eipId) + if err != nil { + createSnatEntryErr = fmt.Errorf("failed to release EIP(ID=%s), it should be manually released: %v: %v", eipId, err, createSnatEntryErr) + } + + createSnatEntryErr = fmt.Errorf("failed to create a SNAT etnry: %v", createSnatEntryErr) + return createSnatEntryErr + } + cblogger.Debug("Successfully created SNAT Entry in NAT Gateway: ID=", *natGatewayId) + + return nil +} + +func deleteNatGatewayWithEip(vpcClient *vpc2016.Client, regionId, vpcId, vSwitchId string) (resultErr error) { + natGatewayList, err := aliDescribeNatGateways(vpcClient, regionId, vpcId, vSwitchId, "internet") + if err != nil { + resultErr = fmt.Errorf("failed to get NAT Gateways: %v", err) + return resultErr + } + + var ngwForCluster *vpc2016.DescribeNatGatewaysResponseBodyNatGatewaysNatGateway = nil + for _, ngw := range natGatewayList { + ngwVpcId := "" + re := regexp.MustCompile(`\S*` + delimiterVpcId + `\S*`) + found := re.FindString(*ngw.Description) + if found != "" { + split := strings.Split(found, delimiterVpcId) + ngwVpcId = split[1] + if strings.EqualFold(ngwVpcId, vpcId) { + ngwForCluster = ngw + break + } + } + } + + if ngwForCluster == nil { + resultErr = fmt.Errorf("no NAT Gateway in %s%s", delimiterVpcId, vpcId) + return resultErr + } + + cblogger.Debug("Request to delete NAT Gateway: ID=", *ngwForCluster.NatGatewayId) + err = aliDeleteNatGateway(vpcClient, regionId, *ngwForCluster.NatGatewayId) + if err != nil { + cblogger.Debug("Failed to delete NAT Gateway: ID=", *ngwForCluster.NatGatewayId, ": ", err) + resultErr = fmt.Errorf("failed to delete NAT Gateway(ID=%s): %v", *ngwForCluster.NatGatewayId, err) + } + cblogger.Debug("Successfully deleted NAT Gateway: ID=", *ngwForCluster.NatGatewayId) + + resultErr = nil + eipAddressList, err := aliDescribeEipAddressesWithNat(vpcClient, regionId, *ngwForCluster.NatGatewayId) + if err != nil { + if resultErr != nil { + resultErr = fmt.Errorf("%v - no EIP with NAT Gateway(ID=%s): %v", resultErr, *ngwForCluster.NatGatewayId, err) + } else { + resultErr = fmt.Errorf("no EIP with NAT Gateway(ID=%s): %v", *ngwForCluster.NatGatewayId, err) + } + } else { + for _, eipAddr := range eipAddressList { + eipVpcId := "" + re := regexp.MustCompile(`\S*` + delimiterVpcId + `\S*`) + found := re.FindString(*eipAddr.Description) + if found != "" { + split := strings.Split(found, delimiterVpcId) + eipVpcId = split[1] + if strings.EqualFold(eipVpcId, vpcId) { + cblogger.Debug("Wait until EIP is Available: IP=", eipAddr.IpAddress) + waitUntilEipIsStatus(vpcClient, regionId, *eipAddr.AllocationId, "", "Available") + + cblogger.Debug("Request to release EIP: IP=", eipAddr.IpAddress) + err = aliReleaseEipAddress(vpcClient, regionId, *eipAddr.AllocationId) + if err != nil { + cblogger.Debug("Failed to release EIP: IP=", eipAddr.IpAddress, ": ", err) + } + } + } + } + } + + return resultErr +} + +func waitUntilClusterSecurityGroupIdIsExist(csClient *cs2015.Client, clusterId string) error { + apiCallCount := 0 + maxAPICallCount := 20 + + var waitingErr error + for { + cluster, err := aliDescribeClusterDetail(csClient, clusterId) + if err != nil { + maxAPICallCount = maxAPICallCount / 2 + } + if !strings.EqualFold(tea.StringValue(cluster.SecurityGroupId), "") { + return nil + } + apiCallCount++ + if apiCallCount >= maxAPICallCount { + waitingErr = fmt.Errorf("failed to get cluster's security group id: The maximum number of verification requests has been exceeded while waiting for availability of that resource") + break + } + time.Sleep(5 * time.Second) + cblogger.Info("Wait until cluster's security group id is exist") + } + + return waitingErr +} +*/ +/* + // + // Check whether if a nat gateway is created with the cluster or not + // + cblogger.Debug(fmt.Sprintf("Check if NAT Gateway is Automatically Created.")) + + tagKey := tagKeyAckAliyunCom + tagValue := tea.StringValue(clusterId) + ngwsWithTag, err := getInternetNatGatewaysWithTagInVpc(ach.VpcClient, regionId, vpcId, tagKey, tagValue) + if err != nil { + createErr = fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(createErr) + LoggingError(hiscallInfo, err) + return emptyClusterInfo, createErr + } + if len(ngwsWithTag) > 0 { + cblogger.Debug(fmt.Sprintf("NAT Gateway(%s) is Automatically Created.", tea.StringValue(ngwsWithTag[0].NatGatewayId))) + err = aliTagNatGateway(ach.VpcClient, regionId, tea.StringValue(ngwsWithTag[0].NatGatewayId), tagKeyCbSpiderPmksNatGateway, tagValueOwned) + if err != nil { + createErr = fmt.Errorf("Failed to Create Cluster: %v", err) + cblogger.Error(createErr) + LoggingError(hiscallInfo, createErr) + return emptyClusterInfo, createErr + } + } else { + cblogger.Debug(fmt.Sprintf("No Created NAT Gateway.")) + } +*/ diff --git a/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba/alibaba_api.go b/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba/alibaba_api.go deleted file mode 100644 index 42b9e6d5d..000000000 --- a/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba/alibaba_api.go +++ /dev/null @@ -1,389 +0,0 @@ -// Alibaba Driver of CB-Spider. -// The CB-Spider is a sub-Framework of the Cloud-Barista Multi-Cloud Project. -// The CB-Spider Mission is to connect all the clouds with a single interface. -// -// * Cloud-Barista: https://github.com/cloud-barista -// -// This is Alibaba Driver. -// -// by CB-Spider Team, 2022.09. - -package alibaba - -import ( - "strings" - - "github.com/aliyun/alibaba-cloud-sdk-go/sdk" - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials" - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" - vpc "github.com/aliyun/alibaba-cloud-sdk-go/services/vpc" -) - -func CreateCluster(access_key string, access_secret string, region_id string, body string) (string, error) { - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - panic(err) - } - - request := requests.NewCommonRequest() - - request.Method = "POST" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters" - request.Headers["Content-Type"] = "application/json" - - request.Content = []byte(body) - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func GetClusters(access_key string, access_secret string, region_id string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - panic(err) - } - - request := requests.NewCommonRequest() - - request.Method = "GET" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/api/v1/clusters" - request.Headers["Content-Type"] = "application/json" - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func GetCluster(access_key string, access_secret string, region_id string, cluster_id string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "GET" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id - request.Headers["Content-Type"] = "application/json" - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func GetClusterKubeConfig(access_key string, access_secret string, region_id string, cluster_id string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "GET" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id - request.PathPattern = "/k8s/" + cluster_id + "/user_config" - request.Headers["Content-Type"] = "application/json" - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func DeleteCluster(access_key string, access_secret string, region_id string, cluster_id string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - panic(err) - } - - request := requests.NewCommonRequest() - - request.Method = "DELETE" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id - request.Headers["Content-Type"] = "application/json" - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func CreateNodeGroup(access_key string, access_secret string, region_id string, cluster_id string, body string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "POST" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id + "/nodepools" - request.Headers["Content-Type"] = "application/json" - - request.Content = []byte(body) - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func ListNodeGroup(access_key string, access_secret string, region_id string, cluster_id string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "GET" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id + "/nodepools" - request.Headers["Content-Type"] = "application/json" - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -// package main - -// import ( -// "fmt" - -// "github.com/aliyun/alibaba-cloud-sdk-go/sdk" -// "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" -// ) - -func DescribeClusterNodes(access_key string, access_secret string, region_id string, cluster_id string, nodepool_id string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - request.Method = "GET" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id + "/nodes" - request.Headers["Content-Type"] = "application/json" - request.QueryParams["nodepool_id"] = nodepool_id - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func GetNodeGroup(access_key string, access_secret string, region_id string, cluster_id string, nodepool_id string) (string, error) { - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "GET" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id + "/nodepools/" + nodepool_id - request.Headers["Content-Type"] = "application/json" - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func DeleteNodeGroup(access_key string, access_secret string, region_id string, cluster_id string, nodepool_id string) (string, error) { - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "DELETE" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id + "/nodepools/" + nodepool_id - request.Headers["Content-Type"] = "application/json" - request.QueryParams["force"] = "true" - - // 노드그룹을 삭제 또는 오토스케일러 변경을 최초 호출하면 에러가 발생한다. - // 어떻게 하면 좋은가? - // 첫번째 호출에서 에러가 없으면 리턴! // 최초 호출이 아닌 경우 - // 첫번째 호출에서 에러 발생하면 한번 더 호출! // 최초 호출인 경우 - response, err := client.ProcessCommonRequest(request) - if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "cluster-autoscaler") { - response, err = client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - } else { - return "", err - } - } - - return response.GetHttpContentString(), nil -} - -func ModifyNodeGroup(access_key string, access_secret string, region_id string, cluster_id string, nodepool_id string, body string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "PUT" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/clusters/" + cluster_id + "/nodepools/" + nodepool_id - request.Headers["Content-Type"] = "application/json" - request.Content = []byte(body) - - // 노드그룹을 삭제 또는 오토스케일러 변경을 최초 호출하면 에러가 발생한다. - // 어떻게 하면 좋은가? - // 첫번째 호출에서 에러가 없으면 리턴! // 최초 호출이 아닌 경우 - // 첫번째 호출에서 에러 발생하면 한번 더 호출! // 최초 호출인 경우 - response, err := client.ProcessCommonRequest(request) - if err != nil { - if strings.Contains(strings.ToLower(err.Error()), "cluster-autoscaler") { - response, err = client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - } else { - return "", err - } - } - - return response.GetHttpContentString(), nil -} - -func UpgradeCluster(access_key string, access_secret string, region_id string, cluster_id string, body string) (string, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := sdk.NewClientWithOptions(region_id, config, credential) - if err != nil { - return "", err - } - - request := requests.NewCommonRequest() - - request.Method = "POST" - request.Scheme = "https" // https | http - request.Domain = "cs." + region_id + ".aliyuncs.com" - request.Version = "2015-12-15" - request.PathPattern = "/api/v2/clusters/" + cluster_id + "/upgrade" - request.Headers["Content-Type"] = "application/json" - - request.Content = []byte(body) - - response, err := client.ProcessCommonRequest(request) - if err != nil { - return "", err - } - - return response.GetHttpContentString(), nil -} - -func DescribeVSwitches(access_key string, access_secret string, region_id string, vpc_id string) (*vpc.DescribeVSwitchesResponse, error) { - - config := sdk.NewConfig() - credential := credentials.NewAccessKeyCredential(access_key, access_secret) - client, err := vpc.NewClientWithOptions(region_id, config, credential) - if err != nil { - return nil, err - } - - request := vpc.CreateDescribeVSwitchesRequest() - request.Scheme = "https" - request.VpcId = vpc_id - response, err := client.DescribeVSwitches(request) - if err != nil { - return nil, err - } - - return response, nil -} diff --git a/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba_api_test.go b/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba_api_test.go deleted file mode 100644 index bad940944..000000000 --- a/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba_api_test.go +++ /dev/null @@ -1,547 +0,0 @@ -// Alibaba Driver of CB-Spider. -// The CB-Spider is a sub-Framework of the Cloud-Barista Multi-Cloud Project. -// The CB-Spider Mission is to connect all the clouds with a single interface. -// -// * Cloud-Barista: https://github.com/cloud-barista -// -// This is Alibaba Driver. -// -// by CB-Spider Team, 2022.09. - -package main - -import ( - "encoding/json" - "os" - "testing" - - alibaba "github.com/cloud-barista/cb-spider/cloud-control-manager/cloud-driver/drivers/alibaba/utils/alibaba" -) - -var access_key string -var access_secret string -var region_id string - -func setup() { - println("setup") - access_key = os.Getenv("CLIENT_ID") - access_secret = os.Getenv("CLIENT_SECRET") - region_id = os.Getenv("REGION_ID") -} - -func shutdown() { - println("shutdown") -} - -func TestMain(m *testing.M) { - setup() - code := m.Run() - shutdown() - os.Exit(code) -} - -func TestCreateClusterOnly(t *testing.T) { - - t.Log("클러스터만 생성하기") - - body := `{ - "name": "cluster_2", - "region_id": "cn-beijing", - "cluster_type": "ManagedKubernetes", - "kubernetes_version": "1.22.10-aliyun.1", - "vpcid": "vpc-2zek5slojo5bh621ftnrg", - "container_cidr": "172.24.0.0/16", - "service_cidr": "172.23.0.0/16", - "num_of_nodes": 0, - "master_vswitch_ids": [ - "vsw-2ze0qpwcio7r5bx3nqbp1" - ] - }` - - result, err := alibaba.CreateCluster(access_key, access_secret, region_id, body) - if err != nil { - t.Errorf("Failed to create cluster: %v", err) - } - println(result) -} - -func TestCreateClusterWithNodeGroup(t *testing.T) { - - t.Log("클러스터 + 노드그룹(1) 생성하기") - - body := `{ - "name": "cluster_0", - "region_id": "cn-beijing", - "cluster_type": "ManagedKubernetes", - "kubernetes_version": "1.22.10-aliyun.1", - "vpcid": "vpc-2zek5slojo5bh621ftnrg", - "container_cidr": "172.21.0.0/16", - "service_cidr": "172.22.0.0/16", - "key_pair": "kp1", - "login_password": "$etri2ETRI", - "master_vswitch_ids": [ - "vsw-2ze0qpwcio7r5bx3nqbp1" - ], - "master_instance_types": [ - "ecs.g7ne.xlarge,ecs.c7.xlarge" - ], - "master_system_disk_category": "cloud_essd", - "master_system_disk_size": 120, - "num_of_nodes": 0, - "vswitch_ids": [ - "vsw-2ze0qpwcio7r5bx3nqbp1" - ], - "worker_vswitch_ids": [ - "vsw-2ze0qpwcio7r5bx3nqbp1" - ], - "worker_instance_types": [ - "ecs.g7ne.xlarge,ecs.c7.xlarge" - ], - "worker_system_disk_category": "cloud_essd", - "worker_system_disk_size": 120, - "worker_data_disks": [ - { - "category": "cloud_essd", - "size": "120" - } - ] - }` - - result, err := alibaba.CreateCluster(access_key, access_secret, region_id, body) - if err != nil { - t.Errorf("Failed to create cluster: %v", err) - } - println(result) -} - -func TestGetClusters(t *testing.T) { - result, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(result) -} - -func TestGetCluster(t *testing.T) { - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - println(cluster_id) - - cluster_json_str, err := alibaba.GetCluster(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to get cluster: %v", err) - } - println(cluster_json_str) - - cluster_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(cluster_json_str), &cluster_json_obj) - println(cluster_json_obj["cluster_id"].(string)) - - //"{\"api_server_endpoint\":\"https://47.74.22.109:6443\",\"dashboard_endpoint\":\"\",\"intranet_api_server_endpoint\":\"https://10.0.11.77:6443\"}" - println(cluster_json_obj["master_url"].(string)) - master_url_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(cluster_json_obj["master_url"].(string)), &master_url_json_obj) - println(master_url_json_obj["api_server_endpoint"].(string)) - } -} - -func TestGetClusterKubeConfig(t *testing.T) { - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - println(cluster_id) - - cluster_kube_config_json_str, err := alibaba.GetClusterKubeConfig(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to get cluster: %v", err) - } - println(cluster_kube_config_json_str) - - cluster_kube_config_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(cluster_kube_config_json_str), &cluster_kube_config_json_obj) - println(cluster_kube_config_json_obj["config"].(string)) - } -} - -func TestDeleteCluster(t *testing.T) { - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - println(cluster_id) - temp, err := alibaba.DeleteCluster(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to delete cluster: %v", err) - } - println(temp) - } -} - -func TestCreateNodeGroup(t *testing.T) { - - body := `{ - "auto_scaling": { - "enable": true, - "max_instances": 5, - "min_instances": 0 - }, - "kubernetes_config": { - "runtime": "containerd", - "runtime_version": "1.5.10" - }, - "nodepool_info": { - "name": "nodepoolx" - }, - "scaling_group": { - "instance_charge_type": "PostPaid", - "instance_types": [ - "ecs.c6.xlarge" - ], - "key_pair": "kp1", - "system_disk_category": "cloud_essd", - "system_disk_size": 70, - "vswitch_ids": [ - "vsw-2ze0qpwcio7r5bx3nqbp1" - ] - }, - "management": { - " enable":true - } - }` - - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - println(cluster_id) - - clusters_json_str, err := alibaba.CreateNodeGroup(access_key, access_secret, region_id, cluster_id, body) - if err != nil { - t.Errorf("Failed to create node group: %v", err) - } - println(clusters_json_str) - } - -} - -func TestCreateNodeGroup2(t *testing.T) { - - body := `{ - "nodepool_info": { - "name": "nodepoolx3" - }, - "auto_scaling": { - "enable": true, - "max_instances": 5, - "min_instances": 2 - }, - "scaling_group": { - "instance_types": ["ecs.c6.xlarge"], - "key_pair": "kp1", - "system_disk_category": "cloud_essd", - "system_disk_size": 70, - "vswitch_ids": ["vsw-2ze0qpwcio7r5bx3nqbp1"] - }, - "management": { - "enable":true - } - }` - - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - println(cluster_id) - - clusters_json_str, err := alibaba.CreateNodeGroup(access_key, access_secret, region_id, cluster_id, body) - if err != nil { - t.Errorf("Failed to create node group: %v", err) - } - println(clusters_json_str) - } - -} - -func TestListNodeGroup(t *testing.T) { - - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - - nodepools_json_str, err := alibaba.ListNodeGroup(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to list node group: %v", err) - } - println(nodepools_json_str) - nodepools_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(nodepools_json_str), &nodepools_json_obj) - nodepools := nodepools_json_obj["nodepools"].([]interface{}) - for _, v := range nodepools { - node_group_id := v.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) - println(node_group_id) - - // get node group - nodepool_json_str, err := alibaba.GetNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id) - if err != nil { - t.Errorf("Failed to get node group: %v", err) - } - println(nodepool_json_str) - } - } -} - -func TestGetNodeGroup(t *testing.T) { - - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - - nodepools_json_str, err := alibaba.ListNodeGroup(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to list node group: %v", err) - } - - println(nodepools_json_str) - nodepools_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(nodepools_json_str), &nodepools_json_obj) - nodepools := nodepools_json_obj["nodepools"].([]interface{}) - for _, v := range nodepools { - node_group_id := v.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) - println(node_group_id) - - // get node group - nodepool_json_str, err := alibaba.GetNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id) - if err != nil { - t.Errorf("Failed to get node group: %v", err) - } - println(nodepool_json_str) - } - } -} - -func TestSetNodeGroupAutoScaling(t *testing.T) { - // https://next.api.alibabacloud.com/api/CS/2015-12-15/ModifyClusterNodePool?sdkStyle=old¶ms={} - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - - nodepools_json_str, err := alibaba.ListNodeGroup(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to list node group: %v", err) - } - println(nodepools_json_str) - nodepools_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(nodepools_json_str), &nodepools_json_obj) - nodepools := nodepools_json_obj["nodepools"].([]interface{}) - for _, v := range nodepools { - node_group_id := v.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) - println(node_group_id) - - body := `{"auto_scaling":{"enable":false}}` - res, err := alibaba.ModifyNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id, body) - if err != nil { - t.Errorf("Failed to modify node group: %v", err) - } - println(res) - - body = `{"auto_scaling":{"enable":true}}` - res, err = alibaba.ModifyNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id, body) - if err != nil { - t.Errorf("Failed to modify node group: %v", err) - } - println(res) - // "{\"code\":\"ErrDefaultNodePool\",\"message\":\" Nodepool is default, cannot enable autoscaling\"}\n" - // default node pool: cannot enable autoscaling - // custom, managed node pool: can enable autoscaling - // https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/manage-node-pools - // body := `{"auto_scaling":{"enable":true,"max_instances":10 ,"min_instances":1}, "scaling_group":{"desired_size":5},"management":{" enable":true}}` - } - } -} - -func TestChangeNodeGroupScaling(t *testing.T) { - // modify (set auto scaling) on/off count - // body := `{"auto_scaling":{"enable":false}}` - // body := `{"auto_scaling":{"enable":true}}` - // body := `{"auto_scaling":{"max_instances":5,"min_instances":0},"scaling_group":{"desired_size":2}}` - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - - nodepools_json_str, err := alibaba.ListNodeGroup(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to list node group: %v", err) - } - println(nodepools_json_str) - nodepools_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(nodepools_json_str), &nodepools_json_obj) - nodepools := nodepools_json_obj["nodepools"].([]interface{}) - for _, v := range nodepools { - node_group_id := v.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) - println(node_group_id) - - body := `{"auto_scaling":{"max_instances":10,"min_instances":0},"scaling_group":{"desired_size":2}}` - res, err := alibaba.ModifyNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id, body) - if err != nil { - t.Errorf("Failed to modify node group: %v", err) - } - println(res) - - body = `{"auto_scaling":{"max_instances":3,"min_instances":1},"scaling_group":{"desired_size":1}}` - res, err = alibaba.ModifyNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id, body) - if err != nil { - t.Errorf("Failed to modify node group: %v", err) - } - println(res) - } - } -} - -func TestDeleteNodeGroup(t *testing.T) { - - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - - nodepools_json_str, err := alibaba.ListNodeGroup(access_key, access_secret, region_id, cluster_id) - if err != nil { - t.Errorf("Failed to list node group: %v", err) - } - println(nodepools_json_str) - nodepools_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(nodepools_json_str), &nodepools_json_obj) - nodepools := nodepools_json_obj["nodepools"].([]interface{}) - for _, v := range nodepools { - node_group_id := v.(map[string]interface{})["nodepool_info"].(map[string]interface{})["nodepool_id"].(string) - println(node_group_id) - - name := v.(map[string]interface{})["nodepool_info"].(map[string]interface{})["name"].(string) - println(name) - - if name == "nodepoolx" { - temp, err := alibaba.DeleteNodeGroup(access_key, access_secret, region_id, cluster_id, node_group_id) - if err != nil { - t.Errorf("Failed to delete node group: %v", err) - } - println(temp) - } - } - } -} - -func TestUpgradeCluster(t *testing.T) { - //https://www.alibabacloud.com/help/en/container-service-for-kubernetes/latest/kubernetes-1-22-release-notes#concept-2170736 - // 1.22.3-aliyun.1 - // { - // "next_version" : "1.22.3-aliyun.1" - // } - clusters_json_str, err := alibaba.GetClusters(access_key, access_secret, region_id) - if err != nil { - t.Errorf("Failed to get clusters: %v", err) - } - println(clusters_json_str) - - clusters_json_obj := make(map[string]interface{}) - json.Unmarshal([]byte(clusters_json_str), &clusters_json_obj) - - clusters := clusters_json_obj["clusters"].([]interface{}) - for _, v := range clusters { - cluster_id := v.(map[string]interface{})["cluster_id"].(string) - - version := `{"next_version" : "1.22.3-aliyun.1"}` - res, err := alibaba.UpgradeCluster(access_key, access_secret, region_id, cluster_id, version) - if err != nil { - t.Errorf("Failed to upgrade cluster: %v", err) - } - println(res) - } -} diff --git a/go.mod b/go.mod index 6b9459db9..6332d502a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 replace ( github.com/IBM/vpc-go-sdk/0.23.0 => github.com/IBM/vpc-go-sdk v0.23.0 + github.com/alibabacloud-go/ecs-20140526/v4 => github.com/alibabacloud-go/ecs-20140526/v4 v4.0.1 github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.1+incompatible ) @@ -16,7 +17,7 @@ require ( github.com/IBM/vpc-go-sdk/0.23.0 v0.23.0 github.com/Microsoft/go-winio v0.5.2 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 - github.com/aliyun/alibaba-cloud-sdk-go v1.61.1743 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.684 github.com/aws/aws-sdk-go v1.39.4 github.com/bramvdbogaerde/go-scp v1.0.0 github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c @@ -24,7 +25,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 - github.com/fsnotify/fsnotify v1.4.9 + github.com/fsnotify/fsnotify v1.7.0 github.com/go-resty/resty/v2 v2.6.0 github.com/gogo/protobuf v1.3.2 github.com/golang-jwt/jwt/v4 v4.0.0 @@ -33,10 +34,10 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/opentracing/opentracing-go v1.2.0 + github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b github.com/prometheus/client_golang v1.11.1 github.com/rs/xid v1.5.0 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.3 github.com/snowzach/rotatefilehook v0.0.0-20220211133110-53752135082d github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 @@ -46,8 +47,8 @@ require ( github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.493 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm v1.0.493 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc v1.0.206 - github.com/uber/jaeger-client-go v2.29.1+incompatible - golang.org/x/crypto v0.17.0 + github.com/uber/jaeger-client-go v2.30.0+incompatible + golang.org/x/crypto v0.20.0 golang.org/x/oauth2 v0.7.0 google.golang.org/api v0.114.0 google.golang.org/grpc v1.56.3 @@ -57,7 +58,13 @@ require ( require ( github.com/IBM/platform-services-go-sdk v0.30.0 + github.com/Masterminds/semver/v3 v3.2.1 github.com/NaverCloudPlatform/ncloud-sdk-go-v2 v1.6.6 + github.com/alibabacloud-go/cs-20151215/v4 v4.5.8 + github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.5 + github.com/alibabacloud-go/ecs-20140526/v4 v4.24.17 + github.com/alibabacloud-go/tea v1.2.2 + github.com/alibabacloud-go/vpc-20160428/v6 v6.4.0 github.com/cloud-barista/ktcloud-sdk-go v0.2.1-0.20240123114820-8684cfc5deeb github.com/cloud-barista/nhncloud-sdk-go v0.0.0-20240306113442-b0c008fddaf5 github.com/go-openapi/strfmt v0.21.3 @@ -72,11 +79,21 @@ require ( ) require ( + github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect + github.com/alibabacloud-go/debug v1.0.0 // indirect + github.com/alibabacloud-go/endpoint-util v1.1.1 // indirect + github.com/alibabacloud-go/openapi-util v0.1.0 // indirect + github.com/alibabacloud-go/tea-utils v1.4.5 // indirect + github.com/alibabacloud-go/tea-utils/v2 v2.0.4 // indirect + github.com/alibabacloud-go/tea-xml v1.1.3 // indirect + github.com/aliyun/credentials-go v1.3.2 // indirect + github.com/clbanning/mxj/v2 v2.7.0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/mattn/go-sqlite3 v1.14.17 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect ) require ( @@ -90,7 +107,6 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -154,15 +170,15 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/swaggo/files v0.0.0-20190704085106-630677cd5c14 // indirect - github.com/uber/jaeger-lib v2.4.0+incompatible // indirect + github.com/uber/jaeger-lib v2.4.1+incompatible // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.1 // indirect go.mongodb.org/mongo-driver v1.10.0 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect golang.org/x/tools v0.6.0 // indirect @@ -171,10 +187,10 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - gorm.io/driver/sqlite v1.5.2 - gorm.io/gorm v1.25.2 + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gorm.io/driver/sqlite v1.5.5 + gorm.io/gorm v1.25.7 gotest.tools/v3 v3.0.3 // indirect k8s.io/klog/v2 v2.30.0 // indirect k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect diff --git a/go.sum b/go.sum index 623e75919..e7f3735f0 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw= -github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/IBM/go-sdk-core/v5 v5.5.1/go.mod h1:Sn+z+qTDREQvCr+UFa22TqqfXNxx3o723y8GsfLV8e0= github.com/IBM/go-sdk-core/v5 v5.9.5/go.mod h1:YlOwV9LeuclmT/qi/LAK2AsobbAP42veV0j68/rlZsE= github.com/IBM/go-sdk-core/v5 v5.10.2 h1:bfqhYNwwpJ3zJQSYpF3umhmRIKaa762itvJkTAWCCLU= @@ -92,6 +92,8 @@ github.com/IBM/vpc-go-sdk v0.23.0 h1:C26g02pPtTEWyLHNKK0mlFdpHeb9+noku9Q6qJLmdkc github.com/IBM/vpc-go-sdk v0.23.0/go.mod h1:YPyIfI+/qhPqlYp+I7dyx2U1GLcXgp/jzVvsZfUH4y8= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -101,14 +103,51 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1743 h1:CYqFCr/akroZIHoFpohoPoPMQ3vq5OxYwKnygUzKRl4= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1743/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= +github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 h1:iC9YFYKDGEy3n/FtqJnOkZsene9olVspKmkX5A2YBEo= +github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc= +github.com/alibabacloud-go/cs-20151215/v4 v4.5.8 h1:tzhBIwMFx+VNHWq6hHswXw3Jvh+Qr/Mg2u7Sbw8BkUo= +github.com/alibabacloud-go/cs-20151215/v4 v4.5.8/go.mod h1:hXoCTv/6PMsi8nLjY8x2/iUhPX0/LRSixVWyL/GthSs= +github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.5 h1:yyolbgHfV2Tp91vMjO/CF5aOxKG+UgdVAeUoloEQI3E= +github.com/alibabacloud-go/darabonba-openapi/v2 v2.0.5/go.mod h1:kUe8JqFmoVU7lfBauaDD5taFaW7mBI+xVsyHutYtabg= +github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68/go.mod h1:6pb/Qy8c+lqua8cFpEy7g39NRRqOWc3rOwAy8m5Y2BY= +github.com/alibabacloud-go/debug v1.0.0 h1:3eIEQWfay1fB24PQIEzXAswlVJtdQok8f3EVN5VrBnA= +github.com/alibabacloud-go/debug v1.0.0/go.mod h1:8gfgZCCAC3+SCzjWtY053FrOcd4/qlH6IHTI4QyICOc= +github.com/alibabacloud-go/ecs-20140526/v4 v4.0.1 h1:/1le0GOdj9yNChRlP5oSbRMFR2qEDSCfVWja2yRBg9w= +github.com/alibabacloud-go/ecs-20140526/v4 v4.0.1/go.mod h1:/rwBz+SX1rT5QBlZYJiRchDr+jLMiHEaIxyd88BPhxs= +github.com/alibabacloud-go/endpoint-util v1.1.0/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE= +github.com/alibabacloud-go/endpoint-util v1.1.1 h1:ZkBv2/jnghxtU0p+upSU0GGzW1VL9GQdZO3mcSUTUy8= +github.com/alibabacloud-go/endpoint-util v1.1.1/go.mod h1:O5FuCALmCKs2Ff7JFJMudHs0I5EBgecXXxZRyswlEjE= +github.com/alibabacloud-go/openapi-util v0.1.0 h1:0z75cIULkDrdEhkLWgi9tnLe+KhAFE/r5Pb3312/eAY= +github.com/alibabacloud-go/openapi-util v0.1.0/go.mod h1:sQuElr4ywwFRlCCberQwKRFhRzIyG4QTP/P4y1CJ6Ws= +github.com/alibabacloud-go/tea v1.1.0/go.mod h1:IkGyUSX4Ba1V+k4pCtJUc6jDpZLFph9QMy2VUPTwukg= +github.com/alibabacloud-go/tea v1.1.7/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.8/go.mod h1:/tmnEaQMyb4Ky1/5D+SE1BAsa5zj/KeGOFfwYm3N/p4= +github.com/alibabacloud-go/tea v1.1.17/go.mod h1:nXxjm6CIFkBhwW4FQkNrolwbfon8Svy6cujmKFUq98A= +github.com/alibabacloud-go/tea v1.2.1/go.mod h1:qbzof29bM/IFhLMtJPrgTGK3eauV5J2wSyEUo4OEmnA= +github.com/alibabacloud-go/tea v1.2.2 h1:aTsR6Rl3ANWPfqeQugPglfurloyBJY85eFy7Gc1+8oU= +github.com/alibabacloud-go/tea v1.2.2/go.mod h1:CF3vOzEMAG+bR4WOql8gc2G9H3EkH3ZLAQdpmpXMgwk= +github.com/alibabacloud-go/tea-utils v1.3.1/go.mod h1:EI/o33aBfj3hETm4RLiAxF/ThQdSngxrpF8rKUDJjPE= +github.com/alibabacloud-go/tea-utils v1.4.5 h1:h0/6Xd2f3bPE4XHTvkpjwxowIwRCJAJOqY6Eq8f3zfA= +github.com/alibabacloud-go/tea-utils v1.4.5/go.mod h1:KNcT0oXlZZxOXINnZBs6YvgOd5aYp9U67G+E3R8fcQw= +github.com/alibabacloud-go/tea-utils/v2 v2.0.4 h1:SoFgjJuO7pze88j9RBJNbKb7AgTS52O+J5ITxc00lCs= +github.com/alibabacloud-go/tea-utils/v2 v2.0.4/go.mod h1:sj1PbjPodAVTqGTA3olprfeeqqmwD0A5OQz94o9EuXQ= +github.com/alibabacloud-go/tea-xml v1.1.3 h1:7LYnm+JbOq2B+T/B0fHC4Ies4/FofC4zHzYtqw7dgt0= +github.com/alibabacloud-go/tea-xml v1.1.3/go.mod h1:Rq08vgCcCAjHyRi/M7xlHKUykZCEtyBy9+DPF6GgEu8= +github.com/alibabacloud-go/vpc-20160428/v6 v6.4.0 h1:guVystsjRoYjwCxLRxdND/fGoMjWHdMqmqxbxzD9ILw= +github.com/alibabacloud-go/vpc-20160428/v6 v6.4.0/go.mod h1:MfCvuRMtUrZe7H62t7pRqeb9Mp60C0flhT5VAkvMkV0= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.684 h1:7zROQ8F1/iNCt8e0Lguk2xJTp31rPPku7X4527XRUHg= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.684/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= +github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6qTJya2bDq4X1Tw= +github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= +github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g= +github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -136,6 +175,9 @@ github.com/chyeh/pubip v0.0.0-20170203095919-b7e679cf541c/go.mod h1:C7ma6h458jTW github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clbanning/mxj/v2 v2.5.5/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= +github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= +github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloud-barista/cb-log v0.8.0 h1:ArWCs1EgpoD3ZnBgcC4cAw5ufI/JHmFKfJswlv4whgk= github.com/cloud-barista/cb-log v0.8.0/go.mod h1:nGgfTFMPwl1MpCO3FBjexUkNdOYA0BNJoyM9Pd0lMms= @@ -178,12 +220,14 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -265,6 +309,7 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0 h1:RAqyYixv1p7uEnocuy8P1nru5wprCh/MH2BIlW5z5/o= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -351,6 +396,7 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy github.com/gophercloud/gophercloud v1.3.0 h1:RUKyCMiZoQR3VlVR5E3K7PK1AC3/qppsWYo6dtBiqs8= github.com/gophercloud/gophercloud v1.3.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= @@ -428,6 +474,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -474,8 +521,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -540,8 +587,8 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -594,9 +641,10 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/snowzach/rotatefilehook v0.0.0-20220211133110-53752135082d h1:4660u5vJtsyrn3QwJNfESwCws+TM1CMhRn123xjVyQ8= github.com/snowzach/rotatefilehook v0.0.0-20220211133110-53752135082d/go.mod h1:ZLVe3VfhAuMYLYWliGEydMBoRnfib8EFSqkBYu1ck9E= @@ -619,6 +667,7 @@ github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -657,10 +706,13 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc v1.0.206/go.mod h1: github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/uber/jaeger-client-go v2.29.1+incompatible h1:R9ec3zO3sGpzs0abd43Y+fBZRJ9uiH6lXyR/+u6brW4= -github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ= -github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= +github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -675,6 +727,7 @@ github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgk github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.30/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -697,6 +750,7 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -712,17 +766,26 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -732,6 +795,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -794,6 +858,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -809,9 +874,14 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -840,6 +910,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -877,6 +948,7 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -914,15 +986,23 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -933,6 +1013,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -941,8 +1024,10 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 h1:GZokNIeuVkl3aZHJchRrr13WCsols02MLUcz1U9is6M= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -982,6 +1067,7 @@ golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1007,6 +1093,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1134,11 +1224,13 @@ gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+a gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1154,12 +1246,13 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc= -gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4= -gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho= -gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k= +gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= +gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= +gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= @@ -1186,6 +1279,7 @@ k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=