Skip to content

Commit

Permalink
Remove resource group concept in azure for cb-sp v0.8.11
Browse files Browse the repository at this point in the history
  • Loading branch information
@cb-github-robot
cb-github-robot authored Apr 16, 2024
2 parents 6c13d8e + f0c0fc4 commit b91e74a
Show file tree
Hide file tree
Showing 11 changed files with 281 additions and 548 deletions.
265 changes: 22 additions & 243 deletions assets/cloudimage.csv
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions assets/cloudlocation.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ tencent,ap-chongqing,Southwest China (Chongqing),29.431586,106.912251
tencent,ap-hongkong,Hong Kong,22.396228,114.108497
ktcloud,kr-central-a,Cheon-an A - South Korea,36.812483,127.107005
ktcloud,kr-central-b,Cheon-an B - South Korea,36.832583,127.157205
ktcloud,kr-seoul-m,Seoul M- South Korea,37.566535,126.977969
ktcloud,kr-seoul-m2,Seoul M2 - South Korea,37.466535,126.877969
ktcloud,kor-seoul-m,Seoul M- South Korea,37.566535,126.977969
ktcloud,kor-seoul-m2,Seoul M2 - South Korea,37.466535,126.877969
ktcloud,kr-kimhae,Kimhae - South Korea,35.228545,128.889352
ktcloudvpc,us,LA - US,34.054234,-118.243685
ibm,us-south,Dallas (US South),32.776664,-96.796988
Expand Down
4 changes: 2 additions & 2 deletions assets/cloudspec.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ NHNCLOUD,nhncloud-kr2,m2.c4m8,,,,,,,,,,,,default,default
NHNCLOUD,nhncloud-jp1,m2.c4m8,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kor-central-a,d3530ad2-462b-43ad-97d5-e1087b952b7d!_disk20GB,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kor-central-b,385df834-6cf7-485b-bca8-de4d4f59b0b0,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kr-seoul-m,385df834-6cf7-485b-bca8-de4d4f59b0b0,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kr-seoul-m2,753c80b0-b03f-4a91-98a7-6b11233f85a8,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kor-seoul-m,385df834-6cf7-485b-bca8-de4d4f59b0b0,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kor-seoul-m2,753c80b0-b03f-4a91-98a7-6b11233f85a8,,,,,,,,,,,,default,default
KTCLOUD,ktcloud-kr-kimhae,385df834-6cf7-485b-bca8-de4d4f59b0b0,,,,,,,,,,,,default,default
KTCLOUDVPC,ktcloudvpc-kr-dx-m1,4x16.itl,,,,,,,,,,,,default,default
8 changes: 4 additions & 4 deletions scripts/initMultiCloudEnv.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ start_time=$(date +%s)


# Execute the load-common-resource script and capture its output
EXPECTED_DURATION=300 # 5 minutes
EXPECTED_DURATION=240 # 4 minutes
progress_time=$(date +%s)

"$CBTUMBLEBUG_ROOT"/src/testclient/scripts/2.configureTumblebug/load-common-resource.sh -n tb > initTmp.out &
Expand Down Expand Up @@ -86,7 +86,7 @@ done

echo ""
echo ""
echo " Done"
echo "Done"
output=$(<initTmp.out)
rm initTmp.out

Expand Down Expand Up @@ -127,7 +127,7 @@ done <<< "$output"

# Display the counts
echo ""
echo "- Image Success count: $successImageCount (Failed count: $failedImageCount)"
echo "- Spec Success count: $successSpecCount (Failed count: $failedSpecCount)"
echo "- Image Success count: $successImageCount (not registered: $failedImageCount)"
echo "- Spec Success count: $successSpecCount (not registered: $failedSpecCount)"
echo ""
echo "Total duration: $duration seconds."
2 changes: 1 addition & 1 deletion scripts/runSpider.sh
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash

CONTAINER_NAME_READ="CB-Spider"
CONTAINER_VERSION="0.8.10"
CONTAINER_VERSION="0.8.11"
CONTAINER_PORT="-p 1024:1024 -p 2048:2048"
CONTAINER_DATA_PATH="/root/go/src/github.com/cloud-barista/cb-spider/meta_db"

Expand Down
197 changes: 102 additions & 95 deletions src/core/mcir/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1421,6 +1421,8 @@ func LoadCommonResource() (common.IdList, error) {

// LookupSpecList and LookupImageList of all connections in parallel
var specMap sync.Map
// ignoreConnectionMap is used to store connection names that failed to lookup specs
var ignoreConnectionMap sync.Map
startTime := time.Now()
var wg sync.WaitGroup
for _, connConfig := range connectionList.Connectionconfig {
Expand All @@ -1430,9 +1432,10 @@ func LoadCommonResource() (common.IdList, error) {
specsInConnection, err := LookupSpecList(connConfig.ConfigName)
if err != nil {
log.Error().Err(err).Msgf("Cannot LookupSpecList in %s", connConfig.ConfigName)
ignoreConnectionMap.Store(connConfig.ConfigName, err)
return
}
log.Info().Msgf("[%s]#Spec: %d", connConfig.ConfigName, len(specsInConnection.Vmspec))
log.Info().Msgf("[%s] #Spec: %d", connConfig.ConfigName, len(specsInConnection.Vmspec))
for _, spec := range specsInConnection.Vmspec {
key := GenSpecMapKey(connConfig.RegionName, spec.Name)
// instead of connConfig.RegionName, spec.Region will be used in the future
Expand All @@ -1454,7 +1457,7 @@ func LoadCommonResource() (common.IdList, error) {
// if err != nil {
// log.Error().Err(err).Msgf("Cannot LookupImageList in %s", cc.ConfigName)
// } else {
// log.Info().Msgf("[%s]#Image: %d", cc.ConfigName, len(imagesInConnection.Image))
// log.Info().Msgf("[%s] #Image: %d", cc.ConfigName, len(imagesInConnection.Image))
// }
// }(connConfig)
// }
Expand Down Expand Up @@ -1557,85 +1560,89 @@ func LoadCommonResource() (common.IdList, error) {
providerName := row[0]
specReqTmp.ConnectionName = row[1]
specReqTmp.CspSpecName = row[2]
// Give a name for spec object by combining ConnectionName and CspSpecName
// To avoid naming-rule violation, modify the string
specReqTmp.Name = specReqTmp.ConnectionName + "-" + specReqTmp.CspSpecName
specReqTmp.Name = ToNamingRuleCompatible(specReqTmp.Name)
specInfoId := specReqTmp.Name
rootDiskType := row[14]
rootDiskSize := row[15]
specReqTmp.Description = "Common Spec Resource"

regionName := specReqTmp.ConnectionName // assume regionName is the same as connectionName
regiesteredStatus = ""

var errRegisterSpec error

log.Trace().Msgf("[%d] register Common Spec: %s", i, specReqTmp.Name)

// Register Spec object
searchKey := GenSpecMapKey(regionName, specReqTmp.CspSpecName)
value, ok := specMap.Load(searchKey)
if ok {
spiderSpec := value.(SpiderSpecInfo)
//log.Info().Msgf("Found spec in the map: %s", spiderSpec.Name)
tumblebugSpec, errConvert := ConvertSpiderSpecToTumblebugSpec(spiderSpec)
if errConvert != nil {
log.Error().Err(errConvert).Msg("Cannot ConvertSpiderSpecToTumblebugSpec")
}

tumblebugSpec.Name = specInfoId
tumblebugSpec.ConnectionName = specReqTmp.ConnectionName
_, errRegisterSpec = RegisterSpecWithInfo(common.SystemCommonNs, &tumblebugSpec, true)
if errRegisterSpec != nil {
log.Info().Err(errRegisterSpec).Msg("RegisterSpec WithInfo failed")
}
_, ok := ignoreConnectionMap.Load(specReqTmp.ConnectionName)
if !ok {
// Give a name for spec object by combining ConnectionName and CspSpecName
// To avoid naming-rule violation, modify the string
specReqTmp.Name = specReqTmp.ConnectionName + "-" + specReqTmp.CspSpecName
specReqTmp.Name = ToNamingRuleCompatible(specReqTmp.Name)
specInfoId := specReqTmp.Name
rootDiskType := row[14]
rootDiskSize := row[15]
specReqTmp.Description = "Common Spec Resource"

regionName := specReqTmp.ConnectionName // assume regionName is the same as connectionName
regiesteredStatus = ""

} else {
errRegisterSpec = fmt.Errorf("Not Found spec from the fetched spec list: %s", searchKey)
log.Info().Err(errRegisterSpec).Msgf("")
// _, errRegisterSpec = RegisterSpecWithCspSpecName(common.SystemCommonNs, &specReqTmp, true)
// if errRegisterSpec != nil {
// log.Error().Err(errRegisterSpec).Msg("RegisterSpec WithCspSpecName failed")
// }
}
var errRegisterSpec error

if errRegisterSpec != nil {
regiesteredStatus += " [Failed] " + errRegisterSpec.Error()
} else {
// Update registered Spec object with givn info from asset file
// Update registered Spec object with Cost info
costPerHour, err2 := strconv.ParseFloat(strings.ReplaceAll(row[3], " ", ""), 32)
if err2 != nil {
log.Error().Err(err2).Msg("Not valid CostPerHour value in the asset")
costPerHour = 99999999.9
}
evaluationScore01, err2 := strconv.ParseFloat(strings.ReplaceAll(row[4], " ", ""), 32)
if err2 != nil {
log.Error().Err(err2).Msg("Not valid evaluationScore01 value in the asset")
evaluationScore01 = -99.9
log.Trace().Msgf("[%d] register Common Spec: %s", i, specReqTmp.Name)

// Register Spec object
searchKey := GenSpecMapKey(regionName, specReqTmp.CspSpecName)
value, ok := specMap.Load(searchKey)
if ok {
spiderSpec := value.(SpiderSpecInfo)
//log.Info().Msgf("Found spec in the map: %s", spiderSpec.Name)
tumblebugSpec, errConvert := ConvertSpiderSpecToTumblebugSpec(spiderSpec)
if errConvert != nil {
log.Error().Err(errConvert).Msg("Cannot ConvertSpiderSpecToTumblebugSpec")
}

tumblebugSpec.Name = specInfoId
tumblebugSpec.ConnectionName = specReqTmp.ConnectionName
_, errRegisterSpec = RegisterSpecWithInfo(common.SystemCommonNs, &tumblebugSpec, true)
if errRegisterSpec != nil {
log.Info().Err(errRegisterSpec).Msg("RegisterSpec WithInfo failed")
}

} else {
errRegisterSpec = fmt.Errorf("Not Found spec from the fetched spec list: %s", searchKey)
log.Info().Msgf(errRegisterSpec.Error())
// _, errRegisterSpec = RegisterSpecWithCspSpecName(common.SystemCommonNs, &specReqTmp, true)
// if errRegisterSpec != nil {
// log.Error().Err(errRegisterSpec).Msg("RegisterSpec WithCspSpecName failed")
// }
}
specUpdateRequest :=
TbSpecInfo{
ProviderName: providerName,
RegionName: regionName,
CostPerHour: float32(costPerHour),
RootDiskType: rootDiskType,
RootDiskSize: rootDiskSize,
EvaluationScore01: float32(evaluationScore01),

if errRegisterSpec != nil {
regiesteredStatus += " [Failed] " + errRegisterSpec.Error()
} else {
// Update registered Spec object with givn info from asset file
// Update registered Spec object with Cost info
costPerHour, err2 := strconv.ParseFloat(strings.ReplaceAll(row[3], " ", ""), 32)
if err2 != nil {
log.Error().Err(err2).Msg("Not valid CostPerHour value in the asset")
costPerHour = 99999999.9
}
evaluationScore01, err2 := strconv.ParseFloat(strings.ReplaceAll(row[4], " ", ""), 32)
if err2 != nil {
log.Error().Err(err2).Msg("Not valid evaluationScore01 value in the asset")
evaluationScore01 = -99.9
}
specUpdateRequest :=
TbSpecInfo{
ProviderName: providerName,
RegionName: regionName,
CostPerHour: float32(costPerHour),
RootDiskType: rootDiskType,
RootDiskSize: rootDiskSize,
EvaluationScore01: float32(evaluationScore01),
}

_, err3 := UpdateSpec(common.SystemCommonNs, specInfoId, specUpdateRequest)
if err3 != nil {
log.Error().Err(err3).Msg("UpdateSpec failed")
regiesteredStatus += " [Failed] " + err3.Error()
_, err3 := UpdateSpec(common.SystemCommonNs, specInfoId, specUpdateRequest)
if err3 != nil {
log.Error().Err(err3).Msg("UpdateSpec failed")
regiesteredStatus += " [Failed] " + err3.Error()
}
//fmt.Printf("[%d] Registered Common Spec\n", i)
//common.PrintJsonPretty(updatedSpecInfo)
}
//fmt.Printf("[%d] Registered Common Spec\n", i)
//common.PrintJsonPretty(updatedSpecInfo)
}

regiesteredIds.AddItem(common.StrSpec + ": " + specInfoId + regiesteredStatus)
// }(i, row, lenSpecs)
regiesteredIds.AddItem(common.StrSpec + ": " + specInfoId + regiesteredStatus)
// }(i, row, lenSpecs)
}
}
wait.Wait()
}(rowsSpec)
Expand All @@ -1650,35 +1657,35 @@ func LoadCommonResource() (common.IdList, error) {
// goroutine
go func(i int, row []string, lenImages int) {
defer wait.Done()
// RandomSleep for safe parallel executions
common.RandomSleep(0, lenImages/10)

imageReqTmp := TbImageReq{}
// row0: ProviderName
// row1: connectionName
// row2: cspImageId
// row3: OsType
imageReqTmp.ConnectionName = row[1]
imageReqTmp.CspImageId = row[2]
osType := strings.ReplaceAll(row[3], " ", "")
// Give a name for spec object by combining ConnectionName and OsType
// To avoid naming-rule violation, modify the string
imageReqTmp.Name = imageReqTmp.ConnectionName + "-" + osType
imageReqTmp.Name = ToNamingRuleCompatible(imageReqTmp.Name)
imageInfoId := imageReqTmp.Name
imageReqTmp.Description = "Common Image Resource"
_, ok := ignoreConnectionMap.Load(imageReqTmp.ConnectionName)
if !ok {
// RandomSleep for safe parallel executions
common.RandomSleep(0, lenImages/8)

log.Trace().Msgf("[%d] register Common Image: %s", i, imageReqTmp.Name)
imageReqTmp.CspImageId = row[2]
osType := strings.ReplaceAll(row[3], " ", "")
// Give a name for spec object by combining ConnectionName and OsType
// To avoid naming-rule violation, modify the string
imageReqTmp.Name = imageReqTmp.ConnectionName + "-" + osType
imageReqTmp.Name = ToNamingRuleCompatible(imageReqTmp.Name)
imageInfoId := imageReqTmp.Name
imageReqTmp.Description = "Common Image Resource"

log.Trace().Msgf("[%d] register Common Image: %s", i, imageReqTmp.Name)

// Register Spec object
regiesteredStatus = ""

// Register Spec object
regiesteredStatus = ""
_, err := common.GetConnConfig(imageReqTmp.ConnectionName)
if err != nil {
log.Info().Err(err).Msgf("[%s] Cannot GetConnConfig ", imageReqTmp.ConnectionName)
regiesteredStatus += " [Failed] " + err.Error()
} else {
_, err1 := RegisterImageWithId(common.SystemCommonNs, &imageReqTmp, true)
if err1 != nil {
log.Info().Err(err1).Msg("")
log.Info().Msg(err1.Error())
regiesteredStatus += " [Failed] " + err1.Error()
} else {
// Update registered image object with OsType info
Expand All @@ -1689,10 +1696,10 @@ func LoadCommonResource() (common.IdList, error) {
regiesteredStatus += " [Failed] " + err2.Error()
}
}
}

//regiesteredStatus = strings.Replace(regiesteredStatus, "\\", "", -1)
regiesteredIds.AddItem(common.StrImage + ": " + imageInfoId + regiesteredStatus)
//regiesteredStatus = strings.Replace(regiesteredStatus, "\\", "", -1)
regiesteredIds.AddItem(common.StrImage + ": " + imageInfoId + regiesteredStatus)
}
}(i, row, lenImages)
}
wait.Wait()
Expand Down
Loading

0 comments on commit b91e74a

Please sign in to comment.