Skip to content

Commit

Permalink
Template and parameters deployed on 4-2-2024 10:6:25, based on the co…
Browse files Browse the repository at this point in the history 
…llaboration branch's commit ID: 0bd0394
  • Loading branch information
@tobiasny
tobiasny committed Apr 2, 2024
1 parent 4de092a commit 0b2adf5
Showing 1 changed file with 113 additions and 23 deletions.
136 changes: 113 additions & 23 deletions s037-cost-management/TemplateForWorkspace.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25257,6 +25257,75 @@
},
"dependsOn": []
},
{
"name": "[concat(parameters('workspaceName'), '/2024-03-25 Cost Query Sub-RG')]",
"type": "Microsoft.Synapse/workspaces/sqlscripts",
"apiVersion": "2019-06-01-preview",
"properties": {
"folder": {
"name": "ACDC"
},
"content": {
"query": "\nSELECT\n --Top 100\n upper(SubscriptionName) as SubscriptionName\n ,upper(ResourceGroup) as ResourceGroup\n --,CostInBillingCurrency as Cost\n --,*\n ,ActiveWBS\n ,CostAllocationType\n ,sum(CostInBillingCurrency) as Cost\n --,*\nFROM\n OPENROWSET(\n BULK 'https://s037costmgmt.dfs.core.windows.net/usage/exports/monthly/ACMMonthlyAmortizedCost/20240201-20240229/Extended_v3_ACMMonthlyAmortizedCost_20240201-20240229.parquet/**',\n FORMAT = 'PARQUET'\n ) AS [result]\n where \n --upper(ActiveWBS) <> 'C.TDI.IT.00010' and\n --upper(ActiveWBS) = 'E.RDI.10512' and\n upper(SubscriptionName) like 'S913-AURORA-NON-PRODUCTION'\n and lower(ResourceGroup) = 'mc_dev-aurora-ngrm-00_dev-aurora-ngrm-00_northeurope'\n group by upper(SubscriptionName), upper(ResourceGroup), ActiveWBS, CostAllocationType\n\n ",
"metadata": {
"language": "sql"
},
"currentConnection": {
"databaseName": "costmgmt",
"poolName": "Built-in"
},
"resultLimit": -1
},
"type": "SqlQuery"
},
"dependsOn": []
},
{
"name": "[concat(parameters('workspaceName'), '/2024-03-25 Cost Query WBS')]",
"type": "Microsoft.Synapse/workspaces/sqlscripts",
"apiVersion": "2019-06-01-preview",
"properties": {
"folder": {
"name": "ACDC"
},
"content": {
"query": "-- This is auto-generated code\nSELECT\n --TOP 100 \n *\nFROM\n OPENROWSET(\n BULK 'https://omniaacdcdlsprod.dfs.core.windows.net/costreports/monthly/2024/02/omnia_acdc_cost_details_with_sap_response.csv',\n FORMAT = 'CSV',\n        PARSER_VERSION = '2.0',\n HEADER_ROW=TRUE\n ) AS [result]\n --where WBS = 'C.TDI.IT.00010'\n where WBS = 'E.RDI.10512'\n",
"metadata": {
"language": "sql"
},
"currentConnection": {
"databaseName": "costmgmt",
"poolName": "Built-in"
},
"resultLimit": -1
},
"type": "SqlQuery"
},
"dependsOn": []
},
{
"name": "[concat(parameters('workspaceName'), '/2024-03-25 Cost Query excl WBS Tag')]",
"type": "Microsoft.Synapse/workspaces/sqlscripts",
"apiVersion": "2019-06-01-preview",
"properties": {
"folder": {
"name": "ACDC"
},
"content": {
"query": "\nSELECT\n --upper(SubscriptionName) as SubscriptionName\n --,upper(ResourceGroup) as ResourceGroup\n --,CostInBillingCurrency as Cost\n --,Tags\n sum(CostInBillingCurrency)\nFROM\n OPENROWSET(\n BULK 'https://s037costmgmt.dfs.core.windows.net/usage/exports/monthly/ACMMonthlyAmortizedCost/20240201-20240229/ACMMonthlyAmortizedCost_20240201-20240229.parquet',\n FORMAT = 'PARQUET'\n ) AS [result]\n where \n upper(Tags) not like N'%\"COSTALLOCATIONTYPE\": \"WBS\"%' AND \n upper(SubscriptionName) like 'S913-AURORA-NON-PRODUCTION'\n and lower(ResourceGroup) = 'mc_dev-aurora-ngrm-00_dev-aurora-ngrm-00_northeurope'\n",
"metadata": {
"language": "sql"
},
"currentConnection": {
"databaseName": "costmgmt",
"poolName": "Built-in"
},
"resultLimit": -1
},
"type": "SqlQuery"
},
"dependsOn": []
},
{
"name": "[concat(parameters('workspaceName'), '/All non RI VMs')]",
"type": "Microsoft.Synapse/workspaces/sqlscripts",
Expand Down Expand Up @@ -28342,6 +28411,29 @@
},
"dependsOn": []
},
{
"name": "[concat(parameters('workspaceName'), '/Sentinel Pricing')]",
"type": "Microsoft.Synapse/workspaces/sqlscripts",
"apiVersion": "2019-06-01-preview",
"properties": {
"folder": {
"name": "Ad Hoc Cost Report Requests"
},
"content": {
"query": "-- This is auto-generated code\nSELECT\n MeterName\n ,UnitPrice\n ,UnitOfMeasure\n ,PartNumber\n ,BasePrice\n ,MarketPrice\n ,MeterType\n ,MeterCategory\n ,ServiceFamily\n ,Product\n ,MeterRegion\n\n --distinct(ServiceFamily)\nFROM\n OPENROWSET(\n BULK 'https://s037costmgmt.dfs.core.windows.net/usage/pricesheet/portal-export/pricesheet-latest/*.parquet',\n FORMAT = 'PARQUET'\n ) AS [result]\n where \n MeterCategory = 'Sentinel'\n and OfferID = 'MS-AZR-0017P'\n and MeterRegion = 'EU West'\n and MeterName not like '%Classic%'\n order by UnitPrice\n \n",
"metadata": {
"language": "sql"
},
"currentConnection": {
"databaseName": "costmgmt",
"poolName": "Built-in"
},
"resultLimit": -1
},
"type": "SqlQuery"
},
"dependsOn": []
},
{
"name": "[concat(parameters('workspaceName'), '/SingularMonthly')]",
"type": "Microsoft.Synapse/workspaces/sqlscripts",
Expand Down Expand Up @@ -36288,7 +36380,7 @@
"spark.dynamicAllocation.enabled": "true",
"spark.dynamicAllocation.minExecutors": "1",
"spark.dynamicAllocation.maxExecutors": "4",
"spark.autotune.trackingId": "a569f42a-842d-4574-85c1-81c335059130"
"spark.autotune.trackingId": "178a4842-4de1-4880-b84e-9c1d23f3aae9"
}
},
"metadata": {
Expand All @@ -36313,8 +36405,7 @@
"sparkVersion": "3.3",
"nodeCount": 3,
"cores": 16,
"memory": 112,
"automaticScaleJobs": true
"memory": 112
},
"sessionKeepAliveTimeout": 30
},
Expand Down Expand Up @@ -36701,7 +36792,7 @@
"spark.dynamicAllocation.enabled": "true",
"spark.dynamicAllocation.minExecutors": "1",
"spark.dynamicAllocation.maxExecutors": "4",
"spark.autotune.trackingId": "5bf02eec-dc2a-47b6-a60d-0d03909705ed"
"spark.autotune.trackingId": "fc52e4bd-3998-488c-9a0f-53bd61d1b6a3"
}
},
"metadata": {
Expand All @@ -36726,8 +36817,7 @@
"sparkVersion": "3.3",
"nodeCount": 3,
"cores": 16,
"memory": 112,
"automaticScaleJobs": true
"memory": 112
},
"sessionKeepAliveTimeout": 30
},
Expand Down Expand Up @@ -43003,7 +43093,7 @@
"spark.dynamicAllocation.enabled": "true",
"spark.dynamicAllocation.minExecutors": "1",
"spark.dynamicAllocation.maxExecutors": "5",
"spark.autotune.trackingId": "6b3d1fdc-3ee4-4e9c-9f70-a7f21e686b55"
"spark.autotune.trackingId": "46341c8e-a04b-4e8f-a9af-2832cc352e0c"
}
},
"metadata": {
Expand Down Expand Up @@ -43366,13 +43456,16 @@
"\r\n",
" # Compute license priceses for ADF SSIS SQL compute\r\n",
" adf_pricesheet = pricesheet_df.where(is_sql_adf)\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('LicenseType', F.split('MeterSubCategory', ' ').getItem(1))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('ComputeResource', F.concat(F.split('MeterName', ' ').getItem(0), F.lit(' '), F.split('MeterName', ' ').getItem(1)))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('PricesheetJoinKey', F.concat(F.lit('SSIS '), F.col('ComputeResource'), F.lit(' - '), F.col('LicenseType')))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('AHB', F.when(F.col('MeterType').like('%AHB%'), 'AHBUnitPrice').otherwise('PayGUnitPrice'))\r\n",
" adf_pricesheet = adf_pricesheet.groupBy('PricesheetJoinKey').pivot('AHB').agg(F.first('NormalizedUnitPrice'))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('LicensePayGUnitPrice', F.col('PayGUnitPrice') - F.col('AHBUnitPrice'))\r\n",
" adf_pricesheet = adf_pricesheet.select('PricesheetJoinKey', 'LicensePayGUnitPrice')\r\n",
"\r\n",
" # Only process if adf related rows exists -> if not the pivoting will fail\r\n",
" if adf_pricesheet.count() > 0:\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('LicenseType', F.split('MeterSubCategory', ' ').getItem(1))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('ComputeResource', F.concat(F.split('MeterName', ' ').getItem(0), F.lit(' '), F.split('MeterName', ' ').getItem(1)))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('PricesheetJoinKey', F.concat(F.lit('SSIS '), F.col('ComputeResource'), F.lit(' - '), F.col('LicenseType')))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('AHB', F.when(F.col('MeterType').like('%AHB%'), 'AHBUnitPrice').otherwise('PayGUnitPrice'))\r\n",
" adf_pricesheet = adf_pricesheet.groupBy('PricesheetJoinKey').pivot('AHB').agg(F.first('NormalizedUnitPrice'))\r\n",
" adf_pricesheet = adf_pricesheet.withColumn('LicensePayGUnitPrice', F.col('PayGUnitPrice') - F.col('AHBUnitPrice'))\r\n",
" adf_pricesheet = adf_pricesheet.select('PricesheetJoinKey', 'LicensePayGUnitPrice')\r\n",
"\r\n",
" pricesheet_df = pricesheet_df \\\r\n",
" .select('Product', 'NormalizedUnitPrice') \\\r\n",
Expand Down Expand Up @@ -44385,7 +44478,7 @@
"spark.dynamicAllocation.enabled": "true",
"spark.dynamicAllocation.minExecutors": "1",
"spark.dynamicAllocation.maxExecutors": "4",
"spark.autotune.trackingId": "3ee2a5d6-bdcd-4b53-9baf-b6a9d86d3e96"
"spark.autotune.trackingId": "bfae63e0-183a-40d1-ab94-6811865dc39a"
}
},
"metadata": {
Expand All @@ -44410,7 +44503,8 @@
"sparkVersion": "3.3",
"nodeCount": 3,
"cores": 16,
"memory": 112
"memory": 112,
"automaticScaleJobs": true
},
"sessionKeepAliveTimeout": 30
},
Expand Down Expand Up @@ -46798,7 +46892,7 @@
"spark.dynamicAllocation.enabled": "true",
"spark.dynamicAllocation.minExecutors": "1",
"spark.dynamicAllocation.maxExecutors": "5",
"spark.autotune.trackingId": "a8c22b30-0326-4d55-991d-6847f4b96557"
"spark.autotune.trackingId": "ba52a84c-b785-4f63-a166-d9fb13d4c25d"
}
},
"metadata": {
Expand All @@ -46823,8 +46917,7 @@
"sparkVersion": "3.3",
"nodeCount": 3,
"cores": 16,
"memory": 112,
"automaticScaleJobs": true
"memory": 112
},
"sessionKeepAliveTimeout": 30
},
Expand Down Expand Up @@ -46892,7 +46985,6 @@
},
{
"cell_type": "code",
"metadata": {},
"source": [
"test_path = f'abfss://win-hub-logs@hubautomation.dfs.core.windows.net/{deploy_date}-AHUB-Deployment.csv'\r\n",
"csv_options = {'header' : True,\r\n",
Expand Down Expand Up @@ -48604,8 +48696,7 @@
"sparkVersion": "3.3",
"nodeCount": 3,
"cores": 16,
"memory": 112,
"automaticScaleJobs": true
"memory": 112
},
"sessionKeepAliveTimeout": 30
},
Expand All @@ -48625,7 +48716,6 @@
},
{
"cell_type": "code",
"metadata": {},
"source": [
"from datetime import timedelta, datetime\r\n",
"from dateutil.relativedelta import relativedelta\r\n",
Expand Down

0 comments on commit 0b2adf5

Please sign in to comment.