From 38d934dd477947c66c8732dd5ae209bbf26dfd23 Mon Sep 17 00:00:00 2001 From: tobiasny <31841479+tobiasny@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:24:44 +0200 Subject: [PATCH] Template and parameters deployed on 8-21-2024 9:24:41, based on the collaboration branch's commit ID: 0c8dff7eab30a2571a707cf11994986fb7613637 --- .../TemplateForWorkspace.json | 637 ++++++++++-------- .../TemplateParametersForWorkspace.json | 53 +- 2 files changed, 405 insertions(+), 285 deletions(-) diff --git a/s037-cost-management/TemplateForWorkspace.json b/s037-cost-management/TemplateForWorkspace.json index 4cf321c..900193f 100644 --- a/s037-cost-management/TemplateForWorkspace.json +++ b/s037-cost-management/TemplateForWorkspace.json @@ -757,6 +757,18 @@ "type": "string", "defaultValue": "https://s037-cost-management.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sprkpool33large" }, + "compute-consumption-plan_notebookSparkPoolNameRef": { + "type": "string", + "defaultValue": "sprkpool33large" + }, + "compute-consumption-plan_notebookSparkPoolIdRef": { + "type": "string", + "defaultValue": "/subscriptions/13d66f54-0a19-4912-b4f3-54d15897368d/resourceGroups/Synapse/providers/Microsoft.Synapse/workspaces/s037-cost-management/bigDataPools/sprkpool33large" + }, + "compute-consumption-plan_notebookSparkPoolEndpointRef": { + "type": "string", + "defaultValue": "https://s037-cost-management.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sprkpool33large" + }, "compute-cost-drivers_notebookSparkPoolNameRef": { "type": "string", "defaultValue": "sprkpool33large" @@ -1045,29 +1057,29 @@ "type": "string", "defaultValue": "https://s037-cost-management.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sprkpool33large" }, - "prodcluster_delayInMinutes": { + "sparkpool32_delayInMinutes": { "type": "int", - "defaultValue": 30 + "defaultValue": 15 }, - "prodcluster_maxNodeCount": { + "sparkpool32_maxNodeCount": { "type": "int", - "defaultValue": 10 + "defaultValue": 5 }, - "prodcluster_minNodeCount": { + "sparkpool32_minNodeCount": { "type": "int", "defaultValue": 3 }, - "prodcluster_nodeCount": { + "sparkpool32_nodeCount": { "type": "int", - "defaultValue": 10 + "defaultValue": 0 }, - "prodcluster_nodeSize": { + "sparkpool32_nodeSize": { "type": "string", - "defaultValue": "Large" + "defaultValue": "Medium" }, - "prodcluster_sparkVersion": { + "sparkpool32_sparkVersion": { "type": "string", - "defaultValue": "3.4" + "defaultValue": "3.2" }, "sprkpool33large_delayInMinutes": { "type": "int", @@ -1093,29 +1105,29 @@ "type": "string", "defaultValue": "3.3" }, - "sparkpool32_delayInMinutes": { + "prodcluster_delayInMinutes": { "type": "int", - "defaultValue": 15 + "defaultValue": 30 }, - "sparkpool32_maxNodeCount": { + "prodcluster_maxNodeCount": { "type": "int", - "defaultValue": 5 + "defaultValue": 10 }, - "sparkpool32_minNodeCount": { + "prodcluster_minNodeCount": { "type": "int", "defaultValue": 3 }, - "sparkpool32_nodeCount": { + "prodcluster_nodeCount": { "type": "int", - "defaultValue": 0 + "defaultValue": 10 }, - "sparkpool32_nodeSize": { + "prodcluster_nodeSize": { "type": "string", - "defaultValue": "Medium" + "defaultValue": "Large" }, - "sparkpool32_sparkVersion": { + "prodcluster_sparkVersion": { "type": "string", - "defaultValue": "3.2" + "defaultValue": "3.4" } }, "variables": { @@ -24741,6 +24753,38 @@ "[concat(variables('workspaceId'), '/integrationRuntimes/AutoResolveIntegrationRuntime')]" ] }, + { + "name": "[concat(parameters('workspaceName'), '/subscriptions_bronze_dev_lakehouse')]", + "type": "Microsoft.Synapse/workspaces/linkedServices", + "apiVersion": "2019-06-01-preview", + "properties": { + "annotations": [], + "type": "Lakehouse", + "typeProperties": { + "workspaceId": "c7651b6f-1da5-4098-b89c-912f51ebfbe2", + "artifactId": "bad450e1-2e58-4da2-9f00-5b500f717224", + "tenant": "3aa4a235-b6e2-48d5-9195-7fcf05b459b0", + "servicePrincipalId": "4c1cded2-f7a7-4c00-8d65-0a3287cbd682", + "servicePrincipalCredentialType": "ServicePrincipalKey", + "servicePrincipalCredential": { + "type": "AzureKeyVaultSecret", + "store": { + "referenceName": "ACM_Toolkit_kv", + "type": "LinkedServiceReference" + }, + "secretName": "sp-password" + } + }, + "connectVia": { + "referenceName": "AutoResolveIntegrationRuntime", + "type": "IntegrationRuntimeReference" + } + }, + "dependsOn": [ + "[concat(variables('workspaceId'), '/integrationRuntimes/AutoResolveIntegrationRuntime')]", + "[concat(variables('workspaceId'), '/linkedServices/ACM_Toolkit_kv')]" + ] + }, { "name": "[concat(parameters('workspaceName'), '/5th of every month at 5am')]", "type": "Microsoft.Synapse/workspaces/triggers", @@ -37604,7 +37648,7 @@ "spark.dynamicAllocation.enabled": "false", "spark.dynamicAllocation.minExecutors": "2", "spark.dynamicAllocation.maxExecutors": "2", - "spark.autotune.trackingId": "9a08c812-a4a2-45f4-a25c-91523484dab6" + "spark.autotune.trackingId": "69b086fb-0f83-4dce-aec4-b38a990f9266" } }, "metadata": { @@ -37629,8 +37673,7 @@ "sparkVersion": "3.2", "nodeCount": 3, "cores": 8, - "memory": 56, - "automaticScaleJobs": false + "memory": 56 }, "sessionKeepAliveTimeout": 30 }, @@ -40527,6 +40570,78 @@ }, "dependsOn": [] }, + { + "name": "[concat(parameters('workspaceName'), '/compute-consumption-plan')]", + "type": "Microsoft.Synapse/workspaces/notebooks", + "apiVersion": "2019-06-01-preview", + "properties": { + "folder": { + "name": "NotebookInProduction/Cost Prediction" + }, + "nbformat": 4, + "nbformat_minor": 2, + "bigDataPool": { + "referenceName": "[parameters('compute-consumption-plan_notebookSparkPoolNameRef')]", + "type": "BigDataPoolReference" + }, + "sessionProperties": { + "driverMemory": "112g", + "driverCores": 16, + "executorMemory": "112g", + "executorCores": 16, + "numExecutors": 1, + "conf": { + "spark.dynamicAllocation.enabled": "true", + "spark.dynamicAllocation.minExecutors": "1", + "spark.dynamicAllocation.maxExecutors": "5", + "spark.autotune.trackingId": "ce527ec2-dbcf-4905-bdb3-adf86dbe7676" + } + }, + "metadata": { + "saveOutput": true, + "enableDebugMode": false, + "kernelspec": { + "name": "synapse_pyspark", + "display_name": "python" + }, + "language_info": { + "name": "python" + }, + "a365ComputeOptions": { + "id": "[parameters('compute-consumption-plan_notebookSparkPoolIdRef')]", + "name": "[parameters('compute-consumption-plan_notebookSparkPoolNameRef')]", + "type": "Spark", + "endpoint": "[parameters('compute-consumption-plan_notebookSparkPoolEndpointRef')]", + "auth": { + "type": "AAD", + "authResource": "https://dev.azuresynapse.net" + }, + "sparkVersion": "3.3", + "nodeCount": 3, + "cores": 16, + "memory": 112, + "automaticScaleJobs": true + }, + "sessionKeepAliveTimeout": 30 + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "tags": [ + "parameters" + ] + }, + "source": [ + "storageAccount = 's037costmgmt'" + ], + "outputs": [], + "execution_count": null + } + ] + }, + "dependsOn": [] + }, { "name": "[concat(parameters('workspaceName'), '/compute-cost-drivers')]", "type": "Microsoft.Synapse/workspaces/notebooks", @@ -40765,7 +40880,7 @@ "spark.dynamicAllocation.enabled": "true", "spark.dynamicAllocation.minExecutors": "1", "spark.dynamicAllocation.maxExecutors": "5", - "spark.autotune.trackingId": "3516f8af-6547-4077-9a16-b336b0a39349" + "spark.autotune.trackingId": "fe08a4a8-bf4e-44a5-bd32-038b102d6210" } }, "metadata": { @@ -40815,19 +40930,19 @@ "storageAccount = 's037costmgmt'" ], "outputs": [], - "execution_count": 248 + "execution_count": 2 }, { "cell_type": "code", "source": [ - "import pyspark.sql.functions as F\r\n", - "import pyspark.sql.window as W\r\n", - "import statsmodels.api as sm\r\n", - "import pandas as pd\r\n", + "import pyspark.sql.functions as F\n", + "import pyspark.sql.window as W\n", + "import statsmodels.api as sm\n", + "import pandas as pd\n", "import numpy as np" ], "outputs": [], - "execution_count": 2 + "execution_count": 3 }, { "cell_type": "code", @@ -40843,11 +40958,11 @@ } }, "source": [ - "# Define rolling window for accumulation\r\n", + "# Define rolling window for accumulation\n", "window = W.Window.orderBy(\"Date\").rowsBetween(W.Window.unboundedPreceding, 0)" ], "outputs": [], - "execution_count": 3 + "execution_count": 4 }, { "cell_type": "markdown", @@ -40876,25 +40991,25 @@ } }, "source": [ - "# Load usage source\r\n", - "cost_path = f'abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/ACMMonthlyActualCost/*/Extended_v3_ACMMonthlyActualCost_*.parquet'\r\n", - "cost_df = spark.read.format('parquet').load(cost_path)\r\n", - "\r\n", - "# Select appropriate columns\r\n", - "cost_df = cost_df.select('Date', 'CostInBillingCurrency', 'PayGPrice', 'Quantity')\r\n", - "cost_df = cost_df.withColumn('Date', F.date_trunc('month', 'Date'))\r\n", - "cost_df = cost_df.withColumn('RetailCost', F.col('PayGPrice') * F.col('Quantity')).drop('Quantity', 'PayGPrice')\r\n", - "cost_df = cost_df.groupBy('Date').agg(F.sum('CostInBillingCurrency').alias('Cost'), F.sum('RetailCost').alias('RetailCost')).orderBy('Date')\r\n", - "\r\n", - "# Filter away latest month - as we predict cost per month, it will mess up future predictions\r\n", - "cost_df = cost_df.filter(F.col('Date') < F.concat(F.date_format(F.current_date(), 'yyyy'), F.lit('-'), F.date_format(F.current_date(), 'MM'), F.lit('-'), F.lit('01')))\r\n", - "\r\n", - "# Split dataframe into discounted and retail cost\r\n", - "discount_df = cost_df.alias('discount_df').drop('RetailCost')\r\n", + "# Load usage source\n", + "cost_path = f'abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/ACMMonthlyActualCost/*/Extended_v3_ACMMonthlyActualCost_*.parquet'\n", + "cost_df = spark.read.format('parquet').load(cost_path)\n", + "\n", + "# Select appropriate columns\n", + "cost_df = cost_df.select('Date', 'CostInBillingCurrency', 'PayGPrice', 'Quantity')\n", + "cost_df = cost_df.withColumn('Date', F.date_trunc('month', 'Date'))\n", + "cost_df = cost_df.withColumn('RetailCost', F.col('PayGPrice') * F.col('Quantity')).drop('Quantity', 'PayGPrice')\n", + "cost_df = cost_df.groupBy('Date').agg(F.sum('CostInBillingCurrency').alias('Cost'), F.sum('RetailCost').alias('RetailCost')).orderBy('Date')\n", + "\n", + "# Filter away latest month - as we predict cost per month, it will mess up future predictions\n", + "cost_df = cost_df.filter(F.col('Date') < F.concat(F.date_format(F.current_date(), 'yyyy'), F.lit('-'), F.date_format(F.current_date(), 'MM'), F.lit('-'), F.lit('01')))\n", + "\n", + "# Split dataframe into discounted and retail cost\n", + "discount_df = cost_df.alias('discount_df').drop('RetailCost')\n", "retail_df = cost_df.alias('retail_df').drop('Cost').withColumnRenamed('RetailCost', 'Cost')" ], "outputs": [], - "execution_count": 165 + "execution_count": 5 }, { "cell_type": "markdown", @@ -40924,20 +41039,20 @@ "collapsed": false }, "source": [ - "# Load billing source\r\n", - "billing_path = f'abfss://usage@{storageAccount}.dfs.core.windows.net/billing/BillingPeriods.parquet'\r\n", - "billing_df = spark.read.format('parquet').load(billing_path)\r\n", - "\r\n", - "# Compute invoiced amount\r\n", - "billing_df = billing_df \\\r\n", - " .withColumn('Cost', F.col('TotalUsage') - F.col('Adjustments') + F.col('AzureMarketplaceServiceCharges')) \\\r\n", - " .withColumn('Date', F.to_date(F.col('BillingPeriod'), 'yyyyMM'))\\\r\n", - " .select('Cost', 'Date') \\\r\n", - " .where(F.col('Date') < '2024-04-01') \\\r\n", + "# Load billing source\n", + "billing_path = f'abfss://usage@{storageAccount}.dfs.core.windows.net/billing/BillingPeriods.parquet'\n", + "billing_df = spark.read.format('parquet').load(billing_path)\n", + "\n", + "# Compute invoiced amount\n", + "billing_df = billing_df \\\n", + " .withColumn('Cost', F.col('TotalUsage') - F.col('Adjustments') + F.col('AzureMarketplaceServiceCharges')) \\\n", + " .withColumn('Date', F.to_date(F.col('BillingPeriod'), 'yyyyMM'))\\\n", + " .select('Cost', 'Date') \\\n", + " .where(F.col('Date') < '2024-04-01') \\\n", " .orderBy('Date')" ], "outputs": [], - "execution_count": 166 + "execution_count": 6 }, { "cell_type": "markdown", @@ -40966,20 +41081,20 @@ } }, "source": [ - "# 9% cost increase April 2021\r\n", - "COST_INCREASE_2021 = 1.09\r\n", - "COST_INCREASE_DATE_2021 = '2021-04-01'\r\n", - "\r\n", - "# 11% cost increase April 2023\r\n", - "COST_INCREASE_2023 = 1.11\r\n", - "COST_INCREASE_DATE_2023 = '2023-04-01'\r\n", - "\r\n", - "# 12% cost increase February 2024\r\n", - "COST_INCREASE_2024 = 1.12\r\n", + "# 9% cost increase April 2021\n", + "COST_INCREASE_2021 = 1.09\n", + "COST_INCREASE_DATE_2021 = '2021-04-01'\n", + "\n", + "# 11% cost increase April 2023\n", + "COST_INCREASE_2023 = 1.11\n", + "COST_INCREASE_DATE_2023 = '2023-04-01'\n", + "\n", + "# 12% cost increase February 2024\n", + "COST_INCREASE_2024 = 1.12\n", "COST_INCREASE_DATE_2024 = '2024-02-01'" ], "outputs": [], - "execution_count": 167 + "execution_count": 7 }, { "cell_type": "code", @@ -40995,15 +41110,15 @@ } }, "source": [ - "adjusted_billing_df = billing_df.withColumn('Cost',\r\n", - " F.when((F.col('Date') >= COST_INCREASE_DATE_2021) & (F.col('Date') < COST_INCREASE_DATE_2023), F.col('Cost') / COST_INCREASE_2021)\r\n", - " .when((F.col('Date') >= COST_INCREASE_DATE_2023) & (F.col('Date') < COST_INCREASE_DATE_2024), F.col('Cost') / (COST_INCREASE_2021 * COST_INCREASE_2023))\r\n", - " .when(F.col('Date') >= COST_INCREASE_DATE_2024, F.col('Cost') / (COST_INCREASE_2021 * COST_INCREASE_2023))\r\n", - " .otherwise(F.col('Cost'))\r\n", + "adjusted_billing_df = billing_df.withColumn('Cost',\n", + " F.when((F.col('Date') >= COST_INCREASE_DATE_2021) & (F.col('Date') < COST_INCREASE_DATE_2023), F.col('Cost') / COST_INCREASE_2021)\n", + " .when((F.col('Date') >= COST_INCREASE_DATE_2023) & (F.col('Date') < COST_INCREASE_DATE_2024), F.col('Cost') / (COST_INCREASE_2021 * COST_INCREASE_2023))\n", + " .when(F.col('Date') >= COST_INCREASE_DATE_2024, F.col('Cost') / (COST_INCREASE_2021 * COST_INCREASE_2023 * COST_INCREASE_2024))\n", + " .otherwise(F.col('Cost'))\n", ")" ], "outputs": [], - "execution_count": 168 + "execution_count": 8 }, { "cell_type": "markdown", @@ -41032,34 +41147,34 @@ } }, "source": [ - "def compute_ols_prediction(input_df, lookback, prediction_interval):\r\n", - " input_df['x'] = range(len(input_df))\r\n", - "\r\n", - " # Estimate OLS model\r\n", - " y = input_df['Cost'].values if lookback == -1 else input_df['Cost'].tail(lookback)\r\n", - " x = input_df['x'].values if lookback == -1 else input_df['x'].tail(lookback)\r\n", - " model = sm.OLS(y, sm.add_constant(x))\r\n", - " result = model.fit()\r\n", - "\r\n", - " # Configure prediction period\r\n", - " future_months = pd.date_range(start=input_df.index[-1], periods=prediction_interval, freq=\"MS\")\r\n", - " # future_months = future_months[1:]\r\n", - " future_x = np.arange(input_df['x'][-1] + 1, input_df['x'][-1] + prediction_interval + 1)\r\n", - "\r\n", - " # Predict future cost\r\n", - " x = sm.add_constant(future_x)\r\n", - " predicted_cost = result.predict(x)\r\n", - "\r\n", - " # Create the predicted cost column label\r\n", - " column_label = \"FullLookback\" if lookback == -1 else f'{lookback}MonthLookback'\r\n", - "\r\n", - " predicted_df = pd.DataFrame({\"Date\": future_months, f\"PredictedCost{column_label}\": predicted_cost})\r\n", - " predicted_df = spark.createDataFrame(predicted_df)\r\n", - " predicted_df = predicted_df.withColumn(f'PredictedCumulativeCost{column_label}', F.sum(f\"PredictedCost{column_label}\").over(window) + input_df['CumulativeCost'][-1])\r\n", + "def compute_ols_prediction(input_df, lookback, prediction_interval):\n", + " input_df['x'] = range(len(input_df))\n", + "\n", + " # Estimate OLS model\n", + " y = input_df['Cost'].values if lookback == -1 else input_df['Cost'].tail(lookback)\n", + " x = input_df['x'].values if lookback == -1 else input_df['x'].tail(lookback)\n", + " model = sm.OLS(y, sm.add_constant(x))\n", + " result = model.fit()\n", + "\n", + " # Configure prediction period\n", + " future_months = pd.date_range(start=input_df.index[-1], periods=prediction_interval, freq=\"MS\")\n", + " # future_months = future_months[1:]\n", + " future_x = np.arange(input_df['x'][-1] + 1, input_df['x'][-1] + prediction_interval + 1)\n", + "\n", + " # Predict future cost\n", + " x = sm.add_constant(future_x)\n", + " predicted_cost = result.predict(x)\n", + "\n", + " # Create the predicted cost column label\n", + " column_label = \"FullLookback\" if lookback == -1 else f'{lookback}MonthLookback'\n", + "\n", + " predicted_df = pd.DataFrame({\"Date\": future_months, f\"PredictedCost{column_label}\": predicted_cost})\n", + " predicted_df = spark.createDataFrame(predicted_df)\n", + " predicted_df = predicted_df.withColumn(f'PredictedCumulativeCost{column_label}', F.sum(f\"PredictedCost{column_label}\").over(window) + input_df['CumulativeCost'][-1])\n", " return predicted_df" ], "outputs": [], - "execution_count": 169 + "execution_count": 9 }, { "cell_type": "code", @@ -41075,29 +41190,29 @@ } }, "source": [ - "def produce_ols_results(input_df, interval):\r\n", - " # Compute cumulative cost and prepare for prediction\r\n", - " input_df = input_df.withColumn(\"CumulativeCost\", F.sum(\"Cost\").over(window))\r\n", - " pdf = input_df.toPandas()\r\n", - " pdf.set_index(\"Date\", inplace=True)\r\n", - " \r\n", - " # Predict cost based on different lookback periods\r\n", - " four_month_lookback_df = compute_ols_prediction(pdf, lookback=6, prediction_interval=90)\r\n", - " eight_month_lookback_df = compute_ols_prediction(pdf, lookback=12, prediction_interval=90)\r\n", - " twelve_month_lookback_df = compute_ols_prediction(pdf, lookback=24, prediction_interval=90)\r\n", - " sixteen_month_lookback_df = compute_ols_prediction(pdf, lookback=-1, prediction_interval=90)\r\n", - "\r\n", - " # Combine prediction results into a single dataframe\r\n", - " input_df = input_df.join(four_month_lookback_df, on='Date', how='outer')\r\n", - " input_df = input_df.join(eight_month_lookback_df, on='Date', how='outer')\r\n", - " input_df = input_df.join(twelve_month_lookback_df, on='Date', how='outer')\r\n", - " input_df = input_df.join(sixteen_month_lookback_df, on='Date', how='outer')\r\n", - " input_df = input_df.withColumn('Date', F.to_date('Date'))\r\n", - "\r\n", + "def produce_ols_results(input_df, interval):\n", + " # Compute cumulative cost and prepare for prediction\n", + " input_df = input_df.withColumn(\"CumulativeCost\", F.sum(\"Cost\").over(window))\n", + " pdf = input_df.toPandas()\n", + " pdf.set_index(\"Date\", inplace=True)\n", + " \n", + " # Predict cost based on different lookback periods\n", + " four_month_lookback_df = compute_ols_prediction(pdf, lookback=6, prediction_interval=90)\n", + " eight_month_lookback_df = compute_ols_prediction(pdf, lookback=12, prediction_interval=90)\n", + " twelve_month_lookback_df = compute_ols_prediction(pdf, lookback=24, prediction_interval=90)\n", + " sixteen_month_lookback_df = compute_ols_prediction(pdf, lookback=-1, prediction_interval=90)\n", + "\n", + " # Combine prediction results into a single dataframe\n", + " input_df = input_df.join(four_month_lookback_df, on='Date', how='outer')\n", + " input_df = input_df.join(eight_month_lookback_df, on='Date', how='outer')\n", + " input_df = input_df.join(twelve_month_lookback_df, on='Date', how='outer')\n", + " input_df = input_df.join(sixteen_month_lookback_df, on='Date', how='outer')\n", + " input_df = input_df.withColumn('Date', F.to_date('Date'))\n", + "\n", " return input_df" ], "outputs": [], - "execution_count": 170 + "execution_count": 10 }, { "cell_type": "code", @@ -41113,13 +41228,13 @@ } }, "source": [ - "billing_df = produce_ols_results(billing_df, interval=90)\r\n", - "adjusted_billing_df = produce_ols_results(adjusted_billing_df, interval=90)\r\n", - "discount_df = produce_ols_results(discount_df, interval=90)\r\n", + "billing_df = produce_ols_results(billing_df, interval=90)\n", + "adjusted_billing_df = produce_ols_results(adjusted_billing_df, interval=90)\n", + "discount_df = produce_ols_results(discount_df, interval=90)\n", "retail_df = produce_ols_results(retail_df, interval=90)" ], "outputs": [], - "execution_count": 171 + "execution_count": 11 }, { "cell_type": "markdown", @@ -41148,11 +41263,11 @@ } }, "source": [ - "billing_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/cumulative-cost-prediction.parquet\"\r\n", + "billing_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/cumulative-cost-prediction.parquet\"\n", "billing_df.write.format('parquet').mode('overwrite').option('overwriteSchema', 'true').save(billing_path)" ], "outputs": [], - "execution_count": 173 + "execution_count": 12 }, { "cell_type": "code", @@ -41168,11 +41283,11 @@ } }, "source": [ - "adjusted_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/adjusted-cumulative-cost-prediction.parquet\"\r\n", + "adjusted_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/adjusted-cumulative-cost-prediction.parquet\"\n", "adjusted_billing_df.write.format('parquet').mode('overwrite').option('overwriteSchema', 'true').save(adjusted_path)" ], "outputs": [], - "execution_count": 174 + "execution_count": 13 }, { "cell_type": "code", @@ -41188,11 +41303,11 @@ } }, "source": [ - "discount_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/discounted-cost-prediction.parquet\"\r\n", + "discount_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/discounted-cost-prediction.parquet\"\n", "discount_df.write.format('parquet').mode('overwrite').option('overwriteSchema', 'true').save(discount_path)" ], "outputs": [], - "execution_count": 175 + "execution_count": 14 }, { "cell_type": "code", @@ -41208,11 +41323,11 @@ } }, "source": [ - "retail_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/retail-cost-prediction.parquet\"\r\n", + "retail_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/exports/monthly/retail-cost-prediction.parquet\"\n", "retail_df.write.format('parquet').mode('overwrite').option('overwriteSchema', 'true').save(retail_path)" ], "outputs": [], - "execution_count": 176 + "execution_count": 15 }, { "cell_type": "markdown", @@ -41241,16 +41356,16 @@ } }, "source": [ - "# def predict_cost(input_df, p, d, q):\r\n", - "# model = sm.tsa.ARIMA(input_df[\"Cost\"], order=(p, d, q)).fit()\r\n", - "# future_months = pd.date_range(start=input_df.index[-1], periods=72, freq=\"MS\")\r\n", - "# predicted_cost = model.predict(start=input_df.index[-1], end=future_months[-1], typ=\"levels\")\r\n", - "# predicted_df = pd.DataFrame({\"Date\": future_months, \"PredictedCost\": predicted_cost})\r\n", - "# predicted_df = spark.createDataFrame(predicted_df)\r\n", + "# def predict_cost(input_df, p, d, q):\n", + "# model = sm.tsa.ARIMA(input_df[\"Cost\"], order=(p, d, q)).fit()\n", + "# future_months = pd.date_range(start=input_df.index[-1], periods=72, freq=\"MS\")\n", + "# predicted_cost = model.predict(start=input_df.index[-1], end=future_months[-1], typ=\"levels\")\n", + "# predicted_df = pd.DataFrame({\"Date\": future_months, \"PredictedCost\": predicted_cost})\n", + "# predicted_df = spark.createDataFrame(predicted_df)\n", "# return predicted_df" ], "outputs": [], - "execution_count": 133 + "execution_count": 16 } ] }, @@ -43315,7 +43430,7 @@ "spark.dynamicAllocation.enabled": "true", "spark.dynamicAllocation.minExecutors": "1", "spark.dynamicAllocation.maxExecutors": "5", - "spark.autotune.trackingId": "bbe6faaa-9363-4aae-b478-f10bf8971948" + "spark.autotune.trackingId": "2e18fef4-2d9c-4f74-922a-a2bcf499f91a" } }, "metadata": { @@ -43340,8 +43455,7 @@ "sparkVersion": "3.3", "nodeCount": 3, "cores": 16, - "memory": 112, - "automaticScaleJobs": true + "memory": 112 }, "sessionKeepAliveTimeout": 30 }, @@ -43419,7 +43533,6 @@ }, { "cell_type": "code", - "metadata": {}, "source": [ "def get_last_day_prev_month(to_date):\r\n", " current_month = int(to_date[4:6])\r\n", @@ -44475,7 +44588,7 @@ "spark.dynamicAllocation.enabled": "true", "spark.dynamicAllocation.minExecutors": "1", "spark.dynamicAllocation.maxExecutors": "5", - "spark.autotune.trackingId": "82d30e90-6fd9-47cc-b9fb-9a6784b32a20" + "spark.autotune.trackingId": "7ef0c6d9-a33c-4680-9e43-584afbd9096d" } }, "metadata": { @@ -44500,8 +44613,7 @@ "sparkVersion": "3.3", "nodeCount": 3, "cores": 16, - "memory": 112, - "automaticScaleJobs": true + "memory": 112 }, "sessionKeepAliveTimeout": 30 }, @@ -44533,13 +44645,13 @@ } }, "source": [ - "import requests\r\n", - "import datetime\r\n", - "import pyspark.sql.types as T\r\n", + "import requests\n", + "import datetime\n", + "import pyspark.sql.types as T\n", "import pyspark.sql.functions as F" ], "outputs": [], - "execution_count": 8 + "execution_count": 2 }, { "cell_type": "code", @@ -44552,7 +44664,7 @@ "storageAccount = 's037costmgmt'" ], "outputs": [], - "execution_count": 9 + "execution_count": 3 }, { "cell_type": "code", @@ -44568,11 +44680,11 @@ } }, "source": [ - "BASE_URL = \"https://management.azure.com/providers/Microsoft.Billing/\"\r\n", + "BASE_URL = \"https://management.azure.com/providers/Microsoft.Billing/\"\n", "BILLING_ACCOUNT_ID = \"57950773\"" ], "outputs": [], - "execution_count": 10 + "execution_count": 4 }, { "cell_type": "code", @@ -44588,25 +44700,25 @@ } }, "source": [ - "def generate_billing_periods(initial_year, initial_month):\r\n", - " now = datetime.datetime.now()\r\n", - " current_year = now.year\r\n", - " current_month = now.month\r\n", - "\r\n", - " billing_periods = []\r\n", - "\r\n", - " # Loop through the years and months and add the dates to the list\r\n", - " for year in range(initial_year, current_year + 1):\r\n", - " start_month = 1 if year > initial_year else initial_month\r\n", - " end_month = current_month if year == current_year else 12\r\n", - " for month in range(start_month, end_month + 1):\r\n", - " date_str = f\"{year}{month:02}\"\r\n", - " billing_periods.append(date_str)\r\n", - "\r\n", + "def generate_billing_periods(initial_year, initial_month):\n", + " now = datetime.datetime.now()\n", + " current_year = now.year\n", + " current_month = now.month\n", + "\n", + " billing_periods = []\n", + "\n", + " # Loop through the years and months and add the dates to the list\n", + " for year in range(initial_year, current_year + 1):\n", + " start_month = 1 if year > initial_year else initial_month\n", + " end_month = current_month if year == current_year else 12\n", + " for month in range(start_month, end_month + 1):\n", + " date_str = f\"{year}{month:02}\"\n", + " billing_periods.append(date_str)\n", + "\n", " return billing_periods" ], "outputs": [], - "execution_count": 11 + "execution_count": 5 }, { "cell_type": "code", @@ -44622,24 +44734,24 @@ } }, "source": [ - "def fetch_billing_details(billing_account_id, billing_period, api_version, access_token):\r\n", - "\r\n", - " # Build Azure management API billing endpoint\r\n", - " endpoint_url = BASE_URL\r\n", - " endpoint_url += f\"billingAccounts/{billing_account_id}/\"\r\n", - " endpoint_url += f\"billingPeriods/{billing_period}/providers/Microsoft.Consumption/balances?\"\r\n", - " endpoint_url += f\"api-version={api_version}\"\r\n", - "\r\n", - " headers = {\r\n", - " 'Authorization': 'Bearer ' + access_token\r\n", - " }\r\n", - "\r\n", - " response = requests.get(endpoint_url, headers=headers)\r\n", - " result = response.json()\r\n", + "def fetch_billing_details(billing_account_id, billing_period, api_version, access_token):\n", + "\n", + " # Build Azure management API billing endpoint\n", + " endpoint_url = BASE_URL\n", + " endpoint_url += f\"billingAccounts/{billing_account_id}/\"\n", + " endpoint_url += f\"billingPeriods/{billing_period}/providers/Microsoft.Consumption/balances?\"\n", + " endpoint_url += f\"api-version={api_version}\"\n", + "\n", + " headers = {\n", + " 'Authorization': 'Bearer ' + access_token\n", + " }\n", + "\n", + " response = requests.get(endpoint_url, headers=headers)\n", + " result = response.json()\n", " return result" ], "outputs": [], - "execution_count": 12 + "execution_count": 6 }, { "cell_type": "code", @@ -44655,10 +44767,10 @@ } }, "source": [ - "access_token = \"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6InEtMjNmYWxldlpoaEQzaG05Q1Fia1A1TVF5VSIsImtpZCI6InEtMjNmYWxldlpoaEQzaG05Q1Fia1A1TVF5VSJ9.eyJhdWQiOiJodHRwczovL21hbmFnZW1lbnQuY29yZS53aW5kb3dzLm5ldCIsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LzNhYTRhMjM1LWI2ZTItNDhkNS05MTk1LTdmY2YwNWI0NTliMC8iLCJpYXQiOjE3MTM4NzgxODgsIm5iZiI6MTcxMzg3ODE4OCwiZXhwIjoxNzEzODgzMDQ3LCJhY3IiOiIxIiwiYWlvIjoiQVZRQXEvOFdBQUFBMzVCTjdSSDdBOHJNMjdoRWl2RGdtSVYrTm9jUkdBZ2lBR1E4ZjUxVnI4U0RkODlsbHYzSEdwMi85Vng3UG9Va1lCQkRIem9xdGVsam9KTGx6bU5ISlBDaStCMG90V0VaTFRnakxIZUk0MDQ9IiwiYW1yIjpbInJzYSIsIm1mYSJdLCJhcHBpZCI6IjE4ZmJjYTE2LTIyMjQtNDVmNi04NWIwLWY3YmYyYjM5YjNmMyIsImFwcGlkYWNyIjoiMCIsImRldmljZWlkIjoiMzc1ZmY4YTUtMTk4OS00NDVhLWE5M2ItZmRmYjA4ODdmNWZiIiwiZmFtaWx5X25hbWUiOiJGbG9vZCIsImdpdmVuX25hbWUiOiJEYXZpZCIsImdyb3VwcyI6WyI5NGNkZGYwMC1kZTBmLTRjNTgtYTAzNC1kZTM0NjQ1N2ZkYTAiLCI2YTBmNzMwMS0wMzhlLTQyMzEtYWQ1Ni02ZDAxYWRjYjg0ZWYiLCJkODYyY2EwMi05ZDk4LTQ3M2QtOTc2MC01MWM4ZWY2OWJlZmYiLCJmZmFlYTYwMy03NTlhLTQ0OTAtOTM3Yy04NzUyNmY0MmQzMWUiLCJlZjI5ZGEwNS1hNDg5LTRhODQtOTgzZC02ZTJjZTMyYTFjNGYiLCJmYWNiMDQwNi1hOTllLTRiMWMtOWJkMS1iNTg5MDNhZDYwYzkiLCJjYmI5ZmMwNi1jNWQ2LTQ4NzItOGM4NS00ODAyNGU4OWM2NDkiLCI5ZWIyNjMwOS0zNWUwLTRhODAtOTZhZC1lNGM5ZWM1NzExYmUiLCI5NjI5YTIwYS1lYWVjLTQ5YjMtOTc4NC0wYWU5YTY1NzBmYzkiLCIxZGI2YmEwYy0xZDJmLTRkNzYtOWRhZS0wODgxZTU5MTNjNWMiLCJjYjM4YzIwZC1mNGEzLTRlYzMtYjI1OC1lZDAyYzkzNjg0ZjAiLCJmYjNkY2IwZC0yMTJkLTRlNzgtYjUzZi0zYzdiOTMyZWUyNWIiLCIxOTNjZjEwZC03ZGQwLTQ2ZDctYTEwMS00NTU4MjAyMzRhNTYiLCJmNjM4NDgwZi1iMjk1LTQ3N2YtYmRlYS1iOTllZDQyMWRiYTIiLCIzMTk4OWUwZi1kNmU1LTQ4ZTctOTk1Mi0xMGE3ZmZhZTNiYjgiLCIwNzdlMzgxMS03ODNhLTRiOWEtOWE0Yi01Yjk5ZTdlNjcxYzAiLCIzMTAwNDcxMi1kOTBlLTQ5ZjYtOWU3OS0yZTk3ZjY4ZjZkNTAiLCIwNDdmYjAxMi02OTRjLTQ0YzMtOGQ0OC05MjdjYjM0OTc4OGMiLCJiNzJiNjgxNC1kMTcyLTRlMzYtYmI5MC0wYjNmMTVhOTE5NzciLCJmMmM5ODgxNS1lMTY2LTRmOTAtYTVmNC0xZjUzNWY1ZWRjYzgiLCI3MTEzYzMxNS1jNGIxLTRkYzYtYWQ2Yi0wYTFlMjEyMjhhMDEiLCIwNWI5NmQxNi1iNjQ0LTRmY2YtOGFlNS1mZjBjMGY4MjViNWYiLCJiYzM5ZjIxNy00NDE5LTQyOTEtOTUyNy0zNTYxNDc3YTJmYjciLCIwMDAzMTExYi0wYzJlLTQwNDEtYjFkZC0wYjJlZDFlZjZmNzgiLCJiZmMxMzkxYy02ODE0LTQxNDMtODNjNy0zYzQ2YWY4Yzc4YTgiLCIzMDcyOGUxZS00NmYwLTRiNjEtYTBiYy02YmMzZjczNjBiOGEiLCJjNjhlNTAxZi02ZDFmLTRiNGItOGYzNy0wYzQ2ODE1NmRkZjIiLCI1YWYwOTkyMS1mZDZhLTQ2MTEtOWVhOS1hM2JmYjFmMzViYmUiLCI3ZWYzNmMyMy0zZjZiLTQzMjEtYjAyOC1iODFhOWI0NTk5OTEiLCJkZjY1ZWEyMy04OGMzLTQzMWEtYTg3Zi02MDU1Y2Y5YmQ1NDkiLCIzYjk1ZGEyNy0zMzdhLTQ0YzEtYjY5ZC01NjQ0YjUxYTQ4MzYiLCI4ZmEzYmIyOC04NGY5LTRiMDYtYTRkNS0wODkwN2U4ZGQ2ZmEiLCJlMDQwYzQyOC1hYzU3LTQ1NDItYTMyOC0yZjY3MmEzNGNjYjgiLCJlMWNhYTQyYS1iZjY5LTQzOTEtOWE1ZS1jOGNmZGFlMWI0MTciLCI5ZmU2ZTYyYS0zMTE4LTQ3MDQtYWZiZS04MzQ4MjJjYjBkNDMiLCIwYjEwMmIyYy0zYzk3LTRiNzYtYjRkZi1lMDM5ODY4NmFmNTYiLCI3NDhhMDIyZS05NjhlLTRhOTQtYWRlMy05MjRlMjZiYWRhYTgiLCJkYTVhMzMyZS0xNDcwLTQwZjMtOTgwMy00OTc1ZTU2OTRlNWIiLCI0ZjZmNGUyZi1iMjIzLTRlNzMtYjMzMy01YmIxZjc2MDE0ZjQiLCIyOWUwMzUzMC1mODE4LTQwMzYtYjI4Yi05N2I4Yjk0ZjI2NTAiLCJhZDU3NzgzMi1hMGUxLTRhMDgtYmJmNC0xMjcyNDNjNmE3NTIiLCIxZGZlYmQzNS03ZTRkLTRmMzktOTIyNS0wMTFkYWRkZGRlYjAiLCIzMDQ5NGUzNi0yMDQxLTQwOTYtYWNmOC1iMzY4NGE2MzA5ZTQiLCI1ZGQ1NWMzNy0wYjg0LTQwYjQtODc2MS01M2JiOWE5M2FjOGYiLCJhNWVkZjAzOC02YzJmLTQ0MzgtOGYyZi1jZGY1YWViOTFhYzUiLCIzOTI3MDkzYS04ZDU5LTQxMTktODMzZS1lN2EwMmZiODg3OGMiLCI2MTJhY2IzZi1lMmQ4LTRiNTYtYjU1MC0wZmU1NTY4OTQyODQiLCJiNDA3NGY0Mi00MDFiLTRjNWUtODE3ZS02ZWY4MzM1OWI3YmYiLCI4NDljZjQ0My05YzVjLTRlMzItYTJlYS01NWVhZWRkOWU2YTYiLCIyNjE1NDA0NC0yYTc4LTRjODItOTk5My0xNmUwODQ1YzU1ODIiLCJiMzZiMjk0NS0xZDI0LTRjODMtOWJhNS0yNTE0YTU2ZjQ5YTMiLCI2YzRhMjU0OS04MGViLTQ2YTktOTk3Mi05MjcxNzhmNGM0ZmYiLCJjNWQyNjI0Yi02NWY4LTQzZDQtYTFhOC01MjhmMWM1Mjk3ZDMiLCIyNDFmMWM0Yy02MjExLTQ4YWMtYTA4OC1iNjJlMWMwYjMzYWMiLCI0NTIyYTQ0ZC02NjBkLTQ0ZWMtODA3My1hZmRjMTZlNGFkN2IiLCI1MzAxYjQ0ZC1mNTA1LTQ0MDctODc2ZC1lZmFlNWI3YmU5ODAiLCJlNzE2Nzc1MC01YzViLTRiNzYtODhiMi02NThlYmYyMTNmMWUiLCI5MWE0YWI1MC1jNDQxLTRhOWQtYmU0My1hMDJjNDg1Zjk1ZjMiLCIxYWIzMTg1MS1jMzZjLTQ5NmYtOGQ0Yy0zYTdhYTQ1OTM0YzgiLCIzMmRhZDg1MS0wYjZhLTQ1MzEtYjgzNS0zMDcwY2JlOTBjNTAiLCI2NDgwYzQ1NC01ZjZhLTRkMTktYWQ0Zi01ZDA0Yzg2NWJhODUiLCJhZjM0ZTA1NS0yZTg1LTQ0NzItOGI3My1lYWI0MTBhN2ExNWEiLCJiMWMyNjI1Ni05NmRmLTQzN2YtYTNhYi04ZTE2NjNlNjZhYjEiLCJmMjlmMjQ1Ny1jMDYwLTRjODMtYTY2Ni0zNDU5MmMyY2MxZmIiLCJkZGZjMjY1YS0wYmJmLTQ5Y2UtYmIwMi02N2Q1MTNiNzdjMmIiLCJlYWI3N2M1Yi0wZTZkLTQ1ZGYtODNmOC1hNjVkOTcwNzIxNWQiLCJiYzdmNmY1ZS1iNGIxLTRjZjMtYWQ4Ny1kZTZlMWVjMjdlNDMiLCJlMmNlOTM2MC1kYmQ2LTRkZDQtODU1Mi04MDYyYTUxN2I1ZGQiLCI3MzI5OWI2MC02MDIzLTRjZmQtOThjMS01ZDY1ZTJmY2I0N2YiLCIwZmUxZWU2MC02MzA1LTRkMGEtOWIxMC0zMWI1MDI2ZjI1YzciLCI5MDA3MGY2MS02NTcyLTRkNGYtOGQwNC0yNWE0YWViNWIzNzkiLCI3NTJhZjU2MS0yOTg2LTRkNDEtOTYwNC1mZTY5MjhhNmU1NjUiLCIzZTEzNjk2My0xZjM5LTQ4YjgtYjVmMi0yNjRkYTAxYTdhYTUiLCJiMDMxNTk2NC04Y2IzLTRkMTItOGFkNy05NGU4OGNhNmI3MDciLCIxMmVlYjk2NS1jNTVmLTRiYjQtOGRiYi1mNGU2ZTA4Y2MzMGEiLCI5MzQ1ZTg2Ni1mYWZmLTRlN2UtYjljMC02ZGIyZDY1OGY3OTMiLCI0OGVkNGM2Ny1jN2E3LTQ3M2ItYWI3YS02NTEyMWE3NDczNzkiLCI5YjYxNzU2OS02NzQ1LTQ1YTUtYTdmMi0yMGE3NmZjZGE4MGYiLCI2OGI2ZDQ2YS05NjkyLTQ5ZTEtYTFhNi1iMDc1ZjZhMmU5ZDkiLCI5ZjYyZWM2Yy1lZTQxLTQ0ZTktOTEzMS05OWI3OTI1NDEzODIiLCJmYzY0MzQ2ZS0wZjUyLTRjZDItOTU0Ni01OTY1ZmQ0MmRlMjUiLCIxYWMxMzc2ZS1mMDRhLTQ0YjYtYmU1OC1hYTE2MDAzZWE5NmUiLCI5YzM4ZmE2Zi00YThmLTQyNWQtYWI0OC1iNzc5N2FjY2Y4ZGUiLCJmZWNlNGE3MC1hOTY4LTRiN2QtOThiYS05YWUxMzA5NzhjOWMiLCIyYzg4Zjk3MS05NzJjLTRjNDgtYjZjNi0wODYzMWMxOTE0YTciLCIwZGY1Mjk3NS1kYjE5LTRmYTMtYmNkOC02MWQ4YjJlMjgyZDgiLCJjNzZlMWU3Ni1hZjZmLTQyYzUtOGRiMi1jYWVjOGRjODFjYmUiLCJkYWVmMDQ3Ny0zNDFiLTRkZGUtOTNmYy00YjgxOTAxMWIyOTQiLCI4OTQ5YTI3Ny1kYTM0LTRiYjAtYWYwMS1iZGJkZmY2NWViNmEiLCJjNDBkZTA3OS0yYTU2LTQ5MzItOGYzNS0wNTI5NzliZGMwZmMiLCIxYjRjOGU3Yi1hYzBjLTQxOWEtYjg3MS03ODllZmE2MWE2N2UiLCJmYjdiNjU3ZS1iYTU5LTQ4YzMtODY1Ni0yMjQ0Zjc3MDMzY2QiLCIzMDAxNjA4MC0yODVjLTQ3YzctODhkMC0yMGYyYWI1OGI3MzAiLCI0NzMxM2Y4MS0wNmNkLTQ5NzktODAwOC00MWNhNmE5MDdhZTciLCJjNjNjNzU4NC1kNTE4LTRjNTYtODEzOC1lN2EwMzIzYjczYjgiLCI0MTIxN2E4Ni1lNTVlLTRkODgtYmM3Yy1hMWUxMDExZjc2NGIiLCI3OThmOWM4OC0wZTFhLTQyODgtYjcwNC0wZDY4MzQ1MDcwZTQiLCJiZjNjMGY4YS1iNDlhLTQzYzQtYjg3YS04ZDgyZTQ4ZGRhOGIiLCJmMmM4Y2I4Yi0yNzY5LTRiNDYtYTk2Ni02NzhhMjVhOTFkM2MiLCI2YzU4ZWY4Yi1jNGIyLTQzOGYtOTQ3NS1iOWRmMWNhYjhjMGYiLCIzZWFiOTk4Yy04MjRkLTRhYjktODVhNS1lNDI0YTBhMjExYzEiLCJlNDExYzA4ZC03MDQzLTRkMWItYmJjMS0xMTY1MGVjMmUyYWUiLCI3ZDYzZGE4ZC00NzMwLTRiYTAtOTZiZS1hZWVhYmU2NmVmODEiLCIxYWJjYjM5MC04MDEyLTQ1MGYtODlkZS1iOWYwZmQ3ZjIwMzIiLCJjY2MzZjc5MC0yNmMyLTQyMDgtYmM1Zi05NzNmNmVjY2JlNDciLCIwMDA1ZDM5Mi03NTdkLTRhYzUtOTNjMi1mYTQxNTg2YmIzYmYiLCJiYjIzZmE5Ni1lODU5LTQwZmUtOWU1Yy0zOTVmYzYwZmFiYWQiLCI4MDY3ZDI5OC0zZjU1LTQ0MjItOWU2MS01NzFhYzhkYjY4NGIiLCIwMDliNTM5Yy1kY2UzLTQ3M2EtODY0My03ZGI5NGQxNjI4NDQiLCIzN2I0ODM5ZS01YjBjLTQyOTMtYWY4OC0xYTI1YWE4MDBmNjciLCJhNjM2MjY5Zi1hOTY3LTQ1YjItYmI0ZS0zZTAyZGJiNjI0YTIiLCJhMmY3MzFhMi05MDMzLTQzMGMtOGRjMC0yNjliYmZmNWY1NzYiLCJhNDdkZjJhMi03OTlhLTRkOGQtYTc0Ni01Nzk0MTFhY2QxODUiLCJkZjQxZTVhNS05NzQ5LTQzMGYtYTQwOS0yYjYzNTM0MzMwM2EiLCJkNjEzMDRhNi0yNDM2LTQ4NmUtOGI0MC1kNTM1MTJmM2E1YmYiLCIxYzE3NDNhOC04OTcxLTQ1MmItYjNmZC02OWFlYTNhOTc1MjMiLCJlZDg5ODNhOC1hZjc0LTQzNDYtODU2Zi1iNjMxOGYyNWI2MjgiLCJiNjI3NTFhYS1iOTAzLTQ4MTUtYmY0YS02Y2ExZmQ2NDNlZTciLCJlNTI2ZWVhYy1hNDQwLTQzNjQtOTE0ZC0xMDljZmUzNzk0MWUiLCJhZmY0YjBhZC03YzNjLTRkOTUtODNkZi1iYjU2MThmNjFkMGIiLCIzZWRiNjhhZS1mYWQ0LTQ2NmYtYTEyZC1hNjc1MzQ4MGY4N2IiLCIyOGU0NjFiMS0wZGNkLTRlNmQtYjM2Zi00ZjQ3NDdmNzI5YWIiLCJjNzU1NTViMi1jMjU1LTRkNTItOTY5NS05MDFlOTcyOGMwOGUiLCIxZmJkZGViMi0xZGJhLTQxYmEtOWNhNS1kYzRiZDFjMTUwYWIiLCI1MjkwMzRiMy1jNDUwLTQwYTgtYmNjNS1jNDIzODMyZjQ1NjIiLCJmOTk1MjliNC1hNTljLTRiYTEtYTMyZC1mYWQwZGUzMDA5YWMiLCIxYmFmNjhiNC0yNWM4LTQ5OTgtYjE1OS1jYWU5ZDQ4OTM3N2UiLCJmYTQxNzFiNC0yNzQwLTRiMzYtYTAyNi04MGRlY2NlYzU1NDAiLCIyOTM5ODRiNC1lNTE5LTQ2NzYtOWQyYS02ZmEzZjY3MmJkMWMiLCJjYjQyMjViNy0zNWFmLTQ1MzMtODg5Zi03MjQxYzk5ZjE1MWMiLCIwZDdhZGFiOS1mMTlkLTRhZjktOWU2Yi03ZGQ1YTNkN2Q5OWYiLCIzZWE4MzZiYy00YTI1LTRmZGItOWUxOC00NjU0NzU0OTNkNmYiLCIxNmY4MTBiZS1lMjc5LTRlMjItODFjMS02MmY0OTQ2Y2E1MDYiLCJkZTUwZTdiZS0yNWM3LTRmOTMtYWQ1Mi0zYmY1YWU4MTUzMjIiLCJhNTEwY2ViZi0xZTc5LTRjOWItYjIxZS02OTlmOWQ3NTE5ZjkiLCJiZDYzZjZjMC1jYzYxLTQ1ZWItYmIwYS05NzUxODVmYzYwMWYiLCI1ZjA2ZGNjMS0zMjYzLTQxZjYtOWJmYS1mYTZmMGNlYjg2ZDkiLCI0ZjYwODBjOS1jZTM4LTRkNWUtYjE2Ny1lYzUzNzdiZGRmODkiLCJiNDliMTRjYS0wM2M2LTRlYzMtYmNhNS0wZDQzNjkzNTA2YWIiLCJkYTMzNzFjYS04MDYyLTQ2NzEtYjBjOC01MzhhOWM2ZjJhZGQiLCJlODkxNDhjYi00YTFkLTQzNjctOGFhNC1jN2FhNjQ0MmY0NzEiLCJiNGM0NzFjYi1mMjYzLTRlMjItYTAzNi0xNmQ4ZDE2NDQ2ZmUiLCJhODZiODFjYy1hMGZhLTRkY2QtYTQ3MC1jOWQ0ZmZiNGQzNDciLCI0ZWIxM2ZjZi0yMGI2LTQ3ODQtYWU1My0zMTQ1MmFlN2ZlZDEiLCI3MTMzN2FkMy0xZjZiLTQ5N2UtOWE0NS1mYzZlOTc4MjM3OGIiLCIwYmFmYTBkNy0zYTA0LTRhMGYtOTE3ZC00NzQyZmI0MTM1ZGUiLCJhOTNhMTNkOS02MzExLTQxZTEtYjNiNi1mOTlkNGZiMDJlYWQiLCJmMTY3OTZkYS0zNGYzLTRmNmEtYWM0My1jNmExNjkxZmJiM2MiLCJmYTIwNjJkYy02MDY4LTQ2MWMtOTJhZC1iYmZmNjAxMjZkMTkiLCI0NjI5M2JkZC0xOGMxLTRkYWMtYWFhNC0xODBhYTRiMzM3YWQiLCJiZDI1YzRkZC03MWI4LTQ1ZDMtODcwMi05OGM0N2ZlMDAxOWQiLCIwOTQ1YzRkZi0wMWYxLTRkMWMtOTE5Ny02YWE3YWY0Y2U3MzciLCI0Y2JiZDVkZi1hYzI5LTQ2ZmMtYTI2ZS01NzBjMWY0ODU5ODgiLCJiOTA2NDBlMC1jYzU4LTRlNWItYjIxYi05YzE3OTcwODBkYjMiLCI2MWM5NzNlMC1hMDMzLTRiYmEtOWEzOS00ZGZmYThhMzEyZDIiLCJlZmIwNzFlMi1jZmFhLTQwYzctOThlNS03MjA1NjZhMGYyMjUiLCJiOGEzYjNlMy1kOWRmLTQ5NTYtOWZhNy02ZGFhNjUwNjI1NzgiLCIxYWIzYzllNy1kZGUwLTRhZjEtYWQ4OS1lN2ZkZWI3YmI4ZDgiLCIyZWYzMmFlZC0wZjhjLTQxNDMtOGUyMi1jNTFmODYxYjM1NzciLCIyNmIzYmRlZS0xYzhmLTQ0ZWMtODliZC0wNmY4MDNiMTc3ODciLCJkOTljZWRlZi1iYzg3LTQxY2EtYmFhZC01NzNmYzNlZjIwZWQiLCI4MmY4ZjNlZi05MDYwLTRhMmYtYTU4Ny01ZjQ3Y2EyMzhhYjciLCJjYTQ1ZjllZi1hMTc5LTQ4ZjgtODMxZC05NDFjNGVjMjIxMTMiLCJiMGFjYjdmMC00YmRkLTRkODMtYWNiMC0yNGRkODZjYWQ5MDQiLCJjMTliOTNmMS01ZWY2LTQwNzctYTU4Zi02YjdjZWY2NDY3MDEiLCIxMDRiMjhmMi1lYjJkLTQ0NWUtOGQ0Yi0wMDExZTExOTZmYWQiLCIyMTE1NDJmMy1jY2JkLTRkOWEtOTNhNi1jNWZiYmQ5MjE5YTMiLCI3OTg2MzJmOC1iMDQ1LTRjYzEtYTg0ZC04MWVkNTY3NzFjYWIiLCI1NjdmYjZmOC03MDU4LTQ5OTYtYTUzYi0yZjJjMWJkMjE4NGQiLCIyZjRhYjNmOS02ODJkLTQ2ZDYtOTIwNC03ZWJkYTdiYmQ0MWMiLCIyYzVhMDlmYS1kOTM3LTRiM2EtYWFhZi1mNzc0ZDg3MWQ2YzUiLCI5NjAxODhmYS00MDhlLTQ5MDEtOTUwZS1kNmUxOWZlZDU0ZWYiLCI5MGI2ZDdmYS1mYTI2LTRhODUtOTJlMy05ZGEyMDVlZDQ2YzUiLCJhYWMzN2JmYi03ZTgxLTQzMWMtYmI3MC01Y2U0ZTgwYmNiZGQiLCI5OWU2MmFmZi00Zjc2LTQ2MGEtYTYxMS1mOGE0OGU5OGY3NDEiXSwiaWR0eXAiOiJ1c2VyIiwiaXBhZGRyIjoiMTQzLjk3LjIuNDEiLCJuYW1lIjoiRGF2aWQgRmxvb2QiLCJvaWQiOiJkNTRlYTg5ZS03ZTJkLTQzYTQtYmU0NS01NmMzNTA5ZWQzYTYiLCJvbnByZW1fc2lkIjoiUy0xLTUtMjEtMjIwNTIzMzg4LTEwODUwMzEyMTQtNzI1MzQ1NTQzLTIyMjIyODEiLCJwdWlkIjoiMTAwMzNGRkY4REJDNUUwNyIsInJoIjoiMC5BUUlBTmFLa091SzIxVWlSbFhfUEJiUlpzRVpJZjNrQXV0ZFB1a1Bhd2ZqMk1CTUNBSjAuIiwic2NwIjoidXNlcl9pbXBlcnNvbmF0aW9uIiwic3ViIjoid29sTWxiQWlrRC1Mbi13QUg0dDV3Z3EwajRCX3pUb2ZteGFkaVl0Sy1LQSIsInRpZCI6IjNhYTRhMjM1LWI2ZTItNDhkNS05MTk1LTdmY2YwNWI0NTliMCIsInVuaXF1ZV9uYW1lIjoiREZMT09AZXF1aW5vci5jb20iLCJ1cG4iOiJERkxPT0BlcXVpbm9yLmNvbSIsInV0aSI6ImpMemtiMGtROEVHZGNDU2k2UjlyQUEiLCJ2ZXIiOiIxLjAiLCJ3aWRzIjpbImI3OWZiZjRkLTNlZjktNDY4OS04MTQzLTc2YjE5NGU4NTUwOSJdLCJ4bXNfdGNkdCI6MTM0NDU4MzYwMn0.1Uv93VEKN2p57uDZRgZzLoGnFyMr82494gVxx5IVW5G_Syw3s29GyvQvB8fvTiL51nR0colOh-ZvPVBiDm6DMLeMsV5VjJRpCA8I9HuszR5RsnC-di1cUjFROJs-JHQLtOJw4cSI5uJQ9AhHNot2yBrWHovOax4FVGY8hI0m388OE85Srb5NChmPwsjF2oEhT_a7E5ZVuQA9DVlhDip0dmLBrF2cMupjoOHXeepmxsW7fvQczqjVhhZmWhMPjvtWikLFHiyGo5fKDTjCSsqzRisTw-vHF-j3OxfNHmFCTnhxnYA3oYXHRvz0JqSkpEn6pvnNpn4mY2TX40Ls3OTeZw\"" + "access_token = \"eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiIsIng1dCI6IktRMnRBY3JFN2xCYVZWR0JtYzVGb2JnZEpvNCIsImtpZCI6IktRMnRBY3JFN2xCYVZWR0JtYzVGb2JnZEpvNCJ9.eyJhdWQiOiJodHRwczovL21hbmFnZW1lbnQuY29yZS53aW5kb3dzLm5ldCIsImlzcyI6Imh0dHBzOi8vc3RzLndpbmRvd3MubmV0LzNhYTRhMjM1LWI2ZTItNDhkNS05MTk1LTdmY2YwNWI0NTliMC8iLCJpYXQiOjE3MjQxNjE0MTUsIm5iZiI6MTcyNDE2MTQxNSwiZXhwIjoxNzI0MTY2NTM2LCJhY3IiOiIxIiwiYWlvIjoiQWJRQVMvOFhBQUFBMFQvWmdFL3FndFlJUVJMZGNWK1dVZUJXWXFiVmF1dFRHNStBMjN6a3duRElJenkwZDd1cHJVTFpuTjdvUURZbUo5TEovbVVNV2pxQWU5clgzUWVYd29OK0QyLytwSU1xNzg5ZzhYcW0rTmRETGNhRmd6cSt5Y2d4dlJ1VVdCdVNyb001S00yUGRXOCtaendBR0hmV2E1L3E5Vll4RWVBR2M1d00yUmJMSzZlbUl2TU8wQThtbGc1YldWWFlVUnB4M21DVnNrcGZmWmZ1cDZ6bHVNQng5bXR2U1gvMUs0eHFBSUdXOGxTWnhSND0iLCJhbXIiOlsicnNhIiwibWZhIl0sImFwcGlkIjoiMThmYmNhMTYtMjIyNC00NWY2LTg1YjAtZjdiZjJiMzliM2YzIiwiYXBwaWRhY3IiOiIwIiwiZGV2aWNlaWQiOiIzNzVmZjhhNS0xOTg5LTQ0NWEtYTkzYi1mZGZiMDg4N2Y1ZmIiLCJmYW1pbHlfbmFtZSI6IkZsb29kIiwiZ2l2ZW5fbmFtZSI6IkRhdmlkIiwiZ3JvdXBzIjpbIjk0Y2RkZjAwLWRlMGYtNGM1OC1hMDM0LWRlMzQ2NDU3ZmRhMCIsIjZhMGY3MzAxLTAzOGUtNDIzMS1hZDU2LTZkMDFhZGNiODRlZiIsImQ4NjJjYTAyLTlkOTgtNDczZC05NzYwLTUxYzhlZjY5YmVmZiIsImZmYWVhNjAzLTc1OWEtNDQ5MC05MzdjLTg3NTI2ZjQyZDMxZSIsImVmMjlkYTA1LWE0ODktNGE4NC05ODNkLTZlMmNlMzJhMWM0ZiIsImZhY2IwNDA2LWE5OWUtNGIxYy05YmQxLWI1ODkwM2FkNjBjOSIsImNiYjlmYzA2LWM1ZDYtNDg3Mi04Yzg1LTQ4MDI0ZTg5YzY0OSIsIjllYjI2MzA5LTM1ZTAtNGE4MC05NmFkLWU0YzllYzU3MTFiZSIsIjk2MjlhMjBhLWVhZWMtNDliMy05Nzg0LTBhZTlhNjU3MGZjOSIsIjFkYjZiYTBjLTFkMmYtNGQ3Ni05ZGFlLTA4ODFlNTkxM2M1YyIsImNiMzhjMjBkLWY0YTMtNGVjMy1iMjU4LWVkMDJjOTM2ODRmMCIsImZiM2RjYjBkLTIxMmQtNGU3OC1iNTNmLTNjN2I5MzJlZTI1YiIsIjE5M2NmMTBkLTdkZDAtNDZkNy1hMTAxLTQ1NTgyMDIzNGE1NiIsImY2Mzg0ODBmLWIyOTUtNDc3Zi1iZGVhLWI5OWVkNDIxZGJhMiIsIjMxOTg5ZTBmLWQ2ZTUtNDhlNy05OTUyLTEwYTdmZmFlM2JiOCIsIjQ5MWYyMjEwLTQ2NWEtNDcwZC1hZTkwLWQwMGRmZDI3Yzk3YSIsIjA3N2UzODExLTc4M2EtNGI5YS05YTRiLTViOTllN2U2NzFjMCIsIjMxMDA0NzEyLWQ5MGUtNDlmNi05ZTc5LTJlOTdmNjhmNmQ1MCIsIjA0N2ZiMDEyLTY5NGMtNDRjMy04ZDQ4LTkyN2NiMzQ5Nzg4YyIsIjMzZThjZDEzLTg1YzUtNGVkYS04NTM3LTBiZjIwMDkwZjA0YyIsImI3MmI2ODE0LWQxNzItNGUzNi1iYjkwLTBiM2YxNWE5MTk3NyIsImYyYzk4ODE1LWUxNjYtNGY5MC1hNWY0LTFmNTM1ZjVlZGNjOCIsIjcxMTNjMzE1LWM0YjEtNGRjNi1hZDZiLTBhMWUyMTIyOGEwMSIsIjA1Yjk2ZDE2LWI2NDQtNGZjZi04YWU1LWZmMGMwZjgyNWI1ZiIsImJjMzlmMjE3LTQ0MTktNDI5MS05NTI3LTM1NjE0NzdhMmZiNyIsImMwMzNlZDE5LTBiMWMtNGQ1OS04ZTAxLTA4M2VkN2NkMzQ5ZCIsIjAwMDMxMTFiLTBjMmUtNDA0MS1iMWRkLTBiMmVkMWVmNmY3OCIsImJmYzEzOTFjLTY4MTQtNDE0My04M2M3LTNjNDZhZjhjNzhhOCIsIjMwNzI4ZTFlLTQ2ZjAtNGI2MS1hMGJjLTZiYzNmNzM2MGI4YSIsImM2OGU1MDFmLTZkMWYtNGI0Yi04ZjM3LTBjNDY4MTU2ZGRmMiIsIjVhZjA5OTIxLWZkNmEtNDYxMS05ZWE5LWEzYmZiMWYzNWJiZSIsIjdlZjM2YzIzLTNmNmItNDMyMS1iMDI4LWI4MWE5YjQ1OTk5MSIsImRmNjVlYTIzLTg4YzMtNDMxYS1hODdmLTYwNTVjZjliZDU0OSIsIjNiOTVkYTI3LTMzN2EtNDRjMS1iNjlkLTU2NDRiNTFhNDgzNiIsIjhmYTNiYjI4LTg0ZjktNGIwNi1hNGQ1LTA4OTA3ZThkZDZmYSIsImUwNDBjNDI4LWFjNTctNDU0Mi1hMzI4LTJmNjcyYTM0Y2NiOCIsImUxY2FhNDJhLWJmNjktNDM5MS05YTVlLWM4Y2ZkYWUxYjQxNyIsIjlmZTZlNjJhLTMxMTgtNDcwNC1hZmJlLTgzNDgyMmNiMGQ0MyIsIjBiMTAyYjJjLTNjOTctNGI3Ni1iNGRmLWUwMzk4Njg2YWY1NiIsIjc0OGEwMjJlLTk2OGUtNGE5NC1hZGUzLTkyNGUyNmJhZGFhOCIsImRhNWEzMzJlLTE0NzAtNDBmMy05ODAzLTQ5NzVlNTY5NGU1YiIsImIzZGU0MzJmLWRmNzUtNGJjYS1iMDg2LTlmODg4M2Q2OGRkMCIsIjRmNmY0ZTJmLWIyMjMtNGU3My1iMzMzLTViYjFmNzYwMTRmNCIsIjI5ZTAzNTMwLWY4MTgtNDAzNi1iMjhiLTk3YjhiOTRmMjY1MCIsImFkNTc3ODMyLWEwZTEtNGEwOC1iYmY0LTEyNzI0M2M2YTc1MiIsIjA1MDYwYTM0LTgwYjAtNDc4My1iYzRmLWIyNmZmNmRiODY5MSIsIjFkZmViZDM1LTdlNGQtNGYzOS05MjI1LTAxMWRhZGRkZGViMCIsIjMwNDk0ZTM2LTIwNDEtNDA5Ni1hY2Y4LWIzNjg0YTYzMDllNCIsIjVkZDU1YzM3LTBiODQtNDBiNC04NzYxLTUzYmI5YTkzYWM4ZiIsImE1ZWRmMDM4LTZjMmYtNDQzOC04ZjJmLWNkZjVhZWI5MWFjNSIsIjM5MjcwOTNhLThkNTktNDExOS04MzNlLWU3YTAyZmI4ODc4YyIsIjYxMmFjYjNmLWUyZDgtNGI1Ni1iNTUwLTBmZTU1Njg5NDI4NCIsImI0MDc0ZjQyLTQwMWItNGM1ZS04MTdlLTZlZjgzMzU5YjdiZiIsIjhjYmVmZjQyLTdjNGYtNDFkNy1iODBlLWYyNDJmMTc4MWJlYiIsIjg0OWNmNDQzLTljNWMtNGUzMi1hMmVhLTU1ZWFlZGQ5ZTZhNiIsIjI2MTU0MDQ0LTJhNzgtNGM4Mi05OTkzLTE2ZTA4NDVjNTU4MiIsImIzNmIyOTQ1LTFkMjQtNGM4My05YmE1LTI1MTRhNTZmNDlhMyIsIjZjNGEyNTQ5LTgwZWItNDZhOS05OTcyLTkyNzE3OGY0YzRmZiIsImE4ZmI1YTQ5LTIyMjUtNDc1Ny04NGZjLWQzZmRjMDc4YzQzZSIsImM1ZDI2MjRiLTY1ZjgtNDNkNC1hMWE4LTUyOGYxYzUyOTdkMyIsIjI0MWYxYzRjLTYyMTEtNDhhYy1hMDg4LWI2MmUxYzBiMzNhYyIsIjQ1MjJhNDRkLTY2MGQtNDRlYy04MDczLWFmZGMxNmU0YWQ3YiIsIjUzMDFiNDRkLWY1MDUtNDQwNy04NzZkLWVmYWU1YjdiZTk4MCIsIjYyYzRjYTRlLTAzM2ItNDZkYS1iNjc1LTFmMzg2YTkwMzIyYSIsImU3MTY3NzUwLTVjNWItNGI3Ni04OGIyLTY1OGViZjIxM2YxZSIsIjkxYTRhYjUwLWM0NDEtNGE5ZC1iZTQzLWEwMmM0ODVmOTVmMyIsIjFhYjMxODUxLWMzNmMtNDk2Zi04ZDRjLTNhN2FhNDU5MzRjOCIsIjMyZGFkODUxLTBiNmEtNDUzMS1iODM1LTMwNzBjYmU5MGM1MCIsIjY0ODBjNDU0LTVmNmEtNGQxOS1hZDRmLTVkMDRjODY1YmE4NSIsImFmMzRlMDU1LTJlODUtNDQ3Mi04YjczLWVhYjQxMGE3YTE1YSIsImIxYzI2MjU2LTk2ZGYtNDM3Zi1hM2FiLThlMTY2M2U2NmFiMSIsImYyOWYyNDU3LWMwNjAtNGM4My1hNjY2LTM0NTkyYzJjYzFmYiIsImRkZmMyNjVhLTBiYmYtNDljZS1iYjAyLTY3ZDUxM2I3N2MyYiIsImVhYjc3YzViLTBlNmQtNDVkZi04M2Y4LWE2NWQ5NzA3MjE1ZCIsIjIwNGVkNjViLWMzODYtNDI1NC1hNGU0LTg1ODM1MWVjZjU0NiIsImJjN2Y2ZjVlLWI0YjEtNGNmMy1hZDg3LWRlNmUxZWMyN2U0MyIsImQwMGZhODVmLTcyNzMtNGY5MS1iYWI4LTJlY2E4NTYxMmJhOSIsImUyY2U5MzYwLWRiZDYtNGRkNC04NTUyLTgwNjJhNTE3YjVkZCIsIjczMjk5YjYwLTYwMjMtNGNmZC05OGMxLTVkNjVlMmZjYjQ3ZiIsIjBmZTFlZTYwLTYzMDUtNGQwYS05YjEwLTMxYjUwMjZmMjVjNyIsIjkwMDcwZjYxLTY1NzItNGQ0Zi04ZDA0LTI1YTRhZWI1YjM3OSIsIjc1MmFmNTYxLTI5ODYtNGQ0MS05NjA0LWZlNjkyOGE2ZTU2NSIsIjNlMTM2OTYzLTFmMzktNDhiOC1iNWYyLTI2NGRhMDFhN2FhNSIsImIwMzE1OTY0LThjYjMtNGQxMi04YWQ3LTk0ZTg4Y2E2YjcwNyIsIjEyZWViOTY1LWM1NWYtNGJiNC04ZGJiLWY0ZTZlMDhjYzMwYSIsIjkzNDVlODY2LWZhZmYtNGU3ZS1iOWMwLTZkYjJkNjU4Zjc5MyIsIjQ4ZWQ0YzY3LWM3YTctNDczYi1hYjdhLTY1MTIxYTc0NzM3OSIsIjliNjE3NTY5LTY3NDUtNDVhNS1hN2YyLTIwYTc2ZmNkYTgwZiIsImIxMzFkNTY5LWVlM2UtNDY4ZS1iM2FiLTYxMDQzZDlhMjY1ZSIsIjY4YjZkNDZhLTk2OTItNDllMS1hMWE2LWIwNzVmNmEyZTlkOSIsIjlmNjJlYzZjLWVlNDEtNDRlOS05MTMxLTk5Yjc5MjU0MTM4MiIsImZjNjQzNDZlLTBmNTItNGNkMi05NTQ2LTU5NjVmZDQyZGUyNSIsIjFhYzEzNzZlLWYwNGEtNDRiNi1iZTU4LWFhMTYwMDNlYTk2ZSIsIjljMzhmYTZmLTRhOGYtNDI1ZC1hYjQ4LWI3Nzk3YWNjZjhkZSIsImZlY2U0YTcwLWE5NjgtNGI3ZC05OGJhLTlhZTEzMDk3OGM5YyIsIjJjODhmOTcxLTk3MmMtNGM0OC1iNmM2LTA4NjMxYzE5MTRhNyIsIjBkZjUyOTc1LWRiMTktNGZhMy1iY2Q4LTYxZDhiMmUyODJkOCIsImMxMGU3YTc2LWMyZGMtNDkzYy04YjUyLWY2MWFlMmM4YmUwNiIsImRhZWYwNDc3LTM0MWItNGRkZS05M2ZjLTRiODE5MDExYjI5NCIsIjg5NDlhMjc3LWRhMzQtNGJiMC1hZjAxLWJkYmRmZjY1ZWI2YSIsImM0MGRlMDc5LTJhNTYtNDkzMi04ZjM1LTA1Mjk3OWJkYzBmYyIsIjFiNGM4ZTdiLWFjMGMtNDE5YS1iODcxLTc4OWVmYTYxYTY3ZSIsImZiN2I2NTdlLWJhNTktNDhjMy04NjU2LTIyNDRmNzcwMzNjZCIsIjMwMDE2MDgwLTI4NWMtNDdjNy04OGQwLTIwZjJhYjU4YjczMCIsIjQ3MzEzZjgxLTA2Y2QtNDk3OS04MDA4LTQxY2E2YTkwN2FlNyIsImM2M2M3NTg0LWQ1MTgtNGM1Ni04MTM4LWU3YTAzMjNiNzNiOCIsIjc1ZmZiMzg0LTRlZDktNDFjYy04ZjMyLTE2N2I4NmJlMzhkMiIsIjUxOGIwYzg2LWZhYWEtNGJiNi1hMTg4LTlkZmM0NzliNDkxMSIsIjQxMjE3YTg2LWU1NWUtNGQ4OC1iYzdjLWExZTEwMTFmNzY0YiIsIjc5OGY5Yzg4LTBlMWEtNDI4OC1iNzA0LTBkNjgzNDUwNzBlNCIsImJmM2MwZjhhLWI0OWEtNDNjNC1iODdhLThkODJlNDhkZGE4YiIsImYyYzhjYjhiLTI3NjktNGI0Ni1hOTY2LTY3OGEyNWE5MWQzYyIsIjZjNThlZjhiLWM0YjItNDM4Zi05NDc1LWI5ZGYxY2FiOGMwZiIsIjNlYWI5OThjLTgyNGQtNGFiOS04NWE1LWU0MjRhMGEyMTFjMSIsImU0MTFjMDhkLTcwNDMtNGQxYi1iYmMxLTExNjUwZWMyZTJhZSIsIjdkNjNkYThkLTQ3MzAtNGJhMC05NmJlLWFlZWFiZTY2ZWY4MSIsIjFhYmNiMzkwLTgwMTItNDUwZi04OWRlLWI5ZjBmZDdmMjAzMiIsImNjYzNmNzkwLTI2YzItNDIwOC1iYzVmLTk3M2Y2ZWNjYmU0NyIsIjAwMDVkMzkyLTc1N2QtNGFjNS05M2MyLWZhNDE1ODZiYjNiZiIsImJiMjNmYTk2LWU4NTktNDBmZS05ZTVjLTM5NWZjNjBmYWJhZCIsIjFlOTdhYzk3LTY5NjAtNDA3MC04ZTQxLTczYTFjMWM5YWE4NCIsIjgwNjdkMjk4LTNmNTUtNDQyMi05ZTYxLTU3MWFjOGRiNjg0YiIsIjAwOWI1MzljLWRjZTMtNDczYS04NjQzLTdkYjk0ZDE2Mjg0NCIsIjM3YjQ4MzllLTViMGMtNDI5My1hZjg4LTFhMjVhYTgwMGY2NyIsImE2MzYyNjlmLWE5NjctNDViMi1iYjRlLTNlMDJkYmI2MjRhMiIsImE0N2RmMmEyLTc5OWEtNGQ4ZC1hNzQ2LTU3OTQxMWFjZDE4NSIsImRmNDFlNWE1LTk3NDktNDMwZi1hNDA5LTJiNjM1MzQzMzAzYSIsImQ2MTMwNGE2LTI0MzYtNDg2ZS04YjQwLWQ1MzUxMmYzYTViZiIsIjFjMTc0M2E4LTg5NzEtNDUyYi1iM2ZkLTY5YWVhM2E5NzUyMyIsImVkODk4M2E4LWFmNzQtNDM0Ni04NTZmLWI2MzE4ZjI1YjYyOCIsImI2Mjc1MWFhLWI5MDMtNDgxNS1iZjRhLTZjYTFmZDY0M2VlNyIsImU1MjZlZWFjLWE0NDAtNDM2NC05MTRkLTEwOWNmZTM3OTQxZSIsImFmZjRiMGFkLTdjM2MtNGQ5NS04M2RmLWJiNTYxOGY2MWQwYiIsIjI4ZTQ2MWIxLTBkY2QtNGU2ZC1iMzZmLTRmNDc0N2Y3MjlhYiIsIjFmYmRkZWIyLTFkYmEtNDFiYS05Y2E1LWRjNGJkMWMxNTBhYiIsIjUyOTAzNGIzLWM0NTAtNDBhOC1iY2M1LWM0MjM4MzJmNDU2MiIsImQxN2JmMGIzLTkzOTgtNGI3NC1hZjNjLWNjZTA1NGQ2ZWY1MiIsImY5OTUyOWI0LWE1OWMtNGJhMS1hMzJkLWZhZDBkZTMwMDlhYyIsImZhNDE3MWI0LTI3NDAtNGIzNi1hMDI2LTgwZGVjY2VjNTU0MCIsIjI5Mzk4NGI0LWU1MTktNDY3Ni05ZDJhLTZmYTNmNjcyYmQxYyIsImNiNDIyNWI3LTM1YWYtNDUzMy04ODlmLTcyNDFjOTlmMTUxYyIsIjBkN2FkYWI5LWYxOWQtNGFmOS05ZTZiLTdkZDVhM2Q3ZDk5ZiIsIjNlYTgzNmJjLTRhMjUtNGZkYi05ZTE4LTQ2NTQ3NTQ5M2Q2ZiIsIjE2ZjgxMGJlLWUyNzktNGUyMi04MWMxLTYyZjQ5NDZjYTUwNiIsImRlNTBlN2JlLTI1YzctNGY5My1hZDUyLTNiZjVhZTgxNTMyMiIsImE1MTBjZWJmLTFlNzktNGM5Yi1iMjFlLTY5OWY5ZDc1MTlmOSIsImJkNjNmNmMwLWNjNjEtNDVlYi1iYjBhLTk3NTE4NWZjNjAxZiIsIjVmMDZkY2MxLTMyNjMtNDFmNi05YmZhLWZhNmYwY2ViODZkOSIsIjRmNjA4MGM5LWNlMzgtNGQ1ZS1iMTY3LWVjNTM3N2JkZGY4OSIsImI0OWIxNGNhLTAzYzYtNGVjMy1iY2E1LTBkNDM2OTM1MDZhYiIsImRhMzM3MWNhLTgwNjItNDY3MS1iMGM4LTUzOGE5YzZmMmFkZCIsImU4OTE0OGNiLTRhMWQtNDM2Ny04YWE0LWM3YWE2NDQyZjQ3MSIsImI0YzQ3MWNiLWYyNjMtNGUyMi1hMDM2LTE2ZDhkMTY0NDZmZSIsImE4NmI4MWNjLWEwZmEtNGRjZC1hNDcwLWM5ZDRmZmI0ZDM0NyIsIjRlYjEzZmNmLTIwYjYtNDc4NC1hZTUzLTMxNDUyYWU3ZmVkMSIsIjA3Yzk2YmQyLWJkNjItNDNkNS05Nzg4LWE0ZDA0Mzg2Y2U1OCIsIjcxMzM3YWQzLTFmNmItNDk3ZS05YTQ1LWZjNmU5NzgyMzc4YiIsIjBiYWZhMGQ3LTNhMDQtNGEwZi05MTdkLTQ3NDJmYjQxMzVkZSIsImE5M2ExM2Q5LTYzMTEtNDFlMS1iM2I2LWY5OWQ0ZmIwMmVhZCIsImYxNjc5NmRhLTM0ZjMtNGY2YS1hYzQzLWM2YTE2OTFmYmIzYyIsImZhMjA2MmRjLTYwNjgtNDYxYy05MmFkLWJiZmY2MDEyNmQxOSIsIjQ2MjkzYmRkLTE4YzEtNGRhYy1hYWE0LTE4MGFhNGIzMzdhZCIsImJkMjVjNGRkLTcxYjgtNDVkMy04NzAyLTk4YzQ3ZmUwMDE5ZCIsIjA5NDVjNGRmLTAxZjEtNGQxYy05MTk3LTZhYTdhZjRjZTczNyIsIjRjYmJkNWRmLWFjMjktNDZmYy1hMjZlLTU3MGMxZjQ4NTk4OCIsImI5MDY0MGUwLWNjNTgtNGU1Yi1iMjFiLTljMTc5NzA4MGRiMyIsIjYxYzk3M2UwLWEwMzMtNGJiYS05YTM5LTRkZmZhOGEzMTJkMiIsImVmYjA3MWUyLWNmYWEtNDBjNy05OGU1LTcyMDU2NmEwZjIyNSIsImI4YTNiM2UzLWQ5ZGYtNDk1Ni05ZmE3LTZkYWE2NTA2MjU3OCIsIjYyMGEyMGU2LWNmMDYtNDA2Yi1hNzUwLTFmM2M0Y2MwYWMzMyIsIjFhYjNjOWU3LWRkZTAtNGFmMS1hZDg5LWU3ZmRlYjdiYjhkOCIsIjJlZjMyYWVkLTBmOGMtNDE0My04ZTIyLWM1MWY4NjFiMzU3NyIsIjI2YjNiZGVlLTFjOGYtNDRlYy04OWJkLTA2ZjgwM2IxNzc4NyIsImQ5OWNlZGVmLWJjODctNDFjYS1iYWFkLTU3M2ZjM2VmMjBlZCIsIjgyZjhmM2VmLTkwNjAtNGEyZi1hNTg3LTVmNDdjYTIzOGFiNyIsImNhNDVmOWVmLWExNzktNDhmOC04MzFkLTk0MWM0ZWMyMjExMyIsImIwYWNiN2YwLTRiZGQtNGQ4My1hY2IwLTI0ZGQ4NmNhZDkwNCIsIjEwNGIyOGYyLWViMmQtNDQ1ZS04ZDRiLTAwMTFlMTE5NmZhZCIsImVmN2ZjMGYyLWU4NTUtNGI3ZS05NzZlLWY0ZDhhZTcwYjVhOCIsIjIxMTU0MmYzLWNjYmQtNGQ5YS05M2E2LWM1ZmJiZDkyMTlhMyIsImZmZDY0NmY0LTkxOTQtNGU2Ny04YTM5LWI2ZjkyZTc3NzU2NCIsIjc5ODYzMmY4LWIwNDUtNGNjMS1hODRkLTgxZWQ1Njc3MWNhYiIsIjU2N2ZiNmY4LTcwNTgtNDk5Ni1hNTNiLTJmMmMxYmQyMTg0ZCIsIjJmNGFiM2Y5LTY4MmQtNDZkNi05MjA0LTdlYmRhN2JiZDQxYyIsIjk2MDE4OGZhLTQwOGUtNDkwMS05NTBlLWQ2ZTE5ZmVkNTRlZiIsIjkwYjZkN2ZhLWZhMjYtNGE4NS05MmUzLTlkYTIwNWVkNDZjNSIsImFhYzM3YmZiLTdlODEtNDMxYy1iYjcwLTVjZTRlODBiY2JkZCIsIjk5ZTYyYWZmLTRmNzYtNDYwYS1hNjExLWY4YTQ4ZTk4Zjc0MSJdLCJpZHR5cCI6InVzZXIiLCJpcGFkZHIiOiI5MC4yNTAuOS40OCIsIm5hbWUiOiJEYXZpZCBGbG9vZCIsIm9pZCI6ImQ1NGVhODllLTdlMmQtNDNhNC1iZTQ1LTU2YzM1MDllZDNhNiIsIm9ucHJlbV9zaWQiOiJTLTEtNS0yMS0yMjA1MjMzODgtMTA4NTAzMTIxNC03MjUzNDU1NDMtMjIyMjI4MSIsInB1aWQiOiIxMDAzM0ZGRjhEQkM1RTA3IiwicmgiOiIwLkFRSUFOYUtrT3VLMjFVaVJsWF9QQmJSWnNFWklmM2tBdXRkUHVrUGF3ZmoyTUJNQ0FKMC4iLCJzY3AiOiJ1c2VyX2ltcGVyc29uYXRpb24iLCJzdWIiOiJ3b2xNbGJBaWtELUxuLXdBSDR0NXdncTBqNEJfelRvZm14YWRpWXRLLUtBIiwidGlkIjoiM2FhNGEyMzUtYjZlMi00OGQ1LTkxOTUtN2ZjZjA1YjQ1OWIwIiwidW5pcXVlX25hbWUiOiJERkxPT0BlcXVpbm9yLmNvbSIsInVwbiI6IkRGTE9PQGVxdWlub3IuY29tIiwidXRpIjoiMTdqd0IyaUVTVTZ0QWRoZU9wb19BQSIsInZlciI6IjEuMCIsIndpZHMiOlsiYjc5ZmJmNGQtM2VmOS00Njg5LTgxNDMtNzZiMTk0ZTg1NTA5Il0sInhtc19pZHJlbCI6IjEgMjIiLCJ4bXNfdGNkdCI6MTM0NDU4MzYwMn0.xqGlTWIA2l1oVqCMKNFt4L1_n5m2DrJPgmbH18MB8psacqNrWEpnL1pGhN8zFprrnN-BHksgou5oo8kuUvB3k-v0KY8HOZAXwZUUeXdwRASTrSwpLbR5qI3ICN2tXGeWQaWdnfe0M3fCoGuguazOmS4i9P60WwGDAYTdjLYUwmsgA6eonneXCN_1-aMWyHl6YwIKaH_aVylN5zozYtwIrOA7g6W9cOguVmj8qoHKZQDm8VvwSGfsRu9jr8Cr_PIqySMDl1DTdtRVm8fCsDHFhqk_vWTS8FGFC4CSoSNtfgZTGIS1vVeJbLovcckZSmcq2sJ8M9upvXFsHHRoFRW6jQ\"" ], "outputs": [], - "execution_count": 13 + "execution_count": 7 }, { "cell_type": "code", @@ -44674,57 +44786,57 @@ } }, "source": [ - "billing_periods = generate_billing_periods(2018, 5)\r\n", - "\r\n", - "schema = T.StructType([\r\n", - " T.StructField(\"Id\", T.StringType(), True),\r\n", - " T.StructField(\"BillingPeriod\", T.StringType(), True),\r\n", - " T.StructField(\"BillingAccountId\", T.StringType(), True),\r\n", - " T.StructField(\"Currency\", T.StringType(), True),\r\n", - " T.StructField(\"BeginningBalance\", T.DoubleType(), True),\r\n", - " T.StructField(\"EndingBalance\", T.DoubleType(), True),\r\n", - " T.StructField(\"NewPurchases\", T.DoubleType(), True),\r\n", - " T.StructField(\"Adjustments\", T.DoubleType(), True),\r\n", - " T.StructField(\"Utilized\", T.DoubleType(), True),\r\n", - " T.StructField(\"ServiceOverage\", T.DoubleType(), True),\r\n", - " T.StructField(\"ChargesBilledSeparately\", T.DoubleType(), True),\r\n", - " T.StructField(\"TotalOverage\", T.DoubleType(), True),\r\n", - " T.StructField(\"TotalUsage\", T.DoubleType(), True),\r\n", - " T.StructField(\"AzureMarketplaceServiceCharges\", T.DoubleType(), True),\r\n", - " T.StructField(\"BillingFrequency\", T.StringType(), True),\r\n", - " T.StructField(\"PriceHidden\", T.BooleanType(), True)\r\n", - "])\r\n", - "df = spark.createDataFrame([], schema)\r\n", - "\r\n", - "for billing_period in billing_periods:\r\n", - " print(f'Fetching billing period {billing_period}')\r\n", - "\r\n", - " data = fetch_billing_details(BILLING_ACCOUNT_ID, billing_period, '2019-10-01', access_token)\r\n", - "\r\n", - " new_row = [(\r\n", - " data['id'],\r\n", - " billing_period,\r\n", - " BILLING_ACCOUNT_ID,\r\n", - " data['properties']['currency'],\r\n", - " data['properties']['beginningBalance'],\r\n", - " data['properties']['endingBalance'],\r\n", - " data['properties']['newPurchases'],\r\n", - " data['properties']['adjustments'],\r\n", - " data['properties']['utilized'],\r\n", - " data['properties']['serviceOverage'],\r\n", - " data['properties']['chargesBilledSeparately'],\r\n", - " data['properties']['totalOverage'],\r\n", - " data['properties']['totalUsage'],\r\n", - " data['properties']['azureMarketplaceServiceCharges'],\r\n", - " data['properties']['billingFrequency'],\r\n", - " data['properties']['priceHidden']\r\n", - " )]\r\n", - " new_df = spark.createDataFrame(new_row, schema)\r\n", - "\r\n", + "billing_periods = generate_billing_periods(2018, 5)\n", + "\n", + "schema = T.StructType([\n", + " T.StructField(\"Id\", T.StringType(), True),\n", + " T.StructField(\"BillingPeriod\", T.StringType(), True),\n", + " T.StructField(\"BillingAccountId\", T.StringType(), True),\n", + " T.StructField(\"Currency\", T.StringType(), True),\n", + " T.StructField(\"BeginningBalance\", T.DoubleType(), True),\n", + " T.StructField(\"EndingBalance\", T.DoubleType(), True),\n", + " T.StructField(\"NewPurchases\", T.DoubleType(), True),\n", + " T.StructField(\"Adjustments\", T.DoubleType(), True),\n", + " T.StructField(\"Utilized\", T.DoubleType(), True),\n", + " T.StructField(\"ServiceOverage\", T.DoubleType(), True),\n", + " T.StructField(\"ChargesBilledSeparately\", T.DoubleType(), True),\n", + " T.StructField(\"TotalOverage\", T.DoubleType(), True),\n", + " T.StructField(\"TotalUsage\", T.DoubleType(), True),\n", + " T.StructField(\"AzureMarketplaceServiceCharges\", T.DoubleType(), True),\n", + " T.StructField(\"BillingFrequency\", T.StringType(), True),\n", + " T.StructField(\"PriceHidden\", T.BooleanType(), True)\n", + "])\n", + "df = spark.createDataFrame([], schema)\n", + "\n", + "for billing_period in billing_periods:\n", + " print(f'Fetching billing period {billing_period}')\n", + "\n", + " data = fetch_billing_details(BILLING_ACCOUNT_ID, billing_period, '2019-10-01', access_token)\n", + "\n", + " new_row = [(\n", + " data['id'],\n", + " billing_period,\n", + " BILLING_ACCOUNT_ID,\n", + " data['properties']['currency'],\n", + " data['properties']['beginningBalance'],\n", + " data['properties']['endingBalance'],\n", + " data['properties']['newPurchases'],\n", + " data['properties']['adjustments'],\n", + " data['properties']['utilized'],\n", + " data['properties']['serviceOverage'],\n", + " data['properties']['chargesBilledSeparately'],\n", + " data['properties']['totalOverage'],\n", + " data['properties']['totalUsage'],\n", + " data['properties']['azureMarketplaceServiceCharges'],\n", + " data['properties']['billingFrequency'],\n", + " data['properties']['priceHidden']\n", + " )]\n", + " new_df = spark.createDataFrame(new_row, schema)\n", + "\n", " df = df.union(new_df)" ], "outputs": [], - "execution_count": 14 + "execution_count": 8 }, { "cell_type": "code", @@ -44740,11 +44852,11 @@ } }, "source": [ - "target_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/billing/BillingPeriods.parquet\"\r\n", + "target_path = f\"abfss://usage@{storageAccount}.dfs.core.windows.net/billing/BillingPeriods.parquet\"\n", "df.write.format('parquet').mode('overwrite').save(target_path)" ], "outputs": [], - "execution_count": 15 + "execution_count": 9 } ] }, @@ -45098,7 +45210,7 @@ "spark.dynamicAllocation.enabled": "true", "spark.dynamicAllocation.minExecutors": "1", "spark.dynamicAllocation.maxExecutors": "5", - "spark.autotune.trackingId": "cca96f11-90e2-43e8-8e81-8e89874de2fa" + "spark.autotune.trackingId": "c88fc8da-726a-422b-8a19-80439853392f" } }, "metadata": { @@ -45123,8 +45235,7 @@ "sparkVersion": "3.3", "nodeCount": 3, "cores": 16, - "memory": 112, - "automaticScaleJobs": true + "memory": 112 }, "sessionKeepAliveTimeout": 30 }, @@ -50420,25 +50531,30 @@ "dependsOn": [] }, { - "name": "[concat(parameters('workspaceName'), '/prodcluster')]", + "name": "[concat(parameters('workspaceName'), '/sparkpool32')]", "type": "Microsoft.Synapse/workspaces/bigDataPools", "apiVersion": "2019-06-01-preview", "properties": { "autoPause": { "enabled": true, - "delayInMinutes": "[parameters('prodcluster_delayInMinutes')]" + "delayInMinutes": "[parameters('sparkpool32_delayInMinutes')]" }, "autoScale": { "enabled": true, - "maxNodeCount": "[parameters('prodcluster_maxNodeCount')]", - "minNodeCount": "[parameters('prodcluster_minNodeCount')]" + "maxNodeCount": "[parameters('sparkpool32_maxNodeCount')]", + "minNodeCount": "[parameters('sparkpool32_minNodeCount')]" }, - "nodeCount": "[parameters('prodcluster_nodeCount')]", - "nodeSize": "[parameters('prodcluster_nodeSize')]", + "nodeCount": "[parameters('sparkpool32_nodeCount')]", + "nodeSize": "[parameters('sparkpool32_nodeSize')]", "nodeSizeFamily": "MemoryOptimized", - "sparkVersion": "[parameters('prodcluster_sparkVersion')]", + "sparkVersion": "[parameters('sparkpool32_sparkVersion')]", + "libraryRequirements": { + "content": "azure.mgmt.billing\nazure.mgmt.consumption\nazure.mgmt.resource\nazure.identity\nazure.keyvault\n\n", + "filename": "requirements.txt", + "time": "2023-01-25T09:06:41.7702231Z" + }, "isComputeIsolationEnabled": false, - "sessionLevelPackagesEnabled": false, + "sessionLevelPackagesEnabled": true, "annotations": [] }, "dependsOn": [], @@ -50470,30 +50586,25 @@ "location": "northeurope" }, { - "name": "[concat(parameters('workspaceName'), '/sparkpool32')]", + "name": "[concat(parameters('workspaceName'), '/prodcluster')]", "type": "Microsoft.Synapse/workspaces/bigDataPools", "apiVersion": "2019-06-01-preview", "properties": { "autoPause": { "enabled": true, - "delayInMinutes": "[parameters('sparkpool32_delayInMinutes')]" + "delayInMinutes": "[parameters('prodcluster_delayInMinutes')]" }, "autoScale": { "enabled": true, - "maxNodeCount": "[parameters('sparkpool32_maxNodeCount')]", - "minNodeCount": "[parameters('sparkpool32_minNodeCount')]" + "maxNodeCount": "[parameters('prodcluster_maxNodeCount')]", + "minNodeCount": "[parameters('prodcluster_minNodeCount')]" }, - "nodeCount": "[parameters('sparkpool32_nodeCount')]", - "nodeSize": "[parameters('sparkpool32_nodeSize')]", + "nodeCount": "[parameters('prodcluster_nodeCount')]", + "nodeSize": "[parameters('prodcluster_nodeSize')]", "nodeSizeFamily": "MemoryOptimized", - "sparkVersion": "[parameters('sparkpool32_sparkVersion')]", - "libraryRequirements": { - "content": "azure.mgmt.billing\nazure.mgmt.consumption\nazure.mgmt.resource\nazure.identity\nazure.keyvault\n\n", - "filename": "requirements.txt", - "time": "2023-01-25T09:06:41.7702231Z" - }, + "sparkVersion": "[parameters('prodcluster_sparkVersion')]", "isComputeIsolationEnabled": false, - "sessionLevelPackagesEnabled": true, + "sessionLevelPackagesEnabled": false, "annotations": [] }, "dependsOn": [], diff --git a/s037-cost-management/TemplateParametersForWorkspace.json b/s037-cost-management/TemplateParametersForWorkspace.json index 9d21b30..34c38cd 100644 --- a/s037-cost-management/TemplateParametersForWorkspace.json +++ b/s037-cost-management/TemplateParametersForWorkspace.json @@ -566,6 +566,15 @@ "combine-recommendations-and-autofitcombometer_notebookSparkPoolEndpointRef": { "value": "https://s037-cost-management.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sprkpool33large" }, + "compute-consumption-plan_notebookSparkPoolNameRef": { + "value": "sprkpool33large" + }, + "compute-consumption-plan_notebookSparkPoolIdRef": { + "value": "/subscriptions/13d66f54-0a19-4912-b4f3-54d15897368d/resourceGroups/Synapse/providers/Microsoft.Synapse/workspaces/s037-cost-management/bigDataPools/sprkpool33large" + }, + "compute-consumption-plan_notebookSparkPoolEndpointRef": { + "value": "https://s037-cost-management.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sprkpool33large" + }, "compute-cost-drivers_notebookSparkPoolNameRef": { "value": "sprkpool33large" }, @@ -782,23 +791,23 @@ "vm-hub-deployments_notebookSparkPoolEndpointRef": { "value": "https://s037-cost-management.dev.azuresynapse.net/livyApi/versions/2019-11-01-preview/sparkPools/sprkpool33large" }, - "prodcluster_delayInMinutes": { - "value": 30 + "sparkpool32_delayInMinutes": { + "value": 15 }, - "prodcluster_maxNodeCount": { - "value": 10 + "sparkpool32_maxNodeCount": { + "value": 5 }, - "prodcluster_minNodeCount": { + "sparkpool32_minNodeCount": { "value": 3 }, - "prodcluster_nodeCount": { - "value": 10 + "sparkpool32_nodeCount": { + "value": 0 }, - "prodcluster_nodeSize": { - "value": "Large" + "sparkpool32_nodeSize": { + "value": "Medium" }, - "prodcluster_sparkVersion": { - "value": "3.4" + "sparkpool32_sparkVersion": { + "value": "3.2" }, "sprkpool33large_delayInMinutes": { "value": 10 @@ -818,23 +827,23 @@ "sprkpool33large_sparkVersion": { "value": "3.3" }, - "sparkpool32_delayInMinutes": { - "value": 15 + "prodcluster_delayInMinutes": { + "value": 30 }, - "sparkpool32_maxNodeCount": { - "value": 5 + "prodcluster_maxNodeCount": { + "value": 10 }, - "sparkpool32_minNodeCount": { + "prodcluster_minNodeCount": { "value": 3 }, - "sparkpool32_nodeCount": { - "value": 0 + "prodcluster_nodeCount": { + "value": 10 }, - "sparkpool32_nodeSize": { - "value": "Medium" + "prodcluster_nodeSize": { + "value": "Large" }, - "sparkpool32_sparkVersion": { - "value": "3.2" + "prodcluster_sparkVersion": { + "value": "3.4" } } } \ No newline at end of file