Merge branch 'master' into pandas_warning

UNSW-CEEM · May 15, 2024 · 43b8631 · 43b8631
2 parents 6e4799c + 7fbc99f
commit 43b8631
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 4 deletions.
diff --git a/.gitignore b/.gitignore
@@ -29,3 +29,8 @@ nemosis/check_new_bid_table_functionality.py
 *.pyc
 smoke.py
 gui.exe
+nemosis_test_cache/
+line-profiler.py
+profiler.py
+test.ipynb
+.vscode/settings.json
diff --git a/nemosis/data_fetch_methods.py b/nemosis/data_fetch_methods.py
@@ -2,6 +2,7 @@
 import os as _os
 import glob as _glob
 import pandas as _pd
+import warnings
 from datetime import datetime as _datetime
 from . import filters as _filters
 from . import downloader as _downloader
@@ -13,6 +14,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 def dynamic_data_compiler(
     start_time,
     end_time,
@@ -762,7 +764,7 @@ def _download_data(
     )
     return
 
-
+  
 # GUI wrappers and mappers below
 
 
@@ -833,5 +835,6 @@ def _static_table_wrapper_for_gui(
     "INTERCONNECTORCONSTRAINT": _dynamic_data_wrapper_for_gui,
     "MARKET_PRICE_THRESHOLDS": _dynamic_data_wrapper_for_gui,
     "DAILY_REGION_SUMMARY": _dynamic_data_wrapper_for_gui,
-    "NEXT_DAY_DISPATCHLOAD": _dynamic_data_wrapper_for_gui
+    "NEXT_DAY_DISPATCHLOAD": _dynamic_data_wrapper_for_gui,
+    "ROOFTOP_PV_ACTUAL": _dynamic_data_wrapper_for_gui
 }
diff --git a/nemosis/defaults.py b/nemosis/defaults.py
@@ -34,7 +34,8 @@
     "TRADINGREGIONSUM": "PUBLIC_DVD_TRADINGREGIONSUM",
     "TRADINGINTERCONNECT": "PUBLIC_DVD_TRADINGINTERCONNECT",
     "MARKET_PRICE_THRESHOLDS": "PUBLIC_DVD_MARKET_PRICE_THRESHOLDS",
-    "DAILY_REGION_SUMMARY": "PUBLIC_DAILY_REGION_SUMMARY"
+    "DAILY_REGION_SUMMARY": "PUBLIC_DAILY_REGION_SUMMARY",
+    "ROOFTOP_PV_ACTUAL": "PUBLIC_DVD_ROOFTOP_PV_ACTUAL",
 }
 
 table_types = {
@@ -72,7 +73,8 @@
     "TRADINGREGIONSUM": "MMS",
     "TRADINGINTERCONNECT": "MMS",
     "MARKET_PRICE_THRESHOLDS": "MMS",
-    "DAILY_REGION_SUMMARY": "DAILY_REGION_SUMMARY"
+    "DAILY_REGION_SUMMARY": "DAILY_REGION_SUMMARY",
+    "ROOFTOP_PV_ACTUAL": "MMS",
 }
 
 dynamic_tables = [
@@ -109,6 +111,7 @@
     "TRADINGPRICE",
     "TRADINGREGIONSUM",
     "TRADINGINTERCONNECT",
+    "ROOFTOP_PV_ACTUAL",
 ]
 
 display_as_Custom = ["FCAS_4s_SCADA_MAP", "PLANTSTATS"]
@@ -171,6 +174,7 @@
     "TRADINGREGIONSUM": "aemo_data_url",
     "TRADINGINTERCONNECT": "aemo_data_url",
     "MARKET_PRICE_THRESHOLDS": "aemo_data_url",
+    "ROOFTOP_PV_ACTUAL": "aemo_data_url",
 }
 
 filterable_cols = [
@@ -214,6 +218,7 @@
     "Max Lower Angle",
     "Max Upper Angle",
     "Bid Type",
+    "TYPE",
 ]
 
 table_columns = {
@@ -230,9 +235,11 @@
         "LOWER5MIN",
         "LOWER60SEC",
         "LOWER6SEC",
+        "LOWER1SEC",
         "RAISE5MIN",
         "RAISE60SEC",
         "RAISE6SEC",
+        "RAISE1SEC",
         "LOWERREG",
         "RAISEREG",
         "SEMIDISPATCHCAP",
@@ -255,9 +262,11 @@
         "LOWER5MIN",
         "LOWER60SEC",
         "LOWER6SEC",
+        "LOWER1SEC",
         "RAISE5MIN",
         "RAISE60SEC",
         "RAISE6SEC",
+        "RAISE1SEC",
         "LOWERREG",
         "RAISEREG",
         "SEMIDISPATCHCAP",
@@ -671,6 +680,14 @@
         "DISPATCHABLELOAD",
         "NETINTERCHANGE"
     ],
+    "ROOFTOP_PV_ACTUAL": [
+        "INTERVAL_DATETIME",
+        "REGIONID",
+        "POWER",
+        "QI",
+        "TYPE",
+        "LASTCHANGED"
+    ],
 }
 
 table_primary_keys = {
@@ -767,6 +784,11 @@
         "INTERVENTION",
         "REGIONID",
     ],
+    "ROOFTOP_PV_ACTUAL": [
+        "INTERVAL_DATETIME",
+        "REGIONID",
+        "TYPE",
+    ],
 }
 
 effective_date_group_col = {
@@ -823,6 +845,7 @@
     "FCAS_4s_SCADA_MAP": None,
     "MARKET_PRICE_THRESHOLDS": "EFFECTIVEDATE",
     "DAILY_REGION_SUMMARY": "SETTLEMENTDATE",
+    "ROOFTOP_PV_ACTUAL": "INTERVAL_DATETIME",
 }
 
 reg_exemption_list_tabs = {

diff --git a/nemosis/processing_info_maps.py b/nemosis/processing_info_maps.py
@@ -44,6 +44,7 @@
     "INTERCONNECTORCONSTRAINT": None,
     "MARKET_PRICE_THRESHOLDS": None,
     "DAILY_REGION_SUMMARY": None,
+    "ROOFTOP_PV_ACTUAL": None,
 }
 
 search_type = {
@@ -81,6 +82,7 @@
     "INTERCONNECTORCONSTRAINT": "all",
     "MARKET_PRICE_THRESHOLDS": "all",
     "DAILY_REGION_SUMMARY": 'start_to_end',
+    "ROOFTOP_PV_ACTUAL": "start_to_end",
 }
 
 date_cols = {
@@ -118,6 +120,7 @@
     "INTERCONNECTORCONSTRAINT": ["EFFECTIVEDATE"],
     "MARKET_PRICE_THRESHOLDS": ["EFFECTIVEDATE"],
     "DAILY_REGION_SUMMARY": ['SETTLEMENTDATE'],
+    "ROOFTOP_PV_ACTUAL": ["INTERVAL_DATETIME"],
 }
 
 filter = {
@@ -155,6 +158,7 @@
     "INTERCONNECTORCONSTRAINT": filters.filter_on_effective_date,
     "MARKET_PRICE_THRESHOLDS": filters.filter_on_effective_date,
     "DAILY_REGION_SUMMARY": filters.filter_on_settlementdate,
+    "ROOFTOP_PV_ACTUAL": filters.filter_on_interval_datetime,
 }
 
 finalise = {
@@ -233,6 +237,7 @@
     ],
     "MARKET_PRICE_THRESHOLDS": None,
     "DAILY_REGION_SUMMARY": None,
+    "ROOFTOP_PV_ACTUAL": None,
 }
 
 date_gen = {

diff --git a/tests/test_data_fetch_methods.py b/tests/test_data_fetch_methods.py
@@ -8,6 +8,7 @@
 from pandas._testing import assert_frame_equal
 import pytz
 
+
 class TestDynamicDataCompilerWithSettlementDateFiltering(unittest.TestCase):
     def setUp(self):
         self.table_names = [
@@ -23,6 +24,7 @@ def setUp(self):
             "TRADINGPRICE",
             "TRADINGREGIONSUM",
             "TRADINGINTERCONNECT",
+            "ROOFTOP_PV_ACTUAL",
         ]
 
         self.table_types = {
@@ -38,6 +40,7 @@ def setUp(self):
             "TRADINGPRICE": "REGIONIDONLY",
             "TRADINGREGIONSUM": "REGIONIDONLY",
             "TRADINGINTERCONNECT": "INTERCONNECTORIDONLY",
+            "ROOFTOP_PV_ACTUAL": "REGIONID-TYPE",
         }
 
         self.table_filters = {
@@ -53,6 +56,7 @@ def setUp(self):
             "TRADINGPRICE": ["REGIONID"],
             "TRADINGREGIONSUM": ["REGIONID"],
             "TRADINGINTERCONNECT": ["INTERCONNECTORID"],
+            "ROOFTOP_PV_ACTUAL": ["REGIONID", "TYPE"],
         }
 
         self.filter_values = {
@@ -64,6 +68,7 @@ def setUp(self):
             "INTERCONNECTORIDONLY": (["VIC1-NSW1"],),
             "CONSTRAINTID": (["DATASNAP_DFS_Q_CLST"], [0]),
             "DUID-BIDTYPE": (["AGLHAL"], ["ENERGY"]),
+            "REGIONID-TYPE": (['NSW1'], ['DAILY']),
         }
 
     def test_dispatch_tables_start_of_month(self):
@@ -86,6 +91,7 @@ def test_dispatch_tables_start_of_month(self):
                 "TRADINGPRICE",
                 "TRADINGREGIONSUM",
                 "TRADINGINTERCONNECT",
+                "ROOFTOP_PV_ACTUAL"
             ]:
                 expected_length = 10
                 expected_first_time = "2018/02/01 00:30:00"
@@ -146,6 +152,7 @@ def test_dispatch_tables_end_of_month(self):
                 "TRADINGPRICE",
                 "TRADINGREGIONSUM",
                 "TRADINGINTERCONNECT",
+                "ROOFTOP_PV_ACTUAL"
             ]:
                 expected_length = 6
                 expected_first_time = "2018/01/31 21:30:00"
@@ -197,6 +204,7 @@ def test_dispatch_tables_straddle_2_months(self):
                 "TRADINGPRICE",
                 "TRADINGREGIONSUM",
                 "TRADINGINTERCONNECT",
+                "ROOFTOP_PV_ACTUAL"
             ]:
                 expected_length = 48
                 expected_first_time = "2018/02/28 21:30:00"
@@ -248,6 +256,7 @@ def test_dispatch_tables_start_of_year(self):
                 "TRADINGPRICE",
                 "TRADINGREGIONSUM",
                 "TRADINGINTERCONNECT",
+                "ROOFTOP_PV_ACTUAL"
             ]:
                 expected_length = 2
                 expected_first_time = "2018/01/01 00:30:00"
@@ -303,6 +312,7 @@ def test_dispatch_tables_end_of_year(self):
                 "TRADINGPRICE",
                 "TRADINGREGIONSUM",
                 "TRADINGINTERCONNECT",
+                "ROOFTOP_PV_ACTUAL"
             ]:
                 expected_length = 2
                 expected_first_time = "2017/12/31 23:30:00"
@@ -354,6 +364,7 @@ def test_dispatch_tables_straddle_years(self):
                 "TRADINGPRICE",
                 "TRADINGREGIONSUM",
                 "TRADINGINTERCONNECT",
+                "ROOFTOP_PV_ACTUAL"
             ]:
                 expected_length = 4
                 expected_first_time = "2017/12/31 23:30:00"