Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

added time range splitting and process status management #41

Merged
merged 14 commits into from
Jun 27, 2022
Merged

added time range splitting and process status management #41

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
88ad96b
added time range splitting and process status management
Dolgalad Apr 19, 2022
f5ced69
replace parameter_concat function by speasy.products.variable.merge
Dolgalad Jun 1, 2022
03bde87
removed datetime check in dl_parameter
Dolgalad Jun 1, 2022
46e32a8
seems to work
Dolgalad Jun 1, 2022
81d5540
removed print, use logger
Dolgalad Jun 1, 2022
7490295
added dl_parameter_chunk
Dolgalad Jun 1, 2022
4fa5e20
[AMDA] Makes max chunk size configurable
jeandet Jun 26, 2022
1fbdea4
[AMDA] Too long request message has to be a warning, not a debug message
jeandet Jun 26, 2022
6942230
[AMDA] Adds test for long requests which is disabled by default
jeandet Jun 26, 2022
228439e
[AMDA] Fix typo
jeandet Jun 26, 2022
d6ac8aa
[Doc] set language to avoid sphinx warning
jeandet Jun 26, 2022
9ac0d49
[GH Actions] Adds long tests for CI
jeandet Jun 26, 2022
7390321
[LGTM fix]
jeandet Jun 26, 2022
482c3c6
[AMDA] increase long request test coverage
jeandet Jun 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[AMDA] Makes max chunk size configurable
  • Loading branch information
@jeandet
jeandet committed Jun 26, 2022
commit 4fa5e20f6353f4da72917558c05c3fad2a65e8e6
1 change: 1 addition & 0 deletions speasy/config/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,4 @@ def remove_entry(entry: ConfigEntry):
amda_username = ConfigEntry("AMDA", "username")
amda_password = ConfigEntry("AMDA", "password")
amda_user_cache_retention = ConfigEntry("AMDA", "user_cache_retention", "900") # 60 * 15 seconds
amda_max_chunk_size_days = ConfigEntry("AMDA", "max_chunk_size_days", "10") # 60 * 15 seconds
14 changes: 8 additions & 6 deletions speasy/webservices/amda/_impl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from typing import Optional

# General modules
from ...config import amda_password, amda_username, amda_user_cache_retention
from ...config import amda_password, amda_username, amda_user_cache_retention, amda_max_chunk_size_days
from ...products.variable import SpeasyVariable, merge
from ...inventory import data_tree, flat_inventories
from ...inventory import reset_amda_inventory as reset_amda_flat_inventory
Expand Down Expand Up @@ -85,9 +85,12 @@ def update_inventory(self):
data_tree.amda.Parameters.__dict__.update(data.dataRoot.AMDA.__dict__)

self._update_lists()
def dl_parameter_chunk(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[SpeasyVariable]:

def dl_parameter_chunk(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[
SpeasyVariable]:
url = rest_client.get_parameter(
startTime=start_time.timestamp(), stopTime=stop_time.timestamp(), parameterID=parameter_id, timeFormat='UNIXTIME',
startTime=start_time.timestamp(), stopTime=stop_time.timestamp(), parameterID=parameter_id,
timeFormat='UNIXTIME',
server_url=self.server_url, **kwargs)
# check status until done
if url is not None:
Expand All @@ -100,17 +103,16 @@ def dl_parameter_chunk(self, start_time: datetime, stop_time: datetime, paramete
return var
return None


def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[
SpeasyVariable]:
dt = timedelta(days=1)
dt = timedelta(days=int(amda_max_chunk_size_days))

if stop_time - start_time > dt:
var = None
curr_t = start_time
while curr_t < stop_time:
if curr_t + dt < stop_time:
var = merge([var, self.dl_parameter_chunk(curr_t, curr_t+dt, parameter_id, **kwargs)])
var = merge([var, self.dl_parameter_chunk(curr_t, curr_t + dt, parameter_id, **kwargs)])
else:
var = merge([var, self.dl_parameter_chunk(curr_t, stop_time, parameter_id, **kwargs)])
curr_t += dt
Expand Down