added time range splitting and process status management

.github/workflows/PRs.yml

@@ -35,10 +35,18 @@ jobs:
     - name: install texlive for Pandoc
       run: sudo apt update && sudo apt install -y texlive pandoc && pip install wheel
     - name: Test with pytest
+      if: matrix.python-version != '3.8'
       run: |
         pip install pytest pytest-cov sphinx pandoc
         pip install -r docs/requirements.txt
-        pytest --cov=./ --cov-report=xml
+        pytest
+        make doctest
+    - name: Test with pytest (coverage + long tests)
+      if: matrix.python-version == '3.8'
+      run: |
+        pip install pytest pytest-cov sphinx pandoc
+        pip install -r docs/requirements.txt
+        SPEASY_AMDA_MAX_CHUNK_SIZE_DAYS=25 SPEASY_LONG_TESTS="" pytest --cov=./ --cov-report=xml
         make doctest
     - name: Check that release process is not broken
       if: matrix.python-version == '3.7'

.github/workflows/tests.yml

@@ -35,10 +35,18 @@ jobs:
     - name: install texlive for Pandoc
       run: sudo apt update && sudo apt install -y texlive pandoc && pip install wheel
     - name: Test with pytest
+      if: matrix.python-version != '3.8'
       run: |
         pip install pytest pytest-cov sphinx pandoc
         pip install -r docs/requirements.txt
-        pytest --cov=./ --cov-report=xml
+        pytest
+        make doctest
+    - name: Test with pytest (coverage + long tests)
+      if: matrix.python-version == '3.8'
+      run: |
+        pip install pytest pytest-cov sphinx pandoc
+        pip install -r docs/requirements.txt
+        SPEASY_AMDA_MAX_CHUNK_SIZE_DAYS=25 SPEASY_LONG_TESTS="" pytest --cov=./ --cov-report=xml
         make doctest
     - name: Check that release process is not broken
       if: matrix.python-version == '3.7'

docs/conf.py

@@ -31,7 +31,6 @@
 #
 # needs_sphinx = '1.0'
 
-# Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'sphinx.ext.autodoc',
@@ -130,7 +129,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.

speasy/config/__init__.py

@@ -109,3 +109,4 @@ def remove_entry(entry: ConfigEntry):
 amda_username = ConfigEntry("AMDA", "username")
 amda_password = ConfigEntry("AMDA", "password")
 amda_user_cache_retention = ConfigEntry("AMDA", "user_cache_retention", "900")  # 60 * 15 seconds
+amda_max_chunk_size_days = ConfigEntry("AMDA", "max_chunk_size_days", "10")  # 60 * 15 seconds

speasy/webservices/amda/_impl.py

@@ -6,12 +6,12 @@
 from .rest_client import auth_args
 from .exceptions import MissingCredentials
 
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Optional
 
 # General modules
-from ...config import amda_password, amda_username, amda_user_cache_retention
-from ...products.variable import SpeasyVariable
+from ...config import amda_password, amda_username, amda_user_cache_retention, amda_max_chunk_size_days
+from ...products.variable import SpeasyVariable, merge
 from ...inventory import data_tree, flat_inventories
 from ...inventory import reset_amda_inventory as reset_amda_flat_inventory
 from ...core.cache import CacheCall
@@ -85,14 +85,13 @@ def update_inventory(self):
 
         self._update_lists()
 
-    def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[
+    def dl_parameter_chunk(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[
         SpeasyVariable]:
-
-        start_time = start_time.timestamp()
-        stop_time = stop_time.timestamp()
         url = rest_client.get_parameter(
-            startTime=start_time, stopTime=stop_time, parameterID=parameter_id, timeFormat='UNIXTIME',
+            startTime=start_time.timestamp(), stopTime=stop_time.timestamp(), parameterID=parameter_id,
+            timeFormat='UNIXTIME',
             server_url=self.server_url, **kwargs)
+        # check status until done
         if url is not None:
             var = load_csv(url)
             if len(var):
@@ -103,6 +102,23 @@ def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id:
             return var
         return None
 
+    def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[
+        SpeasyVariable]:
+        dt = timedelta(days=int(amda_max_chunk_size_days.get()))
+
+        if stop_time - start_time > dt:
+            var = None
+            curr_t = start_time
+            while curr_t < stop_time:
+                if curr_t + dt < stop_time:
+                    var = merge([var, self.dl_parameter_chunk(curr_t, curr_t + dt, parameter_id, **kwargs)])
+                else:
+                    var = merge([var, self.dl_parameter_chunk(curr_t, stop_time, parameter_id, **kwargs)])
+                curr_t += dt
+            return var
+        else:
+            return self.dl_parameter_chunk(start_time, stop_time, parameter_id, **kwargs)
+
     def dl_user_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[
         SpeasyVariable]:
         username, password = _get_credentials()

speasy/webservices/amda/rest_client.py

@@ -1,4 +1,5 @@
 import logging
+import time
 from enum import Enum
 
 
@@ -27,6 +28,8 @@ class Endpoint(Enum):
     GETCAT = "getCatalog.php"
     GETPARAM = "getParameter.php"
 
+    GETSTATUS = "getStatus.php"
+
 
 def auth_args(username: str, password: str) -> dict:
     return {'userID': username, 'password': password}
@@ -175,6 +178,21 @@ def send_request_json(endpoint: Endpoint, params: Dict = None, n_try: int = 3,
             'dataFileURLs' in js:
             log.debug(f"success: {js['dataFileURLs']}")
             return js['dataFileURLs']
+        elif "success" in js and \
+                js["success"] is True and \
+                "status" in js and \
+                js["status"]=="in progress":
+            log.warning("This request duration is too long, consider reducing time range")
+            while True:
+                default_sleep_time = 10.
+                time.sleep(default_sleep_time)
+                url = request_url(Endpoint.GETSTATUS, server_url=server_url)
+
+                status = http.get(url, params=js).json()
+                if status is not None and status["status"] == "done":
+                    return status["dataFileURLs"]
+
+
         else:
             log.debug(f"Failed: {r.text}")
     return None

tests/test_amda.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 """Tests for `amda` package."""
-
+import logging
 import unittest
 from ddt import ddt, data, unpack
 import os
@@ -72,6 +72,19 @@ def test_get_variable_over_midnight(self):
                                         disable_cache=True)
         self.assertIsNotNone(result)
 
+    def test_get_variable_long_request(self):
+        if "SPEASY_LONG_TESTS" not in os.environ:
+            self.skipTest("Long tests disabled")
+        with self.assertLogs('speasy.webservices.amda.rest_client', level='WARNING') as cm:
+            start_date = datetime(2021, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
+            stop_date = datetime(2021, 1, 30, 0, 0, 0, tzinfo=timezone.utc)
+            parameter_id = "mms1_b_gse"
+            result = spz.amda.get_parameter(parameter_id, start_date, stop_date, disable_proxy=True,
+                                            disable_cache=True)
+            self.assertIsNotNone(result)
+            self.assertTrue(
+                any(["This request duration is too long, consider reducing time range" in line for line in cm.output]))
+
     def test_get_product_range(self):
         param_range = spz.amda.parameter_range(spz.amda.list_parameters()[0])
         self.assertIsNotNone(param_range)
@@ -111,11 +124,11 @@ def test_get_timetable_from_Index(self):
     def test_get_catalog_from_Index(self):
         r = spz.amda.get_catalog(spz.amda.list_catalogs()[-1])
         self.assertIsNotNone(r)
+
     def test_get_multidimensional_data(self):
         r = spz.amda.get_data("psp_spe_EvsEvspa", "2021-07-30T00:00:00", "2021-07-30T00:05:00")
         self.assertIsNotNone(r)
         self.assertIsNotNone(r.data)
-
 
 
 class PrivateProductsRequests(unittest.TestCase):