diff --git a/.github/workflows/PRs.yml b/.github/workflows/PRs.yml index f62a9fe3..4294f202 100644 --- a/.github/workflows/PRs.yml +++ b/.github/workflows/PRs.yml @@ -35,10 +35,18 @@ jobs: - name: install texlive for Pandoc run: sudo apt update && sudo apt install -y texlive pandoc && pip install wheel - name: Test with pytest + if: matrix.python-version != '3.8' run: | pip install pytest pytest-cov sphinx pandoc pip install -r docs/requirements.txt - pytest --cov=./ --cov-report=xml + pytest + make doctest + - name: Test with pytest (coverage + long tests) + if: matrix.python-version == '3.8' + run: | + pip install pytest pytest-cov sphinx pandoc + pip install -r docs/requirements.txt + SPEASY_AMDA_MAX_CHUNK_SIZE_DAYS=25 SPEASY_LONG_TESTS="" pytest --cov=./ --cov-report=xml make doctest - name: Check that release process is not broken if: matrix.python-version == '3.7' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a541b8c9..a56948b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,10 +35,18 @@ jobs: - name: install texlive for Pandoc run: sudo apt update && sudo apt install -y texlive pandoc && pip install wheel - name: Test with pytest + if: matrix.python-version != '3.8' run: | pip install pytest pytest-cov sphinx pandoc pip install -r docs/requirements.txt - pytest --cov=./ --cov-report=xml + pytest + make doctest + - name: Test with pytest (coverage + long tests) + if: matrix.python-version == '3.8' + run: | + pip install pytest pytest-cov sphinx pandoc + pip install -r docs/requirements.txt + SPEASY_AMDA_MAX_CHUNK_SIZE_DAYS=25 SPEASY_LONG_TESTS="" pytest --cov=./ --cov-report=xml make doctest - name: Check that release process is not broken if: matrix.python-version == '3.7' diff --git a/docs/conf.py b/docs/conf.py index c3b8c143..5a1fc9af 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,7 +31,6 @@ # # needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', @@ -130,7 +129,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/speasy/config/__init__.py b/speasy/config/__init__.py index f98e36a7..1f3da08a 100644 --- a/speasy/config/__init__.py +++ b/speasy/config/__init__.py @@ -109,3 +109,4 @@ def remove_entry(entry: ConfigEntry): amda_username = ConfigEntry("AMDA", "username") amda_password = ConfigEntry("AMDA", "password") amda_user_cache_retention = ConfigEntry("AMDA", "user_cache_retention", "900") # 60 * 15 seconds +amda_max_chunk_size_days = ConfigEntry("AMDA", "max_chunk_size_days", "10") # 60 * 15 seconds diff --git a/speasy/webservices/amda/_impl.py b/speasy/webservices/amda/_impl.py index a7401c17..3a71e850 100644 --- a/speasy/webservices/amda/_impl.py +++ b/speasy/webservices/amda/_impl.py @@ -6,12 +6,12 @@ from .rest_client import auth_args from .exceptions import MissingCredentials -from datetime import datetime +from datetime import datetime, timedelta from typing import Optional # General modules -from ...config import amda_password, amda_username, amda_user_cache_retention -from ...products.variable import SpeasyVariable +from ...config import amda_password, amda_username, amda_user_cache_retention, amda_max_chunk_size_days +from ...products.variable import SpeasyVariable, merge from ...inventory import data_tree, flat_inventories from ...inventory import reset_amda_inventory as reset_amda_flat_inventory from ...core.cache import CacheCall @@ -85,14 +85,13 @@ def update_inventory(self): self._update_lists() - def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[ + def dl_parameter_chunk(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[ SpeasyVariable]: - - start_time = start_time.timestamp() - stop_time = stop_time.timestamp() url = rest_client.get_parameter( - startTime=start_time, stopTime=stop_time, parameterID=parameter_id, timeFormat='UNIXTIME', + startTime=start_time.timestamp(), stopTime=stop_time.timestamp(), parameterID=parameter_id, + timeFormat='UNIXTIME', server_url=self.server_url, **kwargs) + # check status until done if url is not None: var = load_csv(url) if len(var): @@ -103,6 +102,23 @@ def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: return var return None + def dl_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[ + SpeasyVariable]: + dt = timedelta(days=int(amda_max_chunk_size_days.get())) + + if stop_time - start_time > dt: + var = None + curr_t = start_time + while curr_t < stop_time: + if curr_t + dt < stop_time: + var = merge([var, self.dl_parameter_chunk(curr_t, curr_t + dt, parameter_id, **kwargs)]) + else: + var = merge([var, self.dl_parameter_chunk(curr_t, stop_time, parameter_id, **kwargs)]) + curr_t += dt + return var + else: + return self.dl_parameter_chunk(start_time, stop_time, parameter_id, **kwargs) + def dl_user_parameter(self, start_time: datetime, stop_time: datetime, parameter_id: str, **kwargs) -> Optional[ SpeasyVariable]: username, password = _get_credentials() diff --git a/speasy/webservices/amda/rest_client.py b/speasy/webservices/amda/rest_client.py index da2e344c..4728565b 100644 --- a/speasy/webservices/amda/rest_client.py +++ b/speasy/webservices/amda/rest_client.py @@ -1,4 +1,5 @@ import logging +import time from enum import Enum @@ -27,6 +28,8 @@ class Endpoint(Enum): GETCAT = "getCatalog.php" GETPARAM = "getParameter.php" + GETSTATUS = "getStatus.php" + def auth_args(username: str, password: str) -> dict: return {'userID': username, 'password': password} @@ -175,6 +178,21 @@ def send_request_json(endpoint: Endpoint, params: Dict = None, n_try: int = 3, 'dataFileURLs' in js: log.debug(f"success: {js['dataFileURLs']}") return js['dataFileURLs'] + elif "success" in js and \ + js["success"] is True and \ + "status" in js and \ + js["status"]=="in progress": + log.warning("This request duration is too long, consider reducing time range") + while True: + default_sleep_time = 10. + time.sleep(default_sleep_time) + url = request_url(Endpoint.GETSTATUS, server_url=server_url) + + status = http.get(url, params=js).json() + if status is not None and status["status"] == "done": + return status["dataFileURLs"] + + else: log.debug(f"Failed: {r.text}") return None diff --git a/tests/test_amda.py b/tests/test_amda.py index 7af60408..071c56b8 100644 --- a/tests/test_amda.py +++ b/tests/test_amda.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """Tests for `amda` package.""" - +import logging import unittest from ddt import ddt, data, unpack import os @@ -72,6 +72,19 @@ def test_get_variable_over_midnight(self): disable_cache=True) self.assertIsNotNone(result) + def test_get_variable_long_request(self): + if "SPEASY_LONG_TESTS" not in os.environ: + self.skipTest("Long tests disabled") + with self.assertLogs('speasy.webservices.amda.rest_client', level='WARNING') as cm: + start_date = datetime(2021, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + stop_date = datetime(2021, 1, 30, 0, 0, 0, tzinfo=timezone.utc) + parameter_id = "mms1_b_gse" + result = spz.amda.get_parameter(parameter_id, start_date, stop_date, disable_proxy=True, + disable_cache=True) + self.assertIsNotNone(result) + self.assertTrue( + any(["This request duration is too long, consider reducing time range" in line for line in cm.output])) + def test_get_product_range(self): param_range = spz.amda.parameter_range(spz.amda.list_parameters()[0]) self.assertIsNotNone(param_range) @@ -111,11 +124,11 @@ def test_get_timetable_from_Index(self): def test_get_catalog_from_Index(self): r = spz.amda.get_catalog(spz.amda.list_catalogs()[-1]) self.assertIsNotNone(r) + def test_get_multidimensional_data(self): r = spz.amda.get_data("psp_spe_EvsEvspa", "2021-07-30T00:00:00", "2021-07-30T00:05:00") self.assertIsNotNone(r) self.assertIsNotNone(r.data) - class PrivateProductsRequests(unittest.TestCase):