From 8cfcdb24b80de35f89a78916beadce03b469a71a Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Mon, 25 Mar 2024 18:23:44 +0100 Subject: [PATCH 1/3] Changed env var warning to info + using PyPI csv2bufr --- README.md | 2 +- requirements.txt | 2 +- setup.py | 4 ---- synop2bufr/__init__.py | 28 +++++++++++++++------------- tests/test_synop2bufr.py | 2 +- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4cabb81..396cf3c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Dependencies are listed in [requirements.txt](https://github.com/wmo-im/synop2bu ### Setting Environment Variables -Before using synop2bufr, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (255).** +Before using synop2bufr, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (65535).** You can set these environment variables in your shell if you want to run synop2bufr on your local machine. Here's how you can do it in a Bash shell: diff --git a/requirements.txt b/requirements.txt index d19c2c5..a2c9882 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ click pymetdecoder-wmo -csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git \ No newline at end of file +csv2bufr \ No newline at end of file diff --git a/setup.py b/setup.py index 8486b23..614a0ae 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ import os import re from setuptools import Command, find_packages, setup -import subprocess class PyTest(Command): @@ -74,9 +73,6 @@ def get_package_version(): if (os.path.exists('MANIFEST')): os.unlink('MANIFEST') -# Install dependencies not on PyPI -subprocess.check_call("pip install https://github.com/wmo-im/csv2bufr/archive/main.zip", shell=True) # noqa - setup( name='synop2bufr', version=get_package_version(), diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 16a2902..5691918 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1293,7 +1293,7 @@ def transform(data: str, metadata: str, year: int, # each report necessitates this approach, because # we want to ensure the warning is only appended # to the first conversion - can_var_warning_be_displayed = True + can_var_info_be_displayed = True # =================== # First parse metadata file @@ -1544,34 +1544,36 @@ def truncate_to_twenty(name: str) -> str: # and subcentre codes are present missing_env_vars = [] - if os.environ.get("BUFR_ORIGINATING_CENTRE") is None: + ORIGINATING_CENTRE = os.environ.get("BUFR_ORIGINATING_CENTRE", 65535) # noqa + ORIGINATING_SUBCENTRE = os.environ.get("BUFR_ORIGINATING_SUBCENTRE", 65535) # noqa + + if ORIGINATING_CENTRE == 65535: missing_env_vars.append("BUFR_ORIGINATING_CENTRE") else: # Add the BUFR header centre and subcentre to mappings mapping["header"].append({ "eccodes_key": "bufrHeaderCentre", - "value": f"const:{os.environ.get('BUFR_ORIGINATING_CENTRE')}" # noqa + "value": f"const:{ORIGINATING_CENTRE}" # noqa }) - if os.environ.get("BUFR_ORIGINATING_SUBCENTRE") is None: + if ORIGINATING_SUBCENTRE == 65535: missing_env_vars.append("BUFR_ORIGINATING_SUBCENTRE") else: mapping["header"].append({ "eccodes_key": "bufrHeaderSubCentre", - "value": f"const:{os.environ.get('BUFR_ORIGINATING_SUBCENTRE')}" # noqa + "value": f"const:{ORIGINATING_SUBCENTRE}" # noqa }) # If either of these environment variables are not set, - # we will default to missing and warn the user once - if missing_env_vars and can_var_warning_be_displayed: - # Display ewarning messages + # we will default to missing and inform the user once + if missing_env_vars and can_var_info_be_displayed: + # Display info messages for var in missing_env_vars: - var_warning = f"The {var} environment variable is not set, will default to missing!" # noqa - LOGGER.warning(var_warning) - warning_msgs.append(var_warning) - can_var_warning_be_displayed = False + var_info = f"The {var} environment variable is not set, will default to missing!" # noqa + LOGGER.info(var_info) + can_var_info_be_displayed = False # Stop duplicated warnings - can_var_warning_be_displayed = False + can_var_info_be_displayed = False # Define a new method which handles the updating of # the mapping file with section 3 and 4 cloud data diff --git a/tests/test_synop2bufr.py b/tests/test_synop2bufr.py index d0d633f..b6ab9bf 100644 --- a/tests/test_synop2bufr.py +++ b/tests/test_synop2bufr.py @@ -48,7 +48,7 @@ def multiple_reports_307096(): def single_report(): return """AAXX 21121 15001 05515 32931 10103 21090 39765 42250 57020 60071 72006 82110 91155 - 333 10178 21073 34101 55055 00010 20003 30002 50001 60004 + 222// 06070 20502 333 10178 21073 34101 55055 00010 20003 30002 50001 60004 60035 70500 83145 81533 91008 91111 444 18031 22053 """ From 20c655924143be0e0d730290edf4896d7cf9f108 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:54:32 +0100 Subject: [PATCH 2/3] Updated handling of env vars + removed bufr centre/subcentre from original mappings --- synop2bufr/__init__.py | 31 ++++++++++--------- .../resources/synop-mappings-307080.json | 2 -- .../resources/synop-mappings-307096.json | 2 -- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 5691918..bb50e9f 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1544,25 +1544,26 @@ def truncate_to_twenty(name: str) -> str: # and subcentre codes are present missing_env_vars = [] - ORIGINATING_CENTRE = os.environ.get("BUFR_ORIGINATING_CENTRE", 65535) # noqa - ORIGINATING_SUBCENTRE = os.environ.get("BUFR_ORIGINATING_SUBCENTRE", 65535) # noqa + ORIGINATING_CENTRE = os.environ.get("BUFR_ORIGINATING_CENTRE", None) # noqa + ORIGINATING_SUBCENTRE = os.environ.get("BUFR_ORIGINATING_SUBCENTRE", None) # noqa - if ORIGINATING_CENTRE == 65535: + if ORIGINATING_CENTRE is None: missing_env_vars.append("BUFR_ORIGINATING_CENTRE") - else: - # Add the BUFR header centre and subcentre to mappings - mapping["header"].append({ - "eccodes_key": "bufrHeaderCentre", - "value": f"const:{ORIGINATING_CENTRE}" # noqa - }) + ORIGINATING_CENTRE = 65535 - if ORIGINATING_SUBCENTRE == 65535: + if ORIGINATING_SUBCENTRE is None: missing_env_vars.append("BUFR_ORIGINATING_SUBCENTRE") - else: - mapping["header"].append({ - "eccodes_key": "bufrHeaderSubCentre", - "value": f"const:{ORIGINATING_SUBCENTRE}" # noqa - }) + ORIGINATING_SUBCENTRE = 65535 + + # Add the BUFR header centre and subcentre to mappings + mapping["header"].append({ + "eccodes_key": "bufrHeaderCentre", + "value": f"const:{ORIGINATING_CENTRE}" # noqa + }) + mapping["header"].append({ + "eccodes_key": "bufrHeaderSubCentre", + "value": f"const:{ORIGINATING_SUBCENTRE}" # noqa + }) # If either of these environment variables are not set, # we will default to missing and inform the user once diff --git a/synop2bufr/resources/synop-mappings-307080.json b/synop2bufr/resources/synop-mappings-307080.json index 066c361..aa7e984 100644 --- a/synop2bufr/resources/synop-mappings-307080.json +++ b/synop2bufr/resources/synop-mappings-307080.json @@ -19,8 +19,6 @@ "header":[ {"eccodes_key": "edition", "value": "const:4"}, {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:65535"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:65535"}, {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, {"eccodes_key": "dataCategory", "value": "const:0"}, {"eccodes_key": "internationalDataSubCategory", "value": "const:2"}, diff --git a/synop2bufr/resources/synop-mappings-307096.json b/synop2bufr/resources/synop-mappings-307096.json index 716c36e..aa83af5 100644 --- a/synop2bufr/resources/synop-mappings-307096.json +++ b/synop2bufr/resources/synop-mappings-307096.json @@ -19,8 +19,6 @@ "header":[ {"eccodes_key": "edition", "value": "const:4"}, {"eccodes_key": "masterTableNumber", "value": "const:0"}, - {"eccodes_key": "bufrHeaderCentre", "value": "const:65535"}, - {"eccodes_key": "bufrHeaderSubCentre", "value": "const:65535"}, {"eccodes_key": "updateSequenceNumber", "value": "const:0"}, {"eccodes_key": "dataCategory", "value": "const:0"}, {"eccodes_key": "internationalDataSubCategory", "value": "const:2"}, From b0a992b270ef9abf0e0478aae23ffbc46b1cab48 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:58:34 +0100 Subject: [PATCH 3/3] Updated tests.yml env variables --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c531e8b..dba9ab0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,8 +9,8 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] env: - BUFR_ORIGINATING_CENTRE: 123 - BUFR_ORIGINATING_SUBCENTRE: 123 + BUFR_ORIGINATING_CENTRE: 65535 + BUFR_ORIGINATING_SUBCENTRE: 65535 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5