From 767ef1f8cf43f4f3dd9b982d16d4fd65f504016c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 30 Sep 2021 15:05:00 +0200 Subject: [PATCH 1/5] only anncept yaml extensions as yaml extensions --- lib/galaxy/tool_util/loader_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/loader_directory.py b/lib/galaxy/tool_util/loader_directory.py index c3d22c0d12c4..e4d27d9cd410 100644 --- a/lib/galaxy/tool_util/loader_directory.py +++ b/lib/galaxy/tool_util/loader_directory.py @@ -21,7 +21,7 @@ TOOL_REGEX = re.compile(r" Date: Wed, 23 Mar 2022 12:16:11 +0100 Subject: [PATCH 2/5] Revert "only anncept yaml extensions as yaml extensions" This reverts commit 767ef1f8cf43f4f3dd9b982d16d4fd65f504016c. --- lib/galaxy/tool_util/loader_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/loader_directory.py b/lib/galaxy/tool_util/loader_directory.py index e4d27d9cd410..c3d22c0d12c4 100644 --- a/lib/galaxy/tool_util/loader_directory.py +++ b/lib/galaxy/tool_util/loader_directory.py @@ -21,7 +21,7 @@ TOOL_REGEX = re.compile(r" Date: Wed, 23 Mar 2022 12:37:47 +0100 Subject: [PATCH 3/5] check 5k prefix for class for checking potential yaml and cwl tools --- lib/galaxy/tool_util/loader_directory.py | 46 ++++++++++++++---------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/tool_util/loader_directory.py b/lib/galaxy/tool_util/loader_directory.py index c3d22c0d12c4..5d8ecbf0d153 100644 --- a/lib/galaxy/tool_util/loader_directory.py +++ b/lib/galaxy/tool_util/loader_directory.py @@ -188,22 +188,41 @@ def looks_like_a_data_manager_xml(path): return looks_like_xml(path=path, regex=DATA_MANAGER_REGEX) -def is_a_yaml_with_class(path, classes): - """Determine if a file is a valid YAML with a supplied ``class`` entry.""" - if not _has_extension(path, YAML_EXTENSIONS): - return False +def as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes): + """ + get a dict from yaml file if it contains `class: CLASS`, where CLASS is + any string given in CLASSES. must appear in the first 5k and also load + properly in total. + """ + with open(path, encoding="utf-8") as f: + try: + start_contents = f.read(5 * 1024) + except UnicodeDecodeError: + return None + if re.search(rf"\nclass:\s+({'|'.join(classes)})\s*\n", start_contents) is None: + return None with open(path) as f: try: as_dict = yaml.safe_load(f) except Exception: - return False + return None if not isinstance(as_dict, dict): - return False + return None file_class = as_dict.get("class", None) - return file_class in classes + if file_class not in classes: + return None + + return as_dict + + +def is_a_yaml_with_class(path, classes): + """Determine if a file is a valid YAML with a supplied ``class`` entry.""" + if not _has_extension(path, YAML_EXTENSIONS): + return False + return as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) is not None def looks_like_a_tool_yaml(path): @@ -216,17 +235,8 @@ def looks_like_a_cwl_artifact(path, classes=None): if not _has_extension(path, CWL_EXTENSIONS): return False - with open(path) as f: - try: - as_dict = yaml.safe_load(f) - except Exception: - return False - - if not isinstance(as_dict, dict): - return False - - file_class = as_dict.get("class", None) - if classes is not None and file_class not in classes: + as_dict = as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) + if as_dict is None: return False file_cwl_version = as_dict.get("cwlVersion", None) From 4d2ee6e2d02052af42a96f969be86cfb762031d5 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 22 Apr 2022 17:54:35 +0200 Subject: [PATCH 4/5] make return values more explicit --- lib/galaxy/tool_util/loader_directory.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tool_util/loader_directory.py b/lib/galaxy/tool_util/loader_directory.py index 5d8ecbf0d153..bb0c307b1f63 100644 --- a/lib/galaxy/tool_util/loader_directory.py +++ b/lib/galaxy/tool_util/loader_directory.py @@ -198,31 +198,32 @@ def as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes): try: start_contents = f.read(5 * 1024) except UnicodeDecodeError: - return None + return False, None if re.search(rf"\nclass:\s+({'|'.join(classes)})\s*\n", start_contents) is None: - return None + return False, None with open(path) as f: try: as_dict = yaml.safe_load(f) except Exception: - return None + return False, None if not isinstance(as_dict, dict): - return None + return False, None file_class = as_dict.get("class", None) if file_class not in classes: - return None + return False, None - return as_dict + return True, as_dict def is_a_yaml_with_class(path, classes): """Determine if a file is a valid YAML with a supplied ``class`` entry.""" if not _has_extension(path, YAML_EXTENSIONS): return False - return as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) is not None + is_yaml, as_dict = as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) + return is_yaml def looks_like_a_tool_yaml(path): @@ -235,8 +236,8 @@ def looks_like_a_cwl_artifact(path, classes=None): if not _has_extension(path, CWL_EXTENSIONS): return False - as_dict = as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) - if as_dict is None: + is_yaml, as_dict = as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) + if not is_yaml is None: return False file_cwl_version = as_dict.get("cwlVersion", None) From c5f3535fd331cb5d59caea357d279d460e3cd41e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 22 Apr 2022 17:57:18 +0200 Subject: [PATCH 5/5] Update lib/galaxy/tool_util/loader_directory.py Co-authored-by: Marius van den Beek --- lib/galaxy/tool_util/loader_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/loader_directory.py b/lib/galaxy/tool_util/loader_directory.py index bb0c307b1f63..fd0e73e58f5b 100644 --- a/lib/galaxy/tool_util/loader_directory.py +++ b/lib/galaxy/tool_util/loader_directory.py @@ -237,7 +237,7 @@ def looks_like_a_cwl_artifact(path, classes=None): return False is_yaml, as_dict = as_dict_if_looks_like_yaml_or_cwl_with_class(path, classes) - if not is_yaml is None: + if not is_yaml: return False file_cwl_version = as_dict.get("cwlVersion", None)