From 58228c6798b98ae284228c782b18e25b55cfd247 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Wed, 7 Sep 2022 11:09:38 +0100 Subject: [PATCH] cylc vip --- CHANGES.md | 3 + cylc/flow/option_parsers.py | 479 +++++++++++++----- cylc/flow/scheduler_cli.py | 311 +++++++----- cylc/flow/scripts/cylc.py | 9 +- cylc/flow/scripts/install.py | 107 ++-- cylc/flow/scripts/validate.py | 80 ++- cylc/flow/scripts/validate_install_play.py | 110 ++++ setup.cfg | 1 + .../cylc-combination-scripts/00-vip.t | 43 ++ .../cylc-combination-scripts/00-vip/flow.cylc | 7 + .../00-vip/reference.log | 1 + .../cylc-combination-scripts/test_header | 1 + .../database/04-lock-recover/bin/cylc-db-lock | 2 +- .../05-lock-recover-100/bin/cylc-db-lock | 2 +- tests/functional/env-filter/00-filter.t | 2 +- tests/functional/include-files/00-basic.t | 4 +- tests/functional/restart/09-reload.t | 2 +- tests/functional/workflow-state/00-polling.t | 4 +- tests/functional/workflow-state/01-polling.t | 4 +- tests/functional/xtriggers/02-persistence.t | 4 +- tests/unit/test_option_parsers.py | 425 +++++++++++++++- 21 files changed, 1254 insertions(+), 347 deletions(-) create mode 100644 cylc/flow/scripts/validate_install_play.py create mode 100644 tests/functional/cylc-combination-scripts/00-vip.t create mode 100644 tests/functional/cylc-combination-scripts/00-vip/flow.cylc create mode 100644 tests/functional/cylc-combination-scripts/00-vip/reference.log create mode 120000 tests/functional/cylc-combination-scripts/test_header diff --git a/CHANGES.md b/CHANGES.md index 5b42eda3f8f..3d5ea56522b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,9 @@ ones in. --> [#5184](https://github.com/cylc/cylc-flow/pull/5184) - scan for active runs of the same workflow at install time. +[#5094](https://github.com/cylc/cylc-flow/pull/5094) - Added a single +command to validate, install and play a workflow. + [#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of 100 for the "default" queue. diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py index a0609e0e172..142d7e670c0 100644 --- a/cylc/flow/option_parsers.py +++ b/cylc/flow/option_parsers.py @@ -17,6 +17,7 @@ from contextlib import suppress import logging +from itertools import product from optparse import ( OptionParser, OptionConflictError, @@ -33,7 +34,7 @@ import sys from textwrap import dedent -from typing import Any, Dict, Optional, List, Tuple +from typing import Any, Dict, Optional, List, Tuple, Union from cylc.flow import LOG from cylc.flow.terminal import supports_color, DIM @@ -51,20 +52,94 @@ SHORTLINK_TO_ICP_DOCS = "https://bit.ly/3MYHqVh" -icp_option = Option( - "--initial-cycle-point", "--icp", - metavar="CYCLE_POINT or OFFSET", +APPEND = 'append' +ARGS = 'args' +KWARGS = 'kwargs' +HELP = 'help' +ACTION = 'action' +STORE = 'store' +STORE_CONST = 'store_const' +STORE_TRUE = 'store_true' +SOURCES = 'sources' +DEFAULT = 'default' +DEST = 'dest' +METAVAR = 'metavar' +CHOICES = 'choices' +OPTS = 'opts' +ALL = 'all' +USEIF = 'useif' +DOUBLEDASH = '--' + + +class OptionSettings(): + """Container for info about a command line option + + Despite some similarities this is not to be confused with + optparse.Option: This a container for information which may or may + not be passed to optparse depending on the results of + cylc.flow.option_parsers(thismodule).combine_options_pair. + """ + + def __init__(self, argslist, sources=None, useif=None, **kwargs): + """Init function: + + Args: + arglist: list of arguments for optparse.Option. + sources: set of CLI scripts which use this option. + useif: badge for use by Cylc optionparser. + **kwargs: kwargs for optparse.option. + """ + self.kwargs: Dict[str, str] = {} + self.sources: set = sources if sources is not None else set() + self.useif: str = useif if useif is not None else '' + + self.args: list[str] = argslist + for kwarg, value in kwargs.items(): + self.kwargs.update({kwarg: value}) + + def __eq__(self, other): + """Args and Kwargs, but not other props equal.""" + return ( + self.kwargs == other.kwargs + and self.args == other.args + ) + + def __and__(self, other): + """Is there a set intersection between arguments.""" + return list(set(self.args).intersection(set(other.args))) + + def __sub__(self, other): + """Set difference on args.""" + return list(set(self.args) - set(other.args)) + + def _in_list(self, others): + """CLI arguments for this option found in any of a list of + other options.""" + return any([self & other for other in others]) + + def _update_sources(self, other): + """Update the sources from this and 1 other OptionSettings object""" + self.sources = {*self.sources, *other.sources} + + +ICP_OPTION = OptionSettings( + ["--initial-cycle-point", "--icp"], help=( - "Set the initial cycle point. " - "Required if not defined in flow.cylc." - "\nMay be either an absolute point or an offset: See " - f"{SHORTLINK_TO_ICP_DOCS} (Cylc documentation link)." + "Set the initial cycle point." + " Required if not defined in flow.cylc." + "\nMay be either an absolute point or an offset: See" + f" {SHORTLINK_TO_ICP_DOCS} (Cylc documentation link)." ), - action="store", - dest="icp", + metavar="CYCLE_POINT or OFFSET", + action=STORE, + dest="icp" ) +icp_option = Option( + *ICP_OPTION.args, **ICP_OPTION.kwargs) # type: ignore[arg-type] + + def format_shell_examples(string): """Put comments in the terminal "diminished" colour.""" return cparse( @@ -169,7 +244,7 @@ def verbosity_to_env(verb: int) -> Dict[str, str]: } -def env_to_verbosity(env: dict) -> int: +def env_to_verbosity(env: Union[Dict, os._Environ]) -> int: """Extract verbosity from environment variables. Examples: @@ -283,6 +358,68 @@ class CylcOptionParser(OptionParser): See `cylc help id` for more details. ''')) + STD_OPTIONS = [ + OptionSettings( + ['-q', '--quiet'], help='Decrease verbosity.', + action='decrement', dest='verbosity', useif=ALL), + OptionSettings( + ['-v', '--verbose'], help='Increase Verbosity', + dest='verbosity', action='count', + default=env_to_verbosity(os.environ), useif=ALL), + OptionSettings( + ['--debug'], help='Equivalent to -v -v', + dest='verbosity', action=STORE_CONST, const=2, useif=ALL), + OptionSettings( + ['--no-timestamp'], help='Don\'t timestamp logged messages.', + action='store_false', dest='log_timestamp', + default=True, useif=ALL), + OptionSettings( + ['--color', '--color'], metavar='WHEN', action=STORE, + default='auto', choices=['never', 'auto', 'always'], + help=( + "When to use color/bold text in terminal output." + " Options are 'never', 'auto' and 'always'." + ), + useif='color'), + OptionSettings( + ['--comms-timeout'], metavar='SEC', + help=( + "Set a timeout for network connections" + " to the running workflow. The default is no timeout." + " For task messaging connections see" + " site/user config file documentation." + ), + action=STORE, default=None, dest='comms_timeout', useif='comms'), + OptionSettings( + ['-s', '--set'], metavar='NAME=VALUE', + help=( + "Set the value of a Jinja2 template variable in the" + " workflow definition." + " Values should be valid Python literals so strings" + " must be quoted" + " e.g. 'STR=\"string\"', INT=43, BOOL=True." + " This option can be used multiple " + " times on the command line." + " NOTE: these settings persist across workflow restarts," + " but can be set again on the \"cylc play\"" + " command line if they need to be overridden." + ), + action='append', default=[], dest='templatevars', useif='jset'), + OptionSettings( + ['--set-file'], metavar='FILE', + help=( + "Set the value of Jinja2 template variables in the" + " workflow definition from a file containing NAME=VALUE" + " pairs (one per line)." + " As with --set values should be valid Python literals " + " so strings must be quoted e.g. STR='string'." + " NOTE: these settings persist across workflow restarts," + " but can be set again on the \"cylc play\"" + " command line if they need to be overridden." + ), + action=STORE, default=None, dest='templatevars_file', useif='jset') + ] + def __init__( self, usage: str, @@ -356,6 +493,17 @@ def __init__( formatter=CylcHelpFormatter() ) + def get_std_options(self): + """Get a data-structure of standard options""" + opts = [] + for opt in self.STD_OPTIONS: + if ( + opt.useif == ALL + or hasattr(self, opt.useif) and getattr(self, opt.useif) + ): + opts.append(opt) + return opts + def add_std_option(self, *args, **kwargs): """Add a standard option, ignoring override.""" with suppress(OptionConflictError): @@ -363,121 +511,53 @@ def add_std_option(self, *args, **kwargs): def add_std_options(self): """Add standard options if they have not been overridden.""" - self.add_std_option( - "-q", "--quiet", - help="Decrease verbosity.", - action='decrement', - dest='verbosity', - ) - self.add_std_option( - "-v", "--verbose", - help="Increase verbosity.", - dest='verbosity', - action='count', - default=env_to_verbosity(os.environ) - ) - self.add_std_option( - "--debug", - help="Equivalent to -v -v", - dest="verbosity", - action='store_const', - const=2 - ) - self.add_std_option( - "--no-timestamp", - help="Don't timestamp logged messages.", - action="store_false", dest="log_timestamp", default=True) - - if self.color: - self.add_std_option( - '--color', '--colour', metavar='WHEN', action='store', - default='auto', choices=['never', 'auto', 'always'], - help=( - "When to use color/bold text in terminal output." - " Options are 'never', 'auto' and 'always'." - ) - ) + for option in self.get_std_options(): + if not any(self.has_option(i) for i in option.args): + self.add_option(*option.args, **option.kwargs) - if self.comms: - self.add_std_option( - "--comms-timeout", metavar='SEC', + @staticmethod + def get_cylc_rose_options(): + """Returns a list of option dictionaries if Cylc Rose exists.""" + try: + __import__('cylc.rose') + except ImportError: + return [] + return [ + OptionSettings( + ["--opt-conf-key", "-O"], help=( - "Set a timeout for network connections " - "to the running workflow. The default is no timeout. " - "For task messaging connections see " - "site/user config file documentation." - ), - action="store", default=None, dest="comms_timeout") - - if self.jset: - self.add_std_option( - "-s", "--set", metavar="NAME=VALUE", + "Use optional Rose Config Setting" + " (If Cylc-Rose is installed)"), + action="append", default=[], dest="opt_conf_keys", + sources={'cylc-rose'}, + ), + OptionSettings( + ["--define", '-D'], help=( - "Set the value of a Jinja2 template variable in the" - " workflow definition." - " Values should be valid Python literals so strings" - " must be quoted" - " e.g. 'STR=\"string\"', INT=43, BOOL=True." - " This option can be used multiple " - " times on the command line." - " NOTE: these settings persist across workflow restarts," - " but can be set again on the \"cylc play\"" - " command line if they need to be overridden." - ), - action="append", default=[], dest="templatevars") - - self.add_std_option( - "--set-file", metavar="FILE", + "Each of these overrides the `[SECTION]KEY` setting" + " in a `rose-suite.conf` file." + " Can be used to disable a setting using the syntax" + " `--define=[SECTION]!KEY` or" + " even `--define=[!SECTION]`."), + action="append", default=[], dest="defines", + sources={'cylc-rose'}), + OptionSettings( + ["--rose-template-variable", '-S', '--define-suite'], help=( - "Set the value of Jinja2 template variables in the " - "workflow definition from a file containing NAME=VALUE " - "pairs (one per line). " - "As with --set values should be valid Python literals " - "so strings must be quoted e.g. STR='string'. " - "NOTE: these settings persist across workflow restarts, " - "but can be set again on the \"cylc play\" " - "command line if they need to be overridden." - ), - action="store", default=None, dest="templatevars_file") + "As `--define`, but with an implicit `[SECTION]` for" + " workflow variables."), + action="append", default=[], dest="rose_template_vars", + sources={'cylc-rose'}, + ) + ] def add_cylc_rose_options(self) -> None: - """Add extra options for cylc-rose plugin if it is installed.""" - try: - __import__('cylc.rose') - except ImportError: - return - self.add_option( - "--opt-conf-key", "-O", - help=( - "Use optional Rose Config Setting " - "(If Cylc-Rose is installed)" - ), - action="append", - default=[], - dest="opt_conf_keys" - ) - self.add_option( - "--define", '-D', - help=( - "Each of these overrides the `[SECTION]KEY` setting in a " - "`rose-suite.conf` file. " - "Can be used to disable a setting using the syntax " - "`--define=[SECTION]!KEY` or even `--define=[!SECTION]`." - ), - action="append", - default=[], - dest="defines" - ) - self.add_option( - "--rose-template-variable", '-S', '--define-suite', - help=( - "As `--define`, but with an implicit `[SECTION]` for " - "workflow variables." - ), - action="append", - default=[], - dest="rose_template_vars" - ) + """Add extra options for cylc-rose plugin if it is installed. + + Now a vestigal interface for get_cylc_rose_options. + """ + for option in self.get_cylc_rose_options(): + self.add_option(*option.args, **option.kwargs) def parse_args(self, api_args, remove_opts=None): """Parse options and arguments, overrides OptionParser.parse_args. @@ -607,3 +687,162 @@ def __call__(self, **kwargs) -> Values: setattr(opts, key, value) return opts + + +def appendif(list_, item): + """Avoid duplicating items in output list""" + if item not in list_: + list_.append(item) + return list_ + + +def combine_options_pair(first_list, second_list): + """Combine two option lists recording where each came from. + + Scenarios: + - Arguments are identical - return this argument. + - Arguments are not identical but have some common label strings, + i.e. both arguments can be invoked using `-f`. + - If there are non-shared label strings strip the shared ones. + - Otherwise raise an error. + E.g: If `command-A` has an option `-f` or `--file` and + `command-B has an option `-f` or `--fortran`` then + `command-A+B` will have options `--fortran` and `--file` but _not_ + `-f`, which would be confusing. + - Arguments only apply to a single compnent of the compound CLI script. + + """ + output = [] + if not first_list: + output = second_list + elif not second_list: + output = first_list + else: + for first, second in product(first_list, second_list): + # Two options are identical in both args and kwargs: + if first == second: + first._update_sources(second) + output = appendif(output, first) + + # If any of the argument names identical we must remove + # overlapping names (if we can) + # e.g. [-a, --aleph], [-a, --alpha-centuri] -> keep both options + # but neither should have the `-a` short version: + elif ( + first != second + and first & second + ): + # if any of the args are different: + if first.args == second.args: + # if none of the arg names are different. + raise Exception(f'Clashing Options \n{first}\n{second}') + else: + first_args = first - second + second.args = second - first + first.args = first_args + output = appendif(output, first) + output = appendif(output, second) + else: + # Neither option appears in the other list, so it can be + # appended: + if not first._in_list(second_list): + output = appendif(output, first) + if not second._in_list(first_list): + output = appendif(output, second) + + return output + + +def add_sources_to_helps(options, modify=None): + """Prettify format of list of CLI commands this option applies to + and prepend that list to the start of help. + + Arguments: + Options: + Options dicts to modify help upon. + modify: + Dict of items to substitute: Intended to allow one + to replace cylc-rose with the names of the sub-commands + cylc rose options apply to. + """ + modify = {} if modify is None else modify + for option in options: + if hasattr(option, SOURCES): + sources = list(option.sources) + for match, sub in modify.items(): + if match in option.sources: + sources.append(sub) + sources.remove(match) + + option.kwargs[HELP] = cparse( + f'[{", ".join(sources)}]' + f' {option.kwargs[HELP]}') + return options + + +def combine_options(*args, modify=None): + """Combine a list of argument dicts. + + Ordering should be irrelevant because combine_options_pair should + be commutative, and the overall order of args is not relevant. + """ + list_ = list(args) + output = list_[0] + for arg in list_[1:]: + output = combine_options_pair(arg, output) + + return add_sources_to_helps(output, modify) + + +def cleanup_sysargv( + script_name, + workflow_id, + options, + compound_script_opts, + script_opts, + source, +): + """Remove unwanted options from sys.argv + + Some cylc scripts (notably Cylc Play when it is re-invoked on a scheduler + server) require the correct content in sys.argv. + """ + # Organize Options by dest. + script_opts_by_dest = { + x.kwargs.get('dest', x.args[0].strip(DOUBLEDASH)): x + for x in script_opts + } + compound_opts_by_dest = { + x.kwargs.get('dest', x.args[0].strip(DOUBLEDASH)): x + for x in compound_script_opts + } + # Filter out non-cylc-play options. + for unwanted_opt in (set(options.__dict__)) - set(script_opts_by_dest): + for arg in compound_opts_by_dest[unwanted_opt].args: + if arg in sys.argv: + index = sys.argv.index(arg) + sys.argv.pop(index) + if ( + compound_opts_by_dest[unwanted_opt].kwargs[ACTION] + not in ['store_true', 'store_false'] + ): + sys.argv.pop(index) + + # replace compound script name: + sys.argv[1] = script_name + + # replace source path with workflow ID. + if str(source) in sys.argv: + sys.argv.remove(str(source)) + sys.argv.append(workflow_id) + + +def log_subcommand(command, workflow_id): + """Log a command run as part of a sequence. + + Example: + >>> log_subcommand('ruin', 'my_workflow') + \x1b[1m\x1b[36m$ cylc ruin my_workflow\x1b[0m\x1b[1m\x1b[0m\n + """ + print(cparse( + f'$ cylc {command} {workflow_id}')) diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index 5682530ee38..ad9bc3f0422 100644 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -17,6 +17,7 @@ from ansimarkup import parse as cparse import asyncio +from copy import deepcopy from functools import lru_cache from itertools import zip_longest from pathlib import Path @@ -41,8 +42,10 @@ from cylc.flow.option_parsers import ( WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, + OptionSettings, Options, - icp_option, + ICP_OPTION, + APPEND, ARGS, KWARGS, STORE, STORE_TRUE ) from cylc.flow.pathutil import get_workflow_run_scheduler_log_path from cylc.flow.remote import cylc_server_cmd @@ -117,152 +120,197 @@ } ''' +PLAY_ICP_OPTION = deepcopy(ICP_OPTION) +PLAY_ICP_OPTION.sources = {'play'} -@lru_cache() -def get_option_parser(add_std_opts: bool = False) -> COP: - """Parse CLI for "cylc play".""" - parser = COP( - PLAY_DOC, - jset=True, - comms=True, - argdoc=[WORKFLOW_ID_ARG_DOC] - ) - - parser.add_option( - "-n", "--no-detach", "--non-daemon", - help="Do not daemonize the scheduler (infers --format=plain)", - action="store_true", dest="no_detach") - - parser.add_option( - "--profile", help="Output profiling (performance) information", - action="store_true", dest="profile_mode") +RUN_MODE = OptionSettings( + ["-m", "--mode"], + help="Run mode: live, dummy, simulation (default live).", + metavar="STRING", action=STORE, dest="run_mode", + choices=['live', 'dummy', 'simulation'], +) - parser.add_option(icp_option) +PLAY_RUN_MODE = deepcopy(RUN_MODE) +PLAY_RUN_MODE.sources = {'play'} - parser.add_option( - "--start-cycle-point", "--startcp", +PLAY_OPTIONS = [ + OptionSettings( + ["-n", "--no-detach", "--non-daemon"], + help="Do not daemonize the scheduler (infers --format=plain)", + action=STORE_TRUE, dest="no_detach", sources={'play'}), + OptionSettings( + ["--profile"], + help="Output profiling (performance) information", + action=STORE_TRUE, + default=False, + dest="profile_mode", + sources={'play'}, + ), + OptionSettings( + ["--start-cycle-point", "--startcp"], help=( - "Set the start cycle point, which may be after the initial cycle " - "point. If the specified start point is not in the sequence, the " - "next on-sequence point will be used. " - "(Not to be confused with the initial cycle point.) " - "This replaces the Cylc 7 --warm option." - ), - metavar="CYCLE_POINT", action="store", dest="startcp") - - parser.add_option( - "--final-cycle-point", "--fcp", + "Set the start cycle point, which may be after" + " the initial cycle point. If the specified start point is" + " not in the sequence, the next on-sequence point will" + " be used. (Not to be confused with the initial cycle point)"), + metavar="CYCLE_POINT", + action=STORE, + dest="startcp", + sources={'play'}, + ), + OptionSettings( + ["--final-cycle-point", "--fcp"], help=( - "Set the final cycle point. " - "This command line option overrides the workflow " - "config option '[scheduling]final cycle point'. " - ), - metavar="CYCLE_POINT", action="store", dest="fcp") - - parser.add_option( - "--stop-cycle-point", "--stopcp", + "Set the final cycle point. This command line option overrides" + " the workflow config option" + " '[scheduling]final cycle point'. "), + metavar="CYCLE_POINT", + action=STORE, + dest="fcp", + sources={'play'}, + ), + OptionSettings( + ["--stop-cycle-point", "--stopcp"], help=( - "Set the stop cycle point. " - "Shut down after all tasks have PASSED this cycle point. " - "(Not to be confused with the final cycle point.) " - "This command line option overrides the workflow " - "config option '[scheduling]stop after cycle point'. " - ), - metavar="CYCLE_POINT", action="store", dest="stopcp") - - parser.add_option( - "--start-task", "--starttask", "-t", + "Set the stop cycle point. Shut down after all" + " have PASSED this cycle point. (Not to be confused" + " the final cycle point.) This command line option overrides" + " the workflow config option" + " '[scheduling]stop after cycle point'."), + metavar="CYCLE_POINT", + action=STORE, + dest="stopcp", + sources={'play'}, + ), + OptionSettings( + ["--start-task", "--starttask", "-t"], help=( - "Start from this task instance, given by '/'. " - "Can be used multiple times " - "to start from multiple tasks at once. Dependence on tasks with " - "with cycle points earlier than the earliest start-task will be " - "ignored. A sub-graph of the workflow will run if selected tasks " - "do not lead on to the full graph." - ), - metavar="TASK_ID", action="append", dest="starttask") - - parser.add_option( - "--pause", + "Start from this task instance, given by '/'." + " This can be used multiple times to start from multiple" + " tasks at once. Dependence on tasks with cycle points earlier" + " than the earliest start-task will be ignored. A" + " sub-graph of the workflow will run if selected tasks do" + " not lead on to the full graph."), + metavar="TASK_ID", + action=APPEND, + dest="starttask", + sources={'play'}, + ), + OptionSettings( + ["--pause"], help="Pause the workflow immediately on start up.", - action="store_true", dest="paused_start") - - parser.add_option( - "--hold-after", "--hold-cycle-point", "--holdcp", + action=STORE_TRUE, + dest="paused_start", + sources={'play'}, + ), + OptionSettings( + ["--hold-after", "--hold-cycle-point", "--holdcp"], help="Hold all tasks after this cycle point.", - metavar="CYCLE_POINT", action="store", dest="holdcp") - - parser.add_option( - "-m", "--mode", - help="Run mode: live, dummy, simulation (default live).", - metavar="STRING", action="store", dest="run_mode", - choices=["live", "dummy", "simulation"]) - - parser.add_option( - "--reference-log", - help="Generate a reference log for use in reference tests.", - action="store_true", default=False, dest="genref") - - parser.add_option( - "--reference-test", - help="Do a test run against a previously generated reference log.", - action="store_true", default=False, dest="reftest") - - # Override standard parser option for specific help description. - parser.add_option( - "--host", + metavar="CYCLE_POINT", + action=STORE, + dest="holdcp", + sources={'play'}, + ), + OptionSettings( + ["--reference-log"], + help="Generate a reference log for use in reference ", + action=STORE_TRUE, + default=False, + dest="genref", + sources={'play'}, + ), + OptionSettings( + ["--reference-test"], + help="Do a test run against a previously generated reference.", + action=STORE_TRUE, + default=False, + dest="reftest", + sources={'play'}, + ), + OptionSettings( + ["--host"], help=( - "Specify the host on which to start-up the workflow. " - "If not specified, a host will be selected using " - "the '[scheduler]run hosts' global config." - ), - metavar="HOST", action="store", dest="host") - - parser.add_option( - "--format", - help="The format of the output: 'plain'=human readable, 'json'", - choices=("plain", "json"), - default="plain" - ) - - parser.add_option( - "--main-loop", + "Specify the host on which to start-up the workflow." + " If not specified, a host will be selected using" + " the '[scheduler]run hosts' global config."), + metavar="HOST", + action=STORE, + dest="host", + sources={'play'}, + ), + OptionSettings( + ["--format"], + help="The format of the output: 'plain'=human readable, 'json", + choices=('plain', 'json'), + default="plain", + dest='format', + sources={'play'}, + ), + OptionSettings( + ["--main-loop"], help=( - "Specify an additional plugin to run in the main loop." - " These are used in combination with those specified in" - " [scheduler][main loop]plugins. Can be used multiple times" - ), - metavar="PLUGIN_NAME", action="append", dest="main_loop" - ) - - parser.add_option( - "--abort-if-any-task-fails", + "Specify an additional plugin to run in the main" + " These are used in combination with those specified" + " [scheduler][main loop]plugins. Can be used multiple times."), + metavar="PLUGIN_NAME", + action=APPEND, + dest="main_loop", + sources={'play'}, + ), + OptionSettings( + ["--abort-if-any-task-fails"], help="If set workflow will abort with status 1 if any task fails.", - action="store_true", default=False, dest="abort_if_any_task_fails" - ) - - parser.add_option( - '--downgrade', + action=STORE_TRUE, + default=False, + dest="abort_if_any_task_fails", + sources={'play'}, + ), + PLAY_ICP_OPTION, + PLAY_RUN_MODE, + OptionSettings( + ['--downgrade'], help=( - 'Allow the workflow to be restarted with an older version of Cylc,' - ' NOT RECOMMENDED.' - ' By default Cylc prevents you from restarting a workflow with an' - ' older version of Cylc than it was previously run with.' - ' Use this flag to disable this check.' + 'Allow the workflow to be restarted with an' + ' older version of Cylc, NOT RECOMMENDED.' + ' By default Cylc prevents you from restarting' + ' a workflow with an older version of Cylc than' + ' it was previously run with. Use this flag' + ' to disable this check.' ), - action='store_true', - default=False - ) - - parser.add_option( - '--upgrade', + action=STORE_TRUE, + default=False, + sources={'play'} + ), + OptionSettings( + ['--upgrade'], help=( - 'Allow the workflow to be restarted with an newer version of Cylc.' + 'Allow the workflow to be restarted with' + ' a newer version of Cylc.' ), - action='store_true', - default=False + action=STORE_TRUE, + default=False, + sources={'play'} + ), +] + + +@lru_cache() +def get_option_parser(add_std_opts: bool = False) -> COP: + """Parse CLI for "cylc play".""" + parser = COP( + PLAY_DOC, + jset=True, + comms=True, + argdoc=[WORKFLOW_ID_ARG_DOC] ) + options = parser.get_cylc_rose_options() + PLAY_OPTIONS + for option in options: + if isinstance(option, OptionSettings): + parser.add_option(*option.args, **option.kwargs) + else: + parser.add_option(*option[ARGS], **option[KWARGS]) + if add_std_opts: # This is for the API wrapper for integration tests. Otherwise (CLI # use) "standard options" are added later in options.parse_args(). @@ -555,6 +603,11 @@ async def _run(scheduler: Scheduler) -> int: @cli_function(get_option_parser) def play(parser: COP, options: 'Values', id_: str): """Implement cylc play.""" + return _play(parser, options, id_) + + +def _play(parser: COP, options: 'Values', id_: str): + """Allows compound scripts to import play, but supply their own COP.""" if options.starttask: options.starttask = upgrade_legacy_ids( *options.starttask, diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py index 39d8d1c8e50..74a37119043 100644 --- a/cylc/flow/scripts/cylc.py +++ b/cylc/flow/scripts/cylc.py @@ -201,7 +201,8 @@ def get_version(long=False): 'ls': 'list', 'shutdown': 'stop', 'task-message': 'message', - 'unhold': 'release' + 'unhold': 'release', + 'validate-install-play': 'vip' } @@ -311,12 +312,6 @@ def match_command(command): for cmd in COMMANDS if cmd.startswith(command) }, - *{ - # search aliases - cmd - for alias, cmd in ALIASES.items() - if alias.startswith(command) - } } if len(possible_cmds) == 0: print( diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py index c157a49d4a8..e926fb41750 100755 --- a/cylc/flow/scripts/install.py +++ b/cylc/flow/scripts/install.py @@ -97,6 +97,7 @@ from cylc.flow.loggingutil import CylcLogFormatter from cylc.flow.option_parsers import ( CylcOptionParser as COP, + OptionSettings, Options ) from cylc.flow.pathutil import ( @@ -112,71 +113,80 @@ from cylc.flow.terminal import cli_function -def get_option_parser() -> COP: - parser = COP( - __doc__, - comms=True, - argdoc=[ - COP.optional( - ('SOURCE_NAME | PATH', - 'Workflow source name or path to source directory') - ) - ] - ) - - parser.add_option( - "--workflow-name", "-n", +INSTALL_OPTIONS = [ + OptionSettings( + ["--workflow-name", "-n"], help="Install into ~/cylc-run//runN ", action="store", metavar="WORKFLOW_NAME", default=None, - dest="workflow_name") - - parser.add_option( - "--symlink-dirs", + dest="workflow_name", + sources={'install'}, + ), + OptionSettings( + ["--symlink-dirs"], help=( - "Enter a comma-delimited list, in the form " - "'log=path/to/store, share = $HOME/some/path, ...'. " - "Use this option to override the global.cylc configuration for " - "local symlinks for the run, log, work, share and " - "share/cycle directories. " - "Enter an empty list '' to skip making localhost symlink dirs." - ), + "Enter a comma-delimited list, in the form" + " 'log=path/to/store, share = $HOME/some/path, ...'." + " Use this option to override the global.cylc configuration" + " for local symlinks for the run, log, work, share and" + " share/cycle directories. Enter" + " an empty list '' to skip making localhost symlink dirs."), action="store", - dest="symlink_dirs" - ) - - parser.add_option( - "--run-name", + dest="symlink_dirs", + sources={'install'}, + ), + OptionSettings( + ["--run-name"], help=( - "Give the run a custom name instead of automatically numbering it." - ), + "Give the run a custom name instead of automatically" + " numbering it."), action="store", metavar="RUN_NAME", default=None, - dest="run_name") - - parser.add_option( - "--no-run-name", + dest="run_name", + sources={'install'}, + ), + OptionSettings( + ["--no-run-name"], help=( - "Install the workflow directly into ~/cylc-run/, " - "without an automatic run number or custom run name." - ), + "Install the workflow directly into" + " ~/cylc-run/," + " without an automatic run number or custom run name."), action="store_true", default=False, - dest="no_run_name") - - parser.add_option( - "--no-ping", + dest="no_run_name", + sources={'install'}, + ), + OptionSettings( + ['--no-ping'], help=( "When scanning for active instances of the workflow, " - "do not attempt to contact the schedulers to get status." - ), - action="store_true", + "do not attempt to contact the schedulers to get status."), + action='store_true', default=False, - dest="no_ping") + dest='no_ping', + sources={'install'}, + ) +] - parser.add_cylc_rose_options() + +def get_option_parser() -> COP: + parser = COP( + __doc__, + comms=True, + argdoc=[ + COP.optional( + ('SOURCE_NAME | PATH', + 'Workflow source name or path to source directory') + ) + ] + ) + + options = parser.get_cylc_rose_options() + INSTALL_OPTIONS + + for option in options: + parser.add_option(*option.args, **option.kwargs) return parser @@ -320,5 +330,4 @@ def install( entry_point.name, exc ) from None - return workflow_name diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py index 6e5031c91ee..aa93752f40c 100755 --- a/cylc/flow/scripts/validate.py +++ b/cylc/flow/scripts/validate.py @@ -26,6 +26,7 @@ """ from ansimarkup import parse as cparse +from copy import deepcopy from optparse import Values import sys @@ -42,13 +43,56 @@ from cylc.flow.option_parsers import ( WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, + OptionSettings, Options, - icp_option, + ICP_OPTION, + ARGS, KWARGS ) from cylc.flow.profiler import Profiler from cylc.flow.task_proxy import TaskProxy from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function +from cylc.flow.scheduler_cli import RUN_MODE + + +VALIDATE_RUN_MODE = deepcopy(RUN_MODE) +VALIDATE_RUN_MODE.sources = {'validate'} +VALIDATE_ICP_OPTION = deepcopy(ICP_OPTION) +VALIDATE_ICP_OPTION.sources = {'validate'} + + +VALIDATE_OPTIONS = [ + OptionSettings( + ["--check-circular"], + help=( + "Check for circular dependencies in graphs when the number of" + " tasks is greater than 100 (smaller graphs are always" + " checked). This can be slow when the number of" + " tasks is high."), + action="store_true", + default=False, + dest="check_circular", + sources={'validate'} + ), + OptionSettings( + ["--output", "-o"], + help="Specify a file name to dump the processed flow.cylc.", + metavar="FILENAME", + action="store", + dest="output", + sources={'validate'} + ), + OptionSettings( + ["--profile"], + help="Output profiling (performance) information", + action="store_true", + default=False, + dest="profile_mode", + sources={'validate'} + ), + VALIDATE_RUN_MODE, + VALIDATE_ICP_OPTION +] def get_option_parser(): @@ -58,31 +102,13 @@ def get_option_parser(): argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], ) - parser.add_option( - "--check-circular", - help="Check for circular dependencies in graphs when the number of " - "tasks is greater than 100 (smaller graphs are always checked). " - "This can be slow when the number of " - "tasks is high.", - action="store_true", default=False, dest="check_circular") - - parser.add_option( - "--output", "-o", - help="Specify a file name to dump the processed flow.cylc.", - metavar="FILENAME", action="store", dest="output") - - parser.add_option( - "--profile", help="Output profiling (performance) information", - action="store_true", default=False, dest="profile_mode") + validate_options = parser.get_cylc_rose_options() + VALIDATE_OPTIONS - parser.add_option( - "-u", "--run-mode", help="Validate for run mode.", action="store", - default="live", dest="run_mode", - choices=['live', 'dummy', 'simulation']) - - parser.add_option(icp_option) - - parser.add_cylc_rose_options() + for option in validate_options: + if isinstance(option, OptionSettings): + parser.add_option(*option.args, **option.kwargs) + else: + parser.add_option(*option[ARGS], **option[KWARGS]) parser.set_defaults(is_validate=True) @@ -102,6 +128,10 @@ def get_option_parser(): @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: + wrapped_main(parser, options, workflow_id) + + +def wrapped_main(parser: COP, options: 'Values', workflow_id: str) -> None: """cylc validate CLI.""" profiler = Profiler(None, options.profile_mode) profiler.start() diff --git a/cylc/flow/scripts/validate_install_play.py b/cylc/flow/scripts/validate_install_play.py new file mode 100644 index 00000000000..7feaac163e4 --- /dev/null +++ b/cylc/flow/scripts/validate_install_play.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 + +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""cylc validate-install-play [OPTIONS] ARGS + +Validate install and play a single workflow. + +This script is equivalent to: + + $ cylc validate /path/to/myworkflow + $ cylc install /path/to/myworkflow + $ cylc play myworkflow + +""" + +from cylc.flow.scripts.validate import ( + VALIDATE_OPTIONS, + wrapped_main as validate_main +) +from cylc.flow.scripts.install import ( + INSTALL_OPTIONS, install, get_source_location +) +from cylc.flow.scheduler_cli import PLAY_OPTIONS +from cylc.flow.option_parsers import ( + CylcOptionParser as COP, + combine_options, + cleanup_sysargv, + log_subcommand, +) +from cylc.flow.scheduler_cli import _play +from cylc.flow.terminal import cli_function + +from typing import TYPE_CHECKING, Optional + + +if TYPE_CHECKING: + from optparse import Values + + +CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options() +VIP_OPTIONS = combine_options( + VALIDATE_OPTIONS, + INSTALL_OPTIONS, + PLAY_OPTIONS, + CYLC_ROSE_OPTIONS, + modify={'cylc-rose': 'validate, install'} +) + + +def get_option_parser() -> COP: + parser = COP( + __doc__, + comms=True, + jset=True, + argdoc=[ + COP.optional(( + 'SOURCE_NAME | PATH', + 'Workflow source name or path to source directory' + )) + ] + ) + for option in VIP_OPTIONS: + parser.add_option(*option.args, **option.kwargs) + return parser + + +@cli_function(get_option_parser) +def main(parser: COP, options: 'Values', workflow_id: Optional[str] = None): + """Run Cylc validate - install - play in sequence.""" + if not workflow_id: + workflow_id = '.' + + orig_source = workflow_id + source = get_source_location(workflow_id) + + log_subcommand('validate', source) + validate_main(parser, options, str(source)) + + log_subcommand('install', '.') + workflow_id = install(options, workflow_id) + + cleanup_sysargv( + 'play', + workflow_id, + options, + compound_script_opts=VIP_OPTIONS, + script_opts=( + PLAY_OPTIONS + CYLC_ROSE_OPTIONS + + parser.get_std_options() + ), + source=orig_source, + ) + + log_subcommand('play', workflow_id) + _play(parser, options, workflow_id) diff --git a/setup.cfg b/setup.cfg index 3aa0282073f..f6ee306837f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -199,6 +199,7 @@ cylc.command = trigger = cylc.flow.scripts.trigger:main validate = cylc.flow.scripts.validate:main view = cylc.flow.scripts.view:main + vip = cylc.flow.scripts.validate_install_play:main # async functions to run within the scheduler main loop cylc.main_loop = health_check = cylc.flow.main_loop.health_check diff --git a/tests/functional/cylc-combination-scripts/00-vip.t b/tests/functional/cylc-combination-scripts/00-vip.t new file mode 100644 index 00000000000..4657013db4f --- /dev/null +++ b/tests/functional/cylc-combination-scripts/00-vip.t @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#------------------------------------------------------------------------------ +# Test `cylc vip` (Validate Install Play) + +. "$(dirname "$0")/test_header" +set_test_number 5 + +WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" + +run_ok "${TEST_NAME_BASE}-from-path" \ + cylc vip --no-detach --debug \ + --workflow-name "${WORKFLOW_NAME}" \ + --initial-cycle-point=1300 \ + --no-run-name \ + --reference-test \ + "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" + +grep_ok "13000101T0000Z" "${TEST_NAME_BASE}-from-path.stdout" + +grep "\$" "${TEST_NAME_BASE}-from-path.stdout" > VIPOUT.txt + +named_grep_ok "${TEST_NAME_BASE}-it-validated" "$ cylc validate" "VIPOUT.txt" +named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc install" "VIPOUT.txt" +named_grep_ok "${TEST_NAME_BASE}-it-played" "$ cylc play" "VIPOUT.txt" + +purge +exit 0 diff --git a/tests/functional/cylc-combination-scripts/00-vip/flow.cylc b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc new file mode 100644 index 00000000000..745bf26cb2f --- /dev/null +++ b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc @@ -0,0 +1,7 @@ +[scheduler] + allow implicit tasks = True + +[scheduling] + initial cycle point = 1500 + [[graph]] + R1 = foo diff --git a/tests/functional/cylc-combination-scripts/00-vip/reference.log b/tests/functional/cylc-combination-scripts/00-vip/reference.log new file mode 100644 index 00000000000..70c76fc307c --- /dev/null +++ b/tests/functional/cylc-combination-scripts/00-vip/reference.log @@ -0,0 +1 @@ +13000101T0000Z/foo -triggered off [] in flow 1 diff --git a/tests/functional/cylc-combination-scripts/test_header b/tests/functional/cylc-combination-scripts/test_header new file mode 120000 index 00000000000..90bd5a36f92 --- /dev/null +++ b/tests/functional/cylc-combination-scripts/test_header @@ -0,0 +1 @@ +../lib/bash/test_header \ No newline at end of file diff --git a/tests/functional/database/04-lock-recover/bin/cylc-db-lock b/tests/functional/database/04-lock-recover/bin/cylc-db-lock index cdef1b2b8f1..90213dbf5e2 100755 --- a/tests/functional/database/04-lock-recover/bin/cylc-db-lock +++ b/tests/functional/database/04-lock-recover/bin/cylc-db-lock @@ -10,7 +10,7 @@ def main(): os.path.join(os.getenv("CYLC_WORKFLOW_RUN_DIR"), "log", "db")) lockf(handle, LOCK_SH) call([ - "cylc", "task", "message", "I have locked the public database file"]) + "cylc", "task-message", "I have locked the public database file"]) workflow_log_dir = os.getenv("CYLC_WORKFLOW_LOG_DIR") while True: for line in open(os.path.join(workflow_log_dir, "log")): diff --git a/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock b/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock index 6cc995e39a3..4a8846180ce 100755 --- a/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock +++ b/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock @@ -11,7 +11,7 @@ def main(): os.path.join(os.getenv("CYLC_WORKFLOW_RUN_DIR"), "log", "db")) lockf(handle, LOCK_SH) call([ - "cylc", "task", "message", "I have locked the public database file"]) + "cylc", "task-message", "I have locked the public database file"]) workflow_log_dir = os.getenv("CYLC_WORKFLOW_LOG_DIR") while True: for line in open(os.path.join(workflow_log_dir, "log")): diff --git a/tests/functional/env-filter/00-filter.t b/tests/functional/env-filter/00-filter.t index 30cd9027c28..6078258dd14 100644 --- a/tests/functional/env-filter/00-filter.t +++ b/tests/functional/env-filter/00-filter.t @@ -52,7 +52,7 @@ __FLOW_CONFIG__ #------------------------------------------------------------------------------- # check validation TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc val "${WORKFLOW_NAME}" +run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # check that config retrieves only the filtered environment TEST_NAME=${TEST_NAME_BASE}-config diff --git a/tests/functional/include-files/00-basic.t b/tests/functional/include-files/00-basic.t index 19fe3a9531b..23a48c86eec 100644 --- a/tests/functional/include-files/00-basic.t +++ b/tests/functional/include-files/00-basic.t @@ -24,12 +24,12 @@ install_workflow "${TEST_NAME_BASE}" workflow #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" # test raw workflow validates -run_ok "${TEST_NAME}.1" cylc val "${WORKFLOW_NAME}" +run_ok "${TEST_NAME}.1" cylc validate "${WORKFLOW_NAME}" # test workflow validates as inlined during editing mkdir inlined cylc view --inline "${WORKFLOW_NAME}" > inlined/flow.cylc -run_ok "${TEST_NAME}.2" cylc val ./inlined +run_ok "${TEST_NAME}.2" cylc validate ./inlined #------------------------------------------------------------------------------- # compare inlined workflow def with reference copy TEST_NAME=${TEST_NAME_BASE}-compare diff --git a/tests/functional/restart/09-reload.t b/tests/functional/restart/09-reload.t index ced8376b242..e84008bfccf 100755 --- a/tests/functional/restart/09-reload.t +++ b/tests/functional/restart/09-reload.t @@ -23,7 +23,7 @@ set_test_number 3 install_workflow "${TEST_NAME_BASE}" reload #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-validate" -run_ok "${TEST_NAME}" cylc val "${WORKFLOW_NAME}" +run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}" diff --git a/tests/functional/workflow-state/00-polling.t b/tests/functional/workflow-state/00-polling.t index b858f0767f0..f073f6e3c02 100644 --- a/tests/functional/workflow-state/00-polling.t +++ b/tests/functional/workflow-state/00-polling.t @@ -32,11 +32,11 @@ cylc install "${TEST_DIR}/upstream" --workflow-name="${UPSTREAM}" --no-run-name #------------------------------------------------------------------------------- # validate both workflows as tests TEST_NAME="${TEST_NAME_BASE}-validate-upstream" -run_ok "${TEST_NAME}" cylc val --debug "${UPSTREAM}" +run_ok "${TEST_NAME}" cylc validate --debug "${UPSTREAM}" TEST_NAME=${TEST_NAME_BASE}-validate-polling run_ok "${TEST_NAME}" \ - cylc val --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" + cylc validate --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # run the upstream workflow and detach (not a test) diff --git a/tests/functional/workflow-state/01-polling.t b/tests/functional/workflow-state/01-polling.t index 636d39e5211..b4694c35a6c 100644 --- a/tests/functional/workflow-state/01-polling.t +++ b/tests/functional/workflow-state/01-polling.t @@ -32,11 +32,11 @@ cylc install "${TEST_DIR}/upstream" --workflow-name="${UPSTREAM}" --no-run-name #------------------------------------------------------------------------------- # validate both workflows as tests TEST_NAME="${TEST_NAME_BASE}-validate-upstream" -run_ok "${TEST_NAME}" cylc val --debug "${UPSTREAM}" +run_ok "${TEST_NAME}" cylc validate --debug "${UPSTREAM}" TEST_NAME="${TEST_NAME_BASE}-validate-polling" run_ok "${TEST_NAME}" \ - cylc val --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" + cylc validate --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- # check auto-generated task script for lbad diff --git a/tests/functional/xtriggers/02-persistence.t b/tests/functional/xtriggers/02-persistence.t index 1cc9d18ffee..2b424473f7c 100644 --- a/tests/functional/xtriggers/02-persistence.t +++ b/tests/functional/xtriggers/02-persistence.t @@ -36,7 +36,7 @@ mkdir -p 'lib/python' cp "${WORKFLOW_RUN_DIR}/faker_succ.py" 'lib/python/faker.py' # Validate the test workflow. -run_ok "${TEST_NAME_BASE}-val" cylc val --debug "${WORKFLOW_NAME}" +run_ok "${TEST_NAME_BASE}-val" cylc validate --debug "${WORKFLOW_NAME}" # Run the first cycle, till auto shutdown by task. TEST_NAME="${TEST_NAME_BASE}-run" @@ -50,7 +50,7 @@ grep_ok 'NAME is bob' '2010.foo.out' cp "${WORKFLOW_RUN_DIR}/faker_fail.py" 'lib/python/faker.py' # Validate again (with the new xtrigger function). -run_ok "${TEST_NAME_BASE}-val2" cylc val --debug "${WORKFLOW_NAME}" +run_ok "${TEST_NAME_BASE}-val2" cylc validate --debug "${WORKFLOW_NAME}" # Restart the workflow, to run the final cycle point. TEST_NAME="${TEST_NAME_BASE}-restart" diff --git a/tests/unit/test_option_parsers.py b/tests/unit/test_option_parsers.py index dcd2f6e8977..5a5e4cbad71 100644 --- a/tests/unit/test_option_parsers.py +++ b/tests/unit/test_option_parsers.py @@ -14,14 +14,21 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import pytest +from contextlib import redirect_stdout +import io +import sys +from types import SimpleNamespace from typing import List -import sys -import io -from contextlib import redirect_stdout +import pytest +from pytest import param + import cylc.flow.flags -from cylc.flow.option_parsers import CylcOptionParser as COP, Options +from cylc.flow.option_parsers import ( + CylcOptionParser as COP, Options, combine_options, combine_options_pair, + OptionSettings, + ARGS, HELP, KWARGS, SOURCES, cleanup_sysargv, USEIF +) USAGE_WITH_COMMENT = "usage \n # comment" @@ -93,3 +100,411 @@ def test_Options_std_opts(): MyOptions = Options(parser) MyValues = MyOptions(verbosity=1) assert MyValues.verbosity == 1 + + +# Add overlapping args tomorrow +@pytest.mark.parametrize( + 'first, second, expect', + [ + param( + [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'do'}}], + [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'dont'}}], + ( + [{ + ARGS: ['-f', '--foo'], KWARGS: {}, + SOURCES: {'do', 'dont'}, USEIF: '' + }] + ), + id='identical arg lists unchanged' + ), + param( + [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'fall'}}], + [{ + ARGS: ['-f', '--foolish'], + KWARGS: {'help': 'not identical'}, + SOURCES: {'fold'}}], + ( + [ + { + ARGS: ['--foo'], KWARGS: {}, SOURCES: {'fall'}, + USEIF: '' + }, + { + ARGS: ['--foolish'], + KWARGS: {'help': 'not identical'}, + SOURCES: {'fold'}, + USEIF: '' + } + ] + ), + id='different arg lists lose shared names' + ), + param( + [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'cook'}}], + [{ + ARGS: ['-f', '--foo'], + KWARGS: {'help': 'not identical'}, + SOURCES: {'bake'}, + USEIF: '' + }], + None, + id='different args identical arg list cause exception' + ), + param( + [{ARGS: ['-g', '--goo'], KWARGS: {}, SOURCES: {'knit'}}], + [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'feed'}}], + [ + { + ARGS: ['-g', '--goo'], KWARGS: {}, + SOURCES: {'knit'}, USEIF: '' + }, + { + ARGS: ['-f', '--foo'], KWARGS: {}, + SOURCES: {'feed'}, USEIF: '' + }, + ], + id='all unrelated args added' + ), + param( + [ + {ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'work'}}, + {ARGS: ['-r', '--redesdale'], KWARGS: {}, SOURCES: {'work'}} + ], + [ + {ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'sink'}}, + { + ARGS: ['-b', '--buttered-peas'], + KWARGS: {}, SOURCES: {'sink'} + } + ], + [ + { + ARGS: ['-f', '--foo'], + KWARGS: {}, + SOURCES: {'work', 'sink'}, + USEIF: '' + }, + { + ARGS: ['-b', '--buttered-peas'], + KWARGS: {}, + SOURCES: {'sink'}, + USEIF: '' + }, + { + ARGS: ['-r', '--redesdale'], + KWARGS: {}, + SOURCES: {'work'}, + USEIF: '' + }, + ], + id='do not repeat args' + ), + param( + [ + { + ARGS: ['-f', '--foo'], + KWARGS: {}, + SOURCES: {'push'} + }, + ], + [], + [ + { + ARGS: ['-f', '--foo'], + KWARGS: {}, + SOURCES: {'push'}, + USEIF: '' + }, + ], + id='one empty list is fine' + ) + ] +) +def test_combine_options_pair(first, second, expect): + """It combines sets of options""" + first = [ + OptionSettings(i[ARGS], sources=i[SOURCES], **i[KWARGS]) + for i in first + ] + second = [ + OptionSettings(i[ARGS], sources=i[SOURCES], **i[KWARGS]) + for i in second + ] + if expect is not None: + result = combine_options_pair(first, second) + assert [i.__dict__ for i in result] == expect + else: + with pytest.raises(Exception, match='Clashing Options'): + combine_options_pair(first, second) + + +@pytest.mark.parametrize( + 'inputs, expect', + [ + param( + [ + ([OptionSettings( + ['-i', '--inflammable'], help='', sources={'wish'} + )]), + ([OptionSettings( + ['-f', '--flammable'], help='', sources={'rest'} + )]), + ([OptionSettings( + ['-n', '--non-flammable'], help='', sources={'swim'} + )]), + ], + [ + {ARGS: ['-i', '--inflammable']}, + {ARGS: ['-f', '--flammable']}, + {ARGS: ['-n', '--non-flammable']} + ], + id='merge three argsets no overlap' + ), + param( + [ + [ + OptionSettings( + ['-m', '--morpeth'], help='', sources={'stop'}), + OptionSettings( + ['-r', '--redesdale'], help='', sources={'stop'}), + ], + [ + OptionSettings( + ['-b', '--byker'], help='', sources={'walk'}), + OptionSettings( + ['-r', '--roxborough'], help='', sources={'walk'}), + ], + [ + OptionSettings( + ['-b', '--bellingham'], help='', sources={'leap'}), + ] + ], + [ + {ARGS: ['--bellingham']}, + {ARGS: ['--roxborough']}, + {ARGS: ['--redesdale']}, + {ARGS: ['--byker']}, + {ARGS: ['-m', '--morpeth']} + ], + id='merge three overlapping argsets' + ), + param( + [ + ([]), + ( + [ + OptionSettings( + ['-c', '--campden'], help='x', sources={'foo'}) + ] + ) + ], + [ + {ARGS: ['-c', '--campden']} + ], + id="empty list doesn't clear result" + ), + ] +) +def test_combine_options(inputs, expect): + """It combines multiple input sets""" + result = combine_options(*inputs) + result_args = [i.args for i in result] + + # Order of args irrelevent to test + for option in expect: + assert option[ARGS] in result_args + + +@pytest.mark.parametrize( + 'argv_before, kwargs, expect', + [ + param( + 'vip myworkflow --foo something'.split(), + { + 'script_name': 'play', + 'workflow_id': 'myworkflow', + 'compound_script_opts': [ + OptionSettings(['--foo', '-f']), + ], + 'script_opts': [ + OptionSettings(['--foo', '-f'])] + }, + 'play myworkflow --foo something'.split(), + id='no opts to remove' + ), + param( + 'vip myworkflow -f something -b something_else --baz'.split(), + { + 'script_name': 'play', + 'workflow_id': 'myworkflow', + 'compound_script_opts': [ + OptionSettings(['--foo', '-f']), + OptionSettings(['--bar', '-b'], action='store'), + OptionSettings(['--baz'], action='store_true'), + ], + 'script_opts': [ + OptionSettings(['--foo', '-f']), + ] + }, + 'play myworkflow -f something'.split(), + id='remove some opts' + ), + param( + 'vip myworkflow'.split(), + { + 'script_name': 'play', + 'workflow_id': 'myworkflow', + 'compound_script_opts': [ + OptionSettings(['--foo', '-f']), + OptionSettings(['--bar', '-b']), + OptionSettings(['--baz']), + ], + 'script_opts': [] + }, + 'play myworkflow'.split(), + id='no opts to keep' + ), + param( + 'vip ./myworkflow --foo something'.split(), + { + 'script_name': 'play', + 'workflow_id': 'myworkflow', + 'compound_script_opts': [ + OptionSettings(['--foo', '-f'])], + 'script_opts': [ + OptionSettings(['--foo', '-f']), + ], + 'source': './myworkflow', + }, + 'play --foo something myworkflow'.split(), + id='replace path' + ), + ] +) +def test_cleanup_sysargv(monkeypatch, argv_before, kwargs, expect): + """It replaces the contents of sysargv with Cylc Play argv items. + """ + # Fake up sys.argv: for this test. + dummy_cylc_path = ['/pathto/my/cylc/bin/cylc'] + monkeypatch.setattr(sys, 'argv', dummy_cylc_path + argv_before) + # Fake options too: + opts = SimpleNamespace(**{ + i.args[0].replace('--', ''): i for i in kwargs['compound_script_opts'] + }) + + kwargs.update({'options': opts}) + if not kwargs.get('source', None): + kwargs.update({'source': ''}) + + # Test the script: + cleanup_sysargv(**kwargs) + assert sys.argv == dummy_cylc_path + expect + + +class TestOptionSettings(): + @staticmethod + def test_init(): + args = ['--foo', '-f'] + kwargs = {'bar': 42} + sources = {'touch'} + useif = 'hello' + + result = OptionSettings( + args, sources=sources, useif=useif, **kwargs) + + assert result.__dict__ == { + 'kwargs': kwargs, 'sources': sources, + 'useif': useif, 'args': args + } + + @staticmethod + @pytest.mark.parametrize( + 'first, second, expect', + ( + param( + (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'), + (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'), + True, id='Totally the same'), + param( + (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'), + (['--foo', '-f'], {'bar': 42}, {'wibble'}, 'byee'), + True, id='Differing extras'), + param( + (['-f'], {'bar': 42}, {'touch'}, 'hello'), + (['--foo', '-f'], {'bar': 42}, {'wibble'}, 'byee'), + False, id='Not equal args'), + ) + ) + def test___eq__args_intersection(first, second, expect): + args, kwargs, sources, useif = first + first = OptionSettings( + args, sources=sources, useif=useif, **kwargs) + args, kwargs, sources, useif = second + second = OptionSettings( + args, sources=sources, useif=useif, **kwargs) + assert (first == second) == expect + + @staticmethod + @pytest.mark.parametrize( + 'first, second, expect', + ( + param( + ['--foo', '-f'], + ['--foo', '-f'], + ['--foo', '-f'], + id='Totally the same'), + param( + ['--foo', '-f'], + ['--foolish', '-f'], + ['-f'], + id='Some overlap'), + param( + ['--foo', '-f'], + ['--bar', '-b'], + [], + id='No overlap'), + ) + ) + def test___and__(first, second, expect): + first = OptionSettings(first) + second = OptionSettings(second) + assert sorted(first & second) == sorted(expect) + + @staticmethod + @pytest.mark.parametrize( + 'first, second, expect', + ( + param( + ['--foo', '-f'], + ['--foo', '-f'], + [], + id='Totally the same'), + param( + ['--foo', '-f'], + ['--foolish', '-f'], + ['--foo'], + id='Some overlap'), + param( + ['--foolish', '-f'], + ['--foo', '-f'], + ['--foolish'], + id='Some overlap not commuting'), + param( + ['--foo', '-f'], + ['--bar', '-b'], + ['--foo', '-f'], + id='No overlap'), + ) + ) + def test___sub__args_subtraction(first, second, expect): + first = OptionSettings(first) + second = OptionSettings(second) + assert sorted(first - second) == sorted(expect) + + @staticmethod + def test__in_list(): + """It is in a list.""" + first = OptionSettings(['--foo']) + second = OptionSettings(['--foo']) + third = OptionSettings(['--bar']) + assert first._in_list([second, third]) == True