diff --git a/CHANGES.md b/CHANGES.md index c5115a3ff6a..645010fd129 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,9 @@ ones in. --> ### Enhancements +[#5094](https://github.com/cylc/cylc-flow/pull/5094) - Added a single +command to validate, install and play a workflow. + [#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of 100 for the "default" queue. diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py index 4bd4ee8797a..056f7a6072d 100644 --- a/cylc/flow/option_parsers.py +++ b/cylc/flow/option_parsers.py @@ -17,6 +17,7 @@ from contextlib import suppress import logging +from itertools import product from optparse import ( OptionParser, OptionConflictError, @@ -33,7 +34,7 @@ import sys from textwrap import dedent -from typing import Any, Dict, Optional, List, Tuple +from typing import Any, Dict, Optional, List, Tuple, Union from cylc.flow import LOG from cylc.flow.terminal import supports_color, DIM @@ -51,6 +52,22 @@ SHORTLINK_TO_ICP_DOCS = "https://bit.ly/3MYHqVh" +ARGS = 'args' +KWARGS = 'kwargs' +HELP = 'help' +ACTION = 'action' +STORE = 'store' +STORE_CONST = 'store_const' +DEFAULT = 'default' +DEST = 'dest' +METAVAR = 'metavar' +CHOICES = 'choices' +OPTS = 'opts' +ALL = 'all' +USEIF = 'useif' +DOUBLEDASH = '--' + + icp_option = Option( "--initial-cycle-point", "--icp", metavar="CYCLE_POINT or OFFSET", @@ -65,6 +82,21 @@ ) +ICP_OPTION = { + 'args': ["--initial-cycle-point", "--icp"], + 'kwargs': { + "metavar": "CYCLE_POINT or OFFSET", + "help": + "Set the initial cycle point. " + "Required if not defined in flow.cylc." + "\nMay be either an absolute point or an offset: See " + f"{SHORTLINK_TO_ICP_DOCS} (Cylc documentation link).", + "action": "store", + "dest": "icp", + } +} + + def format_shell_examples(string): """Put comments in the terminal "diminished" colour.""" return cparse( @@ -145,7 +177,7 @@ def verbosity_to_env(verb: int) -> Dict[str, str]: } -def env_to_verbosity(env: dict) -> int: +def env_to_verbosity(env: Union[Dict, os._Environ]) -> int: """Extract verbosity from environment variables. Examples: @@ -259,6 +291,116 @@ class CylcOptionParser(OptionParser): See `cylc help id` for more details. ''')) + STD_OPTIONS = [ + { + ARGS: ['-q', '--quiet'], + KWARGS: { + HELP: 'Decrease verbosity.', + ACTION: 'decrement', + DEST: 'verbosity' + }, + USEIF: ALL + }, + { + ARGS: ['-v', '--verbose'], + KWARGS: { + HELP: 'Increase Verbosity', + DEST: 'verbosity', + ACTION: 'count', + DEFAULT: env_to_verbosity(os.environ) + }, + USEIF: ALL + }, + { + ARGS: ['--debug'], + KWARGS: { + HELP: 'Equivalent to -v -v', + DEST: 'verbosity', + ACTION: STORE_CONST, + 'const': 2 + }, + USEIF: ALL + }, + { + ARGS: ['--no-timestamp'], + KWARGS: { + HELP: 'Don\'t timestamp logged messages.', + ACTION: 'store_false', + DEST: 'log_timestamp', + DEFAULT: True + }, + USEIF: ALL + }, + { + ARGS: ['--color', '--color'], + KWARGS: { + METAVAR: 'WHEN', + ACTION: STORE, + DEFAULT: 'auto', + CHOICES: ['never', 'auto', 'always'], + HELP: + "When to use color/bold text in terminal output." + " Options are 'never', 'auto' and 'always'." + }, + USEIF: 'color' + }, + { + ARGS: ['--comms-timeout'], + KWARGS: { + METAVAR: 'SEC', + HELP: + "Set a timeout for network connections " + "to the running workflow. The default is no timeout. " + "For task messaging connections see " + "site/user config file documentation.", + ACTION: STORE, + DEFAULT: None, + DEST: 'comms_timeout' + }, + USEIF: 'comms' + }, + { + ARGS: ['-s', '--set'], + KWARGS: { + METAVAR: 'NAME=VALUE', + HELP: + "Set the value of a Jinja2 template variable in the" + " workflow definition." + " Values should be valid Python literals so strings" + " must be quoted" + " e.g. 'STR=\"string\"', INT=43, BOOL=True." + " This option can be used multiple " + " times on the command line." + " NOTE: these settings persist across workflow restarts," + " but can be set again on the \"cylc play\"" + " command line if they need to be overridden.", + ACTION: 'append', + DEFAULT: [], + DEST: 'templatevars' + }, + USEIF: 'jset' + }, + { + ARGS: ['--set-file'], + KWARGS: { + METAVAR: 'FILE', + HELP: + "Set the value of Jinja2 template variables in the " + "workflow definition from a file containing NAME=VALUE " + "pairs (one per line). " + "As with --set values should be valid Python literals " + "so strings must be quoted e.g. STR='string'. " + "NOTE: these settings persist across workflow restarts, " + "but can be set again on the \"cylc play\" " + "command line if they need to be overridden.", + ACTION: STORE, + DEFAULT: None, + DEST: 'templatevars_file' + }, + USEIF: 'jset' + } + ] + def __init__( self, usage: str, @@ -332,6 +474,17 @@ def __init__( formatter=CylcHelpFormatter() ) + def get_std_options(self): + """Get a data-structure of standard options""" + opts = [] + for opt in self.STD_OPTIONS: + if ( + opt[USEIF] == ALL + or hasattr(self, opt[USEIF]) and getattr(self, opt[USEIF]) + ): + opts.append(opt) + return opts + def add_std_option(self, *args, **kwargs): """Add a standard option, ignoring override.""" with suppress(OptionConflictError): @@ -339,121 +492,63 @@ def add_std_option(self, *args, **kwargs): def add_std_options(self): """Add standard options if they have not been overridden.""" - self.add_std_option( - "-q", "--quiet", - help="Decrease verbosity.", - action='decrement', - dest='verbosity', - ) - self.add_std_option( - "-v", "--verbose", - help="Increase verbosity.", - dest='verbosity', - action='count', - default=env_to_verbosity(os.environ) - ) - self.add_std_option( - "--debug", - help="Equivalent to -v -v", - dest="verbosity", - action='store_const', - const=2 - ) - self.add_std_option( - "--no-timestamp", - help="Don't timestamp logged messages.", - action="store_false", dest="log_timestamp", default=True) - - if self.color: - self.add_std_option( - '--color', '--colour', metavar='WHEN', action='store', - default='auto', choices=['never', 'auto', 'always'], - help=( - "When to use color/bold text in terminal output." - " Options are 'never', 'auto' and 'always'." - ) - ) - - if self.comms: - self.add_std_option( - "--comms-timeout", metavar='SEC', - help=( - "Set a timeout for network connections " - "to the running workflow. The default is no timeout. " - "For task messaging connections see " - "site/user config file documentation." - ), - action="store", default=None, dest="comms_timeout") - - if self.jset: - self.add_std_option( - "-s", "--set", metavar="NAME=VALUE", - help=( - "Set the value of a Jinja2 template variable in the" - " workflow definition." - " Values should be valid Python literals so strings" - " must be quoted" - " e.g. 'STR=\"string\"', INT=43, BOOL=True." - " This option can be used multiple " - " times on the command line." - " NOTE: these settings persist across workflow restarts," - " but can be set again on the \"cylc play\"" - " command line if they need to be overridden." - ), - action="append", default=[], dest="templatevars") - - self.add_std_option( - "--set-file", metavar="FILE", - help=( - "Set the value of Jinja2 template variables in the " - "workflow definition from a file containing NAME=VALUE " - "pairs (one per line). " - "As with --set values should be valid Python literals " - "so strings must be quoted e.g. STR='string'. " - "NOTE: these settings persist across workflow restarts, " - "but can be set again on the \"cylc play\" " - "command line if they need to be overridden." - ), - action="store", default=None, dest="templatevars_file") + for option in self.get_std_options(): + if not any([self.has_option(i) for i in option[ARGS]]): + self.add_option(*option[ARGS], **option[KWARGS]) - def add_cylc_rose_options(self) -> None: - """Add extra options for cylc-rose plugin if it is installed.""" + @staticmethod + def get_cylc_rose_options(): + """Returns a list of option dictionaries if Cylc Rose exists.""" try: __import__('cylc.rose') except ImportError: - return - self.add_option( - "--opt-conf-key", "-O", - help=( - "Use optional Rose Config Setting " - "(If Cylc-Rose is installed)" - ), - action="append", - default=[], - dest="opt_conf_keys" - ) - self.add_option( - "--define", '-D', - help=( - "Each of these overrides the `[SECTION]KEY` setting in a " - "`rose-suite.conf` file. " - "Can be used to disable a setting using the syntax " - "`--define=[SECTION]!KEY` or even `--define=[!SECTION]`." - ), - action="append", - default=[], - dest="defines" - ) - self.add_option( - "--rose-template-variable", '-S', '--define-suite', - help=( - "As `--define`, but with an implicit `[SECTION]` for " - "workflow variables." - ), - action="append", - default=[], - dest="rose_template_vars" - ) + return [] + return [ + { + ARGS: ["--opt-conf-key", "-O"], + KWARGS: { + HELP: + "Use optional Rose Config Setting " + "(If Cylc-Rose is installed)", + ACTION: "append", + DEFAULT: [], + DEST: "opt_conf_keys" + } + }, + { + ARGS: ["--define", '-D'], + KWARGS: { + HELP: + "Each of these overrides the `[SECTION]KEY` setting" + "in a `rose-suite.conf` file. " + "Can be used to disable a setting using the syntax " + "`--define=[SECTION]!KEY` or " + "even `--define=[!SECTION]`.", + ACTION: "append", + DEFAULT: [], + DEST: "defines" + } + }, + { + ARGS: ["--rose-template-variable", '-S', '--define-suite'], + KWARGS: { + HELP: + "As `--define`, but with an implicit `[SECTION]` for " + "workflow variables.", + ACTION: "append", + DEFAULT: [], + DEST: "rose_template_vars" + } + } + ] + + def add_cylc_rose_options(self) -> None: + """Add extra options for cylc-rose plugin if it is installed. + + Now a vestigal interface for get_cylc_rose_options. + """ + for option in self.get_cylc_rose_options(): + self.add_option(*option[ARGS], **option[KWARGS]) def parse_args(self, api_args, remove_opts=None): """Parse options and arguments, overrides OptionParser.parse_args. @@ -583,3 +678,108 @@ def __call__(self, **kwargs) -> Values: setattr(opts, key, value) return opts + + +def combine_options_pair(first_list, second_list): + """Combine two option lists. + + Scenarios: + - Arguments are identical - return this argument. + - Arguments are not identical but have some common label strings, + i.e. both arguments can be invoked using `-f`. + - If there are non-shared label strings strip the shared ones. + - Otherwise raise an error. + E.g: If `command-A` has an option `-f` or `--file` and + `command-B has an option `-f` or `--fortran`` then + `command-A+B` will have options `--fortran` and `--file` but _not_ + `-f`, which would be confusing. + + TODO: Record which list each option came from, so that you can tell + the user which component script is being referred to by each option. + """ + def appendif(list_, item): + """Avoid duplicating items in output list""" + if item not in list_: + list_.append(item) + return list_ + + output = [] + + for first, second in product(first_list, second_list): + if first == second and first not in output: + output = appendif(output, first) + + # If any of the argument name identical: + elif ( + any([i[0] == i[1] for i in product(second[ARGS], first[ARGS])]) + and first != second + ): + # if any of the args are different: + if first[ARGS] != second[ARGS]: + first_args = list(set(first[ARGS]) - set(second[ARGS])) + second[ARGS] = list(set(second[ARGS]) - set(first[ARGS])) + first[ARGS] = first_args + output = appendif(output, first) + output = appendif(output, second) + else: + # if none of the arg names are different. + raise Exception(f'Clashing Options \n{first}\n{second}') + else: + if all([first[ARGS] != i[ARGS] for i in second_list]): + output = appendif(output, first) + if all([second[ARGS] != i[ARGS] for i in first_list]): + output = appendif(output, second) + return output + + +def combine_options(*args): + """Combine a list of argument dicts. + + Ordering should be irrelevant because combine_options_pair should + be commutative, and the overall order of args is not relevant. + """ + list_ = list(args) + output = list_[0] + for arg in list_[1:]: + if arg and output: + output = combine_options_pair(arg, output) + elif arg: + output = arg + return output + + +def cleanup_sysargv( + script_name, + options, + compound_script_opts, + script_opts +): + """Remove unwanted options from sys.argv + + Some cylc scripts (notably Cylc Play when it is re-invoked on a scheduler + server) require the correct content in sys.argv. + """ + # Organize Options by dest. + script_opts_by_dest = { + x[KWARGS].get('dest', x[ARGS][0].strip(DOUBLEDASH)): x + for x in script_opts + } + compound_opts_by_dest = { + x[KWARGS].get('dest', x[ARGS][0].strip(DOUBLEDASH)): x + for x in compound_script_opts + } + + # Filter out non-cylc-play options. + for unwanted_opt in (set(options.__dict__)) - set(script_opts_by_dest): + for arg in compound_opts_by_dest[unwanted_opt][ARGS]: + if arg in sys.argv: + index = sys.argv.index(arg) + sys.argv.pop(index) + if ( + compound_opts_by_dest[unwanted_opt][KWARGS][ACTION] + not in ['store_true', 'store_false'] + ): + sys.argv.pop(index + 1) + + # replace compound script name: + sys.argv[1] = script_name diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py index 5091a1a1400..275069a0bc5 100644 --- a/cylc/flow/scheduler_cli.py +++ b/cylc/flow/scheduler_cli.py @@ -38,7 +38,8 @@ WORKFLOW_ID_ARG_DOC, CylcOptionParser as COP, Options, - icp_option, + ICP_OPTION, + ARGS, KWARGS, HELP, ACTION, DEFAULT, DEST, METAVAR, CHOICES ) from cylc.flow.pathutil import get_workflow_run_scheduler_log_path from cylc.flow.remote import _remote_cylc_cmd @@ -107,6 +108,169 @@ } ''' +PLAY_OPTIONS = [ + { + ARGS: ["-n", "--no-detach", "--non-daemon"], + KWARGS: { + HELP: "Do not daemonize the scheduler (infers --format=plain)", + ACTION: "store_true", + DEST: "no_detach", + } + }, + { + ARGS: ["--profile"], + KWARGS: { + HELP: "Output profiling (performance) information", + ACTION: "store_true", + DEFAULT: False, + DEST: "profile_mode" + } + }, + { + ARGS: ["--start-cycle-point", "--startcp"], + KWARGS: { + HELP: + "Set the start cycle point, which may be after " + "the initial cycle point. If the specified start point is " + "not in the sequence, the next on-sequence point will " + "be used. (Not to be confused with the initial cycle point)", + METAVAR: "CYCLE_POINT", + ACTION: "store", + DEST: "startcp", + } + }, + { + ARGS: ["--final-cycle-point", "--fcp"], + KWARGS: { + HELP: + "Set the final cycle point. This command line option overrides" + "the workflow config option '[scheduling]final cycle point'. ", + METAVAR: "CYCLE_POINT", + ACTION: "store", + DEST: "fcp", + } + }, + { + ARGS: ["--stop-cycle-point", "--stopcp"], + KWARGS: { + HELP: + "Set the stop cycle point. Shut down after all " + "have PASSED this cycle point. (Not to be confused " + "the final cycle point.) This command line option overrides" + "the workflow config option " + "'[scheduling]stop after cycle point'. ", + METAVAR: "CYCLE_POINT", + ACTION: "store", + DEST: "stopcp", + } + }, + { + ARGS: ["--start-task", "--starttask", "-t"], + KWARGS: { + HELP: + "Start from this task instance, given by '/'. " + "This can be used multiple times to start from multiple " + "tasks at once. Dependence on tasks with cycle points earlier" + "than the earliest start-task will be ignored. A" + "sub-graph of the workflow will run if selected tasks do " + "not lead on to the full graph.", + METAVAR: "TASK_ID", + ACTION: "append", + DEST: "starttask", + } + }, + { + ARGS: ["--pause"], + KWARGS: { + HELP: "Pause the workflow immediately on start up.", + ACTION: "store_true", + DEST: "paused_start", + } + }, + { + ARGS: ["--hold-after", "--hold-cycle-point", "--holdcp"], + KWARGS: { + HELP: "Hold all tasks after this cycle point.", + METAVAR: "CYCLE_POINT", + ACTION: "store", + DEST: "holdcp", + } + }, + { + ARGS: ["-m", "--mode"], + KWARGS: { + HELP: "Run mode: live, dummy, simulation (default live).", + METAVAR: "STRING", + ACTION: "store", + DEST: "run_mode", + CHOICES: ['live', 'dummy', 'simulation'], + } + }, + { + ARGS: ["--reference-log"], + KWARGS: { + HELP: "Generate a reference log for use in reference ", + ACTION: "store_true", + DEFAULT: False, + DEST: "genref", + } + }, + { + ARGS: ["--reference-test"], + KWARGS: { + HELP: + "Do a test run against a previously generated reference ", + ACTION: "store_true", + DEFAULT: False, + DEST: "reftest", + } + }, + { + ARGS: ["--host"], + KWARGS: { + HELP: + "Specify the host on which to start-up the workflow. " + "not specified, a host will be selected using the ", + METAVAR: "HOST", + ACTION: "store", + DEST: "host", + } + }, + { + ARGS: ["--format"], + KWARGS: { + HELP: + "The format of the output: 'plain'=human readable, ", + CHOICES: ('plain', 'json'), + DEFAULT: "plain", + DEST: 'format' + } + }, + { + ARGS: ["--main-loop"], + KWARGS: { + HELP: + "Specify an additional plugin to run in the main " + "These are used in combination with those specified " + "[scheduler][main loop]plugins. Can be used multiple times.", + METAVAR: "PLUGIN_NAME", + ACTION: "append", + DEST: "main_loop", + } + }, + { + ARGS: ["--abort-if-any-task-fails"], + KWARGS: { + HELP: + "If set workflow will abort with status 1 if any task fails.", + ACTION: "store_true", + DEFAULT: False, + DEST: "abort_if_any_task_fails", + } + }, + ICP_OPTION +] + @lru_cache() def get_option_parser(add_std_opts: bool = False) -> COP: @@ -118,118 +282,9 @@ def get_option_parser(add_std_opts: bool = False) -> COP: argdoc=[WORKFLOW_ID_ARG_DOC] ) - parser.add_option( - "-n", "--no-detach", "--non-daemon", - help="Do not daemonize the scheduler (infers --format=plain)", - action="store_true", dest="no_detach") - - parser.add_option( - "--profile", help="Output profiling (performance) information", - action="store_true", dest="profile_mode") - - parser.add_option(icp_option) - - parser.add_option( - "--start-cycle-point", "--startcp", - help=( - "Set the start cycle point, which may be after the initial cycle " - "point. If the specified start point is not in the sequence, the " - "next on-sequence point will be used. " - "(Not to be confused with the initial cycle point.) " - "This replaces the Cylc 7 --warm option." - ), - metavar="CYCLE_POINT", action="store", dest="startcp") - - parser.add_option( - "--final-cycle-point", "--fcp", - help=( - "Set the final cycle point. " - "This command line option overrides the workflow " - "config option '[scheduling]final cycle point'. " - ), - metavar="CYCLE_POINT", action="store", dest="fcp") - - parser.add_option( - "--stop-cycle-point", "--stopcp", - help=( - "Set the stop cycle point. " - "Shut down after all tasks have PASSED this cycle point. " - "(Not to be confused with the final cycle point.) " - "This command line option overrides the workflow " - "config option '[scheduling]stop after cycle point'. " - ), - metavar="CYCLE_POINT", action="store", dest="stopcp") - - parser.add_option( - "--start-task", "--starttask", "-t", - help=( - "Start from this task instance, given by '/'. " - "Can be used multiple times " - "to start from multiple tasks at once. Dependence on tasks with " - "with cycle points earlier than the earliest start-task will be " - "ignored. A sub-graph of the workflow will run if selected tasks " - "do not lead on to the full graph." - ), - metavar="TASK_ID", action="append", dest="starttask") - - parser.add_option( - "--pause", - help="Pause the workflow immediately on start up.", - action="store_true", dest="paused_start") - - parser.add_option( - "--hold-after", "--hold-cycle-point", "--holdcp", - help="Hold all tasks after this cycle point.", - metavar="CYCLE_POINT", action="store", dest="holdcp") - - parser.add_option( - "-m", "--mode", - help="Run mode: live, dummy, simulation (default live).", - metavar="STRING", action="store", dest="run_mode", - choices=["live", "dummy", "simulation"]) - - parser.add_option( - "--reference-log", - help="Generate a reference log for use in reference tests.", - action="store_true", default=False, dest="genref") - - parser.add_option( - "--reference-test", - help="Do a test run against a previously generated reference log.", - action="store_true", default=False, dest="reftest") - - # Override standard parser option for specific help description. - parser.add_option( - "--host", - help=( - "Specify the host on which to start-up the workflow. " - "If not specified, a host will be selected using " - "the '[scheduler]run hosts' global config." - ), - metavar="HOST", action="store", dest="host") - - parser.add_option( - "--format", - help="The format of the output: 'plain'=human readable, 'json'", - choices=("plain", "json"), - default="plain" - ) - - parser.add_option( - "--main-loop", - help=( - "Specify an additional plugin to run in the main loop." - " These are used in combination with those specified in" - " [scheduler][main loop]plugins. Can be used multiple times" - ), - metavar="PLUGIN_NAME", action="append", dest="main_loop" - ) - - parser.add_option( - "--abort-if-any-task-fails", - help="If set workflow will abort with status 1 if any task fails.", - action="store_true", default=False, dest="abort_if_any_task_fails" - ) + options = parser.get_cylc_rose_options() + PLAY_OPTIONS + for option in options: + parser.add_option(*option[ARGS], **option[KWARGS]) if add_std_opts: # This is for the API wrapper for integration tests. Otherwise (CLI @@ -427,6 +482,11 @@ async def _run(scheduler: Scheduler) -> int: @cli_function(get_option_parser) def play(parser: COP, options: 'Values', id_: str): """Implement cylc play.""" + return _play(parser, options, id_) + + +def _play(parser: COP, options: 'Values', id_: str): + """Allows compound scripts to import play, but supply their own COP.""" if options.starttask: options.starttask = upgrade_legacy_ids( *options.starttask, diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py index 761bc7dec23..07ef7935b4e 100755 --- a/cylc/flow/scripts/install.py +++ b/cylc/flow/scripts/install.py @@ -85,7 +85,10 @@ from cylc.flow import iter_entry_points from cylc.flow.exceptions import PluginError, InputError -from cylc.flow.option_parsers import CylcOptionParser as COP +from cylc.flow.option_parsers import ( + CylcOptionParser as COP, + ARGS, KWARGS, HELP, ACTION, DEFAULT, DEST, METAVAR +) from cylc.flow.pathutil import EXPLICIT_RELATIVE_PATH_REGEX, expand_path from cylc.flow.workflow_files import ( install_workflow, search_install_source_dirs, parse_cli_sym_dirs @@ -96,6 +99,58 @@ from optparse import Values +INSTALL_OPTIONS = [ + { + ARGS: ["--workflow-name", "-n"], + KWARGS: { + HELP: "Install into ~/cylc-run//runN ", + ACTION: "store", + METAVAR: "WORKFLOW_NAME", + DEFAULT: None, + DEST: "workflow_name" + } + }, + { + ARGS: ["--symlink-dirs"], + KWARGS: { + HELP: + "Enter a comma-delimited list, in the form " + "'log=path/to/store, share = $HOME/some/path, ...'. " + "Use this option to override the global.cylc configuration " + "for local symlinks for the run, log, work, share and " + "share/cycle directories. Enter " + "an empty list '' to skip making localhost symlink dirs.", + ACTION: "store", + DEST: "symlink_dirs" + } + }, + { + ARGS: ["--run-name"], + KWARGS: { + HELP: + "Give the run a custom name instead of automatically " + "numbering it.", + ACTION: "store", + METAVAR: "RUN_NAME", + DEFAULT: None, + DEST: "run_name" + } + }, + { + ARGS: ["--no-run-name"], + KWARGS: { + HELP: + "Install the workflow directly into " + "~/cylc-run/, " + "without an automatic run number or custom run name.", + ACTION: "store_true", + DEFAULT: False, + DEST: "no_run_name" + } + } +] + + def get_option_parser() -> COP: parser = COP( __doc__, @@ -108,49 +163,10 @@ def get_option_parser() -> COP: ] ) - parser.add_option( - "--workflow-name", "-n", - help="Install into ~/cylc-run//runN ", - action="store", - metavar="WORKFLOW_NAME", - default=None, - dest="workflow_name") - - parser.add_option( - "--symlink-dirs", - help=( - "Enter a comma-delimited list, in the form " - "'log=path/to/store, share = $HOME/some/path, ...'. " - "Use this option to override the global.cylc configuration for " - "local symlinks for the run, log, work, share and " - "share/cycle directories. " - "Enter an empty list '' to skip making localhost symlink dirs." - ), - action="store", - dest="symlink_dirs" - ) + options = parser.get_cylc_rose_options() + INSTALL_OPTIONS - parser.add_option( - "--run-name", - help=( - "Give the run a custom name instead of automatically numbering it." - ), - action="store", - metavar="RUN_NAME", - default=None, - dest="run_name") - - parser.add_option( - "--no-run-name", - help=( - "Install the workflow directly into ~/cylc-run/, " - "without an automatic run number or custom run name." - ), - action="store_true", - default=False, - dest="no_run_name") - - parser.add_cylc_rose_options() + for option in options: + parser.add_option(*option[ARGS], **option[KWARGS]) return parser @@ -178,7 +194,7 @@ def main(parser, opts, reg=None): def install( parser: COP, opts: 'Values', reg: Optional[str] = None -) -> None: +) -> str: if opts.no_run_name and opts.run_name: raise InputError( "options --no-run-name and --run-name are mutually exclusive." @@ -229,3 +245,4 @@ def install( entry_point.name, exc ) from None + return _workflow_name diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py index 6e5031c91ee..47e5f097c90 100755 --- a/cylc/flow/scripts/validate.py +++ b/cylc/flow/scripts/validate.py @@ -43,13 +43,59 @@ WORKFLOW_ID_OR_PATH_ARG_DOC, CylcOptionParser as COP, Options, - icp_option, + ICP_OPTION, + ARGS, KWARGS, HELP, ACTION, DEFAULT, DEST, METAVAR, CHOICES ) from cylc.flow.profiler import Profiler from cylc.flow.task_proxy import TaskProxy from cylc.flow.templatevars import get_template_vars from cylc.flow.terminal import cli_function +VALIDATE_OPTIONS = [ + { + ARGS: ["--check-circular"], + KWARGS: { + HELP: + "Check for circular dependencies in graphs when the number of " + "tasks is greater than 100 (smaller graphs are always " + "checked). This can be slow when the number of " + "tasks is high.", + ACTION: "store_true", + DEFAULT: False, + DEST: "check_circular" + } + }, + { + ARGS: ["--output", "-o"], + KWARGS: { + HELP: "Specify a file name to dump the processed flow.cylc.", + METAVAR: "FILENAME", + ACTION: "store", + DEST: "output" + } + }, + { + ARGS: ["--profile"], + KWARGS: { + HELP: "Output profiling (performance) information", + ACTION: "store_true", + DEFAULT: False, + DEST: "profile_mode" + }, + }, + { + ARGS: ["-u", "--run-mode"], + KWARGS: { + HELP: "Validate for run mode.", + ACTION: "store", + DEFAULT: "live", + DEST: "run_mode", + CHOICES: ["live", "dummy", "simulation"] + } + }, + ICP_OPTION, +] + def get_option_parser(): parser = COP( @@ -58,31 +104,10 @@ def get_option_parser(): argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], ) - parser.add_option( - "--check-circular", - help="Check for circular dependencies in graphs when the number of " - "tasks is greater than 100 (smaller graphs are always checked). " - "This can be slow when the number of " - "tasks is high.", - action="store_true", default=False, dest="check_circular") - - parser.add_option( - "--output", "-o", - help="Specify a file name to dump the processed flow.cylc.", - metavar="FILENAME", action="store", dest="output") + validate_options = parser.get_cylc_rose_options() + VALIDATE_OPTIONS - parser.add_option( - "--profile", help="Output profiling (performance) information", - action="store_true", default=False, dest="profile_mode") - - parser.add_option( - "-u", "--run-mode", help="Validate for run mode.", action="store", - default="live", dest="run_mode", - choices=['live', 'dummy', 'simulation']) - - parser.add_option(icp_option) - - parser.add_cylc_rose_options() + for option in validate_options: + parser.add_option(*option[ARGS], **option[KWARGS]) parser.set_defaults(is_validate=True) @@ -102,6 +127,10 @@ def get_option_parser(): @cli_function(get_option_parser) def main(parser: COP, options: 'Values', workflow_id: str) -> None: + wrapped_main(parser, options, workflow_id) + + +def wrapped_main(parser: COP, options: 'Values', workflow_id: str) -> None: """cylc validate CLI.""" profiler = Profiler(None, options.profile_mode) profiler.start() diff --git a/cylc/flow/scripts/validate_install_play.py b/cylc/flow/scripts/validate_install_play.py new file mode 100644 index 00000000000..9b86025364c --- /dev/null +++ b/cylc/flow/scripts/validate_install_play.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""cylc validate-install-play [OPTIONS] ARGS + +Validate install and play a single workflow. + +This script is equivelent to: + + $ cylc validate /path/to/myworkflow + $ cylc install /path/to/myworkflow + $ cylc play myworkflow + +""" + +from ansimarkup import parse as cparse + +from cylc.flow import LOG +from cylc.flow.scripts.validate import ( + VALIDATE_OPTIONS, + wrapped_main as validate_main +) +from cylc.flow.scripts.install import ( + INSTALL_OPTIONS, install, get_source_location +) +from cylc.flow.scheduler_cli import PLAY_OPTIONS +from cylc.flow.option_parsers import ( + CylcOptionParser as COP, + combine_options, + cleanup_sysargv, + ARGS, KWARGS +) +from cylc.flow.scheduler_cli import _play +from cylc.flow.terminal import cli_function + +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from optparse import Values + + +CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options() +VIP_OPTIONS = combine_options( + VALIDATE_OPTIONS, INSTALL_OPTIONS, + PLAY_OPTIONS, CYLC_ROSE_OPTIONS +) + + +def log_subcommand(command, ident): + """Log a command run as part of a sequence. + + TODO: When implementing other Compound commands, + move this somewhere common. + """ + print(cparse( + f'RUNNING: $ cylc {command} {ident}')) + + +def get_option_parser() -> COP: + parser = COP( + __doc__, + comms=True, + jset=True, + argdoc=[ + COP.optional(( + 'SOURCE_NAME | PATH', + 'Workflow source name or path to source directory' + )) + ] + ) + for option in VIP_OPTIONS: + parser.add_option(*option[ARGS], **option[KWARGS]) + return parser + + +@cli_function(get_option_parser) +def main(parser: COP, options: 'Values', workflow_id: str): + """Run Cylc validate - install - play in sequence.""" + source = get_source_location(workflow_id) + + LOG.info('Running Cylc Validate && Cylc Install && Cylc Play') + + log_subcommand('validate', source) + validate_main(parser, options, str(source)) + + log_subcommand('install', workflow_id) + workflow_id = install(parser, options, workflow_id) + + cleanup_sysargv( + 'play', + options, + compound_script_opts=VIP_OPTIONS, + script_opts=( + PLAY_OPTIONS + CYLC_ROSE_OPTIONS + + parser.get_std_options() + ), + ) + + log_subcommand('play', workflow_id) + _play(parser, options, workflow_id) diff --git a/cylcvip b/cylcvip new file mode 100644 index 00000000000..240a7e42854 --- /dev/null +++ b/cylcvip @@ -0,0 +1,1494 @@ +diff --git a/CHANGES.md b/CHANGES.md +index c5115a3ff..645010fd1 100644 +--- a/CHANGES.md ++++ b/CHANGES.md +@@ -34,6 +34,9 @@ ones in. --> + + ### Enhancements + ++[#5094](https://github.com/cylc/cylc-flow/pull/5094) - Added a single ++command to validate, install and play a workflow. ++ + [#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of + 100 for the "default" queue. + +diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py +index 4bd4ee879..056f7a607 100644 +--- a/cylc/flow/option_parsers.py ++++ b/cylc/flow/option_parsers.py +@@ -17,6 +17,7 @@ + + from contextlib import suppress + import logging ++from itertools import product + from optparse import ( + OptionParser, + OptionConflictError, +@@ -33,7 +34,7 @@ from ansimarkup import ( + + import sys + from textwrap import dedent +-from typing import Any, Dict, Optional, List, Tuple ++from typing import Any, Dict, Optional, List, Tuple, Union + + from cylc.flow import LOG + from cylc.flow.terminal import supports_color, DIM +@@ -51,6 +52,22 @@ FULL_ID_MULTI_ARG_DOC = ('ID ...', 'Cycle/Family/Task ID(s)') + + SHORTLINK_TO_ICP_DOCS = "https://bit.ly/3MYHqVh" + ++ARGS = 'args' ++KWARGS = 'kwargs' ++HELP = 'help' ++ACTION = 'action' ++STORE = 'store' ++STORE_CONST = 'store_const' ++DEFAULT = 'default' ++DEST = 'dest' ++METAVAR = 'metavar' ++CHOICES = 'choices' ++OPTS = 'opts' ++ALL = 'all' ++USEIF = 'useif' ++DOUBLEDASH = '--' ++ ++ + icp_option = Option( + "--initial-cycle-point", "--icp", + metavar="CYCLE_POINT or OFFSET", +@@ -65,6 +82,21 @@ icp_option = Option( + ) + + ++ICP_OPTION = { ++ 'args': ["--initial-cycle-point", "--icp"], ++ 'kwargs': { ++ "metavar": "CYCLE_POINT or OFFSET", ++ "help": ++ "Set the initial cycle point. " ++ "Required if not defined in flow.cylc." ++ "\nMay be either an absolute point or an offset: See " ++ f"{SHORTLINK_TO_ICP_DOCS} (Cylc documentation link).", ++ "action": "store", ++ "dest": "icp", ++ } ++} ++ ++ + def format_shell_examples(string): + """Put comments in the terminal "diminished" colour.""" + return cparse( +@@ -145,7 +177,7 @@ def verbosity_to_env(verb: int) -> Dict[str, str]: + } + + +-def env_to_verbosity(env: dict) -> int: ++def env_to_verbosity(env: Union[Dict, os._Environ]) -> int: + """Extract verbosity from environment variables. + + Examples: +@@ -259,6 +291,116 @@ class CylcOptionParser(OptionParser): + See `cylc help id` for more details. + ''')) + ++ STD_OPTIONS = [ ++ { ++ ARGS: ['-q', '--quiet'], ++ KWARGS: { ++ HELP: 'Decrease verbosity.', ++ ACTION: 'decrement', ++ DEST: 'verbosity' ++ }, ++ USEIF: ALL ++ }, ++ { ++ ARGS: ['-v', '--verbose'], ++ KWARGS: { ++ HELP: 'Increase Verbosity', ++ DEST: 'verbosity', ++ ACTION: 'count', ++ DEFAULT: env_to_verbosity(os.environ) ++ }, ++ USEIF: ALL ++ }, ++ { ++ ARGS: ['--debug'], ++ KWARGS: { ++ HELP: 'Equivalent to -v -v', ++ DEST: 'verbosity', ++ ACTION: STORE_CONST, ++ 'const': 2 ++ }, ++ USEIF: ALL ++ }, ++ { ++ ARGS: ['--no-timestamp'], ++ KWARGS: { ++ HELP: 'Don\'t timestamp logged messages.', ++ ACTION: 'store_false', ++ DEST: 'log_timestamp', ++ DEFAULT: True ++ }, ++ USEIF: ALL ++ }, ++ { ++ ARGS: ['--color', '--color'], ++ KWARGS: { ++ METAVAR: 'WHEN', ++ ACTION: STORE, ++ DEFAULT: 'auto', ++ CHOICES: ['never', 'auto', 'always'], ++ HELP: ++ "When to use color/bold text in terminal output." ++ " Options are 'never', 'auto' and 'always'." ++ }, ++ USEIF: 'color' ++ }, ++ { ++ ARGS: ['--comms-timeout'], ++ KWARGS: { ++ METAVAR: 'SEC', ++ HELP: ++ "Set a timeout for network connections " ++ "to the running workflow. The default is no timeout. " ++ "For task messaging connections see " ++ "site/user config file documentation.", ++ ACTION: STORE, ++ DEFAULT: None, ++ DEST: 'comms_timeout' ++ }, ++ USEIF: 'comms' ++ }, ++ { ++ ARGS: ['-s', '--set'], ++ KWARGS: { ++ METAVAR: 'NAME=VALUE', ++ HELP: ++ "Set the value of a Jinja2 template variable in the" ++ " workflow definition." ++ " Values should be valid Python literals so strings" ++ " must be quoted" ++ " e.g. 'STR=\"string\"', INT=43, BOOL=True." ++ " This option can be used multiple " ++ " times on the command line." ++ " NOTE: these settings persist across workflow restarts," ++ " but can be set again on the \"cylc play\"" ++ " command line if they need to be overridden.", ++ ACTION: 'append', ++ DEFAULT: [], ++ DEST: 'templatevars' ++ }, ++ USEIF: 'jset' ++ }, ++ { ++ ARGS: ['--set-file'], ++ KWARGS: { ++ METAVAR: 'FILE', ++ HELP: ++ "Set the value of Jinja2 template variables in the " ++ "workflow definition from a file containing NAME=VALUE " ++ "pairs (one per line). " ++ "As with --set values should be valid Python literals " ++ "so strings must be quoted e.g. STR='string'. " ++ "NOTE: these settings persist across workflow restarts, " ++ "but can be set again on the \"cylc play\" " ++ "command line if they need to be overridden.", ++ ACTION: STORE, ++ DEFAULT: None, ++ DEST: 'templatevars_file' ++ }, ++ USEIF: 'jset' ++ } ++ ] ++ + def __init__( + self, + usage: str, +@@ -332,6 +474,17 @@ class CylcOptionParser(OptionParser): + formatter=CylcHelpFormatter() + ) + ++ def get_std_options(self): ++ """Get a data-structure of standard options""" ++ opts = [] ++ for opt in self.STD_OPTIONS: ++ if ( ++ opt[USEIF] == ALL ++ or hasattr(self, opt[USEIF]) and getattr(self, opt[USEIF]) ++ ): ++ opts.append(opt) ++ return opts ++ + def add_std_option(self, *args, **kwargs): + """Add a standard option, ignoring override.""" + with suppress(OptionConflictError): +@@ -339,121 +492,63 @@ class CylcOptionParser(OptionParser): + + def add_std_options(self): + """Add standard options if they have not been overridden.""" +- self.add_std_option( +- "-q", "--quiet", +- help="Decrease verbosity.", +- action='decrement', +- dest='verbosity', +- ) +- self.add_std_option( +- "-v", "--verbose", +- help="Increase verbosity.", +- dest='verbosity', +- action='count', +- default=env_to_verbosity(os.environ) +- ) +- self.add_std_option( +- "--debug", +- help="Equivalent to -v -v", +- dest="verbosity", +- action='store_const', +- const=2 +- ) +- self.add_std_option( +- "--no-timestamp", +- help="Don't timestamp logged messages.", +- action="store_false", dest="log_timestamp", default=True) +- +- if self.color: +- self.add_std_option( +- '--color', '--colour', metavar='WHEN', action='store', +- default='auto', choices=['never', 'auto', 'always'], +- help=( +- "When to use color/bold text in terminal output." +- " Options are 'never', 'auto' and 'always'." +- ) +- ) +- +- if self.comms: +- self.add_std_option( +- "--comms-timeout", metavar='SEC', +- help=( +- "Set a timeout for network connections " +- "to the running workflow. The default is no timeout. " +- "For task messaging connections see " +- "site/user config file documentation." +- ), +- action="store", default=None, dest="comms_timeout") +- +- if self.jset: +- self.add_std_option( +- "-s", "--set", metavar="NAME=VALUE", +- help=( +- "Set the value of a Jinja2 template variable in the" +- " workflow definition." +- " Values should be valid Python literals so strings" +- " must be quoted" +- " e.g. 'STR=\"string\"', INT=43, BOOL=True." +- " This option can be used multiple " +- " times on the command line." +- " NOTE: these settings persist across workflow restarts," +- " but can be set again on the \"cylc play\"" +- " command line if they need to be overridden." +- ), +- action="append", default=[], dest="templatevars") +- +- self.add_std_option( +- "--set-file", metavar="FILE", +- help=( +- "Set the value of Jinja2 template variables in the " +- "workflow definition from a file containing NAME=VALUE " +- "pairs (one per line). " +- "As with --set values should be valid Python literals " +- "so strings must be quoted e.g. STR='string'. " +- "NOTE: these settings persist across workflow restarts, " +- "but can be set again on the \"cylc play\" " +- "command line if they need to be overridden." +- ), +- action="store", default=None, dest="templatevars_file") ++ for option in self.get_std_options(): ++ if not any([self.has_option(i) for i in option[ARGS]]): ++ self.add_option(*option[ARGS], **option[KWARGS]) + +- def add_cylc_rose_options(self) -> None: +- """Add extra options for cylc-rose plugin if it is installed.""" ++ @staticmethod ++ def get_cylc_rose_options(): ++ """Returns a list of option dictionaries if Cylc Rose exists.""" + try: + __import__('cylc.rose') + except ImportError: +- return +- self.add_option( +- "--opt-conf-key", "-O", +- help=( +- "Use optional Rose Config Setting " +- "(If Cylc-Rose is installed)" +- ), +- action="append", +- default=[], +- dest="opt_conf_keys" +- ) +- self.add_option( +- "--define", '-D', +- help=( +- "Each of these overrides the `[SECTION]KEY` setting in a " +- "`rose-suite.conf` file. " +- "Can be used to disable a setting using the syntax " +- "`--define=[SECTION]!KEY` or even `--define=[!SECTION]`." +- ), +- action="append", +- default=[], +- dest="defines" +- ) +- self.add_option( +- "--rose-template-variable", '-S', '--define-suite', +- help=( +- "As `--define`, but with an implicit `[SECTION]` for " +- "workflow variables." +- ), +- action="append", +- default=[], +- dest="rose_template_vars" +- ) ++ return [] ++ return [ ++ { ++ ARGS: ["--opt-conf-key", "-O"], ++ KWARGS: { ++ HELP: ++ "Use optional Rose Config Setting " ++ "(If Cylc-Rose is installed)", ++ ACTION: "append", ++ DEFAULT: [], ++ DEST: "opt_conf_keys" ++ } ++ }, ++ { ++ ARGS: ["--define", '-D'], ++ KWARGS: { ++ HELP: ++ "Each of these overrides the `[SECTION]KEY` setting" ++ "in a `rose-suite.conf` file. " ++ "Can be used to disable a setting using the syntax " ++ "`--define=[SECTION]!KEY` or " ++ "even `--define=[!SECTION]`.", ++ ACTION: "append", ++ DEFAULT: [], ++ DEST: "defines" ++ } ++ }, ++ { ++ ARGS: ["--rose-template-variable", '-S', '--define-suite'], ++ KWARGS: { ++ HELP: ++ "As `--define`, but with an implicit `[SECTION]` for " ++ "workflow variables.", ++ ACTION: "append", ++ DEFAULT: [], ++ DEST: "rose_template_vars" ++ } ++ } ++ ] ++ ++ def add_cylc_rose_options(self) -> None: ++ """Add extra options for cylc-rose plugin if it is installed. ++ ++ Now a vestigal interface for get_cylc_rose_options. ++ """ ++ for option in self.get_cylc_rose_options(): ++ self.add_option(*option[ARGS], **option[KWARGS]) + + def parse_args(self, api_args, remove_opts=None): + """Parse options and arguments, overrides OptionParser.parse_args. +@@ -583,3 +678,108 @@ class Options: + setattr(opts, key, value) + + return opts ++ ++ ++def combine_options_pair(first_list, second_list): ++ """Combine two option lists. ++ ++ Scenarios: ++ - Arguments are identical - return this argument. ++ - Arguments are not identical but have some common label strings, ++ i.e. both arguments can be invoked using `-f`. ++ - If there are non-shared label strings strip the shared ones. ++ - Otherwise raise an error. ++ E.g: If `command-A` has an option `-f` or `--file` and ++ `command-B has an option `-f` or `--fortran`` then ++ `command-A+B` will have options `--fortran` and `--file` but _not_ ++ `-f`, which would be confusing. ++ ++ TODO: Record which list each option came from, so that you can tell ++ the user which component script is being referred to by each option. ++ """ ++ def appendif(list_, item): ++ """Avoid duplicating items in output list""" ++ if item not in list_: ++ list_.append(item) ++ return list_ ++ ++ output = [] ++ ++ for first, second in product(first_list, second_list): ++ if first == second and first not in output: ++ output = appendif(output, first) ++ ++ # If any of the argument name identical: ++ elif ( ++ any([i[0] == i[1] for i in product(second[ARGS], first[ARGS])]) ++ and first != second ++ ): ++ # if any of the args are different: ++ if first[ARGS] != second[ARGS]: ++ first_args = list(set(first[ARGS]) - set(second[ARGS])) ++ second[ARGS] = list(set(second[ARGS]) - set(first[ARGS])) ++ first[ARGS] = first_args ++ output = appendif(output, first) ++ output = appendif(output, second) ++ else: ++ # if none of the arg names are different. ++ raise Exception(f'Clashing Options \n{first}\n{second}') ++ else: ++ if all([first[ARGS] != i[ARGS] for i in second_list]): ++ output = appendif(output, first) ++ if all([second[ARGS] != i[ARGS] for i in first_list]): ++ output = appendif(output, second) ++ return output ++ ++ ++def combine_options(*args): ++ """Combine a list of argument dicts. ++ ++ Ordering should be irrelevant because combine_options_pair should ++ be commutative, and the overall order of args is not relevant. ++ """ ++ list_ = list(args) ++ output = list_[0] ++ for arg in list_[1:]: ++ if arg and output: ++ output = combine_options_pair(arg, output) ++ elif arg: ++ output = arg ++ return output ++ ++ ++def cleanup_sysargv( ++ script_name, ++ options, ++ compound_script_opts, ++ script_opts ++): ++ """Remove unwanted options from sys.argv ++ ++ Some cylc scripts (notably Cylc Play when it is re-invoked on a scheduler ++ server) require the correct content in sys.argv. ++ """ ++ # Organize Options by dest. ++ script_opts_by_dest = { ++ x[KWARGS].get('dest', x[ARGS][0].strip(DOUBLEDASH)): x ++ for x in script_opts ++ } ++ compound_opts_by_dest = { ++ x[KWARGS].get('dest', x[ARGS][0].strip(DOUBLEDASH)): x ++ for x in compound_script_opts ++ } ++ ++ # Filter out non-cylc-play options. ++ for unwanted_opt in (set(options.__dict__)) - set(script_opts_by_dest): ++ for arg in compound_opts_by_dest[unwanted_opt][ARGS]: ++ if arg in sys.argv: ++ index = sys.argv.index(arg) ++ sys.argv.pop(index) ++ if ( ++ compound_opts_by_dest[unwanted_opt][KWARGS][ACTION] ++ not in ['store_true', 'store_false'] ++ ): ++ sys.argv.pop(index + 1) ++ ++ # replace compound script name: ++ sys.argv[1] = script_name +diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py +index 5091a1a14..ee29c7265 100644 +--- a/cylc/flow/scheduler_cli.py ++++ b/cylc/flow/scheduler_cli.py +@@ -38,7 +38,8 @@ from cylc.flow.option_parsers import ( + WORKFLOW_ID_ARG_DOC, + CylcOptionParser as COP, + Options, +- icp_option, ++ ICP_OPTION, ++ ARGS, KWARGS, HELP, ACTION, DEFAULT, DEST, METAVAR, CHOICES + ) + from cylc.flow.pathutil import get_workflow_run_scheduler_log_path + from cylc.flow.remote import _remote_cylc_cmd +@@ -107,6 +108,167 @@ mutation ( + } + ''' + ++PLAY_OPTIONS = [ ++ { ++ ARGS: ["-n", "--no-detach", "--non-daemon"], ++ KWARGS: { ++ HELP: "Do not daemonize the scheduler (infers --format=plain)", ++ ACTION: "store_true", ++ DEST: "no_detach", ++ } ++ }, ++ { ++ ARGS: ["--profile"], ++ KWARGS: { ++ HELP: "Output profiling (performance) information", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "profile_mode" ++ } ++ }, ++ { ++ ARGS: ["--start-cycle-point", "--startcp"], ++ KWARGS: { ++ HELP: ++ "Set the start cycle point, which may be after " ++ "the initial cycle point. If the specified start point is " ++ "not in the sequence, the next on-sequence point will " ++ "be used. (Not to be confused with the initial cycle point)", ++ METAVAR: "CYCLE_POINT", ++ ACTION: "store", ++ DEST: "startcp", ++ } ++ }, ++ { ++ ARGS: ["--final-cycle-point", "--fcp"], ++ KWARGS: { ++ HELP: ++ "Set the final cycle point. This command line option overrides" ++ "the workflow config option '[scheduling]final cycle point'. ", ++ METAVAR: "CYCLE_POINT", ++ ACTION: "store", ++ DEST: "fcp", ++ } ++ }, ++ { ++ ARGS: ["--stop-cycle-point", "--stopcp"], ++ KWARGS: { ++ HELP: ++ "Set the stop cycle point. Shut down after all " ++ "have PASSED this cycle point. (Not to be confused " ++ "the final cycle point.) This command line option overrides" ++ "the workflow config option '[scheduling]stop after cycle point'. ", ++ METAVAR: "CYCLE_POINT", ++ ACTION: "store", ++ DEST: "stopcp", ++ } ++ }, ++ { ++ ARGS: ["--start-task", "--starttask", "-t"], ++ KWARGS: { ++ HELP: ++ "Start from this task instance, given by '/'. " ++ "This can be used multiple times to start from multiple " ++ "tasks at once. Dependence on tasks with cycle points earlier" ++ "than the earliest start-task will be ignored. A" ++ "sub-graph of the workflow will run if selected tasks do not lead on to the full graph.", ++ METAVAR: "TASK_ID", ++ ACTION: "append", ++ DEST: "starttask", ++ } ++ }, ++ { ++ ARGS: ["--pause"], ++ KWARGS: { ++ HELP: "Pause the workflow immediately on start up.", ++ ACTION: "store_true", ++ DEST: "paused_start", ++ } ++ }, ++ { ++ ARGS: ["--hold-after", "--hold-cycle-point", "--holdcp"], ++ KWARGS: { ++ HELP: "Hold all tasks after this cycle point.", ++ METAVAR: "CYCLE_POINT", ++ ACTION: "store", ++ DEST: "holdcp", ++ } ++ }, ++ { ++ ARGS: ["-m", "--mode"], ++ KWARGS: { ++ HELP: "Run mode: live, dummy, simulation (default live).", ++ METAVAR: "STRING", ++ ACTION: "store", ++ DEST: "run_mode", ++ CHOICES: ['live', 'dummy', 'simulation'], ++ } ++ }, ++ { ++ ARGS: ["--reference-log"], ++ KWARGS: { ++ HELP: "Generate a reference log for use in reference ", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "genref", ++ } ++ }, ++ { ++ ARGS: ["--reference-test"], ++ KWARGS: { ++ HELP: ++ "Do a test run against a previously generated reference ", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "reftest", ++ } ++ }, ++ { ++ ARGS: ["--host"], ++ KWARGS: { ++ HELP: ++ "Specify the host on which to start-up the workflow. " ++ "not specified, a host will be selected using the ", ++ METAVAR: "HOST", ++ ACTION: "store", ++ DEST: "host", ++ } ++ }, ++ { ++ ARGS: ["--format"], ++ KWARGS: { ++ HELP: ++ "The format of the output: 'plain'=human readable, ", ++ CHOICES: ('plain', 'json'), ++ DEFAULT: "plain", ++ DEST: 'format' ++ } ++ }, ++ { ++ ARGS: ["--main-loop"], ++ KWARGS: { ++ HELP: ++ "Specify an additional plugin to run in the main " ++ "These are used in combination with those specified " ++ "[scheduler][main loop]plugins. Can be used multiple times.", ++ METAVAR: "PLUGIN_NAME", ++ ACTION: "append", ++ DEST: "main_loop", ++ } ++ }, ++ { ++ ARGS: ["--abort-if-any-task-fails"], ++ KWARGS: { ++ HELP: ++ "If set workflow will abort with status 1 if any task fails.", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "abort_if_any_task_fails", ++ } ++ }, ++ ICP_OPTION ++] ++ + + @lru_cache() + def get_option_parser(add_std_opts: bool = False) -> COP: +@@ -118,118 +280,9 @@ def get_option_parser(add_std_opts: bool = False) -> COP: + argdoc=[WORKFLOW_ID_ARG_DOC] + ) + +- parser.add_option( +- "-n", "--no-detach", "--non-daemon", +- help="Do not daemonize the scheduler (infers --format=plain)", +- action="store_true", dest="no_detach") +- +- parser.add_option( +- "--profile", help="Output profiling (performance) information", +- action="store_true", dest="profile_mode") +- +- parser.add_option(icp_option) +- +- parser.add_option( +- "--start-cycle-point", "--startcp", +- help=( +- "Set the start cycle point, which may be after the initial cycle " +- "point. If the specified start point is not in the sequence, the " +- "next on-sequence point will be used. " +- "(Not to be confused with the initial cycle point.) " +- "This replaces the Cylc 7 --warm option." +- ), +- metavar="CYCLE_POINT", action="store", dest="startcp") +- +- parser.add_option( +- "--final-cycle-point", "--fcp", +- help=( +- "Set the final cycle point. " +- "This command line option overrides the workflow " +- "config option '[scheduling]final cycle point'. " +- ), +- metavar="CYCLE_POINT", action="store", dest="fcp") +- +- parser.add_option( +- "--stop-cycle-point", "--stopcp", +- help=( +- "Set the stop cycle point. " +- "Shut down after all tasks have PASSED this cycle point. " +- "(Not to be confused with the final cycle point.) " +- "This command line option overrides the workflow " +- "config option '[scheduling]stop after cycle point'. " +- ), +- metavar="CYCLE_POINT", action="store", dest="stopcp") +- +- parser.add_option( +- "--start-task", "--starttask", "-t", +- help=( +- "Start from this task instance, given by '/'. " +- "Can be used multiple times " +- "to start from multiple tasks at once. Dependence on tasks with " +- "with cycle points earlier than the earliest start-task will be " +- "ignored. A sub-graph of the workflow will run if selected tasks " +- "do not lead on to the full graph." +- ), +- metavar="TASK_ID", action="append", dest="starttask") +- +- parser.add_option( +- "--pause", +- help="Pause the workflow immediately on start up.", +- action="store_true", dest="paused_start") +- +- parser.add_option( +- "--hold-after", "--hold-cycle-point", "--holdcp", +- help="Hold all tasks after this cycle point.", +- metavar="CYCLE_POINT", action="store", dest="holdcp") +- +- parser.add_option( +- "-m", "--mode", +- help="Run mode: live, dummy, simulation (default live).", +- metavar="STRING", action="store", dest="run_mode", +- choices=["live", "dummy", "simulation"]) +- +- parser.add_option( +- "--reference-log", +- help="Generate a reference log for use in reference tests.", +- action="store_true", default=False, dest="genref") +- +- parser.add_option( +- "--reference-test", +- help="Do a test run against a previously generated reference log.", +- action="store_true", default=False, dest="reftest") +- +- # Override standard parser option for specific help description. +- parser.add_option( +- "--host", +- help=( +- "Specify the host on which to start-up the workflow. " +- "If not specified, a host will be selected using " +- "the '[scheduler]run hosts' global config." +- ), +- metavar="HOST", action="store", dest="host") +- +- parser.add_option( +- "--format", +- help="The format of the output: 'plain'=human readable, 'json'", +- choices=("plain", "json"), +- default="plain" +- ) +- +- parser.add_option( +- "--main-loop", +- help=( +- "Specify an additional plugin to run in the main loop." +- " These are used in combination with those specified in" +- " [scheduler][main loop]plugins. Can be used multiple times" +- ), +- metavar="PLUGIN_NAME", action="append", dest="main_loop" +- ) +- +- parser.add_option( +- "--abort-if-any-task-fails", +- help="If set workflow will abort with status 1 if any task fails.", +- action="store_true", default=False, dest="abort_if_any_task_fails" +- ) ++ options = parser.get_cylc_rose_options() + PLAY_OPTIONS ++ for option in options: ++ parser.add_option(*option[ARGS], **option[KWARGS]) + + if add_std_opts: + # This is for the API wrapper for integration tests. Otherwise (CLI +@@ -427,6 +480,11 @@ async def _run(scheduler: Scheduler) -> int: + @cli_function(get_option_parser) + def play(parser: COP, options: 'Values', id_: str): + """Implement cylc play.""" ++ return _play(parser, options, id_) ++ ++ ++def _play(parser: COP, options: 'Values', id_: str): ++ """Allows compound scripts to import play, but supply their own COP.""" + if options.starttask: + options.starttask = upgrade_legacy_ids( + *options.starttask, +diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py +index 761bc7dec..07ef7935b 100755 +--- a/cylc/flow/scripts/install.py ++++ b/cylc/flow/scripts/install.py +@@ -85,7 +85,10 @@ from typing import Optional, TYPE_CHECKING, Dict, Any + + from cylc.flow import iter_entry_points + from cylc.flow.exceptions import PluginError, InputError +-from cylc.flow.option_parsers import CylcOptionParser as COP ++from cylc.flow.option_parsers import ( ++ CylcOptionParser as COP, ++ ARGS, KWARGS, HELP, ACTION, DEFAULT, DEST, METAVAR ++) + from cylc.flow.pathutil import EXPLICIT_RELATIVE_PATH_REGEX, expand_path + from cylc.flow.workflow_files import ( + install_workflow, search_install_source_dirs, parse_cli_sym_dirs +@@ -96,6 +99,58 @@ if TYPE_CHECKING: + from optparse import Values + + ++INSTALL_OPTIONS = [ ++ { ++ ARGS: ["--workflow-name", "-n"], ++ KWARGS: { ++ HELP: "Install into ~/cylc-run//runN ", ++ ACTION: "store", ++ METAVAR: "WORKFLOW_NAME", ++ DEFAULT: None, ++ DEST: "workflow_name" ++ } ++ }, ++ { ++ ARGS: ["--symlink-dirs"], ++ KWARGS: { ++ HELP: ++ "Enter a comma-delimited list, in the form " ++ "'log=path/to/store, share = $HOME/some/path, ...'. " ++ "Use this option to override the global.cylc configuration " ++ "for local symlinks for the run, log, work, share and " ++ "share/cycle directories. Enter " ++ "an empty list '' to skip making localhost symlink dirs.", ++ ACTION: "store", ++ DEST: "symlink_dirs" ++ } ++ }, ++ { ++ ARGS: ["--run-name"], ++ KWARGS: { ++ HELP: ++ "Give the run a custom name instead of automatically " ++ "numbering it.", ++ ACTION: "store", ++ METAVAR: "RUN_NAME", ++ DEFAULT: None, ++ DEST: "run_name" ++ } ++ }, ++ { ++ ARGS: ["--no-run-name"], ++ KWARGS: { ++ HELP: ++ "Install the workflow directly into " ++ "~/cylc-run/, " ++ "without an automatic run number or custom run name.", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "no_run_name" ++ } ++ } ++] ++ ++ + def get_option_parser() -> COP: + parser = COP( + __doc__, +@@ -108,49 +163,10 @@ def get_option_parser() -> COP: + ] + ) + +- parser.add_option( +- "--workflow-name", "-n", +- help="Install into ~/cylc-run//runN ", +- action="store", +- metavar="WORKFLOW_NAME", +- default=None, +- dest="workflow_name") +- +- parser.add_option( +- "--symlink-dirs", +- help=( +- "Enter a comma-delimited list, in the form " +- "'log=path/to/store, share = $HOME/some/path, ...'. " +- "Use this option to override the global.cylc configuration for " +- "local symlinks for the run, log, work, share and " +- "share/cycle directories. " +- "Enter an empty list '' to skip making localhost symlink dirs." +- ), +- action="store", +- dest="symlink_dirs" +- ) ++ options = parser.get_cylc_rose_options() + INSTALL_OPTIONS + +- parser.add_option( +- "--run-name", +- help=( +- "Give the run a custom name instead of automatically numbering it." +- ), +- action="store", +- metavar="RUN_NAME", +- default=None, +- dest="run_name") +- +- parser.add_option( +- "--no-run-name", +- help=( +- "Install the workflow directly into ~/cylc-run/, " +- "without an automatic run number or custom run name." +- ), +- action="store_true", +- default=False, +- dest="no_run_name") +- +- parser.add_cylc_rose_options() ++ for option in options: ++ parser.add_option(*option[ARGS], **option[KWARGS]) + + return parser + +@@ -178,7 +194,7 @@ def main(parser, opts, reg=None): + + def install( + parser: COP, opts: 'Values', reg: Optional[str] = None +-) -> None: ++) -> str: + if opts.no_run_name and opts.run_name: + raise InputError( + "options --no-run-name and --run-name are mutually exclusive." +@@ -229,3 +245,4 @@ def install( + entry_point.name, + exc + ) from None ++ return _workflow_name +diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py +index 6e5031c91..47e5f097c 100755 +--- a/cylc/flow/scripts/validate.py ++++ b/cylc/flow/scripts/validate.py +@@ -43,13 +43,59 @@ from cylc.flow.option_parsers import ( + WORKFLOW_ID_OR_PATH_ARG_DOC, + CylcOptionParser as COP, + Options, +- icp_option, ++ ICP_OPTION, ++ ARGS, KWARGS, HELP, ACTION, DEFAULT, DEST, METAVAR, CHOICES + ) + from cylc.flow.profiler import Profiler + from cylc.flow.task_proxy import TaskProxy + from cylc.flow.templatevars import get_template_vars + from cylc.flow.terminal import cli_function + ++VALIDATE_OPTIONS = [ ++ { ++ ARGS: ["--check-circular"], ++ KWARGS: { ++ HELP: ++ "Check for circular dependencies in graphs when the number of " ++ "tasks is greater than 100 (smaller graphs are always " ++ "checked). This can be slow when the number of " ++ "tasks is high.", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "check_circular" ++ } ++ }, ++ { ++ ARGS: ["--output", "-o"], ++ KWARGS: { ++ HELP: "Specify a file name to dump the processed flow.cylc.", ++ METAVAR: "FILENAME", ++ ACTION: "store", ++ DEST: "output" ++ } ++ }, ++ { ++ ARGS: ["--profile"], ++ KWARGS: { ++ HELP: "Output profiling (performance) information", ++ ACTION: "store_true", ++ DEFAULT: False, ++ DEST: "profile_mode" ++ }, ++ }, ++ { ++ ARGS: ["-u", "--run-mode"], ++ KWARGS: { ++ HELP: "Validate for run mode.", ++ ACTION: "store", ++ DEFAULT: "live", ++ DEST: "run_mode", ++ CHOICES: ["live", "dummy", "simulation"] ++ } ++ }, ++ ICP_OPTION, ++] ++ + + def get_option_parser(): + parser = COP( +@@ -58,31 +104,10 @@ def get_option_parser(): + argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC], + ) + +- parser.add_option( +- "--check-circular", +- help="Check for circular dependencies in graphs when the number of " +- "tasks is greater than 100 (smaller graphs are always checked). " +- "This can be slow when the number of " +- "tasks is high.", +- action="store_true", default=False, dest="check_circular") +- +- parser.add_option( +- "--output", "-o", +- help="Specify a file name to dump the processed flow.cylc.", +- metavar="FILENAME", action="store", dest="output") ++ validate_options = parser.get_cylc_rose_options() + VALIDATE_OPTIONS + +- parser.add_option( +- "--profile", help="Output profiling (performance) information", +- action="store_true", default=False, dest="profile_mode") +- +- parser.add_option( +- "-u", "--run-mode", help="Validate for run mode.", action="store", +- default="live", dest="run_mode", +- choices=['live', 'dummy', 'simulation']) +- +- parser.add_option(icp_option) +- +- parser.add_cylc_rose_options() ++ for option in validate_options: ++ parser.add_option(*option[ARGS], **option[KWARGS]) + + parser.set_defaults(is_validate=True) + +@@ -102,6 +127,10 @@ ValidateOptions = Options( + + @cli_function(get_option_parser) + def main(parser: COP, options: 'Values', workflow_id: str) -> None: ++ wrapped_main(parser, options, workflow_id) ++ ++ ++def wrapped_main(parser: COP, options: 'Values', workflow_id: str) -> None: + """cylc validate CLI.""" + profiler = Profiler(None, options.profile_mode) + profiler.start() +diff --git a/cylc/flow/scripts/validate_install_play.py b/cylc/flow/scripts/validate_install_play.py +new file mode 100644 +index 000000000..bad896aa7 +--- /dev/null ++++ b/cylc/flow/scripts/validate_install_play.py +@@ -0,0 +1,119 @@ ++#!/usr/bin/env python3 ++ ++# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. ++# Copyright (C) NIWA & British Crown (Met Office) & Contributors. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation, either version 3 of the License, or ++# (at your option) any later version. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program. If not, see . ++ ++"""cylc validate-install-play [OPTIONS] ARGS ++ ++ ++- Should validate's arg be WORKFLOW|PATH as described or WORKFLOW|PATH|SOURCE? ++- cylc validate --output-filename probably isn't very useful to VIP and should ++ probably be forced unset. ++- cylc install -n (workflow name) clashed with cylc play -n (no-detach) - ++ proposal -n == --workflow-name because users are more likely to want that. ++ Consider logging a warning bc it's still ambiguous. ++- Possibe DANGER: cylc-rose options for cylc install (-S -O -D) could be ++ over-run at play by --set and --set-file, This can happen anyway, ++ but there's potential for it to be obscured by the VIP command, or even, ++ confusingly, fail validating after install which it passed before. ++""" ++ ++from ansimarkup import parse as cparse ++ ++from cylc.flow import LOG ++from cylc.flow.scripts.validate import ( ++ VALIDATE_OPTIONS, ++ wrapped_main as validate_main ++) ++from cylc.flow.scripts.install import ( ++ INSTALL_OPTIONS, install, get_source_location ++) ++from cylc.flow.scheduler_cli import PLAY_OPTIONS ++from cylc.flow.option_parsers import ( ++ CylcOptionParser as COP, ++ combine_options, ++ cleanup_sysargv, ++ ARGS, KWARGS ++) ++from cylc.flow.scheduler_cli import _play ++from cylc.flow.terminal import cli_function ++ ++from typing import TYPE_CHECKING ++ ++ ++if TYPE_CHECKING: ++ from optparse import Values ++ ++ ++CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options() ++VIP_OPTIONS = combine_options( ++ VALIDATE_OPTIONS, INSTALL_OPTIONS, ++ PLAY_OPTIONS, CYLC_ROSE_OPTIONS ++) ++ ++ ++def log_subcommand(command, ident): ++ """Log a command run as part of a sequence. ++ ++ TODO: When implementing other Compound commands, ++ move this somewhere common. ++ """ ++ print(cparse( ++ f'RUNNING: $ cylc {command} {ident}')) ++ ++ ++def get_option_parser() -> COP: ++ parser = COP( ++ __doc__, ++ comms=True, ++ jset=True, ++ argdoc=[ ++ COP.optional(( ++ 'SOURCE_NAME | PATH', ++ 'Workflow source name or path to source directory' ++ )) ++ ] ++ ) ++ for option in VIP_OPTIONS: ++ parser.add_option(*option[ARGS], **option[KWARGS]) ++ return parser ++ ++ ++@cli_function(get_option_parser) ++def main(parser: COP, options: 'Values', workflow_id: str): ++ """Run Cylc validate - install - play in sequence.""" ++ source = get_source_location(workflow_id) ++ ++ LOG.info('Running Cylc Validate && Cylc Install && Cylc Play') ++ ++ log_subcommand('validate', source) ++ validate_main(parser, options, str(source)) ++ ++ log_subcommand('install', workflow_id) ++ workflow_name = install(parser, options, workflow_id) ++ ++ cleanup_sysargv( ++ 'play', ++ options, ++ compound_script_opts=VIP_OPTIONS, ++ script_opts=( ++ PLAY_OPTIONS + CYLC_ROSE_OPTIONS ++ + parser.get_std_options() ++ ), ++ ) ++ ++ log_subcommand('play', workflow_id) ++ _play(parser, options, workflow_name) +diff --git a/setup.cfg b/setup.cfg +index b07ab0673..4e9ef5790 100644 +--- a/setup.cfg ++++ b/setup.cfg +@@ -196,6 +196,7 @@ cylc.command = + trigger = cylc.flow.scripts.trigger:main + validate = cylc.flow.scripts.validate:main + view = cylc.flow.scripts.view:main ++ vip = cylc.flow.scripts.validate_install_play:main + # async functions to run within the scheduler main loop + cylc.main_loop = + health_check = cylc.flow.main_loop.health_check +diff --git a/tests/functional/cylc-combination-scripts/00-vip.t b/tests/functional/cylc-combination-scripts/00-vip.t +new file mode 100644 +index 000000000..eb7b331c9 +--- /dev/null ++++ b/tests/functional/cylc-combination-scripts/00-vip.t +@@ -0,0 +1,38 @@ ++#!/usr/bin/env bash ++# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. ++# Copyright (C) NIWA & British Crown (Met Office) & Contributors. ++# ++# This program is free software: you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation, either version 3 of the License, or ++# (at your option) any later version. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program. If not, see . ++ ++#------------------------------------------------------------------------------ ++# Test `cylc vip` (Validate Install Play) ++ ++. "$(dirname "$0")/test_header" ++set_test_number 4 ++ ++WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" ++ ++run_ok "${TEST_NAME_BASE}-from-path" \ ++ cylc vip --no-detach --debug \ ++ --workflow-name "${WORKFLOW_NAME}" \ ++ "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" ++ ++grep "RUNNING" "${TEST_NAME_BASE}-from-path.stdout" > VIPOUT.txt ++ ++named_grep_ok "${TEST_NAME_BASE}-it-validated" "cylc validate" "VIPOUT.txt" ++named_grep_ok "${TEST_NAME_BASE}-it-installed" "cylc install" "VIPOUT.txt" ++named_grep_ok "${TEST_NAME_BASE}-it-played" "cylc play" "VIPOUT.txt" ++ ++purge ++exit 0 +diff --git a/tests/functional/cylc-combination-scripts/00-vip/flow.cylc b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc +new file mode 100644 +index 000000000..745bf26cb +--- /dev/null ++++ b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc +@@ -0,0 +1,7 @@ ++[scheduler] ++ allow implicit tasks = True ++ ++[scheduling] ++ initial cycle point = 1500 ++ [[graph]] ++ R1 = foo +diff --git a/tests/functional/cylc-combination-scripts/test_header b/tests/functional/cylc-combination-scripts/test_header +new file mode 120000 +index 000000000..90bd5a36f +--- /dev/null ++++ b/tests/functional/cylc-combination-scripts/test_header +@@ -0,0 +1 @@ ++../lib/bash/test_header +\ No newline at end of file +diff --git a/tests/unit/test_option_parsers.py b/tests/unit/test_option_parsers.py +index dcd2f6e89..f1b5433ec 100644 +--- a/tests/unit/test_option_parsers.py ++++ b/tests/unit/test_option_parsers.py +@@ -14,14 +14,19 @@ + # You should have received a copy of the GNU General Public License + # along with this program. If not, see . + ++from contextlib import redirect_stdout ++import io + import pytest ++from pytest import param ++import sys ++from types import SimpleNamespace + from typing import List + +-import sys +-import io +-from contextlib import redirect_stdout + import cylc.flow.flags +-from cylc.flow.option_parsers import CylcOptionParser as COP, Options ++from cylc.flow.option_parsers import ( ++ CylcOptionParser as COP, Options, combine_options, combine_options_pair, ++ ARGS, KWARGS, cleanup_sysargv ++) + + + USAGE_WITH_COMMENT = "usage \n # comment" +@@ -93,3 +98,186 @@ def test_Options_std_opts(): + MyOptions = Options(parser) + MyValues = MyOptions(verbosity=1) + assert MyValues.verbosity == 1 ++ ++ ++# Add overlapping args tomorrow ++@pytest.mark.parametrize( ++ 'first, second, expect', ++ [ ++ param( ++ [{ARGS: ['-f', '--foo']}], ++ [{ARGS: ['-f', '--foo']}], ++ [{ARGS: ['-f', '--foo']}], ++ id='identical arg lists unchanged' ++ ), ++ param( ++ [{ARGS: ['-f', '--foo']}], ++ [{ARGS: ['-f', '--foolish'], KWARGS: {'help': 'not identical'}}], ++ [ ++ {ARGS: ['--foo']}, ++ {ARGS: ['--foolish'], KWARGS: {'help': 'not identical'}} ++ ], ++ id='different arg lists lose shared names' ++ ), ++ param( ++ [{ARGS: ['-f', '--foo']}], ++ [{ARGS: ['-f', '--foo'], KWARGS: {'help': 'not identical'}}], ++ None, ++ id='different args identical arg list cause exception' ++ ), ++ param( ++ [{ARGS: ['-g', '--goo']}], ++ [{ARGS: ['-f', '--foo']}], ++ [ ++ {ARGS: ['-g', '--goo']}, ++ {ARGS: ['-f', '--foo']}, ++ ], ++ id='all unrelated args added' ++ ), ++ param( ++ [{ARGS: ['-f', '--foo']}, {ARGS: ['-r', '--redesdale']}], ++ [{ARGS: ['-f', '--foo']}, {ARGS: ['-b', '--buttered-peas']}], ++ [ ++ {ARGS: ['-f', '--foo']}, ++ {ARGS: ['-r', '--redesdale']}, ++ {ARGS: ['-b', '--buttered-peas']} ++ ], ++ id='do not repeat args' ++ ) ++ ] ++) ++def test_combine_options_pair(first, second, expect): ++ """It combines sets of options""" ++ if expect is not None: ++ result = combine_options_pair(first, second) ++ ++ # Order of args irrelevent to test ++ expect = sorted(expect, key=lambda x: x[ARGS]) ++ result = sorted(result, key=lambda x: x[ARGS]) ++ ++ assert result == expect ++ else: ++ with pytest.raises(Exception, match='Clashing Options'): ++ combine_options_pair(first, second) ++ ++ ++@pytest.mark.parametrize( ++ 'inputs, expect', ++ [ ++ param( ++ [ ++ [{ARGS: ['-i', '--inflammable']}], ++ [{ARGS: ['-f', '--flammable']}], ++ [{ARGS: ['-n', '--non-flammable']}] ++ ], ++ [ ++ {ARGS: ['-i', '--inflammable']}, ++ {ARGS: ['-f', '--flammable']}, ++ {ARGS: ['-n', '--non-flammable']} ++ ], ++ id='merge three argsets no overlap' ++ ), ++ param( ++ [ ++ [{ARGS: ['-m', '--morpeth']}, {ARGS: ['-r', '--redesdale']}], ++ [{ARGS: ['-b', '--byker']}, {ARGS: ['-r', '--roxborough']}], ++ [{ARGS: ['-b', '--bellingham']}] ++ ++ ], ++ [ ++ {ARGS: ['--bellingham']}, ++ {ARGS: ['--roxborough']}, ++ {ARGS: ['--redesdale']}, ++ {ARGS: ['--byker']}, ++ {ARGS: ['-m', '--morpeth']} ++ ], ++ id='merge three overlapping argsets' ++ ), ++ param( ++ [ ++ [], ++ [{ARGS: ['-c', '--campden']}] ++ ], ++ [ ++ {ARGS: ['-c', '--campden']} ++ ], ++ id="empty list doesn't clear result" ++ ) ++ ] ++) ++def test_combine_options(inputs, expect): ++ """It combines multiple input sets""" ++ result = combine_options(*inputs) ++ ++ # Order of args irrelevent to test ++ expect = sorted(expect, key=lambda x: x[ARGS]) ++ result = sorted(result, key=lambda x: x[ARGS]) ++ ++ assert result == expect ++ ++ ++@pytest.mark.parametrize( ++ 'argv_before, kwargs, expect', ++ [ ++ param( ++ 'vip myworkflow'.split(), ++ { ++ 'script_name': 'play', ++ 'compound_script_opts': [ ++ {ARGS: ['--foo', '-f'], KWARGS: {}}, ++ ], ++ 'script_opts': [{ ++ ARGS: ['--foo', '-f'], ++ KWARGS: {} ++ }] ++ }, ++ 'play myworkflow'.split(), ++ id='no opts to remove' ++ ), ++ param( ++ 'vip myworkflow'.split(), ++ { ++ 'script_name': 'play', ++ 'compound_script_opts': [ ++ {ARGS: ['--foo', '-f'], KWARGS: {}}, ++ {ARGS: ['--bar', '-b'], KWARGS: {}}, ++ {ARGS: ['--baz'], KWARGS: {}}, ++ ], ++ 'script_opts': [{ ++ ARGS: ['--foo', '-f'], ++ KWARGS: {} ++ }] ++ }, ++ 'play myworkflow'.split(), ++ id='remove some opts' ++ ), ++ param( ++ 'vip myworkflow'.split(), ++ { ++ 'script_name': 'play', ++ 'compound_script_opts': [ ++ {ARGS: ['--foo', '-f'], KWARGS: {}}, ++ {ARGS: ['--bar', '-b'], KWARGS: {}}, ++ {ARGS: ['--baz'], KWARGS: {}}, ++ ], ++ 'script_opts': [] ++ }, ++ 'play myworkflow'.split(), ++ id='no opts to keep' ++ ), ++ ] ++) ++def test_cleanup_sysargv(monkeypatch, argv_before, kwargs, expect): ++ """It replaces the contents of sysargv with Cylc Play argv items. ++ """ ++ # Fake up sys.argv: for this test. ++ dummy_cylc_path = ['/pathto/my/cylc/bin/cylc'] ++ monkeypatch.setattr(sys, 'argv', dummy_cylc_path + argv_before) ++ # Fake options too: ++ opts = SimpleNamespace() ++ opts.foo = kwargs['script_opts'] ++ kwargs.update({'options': opts}) ++ ++ # Test the script: ++ cleanup_sysargv(**kwargs) ++ assert sys.argv == dummy_cylc_path + expect diff --git a/setup.cfg b/setup.cfg index b07ab0673f6..4e9ef5790b0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -196,6 +196,7 @@ cylc.command = trigger = cylc.flow.scripts.trigger:main validate = cylc.flow.scripts.validate:main view = cylc.flow.scripts.view:main + vip = cylc.flow.scripts.validate_install_play:main # async functions to run within the scheduler main loop cylc.main_loop = health_check = cylc.flow.main_loop.health_check diff --git a/tests/functional/cylc-combination-scripts/00-vip.t b/tests/functional/cylc-combination-scripts/00-vip.t new file mode 100644 index 00000000000..eb7b331c963 --- /dev/null +++ b/tests/functional/cylc-combination-scripts/00-vip.t @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#------------------------------------------------------------------------------ +# Test `cylc vip` (Validate Install Play) + +. "$(dirname "$0")/test_header" +set_test_number 4 + +WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)" + +run_ok "${TEST_NAME_BASE}-from-path" \ + cylc vip --no-detach --debug \ + --workflow-name "${WORKFLOW_NAME}" \ + "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}" + +grep "RUNNING" "${TEST_NAME_BASE}-from-path.stdout" > VIPOUT.txt + +named_grep_ok "${TEST_NAME_BASE}-it-validated" "cylc validate" "VIPOUT.txt" +named_grep_ok "${TEST_NAME_BASE}-it-installed" "cylc install" "VIPOUT.txt" +named_grep_ok "${TEST_NAME_BASE}-it-played" "cylc play" "VIPOUT.txt" + +purge +exit 0 diff --git a/tests/functional/cylc-combination-scripts/00-vip/flow.cylc b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc new file mode 100644 index 00000000000..745bf26cb2f --- /dev/null +++ b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc @@ -0,0 +1,7 @@ +[scheduler] + allow implicit tasks = True + +[scheduling] + initial cycle point = 1500 + [[graph]] + R1 = foo diff --git a/tests/functional/cylc-combination-scripts/test_header b/tests/functional/cylc-combination-scripts/test_header new file mode 120000 index 00000000000..90bd5a36f92 --- /dev/null +++ b/tests/functional/cylc-combination-scripts/test_header @@ -0,0 +1 @@ +../lib/bash/test_header \ No newline at end of file diff --git a/tests/unit/test_option_parsers.py b/tests/unit/test_option_parsers.py index dcd2f6e8977..f1b5433ec82 100644 --- a/tests/unit/test_option_parsers.py +++ b/tests/unit/test_option_parsers.py @@ -14,14 +14,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from contextlib import redirect_stdout +import io import pytest +from pytest import param +import sys +from types import SimpleNamespace from typing import List -import sys -import io -from contextlib import redirect_stdout import cylc.flow.flags -from cylc.flow.option_parsers import CylcOptionParser as COP, Options +from cylc.flow.option_parsers import ( + CylcOptionParser as COP, Options, combine_options, combine_options_pair, + ARGS, KWARGS, cleanup_sysargv +) USAGE_WITH_COMMENT = "usage \n # comment" @@ -93,3 +98,186 @@ def test_Options_std_opts(): MyOptions = Options(parser) MyValues = MyOptions(verbosity=1) assert MyValues.verbosity == 1 + + +# Add overlapping args tomorrow +@pytest.mark.parametrize( + 'first, second, expect', + [ + param( + [{ARGS: ['-f', '--foo']}], + [{ARGS: ['-f', '--foo']}], + [{ARGS: ['-f', '--foo']}], + id='identical arg lists unchanged' + ), + param( + [{ARGS: ['-f', '--foo']}], + [{ARGS: ['-f', '--foolish'], KWARGS: {'help': 'not identical'}}], + [ + {ARGS: ['--foo']}, + {ARGS: ['--foolish'], KWARGS: {'help': 'not identical'}} + ], + id='different arg lists lose shared names' + ), + param( + [{ARGS: ['-f', '--foo']}], + [{ARGS: ['-f', '--foo'], KWARGS: {'help': 'not identical'}}], + None, + id='different args identical arg list cause exception' + ), + param( + [{ARGS: ['-g', '--goo']}], + [{ARGS: ['-f', '--foo']}], + [ + {ARGS: ['-g', '--goo']}, + {ARGS: ['-f', '--foo']}, + ], + id='all unrelated args added' + ), + param( + [{ARGS: ['-f', '--foo']}, {ARGS: ['-r', '--redesdale']}], + [{ARGS: ['-f', '--foo']}, {ARGS: ['-b', '--buttered-peas']}], + [ + {ARGS: ['-f', '--foo']}, + {ARGS: ['-r', '--redesdale']}, + {ARGS: ['-b', '--buttered-peas']} + ], + id='do not repeat args' + ) + ] +) +def test_combine_options_pair(first, second, expect): + """It combines sets of options""" + if expect is not None: + result = combine_options_pair(first, second) + + # Order of args irrelevent to test + expect = sorted(expect, key=lambda x: x[ARGS]) + result = sorted(result, key=lambda x: x[ARGS]) + + assert result == expect + else: + with pytest.raises(Exception, match='Clashing Options'): + combine_options_pair(first, second) + + +@pytest.mark.parametrize( + 'inputs, expect', + [ + param( + [ + [{ARGS: ['-i', '--inflammable']}], + [{ARGS: ['-f', '--flammable']}], + [{ARGS: ['-n', '--non-flammable']}] + ], + [ + {ARGS: ['-i', '--inflammable']}, + {ARGS: ['-f', '--flammable']}, + {ARGS: ['-n', '--non-flammable']} + ], + id='merge three argsets no overlap' + ), + param( + [ + [{ARGS: ['-m', '--morpeth']}, {ARGS: ['-r', '--redesdale']}], + [{ARGS: ['-b', '--byker']}, {ARGS: ['-r', '--roxborough']}], + [{ARGS: ['-b', '--bellingham']}] + + ], + [ + {ARGS: ['--bellingham']}, + {ARGS: ['--roxborough']}, + {ARGS: ['--redesdale']}, + {ARGS: ['--byker']}, + {ARGS: ['-m', '--morpeth']} + ], + id='merge three overlapping argsets' + ), + param( + [ + [], + [{ARGS: ['-c', '--campden']}] + ], + [ + {ARGS: ['-c', '--campden']} + ], + id="empty list doesn't clear result" + ) + ] +) +def test_combine_options(inputs, expect): + """It combines multiple input sets""" + result = combine_options(*inputs) + + # Order of args irrelevent to test + expect = sorted(expect, key=lambda x: x[ARGS]) + result = sorted(result, key=lambda x: x[ARGS]) + + assert result == expect + + +@pytest.mark.parametrize( + 'argv_before, kwargs, expect', + [ + param( + 'vip myworkflow'.split(), + { + 'script_name': 'play', + 'compound_script_opts': [ + {ARGS: ['--foo', '-f'], KWARGS: {}}, + ], + 'script_opts': [{ + ARGS: ['--foo', '-f'], + KWARGS: {} + }] + }, + 'play myworkflow'.split(), + id='no opts to remove' + ), + param( + 'vip myworkflow'.split(), + { + 'script_name': 'play', + 'compound_script_opts': [ + {ARGS: ['--foo', '-f'], KWARGS: {}}, + {ARGS: ['--bar', '-b'], KWARGS: {}}, + {ARGS: ['--baz'], KWARGS: {}}, + ], + 'script_opts': [{ + ARGS: ['--foo', '-f'], + KWARGS: {} + }] + }, + 'play myworkflow'.split(), + id='remove some opts' + ), + param( + 'vip myworkflow'.split(), + { + 'script_name': 'play', + 'compound_script_opts': [ + {ARGS: ['--foo', '-f'], KWARGS: {}}, + {ARGS: ['--bar', '-b'], KWARGS: {}}, + {ARGS: ['--baz'], KWARGS: {}}, + ], + 'script_opts': [] + }, + 'play myworkflow'.split(), + id='no opts to keep' + ), + ] +) +def test_cleanup_sysargv(monkeypatch, argv_before, kwargs, expect): + """It replaces the contents of sysargv with Cylc Play argv items. + """ + # Fake up sys.argv: for this test. + dummy_cylc_path = ['/pathto/my/cylc/bin/cylc'] + monkeypatch.setattr(sys, 'argv', dummy_cylc_path + argv_before) + # Fake options too: + opts = SimpleNamespace() + opts.foo = kwargs['script_opts'] + kwargs.update({'options': opts}) + + # Test the script: + cleanup_sysargv(**kwargs) + assert sys.argv == dummy_cylc_path + expect