From 58228c6798b98ae284228c782b18e25b55cfd247 Mon Sep 17 00:00:00 2001
From: Tim Pillinger <26465611+wxtim@users.noreply.github.com>
Date: Wed, 7 Sep 2022 11:09:38 +0100
Subject: [PATCH] cylc vip
---
CHANGES.md | 3 +
cylc/flow/option_parsers.py | 479 +++++++++++++-----
cylc/flow/scheduler_cli.py | 311 +++++++-----
cylc/flow/scripts/cylc.py | 9 +-
cylc/flow/scripts/install.py | 107 ++--
cylc/flow/scripts/validate.py | 80 ++-
cylc/flow/scripts/validate_install_play.py | 110 ++++
setup.cfg | 1 +
.../cylc-combination-scripts/00-vip.t | 43 ++
.../cylc-combination-scripts/00-vip/flow.cylc | 7 +
.../00-vip/reference.log | 1 +
.../cylc-combination-scripts/test_header | 1 +
.../database/04-lock-recover/bin/cylc-db-lock | 2 +-
.../05-lock-recover-100/bin/cylc-db-lock | 2 +-
tests/functional/env-filter/00-filter.t | 2 +-
tests/functional/include-files/00-basic.t | 4 +-
tests/functional/restart/09-reload.t | 2 +-
tests/functional/workflow-state/00-polling.t | 4 +-
tests/functional/workflow-state/01-polling.t | 4 +-
tests/functional/xtriggers/02-persistence.t | 4 +-
tests/unit/test_option_parsers.py | 425 +++++++++++++++-
21 files changed, 1254 insertions(+), 347 deletions(-)
create mode 100644 cylc/flow/scripts/validate_install_play.py
create mode 100644 tests/functional/cylc-combination-scripts/00-vip.t
create mode 100644 tests/functional/cylc-combination-scripts/00-vip/flow.cylc
create mode 100644 tests/functional/cylc-combination-scripts/00-vip/reference.log
create mode 120000 tests/functional/cylc-combination-scripts/test_header
diff --git a/CHANGES.md b/CHANGES.md
index 5b42eda3f8f..3d5ea56522b 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -18,6 +18,9 @@ ones in. -->
[#5184](https://github.com/cylc/cylc-flow/pull/5184) - scan for active
runs of the same workflow at install time.
+[#5094](https://github.com/cylc/cylc-flow/pull/5094) - Added a single
+command to validate, install and play a workflow.
+
[#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of
100 for the "default" queue.
diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py
index a0609e0e172..142d7e670c0 100644
--- a/cylc/flow/option_parsers.py
+++ b/cylc/flow/option_parsers.py
@@ -17,6 +17,7 @@
from contextlib import suppress
import logging
+from itertools import product
from optparse import (
OptionParser,
OptionConflictError,
@@ -33,7 +34,7 @@
import sys
from textwrap import dedent
-from typing import Any, Dict, Optional, List, Tuple
+from typing import Any, Dict, Optional, List, Tuple, Union
from cylc.flow import LOG
from cylc.flow.terminal import supports_color, DIM
@@ -51,20 +52,94 @@
SHORTLINK_TO_ICP_DOCS = "https://bit.ly/3MYHqVh"
-icp_option = Option(
- "--initial-cycle-point", "--icp",
- metavar="CYCLE_POINT or OFFSET",
+APPEND = 'append'
+ARGS = 'args'
+KWARGS = 'kwargs'
+HELP = 'help'
+ACTION = 'action'
+STORE = 'store'
+STORE_CONST = 'store_const'
+STORE_TRUE = 'store_true'
+SOURCES = 'sources'
+DEFAULT = 'default'
+DEST = 'dest'
+METAVAR = 'metavar'
+CHOICES = 'choices'
+OPTS = 'opts'
+ALL = 'all'
+USEIF = 'useif'
+DOUBLEDASH = '--'
+
+
+class OptionSettings():
+ """Container for info about a command line option
+
+ Despite some similarities this is not to be confused with
+ optparse.Option: This a container for information which may or may
+ not be passed to optparse depending on the results of
+ cylc.flow.option_parsers(thismodule).combine_options_pair.
+ """
+
+ def __init__(self, argslist, sources=None, useif=None, **kwargs):
+ """Init function:
+
+ Args:
+ arglist: list of arguments for optparse.Option.
+ sources: set of CLI scripts which use this option.
+ useif: badge for use by Cylc optionparser.
+ **kwargs: kwargs for optparse.option.
+ """
+ self.kwargs: Dict[str, str] = {}
+ self.sources: set = sources if sources is not None else set()
+ self.useif: str = useif if useif is not None else ''
+
+ self.args: list[str] = argslist
+ for kwarg, value in kwargs.items():
+ self.kwargs.update({kwarg: value})
+
+ def __eq__(self, other):
+ """Args and Kwargs, but not other props equal."""
+ return (
+ self.kwargs == other.kwargs
+ and self.args == other.args
+ )
+
+ def __and__(self, other):
+ """Is there a set intersection between arguments."""
+ return list(set(self.args).intersection(set(other.args)))
+
+ def __sub__(self, other):
+ """Set difference on args."""
+ return list(set(self.args) - set(other.args))
+
+ def _in_list(self, others):
+ """CLI arguments for this option found in any of a list of
+ other options."""
+ return any([self & other for other in others])
+
+ def _update_sources(self, other):
+ """Update the sources from this and 1 other OptionSettings object"""
+ self.sources = {*self.sources, *other.sources}
+
+
+ICP_OPTION = OptionSettings(
+ ["--initial-cycle-point", "--icp"],
help=(
- "Set the initial cycle point. "
- "Required if not defined in flow.cylc."
- "\nMay be either an absolute point or an offset: See "
- f"{SHORTLINK_TO_ICP_DOCS} (Cylc documentation link)."
+ "Set the initial cycle point."
+ " Required if not defined in flow.cylc."
+ "\nMay be either an absolute point or an offset: See"
+ f" {SHORTLINK_TO_ICP_DOCS} (Cylc documentation link)."
),
- action="store",
- dest="icp",
+ metavar="CYCLE_POINT or OFFSET",
+ action=STORE,
+ dest="icp"
)
+icp_option = Option(
+ *ICP_OPTION.args, **ICP_OPTION.kwargs) # type: ignore[arg-type]
+
+
def format_shell_examples(string):
"""Put comments in the terminal "diminished" colour."""
return cparse(
@@ -169,7 +244,7 @@ def verbosity_to_env(verb: int) -> Dict[str, str]:
}
-def env_to_verbosity(env: dict) -> int:
+def env_to_verbosity(env: Union[Dict, os._Environ]) -> int:
"""Extract verbosity from environment variables.
Examples:
@@ -283,6 +358,68 @@ class CylcOptionParser(OptionParser):
See `cylc help id` for more details.
'''))
+ STD_OPTIONS = [
+ OptionSettings(
+ ['-q', '--quiet'], help='Decrease verbosity.',
+ action='decrement', dest='verbosity', useif=ALL),
+ OptionSettings(
+ ['-v', '--verbose'], help='Increase Verbosity',
+ dest='verbosity', action='count',
+ default=env_to_verbosity(os.environ), useif=ALL),
+ OptionSettings(
+ ['--debug'], help='Equivalent to -v -v',
+ dest='verbosity', action=STORE_CONST, const=2, useif=ALL),
+ OptionSettings(
+ ['--no-timestamp'], help='Don\'t timestamp logged messages.',
+ action='store_false', dest='log_timestamp',
+ default=True, useif=ALL),
+ OptionSettings(
+ ['--color', '--color'], metavar='WHEN', action=STORE,
+ default='auto', choices=['never', 'auto', 'always'],
+ help=(
+ "When to use color/bold text in terminal output."
+ " Options are 'never', 'auto' and 'always'."
+ ),
+ useif='color'),
+ OptionSettings(
+ ['--comms-timeout'], metavar='SEC',
+ help=(
+ "Set a timeout for network connections"
+ " to the running workflow. The default is no timeout."
+ " For task messaging connections see"
+ " site/user config file documentation."
+ ),
+ action=STORE, default=None, dest='comms_timeout', useif='comms'),
+ OptionSettings(
+ ['-s', '--set'], metavar='NAME=VALUE',
+ help=(
+ "Set the value of a Jinja2 template variable in the"
+ " workflow definition."
+ " Values should be valid Python literals so strings"
+ " must be quoted"
+ " e.g. 'STR=\"string\"', INT=43, BOOL=True."
+ " This option can be used multiple "
+ " times on the command line."
+ " NOTE: these settings persist across workflow restarts,"
+ " but can be set again on the \"cylc play\""
+ " command line if they need to be overridden."
+ ),
+ action='append', default=[], dest='templatevars', useif='jset'),
+ OptionSettings(
+ ['--set-file'], metavar='FILE',
+ help=(
+ "Set the value of Jinja2 template variables in the"
+ " workflow definition from a file containing NAME=VALUE"
+ " pairs (one per line)."
+ " As with --set values should be valid Python literals "
+ " so strings must be quoted e.g. STR='string'."
+ " NOTE: these settings persist across workflow restarts,"
+ " but can be set again on the \"cylc play\""
+ " command line if they need to be overridden."
+ ),
+ action=STORE, default=None, dest='templatevars_file', useif='jset')
+ ]
+
def __init__(
self,
usage: str,
@@ -356,6 +493,17 @@ def __init__(
formatter=CylcHelpFormatter()
)
+ def get_std_options(self):
+ """Get a data-structure of standard options"""
+ opts = []
+ for opt in self.STD_OPTIONS:
+ if (
+ opt.useif == ALL
+ or hasattr(self, opt.useif) and getattr(self, opt.useif)
+ ):
+ opts.append(opt)
+ return opts
+
def add_std_option(self, *args, **kwargs):
"""Add a standard option, ignoring override."""
with suppress(OptionConflictError):
@@ -363,121 +511,53 @@ def add_std_option(self, *args, **kwargs):
def add_std_options(self):
"""Add standard options if they have not been overridden."""
- self.add_std_option(
- "-q", "--quiet",
- help="Decrease verbosity.",
- action='decrement',
- dest='verbosity',
- )
- self.add_std_option(
- "-v", "--verbose",
- help="Increase verbosity.",
- dest='verbosity',
- action='count',
- default=env_to_verbosity(os.environ)
- )
- self.add_std_option(
- "--debug",
- help="Equivalent to -v -v",
- dest="verbosity",
- action='store_const',
- const=2
- )
- self.add_std_option(
- "--no-timestamp",
- help="Don't timestamp logged messages.",
- action="store_false", dest="log_timestamp", default=True)
-
- if self.color:
- self.add_std_option(
- '--color', '--colour', metavar='WHEN', action='store',
- default='auto', choices=['never', 'auto', 'always'],
- help=(
- "When to use color/bold text in terminal output."
- " Options are 'never', 'auto' and 'always'."
- )
- )
+ for option in self.get_std_options():
+ if not any(self.has_option(i) for i in option.args):
+ self.add_option(*option.args, **option.kwargs)
- if self.comms:
- self.add_std_option(
- "--comms-timeout", metavar='SEC',
+ @staticmethod
+ def get_cylc_rose_options():
+ """Returns a list of option dictionaries if Cylc Rose exists."""
+ try:
+ __import__('cylc.rose')
+ except ImportError:
+ return []
+ return [
+ OptionSettings(
+ ["--opt-conf-key", "-O"],
help=(
- "Set a timeout for network connections "
- "to the running workflow. The default is no timeout. "
- "For task messaging connections see "
- "site/user config file documentation."
- ),
- action="store", default=None, dest="comms_timeout")
-
- if self.jset:
- self.add_std_option(
- "-s", "--set", metavar="NAME=VALUE",
+ "Use optional Rose Config Setting"
+ " (If Cylc-Rose is installed)"),
+ action="append", default=[], dest="opt_conf_keys",
+ sources={'cylc-rose'},
+ ),
+ OptionSettings(
+ ["--define", '-D'],
help=(
- "Set the value of a Jinja2 template variable in the"
- " workflow definition."
- " Values should be valid Python literals so strings"
- " must be quoted"
- " e.g. 'STR=\"string\"', INT=43, BOOL=True."
- " This option can be used multiple "
- " times on the command line."
- " NOTE: these settings persist across workflow restarts,"
- " but can be set again on the \"cylc play\""
- " command line if they need to be overridden."
- ),
- action="append", default=[], dest="templatevars")
-
- self.add_std_option(
- "--set-file", metavar="FILE",
+ "Each of these overrides the `[SECTION]KEY` setting"
+ " in a `rose-suite.conf` file."
+ " Can be used to disable a setting using the syntax"
+ " `--define=[SECTION]!KEY` or"
+ " even `--define=[!SECTION]`."),
+ action="append", default=[], dest="defines",
+ sources={'cylc-rose'}),
+ OptionSettings(
+ ["--rose-template-variable", '-S', '--define-suite'],
help=(
- "Set the value of Jinja2 template variables in the "
- "workflow definition from a file containing NAME=VALUE "
- "pairs (one per line). "
- "As with --set values should be valid Python literals "
- "so strings must be quoted e.g. STR='string'. "
- "NOTE: these settings persist across workflow restarts, "
- "but can be set again on the \"cylc play\" "
- "command line if they need to be overridden."
- ),
- action="store", default=None, dest="templatevars_file")
+ "As `--define`, but with an implicit `[SECTION]` for"
+ " workflow variables."),
+ action="append", default=[], dest="rose_template_vars",
+ sources={'cylc-rose'},
+ )
+ ]
def add_cylc_rose_options(self) -> None:
- """Add extra options for cylc-rose plugin if it is installed."""
- try:
- __import__('cylc.rose')
- except ImportError:
- return
- self.add_option(
- "--opt-conf-key", "-O",
- help=(
- "Use optional Rose Config Setting "
- "(If Cylc-Rose is installed)"
- ),
- action="append",
- default=[],
- dest="opt_conf_keys"
- )
- self.add_option(
- "--define", '-D',
- help=(
- "Each of these overrides the `[SECTION]KEY` setting in a "
- "`rose-suite.conf` file. "
- "Can be used to disable a setting using the syntax "
- "`--define=[SECTION]!KEY` or even `--define=[!SECTION]`."
- ),
- action="append",
- default=[],
- dest="defines"
- )
- self.add_option(
- "--rose-template-variable", '-S', '--define-suite',
- help=(
- "As `--define`, but with an implicit `[SECTION]` for "
- "workflow variables."
- ),
- action="append",
- default=[],
- dest="rose_template_vars"
- )
+ """Add extra options for cylc-rose plugin if it is installed.
+
+ Now a vestigal interface for get_cylc_rose_options.
+ """
+ for option in self.get_cylc_rose_options():
+ self.add_option(*option.args, **option.kwargs)
def parse_args(self, api_args, remove_opts=None):
"""Parse options and arguments, overrides OptionParser.parse_args.
@@ -607,3 +687,162 @@ def __call__(self, **kwargs) -> Values:
setattr(opts, key, value)
return opts
+
+
+def appendif(list_, item):
+ """Avoid duplicating items in output list"""
+ if item not in list_:
+ list_.append(item)
+ return list_
+
+
+def combine_options_pair(first_list, second_list):
+ """Combine two option lists recording where each came from.
+
+ Scenarios:
+ - Arguments are identical - return this argument.
+ - Arguments are not identical but have some common label strings,
+ i.e. both arguments can be invoked using `-f`.
+ - If there are non-shared label strings strip the shared ones.
+ - Otherwise raise an error.
+ E.g: If `command-A` has an option `-f` or `--file` and
+ `command-B has an option `-f` or `--fortran`` then
+ `command-A+B` will have options `--fortran` and `--file` but _not_
+ `-f`, which would be confusing.
+ - Arguments only apply to a single compnent of the compound CLI script.
+
+ """
+ output = []
+ if not first_list:
+ output = second_list
+ elif not second_list:
+ output = first_list
+ else:
+ for first, second in product(first_list, second_list):
+ # Two options are identical in both args and kwargs:
+ if first == second:
+ first._update_sources(second)
+ output = appendif(output, first)
+
+ # If any of the argument names identical we must remove
+ # overlapping names (if we can)
+ # e.g. [-a, --aleph], [-a, --alpha-centuri] -> keep both options
+ # but neither should have the `-a` short version:
+ elif (
+ first != second
+ and first & second
+ ):
+ # if any of the args are different:
+ if first.args == second.args:
+ # if none of the arg names are different.
+ raise Exception(f'Clashing Options \n{first}\n{second}')
+ else:
+ first_args = first - second
+ second.args = second - first
+ first.args = first_args
+ output = appendif(output, first)
+ output = appendif(output, second)
+ else:
+ # Neither option appears in the other list, so it can be
+ # appended:
+ if not first._in_list(second_list):
+ output = appendif(output, first)
+ if not second._in_list(first_list):
+ output = appendif(output, second)
+
+ return output
+
+
+def add_sources_to_helps(options, modify=None):
+ """Prettify format of list of CLI commands this option applies to
+ and prepend that list to the start of help.
+
+ Arguments:
+ Options:
+ Options dicts to modify help upon.
+ modify:
+ Dict of items to substitute: Intended to allow one
+ to replace cylc-rose with the names of the sub-commands
+ cylc rose options apply to.
+ """
+ modify = {} if modify is None else modify
+ for option in options:
+ if hasattr(option, SOURCES):
+ sources = list(option.sources)
+ for match, sub in modify.items():
+ if match in option.sources:
+ sources.append(sub)
+ sources.remove(match)
+
+ option.kwargs[HELP] = cparse(
+ f'[{", ".join(sources)}]'
+ f' {option.kwargs[HELP]}')
+ return options
+
+
+def combine_options(*args, modify=None):
+ """Combine a list of argument dicts.
+
+ Ordering should be irrelevant because combine_options_pair should
+ be commutative, and the overall order of args is not relevant.
+ """
+ list_ = list(args)
+ output = list_[0]
+ for arg in list_[1:]:
+ output = combine_options_pair(arg, output)
+
+ return add_sources_to_helps(output, modify)
+
+
+def cleanup_sysargv(
+ script_name,
+ workflow_id,
+ options,
+ compound_script_opts,
+ script_opts,
+ source,
+):
+ """Remove unwanted options from sys.argv
+
+ Some cylc scripts (notably Cylc Play when it is re-invoked on a scheduler
+ server) require the correct content in sys.argv.
+ """
+ # Organize Options by dest.
+ script_opts_by_dest = {
+ x.kwargs.get('dest', x.args[0].strip(DOUBLEDASH)): x
+ for x in script_opts
+ }
+ compound_opts_by_dest = {
+ x.kwargs.get('dest', x.args[0].strip(DOUBLEDASH)): x
+ for x in compound_script_opts
+ }
+ # Filter out non-cylc-play options.
+ for unwanted_opt in (set(options.__dict__)) - set(script_opts_by_dest):
+ for arg in compound_opts_by_dest[unwanted_opt].args:
+ if arg in sys.argv:
+ index = sys.argv.index(arg)
+ sys.argv.pop(index)
+ if (
+ compound_opts_by_dest[unwanted_opt].kwargs[ACTION]
+ not in ['store_true', 'store_false']
+ ):
+ sys.argv.pop(index)
+
+ # replace compound script name:
+ sys.argv[1] = script_name
+
+ # replace source path with workflow ID.
+ if str(source) in sys.argv:
+ sys.argv.remove(str(source))
+ sys.argv.append(workflow_id)
+
+
+def log_subcommand(command, workflow_id):
+ """Log a command run as part of a sequence.
+
+ Example:
+ >>> log_subcommand('ruin', 'my_workflow')
+ \x1b[1m\x1b[36m$ cylc ruin my_workflow\x1b[0m\x1b[1m\x1b[0m\n
+ """
+ print(cparse(
+ f'$ cylc {command} {workflow_id}'))
diff --git a/cylc/flow/scheduler_cli.py b/cylc/flow/scheduler_cli.py
index 5682530ee38..ad9bc3f0422 100644
--- a/cylc/flow/scheduler_cli.py
+++ b/cylc/flow/scheduler_cli.py
@@ -17,6 +17,7 @@
from ansimarkup import parse as cparse
import asyncio
+from copy import deepcopy
from functools import lru_cache
from itertools import zip_longest
from pathlib import Path
@@ -41,8 +42,10 @@
from cylc.flow.option_parsers import (
WORKFLOW_ID_ARG_DOC,
CylcOptionParser as COP,
+ OptionSettings,
Options,
- icp_option,
+ ICP_OPTION,
+ APPEND, ARGS, KWARGS, STORE, STORE_TRUE
)
from cylc.flow.pathutil import get_workflow_run_scheduler_log_path
from cylc.flow.remote import cylc_server_cmd
@@ -117,152 +120,197 @@
}
'''
+PLAY_ICP_OPTION = deepcopy(ICP_OPTION)
+PLAY_ICP_OPTION.sources = {'play'}
-@lru_cache()
-def get_option_parser(add_std_opts: bool = False) -> COP:
- """Parse CLI for "cylc play"."""
- parser = COP(
- PLAY_DOC,
- jset=True,
- comms=True,
- argdoc=[WORKFLOW_ID_ARG_DOC]
- )
-
- parser.add_option(
- "-n", "--no-detach", "--non-daemon",
- help="Do not daemonize the scheduler (infers --format=plain)",
- action="store_true", dest="no_detach")
-
- parser.add_option(
- "--profile", help="Output profiling (performance) information",
- action="store_true", dest="profile_mode")
+RUN_MODE = OptionSettings(
+ ["-m", "--mode"],
+ help="Run mode: live, dummy, simulation (default live).",
+ metavar="STRING", action=STORE, dest="run_mode",
+ choices=['live', 'dummy', 'simulation'],
+)
- parser.add_option(icp_option)
+PLAY_RUN_MODE = deepcopy(RUN_MODE)
+PLAY_RUN_MODE.sources = {'play'}
- parser.add_option(
- "--start-cycle-point", "--startcp",
+PLAY_OPTIONS = [
+ OptionSettings(
+ ["-n", "--no-detach", "--non-daemon"],
+ help="Do not daemonize the scheduler (infers --format=plain)",
+ action=STORE_TRUE, dest="no_detach", sources={'play'}),
+ OptionSettings(
+ ["--profile"],
+ help="Output profiling (performance) information",
+ action=STORE_TRUE,
+ default=False,
+ dest="profile_mode",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--start-cycle-point", "--startcp"],
help=(
- "Set the start cycle point, which may be after the initial cycle "
- "point. If the specified start point is not in the sequence, the "
- "next on-sequence point will be used. "
- "(Not to be confused with the initial cycle point.) "
- "This replaces the Cylc 7 --warm option."
- ),
- metavar="CYCLE_POINT", action="store", dest="startcp")
-
- parser.add_option(
- "--final-cycle-point", "--fcp",
+ "Set the start cycle point, which may be after"
+ " the initial cycle point. If the specified start point is"
+ " not in the sequence, the next on-sequence point will"
+ " be used. (Not to be confused with the initial cycle point)"),
+ metavar="CYCLE_POINT",
+ action=STORE,
+ dest="startcp",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--final-cycle-point", "--fcp"],
help=(
- "Set the final cycle point. "
- "This command line option overrides the workflow "
- "config option '[scheduling]final cycle point'. "
- ),
- metavar="CYCLE_POINT", action="store", dest="fcp")
-
- parser.add_option(
- "--stop-cycle-point", "--stopcp",
+ "Set the final cycle point. This command line option overrides"
+ " the workflow config option"
+ " '[scheduling]final cycle point'. "),
+ metavar="CYCLE_POINT",
+ action=STORE,
+ dest="fcp",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--stop-cycle-point", "--stopcp"],
help=(
- "Set the stop cycle point. "
- "Shut down after all tasks have PASSED this cycle point. "
- "(Not to be confused with the final cycle point.) "
- "This command line option overrides the workflow "
- "config option '[scheduling]stop after cycle point'. "
- ),
- metavar="CYCLE_POINT", action="store", dest="stopcp")
-
- parser.add_option(
- "--start-task", "--starttask", "-t",
+ "Set the stop cycle point. Shut down after all"
+ " have PASSED this cycle point. (Not to be confused"
+ " the final cycle point.) This command line option overrides"
+ " the workflow config option"
+ " '[scheduling]stop after cycle point'."),
+ metavar="CYCLE_POINT",
+ action=STORE,
+ dest="stopcp",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--start-task", "--starttask", "-t"],
help=(
- "Start from this task instance, given by '/'. "
- "Can be used multiple times "
- "to start from multiple tasks at once. Dependence on tasks with "
- "with cycle points earlier than the earliest start-task will be "
- "ignored. A sub-graph of the workflow will run if selected tasks "
- "do not lead on to the full graph."
- ),
- metavar="TASK_ID", action="append", dest="starttask")
-
- parser.add_option(
- "--pause",
+ "Start from this task instance, given by '/'."
+ " This can be used multiple times to start from multiple"
+ " tasks at once. Dependence on tasks with cycle points earlier"
+ " than the earliest start-task will be ignored. A"
+ " sub-graph of the workflow will run if selected tasks do"
+ " not lead on to the full graph."),
+ metavar="TASK_ID",
+ action=APPEND,
+ dest="starttask",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--pause"],
help="Pause the workflow immediately on start up.",
- action="store_true", dest="paused_start")
-
- parser.add_option(
- "--hold-after", "--hold-cycle-point", "--holdcp",
+ action=STORE_TRUE,
+ dest="paused_start",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--hold-after", "--hold-cycle-point", "--holdcp"],
help="Hold all tasks after this cycle point.",
- metavar="CYCLE_POINT", action="store", dest="holdcp")
-
- parser.add_option(
- "-m", "--mode",
- help="Run mode: live, dummy, simulation (default live).",
- metavar="STRING", action="store", dest="run_mode",
- choices=["live", "dummy", "simulation"])
-
- parser.add_option(
- "--reference-log",
- help="Generate a reference log for use in reference tests.",
- action="store_true", default=False, dest="genref")
-
- parser.add_option(
- "--reference-test",
- help="Do a test run against a previously generated reference log.",
- action="store_true", default=False, dest="reftest")
-
- # Override standard parser option for specific help description.
- parser.add_option(
- "--host",
+ metavar="CYCLE_POINT",
+ action=STORE,
+ dest="holdcp",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--reference-log"],
+ help="Generate a reference log for use in reference ",
+ action=STORE_TRUE,
+ default=False,
+ dest="genref",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--reference-test"],
+ help="Do a test run against a previously generated reference.",
+ action=STORE_TRUE,
+ default=False,
+ dest="reftest",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--host"],
help=(
- "Specify the host on which to start-up the workflow. "
- "If not specified, a host will be selected using "
- "the '[scheduler]run hosts' global config."
- ),
- metavar="HOST", action="store", dest="host")
-
- parser.add_option(
- "--format",
- help="The format of the output: 'plain'=human readable, 'json'",
- choices=("plain", "json"),
- default="plain"
- )
-
- parser.add_option(
- "--main-loop",
+ "Specify the host on which to start-up the workflow."
+ " If not specified, a host will be selected using"
+ " the '[scheduler]run hosts' global config."),
+ metavar="HOST",
+ action=STORE,
+ dest="host",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--format"],
+ help="The format of the output: 'plain'=human readable, 'json",
+ choices=('plain', 'json'),
+ default="plain",
+ dest='format',
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--main-loop"],
help=(
- "Specify an additional plugin to run in the main loop."
- " These are used in combination with those specified in"
- " [scheduler][main loop]plugins. Can be used multiple times"
- ),
- metavar="PLUGIN_NAME", action="append", dest="main_loop"
- )
-
- parser.add_option(
- "--abort-if-any-task-fails",
+ "Specify an additional plugin to run in the main"
+ " These are used in combination with those specified"
+ " [scheduler][main loop]plugins. Can be used multiple times."),
+ metavar="PLUGIN_NAME",
+ action=APPEND,
+ dest="main_loop",
+ sources={'play'},
+ ),
+ OptionSettings(
+ ["--abort-if-any-task-fails"],
help="If set workflow will abort with status 1 if any task fails.",
- action="store_true", default=False, dest="abort_if_any_task_fails"
- )
-
- parser.add_option(
- '--downgrade',
+ action=STORE_TRUE,
+ default=False,
+ dest="abort_if_any_task_fails",
+ sources={'play'},
+ ),
+ PLAY_ICP_OPTION,
+ PLAY_RUN_MODE,
+ OptionSettings(
+ ['--downgrade'],
help=(
- 'Allow the workflow to be restarted with an older version of Cylc,'
- ' NOT RECOMMENDED.'
- ' By default Cylc prevents you from restarting a workflow with an'
- ' older version of Cylc than it was previously run with.'
- ' Use this flag to disable this check.'
+ 'Allow the workflow to be restarted with an'
+ ' older version of Cylc, NOT RECOMMENDED.'
+ ' By default Cylc prevents you from restarting'
+ ' a workflow with an older version of Cylc than'
+ ' it was previously run with. Use this flag'
+ ' to disable this check.'
),
- action='store_true',
- default=False
- )
-
- parser.add_option(
- '--upgrade',
+ action=STORE_TRUE,
+ default=False,
+ sources={'play'}
+ ),
+ OptionSettings(
+ ['--upgrade'],
help=(
- 'Allow the workflow to be restarted with an newer version of Cylc.'
+ 'Allow the workflow to be restarted with'
+ ' a newer version of Cylc.'
),
- action='store_true',
- default=False
+ action=STORE_TRUE,
+ default=False,
+ sources={'play'}
+ ),
+]
+
+
+@lru_cache()
+def get_option_parser(add_std_opts: bool = False) -> COP:
+ """Parse CLI for "cylc play"."""
+ parser = COP(
+ PLAY_DOC,
+ jset=True,
+ comms=True,
+ argdoc=[WORKFLOW_ID_ARG_DOC]
)
+ options = parser.get_cylc_rose_options() + PLAY_OPTIONS
+ for option in options:
+ if isinstance(option, OptionSettings):
+ parser.add_option(*option.args, **option.kwargs)
+ else:
+ parser.add_option(*option[ARGS], **option[KWARGS])
+
if add_std_opts:
# This is for the API wrapper for integration tests. Otherwise (CLI
# use) "standard options" are added later in options.parse_args().
@@ -555,6 +603,11 @@ async def _run(scheduler: Scheduler) -> int:
@cli_function(get_option_parser)
def play(parser: COP, options: 'Values', id_: str):
"""Implement cylc play."""
+ return _play(parser, options, id_)
+
+
+def _play(parser: COP, options: 'Values', id_: str):
+ """Allows compound scripts to import play, but supply their own COP."""
if options.starttask:
options.starttask = upgrade_legacy_ids(
*options.starttask,
diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py
index 39d8d1c8e50..74a37119043 100644
--- a/cylc/flow/scripts/cylc.py
+++ b/cylc/flow/scripts/cylc.py
@@ -201,7 +201,8 @@ def get_version(long=False):
'ls': 'list',
'shutdown': 'stop',
'task-message': 'message',
- 'unhold': 'release'
+ 'unhold': 'release',
+ 'validate-install-play': 'vip'
}
@@ -311,12 +312,6 @@ def match_command(command):
for cmd in COMMANDS
if cmd.startswith(command)
},
- *{
- # search aliases
- cmd
- for alias, cmd in ALIASES.items()
- if alias.startswith(command)
- }
}
if len(possible_cmds) == 0:
print(
diff --git a/cylc/flow/scripts/install.py b/cylc/flow/scripts/install.py
index c157a49d4a8..e926fb41750 100755
--- a/cylc/flow/scripts/install.py
+++ b/cylc/flow/scripts/install.py
@@ -97,6 +97,7 @@
from cylc.flow.loggingutil import CylcLogFormatter
from cylc.flow.option_parsers import (
CylcOptionParser as COP,
+ OptionSettings,
Options
)
from cylc.flow.pathutil import (
@@ -112,71 +113,80 @@
from cylc.flow.terminal import cli_function
-def get_option_parser() -> COP:
- parser = COP(
- __doc__,
- comms=True,
- argdoc=[
- COP.optional(
- ('SOURCE_NAME | PATH',
- 'Workflow source name or path to source directory')
- )
- ]
- )
-
- parser.add_option(
- "--workflow-name", "-n",
+INSTALL_OPTIONS = [
+ OptionSettings(
+ ["--workflow-name", "-n"],
help="Install into ~/cylc-run//runN ",
action="store",
metavar="WORKFLOW_NAME",
default=None,
- dest="workflow_name")
-
- parser.add_option(
- "--symlink-dirs",
+ dest="workflow_name",
+ sources={'install'},
+ ),
+ OptionSettings(
+ ["--symlink-dirs"],
help=(
- "Enter a comma-delimited list, in the form "
- "'log=path/to/store, share = $HOME/some/path, ...'. "
- "Use this option to override the global.cylc configuration for "
- "local symlinks for the run, log, work, share and "
- "share/cycle directories. "
- "Enter an empty list '' to skip making localhost symlink dirs."
- ),
+ "Enter a comma-delimited list, in the form"
+ " 'log=path/to/store, share = $HOME/some/path, ...'."
+ " Use this option to override the global.cylc configuration"
+ " for local symlinks for the run, log, work, share and"
+ " share/cycle directories. Enter"
+ " an empty list '' to skip making localhost symlink dirs."),
action="store",
- dest="symlink_dirs"
- )
-
- parser.add_option(
- "--run-name",
+ dest="symlink_dirs",
+ sources={'install'},
+ ),
+ OptionSettings(
+ ["--run-name"],
help=(
- "Give the run a custom name instead of automatically numbering it."
- ),
+ "Give the run a custom name instead of automatically"
+ " numbering it."),
action="store",
metavar="RUN_NAME",
default=None,
- dest="run_name")
-
- parser.add_option(
- "--no-run-name",
+ dest="run_name",
+ sources={'install'},
+ ),
+ OptionSettings(
+ ["--no-run-name"],
help=(
- "Install the workflow directly into ~/cylc-run/, "
- "without an automatic run number or custom run name."
- ),
+ "Install the workflow directly into"
+ " ~/cylc-run/,"
+ " without an automatic run number or custom run name."),
action="store_true",
default=False,
- dest="no_run_name")
-
- parser.add_option(
- "--no-ping",
+ dest="no_run_name",
+ sources={'install'},
+ ),
+ OptionSettings(
+ ['--no-ping'],
help=(
"When scanning for active instances of the workflow, "
- "do not attempt to contact the schedulers to get status."
- ),
- action="store_true",
+ "do not attempt to contact the schedulers to get status."),
+ action='store_true',
default=False,
- dest="no_ping")
+ dest='no_ping',
+ sources={'install'},
+ )
+]
- parser.add_cylc_rose_options()
+
+def get_option_parser() -> COP:
+ parser = COP(
+ __doc__,
+ comms=True,
+ argdoc=[
+ COP.optional(
+ ('SOURCE_NAME | PATH',
+ 'Workflow source name or path to source directory')
+ )
+ ]
+ )
+
+ options = parser.get_cylc_rose_options() + INSTALL_OPTIONS
+
+ for option in options:
+ parser.add_option(*option.args, **option.kwargs)
return parser
@@ -320,5 +330,4 @@ def install(
entry_point.name,
exc
) from None
-
return workflow_name
diff --git a/cylc/flow/scripts/validate.py b/cylc/flow/scripts/validate.py
index 6e5031c91ee..aa93752f40c 100755
--- a/cylc/flow/scripts/validate.py
+++ b/cylc/flow/scripts/validate.py
@@ -26,6 +26,7 @@
"""
from ansimarkup import parse as cparse
+from copy import deepcopy
from optparse import Values
import sys
@@ -42,13 +43,56 @@
from cylc.flow.option_parsers import (
WORKFLOW_ID_OR_PATH_ARG_DOC,
CylcOptionParser as COP,
+ OptionSettings,
Options,
- icp_option,
+ ICP_OPTION,
+ ARGS, KWARGS
)
from cylc.flow.profiler import Profiler
from cylc.flow.task_proxy import TaskProxy
from cylc.flow.templatevars import get_template_vars
from cylc.flow.terminal import cli_function
+from cylc.flow.scheduler_cli import RUN_MODE
+
+
+VALIDATE_RUN_MODE = deepcopy(RUN_MODE)
+VALIDATE_RUN_MODE.sources = {'validate'}
+VALIDATE_ICP_OPTION = deepcopy(ICP_OPTION)
+VALIDATE_ICP_OPTION.sources = {'validate'}
+
+
+VALIDATE_OPTIONS = [
+ OptionSettings(
+ ["--check-circular"],
+ help=(
+ "Check for circular dependencies in graphs when the number of"
+ " tasks is greater than 100 (smaller graphs are always"
+ " checked). This can be slow when the number of"
+ " tasks is high."),
+ action="store_true",
+ default=False,
+ dest="check_circular",
+ sources={'validate'}
+ ),
+ OptionSettings(
+ ["--output", "-o"],
+ help="Specify a file name to dump the processed flow.cylc.",
+ metavar="FILENAME",
+ action="store",
+ dest="output",
+ sources={'validate'}
+ ),
+ OptionSettings(
+ ["--profile"],
+ help="Output profiling (performance) information",
+ action="store_true",
+ default=False,
+ dest="profile_mode",
+ sources={'validate'}
+ ),
+ VALIDATE_RUN_MODE,
+ VALIDATE_ICP_OPTION
+]
def get_option_parser():
@@ -58,31 +102,13 @@ def get_option_parser():
argdoc=[WORKFLOW_ID_OR_PATH_ARG_DOC],
)
- parser.add_option(
- "--check-circular",
- help="Check for circular dependencies in graphs when the number of "
- "tasks is greater than 100 (smaller graphs are always checked). "
- "This can be slow when the number of "
- "tasks is high.",
- action="store_true", default=False, dest="check_circular")
-
- parser.add_option(
- "--output", "-o",
- help="Specify a file name to dump the processed flow.cylc.",
- metavar="FILENAME", action="store", dest="output")
-
- parser.add_option(
- "--profile", help="Output profiling (performance) information",
- action="store_true", default=False, dest="profile_mode")
+ validate_options = parser.get_cylc_rose_options() + VALIDATE_OPTIONS
- parser.add_option(
- "-u", "--run-mode", help="Validate for run mode.", action="store",
- default="live", dest="run_mode",
- choices=['live', 'dummy', 'simulation'])
-
- parser.add_option(icp_option)
-
- parser.add_cylc_rose_options()
+ for option in validate_options:
+ if isinstance(option, OptionSettings):
+ parser.add_option(*option.args, **option.kwargs)
+ else:
+ parser.add_option(*option[ARGS], **option[KWARGS])
parser.set_defaults(is_validate=True)
@@ -102,6 +128,10 @@ def get_option_parser():
@cli_function(get_option_parser)
def main(parser: COP, options: 'Values', workflow_id: str) -> None:
+ wrapped_main(parser, options, workflow_id)
+
+
+def wrapped_main(parser: COP, options: 'Values', workflow_id: str) -> None:
"""cylc validate CLI."""
profiler = Profiler(None, options.profile_mode)
profiler.start()
diff --git a/cylc/flow/scripts/validate_install_play.py b/cylc/flow/scripts/validate_install_play.py
new file mode 100644
index 00000000000..7feaac163e4
--- /dev/null
+++ b/cylc/flow/scripts/validate_install_play.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+"""cylc validate-install-play [OPTIONS] ARGS
+
+Validate install and play a single workflow.
+
+This script is equivalent to:
+
+ $ cylc validate /path/to/myworkflow
+ $ cylc install /path/to/myworkflow
+ $ cylc play myworkflow
+
+"""
+
+from cylc.flow.scripts.validate import (
+ VALIDATE_OPTIONS,
+ wrapped_main as validate_main
+)
+from cylc.flow.scripts.install import (
+ INSTALL_OPTIONS, install, get_source_location
+)
+from cylc.flow.scheduler_cli import PLAY_OPTIONS
+from cylc.flow.option_parsers import (
+ CylcOptionParser as COP,
+ combine_options,
+ cleanup_sysargv,
+ log_subcommand,
+)
+from cylc.flow.scheduler_cli import _play
+from cylc.flow.terminal import cli_function
+
+from typing import TYPE_CHECKING, Optional
+
+
+if TYPE_CHECKING:
+ from optparse import Values
+
+
+CYLC_ROSE_OPTIONS = COP.get_cylc_rose_options()
+VIP_OPTIONS = combine_options(
+ VALIDATE_OPTIONS,
+ INSTALL_OPTIONS,
+ PLAY_OPTIONS,
+ CYLC_ROSE_OPTIONS,
+ modify={'cylc-rose': 'validate, install'}
+)
+
+
+def get_option_parser() -> COP:
+ parser = COP(
+ __doc__,
+ comms=True,
+ jset=True,
+ argdoc=[
+ COP.optional((
+ 'SOURCE_NAME | PATH',
+ 'Workflow source name or path to source directory'
+ ))
+ ]
+ )
+ for option in VIP_OPTIONS:
+ parser.add_option(*option.args, **option.kwargs)
+ return parser
+
+
+@cli_function(get_option_parser)
+def main(parser: COP, options: 'Values', workflow_id: Optional[str] = None):
+ """Run Cylc validate - install - play in sequence."""
+ if not workflow_id:
+ workflow_id = '.'
+
+ orig_source = workflow_id
+ source = get_source_location(workflow_id)
+
+ log_subcommand('validate', source)
+ validate_main(parser, options, str(source))
+
+ log_subcommand('install', '.')
+ workflow_id = install(options, workflow_id)
+
+ cleanup_sysargv(
+ 'play',
+ workflow_id,
+ options,
+ compound_script_opts=VIP_OPTIONS,
+ script_opts=(
+ PLAY_OPTIONS + CYLC_ROSE_OPTIONS
+ + parser.get_std_options()
+ ),
+ source=orig_source,
+ )
+
+ log_subcommand('play', workflow_id)
+ _play(parser, options, workflow_id)
diff --git a/setup.cfg b/setup.cfg
index 3aa0282073f..f6ee306837f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -199,6 +199,7 @@ cylc.command =
trigger = cylc.flow.scripts.trigger:main
validate = cylc.flow.scripts.validate:main
view = cylc.flow.scripts.view:main
+ vip = cylc.flow.scripts.validate_install_play:main
# async functions to run within the scheduler main loop
cylc.main_loop =
health_check = cylc.flow.main_loop.health_check
diff --git a/tests/functional/cylc-combination-scripts/00-vip.t b/tests/functional/cylc-combination-scripts/00-vip.t
new file mode 100644
index 00000000000..4657013db4f
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/00-vip.t
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+
+#------------------------------------------------------------------------------
+# Test `cylc vip` (Validate Install Play)
+
+. "$(dirname "$0")/test_header"
+set_test_number 5
+
+WORKFLOW_NAME="cylctb-x$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c6)"
+
+run_ok "${TEST_NAME_BASE}-from-path" \
+ cylc vip --no-detach --debug \
+ --workflow-name "${WORKFLOW_NAME}" \
+ --initial-cycle-point=1300 \
+ --no-run-name \
+ --reference-test \
+ "${TEST_SOURCE_DIR}/${TEST_NAME_BASE}"
+
+grep_ok "13000101T0000Z" "${TEST_NAME_BASE}-from-path.stdout"
+
+grep "\$" "${TEST_NAME_BASE}-from-path.stdout" > VIPOUT.txt
+
+named_grep_ok "${TEST_NAME_BASE}-it-validated" "$ cylc validate" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-installed" "$ cylc install" "VIPOUT.txt"
+named_grep_ok "${TEST_NAME_BASE}-it-played" "$ cylc play" "VIPOUT.txt"
+
+purge
+exit 0
diff --git a/tests/functional/cylc-combination-scripts/00-vip/flow.cylc b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc
new file mode 100644
index 00000000000..745bf26cb2f
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/00-vip/flow.cylc
@@ -0,0 +1,7 @@
+[scheduler]
+ allow implicit tasks = True
+
+[scheduling]
+ initial cycle point = 1500
+ [[graph]]
+ R1 = foo
diff --git a/tests/functional/cylc-combination-scripts/00-vip/reference.log b/tests/functional/cylc-combination-scripts/00-vip/reference.log
new file mode 100644
index 00000000000..70c76fc307c
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/00-vip/reference.log
@@ -0,0 +1 @@
+13000101T0000Z/foo -triggered off [] in flow 1
diff --git a/tests/functional/cylc-combination-scripts/test_header b/tests/functional/cylc-combination-scripts/test_header
new file mode 120000
index 00000000000..90bd5a36f92
--- /dev/null
+++ b/tests/functional/cylc-combination-scripts/test_header
@@ -0,0 +1 @@
+../lib/bash/test_header
\ No newline at end of file
diff --git a/tests/functional/database/04-lock-recover/bin/cylc-db-lock b/tests/functional/database/04-lock-recover/bin/cylc-db-lock
index cdef1b2b8f1..90213dbf5e2 100755
--- a/tests/functional/database/04-lock-recover/bin/cylc-db-lock
+++ b/tests/functional/database/04-lock-recover/bin/cylc-db-lock
@@ -10,7 +10,7 @@ def main():
os.path.join(os.getenv("CYLC_WORKFLOW_RUN_DIR"), "log", "db"))
lockf(handle, LOCK_SH)
call([
- "cylc", "task", "message", "I have locked the public database file"])
+ "cylc", "task-message", "I have locked the public database file"])
workflow_log_dir = os.getenv("CYLC_WORKFLOW_LOG_DIR")
while True:
for line in open(os.path.join(workflow_log_dir, "log")):
diff --git a/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock b/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock
index 6cc995e39a3..4a8846180ce 100755
--- a/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock
+++ b/tests/functional/database/05-lock-recover-100/bin/cylc-db-lock
@@ -11,7 +11,7 @@ def main():
os.path.join(os.getenv("CYLC_WORKFLOW_RUN_DIR"), "log", "db"))
lockf(handle, LOCK_SH)
call([
- "cylc", "task", "message", "I have locked the public database file"])
+ "cylc", "task-message", "I have locked the public database file"])
workflow_log_dir = os.getenv("CYLC_WORKFLOW_LOG_DIR")
while True:
for line in open(os.path.join(workflow_log_dir, "log")):
diff --git a/tests/functional/env-filter/00-filter.t b/tests/functional/env-filter/00-filter.t
index 30cd9027c28..6078258dd14 100644
--- a/tests/functional/env-filter/00-filter.t
+++ b/tests/functional/env-filter/00-filter.t
@@ -52,7 +52,7 @@ __FLOW_CONFIG__
#-------------------------------------------------------------------------------
# check validation
TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc val "${WORKFLOW_NAME}"
+run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
#-------------------------------------------------------------------------------
# check that config retrieves only the filtered environment
TEST_NAME=${TEST_NAME_BASE}-config
diff --git a/tests/functional/include-files/00-basic.t b/tests/functional/include-files/00-basic.t
index 19fe3a9531b..23a48c86eec 100644
--- a/tests/functional/include-files/00-basic.t
+++ b/tests/functional/include-files/00-basic.t
@@ -24,12 +24,12 @@ install_workflow "${TEST_NAME_BASE}" workflow
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
# test raw workflow validates
-run_ok "${TEST_NAME}.1" cylc val "${WORKFLOW_NAME}"
+run_ok "${TEST_NAME}.1" cylc validate "${WORKFLOW_NAME}"
# test workflow validates as inlined during editing
mkdir inlined
cylc view --inline "${WORKFLOW_NAME}" > inlined/flow.cylc
-run_ok "${TEST_NAME}.2" cylc val ./inlined
+run_ok "${TEST_NAME}.2" cylc validate ./inlined
#-------------------------------------------------------------------------------
# compare inlined workflow def with reference copy
TEST_NAME=${TEST_NAME_BASE}-compare
diff --git a/tests/functional/restart/09-reload.t b/tests/functional/restart/09-reload.t
index ced8376b242..e84008bfccf 100755
--- a/tests/functional/restart/09-reload.t
+++ b/tests/functional/restart/09-reload.t
@@ -23,7 +23,7 @@ set_test_number 3
install_workflow "${TEST_NAME_BASE}" reload
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-validate"
-run_ok "${TEST_NAME}" cylc val "${WORKFLOW_NAME}"
+run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
#-------------------------------------------------------------------------------
TEST_NAME="${TEST_NAME_BASE}-run"
workflow_run_ok "${TEST_NAME}" cylc play --debug --no-detach "${WORKFLOW_NAME}"
diff --git a/tests/functional/workflow-state/00-polling.t b/tests/functional/workflow-state/00-polling.t
index b858f0767f0..f073f6e3c02 100644
--- a/tests/functional/workflow-state/00-polling.t
+++ b/tests/functional/workflow-state/00-polling.t
@@ -32,11 +32,11 @@ cylc install "${TEST_DIR}/upstream" --workflow-name="${UPSTREAM}" --no-run-name
#-------------------------------------------------------------------------------
# validate both workflows as tests
TEST_NAME="${TEST_NAME_BASE}-validate-upstream"
-run_ok "${TEST_NAME}" cylc val --debug "${UPSTREAM}"
+run_ok "${TEST_NAME}" cylc validate --debug "${UPSTREAM}"
TEST_NAME=${TEST_NAME_BASE}-validate-polling
run_ok "${TEST_NAME}" \
- cylc val --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}"
+ cylc validate --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}"
#-------------------------------------------------------------------------------
# run the upstream workflow and detach (not a test)
diff --git a/tests/functional/workflow-state/01-polling.t b/tests/functional/workflow-state/01-polling.t
index 636d39e5211..b4694c35a6c 100644
--- a/tests/functional/workflow-state/01-polling.t
+++ b/tests/functional/workflow-state/01-polling.t
@@ -32,11 +32,11 @@ cylc install "${TEST_DIR}/upstream" --workflow-name="${UPSTREAM}" --no-run-name
#-------------------------------------------------------------------------------
# validate both workflows as tests
TEST_NAME="${TEST_NAME_BASE}-validate-upstream"
-run_ok "${TEST_NAME}" cylc val --debug "${UPSTREAM}"
+run_ok "${TEST_NAME}" cylc validate --debug "${UPSTREAM}"
TEST_NAME="${TEST_NAME_BASE}-validate-polling"
run_ok "${TEST_NAME}" \
- cylc val --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}"
+ cylc validate --debug --set="UPSTREAM='${UPSTREAM}'" "${WORKFLOW_NAME}"
#-------------------------------------------------------------------------------
# check auto-generated task script for lbad
diff --git a/tests/functional/xtriggers/02-persistence.t b/tests/functional/xtriggers/02-persistence.t
index 1cc9d18ffee..2b424473f7c 100644
--- a/tests/functional/xtriggers/02-persistence.t
+++ b/tests/functional/xtriggers/02-persistence.t
@@ -36,7 +36,7 @@ mkdir -p 'lib/python'
cp "${WORKFLOW_RUN_DIR}/faker_succ.py" 'lib/python/faker.py'
# Validate the test workflow.
-run_ok "${TEST_NAME_BASE}-val" cylc val --debug "${WORKFLOW_NAME}"
+run_ok "${TEST_NAME_BASE}-val" cylc validate --debug "${WORKFLOW_NAME}"
# Run the first cycle, till auto shutdown by task.
TEST_NAME="${TEST_NAME_BASE}-run"
@@ -50,7 +50,7 @@ grep_ok 'NAME is bob' '2010.foo.out'
cp "${WORKFLOW_RUN_DIR}/faker_fail.py" 'lib/python/faker.py'
# Validate again (with the new xtrigger function).
-run_ok "${TEST_NAME_BASE}-val2" cylc val --debug "${WORKFLOW_NAME}"
+run_ok "${TEST_NAME_BASE}-val2" cylc validate --debug "${WORKFLOW_NAME}"
# Restart the workflow, to run the final cycle point.
TEST_NAME="${TEST_NAME_BASE}-restart"
diff --git a/tests/unit/test_option_parsers.py b/tests/unit/test_option_parsers.py
index dcd2f6e8977..5a5e4cbad71 100644
--- a/tests/unit/test_option_parsers.py
+++ b/tests/unit/test_option_parsers.py
@@ -14,14 +14,21 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
-import pytest
+from contextlib import redirect_stdout
+import io
+import sys
+from types import SimpleNamespace
from typing import List
-import sys
-import io
-from contextlib import redirect_stdout
+import pytest
+from pytest import param
+
import cylc.flow.flags
-from cylc.flow.option_parsers import CylcOptionParser as COP, Options
+from cylc.flow.option_parsers import (
+ CylcOptionParser as COP, Options, combine_options, combine_options_pair,
+ OptionSettings,
+ ARGS, HELP, KWARGS, SOURCES, cleanup_sysargv, USEIF
+)
USAGE_WITH_COMMENT = "usage \n # comment"
@@ -93,3 +100,411 @@ def test_Options_std_opts():
MyOptions = Options(parser)
MyValues = MyOptions(verbosity=1)
assert MyValues.verbosity == 1
+
+
+# Add overlapping args tomorrow
+@pytest.mark.parametrize(
+ 'first, second, expect',
+ [
+ param(
+ [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'do'}}],
+ [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'dont'}}],
+ (
+ [{
+ ARGS: ['-f', '--foo'], KWARGS: {},
+ SOURCES: {'do', 'dont'}, USEIF: ''
+ }]
+ ),
+ id='identical arg lists unchanged'
+ ),
+ param(
+ [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'fall'}}],
+ [{
+ ARGS: ['-f', '--foolish'],
+ KWARGS: {'help': 'not identical'},
+ SOURCES: {'fold'}}],
+ (
+ [
+ {
+ ARGS: ['--foo'], KWARGS: {}, SOURCES: {'fall'},
+ USEIF: ''
+ },
+ {
+ ARGS: ['--foolish'],
+ KWARGS: {'help': 'not identical'},
+ SOURCES: {'fold'},
+ USEIF: ''
+ }
+ ]
+ ),
+ id='different arg lists lose shared names'
+ ),
+ param(
+ [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'cook'}}],
+ [{
+ ARGS: ['-f', '--foo'],
+ KWARGS: {'help': 'not identical'},
+ SOURCES: {'bake'},
+ USEIF: ''
+ }],
+ None,
+ id='different args identical arg list cause exception'
+ ),
+ param(
+ [{ARGS: ['-g', '--goo'], KWARGS: {}, SOURCES: {'knit'}}],
+ [{ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'feed'}}],
+ [
+ {
+ ARGS: ['-g', '--goo'], KWARGS: {},
+ SOURCES: {'knit'}, USEIF: ''
+ },
+ {
+ ARGS: ['-f', '--foo'], KWARGS: {},
+ SOURCES: {'feed'}, USEIF: ''
+ },
+ ],
+ id='all unrelated args added'
+ ),
+ param(
+ [
+ {ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'work'}},
+ {ARGS: ['-r', '--redesdale'], KWARGS: {}, SOURCES: {'work'}}
+ ],
+ [
+ {ARGS: ['-f', '--foo'], KWARGS: {}, SOURCES: {'sink'}},
+ {
+ ARGS: ['-b', '--buttered-peas'],
+ KWARGS: {}, SOURCES: {'sink'}
+ }
+ ],
+ [
+ {
+ ARGS: ['-f', '--foo'],
+ KWARGS: {},
+ SOURCES: {'work', 'sink'},
+ USEIF: ''
+ },
+ {
+ ARGS: ['-b', '--buttered-peas'],
+ KWARGS: {},
+ SOURCES: {'sink'},
+ USEIF: ''
+ },
+ {
+ ARGS: ['-r', '--redesdale'],
+ KWARGS: {},
+ SOURCES: {'work'},
+ USEIF: ''
+ },
+ ],
+ id='do not repeat args'
+ ),
+ param(
+ [
+ {
+ ARGS: ['-f', '--foo'],
+ KWARGS: {},
+ SOURCES: {'push'}
+ },
+ ],
+ [],
+ [
+ {
+ ARGS: ['-f', '--foo'],
+ KWARGS: {},
+ SOURCES: {'push'},
+ USEIF: ''
+ },
+ ],
+ id='one empty list is fine'
+ )
+ ]
+)
+def test_combine_options_pair(first, second, expect):
+ """It combines sets of options"""
+ first = [
+ OptionSettings(i[ARGS], sources=i[SOURCES], **i[KWARGS])
+ for i in first
+ ]
+ second = [
+ OptionSettings(i[ARGS], sources=i[SOURCES], **i[KWARGS])
+ for i in second
+ ]
+ if expect is not None:
+ result = combine_options_pair(first, second)
+ assert [i.__dict__ for i in result] == expect
+ else:
+ with pytest.raises(Exception, match='Clashing Options'):
+ combine_options_pair(first, second)
+
+
+@pytest.mark.parametrize(
+ 'inputs, expect',
+ [
+ param(
+ [
+ ([OptionSettings(
+ ['-i', '--inflammable'], help='', sources={'wish'}
+ )]),
+ ([OptionSettings(
+ ['-f', '--flammable'], help='', sources={'rest'}
+ )]),
+ ([OptionSettings(
+ ['-n', '--non-flammable'], help='', sources={'swim'}
+ )]),
+ ],
+ [
+ {ARGS: ['-i', '--inflammable']},
+ {ARGS: ['-f', '--flammable']},
+ {ARGS: ['-n', '--non-flammable']}
+ ],
+ id='merge three argsets no overlap'
+ ),
+ param(
+ [
+ [
+ OptionSettings(
+ ['-m', '--morpeth'], help='', sources={'stop'}),
+ OptionSettings(
+ ['-r', '--redesdale'], help='', sources={'stop'}),
+ ],
+ [
+ OptionSettings(
+ ['-b', '--byker'], help='', sources={'walk'}),
+ OptionSettings(
+ ['-r', '--roxborough'], help='', sources={'walk'}),
+ ],
+ [
+ OptionSettings(
+ ['-b', '--bellingham'], help='', sources={'leap'}),
+ ]
+ ],
+ [
+ {ARGS: ['--bellingham']},
+ {ARGS: ['--roxborough']},
+ {ARGS: ['--redesdale']},
+ {ARGS: ['--byker']},
+ {ARGS: ['-m', '--morpeth']}
+ ],
+ id='merge three overlapping argsets'
+ ),
+ param(
+ [
+ ([]),
+ (
+ [
+ OptionSettings(
+ ['-c', '--campden'], help='x', sources={'foo'})
+ ]
+ )
+ ],
+ [
+ {ARGS: ['-c', '--campden']}
+ ],
+ id="empty list doesn't clear result"
+ ),
+ ]
+)
+def test_combine_options(inputs, expect):
+ """It combines multiple input sets"""
+ result = combine_options(*inputs)
+ result_args = [i.args for i in result]
+
+ # Order of args irrelevent to test
+ for option in expect:
+ assert option[ARGS] in result_args
+
+
+@pytest.mark.parametrize(
+ 'argv_before, kwargs, expect',
+ [
+ param(
+ 'vip myworkflow --foo something'.split(),
+ {
+ 'script_name': 'play',
+ 'workflow_id': 'myworkflow',
+ 'compound_script_opts': [
+ OptionSettings(['--foo', '-f']),
+ ],
+ 'script_opts': [
+ OptionSettings(['--foo', '-f'])]
+ },
+ 'play myworkflow --foo something'.split(),
+ id='no opts to remove'
+ ),
+ param(
+ 'vip myworkflow -f something -b something_else --baz'.split(),
+ {
+ 'script_name': 'play',
+ 'workflow_id': 'myworkflow',
+ 'compound_script_opts': [
+ OptionSettings(['--foo', '-f']),
+ OptionSettings(['--bar', '-b'], action='store'),
+ OptionSettings(['--baz'], action='store_true'),
+ ],
+ 'script_opts': [
+ OptionSettings(['--foo', '-f']),
+ ]
+ },
+ 'play myworkflow -f something'.split(),
+ id='remove some opts'
+ ),
+ param(
+ 'vip myworkflow'.split(),
+ {
+ 'script_name': 'play',
+ 'workflow_id': 'myworkflow',
+ 'compound_script_opts': [
+ OptionSettings(['--foo', '-f']),
+ OptionSettings(['--bar', '-b']),
+ OptionSettings(['--baz']),
+ ],
+ 'script_opts': []
+ },
+ 'play myworkflow'.split(),
+ id='no opts to keep'
+ ),
+ param(
+ 'vip ./myworkflow --foo something'.split(),
+ {
+ 'script_name': 'play',
+ 'workflow_id': 'myworkflow',
+ 'compound_script_opts': [
+ OptionSettings(['--foo', '-f'])],
+ 'script_opts': [
+ OptionSettings(['--foo', '-f']),
+ ],
+ 'source': './myworkflow',
+ },
+ 'play --foo something myworkflow'.split(),
+ id='replace path'
+ ),
+ ]
+)
+def test_cleanup_sysargv(monkeypatch, argv_before, kwargs, expect):
+ """It replaces the contents of sysargv with Cylc Play argv items.
+ """
+ # Fake up sys.argv: for this test.
+ dummy_cylc_path = ['/pathto/my/cylc/bin/cylc']
+ monkeypatch.setattr(sys, 'argv', dummy_cylc_path + argv_before)
+ # Fake options too:
+ opts = SimpleNamespace(**{
+ i.args[0].replace('--', ''): i for i in kwargs['compound_script_opts']
+ })
+
+ kwargs.update({'options': opts})
+ if not kwargs.get('source', None):
+ kwargs.update({'source': ''})
+
+ # Test the script:
+ cleanup_sysargv(**kwargs)
+ assert sys.argv == dummy_cylc_path + expect
+
+
+class TestOptionSettings():
+ @staticmethod
+ def test_init():
+ args = ['--foo', '-f']
+ kwargs = {'bar': 42}
+ sources = {'touch'}
+ useif = 'hello'
+
+ result = OptionSettings(
+ args, sources=sources, useif=useif, **kwargs)
+
+ assert result.__dict__ == {
+ 'kwargs': kwargs, 'sources': sources,
+ 'useif': useif, 'args': args
+ }
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'first, second, expect',
+ (
+ param(
+ (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'),
+ (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'),
+ True, id='Totally the same'),
+ param(
+ (['--foo', '-f'], {'bar': 42}, {'touch'}, 'hello'),
+ (['--foo', '-f'], {'bar': 42}, {'wibble'}, 'byee'),
+ True, id='Differing extras'),
+ param(
+ (['-f'], {'bar': 42}, {'touch'}, 'hello'),
+ (['--foo', '-f'], {'bar': 42}, {'wibble'}, 'byee'),
+ False, id='Not equal args'),
+ )
+ )
+ def test___eq__args_intersection(first, second, expect):
+ args, kwargs, sources, useif = first
+ first = OptionSettings(
+ args, sources=sources, useif=useif, **kwargs)
+ args, kwargs, sources, useif = second
+ second = OptionSettings(
+ args, sources=sources, useif=useif, **kwargs)
+ assert (first == second) == expect
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'first, second, expect',
+ (
+ param(
+ ['--foo', '-f'],
+ ['--foo', '-f'],
+ ['--foo', '-f'],
+ id='Totally the same'),
+ param(
+ ['--foo', '-f'],
+ ['--foolish', '-f'],
+ ['-f'],
+ id='Some overlap'),
+ param(
+ ['--foo', '-f'],
+ ['--bar', '-b'],
+ [],
+ id='No overlap'),
+ )
+ )
+ def test___and__(first, second, expect):
+ first = OptionSettings(first)
+ second = OptionSettings(second)
+ assert sorted(first & second) == sorted(expect)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ 'first, second, expect',
+ (
+ param(
+ ['--foo', '-f'],
+ ['--foo', '-f'],
+ [],
+ id='Totally the same'),
+ param(
+ ['--foo', '-f'],
+ ['--foolish', '-f'],
+ ['--foo'],
+ id='Some overlap'),
+ param(
+ ['--foolish', '-f'],
+ ['--foo', '-f'],
+ ['--foolish'],
+ id='Some overlap not commuting'),
+ param(
+ ['--foo', '-f'],
+ ['--bar', '-b'],
+ ['--foo', '-f'],
+ id='No overlap'),
+ )
+ )
+ def test___sub__args_subtraction(first, second, expect):
+ first = OptionSettings(first)
+ second = OptionSettings(second)
+ assert sorted(first - second) == sorted(expect)
+
+ @staticmethod
+ def test__in_list():
+ """It is in a list."""
+ first = OptionSettings(['--foo'])
+ second = OptionSettings(['--foo'])
+ third = OptionSettings(['--bar'])
+ assert first._in_list([second, third]) == True