diff --git a/.bandit b/.bandit new file mode 100644 index 00000000000..96b0e563021 --- /dev/null +++ b/.bandit @@ -0,0 +1,20 @@ +# NOTE: At present Bandit does *not* automatically source the .bandit +# configuration file, invoke like so: +# $ bandit -r --ini .bandit +# For development it may be convenient to use flake8-bandit. + +[bandit] +targets: cylc/flow +skips: B404,B607 +# B404: import_subprocess - consider security implications +# * Ignored as the use of subprocesses is a project wide decision. +# * The purpose of cylc is to run user defined code therefore the use +# of subprocesses is unavoidable. +# B607: Starting a process with a partial executable path +# * Ignored as Cylc needs to be able to call out to executables which +# may have to be installed into environments so we cannot specify +# absolute paths. In some cases this may be required for portability. +# * Users can modify their $PATH to get Cylc to call malicious scripts, +# however, the purpose of Cylc Flow is to run user-defined code making +# this a moot point, note all code is run as the user. Cylc Flow does +# *not* provide multi-user functionality. diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 00000000000..0c37c0c2898 --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,4 @@ +exclude_paths: + - 'etc/**' + - 'tests/**' + - 'cylc/flow/**_pb2.py' diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 00000000000..5b3d02564ba --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,39 @@ +# Codecov settings +# After modifying this file, it might be worth to validate it with: +# `curl --data-binary @.codecov.yml https://codecov.io/validate` + +codecov: + notify: + require_ci_to_pass: yes + +# define the colour bar limits here +coverage: + precision: 2 + round: down + range: "75...100" + + # diff type + status: + project: + default: + # commits below this threshold will be marked as failed + target: '92%' + # how much we allow the coverage to drop + threshold: '2%' + patch: + default: + target: '97%' + threshold: '5%' + +# files to ignore +ignore: + - "tests/**" + - "ws_messages_pb2.py" + - "cylc/flow/scripts/report_timings.py" + +flag_management: + default_rules: + carryforward: true + +# turn off comments to pull requests +comment: false diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000000..5a4b396f19c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,59 @@ +# This is the Coverage.py configuration file. This is used by CI when running +# the tests and collecting coverage + +[run] +branch = True +cover_pylib = False +concurrency = thread +data_file = .coverage +disable_warnings = + trace-changed + module-not-python + module-not-imported + no-data-collected + module-not-measured +omit = + tests/* + */cylc/flow/*_pb2.py + cylc/flow/etc/* + cylc/flow/scripts/report_timings.py +parallel = True +source = ./cylc +timid = False + +[report] +exclude_lines = + pragma: no cover + + # Don't complain if tests don't hit defensive assertion code: + raise NotImplementedError + return NotImplemented + + # Ignore type checking code: + if (typing\.)?TYPE_CHECKING: + @overload( |$) + + # Don't complain about ellipsis (exception classes, typing overloads etc): + \.\.\. + + # Ignore abstract methods + @(abc\.)?abstractmethod + +fail_under=0 +ignore_errors = False +omit = + tests/* + */cylc/flow/*_pb2.py + cylc/flow/etc/* +precision = 2 +show_missing = False +skip_covered = False +sort = Name + +[html] +directory = htmlcov + + +[xml] +output = coverage.xml +package_depth = 99 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..25a8d672456 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +* +*.py[cod] +__pycache__ +!.coveragerc +!.docker* +!README.md +!bin +!conda-environment.yml +!cylc +!dockerfiles/* +!setup* +!MANIFEST.in diff --git a/.github/bin/mail b/.github/bin/mail new file mode 100755 index 00000000000..0b7f63446cc --- /dev/null +++ b/.github/bin/mail @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +"""Mock the mailx / mailutils `mail` command""" + +import argparse +import smtplib +import sys +import os + +parser = argparse.ArgumentParser( + description='Cylc functional tests mail command.') +parser.add_argument( + '-s', metavar='subject', dest='subject', type=str, help='e-mail subject') +parser.add_argument( + '-r', metavar='reply_to', dest='sender', type=str, + help='e-mail reply-to address') +parser.add_argument( + 'to', metavar='to', type=str, help='e-mail destination address') +parser.add_argument( + 'body', metavar='body', nargs='?', type=argparse.FileType('r'), + default=sys.stdin, help='e-mail body') + +smtp_server = os.getenv('smtp') +try: + host, port = smtp_server.split(':') + port = int(port) +except (AttributeError, ValueError): + raise OSError( + 'The environment variable "smtp" must be set to a string of the form ' + '"host:port" (e.g. "localhost:8025") for the mocked mail command ' + 'to work.') + +args = parser.parse_args() + +sender_email = args.sender +receiver_email = args.to +message = f"""\ +Subject: {args.subject} + +{args.body.read()}""" + +# https://realpython.com/python-send-email/ +with smtplib.SMTP(host, port) as server: + server.sendmail(sender_email, receiver_email, message) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..73a0df46612 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + labels: + - 'dependencies' + - 'infrastructure' diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml new file mode 100644 index 00000000000..10a700b7ab4 --- /dev/null +++ b/.github/workflows/1_create_release_pr.yml @@ -0,0 +1,59 @@ +name: Release stage 1 - create release PR (Cylc 8+ only) + +on: + workflow_dispatch: + inputs: + version: + description: Version number (PEP 440 compliant e.g., 8.12.4, 8.0a2 etc) + required: true + branch: + description: The branch to open the PR against + required: false + default: 'master' + +jobs: + create-release-pr: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + + - name: Sanitise workflow inputs # Should be 1st step + uses: cylc/release-actions/stage-1/sanitize-inputs@v1 + + - name: Checkout repo + uses: actions/checkout@v4 + with: + ref: ${{ env.BASE_REF }} + fetch-depth: 0 # need to fetch all commits to check contributors + + - name: Check CONTRIBUTING.md + uses: cylc/release-actions/check-shortlog@v1 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Create & checkout PR branch + uses: cylc/release-actions/stage-1/checkout-pr-branch@v1 + + - name: Set the package version + uses: cylc/release-actions/stage-1/set-python-package-version@v1 + with: + init-file: 'cylc/flow/__init__.py' + pypi-package-name: 'cylc-flow' + + - name: Test build + uses: cylc/release-actions/build-python-package@v1 + + - name: Generate changelog + run: | + python3 -m pip install -q towncrier + towncrier build --yes + + - name: Create pull request + uses: cylc/release-actions/stage-1/create-release-pr@v1 + with: + test-workflows: test_fast.yml, test_functional.yml, bash.yml, test_manylinux.yml, test_conda-build.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml new file mode 100644 index 00000000000..15b4a590a69 --- /dev/null +++ b/.github/workflows/2_auto_publish_release.yml @@ -0,0 +1,78 @@ +name: Release stage 2 - auto publish + +on: + pull_request: # types AND paths + types: [closed] + paths: ['cylc/flow/__init__.py'] + # NOTE: While this is too generic, we use the `if` condition of the job to narrow it down + # NOTE: Don't use `branches` as we might create release on any branch + +env: + # Best not to include the GH token here, only do it for the steps that need it + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + CHANGELOG_FILE: CHANGES.md + +jobs: + publish: + if: >- # NOTE: Can't use top-level env here unfortunately + github.event.pull_request.merged == true && + contains(github.event.pull_request.labels.*.name, 'release') + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + + - name: Checkout repo + uses: actions/checkout@v4 + with: + ref: ${{ env.MERGE_SHA }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Get the version number + uses: cylc/release-actions/stage-2/get-version-from-pr@v1 + + - name: Build + uses: cylc/release-actions/build-python-package@v1 + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@v1.10.1 + with: + user: __token__ # uses the API token feature of PyPI - least permissions possible + password: ${{ secrets.PYPI_TOKEN }} + # # Can try using this for testing: + # repository_url: https://test.pypi.org/legacy/ + + - name: Publish GitHub release + id: create-release + uses: cylc/release-actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + commitish: ${{ env.MERGE_SHA }} + tag_name: ${{ env.VERSION }} + release_name: cylc-flow-${{ env.VERSION }} + prerelease: ${{ env.PRERELEASE }} + body: | + See [${{ env.CHANGELOG_FILE }}](https://github.com/${{ github.repository }}/blob/master/${{ env.CHANGELOG_FILE }}) for detail. + + Cylc 8 can be installed via pypi or Conda - you don't need to download this release directly. + See the [installation](https://cylc.github.io/cylc-doc/latest/html/installation.html) section of the documentation. + # TODO: Get topmost changelog section somehow and use that as the body? + + - name: Comment on the release PR with the results & next steps + if: always() + uses: cylc/release-actions/stage-2/comment-on-pr@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + release-url: ${{ steps.create-release.outputs.html_url }} + + - name: Bump dev version + uses: cylc/release-actions/stage-2/bump-dev-version@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + init-file: 'cylc/flow/__init__.py' diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml new file mode 100644 index 00000000000..9938c4c4ec1 --- /dev/null +++ b/.github/workflows/bash.yml @@ -0,0 +1,92 @@ +# Workflow for testing multiple bash versions with Docker +name: bash + +on: + workflow_dispatch: + pull_request: + paths-ignore: + - '.github/workflows/*.ya?ml' + - '!.github/workflows/bash.yml' + - 'cylc/flow/etc/syntax/**' + - 'etc/syntax/**' + - 'tests/unit/**' + - 'tests/integration/**' + - '**.md' + - '**/README*/**' + push: + branches: + - master + - '8.*.x' + paths-ignore: + - '.github/workflows/*.ya?ml' + - '!.github/workflows/bash.yml' + - 'cylc/flow/etc/syntax/**' + - 'etc/syntax/**' + - 'tests/unit/**' + - 'tests/integration/**' + - '**.md' + - '**/README*/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + bash-docker: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + matrix: + bash-version: + - '3.2' + - '4.2' + # if you use a value like 5.0, YAML will round it to 5, which will cause an error later on + - '5.0' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run Docker container + run: | + docker pull ghcr.io/cylc/cylc-bash-testing:master + docker run --name bash -v $(pwd -P):/root/cylc-flow --rm -t -d -m 1G --memory-swap 1G ghcr.io/cylc/cylc-bash-testing:master /bin/bash + docker ps -a + + - name: Install Cylc dependencies in the container + run: | + docker exec bash python3.7 -m pip install six==1.12 + docker exec -w /root/cylc-flow bash python3.7 -m pip install .[all] + + - name: Set the container bash version + run: docker exec bash update-alternatives --set bash /bash/bash-${{ matrix.bash-version }} + + - name: Run functional tests that validate or are related to how Cylc uses bash + run: | + docker exec -w /root/cylc-flow bash \ + ./etc/bin/run-functional-tests -v \ + tests/functional/broadcast/00-simple.t \ + tests/functional/cylc-poll/11-event-time.t \ + tests/functional/cylc-poll/15-job-st-file-no-batch.t \ + tests/functional/events/28-inactivity.t \ + tests/functional/events/34-task-abort.t \ + tests/functional/job-file-trap/00-sigusr1.t \ + tests/functional/job-file-trap/02-pipefail.t \ + tests/functional/job-file-trap/03-user-trap.t \ + tests/functional/jobscript/00-torture.t \ + tests/functional/pause-resume/12-pause-then-retry.t \ + tests/functional/shutdown/09-now2.t \ + tests/functional/shutdown/13-no-port-file-check.t \ + tests/functional/shutdown/14-no-dir-check.t + + - name: Copy cylc-run out of container + if: failure() + run: | + docker cp bash:/root/cylc-run . + + - name: Upload artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: 'cylc-run (bash-${{ matrix.bash-version }})' + path: cylc-run + include-hidden-files: true diff --git a/.github/workflows/branch_sync.yml b/.github/workflows/branch_sync.yml new file mode 100644 index 00000000000..92475094217 --- /dev/null +++ b/.github/workflows/branch_sync.yml @@ -0,0 +1,20 @@ +name: Sync PR + +on: + push: + branches: + - '8.*.x' + schedule: + - cron: '33 04 * * 1-5' # 04:33 UTC Mon-Fri + workflow_dispatch: + inputs: + head_branch: + description: Branch to merge into master + required: true + +jobs: + sync: + uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1 + with: + head_branch: ${{ inputs.head_branch }} + secrets: inherit diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..674542d4e63 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,43 @@ +name: Build + +# build the project whenever the configuration is changed + +on: + workflow_dispatch: + pull_request: + paths: + - 'README.md' # check markdown is valid + - 'MANIFEST.in' # check packaging + - 'pyproject.toml' # check build config + - 'setup.cfg' # check deps and project config + +jobs: + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + matrix: + os: ['ubuntu-latest'] + python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + include: + - os: 'macos-latest' + python: '3.8' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Build + uses: cylc/release-actions/build-python-package@v1 + + - name: Inspect + run: | + unzip -l dist/*.whl | tee files + grep 'cylc_flow.*.dist-info/COPYING' files + grep 'cylc/flow/py.typed' files + grep 'cylc/flow/etc' files + grep 'cylc/flow/etc/cylc-completion.bash' files diff --git a/.github/workflows/shortlog.yml b/.github/workflows/shortlog.yml new file mode 100644 index 00000000000..25e2b0427e3 --- /dev/null +++ b/.github/workflows/shortlog.yml @@ -0,0 +1,22 @@ +name: check contributor list + +on: + workflow_dispatch: + pull_request: + paths: + - 'CONTRIBUTING.md' + - '.github/workflows/shortlog.yml' + - '.mailmap' + +jobs: + test: + runs-on: 'ubuntu-latest' + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 # need to fetch all commits to check contributors + + - name: Check CONTRIBUTING.md + uses: cylc/release-actions/check-shortlog@v1 diff --git a/.github/workflows/test_conda-build.yml b/.github/workflows/test_conda-build.yml new file mode 100644 index 00000000000..91d1ac2ad6b --- /dev/null +++ b/.github/workflows/test_conda-build.yml @@ -0,0 +1,53 @@ +name: conda builds + +on: + pull_request: + paths: + - 'conda-environment.yml' + - '.github/workflows/test_conda-build.yml' + schedule: + - cron: '17 22 * * 6' # Every Saturday at 22:17 + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test_conda_install: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: checkout cylc-flow + uses: actions/checkout@v4 + + - name: build conda env + run: | + # write environment file + env_file='conda-environment.yml' + echo " - pip" >> "$env_file" # list pip as a dependency + echo " - pip:" >> "$env_file" # add a pip section + echo " - ." >> "$env_file" # install cylc-flow (pip install .) + cat "$env_file" + # install environment + conda env create \ + -f "$env_file" \ + --prefix cylc-dev + . /usr/share/miniconda/etc/profile.d/conda.sh + # check cylc-flow was installed correctly + conda run --prefix cylc-dev cylc version --long + + - name: check for activate scripts + run: | + # https://github.com/cylc/cylc-flow/issues/3704#issuecomment-897442365 + # locate all activate scripts + find ./cylc-dev/ -name "activate.d" | tee > activates.txt + # ignore the conda activate script itself + sed -i '/cylc-dev\/etc\/conda\/activate.d/d' activates.txt + # check to make sure no packages have contributed new activate scripts + # (we rely on having a conda activate-less environment) + if [[ $(cat activates.txt | wc -l) -ne 0 ]]; then + echo '::error::Found activate scripts in installation.' + cat activates.txt >&2 + exit 1 + fi diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml new file mode 100644 index 00000000000..b58bc50ed99 --- /dev/null +++ b/.github/workflows/test_fast.yml @@ -0,0 +1,154 @@ +name: fast tests + +on: + pull_request: + workflow_dispatch: + push: + branches: + - master + - '8.*.x' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false # don't stop on first failure + matrix: + os: ['ubuntu-latest'] + python-version: ['3.7', '3.8', '3.10', '3.11', '3'] + include: + # mac os test + - os: 'macos-latest' + python-version: '3.9' # oldest supported version + # non-utc timezone test + - os: 'ubuntu-latest' + python-version: '3.9' # not the oldest, not the most recent version + time-zone: 'XXX-09:35' + + env: + TZ: ${{ matrix.time-zone }} + PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Apt-Get Install + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y sqlite3 + + - name: Patch DNS + uses: cylc/release-actions/patch-dns@v1 + + - name: Install + run: | + pip install -e ."[all]" + + - name: Configure git # Needed by the odd test + uses: cylc/release-actions/configure-git@v1 + + - name: Unit Tests + timeout-minutes: 5 + run: | + pytest cylc/flow tests/unit + + - name: Integration Tests + timeout-minutes: 6 + run: | + pytest tests/integration + + - name: Upload failed tests artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cylc-run (${{ matrix.os }} py-${{ matrix.python-version }}) + path: ~/cylc-run/ + include-hidden-files: true + + - name: Coverage report + run: | + coverage xml + coverage report + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }} + path: coverage.xml + retention-days: 7 + + lint: + runs-on: 'ubuntu-latest' + timeout-minutes: 10 + steps: + - name: Apt-Get Install + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + + - name: Checkout + uses: actions/checkout@v4 + + # note: exclude python 3.10+ from mypy checks as these produce false + # positives in installed libraries for python 3.7 + - name: Configure Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install + run: | + pip install -e ."[tests]" + + - name: Flake8 + run: flake8 + + - name: Bandit + run: | + bandit -r --ini .bandit cylc/flow + + - name: Shellchecker + run: etc/bin/shellchecker + + - name: MyPy + run: mypy + + - name: Towncrier - draft changelog + uses: cylc/release-actions/towncrier-draft@v1 + + - name: Linkcheck + run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py + + codecov: + needs: test + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + + - name: Codecov upload + uses: codecov/codecov-action@v4 + with: + name: ${{ github.workflow }} + flags: fast-tests + fail_ci_if_error: true + verbose: true + # Token not required for public repos, but avoids upload failure due + # to rate-limiting (but not for PRs opened from forks) + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml new file mode 100644 index 00000000000..f055353d904 --- /dev/null +++ b/.github/workflows/test_functional.yml @@ -0,0 +1,346 @@ +name: functional tests + +on: + workflow_dispatch: + pull_request: + paths-ignore: + - '.github/workflows/*.ya?ml' + - '!.github/workflows/test_functional.yml' + - 'cylc/flow/etc/syntax/**' + - 'etc/syntax/**' + - 'tests/conftest.py' + - 'tests/unit/**' + - 'tests/integration/**' + - '**.md' + - '**/README*/**' + push: + branches: + - master + - '8.*.x' + paths-ignore: + - '.github/workflows/*.ya?ml' + - '!.github/workflows/test_functional.yml' + - 'cylc/flow/etc/syntax/**' + - 'etc/syntax/**' + - 'tests/conftest.py' + - 'tests/unit/**' + - 'tests/integration/**' + - '**.md' + - '**/README*/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ${{ matrix.os }} + name: ${{ matrix.name || matrix.chunk }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest'] + python-version: ['3.7'] + test-base: ['tests/f'] + chunk: ['1/4', '2/4', '3/4', '4/4'] + platform: ['_local_background* _local_at*'] + # NOTE: includes must define ALL of the matrix values + include: + # latest python + - name: 'py-3-latest' + os: 'ubuntu-latest' + python-version: '3' + test-base: 'tests/f' + chunk: '1/4' + platform: '_local_background*' + # tests/k + - name: 'flaky' + os: 'ubuntu-latest' + python-version: '3.7' + test-base: 'tests/k' + chunk: '1/1' + platform: '_local_background* _local_at*' + # remote platforms + - name: '_remote_background_indep_poll' + os: 'ubuntu-latest' + python-version: '3.7' + test-base: 'tests/f tests/k' + chunk: '1/1' + platform: '_remote_background_indep_poll _remote_at_indep_poll' + - name: '_remote_background_indep_tcp' + os: 'ubuntu-latest' + test-base: 'tests/f tests/k' + python-version: '3.7' + chunk: '1/1' + platform: '_remote_background_indep_tcp _remote_at_indep_tcp' + # macos + - name: 'macos 1/5' + os: 'macos-latest' + python-version: '3.9' + test-base: 'tests/f' + chunk: '1/5' + platform: '_local_background*' + - name: 'macos 2/5' + os: 'macos-latest' + python-version: '3.9' + test-base: 'tests/f' + chunk: '2/5' + platform: '_local_background*' + + env: + # Use non-UTC time zone + TZ: XXX-05:30 + # these vars are used by etc/bin/run-functional-tests + CYLC_TEST_PLATFORMS: ${{ matrix.platform }} + CYLC_COVERAGE: 1 + REMOTE_PLATFORM: ${{ contains(matrix.platform, '_remote') }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Create global config + run: | + CONF_PATH="$HOME/.cylc/flow/8" + mkdir -p "$CONF_PATH" + touch "$CONF_PATH/global.cylc" + ln -s "$CONF_PATH/global.cylc" "$CONF_PATH/global-tests.cylc" + echo "GLOBAL_CFG_PATH=${CONF_PATH}/global.cylc" >> "$GITHUB_ENV" + + - name: Patch DNS + uses: cylc/release-actions/patch-dns@v1 + + - name: Add localhost entries to global config + if: startsWith(runner.os, 'macos') + run: | + cat >> "$GLOBAL_CFG_PATH" <<__HERE__ + [platforms] + [[localhost, $(hostname -f), $(hostname -s)]] + hosts = localhost + install target = localhost + ssh command = ssh -oBatchMode=yes -oConnectTimeout=8 -oStrictHostKeyChecking=no + __HERE__ + cat "$GLOBAL_CFG_PATH" + + - name: Brew Install + if: startsWith(matrix.os, 'macos') + run: | + # install system deps + brew update + brew install bash coreutils gnu-sed grep + + # add GNU coreutils and sed to the user PATH + # (see instructions in brew install output) + echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" >> "${GITHUB_PATH}" + echo "$(brew --prefix)/opt/grep/libexec/gnubin" >> "${GITHUB_PATH}" + echo "$(brew --prefix)/opt/gnu-sed/libexec/gnubin" >> "${GITHUB_PATH}" + + # add coreutils to the bashrc too (for jobs) + cat >> "${HOME}/.bashrc" <<__HERE__ + PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:$PATH" + PATH="$(brew --prefix)/opt/grep/libexec/gnubin:$PATH" + PATH="$(brew --prefix)/opt/gnu-sed/libexec/gnubin:$PATH" + export PATH + __HERE__ + + - name: Apt-Get Install + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y sqlite3 tree at + + - name: Add .github/bin/ to PATH + # Sets up mocked mail command & any other custom executables + run: echo "${{ github.workspace }}/.github/bin" >> $GITHUB_PATH + + - name: Install + run: | + pip install -e ."[all]" + mkdir "$HOME/cylc-run" + + - name: Configure Atrun + if: contains(matrix.platform, '_local_at') + run: | + cat >> "$GLOBAL_CFG_PATH" << __HERE__ + [platforms] + [[_local_at_indep_tcp]] + hosts = localhost + install target = localhost + job runner = at + __HERE__ + + - name: Swarm Configure + run: | + etc/bin/swarm --yes --debug configure + + - name: Swarm Build + if: env.REMOTE_PLATFORM == 'true' + run: | + # `swarm configure` seems to get ignored so override the user config + cp etc/conf/ssh_config $HOME/.ssh/config + # build and run the swarm + etc/bin/swarm --yes --debug build + etc/bin/swarm --yes --debug run + # test that it's up and running before proceeding + sleep 1 + ssh -vv _remote_background_indep_poll hostname + + - name: Configure git # Needed by the odd test + uses: cylc/release-actions/configure-git@v1 + + - name: Filter Tests + env: + # NOTE: we only want the CHUNK set in this step else we will + # re-chunk tests later when they run + CHUNK: ${{ matrix.chunk }} + run: | + etc/bin/run-functional-tests \ + --dry \ + ${{ matrix.test-base }} \ + > test-file + if [[ $REMOTE_PLATFORM == 'true' ]]; then + # skip tests that don't configure platform requirements + grep -l --color=never REQUIRE_PLATFORM $(cat test-file) > test-file + fi + + - name: Test + id: test + timeout-minutes: 35 + continue-on-error: true + run: | + echo "finished=false" >> $GITHUB_OUTPUT + if [[ '${{ matrix.test-base }}' == 'tests/k' ]]; then + NPROC=4 + else + NPROC=8 + fi + # NOTE: test base is purposefully un-quoted + etc/bin/run-functional-tests \ + -j "${NPROC}" \ + --state=save \ + $(cat test-file) \ + || (echo "finished=true" >> $GITHUB_OUTPUT && false) + + - name: Time Out + if: steps.test.outcome == 'failure' && steps.test.outputs.finished != 'true' + run: | + echo '::error:: tests timed-out' + # help to identify the tests that were running at the time + cylc scan --state=all --format=rich --color-blind + # fail the workflow + false + + - name: Re-run failed tests + timeout-minutes: 10 + if: steps.test.outcome == 'failure' && steps.test.outputs.finished == 'true' + run: | + # re-run failed tests providing that they didn't time out first time + # TODO: make the tests deterministic so we don't need to do this + etc/bin/run-functional-tests \ + -j 1 \ + -v \ + --state=save,failed $(cat test-file) + + - name: Copy cylc-run out of container + if: failure() && steps.test.outcome == 'failure' && env.REMOTE_PLATFORM == 'true' + run: | + # pick the first host in the list + host="$(cut -d ' ' -f 1 <<< "${{ matrix.platform }}")" + # copy back the remote cylc-run dir + rsync -av \ + "${host}:/root/cylc-run/" \ + "${HOME}/cylc-run/${host}/" + + - name: Debug + if: failure() && steps.test.outcome == 'failure' + timeout-minutes: 1 + run: | + find "$HOME/cylc-run" -name '*.err' -type f \ + -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \; + find "$HOME/cylc-run" -name '*.log' -type f \ + -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \; + find "${TMPDIR:-/tmp}/${USER}/cylctb-"* -type f \ + -exec echo \; -exec echo '====== {} ======' \; -exec cat '{}' \; + + - name: Set artifact upload name + if: always() + id: uploadname + run: | + # artifact name cannot contain '/' characters + CID="$(sed 's|/|-|g' <<< "${{ matrix.name || matrix.chunk }}")" + echo "uploadname=$CID" >> $GITHUB_OUTPUT + + - name: Upload failed tests artifact + if: failure() && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: cylc-run (${{ steps.uploadname.outputs.uploadname }}) + path: ~/cylc-run/ + include-hidden-files: true + + - name: Fetch Remote Coverage + if: env.REMOTE_PLATFORM == 'true' + run: | + # pick the first host in the list + host="$(cut -d ' ' -f 1 <<< "${{ matrix.platform }}")" + # copy back the remote coverage files + rsync -av \ + "${host}:/cylc/" \ + '.' \ + --include='.coverage*' \ + --exclude='*' \ + >rsyncout + cat rsyncout + # fiddle the python source location to match the local system + for db in $(grep --color=never '.coverage\.' rsyncout); do + sqlite3 "$db" " + UPDATE file + SET path = REPLACE(path, '/cylc/cylc/', '$PWD/cylc/') + " + done + + - name: Shutdown + if: always() + run: | + etc/bin/swarm kill + + - name: Combine coverage & report + run: | + coverage combine -a + coverage xml + coverage report + + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: coverage_${{ steps.uploadname.outputs.uploadname }} + path: coverage.xml + retention-days: 7 + + codecov: + needs: test + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + + - name: Codecov upload + uses: codecov/codecov-action@v4 + with: + name: ${{ github.workflow }} + flags: functional-tests + fail_ci_if_error: true + verbose: true + # Token not required for public repos, but avoids upload failure due + # to rate-limiting (but not for PRs opened from forks) + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test_manylinux.yml b/.github/workflows/test_manylinux.yml new file mode 100644 index 00000000000..84e1e041283 --- /dev/null +++ b/.github/workflows/test_manylinux.yml @@ -0,0 +1,89 @@ +name: manylinux-compatibility + +# Ensure that cylc-flow remains installable with manylinx1 binaries for use +# with older systems. + +# Ideally we would use the Docker images from PyPa +# (https://github.com/pypa/manylinux). +# The manylinux1 image is based on CentOS=5, glibc=2.5), however, this is so +# old that installing contemporary Python on them is quite hard: +# * native package manager has no package for CentOS5 +# * miniforge can't work with glibc < 2.12 +# * micromamba can't work with glibc < 2.6 +# * need zlib to compile Python / install pip but the package manager can't +# download it because the package stream is offline (past EOL) + +# So this will have to do. This isn't a perfect test as non manylinux compat +# packages may attempt to compile from source. This might succeed here but not +# on an older host. + +on: + workflow_dispatch: + pull_request: + paths: + - 'setup.py' + - 'setup.cfg' + - '.github/workflows/test_manylinux.yml' + +jobs: + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + manylinux: ['1'] + os: ['ubuntu-20.04'] # run on the oldest linux we have access to + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure git # Needed by the odd test + uses: cylc/release-actions/configure-git@v1 + + - name: Configure Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Downgrade PIP + run: | + # pip v22+ will fail on error rather than falling back to the + # previous release + # the old behaviour is better for this check + if (( $(pip --version | sed 's/pip \([0-9]*\).*/\1/') >= 22 )) + then + pip install 'pip<22' + fi + + - name: Configure Manylinux Compatibility + # Make this platform look older than it is. For info see: + # https://stackoverflow.com/questions/37231799/ + # exclude-manylinux-wheels-when-downloading-from-pip + run: | + cat > _manylinux.py <<__HERE__ + manylinux1_compatible = False + manylinux2010_compatible = False + manylinux2014_compatible = False + __HERE__ + echo \ + "manylinux${{ matrix.manylinux }}_compatible = True" \ + >> _manylinux.py + + - name: Install + timeout-minutes: 35 + run: | + PYTHONPATH="$PWD:$PYTHONPATH" pip install ."[all]" + + - name: Test Import + shell: python + run: | + import cylc.flow + import cylc.flow.scheduler + + - name: Test + timeout-minutes: 5 + run: | + pytest -n 5 diff --git a/.github/workflows/test_tutorial_workflow.yml b/.github/workflows/test_tutorial_workflow.yml new file mode 100644 index 00000000000..7859b8588e2 --- /dev/null +++ b/.github/workflows/test_tutorial_workflow.yml @@ -0,0 +1,51 @@ +name: test-tutorial-workflow + +on: + push: + branches: + - master + - '8.*.x' + pull_request: + paths-ignore: + - '.github/workflows/*.ya?ml' + - '!.github/workflows/test_tutorial_workflow.yml' + - 'tests/**' + - '**.md' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + strategy: + matrix: + python-version: ['3.7', '3'] + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: configure python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: checkout + uses: actions/checkout@v4 + + - name: Install docs/tutorial dependencies + uses: cylc/cylc-doc/.github/actions/install-dependencies@master + + - name: install cylc-flow + run: pip install .[all] + + - name: run tutorial workflow + timeout-minutes: 6 + run: | + mkdir -p "${HOME}/cylc-run" + etc/bin/run-validate-tutorials + + - name: debug + if: failure() + run: | + find ~/cylc-run -name job.err -exec cat {} + # cylc error files diff --git a/.github/workflows/update_copyright.yml b/.github/workflows/update_copyright.yml new file mode 100644 index 00000000000..878412a1a27 --- /dev/null +++ b/.github/workflows/update_copyright.yml @@ -0,0 +1,46 @@ +name: Update copyright year + +on: + schedule: + - cron: '0 4 1 1 *' # Every Jan 1st @ 04:00 UTC + +jobs: + update-copyright: + if: github.repository_owner == 'cylc' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Configure git + uses: cylc/release-actions/configure-git@v1 + + - name: Checkout PR branch + uses: cylc/release-actions/checkout-copyright-branch@v1 + + - name: Update copyright year + env: + README_FILE: 'README.md' + SCRIPTS_INIT_FILE: 'cylc/flow/scripts/__init__.py' + run: | + pattern="().*(<\/span>)" + sed -i -E "s/${pattern}/\1${YEAR}\2/" "$README_FILE" + + pattern="(_copyright_year ?= ?)[^ #]*(.*)" + sed -i -E "s/${pattern}/\1${YEAR}\2/" "$SCRIPTS_INIT_FILE" + + - name: Commit & push + run : | + git commit -a -m "Update copyright year" -m "Workflow: ${{ github.workflow }}, run: ${{ github.run_number }}" + git push + + - name: Create pull request + uses: cylc/release-actions/create-pr@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + head: ${{ env.BRANCH_NAME }} + title: 'Auto PR: update copyright year' + body: 'Happy new year!' diff --git a/.gitignore b/.gitignore index 16735ca2333..b18c09ad1eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,20 +1,6 @@ - # python bytecode -*.pyc - -# generated documentation -doc/commands/ -doc/categories/ -doc/cylc.txt -doc/commands.tex -doc/graphics/png-scaled/ -doc/html/ -doc/pdf/ -doc/index.html -doc/cylc-version.txt - -# VERSION FILE -VERSION +*.py[cod] +__pycache__/ # vim backup files *.swp @@ -22,18 +8,40 @@ VERSION # emacs backup files *~ -# ordereddict-0.4.5 build -ext/ordereddict-0.4.5/build - -# site configuration and processed configuration -conf/job-init-env.sh -conf/global.rc* -conf/siterc -conf/*/*.rc.processed +# vscode +.vscode -# processed suite definitions +# processed workflow configs *.rc.processed +*.cylc.processed + +# profiling +.profiling + +# function test log file +.prove + +# coverage +.coverage +.coverage* +coverage.xml +htmlcov/ + +# distutils files +build +dist +MANIFEST +cylc_flow.egg-info +.eggs + +# virtualenv +venv + +# mypy +.mypy_cache/ -# suite passphrases -passphrase +# docker stuffs +.docker* +# operating system junk +.DS_Store diff --git a/.mailmap b/.mailmap index 7a7c4dd4209..71a84fdefc6 100644 --- a/.mailmap +++ b/.mailmap @@ -1,16 +1,59 @@ -Ben Fitzpatrick benfitzpatrick -David Matthews dpmatthews -Hilary James Oliver Hilary James Oliver -Hilary James Oliver Hilary James Oliver -Hilary James Oliver hilary.oliver -Hilary James Oliver Hilary Oliver -Hilary James Oliver Hilary James Oliver -Hilary James Oliver Hilary Oliver -Matt Shin matthew.shin -Matt Shin matthewrmshin -Matt Shin matthewrmshin -Matt Shin Matt Shin +# FORMAT: +# Omit commit-name or commit-email if same as proper + +Alex Reinecke P. A. Reinecke +Alexander Paulsell AlexanderPaulsell +Ben Fitzpatrick benfitzpatrick +Bruno Kinoshita +Bruno Kinoshita +Bruno Kinoshita +David Matthews dpmatthews +David Sutherland +David Sutherland sutherlander +Declan Valters +Declan Valters +Hilary Oliver +Hilary Oliver +Hilary Oliver +Hilary Oliver +Hilary Oliver +Hilary Oliver Hilary James Oliver +Ivor Blockley +Jonathan Thomas +Jonathan Thomas +Jonny Williams williamsjh +Kerry Day Kerry Day kaday -Kerry Day Kerry Day -Patrick Alexander Reinecke P. A. Reinecke -Luis Kornblueh Luis Kornblueh +Lois Huggett lhuggett +Luis Kornblueh +Martin Dix MartinDix +Martin Ryan +Martin Ryan +Matt Shin +Matt Shin +Matt Shin matthew.shin +Matt Shin matthewrmshin +Matt Shin matthewrmshin +Mel Hall <37735232+datamel@users.noreply.github.com> +Oliver Sanders Oliver sanders +Prasanna Challuri challurip +Ronnie Dutta <61982285+MetRonnie@users.noreply.github.com> +Rosalyn Hatcher RosalynHatcher +Sadie Bartholomew <30274190+sadielbartholomew@users.noreply.github.com> +Sadie Bartholomew Sadie L. Bartholomew <30274190+sadielbartholomew@users.noreply.github.com> +Sadie Bartholomew sadielbartholomew +Thomas Coleman ColemanTom +Thomas Coleman Tom Coleman +Thomas Coleman ColemanTom <15375218+ColemanTom@users.noreply.github.com> +Thomas Coleman Tom Coleman <15375218+ColemanTom@users.noreply.github.com> +Tim Pillinger wxtim +Tim Pillinger <26465611+wxtim@users.noreply.github.com> +Tim Whitcomb +Tim Whitcomb trwhitcomb +Tomek Trzeciak +Tomek Trzeciak TomekTrzeciak +Utheri Wagura +Utheri Wagura <36386988+uwagura@users.noreply.github.com> +github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> +github-actions[bot] GitHub Action +Diquan Jabbour <165976689+Diquan-BOM@users.noreply.github.com> diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 78d5a64527b..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,73 +0,0 @@ -# Configuration for the Travis continuous integration system -# ========================================================== -# -# Travis is a free service for running automatic tests on Github repositories. -# This file configures Travis for Cylc, telling it how to install Cylc and run -# the test battery. -# -# Test results get posted back to Github. By default Travis will run tests on any -# pull requests, adding a comment on the pull request page to say if the tests -# pass or fail, it will also test any new commits, showing the test results on -# the branch page, e.g. https://github.com/cylc/cylc/branches. -# -# Connecting a Cylc branch -# ------------------------ -# -# To make use of Travis you will first need to create a fork of Cylc in Github. -# Log in to https://travis-ci.org using your Github credentials, it will ask for -# permission to see your repositories, set the status of branches (whether the -# build passes or fails tests) and create hooks so Travis gets notified of new -# commits. -# -# Travis will create a list of all of your public Github repositories, you can -# enable automatic tests for a repository using the switches in this list. -# -# More information for Travis can be found at http://docs.travis-ci.com/user/getting-started/ - ---- -language: python - -# General environment setup before we start installing stuff -before_install: - # Clear bashrc - the default does nothing if not in an interactive shell. - # SSH connections use the ~/.bashrc file for their environment, so we'll be - # loading our python environment here. - - echo > ~/.bashrc - - # Setup virtualenv (using system packages for pygtk as pip won't install it) - - virtualenv --system-site-packages $HOME/virtualenv/cylc - - echo "source $HOME/virtualenv/cylc/bin/activate" >> ~/.bashrc - - # Make sure Cylc is in PATH when running jobs - - echo "export PATH=$PWD/bin:\$PATH" >> ~/.bashrc - - # Load our new environment - - source ~/.bashrc - - -# These commands are run before the test -install: - # Setup local SSH for Cylc jobs - - ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q - - cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys - - ssh-keyscan -t rsa localhost >> ~/.ssh/known_hosts - - # Install dependencies - - sudo apt-get install build-essential - - sudo apt-get install at python-pip python-dev libgraphviz-dev # python-gtk2-dev - - pip install -r requirements.txt - -# Run tests -script: - # TESTS defines what tests to run - # The following failures appear to be artefacts of running on Travis - # lib/parsec/tests - Cannot find 'validate' library in lib/parsec - # tests/cylcers - Requires GTK for graphing - # tests/restart/04-running.t - Fails only in some runs (timeout?) - - TESTS=$(find . -name *.t -type f -not -path '*/cyclers/*' -not -path '*/parsec/*' -not -path '*/restart/04-running.t') - - cylc test-battery $TESTS -- -j 1 - -# Check output (more useful if you narrow down what tests get run) -after_script: - - for file in $(find $HOME/cylc-run -type f); do echo; echo "== $file =="; cat $file; done - - for file in $(find /tmp/cylc-tests-$USER -type f); do echo; echo "== $file =="; cat $file; done diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000000..15e89f118a2 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,762 @@ +# Changelog + +List of notable changes, for a complete list of changes see the +[closed milestones](https://github.com/cylc/cylc-flow/milestones?state=closed) +for each release. + + + + + +## __cylc-8.3.3 (Released 2024-07-23)__ + +### 🔧 Fixes + +[#6103](https://github.com/cylc/cylc-flow/pull/6103) - Absolute dependencies (dependencies on tasks in a specified cycle rather than at a specified offset) are now visible in the GUI beyond the specified cycle. + +[#6213](https://github.com/cylc/cylc-flow/pull/6213) - Fix bug where the `-S`, `-O` and `-D` options in `cylc vr` would not be applied correctly when restarting a workflow. + +[#6241](https://github.com/cylc/cylc-flow/pull/6241) - Allow flow-merge when triggering n=0 tasks. + +[#6242](https://github.com/cylc/cylc-flow/pull/6242) - Put `share/bin` in the `PATH` of scheduler environment, event handlers therein will now be found. + +[#6249](https://github.com/cylc/cylc-flow/pull/6249), [#6252](https://github.com/cylc/cylc-flow/pull/6252) - Fix a race condition between global config reload and debug logging that caused "platform not defined" errors when running workflows that contained a "rose-suite.conf" file in verbose or debug mode. + +## __cylc-8.3.2 (Released 2024-07-10)__ + +### 🔧 Fixes + +[#6186](https://github.com/cylc/cylc-flow/pull/6186) - Fixed bug where using flow numbers with `cylc set` would not work correctly. + +[#6200](https://github.com/cylc/cylc-flow/pull/6200) - Fixed bug where a stalled paused workflow would be incorrectly reported as running, not paused + +[#6206](https://github.com/cylc/cylc-flow/pull/6206) - Fixes the spawning of multiple parentless tasks off the same sequential wall-clock xtrigger. + +## __cylc-8.3.1 (Released 2024-07-04)__ + +### 🔧 Fixes + +[#6130](https://github.com/cylc/cylc-flow/pull/6130) - Prevent commands accepting job IDs where it doesn't make sense. + +[#6170](https://github.com/cylc/cylc-flow/pull/6170) - Fix an issue where the Cylc logo could appear in the workflow log. + +[#6176](https://github.com/cylc/cylc-flow/pull/6176) - Fix bug where jobs which fail to submit are not shown in GUI/TUI if submission retries are set. + +[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing. + +## __cylc-8.3.0 (Released 2024-06-18)__ + +### ⚠ Breaking Changes + +[#5600](https://github.com/cylc/cylc-flow/pull/5600) - The `cylc dump` command now only shows active tasks (e.g. running & queued + tasks). This restores its behaviour of only showing the tasks which currently + exist in the pool as it did in Cylc 7 and earlier versions of Cylc 8. + +[#5727](https://github.com/cylc/cylc-flow/pull/5727) - Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`. + +[#5794](https://github.com/cylc/cylc-flow/pull/5794) - Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12. + +[#5836](https://github.com/cylc/cylc-flow/pull/5836) - Removed the 'CYLC_TASK_DEPENDENCIES' environment variable + +[#5956](https://github.com/cylc/cylc-flow/pull/5956) - `cylc lint`: deprecated `[cylc-lint]` section in favour of `[tool.cylc.lint]` in `pyproject.toml` + +[#6046](https://github.com/cylc/cylc-flow/pull/6046) - The `submit-fail` and `expire` task outputs must now be + [optional](https://cylc.github.io/cylc-doc/stable/html/glossary.html#term-optional-output) + and can no longer be + [required](https://cylc.github.io/cylc-doc/stable/html/glossary.html#term-required-output). + +### 🚀 Enhancements + +[#5571](https://github.com/cylc/cylc-flow/pull/5571) - Make workflow `CYLC_` variables available to the template processor during parsing. + +[#5658](https://github.com/cylc/cylc-flow/pull/5658) - New "cylc set" command for setting task prerequisites and outputs. + +[#5709](https://github.com/cylc/cylc-flow/pull/5709) - Forward arbitrary environment variables over SSH connections + +[#5721](https://github.com/cylc/cylc-flow/pull/5721) - Allow task simulation mode settings to be changed dynamically using `cylc broadcast`. + +[#5731](https://github.com/cylc/cylc-flow/pull/5731) - Major upgrade to `cylc tui` which now supports larger workflows and can browse installed workflows. + +[#5738](https://github.com/cylc/cylc-flow/pull/5738) - Optionally spawn parentless xtriggered tasks sequentially - i.e., one at a time, after the previous xtrigger is satisfied, instead of all at once out to the runahead limit. The `wall_clock` xtrigger is now sequential by default. + +[#5769](https://github.com/cylc/cylc-flow/pull/5769) - Include task messages and workflow port as appropriate in emails configured by "mail events". + +[#5803](https://github.com/cylc/cylc-flow/pull/5803) - Updated 'reinstall' functionality to support multiple workflows + +[#5809](https://github.com/cylc/cylc-flow/pull/5809) - The workflow-state command and xtrigger are now flow-aware and take universal IDs instead of separate arguments for cycle point, task name, etc. (which are still supported, but deprecated). + +[#5831](https://github.com/cylc/cylc-flow/pull/5831) - Add capability to install xtriggers via a new cylc.xtriggers entry point + +[#5864](https://github.com/cylc/cylc-flow/pull/5864) - Reimplemented the `suite-state` xtrigger for interoperability with Cylc 7. + +[#5872](https://github.com/cylc/cylc-flow/pull/5872) - Improvements to `cylc clean` remote timeout handling. + +[#5873](https://github.com/cylc/cylc-flow/pull/5873) - `cylc lint` improvements: + - Allow use of `#noqa: S001` comments to skip checks for a single line. + - Stop `cylc lint` objecting to `%include ` syntax. + +[#5879](https://github.com/cylc/cylc-flow/pull/5879) - `cylc lint` now warns of use of old templated items such as `%(suite)s` + +[#5890](https://github.com/cylc/cylc-flow/pull/5890) - Lint: Warn users that setting ``CYLC_VERSION``, ``ROSE_VERSION`` or + ``FCM_VERSION`` in the workflow config is deprecated. + +[#5943](https://github.com/cylc/cylc-flow/pull/5943) - The `stop after cycle point` can now be specified as an offset from the inital cycle point. + +[#5955](https://github.com/cylc/cylc-flow/pull/5955) - Support xtrigger argument validation. + +[#6029](https://github.com/cylc/cylc-flow/pull/6029) - Workflow graph window extent is now preserved on reload. + +[#6046](https://github.com/cylc/cylc-flow/pull/6046) - The condition that Cylc uses to evaluate task output completion can now be + customized in the `[runtime]` section with the new `completion` configuration. + This provides a more advanced way to check that tasks generate their required + outputs when run. + +### 🔧 Fixes + +[#5809](https://github.com/cylc/cylc-flow/pull/5809) - Fix bug where the "cylc workflow-state" command only polled for + task-specific status queries and custom outputs. + +[#6008](https://github.com/cylc/cylc-flow/pull/6008) - Fixed bug where the `[scheduler][mail]to/from` settings did not apply as defaults for task event mail. + +[#6036](https://github.com/cylc/cylc-flow/pull/6036) - Fixed bug in simulation mode where repeated submissions were not displaying correctly in TUI/GUI. + +[#6067](https://github.com/cylc/cylc-flow/pull/6067) - Fixed a bug that sometimes allowed suicide-triggered or manually removed tasks to be added back later. + +[#6109](https://github.com/cylc/cylc-flow/pull/6109) - Fixed bug affecting job submission where the list of bad hosts was not always reset correctly. + +[#6123](https://github.com/cylc/cylc-flow/pull/6123) - Allow long-format datetime cycle points in IDs used on the command line. + +## __cylc-8.2.7 (Released 2024-05-15)__ + +### 🔧 Fixes + +[#6096](https://github.com/cylc/cylc-flow/pull/6096) - Fixed bug that caused graph arrows to go missing in the GUI when suicide triggers are present. + +[#6102](https://github.com/cylc/cylc-flow/pull/6102) - Fixed bug introduced in 8.2.6 in `cylc vip` & `cylc vr` when using cylc-rose options (`-S`, `-D`, `-O`). + +## __cylc-8.2.6 (Released 2024-05-02)__ + +### ⚠ Breaking Changes + +[#6068](https://github.com/cylc/cylc-flow/pull/6068) - Removed the Rose Options (`-S`, `-O`, `-D`) from `cylc play`. If you need these use them with `cylc install`. + +### 🚀 Enhancements + +[#6072](https://github.com/cylc/cylc-flow/pull/6072) - Nano Syntax Highlighting now available. + +### 🔧 Fixes + +[#6071](https://github.com/cylc/cylc-flow/pull/6071) - `cylc config` now shows xtrigger function signatures. + +[#6078](https://github.com/cylc/cylc-flow/pull/6078) - Fixed bug where `cylc lint` could hang when checking `inherit` settings in `flow.cylc`. + +## __cylc-8.2.5 (Released 2024-04-04)__ + +### 🔧 Fixes + +[#5924](https://github.com/cylc/cylc-flow/pull/5924) - Validation: a cycle offset can only appear on the right of a dependency if the task's cycling is defined elsewhere with no offset. + +[#5933](https://github.com/cylc/cylc-flow/pull/5933) - Fixed bug in `cylc broadcast` (and the GUI Edit Runtime command) where everything after a `#` character in a setting would be stripped out. + +[#5959](https://github.com/cylc/cylc-flow/pull/5959) - Fix an issue where workflow "timeout" events were not fired in all situations when they should have been. + +[#6011](https://github.com/cylc/cylc-flow/pull/6011) - Fixed a `cylc vip` bug causing remote re-invocation to fail if using `--workflow-name` option. + +[#6031](https://github.com/cylc/cylc-flow/pull/6031) - Fixed workflow-state command and xtrigger for alternate cylc-run directory. + +## __cylc-8.2.4 (Released 2024-01-11)__ + +### 🚀 Enhancements + +[#5772](https://github.com/cylc/cylc-flow/pull/5772) - `cylc lint`: added a check for indentation being 4N spaces. + +[#5838](https://github.com/cylc/cylc-flow/pull/5838) - `cylc lint`: added rule to check for `rose date` usage (should be replaced with `isodatetime`). + +### 🔧 Fixes + +[#5789](https://github.com/cylc/cylc-flow/pull/5789) - Prevent the run mode from being changed on restart. + +[#5801](https://github.com/cylc/cylc-flow/pull/5801) - Fix traceback when using parentheses on right hand side of graph trigger. + +[#5821](https://github.com/cylc/cylc-flow/pull/5821) - Fixed issue where large uncommitted changes could cause `cylc install` to hang. + +[#5841](https://github.com/cylc/cylc-flow/pull/5841) - `cylc lint`: improved handling of S011 to not warn if the `#` is `#$` (e.g. shell base arithmetic). + +[#5885](https://github.com/cylc/cylc-flow/pull/5885) - Fixed bug in using a final cycle point with chained offsets e.g. 'final cycle point = +PT6H+PT1S'. + +[#5893](https://github.com/cylc/cylc-flow/pull/5893) - Fixed bug in computing a time interval-based runahead limit when future triggers are present. + +[#5902](https://github.com/cylc/cylc-flow/pull/5902) - Fixed a bug that prevented unsetting `execution time limit` by broadcast or reload. + +[#5908](https://github.com/cylc/cylc-flow/pull/5908) - Fixed bug causing redundant DB updates when many tasks depend on the same xtrigger. + +[#5909](https://github.com/cylc/cylc-flow/pull/5909) - Fix a bug where Cylc VIP did not remove --workflow-name= from + Cylc play arguments. + +## __cylc-8.2.3 (Released 2023-11-02)__ + +### 🔧 Fixes + +[#5660](https://github.com/cylc/cylc-flow/pull/5660) - Re-worked graph n-window algorithm for better efficiency. + +[#5753](https://github.com/cylc/cylc-flow/pull/5753) - Fixed bug where execution time limit polling intervals could end up incorrectly applied + +[#5776](https://github.com/cylc/cylc-flow/pull/5776) - Ensure that submit-failed tasks are marked as incomplete (so remain visible) when running in back-compat mode. + +[#5791](https://github.com/cylc/cylc-flow/pull/5791) - fix a bug where if multiple clock triggers are set for a task only one was being satisfied. + +## __cylc-8.2.2 (Released 2023-10-05)__ + +### 🚀 Enhancements + +[#5237](https://github.com/cylc/cylc-flow/pull/5237) - Back-compat: allow workflow-state xtriggers (and the `cylc workflow-state` + command) to read Cylc 7 databases. + +### 🔧 Fixes + +[#5693](https://github.com/cylc/cylc-flow/pull/5693) - Log command issuer, if not the workflow owner, for all commands. + +[#5694](https://github.com/cylc/cylc-flow/pull/5694) - Don't fail config file parsing if current working directory does not exist. + (Note however this may not be enough to prevent file parsing commands failing + elsewhere in the Python library). + +[#5704](https://github.com/cylc/cylc-flow/pull/5704) - Fix off-by-one error in automatic upgrade of Cylc 7 "max active cycle points" to Cylc 8 "runahead limit". + +[#5708](https://github.com/cylc/cylc-flow/pull/5708) - Fix runahead limit at start-up, with recurrences that start beyond the limit. + +[#5755](https://github.com/cylc/cylc-flow/pull/5755) - Fixes an issue where submit-failed tasks could be incorrectly considered as completed rather than causing the workflow to stall. + + +## __cylc-8.2.1 (Released 2023-08-14)__ + +### 🔧 Fixes + +[#5631](https://github.com/cylc/cylc-flow/pull/5631) - Fix bug in remote clean for workflows that generated `flow.cylc` files at runtime. + +[#5650](https://github.com/cylc/cylc-flow/pull/5650) - Fix a bug preventing clean-up of finished tasks in the GUI and TUI. + +[#5685](https://github.com/cylc/cylc-flow/pull/5685) - Fix "cylc pause" command help (it targets workflows, not tasks, but was + printing task-matching documentation as well). + + +## __cylc-8.2.0 (Released 2023-07-21)__ + +### Breaking Changes + +[#5600](https://github.com/cylc/cylc-flow/pull/5600) - +The `CYLC_TASK_DEPENDENCIES` environment variable will no longer be exported +in job environments if there are more than 50 dependencies. This avoids an +issue which could cause jobs to fail if this variable became too long. + +### Enhancements + +[#5992](https://github.com/cylc/cylc-flow/pull/5992) - +Before trying to reload the workflow definition, the scheduler will +now wait for preparing tasks to submit, and pause the workflow. +After successful reload the scheduler will unpause the workflow. + +[#5605](https://github.com/cylc/cylc-flow/pull/5605) - Added `-z` shorthand +option for defining a list of strings: +- Before: `cylc command -s "X=['a', 'bc', 'd']"` +- After: `cylc command -z X=a,bc,d`. + +[#5537](https://github.com/cylc/cylc-flow/pull/5537) - Allow parameters +in family names to be split, e.g. `FAM`. + +[#5589](https://github.com/cylc/cylc-flow/pull/5589) - Move to workflow +directory during file parsing, to give the template processor access to +workflow files. + +[#5405](https://github.com/cylc/cylc-flow/pull/5405) - Improve scan command +help, and add scheduler PID to the output. + +[#5461](https://github.com/cylc/cylc-flow/pull/5461) - preserve colour +formatting when starting workflows in distributed mode using `run hosts`. + +[#5291](https://github.com/cylc/cylc-flow/pull/5291) - re-implement old-style +clock triggers as wall_clock xtriggers. + +[#5439](https://github.com/cylc/cylc-flow/pull/5439) - Small CLI short option chages: +Add the `-n` short option for `--workflow-name` to `cylc vip`; rename the `-n` +short option for `--no-detach` to `-N`; add `-r` as a short option for +`--run-name`. + +[#5231](https://github.com/cylc/cylc-flow/pull/5231) - stay up for a timeout +period on restarting a completed workflow, to allow for manual triggering. + +[#5549](https://github.com/cylc/cylc-flow/pull/5549), +[#5546](https://github.com/cylc/cylc-flow/pull/5546) - +Various enhancements to `cylc lint`: +* `cylc lint` will provide a non-zero return code if any issues are identified. + This can be overridden using the new `--exit-zero` flag. +* Fix numbering of lint codes (n.b. lint codes should now be permenantly + unchanging, but may have changed since Cylc 8.1.4, so `pyproject.toml` files + may need updating). +* Check for suicide triggers in `.cylc` files. +* Check for `platform = $(rose host-select)`. +* Check for use of deprecated Cylc commands (and `rose suite-hook`). +* Check for zero prefixed Jinja2 integers. +* Only check for missing Jinja2 shebangs in `flow.cylc` and + `suite.rc` files. + + +[#5525](https://github.com/cylc/cylc-flow/pull/5525) - Jobs can use scripts +in `share/bin` and Python modules in `share/lib/python`. + +### Fixes + +[#5328](https://github.com/cylc/cylc-flow/pull/5328) - +Efficiency improvements to reduce task management overheads on the Scheduler. + +[#5611](https://github.com/cylc/cylc-flow/pull/5611) - +Improve the documentation of the GraphQL schema. + +[#5616](https://github.com/cylc/cylc-flow/pull/5616) - +Improve PBS support for job IDs with trailing components. + +[#5619](https://github.com/cylc/cylc-flow/pull/5619) - +Fix an issue where the `task_pool` table in the database wasn't being updated +in a timely fashion when tasks completed. + +[#5606](https://github.com/cylc/cylc-flow/pull/5606) - +Task outputs and messages are now validated to avoid conflicts with built-in +outputs, messages, qualifiers and Cylc keywords. + +[#5614](https://github.com/cylc/cylc-flow/pull/5614) - +Fix a bug in Cylc 7 compatibility mode where tasks running in the `none` flow +(e.g. via `cylc trigger --flow=none`) would trigger downstream tasks. + +[#5604](https://github.com/cylc/cylc-flow/pull/5604) - +Fix a possible issue where workflows started using +`cylc play --start-cycle-point` could hang during startup. + +[#5573](https://github.com/cylc/cylc-flow/pull/5573) - Fix bug that ran a +queued waiting task even after removal by `cylc remove`. + +[#5524](https://github.com/cylc/cylc-flow/pull/5524) - Logging includes timestamps +for `cylc play` when called by `cylc vip` or `cylc vr`. + +[#5228](https://github.com/cylc/cylc-flow/pull/5228) - +Enabled the "stop", "poll", "kill" and "message" commands to be issued from +the UI whilst the workflow is in the process of shutting down. + +[#5582](https://github.com/cylc/cylc-flow/pull/5582) - Set Cylc 7 compatibility +mode before running pre-configure plugins. + +[#5587](https://github.com/cylc/cylc-flow/pull/5587) - +Permit commas in xtrigger arguments and fix minor issues with the parsing of +xtrigger function signatures. + +[#5618](https://github.com/cylc/cylc-flow/pull/5618) - +Fix a bug when rapidly issuing the same/opposite commands e.g. pausing & +resuming a workflow. + +[#5625](https://github.com/cylc/cylc-flow/pull/5625) - Exclude `setuptools` +version (v67) which results in dependency check failure with editable installs. + +## __cylc-8.1.4 (Released 2023-05-04)__ + +### Fixes + +[#5514](https://github.com/cylc/cylc-flow/pull/5514) - +Ensure `cylc cat-log` directory listings always include the `job-activity.log` +file when present and are able to list submit-failed jobs. + +[#5506](https://github.com/cylc/cylc-flow/pull/5506) - +Fix bug introduced in 8.1.3 where specifying a subshell command for +`flow.cylc[runtime][][remote]host` (e.g. `$(rose host-select)`) +would always result in localhost. + +## __cylc-8.1.3 (Released 2023-04-27)__ + +### Enhancements + +[#5475](https://github.com/cylc/cylc-flow/pull/5475) - much faster computation +of the visualization window around active tasks (at the cost, for now, of not +showing non-active "cousin" nodes). + +[#5453](https://github.com/cylc/cylc-flow/pull/5453) - `cylc cat-log` can now +list and view workflow log files including install logs and workflow +configuration files. + +### Fixes + +[#5495](https://github.com/cylc/cylc-flow/pull/5495) - Fix bug that could cause +invalid parent tasks to appear in the UI datastore. + +[#5334](https://github.com/cylc/cylc-flow/pull/5334) - Apply graph prerequisite +changes to already-spawned tasks after reload or restart. + +[5466](https://github.com/cylc/cylc-flow/pull/5466) - Don't generate duplicate +prerequisites from recurrences with coincident points. + +[5450](https://github.com/cylc/cylc-flow/pull/5450) - Validation provides +better error messages if [sections] and settings are mixed up in a +configuration. + +[5445](https://github.com/cylc/cylc-flow/pull/5445) - Fix remote tidy + bug where install target is not explicit in platform definition. + +[5398](https://github.com/cylc/cylc-flow/pull/5398) - Fix platform from +group selection order bug. + +[#5395](https://github.com/cylc/cylc-flow/pull/5395) - Fix bug where workflow +shuts down if all hosts for all platforms in a platform group are unreachable. + +[#5384](https://github.com/cylc/cylc-flow/pull/5384) - +Fixes `cylc set-verbosity`. + +[#5479](https://github.com/cylc/cylc-flow/pull/5479) - +Fixes `cylc help license` + +[#5394](https://github.com/cylc/cylc-flow/pull/5394) - +Fixes a possible scheduler traceback observed with remote task polling. + +[#5386](https://github.com/cylc/cylc-flow/pull/5386) - Fix bug where +absence of `job name length maximum` in PBS platform settings would cause +Cylc to crash when preparing the job script. + +[#5343](https://github.com/cylc/cylc-flow/pull/5343) - Fix a bug causing +platform names to be checked as if they were hosts. + +[#5359](https://github.com/cylc/cylc-flow/pull/5359) - Fix bug where viewing +a workflow's log in the GUI or using `cylc cat-log` would prevent `cylc clean` +from working. + +## __cylc-8.1.2 (Released 2023-02-20)__ + +### Fixes + +[#5349](https://github.com/cylc/cylc-flow/pull/5349) - Bugfix: `cylc vip --workflow-name` +only worked when used with a space, not an `=`. + +[#5367](https://github.com/cylc/cylc-flow/pull/5367) - Enable using +Rose options (`-O`, `-S` & `-D`) with `cylc view`. + +[#5363](https://github.com/cylc/cylc-flow/pull/5363) Improvements and bugfixes +for `cylc lint`. + +## __cylc-8.1.1 (Released 2023-01-31)__ + +### Fixes + +[#5313](https://github.com/cylc/cylc-flow/pull/5313) - Fix a bug +causing Cylc to be unable to parse previously played Cylc 7 workflows. + +[#5312](https://github.com/cylc/cylc-flow/pull/5312) - task names must be +comma-separated in queue member lists. Any implicit tasks +(i.e. with no task definition under runtime) assigned to a queue will generate a warning. + +[#5314](https://github.com/cylc/cylc-flow/pull/5314) - Fix broken +command option: `cylc vip --run-name`. + +[#5319](https://github.com/cylc/cylc-flow/pull/5319), +[#5321](https://github.com/cylc/cylc-flow/pull/5321), +[#5325](https://github.com/cylc/cylc-flow/pull/5325) - +Various efficiency optimisations to the scheduler which particularly impact +workflows with many-to-many dependencies (e.g. ` => `). + +## __cylc-8.1.0 (Released 2023-01-16)__ + +### Breaking Changes + +* Workflows started with Cylc 8.0 which contain multiple "flows" cannot be + restarted with Cylc 8.1 due to database changes. + +### Enhancements + +[#5229](https://github.com/cylc/cylc-flow/pull/5229) - +- Added a single command to validate a previously run workflow against changes + to its source and reinstall a workflow. +- Allows Cylc commands (including validate, list, view, config, and graph) to load template variables + configured by `cylc install` and `cylc play`. + +[#5121](https://github.com/cylc/cylc-flow/pull/5121) - Added a single +command to validate, install and play a workflow. + +[#5184](https://github.com/cylc/cylc-flow/pull/5184) - Scan for active +runs of the same workflow at install time. + +[#5084](https://github.com/cylc/cylc-flow/pull/5084) - Assign the most recent +previous flow numbers to tasks triggered when no flows are present (e.g. on +restarting a finished workflow). + +[#5032](https://github.com/cylc/cylc-flow/pull/5032) - Set a default limit of +100 for the "default" queue. + +[#5055](https://github.com/cylc/cylc-flow/pull/5055) and +[#5086](https://github.com/cylc/cylc-flow/pull/5086) - Upgrades to `cylc lint` +- Allow users to ignore Cylc Lint issues using `--ignore `. +- Allow settings for `cylc lint` to be recorded in a pyproject.toml file. +- Allow files to be excluded from `cylc lint` checks. + +[#5081](https://github.com/cylc/cylc-flow/pull/5081) - Reduced amount that +gets logged at "INFO" level in scheduler logs. + +[#5259](https://github.com/cylc/cylc-flow/pull/5259) - Add flow_nums +to task_jobs table in the workflow database. + +### Fixes + +[#5286](https://github.com/cylc/cylc-flow/pull/5286) - Fix bug where +`[scheduling][special tasks]clock-trigger` would skip execution retry delays. + +[#5292](https://github.com/cylc/cylc-flow/pull/5292) - +Fix an issue where polling could be repeated if the job's platform +was not available. + +## __cylc-8.0.4 (Released 2022-12-14)__ + +Maintenance release. + +### Fixes + +[##5205](https://github.com/cylc/cylc-flow/pull/#5205) - Fix bug which caused +orphaned running tasks to silently skip remote file installation at scheduler restart. + +[#5224](https://github.com/cylc/cylc-flow/pull/5225) - workflow installation: +disallow reserved names only in the top level source directory. + +[#5211](https://github.com/cylc/cylc-flow/pull/5211) - Provide better +explanation of failure if `icp = next (T-02, T-32)` when list should be +semicolon separated. + +[#5196](https://github.com/cylc/cylc-flow/pull/5196) - Replace traceback +with warning, for scan errors where workflow is stopped. + +[#5199](https://github.com/cylc/cylc-flow/pull/5199) - Fix a problem with +the consolidation tutorial. + +[#5195](https://github.com/cylc/cylc-flow/pull/5195) - +Fix issue where workflows can fail to shutdown due to unavailable remote +platforms and make job log retrieval more robust. + +## __cylc-8.0.3 (Released 2022-10-17)__ + +Maintenance release. + +### Fixes + +[#5192](https://github.com/cylc/cylc-flow/pull/5192) - +Recompute runahead limit after use of `cylc remove`. + +[#5188](https://github.com/cylc/cylc-flow/pull/5188) - +Fix task state selectors in `cylc trigger` and other commands. + +[#5125](https://github.com/cylc/cylc-flow/pull/5125) - Allow rose-suite.conf +changes to be considered by ``cylc reinstall``. + +[#5023](https://github.com/cylc/cylc-flow/pull/5023), +[#5187](https://github.com/cylc/cylc-flow/pull/5187) - +tasks force-triggered +after a shutdown was ordered should submit to run immediately on restart. + +[#5137](https://github.com/cylc/cylc-flow/pull/5137) - +Install the `ana/` directory to remote platforms by default. + +[#5146](https://github.com/cylc/cylc-flow/pull/5146) - no-flow tasks should not +retrigger incomplete children. + +[#5104](https://github.com/cylc/cylc-flow/pull/5104) - Fix retriggering of +failed tasks after a reload. + +[#5139](https://github.com/cylc/cylc-flow/pull/5139) - Fix bug where +`cylc install` could hang if there was a large uncommitted diff in the +source dir (for git/svn repos). + +[#5131](https://github.com/cylc/cylc-flow/pull/5131) - Infer workflow run number +for `workflow_state` xtrigger. + +## __cylc-8.0.2 (Released 2022-09-12)__ + +Maintenance release. + +### Fixes + +[#5115](https://github.com/cylc/cylc-flow/pull/5115) - Updates rsync commands +to make them compatible with latest rsync releases. + +[#5119](https://github.com/cylc/cylc-flow/pull/5119) - Fix formatting of +deprecation warnings at validation. + +[#5067](https://github.com/cylc/cylc-flow/pull/5067) - Datastore fix for +taskdefs removed before restart. + +[#5066](https://github.com/cylc/cylc-flow/pull/5066) - Fix bug where +.cylcignore only found if `cylc install` is run in source directory. + +[#5091](https://github.com/cylc/cylc-flow/pull/5091) - Fix problems with +tutorial workflows. + +[#5098](https://github.com/cylc/cylc-flow/pull/5098) - Fix bug where final task +status updates were not being sent to UI before shutdown. + +[#5114](https://github.com/cylc/cylc-flow/pull/5114) - Fix bug where +validation errors during workflow startup were not printed to stderr before +daemonisation. + +[#5110](https://github.com/cylc/cylc-flow/pull/5110) - Fix bug where reloading +a stalled workflow would cause it stall again. + +## __cylc-8.0.1 (Released 2022-08-16)__ + +Maintenance release. + +### Fixes + +[#5025](https://github.com/cylc/cylc-flow/pull/5025) - Fix a bug where polling +causes a failed task to be shown as submitted when the workflow is reloaded. + +[#5045](https://github.com/cylc/cylc-flow/pull/5045) - +Fix issue where unsatisfied xtriggers could be wiped on reload. + +[#5031](https://github.com/cylc/cylc-flow/pull/5031) - Fix bug where +specifying multiple datetime offsets (e.g. `final cycle point = +P1M-P1D`) +would not obey the given order. + +[#5033](https://github.com/cylc/cylc-flow/pull/5033) - Running `cylc clean` +on a top level dir containing run dir(s) will now remove that top level dir +in addition to the run(s) (if there is nothing else inside it). + +[#5007](https://github.com/cylc/cylc-flow/pull/5007) - Fix for `cylc broadcast` +cycle point validation in the UI. + +[#5037](https://github.com/cylc/cylc-flow/pull/5037) - Fix bug where the +workflow restart number would get wiped on reload. + +[#5049](https://github.com/cylc/cylc-flow/pull/5049) - Fix several small +bugs related to auto restart. + +[#5062](https://github.com/cylc/cylc-flow/pull/5062) - Fix bug where preparing +tasks could sometimes get orphaned when an auto restart occurred. + +## __cylc-8.0.0 (Released 2022-07-28)__ + +Cylc 8 production-ready release. + +### Major Changes + +* Python 2 -> 3. +* Internal communications converted from HTTPS to ZMQ (TCP). +* PyGTK GUIs replaced by: + * Terminal user interface (TUI) included in cylc-flow. + * Web user interface provided by the cylc-uiserver package. +* A new scheduling algorithm with support for branched workflows. +* Command line changes: + * `cylc run` -> `cylc play` + * `cylc restart` -> `cylc play` + * `rose suite-run` -> `cylc install; cylc play ` +* The core package containing Cylc scheduler program has been renamed cylc-flow. +* Cylc review has been removed, the Cylc 7 version remains Cylc 8 compatible. +* [New documentation](https://cylc.github.io/cylc-doc/stable). + +See the [migration guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html) for a full list of changes. + +### Enhancements + +[#4964](https://github.com/cylc/cylc-flow/pull/4964) - +`cylc reinstall` now displays the changes it would make when run +interactively and has improved help / documentaiton. + +[#4836](https://github.com/cylc/cylc-flow/pull/4836) - The log directory has +been tidied. Workflow logs are now found in `log/scheduler` rather than +`log/workflow`, filenames now include `start`/`restart`. Other minor directory +changes. Remote file installation logs are now per install target. + +[#4938](https://github.com/cylc/cylc-flow/pull/4938) - Detect bad Platforms +config: background and at job runners should have a single host. + +[#4877](https://github.com/cylc/cylc-flow/pull/4877) - Upgrade the version of +Jinja2 used by Cylc from 2.11 to 3.0. + +[#4896](https://github.com/cylc/cylc-flow/pull/4896) - Allow the setting of +default job runner directives for platforms. + +[#4900](https://github.com/cylc/cylc-flow/pull/4900) - Added a command to assist +with upgrading Cylc 7 workflows to Cylc 8: Try `cylc lint `. + +[#5009](https://github.com/cylc/cylc-flow/pull/5009) - Added new job +environment variable `$CYLC_WORKFLOW_NAME_BASE` as the basename of +`$CYLC_WORKFLOW_NAME`. + +[#4993](https://github.com/cylc/cylc-flow/pull/4993) - Remove the few remaining +uses of a configured text editor (via `cylc view` and `cylc cat-log` options). +The primary uses of it (`cylc trigger --edit` and `cylc edit` in Cylc 7) have +already been removed from Cylc 8. + +### Fixes + +[#5011](https://github.com/cylc/cylc-flow/pull/5011) - Removes preparing jobs +appearing in UI, and reuse submit number on restart for preparing tasks. + +[#5008](https://github.com/cylc/cylc-flow/pull/5008) - +Autospawn absolute-triggered tasks exactly the same way as parentless tasks. + +[#4984](https://github.com/cylc/cylc-flow/pull/4984) - +Fixes an issue with `cylc reload` which could cause preparing tasks to become +stuck. + +[#4976](https://github.com/cylc/cylc-flow/pull/4976) - Fix bug causing tasks +to be stuck in UI due to discontinued graph of optional outputs. + +[#4975](https://github.com/cylc/cylc-flow/pull/4975) - Fix selection of +platforms from `[job]` and `[remote]` configs. + +[#4948](https://github.com/cylc/cylc-flow/pull/4948) - Fix lack of +errors/warnings for deprecated `[runtime][][remote]retrieve job logs *` +settings. + +[#4970](https://github.com/cylc/cylc-flow/pull/4970) - Fix handling of suicide +triggers in back-compat mode. + +[#4887](https://github.com/cylc/cylc-flow/pull/4887) - Disallow relative paths +in `global.cylc[install]source dirs`. + +[#4906](https://github.com/cylc/cylc-flow/pull/4906) +- Fix delayed spawning of parentless tasks that do have parents in a previous + cycle point. +- Make integer-interval runahead limits consistent with time-interval limits: + `P0` means just the runahead base point; `P1` the base point and the point + (i.e. one cycle interval), and so on. + +[#4936](https://github.com/cylc/cylc-flow/pull/4936) - Fix incorrect +error messages when workflow CLI commands fail. + +[#4941](https://github.com/cylc/cylc-flow/pull/4941) - Fix job state for +platform submit-failures. + +[#4931](https://github.com/cylc/cylc-flow/pull/4931) - Fix cylc install for +installing workflows from multi-level directories. + +[#4926](https://github.com/cylc/cylc-flow/pull/4926) - Fix a docstring +formatting problem presenting in the UI mutation flow argument info. + +[#4891](https://github.com/cylc/cylc-flow/pull/4891) - Fix bug that could cause +past jobs to be omitted in the UI. + +[#4860](https://github.com/cylc/cylc-flow/pull/4860) - Workflow validation +now fails if +[owner setting](https://cylc.github.io/cylc-doc/stable/html/reference/config/workflow.html#flow.cylc[runtime][%3Cnamespace%3E][remote]owner) +is used, as that setting no longer has any effect. + +[#4978](https://github.com/cylc/cylc-flow/pull/4978) - `cylc clean`: fix +occasional failure to clean on remote hosts due to leftover contact file. + +[#4889](https://github.com/cylc/cylc-flow/pull/4889) - `cylc clean`: don't +prompt if no matching workflows. + +[#4890](https://github.com/cylc/cylc-flow/pull/4890) - `cylc install`: don't +overwrite symlink dir targets if they were not cleaned properly before. + +[#4881](https://github.com/cylc/cylc-flow/pull/4881) - Fix bug where commands +targeting a specific cycle point would not work if using an abbreviated +cycle point format. + + +## Older Releases + +* [Cylc 7 changelog](https://github.com/cylc/cylc-flow/blob/7.8.x/CHANGES.md) +* [Cylc 8 pre-release changelog](https://github.com/cylc/cylc-flow/blob/8.0.0/CHANGES.md) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000..5df4403fb8a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,147 @@ +# Cylc: How to Contribute + +Thanks for you interest in the Cylc project! + +Contributions are welcome, please open an issue to discuss changes before +raising a pull request. + +You can also get in touch via: + +* The developers chat: [![chat](https://img.shields.io/matrix/cylc-general:matrix.org)](https://matrix.to/#/#cylc-general:matrix.org) +* The forum: [![forum](https://img.shields.io/discourse/https/cylc.discourse.group/posts.svg)](https://cylc.discourse.group/) + + +## New Contributors + +Please read the [CLA](#contributor-licence-agreement-and-certificate-of-origin). + +Please add your name to the +[Code Contributors](#code-contributors) section of this file as part of your +first Pull Request (for each Cylc repository you contribute to). + + +## Contribute Code + +We use [semver](https://semver.org/) to separate riskier changes (e.g. new features +& code refactors) from bugfixes to provide more stable releases for production environments. + +**Enhancements** are made on the `master` branch and released in the next minor version +(e.g. 8.1, 8.2, 8.3) + +**Bugfixes** and minor usability enhancements are made on bugfix branches and +released as the next maintainance version (e.g. 8.0.1, 8.0.2, 8.0.3). E.G. if the issue is on the `8.0.x` milestone, branch off of `8.0.x` to +develop your bugfix, then raise the pull request against the `8.0.x` branch. We will later merge the `8.0.x` branch into `master`. + +Feel free to ask questions on the issue or +[developers chat](https://matrix.to/#/#cylc-general:matrix.org) if unsure about +anything. + +We use [towncrier](https://towncrier.readthedocs.io/en/stable/index.html) for +generating the changelog. Changelog entries are added by running +``` +towncrier create ..md --content "Short description" +``` + +## Code Contributors + +The following people have contributed to this code under the terms of +the Contributor Licence Agreement and Certificate of Origin detailed +below (_except for the parenthesised names, which represent contributions +from outside of NIWA and the Met Office that predate the explicit introduction +of this Agreement in July 2018; they must be un-parenthesised in future pull +requests_). + + + - Hilary Oliver + - Matt Shin + - Ben Fitzpatrick + - Andrew Clark + - Oliver Sanders + - Declan Valters + - Sadie Bartholomew + - (Luis Kornblueh) + - Kerry Day + - Prasanna Challuri + - David Matthews + - Tim Whitcomb + - Scott Wales + - Tomek Trzeciak + - Thomas Coleman + - Bruno Kinoshita + - (Annette Osprey) + - (Jonathan Thomas) + - Rosalyn Hatcher + - (Domingo Manubens Gil) + - Jonny Williams + - (Milton Woods) + - (Alex Reinecke) + - (Chandin Wilson) + - (Kevin Pulo) + - Lois Huggett + - (Martin Dix) + - (Ivor Blockley) + - Alexander Paulsell + - David Sutherland + - Martin Ryan + - Tim Pillinger + - Samuel Gaist + - Dima Veselov + - Gilliano Menezes + - Mel Hall + - Ronnie Dutta + - John Haiducek + - (Andrew Huang) + - Cheng Da + - Mark Dawson + - Diquan Jabbour + - Shixian Sheng + - Utheri Wagura + - Paul Earnshaw + + +(All contributors are identifiable with email addresses in the git version +control logs or otherwise.) + + +## Contributor Licence Agreement and Certificate of Origin + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I have + the right to submit it, either on my behalf or on behalf of my + employer, under the terms and conditions as described by this file; + or + +(b) The contribution is based upon previous work that, to the best of + my knowledge, is covered under an appropriate licence and I have + the right or permission from the copyright owner under that licence + to submit that work with modifications, whether created in whole or + in part by me, under the terms and conditions as described by + this file; or + +(c) The contribution was provided directly to me by some other person + who certified (a) or (b) and I have not modified it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including my + name and email address) is retained for the full term of + the copyright and may be redistributed consistent with this project + or the licence(s) involved. + +(e) I, or my employer, grant to NIWA and all recipients of + this software a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable copyright licence to reproduce, modify, + prepare derivative works of, publicly display, publicly perform, + sub-licence, and distribute this contribution and such modifications + and derivative works consistent with this project or the licence(s) + involved or other appropriate open source licence(s) specified by + the project and approved by the + [Open Source Initiative (OSI)](http://www.opensource.org/). + +(f) If I become aware of anything that would make any of the above + inaccurate, in any way, I will let NIWA know as soon as + I become aware. + +(The Cylc Contributor Licence Agreement and Certificate of Origin is +inspired that of [Rose](https://github.com/metomi/rose), which in turn was +inspired by the Certificate of Origin used by Enyo and the Linux Kernel.) diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 5c3c8d0c74d..00000000000 --- a/INSTALL +++ /dev/null @@ -1,74 +0,0 @@ - -_______________________________________________________________________ -INSTALLING CYLC FROM A SOURCE TARBALL - -Cylc is typically installed into a designated "cylc admin" user account -(/home/admin/ below, for illustrative purposes). - - % tar xzf cylc-x.y.z.tar.gz - % cd cylc-x.y.z - % export PATH=$PWD/bin:$PATH - % make # (see below to understand what 'make' does here) - -_______________________________________________________________________ -ARRANGING ACCESS TO CYLC - -Users need access to the the "cylc" command from suite and task host user -accounts. Do not simply put the bin directory of a specific cylc release in -$PATH for users, however. Instead install admin/cylc-wrapper as "cylc" in a -central location and ensure that location is in $PATH. The wrapper must be -modified slightly, as described in the file, to point to the location under -which multiple cycle versions will be intalled. For example: - - # Install the central wrapper: - % cp admin/cylc-wrapper /usr/local/bin/cylc - -Then (a) ensure that /usr/local/bin is in $PATH for users, and (b) modify -/usr/local/bin/cylc to point to /home/admin/cylc, where successive releases can -be installed like this: - - # Install cylc releases side by side: - /home/admin/cylc/cylc-6.0.0/ - /home/admin/cylc/cylc-6.1.0/ - /home/admin/cylc/cylc-6.1.2/ - # etc. - /home/admin/cylc/cylc -> cylc-6.3.0 # SYMLINK TO DEFAULT VERSION - -The central wrapper selects from the available versions of cylc according to -the value of an environment variable, $CYLC_VERSION. This allows running -suites to continue to function correctly even if they have not been upgraded -to the latest release. - -_______________________________________________________________________ -CYLC DEVELOPMENT - CLONING THE GIT REPOSITORY - - 1) To clone the cylc repository: - - % git clone git://github.com/cylc/cylc.git - % cd cylc - % make # build ordereddict and User Guide (as above) - % git pull origin master # update official changes - % make # remake documentation after changes - - 2) To participate in cylc development: fork cylc on github, clone your - own fork locally, commit changes in feature branches, push them to - your fork and issue a pull request to the cylc development team. - -_______________________________________________________________________ -WHAT HAPPENS WHEN YOU TYPE 'make' - - 1) a VERSION file is created containing the cylc version string, e.g. - 5.1.0. This is taken from the name of the parent directory - DO NOT - CHANGE THE NAME OF THE UNPACKED SOURCE TREE before running 'make'. - - 2) the Cylc User Guide is generated from LaTeX source files in doc/: - * if you have pdflatex installed, a PDF version is generated - * if you have tex4ht and ImageMagick convert installed, two HTML - versions (single- and multi-page) are generated - * a doc/index.html is created with links to the generated docs. - - 3) A Python module "orrdereddict" will be built from source in - ext/ordereddict-0.4.5. To make it available to cylc you must install - the built module into your $PYTHONPATH. This is not essential as a - Python implementation of the Ordered Dictionary structure will be used - by cylc if necessary. diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 00000000000..ffe0166ed25 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,15 @@ +# Cylc: Quick Installation Guide + +## Cylc 8 + +**See the [Cylc installation guide](https://cylc.github.io/cylc-doc/stable/html/installation.html) +for more detailed information.** + +Cylc must be installed on workflow and job hosts. Third-party dependencies +are not required on job hosts. + +## Cylc 7 + +See [INSTALL.md in the 7.8.x repository branch](https://github.com/cylc/cylc-flow/blob/7.8.x/INSTALL.md), or in your unpacked 7.8.x +release, for how to install Cylc 7. You can download the latest cylc-7 release +tarball from [Cylc Releases](https://github.com/cylc/cylc-flow/releases). diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000000..d2779ebdba8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include cylc/flow/py.typed +recursive-include cylc/flow/etc/ * diff --git a/Makefile b/Makefile deleted file mode 100644 index a7e92d9eee1..00000000000 --- a/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -all: version docs ordereddict - -version: - admin/create-version-file - -docs: - cd doc && $(MAKE) - -ordereddict: - cd ext/ordereddict-0.4.5 && python setup.py build - -clean: - cd doc && $(MAKE) clean diff --git a/README b/README deleted file mode 100644 index 4376c1a7520..00000000000 --- a/README +++ /dev/null @@ -1,35 +0,0 @@ -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -This is the Cylc Suite Engine, a workflow engine and metascheduler. - -Documentation: - * Installation: /path/to/cylc/INSTALL - * User Guide: /path/to/cylc/doc/index.html - * Project Home Page: http://cylc.github.com/cylc - -Code Contributors (git shortlog -s -n): - * Hilary Oliver - * Matt Shin - * Dave Matthews - * Ben Fitzpatrick - * Andrew Clark - * Luis Kornblueh - * Scott Wales - * Kevin Pulo - * Annette Osprey - * Tim Whitcomb - * Alex Reinecke diff --git a/README.md b/README.md new file mode 100644 index 00000000000..08f8d5938d5 --- /dev/null +++ b/README.md @@ -0,0 +1,113 @@ +
+ + +[![PyPI](https://img.shields.io/pypi/v/cylc-flow.svg?color=yellow)](https://pypi.org/project/cylc-flow/) +[![Anaconda-Server Badge](https://anaconda.org/conda-forge/cylc-flow/badges/version.svg)](https://anaconda.org/conda-forge/cylc-flow) +[![chat](https://img.shields.io/matrix/cylc-general:matrix.org)](https://matrix.to/#/#cylc-general:matrix.org) +[![forum](https://img.shields.io/discourse/https/cylc.discourse.group/posts.svg)](https://cylc.discourse.group/) +[![Documentation](https://img.shields.io/website?label=documentation&up_message=live&url=https%3A%2F%2Fcylc.github.io%2Fcylc-doc%2Fstable%2Fhtml%2Findex.html)](https://cylc.github.io/cylc-doc/stable/html/index.html) + +
+ +Cylc (pronounced silk) is a general purpose workflow engine that also +manages cycling systems very efficiently. It is used in production weather, +climate, and environmental forecasting on HPC, but is not specialized to those +domains. + +### Quick Start + + +[Installation](https://cylc.github.io/cylc-doc/stable/html/installation.html) | +[Documentation](https://cylc.github.io/cylc-doc/stable/html/index.html) + +```bash +# install cylc +conda install cylc-flow + +# extract an example to run +cylc get-resources examples/integer-cycling + +# install and run it +cylc vip integer-cycling # vip = validate, install and play + +# watch it run +cylc tui integer-cycling +``` + +### The Cylc Ecosystem + +- [cylc-flow](https://github.com/cylc/cylc-flow) - The core Cylc Scheduler for defining and running workflows. +- [cylc-uiserver](https://github.com/cylc/cylc-uiserver) - The web-based Cylc graphical user interface for monitoring and controlling workflows. +- [cylc-rose](https://github.com/cylc/cylc-rose) - Provides integration with [Rose](http://metomi.github.io/rose/). + +### Migrating From Cylc 7 + +[Migration Guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html) +| [Migration Support](https://cylc.discourse.group/c/cylc/7-to-8/13) + +Cylc 8 can run most Cylc 7 workflows in compatibility mode with little to no +changes, go through the +[migration guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html) +for more details. + +Quick summary of major changes: + +* Python 2 -> 3. +* Internal communications converted from HTTPS to ZMQ (TCP). +* PyGTK GUIs replaced by: + * Terminal user interface (TUI) included in cylc-flow. + * Web user interface provided by the cylc-uiserver package. +* A new scheduling algorithm with support for branched workflows. +* Command line changes: + * `cylc run ` -> `cylc play ` + * `cylc restart ` -> `cylc play ` + * `rose suite-run` -> `cylc install; cylc play ` +* The core package containing Cylc scheduler program has been renamed cylc-flow. +* Cylc review has been removed, the Cylc 7 version remains Cylc 8 compatible. + + +### Citations & Publications + +[![DOI](https://zenodo.org/badge/1836229.svg)](https://zenodo.org/badge/latestdoi/1836229) +[![JOSS](http://joss.theoj.org/papers/10.21105/joss.00737/status.svg)](https://doi.org/10.21105/joss.00737) +[![CISE](https://img.shields.io/website/https/ieeexplore.ieee.org/document/8675433.svg?color=orange&label=CISE&up_message=10.1109%2FMCSE.2019.2906593)](https://ieeexplore.ieee.org/document/8675433) + +### Copyright and Terms of Use + +[![License](https://img.shields.io/github/license/cylc/cylc-flow.svg?color=lightgrey)](https://github.com/cylc/cylc-flow/blob/master/COPYING) + +Copyright (C) 2008-2024 NIWA & British Crown (Met Office) & Contributors. + +Cylc is free software: you can redistribute it and/or modify it under the terms +of the GNU General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. + +Cylc is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +Cylc. If not, see [GNU licenses](http://www.gnu.org/licenses/). + +### Contributing + +[![Contributors](https://img.shields.io/github/contributors/cylc/cylc-flow.svg?color=9cf)](https://github.com/cylc/cylc-flow/graphs/contributors) +[![Commit activity](https://img.shields.io/github/commit-activity/m/cylc/cylc-flow.svg?color=yellowgreen)](https://github.com/cylc/cylc-flow/commits/master) +[![Last commit](https://img.shields.io/github/last-commit/cylc/cylc-flow.svg?color=ff69b4)](https://github.com/cylc/cylc-flow/commits/master) + +Contributions welcome: + +* Read the [contributing](https://github.com/cylc/cylc-flow/blob/master/CONTRIBUTING.md) page. +* Development setup instructions are in the + [developer docs](https://cylc.github.io/cylc-admin/#cylc-8-developer-docs). +* Involved change proposals can be found in the + [admin pages](https://cylc.github.io/cylc-admin/#change-proposals). +* Touch base in the + [developers chat](https://matrix.to/#/#cylc-general:matrix.org). + +This repository contains some code that was generated by GitHub Copilot. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..c1821870264 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,39 @@ +# Security Policies and Procedures + +This document outlines security procedures and general policies for the Cylc +project. + + * [Reporting a Bug](#reporting-a-bug) + * [Disclosure Policy](#disclosure-policy) + * [Comments on this Policy](#comments-on-this-policy) + +## Reporting a Bug + +The Cylc maintainers take security bugs seriously. Thank you for improving the +security of Cylc. We appreciate your efforts and responsible disclosure and +will make every effort to acknowledge your contributions. + +Please report security bugs by sending an email to the lead Cylc maintainers, +[Hilary Oliver](mailto:hilary.oliver@niwa.co.nz) and [Oliver +Sanders](mailto:oliver.sanders@metoffice.gov.uk). If a fix is needed, progress will be +recorded on Cylc repository Issue page on GitHub, and resulting new releases +will be announced on the Cylc [Discourse forum](https://cylc.discourse.group/). + +Report security bugs in third-party modules to the person or team maintaining +the module. + +## Disclosure Policy + +When the Cylc maintainers receive a security bug report, they will assign it to +a primary handler. This person will coordinate the fix and release process as +follows: + + * Confirm the problem and determine the affected versions. + * Audit code to find any potential similar problems. + * Prepare fixes for all releases still under maintenance. These fixes will be + released as fast as possible. + +## Comments on this Policy + +If you have suggestions on how this process could be improved please submit a +pull request. diff --git a/admin/create-version-file b/admin/create-version-file deleted file mode 100755 index 8c2311565fa..00000000000 --- a/admin/create-version-file +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -e - -if [[ -d .git || -f .git ]]; then - echo "(a cylc VERSION file is not needed in a cylc git repo)" >&2 -else - VN=${PWD#*cylc-} - echo "Setting VERSION from parent directory name: $VN" >&2 - echo $VN > VERSION -fi diff --git a/admin/cylc-wrapper b/admin/cylc-wrapper deleted file mode 100755 index c8f853760b4..00000000000 --- a/admin/cylc-wrapper +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Wrapper for selection between installed versions of cylc. -# Author: Matt Shin (Met Office). - -#_____________________________ -# INSTALLATION: -# -# 1. Install this script as "cylc" in $PATH for normal users, AND -# create a symlink gcylc -> cylc to allow continued use of "gcylc" for -# "cylc gui" under this framework. -# -# 2. Install cylc versions as described in the INSTALL file, e.g.: -# /home/admin/cylc/cylc-5.4.4/ -# /home/admin/cylc/cylc-5.4.5/ -# /home/admin/cylc/cylc -> cylc.5.4.5 # symlink DEFAULT version -# -# Environment variables: -# $CYLC_HOME_ROOT: top installation directory, e.g. /home/admin/cylc/ -# $CYLC_HOME: a specific version, e.g. /home/admin/cylc/cylc-5.4.5/ -# $CYLC_VERSION: a specific version string, e.g. 5.4.5 -# -# 3. Set $CYLC_HOME_ROOT for your site - see "!!! EDIT ME !!! below. - -#_____________________________ -# ACTION IN ORDER OF PRIORITY: -# -# 1) If $CYLC_HOME is defined and exists it will be selected. If it does -# not exist the command will fail. -# -# 2) If $CYLC_VERSION is defined and exists it will be selected. If it -# does not exist the default under $CYLC_HOME_ROOT will be selected. -# -# 3) If $CYLC_HOME and $CYLC_VERSION are both undefined, the default -# under $CYLC_HOME_ROOT will be selected. - -#_____________________________ -# NOTES: -# * Users will typically accept the default or set $CYLC_VERSION, but -# they may set $CYLC_HOME to pick up a private cylc installation. -# * Developers may set $CYLC_HOME to pick up their own repository clone. -# * Running suites export $CYLC_VERSION to task environments, so that -# task messaging commands etc. pick up the compatible cylc version. -# * Note that $CYLC_ROOT_HOME can be overridden in the environment too. - -CYLC_HOME_ROOT=${CYLC_HOME_ROOT:-/home/admin/cylc} # !!! EDIT ME !!! - -if [[ -z ${CYLC_HOME:-} ]]; then - if [[ -n ${CYLC_VERSION:-} && -d $CYLC_HOME_ROOT/cylc-$CYLC_VERSION ]]; then - CYLC_HOME=$CYLC_HOME_ROOT/cylc-$CYLC_VERSION - else - CYLC_HOME=$CYLC_HOME_ROOT/cylc - fi -fi -exec $CYLC_HOME/bin/$(basename $0) "$@" diff --git a/admin/get-repo-version b/admin/get-repo-version deleted file mode 100755 index b430b3cc1f3..00000000000 --- a/admin/get-repo-version +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# This script determines the cylc version string in a repository clone -# and echoes it to stdout. It is executed by lib/cylc/version.py when -# cylc is run out of a repository. The version string is returned by -# 'git describe' and then "-dirty" is appended if the working directory -# is not clean. - -LF=' -' - -# move to the repository -cd $( dirname "$0" )/.. -VN=$(git describe --abbrev=4 --tags HEAD 2>/dev/null) -case "$VN" in - *$LF*) (exit 1) ;; - [0-9]*) - # If uncommitted changes exist append "-dirty". - - if [[ -n "$(git status --untracked-files=no --porcelain)" ]]; then - VN="$VN-dirty" - fi -esac -# echo to stdout -echo "$VN" diff --git a/admin/make-tarball b/admin/make-tarball deleted file mode 100755 index ef49efdfaef..00000000000 --- a/admin/make-tarball +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -set -e; trap "echo ERROR" ERR - -function usage { -cat <&2 - exit 1 -fi - -VERSION=$( cylc -v ) -if [[ $VERSION = *-dirty ]]; then - echo "The work tree is not clean: ABORTING" >&2 - exit 1 -fi - -RELEASE=cylc-$VERSION -TARBALL=${RELEASE}.tar.gz -BRANCH=$( git rev-parse --abbrev-ref HEAD ) - -git archive $BRANCH --prefix=$RELEASE/ | gzip > $TARBALL -ls -lh $TARBALL diff --git a/admin/new-release.README b/admin/new-release.README deleted file mode 100644 index e40e11e797e..00000000000 --- a/admin/new-release.README +++ /dev/null @@ -1,11 +0,0 @@ - -How to generate a new cylc release and update the cylc homepage (on -github gh-pages): - - 1 run the test battery - 2 update doc/changes.html and commit the change - 3 tag with the new release number x.y.z - 4 push new commit and the tag to master on cylc/cylc - 5 generate the new documentation and update it to gh-pages: - - % admin/update-gh-pages -p "release x.y.z" diff --git a/admin/update-gh-pages.sh b/admin/update-gh-pages.sh deleted file mode 100755 index acbb031f8c7..00000000000 --- a/admin/update-gh-pages.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -set -e -set -x - -function usage { -cat <$LATESTTAG ($( date +%F )) \2@" index.html - -# in case new files were added: -git add graphics/png/scaled/ -git add html/single/ -git add html/multi/ -git add screenshots/ - -# any changes to update? -git update-index -q --refresh -if [[ -n "$(git diff-index --name-only HEAD --)" ]]; then - echo "committing changes"; - git commit -a -m "$COMMITMSG" -else - # (attempted commit is an error if there's nothing to commit) - echo "no changes to commit"; -fi - -# push to github if requested -$PUSH && git push cylc gh-pages - -# return to master -git checkout master - -echo "DONE: gh-pages updated." diff --git a/bin/cycl b/bin/cycl deleted file mode 100755 index 4272869c501..00000000000 --- a/bin/cycl +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Catch a common command line typo: 'cycl'. This could be done by shell -# alias, but we're relieving the user of the burden. - -echo >&2 -echo "TYPO ALERT: it's \"cylc\", not \"cycl\"!" >&2 -echo >&2 -exec $(dirname $0)/cylc "$@" diff --git a/bin/cylc b/bin/cylc deleted file mode 100755 index ab09ff463e3..00000000000 --- a/bin/cylc +++ /dev/null @@ -1,684 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -import re -import sys - - -def prelude(): - """Ensure cylc library is at the front of "sys.path".""" - lib = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', 'lib') - if lib in sys.path: - sys.path.remove(lib) - sys.path.insert(0, lib) - - -prelude() - - -try: - os.getcwd() -except OSError as exc: - # The current working directory has been deleted (or filesystem - # problems of some kind...). This results in Pyro not being found, - # immediately below. We cannot just chdir to $HOME as gcylc does - # because that would break relative directory path command arguments - # (cylc reg SUITE PATH). - sys.exit(exc) - -# Import cylc to initialise CYLC_DIR and the path for python (__init__.py). -import cylc -from parsec.OrderedDict import OrderedDict - - -class CommandError(Exception): - - def __init__(self, msg): - self.msg = msg - - def __str__(self): - return repr(self.msg) - - -class CommandNotFoundError(CommandError): - pass - - -class CommandNotUniqueError(CommandError): - pass - - -def is_help(str): - if (str == '-h' or str == '--help' or str == '--hlep' or str == 'help' or - str == 'hlep' or str == '?'): - return True - else: - return False - - -def match_dict(abbrev, categories, title): - # allow any unique abbreviation to cylc categories - matches = [] - for cat in categories.keys(): - for alias in categories[cat]: - if re.match('^' + abbrev + '.*', alias): - if cat not in matches: - matches.append(cat) - if len(matches) == 0: - raise CommandNotFoundError(title + ' not found: ' + abbrev) - elif len(matches) > 1: - # multiple matches - res = '' - for cat in matches: - res += ' ' + '|'.join(categories[cat]) - raise CommandNotUniqueError( - title + ' "' + abbrev + '" not unique:' + res) - else: - return matches[0] - - -def match_command(abbrev): - # allow any unique abbreviation to commands when no category is specified - matches = [] - finished_matching = False - for dct in [admin_commands, - license_commands, - database_commands, - preparation_commands, - information_commands, - discovery_commands, - control_commands, - utility_commands, - hook_commands, - task_commands]: - for com in dct.keys(): - if com == abbrev: - matches = [com] - finished_matching = True - break - for alias in dct[com]: - if re.match('^' + abbrev + '.*', alias): - if com not in matches: - matches.append(com) - if finished_matching: - break - if len(matches) == 0: - raise CommandNotFoundError('COMMAND not found: ' + abbrev) - elif len(matches) > 1: - # multiple matches - res = '' - for com in matches: - res += ' ' + '|'.join(all_commands[com]) - raise CommandNotUniqueError( - 'COMMAND "' + abbrev + '" not unique:' + res) - else: - return matches[0] - - -def pretty_print(incom, choose_dict, indent=True, numbered=False, sort=False): - # pretty print commands or topics from a dict: - # (com[item] = description) - - if indent: - spacer = ' ' - else: - spacer = '' - - label = {} - choose = [] - longest = 0 - for item in choose_dict: - choose.append(item) - lbl = '|'.join(choose_dict[item]) - label[item] = lbl - if len(lbl) > longest: - longest = len(lbl) - - count = 0 - pad = False - if len(choose) > 9: - pad = True - - if sort: - choose.sort() - for item in choose: - if item not in incom: - raise SystemExit("ERROR: summary for '" + item + "' not found") - - print spacer, - if numbered: - count += 1 - if pad and count < 10: - digit = ' ' + str(count) - else: - digit = str(count) - print digit + '/', - print label[item], '.'*(longest-len(label[item])) + '...', incom[item] - -# BEGIN MAIN - -# categories[category] = [aliases] -categories = OrderedDict() -categories['all'] = ['all'] -categories['database'] = ['db', 'database'] -categories['preparation'] = ['preparation'] -categories['information'] = ['information'] -categories['discovery'] = ['discovery'] -categories['control'] = ['control'] -categories['utility'] = ['utility'] -categories['task'] = ['task'] -categories['hook'] = ['hook'] -categories['admin'] = ['admin'] -categories['license'] = ['license', 'GPL'] - -information_commands = OrderedDict() - -information_commands['gscan'] = ['gscan', 'gsummary'] -information_commands['gpanel'] = ['gpanel'] -information_commands['gui'] = ['gui', 'gcylc'] -information_commands['list'] = ['list', 'ls'] -information_commands['dump'] = ['dump'] -information_commands['cat-state'] = ['cat-state'] -information_commands['show'] = ['show'] -information_commands['cat-log'] = ['cat-log', 'log'] -information_commands['get-suite-version'] = [ - 'get-suite-version', 'get-cylc-version'] -information_commands['version'] = ['version'] - -information_commands['documentation'] = ['documentation', 'browse'] -information_commands['monitor'] = ['monitor'] -information_commands['get-suite-config'] = ['get-suite-config', 'get-config'] -information_commands['get-site-config'] = [ - 'get-site-config', 'get-global-config'] -information_commands['get-gui-config'] = ['get-gui-config'] - -control_commands = OrderedDict() -control_commands['gui'] = ['gui'] -# NOTE: don't change 'run' to 'start' or the category [control] -# becomes compulsory to disambiguate from 'cylc [task] started'. -# Keeping 'start' as an alias however: 'cylc con start'. -control_commands['run'] = ['run', 'start'] -control_commands['stop'] = ['stop', 'shutdown'] -control_commands['restart'] = ['restart'] -control_commands['trigger'] = ['trigger'] -control_commands['insert'] = ['insert'] -control_commands['remove'] = ['remove'] -control_commands['poll'] = ['poll'] -control_commands['kill'] = ['kill'] -control_commands['hold'] = ['hold'] -control_commands['release'] = ['release', 'unhold'] -control_commands['reset'] = ['reset'] -control_commands['nudge'] = ['nudge'] -control_commands['reload'] = ['reload'] -control_commands['set-runahead'] = ['set-runahead'] -control_commands['set-verbosity'] = ['set-verbosity'] -control_commands['broadcast'] = ['broadcast', 'bcast'] -control_commands['ext-trigger'] = ['ext-trigger', 'external-trigger'] - -utility_commands = OrderedDict() -utility_commands['cycle-point'] = [ - 'cycle-point', 'cyclepoint', 'datetime', 'cycletime'] -utility_commands['random'] = ['random', 'rnd'] -utility_commands['scp-transfer'] = ['scp-transfer'] -utility_commands['suite-state'] = ['suite-state'] - -hook_commands = OrderedDict() -hook_commands['email-suite'] = ['email-suite'] -hook_commands['email-task'] = ['email-task'] -hook_commands['job-logs-retrieve'] = ['job-logs-retrieve'] -hook_commands['check-triggering'] = ['check-triggering'] - -admin_commands = OrderedDict() -admin_commands['test-db'] = ['test-db'] -admin_commands['test-battery'] = ['test-battery'] -admin_commands['import-examples'] = ['import-examples'] -admin_commands['upgrade-db'] = ['upgrade-db'] -admin_commands['upgrade-run-dir'] = ['upgrade-run-dir'] -admin_commands['check-software'] = ['check-software'] - -license_commands = OrderedDict() -license_commands['warranty'] = ['warranty'] -license_commands['conditions'] = ['conditions'] - -database_commands = OrderedDict() -database_commands['register'] = ['register'] -database_commands['reregister'] = ['reregister', 'rename'] -database_commands['unregister'] = ['unregister'] -database_commands['copy'] = ['copy', 'cp'] -database_commands['print'] = ['print'] -database_commands['get-directory'] = ['get-directory'] -database_commands['refresh'] = ['refresh'] - -preparation_commands = OrderedDict() -preparation_commands['edit'] = ['edit'] -preparation_commands['view'] = ['view'] -preparation_commands['validate'] = ['validate'] -preparation_commands['5to6'] = ['5to6'] -preparation_commands['list'] = ['list', 'ls'] -preparation_commands['search'] = ['search', 'grep'] -preparation_commands['graph'] = ['graph'] -preparation_commands['graph-diff'] = ['graph-diff'] -preparation_commands['diff'] = ['diff', 'compare'] -preparation_commands['jobscript'] = ['jobscript'] - -discovery_commands = OrderedDict() -discovery_commands['ping'] = ['ping'] -discovery_commands['scan'] = ['scan'] -discovery_commands['check-versions'] = ['check-versions'] - -task_commands = OrderedDict() -task_commands['submit'] = ['submit', 'single'] -task_commands['message'] = ['message', 'task-message'] -task_commands['jobs-kill'] = ['jobs-kill'] -task_commands['jobs-poll'] = ['jobs-poll'] -task_commands['jobs-submit'] = ['jobs-submit'] -task_commands['job-kill'] = ['job-kill'] -task_commands['job-poll'] = ['job-poll'] -task_commands['job-submit'] = ['job-submit'] - -all_commands = OrderedDict() -for dct in [ - database_commands, - preparation_commands, - information_commands, - discovery_commands, - control_commands, - utility_commands, - task_commands, - admin_commands, - hook_commands, - license_commands]: - for com in dct.keys(): - all_commands[com] = dct[com] - -general_usage = """ -Cylc ("silk") is a suite engine and metascheduler that specializes in -cycling weather and climate forecasting suites and related processing -(but it can also be used for one-off workflows of non-cycling tasks). -For detailed documentation see the Cylc User Guide (cylc doc --help). - -Version __CYLC_VERSION__ - -The graphical user interface for cylc is "gcylc" (a.k.a. "cylc gui"). - -USAGE: - % cylc -v,--version # print cylc version - % cylc version # (ditto, by command) - % cylc help,--help,-h,? # print this help page - - % cylc help CATEGORY # print help by category - % cylc CATEGORY help # (ditto) - - % cylc help [CATEGORY] COMMAND # print command help - % cylc [CATEGORY] COMMAND help,--help # (ditto) - - % cylc [CATEGORY] COMMAND [options] SUITE [arguments] - % cylc [CATEGORY] COMMAND [options] SUITE TASK [arguments]""" - -# topic summaries -catsum = OrderedDict() -catsum['all'] = "The complete command set." -catsum['admin'] = "Cylc installation, testing, and example suites." -catsum['license'] = "Software licensing information (GPL v3.0)." -catsum['database'] = "Suite name registration, copying, deletion, etc." -catsum['information'] = "Interrogate suite definitions and running suites." -catsum['preparation'] = "Suite editing, validation, visualization, etc." -catsum['discovery'] = "Detect running suites." -catsum['control'] = "Suite start up, monitoring, and control." -catsum['task'] = "The task messaging interface." -catsum['hook'] = "Suite and task event hook scripts." -catsum['utility'] = "Cycle arithmetic and templating, etc." - -usage = general_usage + """ - -Commands and categories can both be abbreviated. Use of categories is -optional, but they organize help and disambiguate abbreviated commands: - % cylc control trigger SUITE TASK # trigger TASK in SUITE - % cylc trigger SUITE TASK # ditto - % cylc con trig SUITE TASK # ditto - % cylc c t SUITE TASK # ditto - -CYLC SUITE NAMES AND YOUR REGISTRATION DATABASE - Suites are addressed by hierarchical names such as suite1, nwp.oper, -nwp.test.LAM2, etc. in a "name registration database" ($HOME/.cylc/REGDB) -that simply associates names with the suite definition locations. The -'--db=' command option can be used to view and copy suites from other -users, with access governed by normal filesystem permissions. - -TASK IDENTIFICATION IN CYLC SUITES - Tasks are identified by NAME.CYCLE_POINT where POINT is either a - date-time or an integer. - Date-time cycle points are in an ISO 8601 date-time format, typically - CCYYMMDDThhmm followed by a time zone - e.g. 20101225T0600Z. - Integer cycle points (including those for one-off suites) are integers - - just '1' for one-off suites. - -HOW TO DRILL DOWN TO COMMAND USAGE HELP: - % cylc help # list all available categories (this page) - % cylc help prep # list commands in category 'preparation' - % cylc help prep edit # command usage help for 'cylc [prep] edit' - -Command CATEGORIES:""" - -# Some commands and categories are aliased (db|database, cp|copy) and -# some common typographical errors are corrected (e.g. cycl => cylc). - -# command summaries -comsum = OrderedDict() -# admin -comsum['test-db'] = 'Run an automated suite name database test' -comsum['test-battery'] = 'Run a battery of self-diagnosing test suites' -comsum['import-examples'] = 'Import example suites your suite name database' -comsum['upgrade-db'] = 'Upgrade a pre-cylc-5.4 suite name database' -comsum['upgrade-run-dir'] = 'Upgrade a pre-cylc-6 suite run directory' -comsum['check-software'] = 'Check required software is installed.' -# license -comsum['warranty'] = 'Print the GPLv3 disclaimer of warranty' -comsum['conditions'] = 'Print the GNU General Public License v3.0' -# database -comsum['register'] = 'Register a suite for use' -comsum['reregister'] = 'Change the name of a suite' -comsum['unregister'] = 'Unregister and optionally delete suites' -comsum['copy'] = 'Copy a suite or a group of suites' -comsum['print'] = 'Print registered suites' -comsum['get-directory'] = 'Retrieve suite definition directory paths' -comsum['refresh'] = 'Report invalid registrations and update suite titles' -# preparation -comsum['edit'] = 'Edit suite definitions, optionally inlined' -comsum['view'] = 'View suite definitions, inlined and Jinja2 processed' -comsum['validate'] = 'Parse and validate suite definitions' -comsum['5to6'] = 'Improve the cylc 6 compatibility of a cylc 5 suite file' -comsum['search'] = 'Search in suite definitions' -comsum['graph'] = 'Plot suite dependency graphs and runtime hierarchies' -comsum['graph-diff'] = 'Compare two suite dependencies or runtime hierarchies' -comsum['diff'] = 'Compare two suite definitions and print differences' -# information -comsum['list'] = 'List suite tasks and family namespaces' -comsum['dump'] = 'Print the state of tasks in a running suite' -comsum['cat-state'] = 'Print the state of tasks from the state dump' -comsum['show'] = 'Print task state (prerequisites and outputs etc.)' -comsum['cat-log'] = 'Print various suite and task log files' -comsum['documentation'] = 'Display cylc documentation (User Guide etc.)' -comsum['monitor'] = 'An in-terminal suite monitor (see also gcylc)' -comsum['get-suite-config'] = 'Print suite configuration items' -comsum['get-site-config'] = 'Print site/user configuration items' -comsum['get-gui-config'] = 'Print gcylc configuration items' -comsum['get-suite-version'] = 'Print the cylc version of a suite daemon' -comsum['version'] = 'Print the cylc release version' -comsum['gscan'] = 'Scan GUI for monitoring multiple suites' -comsum['gpanel'] = 'Internal interface for GNOME 2 panel applet' -# control -comsum['gui'] = '(a.k.a. gcylc) cylc GUI for suite control etc.' -comsum['run'] = 'Start a suite at a given cycle point' -comsum['stop'] = 'Shut down running suites' -comsum['restart'] = 'Restart a suite from a previous state' -comsum['trigger'] = 'Manually trigger or re-trigger a task' -comsum['insert'] = 'Insert tasks into a running suite' -comsum['remove'] = 'Remove tasks from a running suite' -comsum['poll'] = 'Poll submitted or running tasks' -comsum['kill'] = 'Kill submitted or running tasks' -comsum['hold'] = 'Hold (pause) suites or individual tasks' -comsum['release'] = 'Release (unpause) suites or individual tasks' -comsum['reset'] = 'Force one or more tasks to change state.' -comsum['nudge'] = 'Cause the cylc task processing loop to be invoked' -comsum['reload'] = 'Reload the suite definition at run time' -comsum['set-runahead'] = 'Change the runahead limit in a running suite.' -comsum['set-verbosity'] = 'Change a running suite\'s logging verbosity' -comsum['ext-trigger'] = 'Report an external trigger event to a suite' -# discovery -comsum['ping'] = 'Check that a suite is running' -comsum['scan'] = 'Scan a host for running suites' -comsum['check-versions'] = 'Compare cylc versions on task host accounts' -# task -comsum['submit'] = 'Run a single task just as its parent suite would' -comsum['message'] = '(task messaging) Report task messages' -comsum['broadcast'] = 'Change suite [runtime] settings on the fly' -comsum['jobs-kill'] = '(Internal) Kill task jobs' -comsum['jobs-poll'] = '(Internal) Retrieve status for task jobs' -comsum['jobs-submit'] = '(Internal) Submit task jobs' -comsum['job-kill'] = '(Internal) Kill a task job' -comsum['job-poll'] = '(Internal) Retrieve status for a task job' -comsum['job-submit'] = '(Internal) Submit a job' - -# utility -comsum['cycle-point'] = 'Cycle point arithmetic and filename templating' -comsum['random'] = 'Generate a random integer within a given range' -comsum['jobscript'] = 'Generate a task job script and print it to stdout' -comsum['scp-transfer'] = 'Scp-based file transfer for cylc suites' -comsum['suite-state'] = 'Query the task states in a suite' - -# hook -comsum['email-task'] = 'A task event hook script that sends email alerts' -comsum['email-suite'] = 'A suite event hook script that sends email alerts' -comsum['job-logs-retrieve'] = ( - '(Internal) Retrieve logs from a remote host for a task job') -comsum['check-triggering'] = 'A suite shutdown event hook for cylc testing' - - -def typo(str): - corrected = str - if str == 'gcycl': - corrected = 'gcylc' - return corrected - - -def category_help(category): - coms = eval(category + '_commands') - alts = '|'.join(categories[category]) - print 'CATEGORY: ' + alts + ' - ' + catsum[category] - if category == 'database': - print "Suite name registrations are held under $HOME/.cylc/REGDB." - print - print 'HELP: cylc [' + alts + '] COMMAND help,--help' - print ' You can abbreviate ' + alts + ' and COMMAND.' - print ' The category ' + alts + ' may be omitted.' - print - print 'COMMANDS:' - pretty_print(comsum, coms, sort=True) - - -def set_environment_vars(args): - """ - Set --env=key=val arguments as environment variables & remove - from argument list - """ - regex = re.compile('\A--env=(\S+)=(\S+)\Z') - for arg in args: - match = regex.match(arg) - if match is None: - continue - os.environ[match.group(1)] = match.group(2) - return filter(lambda i: not regex.search(i), args) - -# no arguments: print help and exit -if len(sys.argv) == 1: - from cylc.version import CYLC_VERSION - print usage.replace("__CYLC_VERSION__", CYLC_VERSION) - pretty_print(catsum, categories) - sys.exit(1) - -args = sys.argv[1:] - -# Set environment variables from arguments like --env=key=val -args = set_environment_vars(args) - -if len(args) == 1: - if args[0] == 'categories': - # secret argument for document processing - keys = catsum.keys() - keys.sort() - for key in keys: - print key - sys.exit(0) - if args[0] == 'commands': - # secret argument for document processing - keys = comsum.keys() - keys.sort() - for key in keys: - print key - sys.exit(0) - if args[0].startswith('category='): - # secret argument for gcylc - category = args[0][9:] - commands = eval(category + '_commands') - for command in commands: - print command - sys.exit(0) - if is_help(args[0]): - # cylc help - from cylc.version import CYLC_VERSION - print usage.replace("__CYLC_VERSION__", CYLC_VERSION) - pretty_print(catsum, categories) - sys.exit(0) - if (args[0] == '-v' or args[0] == '--version'): - from cylc.version import CYLC_VERSION - print CYLC_VERSION - sys.exit(0) - - # cylc CATEGORY with no args => category help - try: - category = match_dict(args[0], categories, 'CATEGORY') - except CommandError, x: - # No matching category - # (no need to print this, the exception will recur below) - # Carry on in case of a no-argument command (e.g. 'cylc scan') - pass - else: - category_help(category) - sys.exit(0) - -command_args = [] - -if len(args) == 2 and (is_help(args[0]) or is_help(args[1])): - # TWO ARGUMENTS, one help - # cylc help CATEGORY - # cylc CATEGORY help - # cylc help COMMAND - # cylc COMMAND help - if is_help(args[1]): - item = args[0] - else: - item = args[1] - try: - category = match_dict(item, categories, 'CATEGORY') - except CommandError, x: - # no matching category, try command - try: - command = match_command(typo(item)) - except CommandError, y: - print >> sys.stderr, x - raise SystemExit(y) - else: - # cylc COMMAND --help - command_args = ['--help'] - else: - # cylc help CATEGORY - category_help(category) - sys.exit(0) - -elif len(args) == 3 and (is_help(args[0]) or is_help(args[2])): - # cylc help CATEGORY COMMAND - # cylc CATEGORY COMMAND help - if is_help(args[2]): - category = args[0] - command = args[1] - else: - category = args[1] - command = args[2] - try: - category = match_dict(category, categories, 'CATEGORY') - except CommandError, x: - raise SystemExit(x) - - coms = eval(category + '_commands') - try: - command = match_dict(command, coms, category + ' COMMAND') - except CommandNotUniqueError, y: - print y - sys.exit(1) - except CommandNotFoundError, y: - print y - print 'COMMANDS available in CATEGORY "' + category + '":' - print coms.keys() - sys.exit(1) - - # cylc COMMAND --help - command_args = ['--help'] - -else: - # two or more args, neither of first two are help - # cylc CATEGORY COMMAND [ARGS] - # cylc COMMAND [ARGS] - try: - category = args[0] - category = match_dict(category, categories, 'CATEGORY') - except CommandError, x: - # no matching category, try command - try: - command = args[0] - command = match_command(typo(command)) - except CommandError, y: - print >> sys.stderr, x - raise SystemExit(y) - else: - # cylc COMMAND [ARGS] - command_args = args[1:] - else: - # cylc CATEGORY COMMAND [ARGS] - coms = eval(category + '_commands') - command = args[1] - try: - command = match_dict(command, coms, category + ' COMMAND') - except CommandNotUniqueError, y: - print y - sys.exit(1) - except CommandNotFoundError, y: - print y - print 'COMMANDS available in CATEGORY "' + category + '":' - print coms.keys() - sys.exit(1) - - else: - # cylc COMMAND [ARGS] - if len(args) > 1: - command_args = args[2:] - else: - command_args = [] - -args_new = [] -for item in command_args: - if is_help(item): - # transform all legal help options to '--help' - args_new.append('--help') - elif item.startswith('--owner'): - # deprecate '--owner' to '--user' - args_new.append(item.replace('--owner', '--user')) - else: - args_new.append(item) -args = args_new - -cmd = sys.argv[0] + '-' + command - -# Replace the current process with that of the sub-command. -try: - os.execvp(cmd, [cmd] + args) -except OSError, exc: - if exc.filename is None: - exc.filename = cmd - raise SystemExit(exc) diff --git a/bin/cylc-5to6 b/bin/cylc-5to6 deleted file mode 100755 index 5241abe5be2..00000000000 --- a/bin/cylc-5to6 +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Transform a cylc 5 suite rc file into a cylc 6 suite rc file where possible. - -usage() { - cat < FILE - -# Save a copy of the changed file. -cylc 5to6 FILE > FILE.5to6 - -# Show the diff of the changed file vs the original file. -diff - <(cylc 5to6 FILE) . - -"""cylc [control] broadcast|bcast [OPTIONS] REG - -Override [runtime] config in targeted namespaces in a running suite. - -Uses for broadcast include making temporary changes to task behaviour, -and task-to-downstream-task communication via environment variables. - -A broadcast can target any [runtime] namespace for all cycles or for a -specific cycle. If a task is affected by specific-cycle and all-cycle -broadcasts at once, the specific takes precedence. If a task is affected -by broadcasts to multiple ancestor namespaces, the result is determined -by normal [runtime] inheritance. In other words, it follows this order: - -all:root -> all:FAM -> all:task -> tag:root -> tag:FAM -> tag:task - -Broadcasts persist, even across suite restarts, until they expire when -their target cycle point is older than the oldest current in the suite, -or until they are explicitly cancelled with this command. All-cycle -broadcasts do not expire. - -For each task the final effect of all broadcasts to all namespaces is -computed on the fly just prior to job submission. The --cancel and ---clear options simply cancel (remove) active broadcasts, they do not -act directly on the final task-level result. Consequently, for example, -you cannot broadcast to "all cycles except Tn" with an all-cycle -broadcast followed by a cancel to Tn (there is no direct broadcast to Tn -to cancel); and you cannot broadcast to "all members of FAMILY except -member_n" with a general broadcast to FAMILY followed by a cancel to -member_n (there is no direct broadcast to member_n to cancel). - -To broadcast a variable to all tasks (quote items with internal spaces): - % cylc broadcast -s "[environment]VERSE = the quick brown fox" REG -To cancel the same broadcast: - % cylc broadcast --cancel "[environment]VERSE" REG - -Use -d/--display to see active broadcasts. Multiple set or cancel -options can be used on the same command line. Broadcast cannot change -[runtime] inheritance. - -See also 'cylc reload' - reload a modified suite definition at run time.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import os -import re - -import cylc.flags -from cylc.broadcast_report import ( - get_broadcast_change_report, get_broadcast_bad_options_report) -from cylc.CylcOptionParsers import cop -from cylc.network.suite_broadcast import BroadcastClient -from cylc.print_tree import print_tree -from cylc.task_id import TaskID -from cylc.cfgspec.suite import SPEC, upg -from parsec.validate import validate - - -def get_padding(settings, level=0, padding=0): - level += 1 - for key, val in settings.items(): - tmp = level*2 + len(key) - if tmp > padding: - padding = tmp - if isinstance(val, dict): - padding = get_padding(val, level, padding) - return padding - - -def get_rdict(left, right=None): - # left is [section]item, or just item - rdict = {} - m = re.match('^\[(.*)\](.*)$', left) - if m: - # [sect]item = right - sect, var = m.groups() - if not var: - rdict = {sect: right} - else: - rdict = {sect: {var: right}} - else: - # item = right - rdict = {left: right} - return rdict - - -def main(): - parser = cop(__doc__, pyro=True) - - parser.add_option( - "-t", "--tag", metavar="CYCLE_POINT", - help="(Deprecated). " - "Target cycle point. More than one can be added. " - "Defaults to '*' for all cycle points with --set and --cancel, " - "and nothing with --clear.", - action="append", dest="point_strings", default=[]) - - parser.add_option( - "-p", "--point", metavar="CYCLE_POINT", - help="Target cycle point. More than one can be added. " - "Defaults to '*' with --set and --cancel, " - "and nothing with --clear.", - action="append", dest="point_strings", default=[]) - - parser.add_option( - "-n", "--namespace", metavar="NAME", - help="Target namespace. Defaults to 'root' with " - "--set and --cancel, and nothing with --clear.", - action="append", dest="namespaces", default=[]) - - parser.add_option( - "-s", "--set", metavar="[SEC]ITEM=VALUE", - help="A [runtime] config item and value to broadcast.", - action="append", dest="set", default=[]) - - parser.add_option( - "-c", "--cancel", metavar="[SEC]ITEM", - help="An item-specific broadcast to cancel.", - action="append", dest="cancel", default=[]) - - parser.add_option( - "-C", "--clear", - help="Cancel all broadcasts, or with -p/--point, " - "-n/--namespace, cancel all broadcasts to targeted " - "namespaces and/or cycle points. Use \"-C -p '*'\" " - "to cancel all all-cycle broadcasts without canceling " - "all specific-cycle broadcasts.", - action="store_true", dest="clear", default=False) - - parser.add_option( - "-e", "--expire", metavar="CYCLE_POINT", - help="Cancel any broadcasts that target cycle " - "points earlier than, but not inclusive of, CYCLE_POINT.", - action="store", default=None, dest="expire") - - parser.add_option( - "-d", "--display", - help="Display active broadcasts.", - action="store_true", default=False, dest="show") - - parser.add_option( - "-k", "--display-task", metavar="TASKID", - help="Print active broadcasts for a given task " - "(" + TaskID.SYNTAX + ").", - action="store", default=None, dest="showtask") - - parser.add_option( - "-b", "--box", - help="Use unicode box characters with -d, -k.", - action="store_true", default=False, dest="unicode") - - parser.add_option( - "-r", "--raw", - help="With -d/--display or -k/--display-task, write out " - "the broadcast config structure in raw Python form.", - action="store_true", default=False, dest="raw") - - (options, args) = parser.parse_args() - suite = args[0] - - debug = False - if cylc.flags.debug: - debug = True - else: - try: - # from task execution environment - if os.environ['CYLC_DEBUG'] == 'True': - debug = True - except KeyError: - pass - - pclient = BroadcastClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - if options.show or options.showtask: - if options.showtask: - try: - name, point_string = TaskID.split(options.showtask) - except ValueError: - parser.error("TASKID must be " + TaskID.SYNTAX) - settings = pclient.broadcast('get', options.showtask) - padding = get_padding(settings) * ' ' - if options.raw: - print str(settings) - else: - print_tree(settings, padding, options.unicode) - sys.exit(0) - - if options.clear: - modified_settings, bad_options = pclient.broadcast( - 'clear', options.point_strings, options.namespaces) - if modified_settings: - print get_broadcast_change_report( - modified_settings, is_cancel=True) - sys.exit(get_broadcast_bad_options_report(bad_options)) - - if options.expire: - modified_settings, bad_options = pclient.broadcast( - 'expire', options.expire) - if modified_settings: - print get_broadcast_change_report( - modified_settings, is_cancel=True) - sys.exit(get_broadcast_bad_options_report(bad_options)) - - # implement namespace and cycle point defaults here - namespaces = options.namespaces - if not namespaces: - namespaces = ["root"] - point_strings = options.point_strings - if not point_strings: - point_strings = ["*"] - - if options.cancel: - settings = [] - for option_item in options.cancel: - if "=" in option_item: - raise ValueError( - "ERROR: --cancel=[SEC]ITEM does not take a value") - option_item = option_item.strip() - if option_item == "inherit": - raise ValueError( - "ERROR: Inheritance cannot be changed by broadcast") - setting = get_rdict(option_item) - upg({'runtime': {'__MANY__': setting}}, 'test') - validate(setting, SPEC['runtime']['__MANY__']) - settings.append(setting) - modified_settings, bad_options = pclient.broadcast( - 'clear', point_strings, namespaces, settings) - if modified_settings: - print get_broadcast_change_report( - modified_settings, is_cancel=True) - sys.exit(get_broadcast_bad_options_report(bad_options)) - - if options.set: - settings = [] - for option_item in options.set: - if "=" not in option_item: - raise ValueError( - "ERROR: --set=[SEC]ITEM=VALUE requires a value") - lhs, rhs = [s.strip() for s in option_item.split("=", 1)] - if lhs == "inherit": - raise ValueError( - "ERROR: Inheritance cannot be changed by broadcast") - setting = get_rdict(lhs, rhs) - upg({'runtime': {'__MANY__': setting}}, 'test') - validate(setting, SPEC['runtime']['__MANY__']) - settings.append(setting) - modified_settings, bad_options = pclient.broadcast( - 'put', point_strings, namespaces, settings) - print get_broadcast_change_report(modified_settings) - sys.exit(get_broadcast_bad_options_report(bad_options, is_set=True)) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-cat-log b/bin/cylc-cat-log deleted file mode 100755 index e9ee70f2dec..00000000000 --- a/bin/cylc-cat-log +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] cat-log|log [OPTIONS] ARGS - -Print the location or content of any suite or task log file, or a listing of a -task log directory on the suite or task host. By default the suite event log -or task job script is printed. For task logs you must use the same cycle -point format as the suite (list the log directory to see what it is).""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import os -from pipes import quote -import shlex -from subprocess import Popen, PIPE -import traceback - -from cylc.CylcOptionParsers import cop -from cylc.owner import is_remote_user -from cylc.rundb import CylcSuiteDAO -from cylc.suite_host import is_remote_host -from cylc.cfgspec.globalcfg import GLOBAL_CFG -from cylc.task_id import TaskID - - -NAME_DEFAULT = "log" -NAME_ERR = "err" -NAME_OUT = "out" -NAME_JOB_ACTIVITY_LOG = "job-activity.log" -NAME_JOB_EDIT_DIFF = "job-edit.diff" -NAME_JOB_STATUS = "job.status" -JOB_LOG_DEST_MAP = { - NAME_DEFAULT: "job", NAME_ERR: "job.err", NAME_OUT: "job.out"} -JOB_LOG_LOCAL_ALWAYS = (NAME_JOB_EDIT_DIFF, NAME_JOB_ACTIVITY_LOG) -LIST_MODE_LOCAL = "local" -LIST_MODE_REMOTE = "remote" - - -def get_option_parser(): - """Set up the CLI option parser.""" - parser = cop( - __doc__, argdoc=[("REG", "Suite name"), ("[TASK-ID]", """Task ID""")]) - - parser.add_option( - "-l", "--location", - help=("Print location of the log file, exit 0 if it exists," + - " exit 1 otherwise"), - action="store_true", default=False, dest="location_mode") - - parser.add_option( - "-o", "--stdout", - help="Suite log: out, task job log: job.out", - action="store_const", const=NAME_OUT, dest="filename") - - parser.add_option( - "-e", "--stderr", - help="Suite log: err, task job log: job.err", - action="store_const", const=NAME_ERR, dest="filename") - - parser.add_option( - "-r", "--rotation", - help="Suite logs log rotation number", metavar="INT", - action="store", dest="rotation_num") - - parser.add_option( - "-a", "--activity", - help="Task job log only: Short for --filename=job-activity.log", - action="store_const", const=NAME_JOB_ACTIVITY_LOG, dest="filename") - - parser.add_option( - "-d", "--diff", - help=("Task job log only: Short for --filename=job-edit.diff" + - " (file present after an edit-run)."), - action="store_const", const=NAME_JOB_EDIT_DIFF, dest="filename") - - parser.add_option( - "-u", "--status", - help="Task job log only: Short for --filename=job.status", - action="store_const", const=NAME_JOB_STATUS, dest="filename") - - parser.add_option( - "-f", "--filename", "-c", "--custom", - help="Name of log file (e.g. 'job.stats').", metavar="FILENAME", - action="store", dest="filename") - - parser.add_option( - "-s", "--submit-number", "-t", "--try-number", - help="Task job log only: submit number (default=NN).", metavar="INT", - action="store", dest="submit_num") - - parser.add_option( - "-x", "--list-local", - help="List a log directory on the suite host", - action="store_const", const=LIST_MODE_LOCAL, dest="list_mode") - - parser.add_option( - "-y", "--list-remote", - help="Task job log only: List log directory on the job host", - action="store_const", const=LIST_MODE_REMOTE, dest="list_mode") - - return parser - - -def get_suite_log_path(options, suite): - """Return file name of a suite log, given the options.""" - if options.list_mode: - basename = "." - else: - if options.filename: - basename = options.filename - else: - basename = NAME_DEFAULT - if options.rotation_num: - basename += "." + options.rotation_num - return os.path.normpath(os.path.join( - GLOBAL_CFG.get_derived_host_item(suite, "suite log directory"), - basename)) - - -def get_task_job_log_path( - options, suite, point, task, submit_num, user_at_host): - """Return file name of a task job log, given the options.""" - if user_at_host and "@" in user_at_host: - owner, host = user_at_host.split("@", 1) - elif user_at_host: - owner, host = (None, user_at_host) - else: - owner, host = (None, None) - if options.list_mode: - basename = "." - elif options.filename: - try: - basename = JOB_LOG_DEST_MAP[options.filename] - except KeyError: - basename = options.filename - else: - basename = JOB_LOG_DEST_MAP[NAME_DEFAULT] - if submit_num != "NN": - submit_num = "%02d" % submit_num - return os.path.normpath(os.path.join( - GLOBAL_CFG.get_derived_host_item( - suite, "suite job log directory", host, owner), - point, task, submit_num, basename)) - - -def get_task_job_attrs(options, suite, point, task, submit_num): - """Return (user@host, command0) of a task job log. - - user@host is set if task job is run remotely and for relevant log files. - command0 is set if task job is running on a batch system that requires a - special command to view stdout/stderr files. - - """ - if (options.filename in JOB_LOG_LOCAL_ALWAYS or - options.list_mode == LIST_MODE_LOCAL): - return (None, None) - suite_dao = CylcSuiteDAO( - os.path.join( - GLOBAL_CFG.get_derived_host_item(suite, "suite run directory"), - CylcSuiteDAO.DB_FILE_BASE_NAME), - is_public=True) - task_job_data = suite_dao.select_task_job(None, point, task, submit_num) - suite_dao.close() - if task_job_data is None: - return (None, None) - if "@" in task_job_data["user_at_host"]: - owner, host = str(task_job_data["user_at_host"]).split("@", 1) - else: - owner, host = (None, str(task_job_data["user_at_host"])) - user_at_host = None - if is_remote_host(host) or is_remote_user(owner): - if host and owner: - user_at_host = owner + "@" + host - elif host: - user_at_host = host - elif owner: - user_at_host = owner + "@localhost" - if (options.list_mode or - options.location_mode or - options.filename not in [ - NAME_ERR, NAME_OUT, - JOB_LOG_DEST_MAP[NAME_ERR], JOB_LOG_DEST_MAP[NAME_OUT]] or - not task_job_data["batch_sys_name"] or - not task_job_data["batch_sys_job_id"] or - not task_job_data["time_run"] or - task_job_data["time_run_exit"]): - return (user_at_host, None) - try: - if user_at_host and "@" in user_at_host: - owner, host = user_at_host.split("@", 1) - else: - owner, host = (None, user_at_host) - if options.filename in (NAME_OUT, JOB_LOG_DEST_MAP[NAME_OUT]): - key = "out viewer" - else: - key = "err viewer" - conf = GLOBAL_CFG.get_host_item("batch systems", host, owner) - command0_tmpl = conf[str(task_job_data["batch_sys_name"])][key] - except (KeyError, TypeError): - return (user_at_host, None) - else: - if command0_tmpl: - return (user_at_host, shlex.split(command0_tmpl % { - "job_id": str(task_job_data["batch_sys_job_id"])})) - else: - return (user_at_host, None) - - -def main(): - """Implement cylc cat-log CLI.""" - parser = get_option_parser() - options, args = parser.parse_args() - suite = args[0] - if options.filename and options.list_mode: - parser.error("Choose either test/print log file or list log directory") - elif len(args) > 1: - # Task job log - try: - task, point = TaskID.split(args[1]) - except ValueError: - parser.error("Illegal task ID: %s" % args[1]) - if options.submit_num in [None, "NN"]: - submit_num = "NN" - else: - try: - submit_num = int(options.submit_num) - except ValueError: - parser.error("Illegal submit number: %s" % options.submit_num) - user_at_host, command0 = get_task_job_attrs( - options, suite, point, task, submit_num) - filename = get_task_job_log_path( - options, args[0], point, task, submit_num, user_at_host) - else: - # Suite log - if options.submit_num or options.list_mode == LIST_MODE_REMOTE: - parser.error("Task log option(s) are not legal for suite logs.") - filename = get_suite_log_path(options, args[0]) - user_at_host, command0 = (None, None) - - # Construct the shell command - commands = [] - if options.location_mode: - if user_at_host is not None: - sys.stdout.write("%s:" % user_at_host) - sys.stdout.write("%s\n" % filename) - commands.append(["test", "-e", filename]) - elif options.list_mode: - commands.append(["ls", filename]) - elif command0 and user_at_host: - commands.append(command0 + ["||", "cat", filename]) - elif command0: - commands.append(command0) - commands.append(["cat", filename]) - else: - commands.append(["cat", filename]) - - # Deal with remote [user@]host - if user_at_host: - if "@" in user_at_host: - owner, host = user_at_host.split("@", 1) - else: - owner, host = (None, user_at_host) - ssh = str(GLOBAL_CFG.get_host_item( - "remote shell template", host, owner)).replace(" %s", "") - for i, command in enumerate(commands): - commands[i] = shlex.split(ssh) + ["-n", user_at_host] + command - - err = None - for command in commands: - stderr=PIPE - if options.debug: - sys.stderr.write(" ".join([quote(item) for item in command]) + "\n") - stderr=None - proc = Popen(command, stderr=stderr) - err = proc.communicate()[1] - ret_code = proc.wait() - if ret_code == 0: - break - if ret_code and err: - sys.stderr.write(err) - sys.exit(ret_code) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-cat-state b/bin/cylc-cat-state deleted file mode 100755 index 6dbe69b06ec..00000000000 --- a/bin/cylc-cat-state +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] cat-state [OPTIONS] REG - -Print the suite state dump file directly to stdout.""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.dump import dump_to_stdout, get_stop_state_summary -from cylc.suite_state_dumping import SuiteStateDumper - - -def main(): - parser = cop(__doc__) - - parser.add_option( - "-d", "--dump", - help="Use the same display format as the 'cylc dump' command.", - action="store_true", default=False, dest="dumpform") - - (options, args) = parser.parse_args() - suite = args[0] - - owner = options.db_owner - f = open(SuiteStateDumper(suite).file_name, 'rb') - lines = f.readlines() - f.close() - - lines = map(str.rstrip, lines) - - if not options.dumpform: - for line in lines: - print line - else: - [glbl, states, fam_states] = get_stop_state_summary( - suite, options.owner, options.host, lines) - dump_to_stdout(states) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-check-software b/bin/cylc-check-software deleted file mode 100755 index 5a550f2f136..00000000000 --- a/bin/cylc-check-software +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# Check for cylc software dependencies - -usage() { - cat <= 2.6 ... " -PVER=$( python -V 2>&1 | awk '{print $2}' ) -echo -n "found ${PVER} ... " -if python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then - echo "ok" -else - echo "ERROR: Python version too old" -fi - -# non-Python packages -echo "Checking for non-Python packages:" -echo -n " + Graphviz ... " -if ! which dot > /dev/null 2>&1; then - echo "NOT FOUND" -else - echo "ok" -fi -echo -n " + sqlite ... " -if ! which sqlite3 > /dev/null 2>&1; then - echo "NOT FOUND" -else - echo "ok" -fi - -# Python packages -# sqlite3 is part of the standard library since Python 2.5 -PKGS="Pyro-3:Pyro.core \ -Jinja2:jinja2 \ -pygraphviz:pygraphviz \ -pygtk:pygtk" - -echo "Checking for Python packages:" - -for ITEM in $PKGS; do - NAME=${ITEM%:*} - MODL=${ITEM#*:} - - echo -n " + $NAME ... " - if ! python -c "import $MODL" > /dev/null 2>&1; then - echo "NOT FOUND" - else - echo "ok" - fi -done diff --git a/bin/cylc-check-triggering b/bin/cylc-check-triggering deleted file mode 100755 index 3f1eaaa47d5..00000000000 --- a/bin/cylc-check-triggering +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [hook] check-triggering ARGS - -This is a cylc shutdown event handler that compares the newly generated -suite log with a previously generated reference log "reference.log" -stored in the suite definition directory. Currently it just compares -runtime triggering information, disregarding event order and timing, and -fails the suite if there is any difference. This should be sufficient to -verify correct scheduling of any suite that is not affected by different -run-to-run conditional triggering. - -1) run your suite with "cylc run --generate-reference-log" to generate -the reference log with resolved triggering information. Check manually -that the reference run was correct. -2) run reference tests with "cylc run --reference-test" - this -automatically sets the shutdown event handler along with a suite timeout -and "abort if shutdown handler fails", "abort on timeout", and "abort if -any task fails". - -Reference tests can use any run mode: - * simulation mode - tests that scheduling is equivalent to the reference - * dummy mode - also tests that task hosting, job submission, job script - evaluation, and cylc messaging are not broken. - * live mode - tests everything (but takes longer with real tasks!) - - If any task fails, or if cylc itself fails, or if triggering is not - equivalent to the reference run, the test will abort with non-zero exit - status - so reference tests can be used as automated tests to check - that changes to cylc have not broken your suites.""" - -import os -import sys - -from cylc.LogDiagnosis import LogAnalyser - -if len(sys.argv) == 2 and sys.argv[1] == '--help': - print __doc__ - sys.exit(0) - -print "\nThis is the cylc check-triggering shutdown event handler" - -event, suite = sys.argv[1], sys.argv[2] - -if event != 'shutdown': - raise SystemExit("ERROR: run this as a shutdown event handler") - -try: - log_dir = os.path.expandvars(os.environ['CYLC_SUITE_LOG_DIR']) - suite_dir = os.path.expandvars(os.environ['CYLC_SUITE_DEF_PATH']) -except KeyError, x: - raise SystemExit(x) - -new_log = os.path.join(log_dir, 'log') -ref_log = os.path.join(suite_dir, 'reference.log') - -try: - lanal = LogAnalyser(ref_log, new_log) - lanal.verify_triggering() -except Exception, x: - print >> sys.stderr, x - raise SystemExit("ERROR: Triggering check FAILED") -else: - print "Triggering check passed" diff --git a/bin/cylc-check-versions b/bin/cylc-check-versions deleted file mode 100755 index 977eb0f74e4..00000000000 --- a/bin/cylc-check-versions +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [discovery] check-versions [OPTIONS] ARGS - -Check the version of cylc invoked on each of SUITE's task host accounts when -CYLC_VERSION is set to *the version running this command line tool*. -Different versions are reported but are not considered an error unless the --e|--error option is specified, because different cylc versions from 6.0.0 -onward should at least be backward compatible. - -It is recommended that cylc versions be installed in parallel and access -configured via the cylc version wrapper as described in the cylc INSTALL -file and User Guide. This must be done on suite and task hosts. Users then get -the latest installed version by default, or (like tasks) a particular version -if $CYLC_VERSION is defined. - -User -v/--verbose to see the command invoked to determine the remote version -(all remote cylc command invocations will be of the same form, which may be -site dependent -- see cylc global config documentation.""" - -import sys -import subprocess - -import cylc.flags -from cylc.remote import remrun -from cylc.CylcOptionParsers import cop -from cylc.version import CYLC_VERSION -from cylc.config import SuiteConfig, SuiteConfigError -from cylc.host_select import get_task_host - - -def main(): - parser = cop(__doc__, prep=True, jset=True) - - parser.add_option( - "-e", "--error", help="Exit with error status " - "if " + CYLC_VERSION + " is not available on all remote accounts.", - action="store_true", default=False, dest="error") - - (options, args) = parser.parse_args(remove_opts=['--host', '--user']) - - # suite name or file path - suite, suiterc, junk = parser.get_suite() - - # extract task host accounts from the suite - config = SuiteConfig.get_inst( - suite, suiterc, - template_vars=options.templatevars, - template_vars_file=options.templatevars_file) - result = config.get_namespace_list('all tasks') - namespaces = result.keys() - accounts = set() - for name in namespaces: - host = get_task_host( - config.get_config(['runtime', name, 'remote', 'host'])) - owner = config.get_config(['runtime', name, 'remote', 'owner']) - accounts.add((owner, host)) - accounts = list(accounts) - - # Interrogate the each remote account with CYLC_VERSION set to our version. - # Post backward compatibility concerns to do this we can just run: - # cylc version --host=HOST --user=USER - # but this command only exists for version > 6.3.0. - # So for the moment generate an actual remote invocation command string for - # "cylc -v". - - # (save verbose flag as gets reset in remrun) - verbose = cylc.flags.verbose - - warn = {} - contacted = 0 - for user, host in accounts: - argv = ["cylc", "version"] - if user and host: - argv += ["--user=%s" % user, "--host=%s" % host] - user_at_host = "%s@%s" % (user, host) - elif user: - argv += ["--user=%s" % user] - user_at_host = "%s@localhost" % user - elif host: - argv += ["--host=%s" % host] - user_at_host = host - if verbose: - print "%s: %s" % (user_at_host, ' '.join(argv)) - p = subprocess.Popen( - argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - res = p.wait() - if res == 0: - if verbose: - print " %s" % out - contacted += 1 - out = out.strip() - if out != CYLC_VERSION: - warn[user_at_host] = out - else: - print >> sys.stderr, 'ERROR ' + user_at_host + ':' - print >> sys.stderr, err - - # report results - if not warn: - if contacted: - print "All", contacted, "accounts have cylc-" + CYLC_VERSION - else: - print "WARNING: failed to invoke cylc-%s on %d accounts:" % ( - CYLC_VERSION, len(warn.keys())) - m = max([len(ac) for ac in warn.keys()]) - for ac, warning in warn.items(): - print ' ', ac.ljust(m), warning - if options.error: - sys.exit(1) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-conditions b/bin/cylc-conditions deleted file mode 100755 index ba9d15dad23..00000000000 --- a/bin/cylc-conditions +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -usage() { - echo "Usage: cylc [license] warranty [--help]" - echo "Cylc is release under the GNU General Public License v3.0" - echo "This command prints the GPL v3.0 license in full." - echo "" - echo "Options:" - echo " --help Print this usage message." -} - -if [[ $# != 0 ]]; then - usage - if [[ $1 == "--help" ]]; then - exit 0 - else - echo "ERROR: illegal command line arguments" - exit 1 - fi -fi - -echo -echo "The GNU General Public License v3.0" -echo -cat $CYLC_DIR/COPYING -echo diff --git a/bin/cylc-copy b/bin/cylc-copy deleted file mode 100755 index cdd34b1e2c9..00000000000 --- a/bin/cylc-copy +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [db] copy|cp [OPTIONS] REG REG2 TOPDIR - -Copy suite or group REG to TOPDIR, and register the copy as REG2. - -Consider the following three suites: - -% cylc db print '^foo' # printed in flat form -foo.bag | "Test Suite Zero" | /home/bob/zero -foo.bar.qux | "Test Suite Two" | /home/bob/two -foo.bar.baz | "Test Suite One" | /home/bob/one - -% cylc db print -t '^foo' # printed in tree from -foo - |-bag "Test Suite Zero" | /home/bob/zero - `-bar - |-baz "Test Suite One" | /home/bob/one - `-qux "Test Suite Two" | /home/bob/two - -These suites are stored in a flat directory structure under /home/bob, -but they are organised in the suite database as a group 'foo' that -contains the suite 'foo.bag' and a group 'foo.bar', which in turn -contains the suites 'foo.bar.baz' and 'foo.bar.qux'. - -When you copy suites with this command, the target registration names -are determined by TARGET and the name structure underneath SOURCE, and -the suite definition directories are copied into a directory tree under -TOPDIR whose structure reflects the target registration names. If this -is not what you want, you can copy suite definition directories manually -and then register the copied directories manually with 'cylc register'. - -EXAMPLES (using the three suites above): - -% cylc db copy foo.bar.baz red /home/bob # suite to suite - Copying suite definition for red -% cylc db print "^red" - red | "Test Suite One" | /home/bob/red - -% cylc copy foo.bar.baz blue.green /home/bob # suite to group - Copying suite definition for blue.green -% cylc db pr "^blue" - blue.green | "Test Suite One" | /home/bob/blue/green - -% cylc copy foo.bar orange /home/bob # group to group - Copying suite definition for orange.qux - Copying suite definition for orange.baz -% cylc db pr "^orange" - orange.qux | "Test Suite Two" | /home/bob/orange/qux - orange.baz | "Test Suite One" | /home/bob/orange/baz""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import os -import re -import shutil - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.mkdir_p import mkdir_p -from cylc.registration import localdb, RegistrationError -from cylc.regpath import RegPath - - -def main(): - parser = cop(__doc__, - argdoc=[("REG", "Source suite name"), - ("REG2", "Target suite name"), - ("TOPDIR", "Top level target directory.")]) - - parser.add_option( - "--db-from", - help="Copy suites from another DB (defaults to --db).", - metavar='PATH', action="store", default=None, dest="dbfrom") - - (options, args) = parser.parse_args() - arg_from = args[0] - arg_to = args[1] - arg_dir = args[2] - - if options.dbfrom: - dbfrom = localdb(file=options.dbfrom) - else: - dbfrom = localdb(file=options.db) - db = localdb(file=options.db) - - flist = dbfrom.get_list('^' + arg_from + r'\b') - if len(flist) == 0: - sys.exit('ERROR, no suites matched: ' + arg_from) - - for item in flist: - freg, fdir, ftitle = item - treg = re.sub(r'\b' + arg_from + r'\b', arg_to, freg) - - tdir = RegPath(treg).get_fpath() - tdir = os.path.join(arg_dir, tdir) - - tdir = os.path.abspath(tdir) - mkdir_p(os.path.dirname(tdir)) - - print 'COPY', fdir, '\n TO', tdir - shutil.copytree(fdir, tdir) - db.register(treg, tdir) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-cycle-point b/bin/cylc-cycle-point deleted file mode 100755 index 59368961bc6..00000000000 --- a/bin/cylc-cycle-point +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [util] cycle-point [OPTIONS] [POINT] - -Cycle point date-time offset computation, and filename templating. - -Filename templating replaces elements of a template string with corresponding -elements of the current or given cycle point. - -The template string format changed at cylc-6, but is backward compatible. - -If a 10-digit pre cylc-6 cycle time is detected, the template string -elements are 'YYYY' (year), 'MM' (month), 'DD' (day), and 'HH' (hour): - % cylc cyclepoint 2010080800 --template foo-YYYY-MM-DD-HH.nc - foo-2010-08-08-00.nc - -Otherwise (cylc-6+) use ISO 8601 or posix date-time format elements: - % cylc cyclepoint 2010080T00 --template foo-CCYY-MM-DD-Thh.nc - foo-2010-08-08-T00.nc - % cylc cyclepoint 2010080T00 --template foo-%Y-%m-%d-Thh.nc - foo-2010-08-08-T00.nc - -Other examples: - -1) print offset from an explicit cycle point: - % cylc [util] cycle-point --offset-hours=6 20100823T1800Z - 20100824T0000Z - -2) print offset from $CYLC_TASK_CYCLE_POINT (as in suite tasks): - % export CYLC_TASK_CYCLE_POINT=20100823T1800Z - % cylc cycle-point --offset-hours=-6 - 20100823T1200Z - -3) cycle point filename templating, explicit template: - % export CYLC_TASK_CYCLE_POINT=2010-08 - % cylc cycle-point --offset-years=2 --template=foo-CCYY-MM.nc - foo-2012-08.nc - -4) cycle point filename templating, template in a variable: - % export CYLC_TASK_CYCLE_POINT=2010-08 - % export MYTEMPLATE=foo-CCYY-MM.nc - % cylc cycle-point --offset-years=2 --template=MYTEMPLATE - foo-2012-08.nc - -Arguments: - [POINT] ISO 8601 date-time, e.g. 20140201T0000Z, default - $CYLC_TASK_CYCLE_POINT""" - -import os -import sys -import re -from optparse import OptionParser - -import cylc.flags -import cylc.cycling.iso8601 -import isodatetime.data -import isodatetime.dumpers -import isodatetime.parsers - - -def main(): - parser = OptionParser(__doc__) - - parser.add_option( - "--offset-hours", metavar="HOURS", - help="Add N hours to CYCLE (may be negative)", - action="store", dest="offsethours") - - parser.add_option( - "--offset-days", metavar="DAYS", - help="Add N days to CYCLE (N may be negative)", - action="store", dest="offsetdays") - - parser.add_option( - "--offset-months", metavar="MONTHS", - help="Add N months to CYCLE (N may be negative)", - action="store", dest="offsetmonths") - - parser.add_option( - "--offset-years", metavar="YEARS", - help="Add N years to CYCLE (N may be negative)", - action="store", dest="offsetyears") - - parser.add_option( - "--offset", metavar="ISO_OFFSET", - help="Add an ISO 8601-based interval representation to CYCLE", - action="store", dest="offset") - - parser.add_option( - "--template", metavar="TEMPLATE", - help="Filename template string or variable", - action="store", dest="template") - - parser.add_option( - "--time-zone", metavar="TEMPLATE", - help="Control the formatting of the result's timezone e.g. " - "(Z, +13:00, -hh", - action="store", default=None, dest="time_zone") - - parser.add_option( - "--num-expanded-year-digits", metavar="NUMBER", - help="Specify a number of expanded year digits to print in the result", - action="store", default=0, dest="num_expanded_year_digits") - - parser.add_option( - "--print-year", help="Print only CCYY of result", - action="store_true", default=False, dest="print_year") - - parser.add_option( - "--print-month", help="Print only MM of result", - action="store_true", default=False, dest="print_month") - - parser.add_option( - "--print-day", help="Print only DD of result", - action="store_true", default=False, dest="print_day") - - parser.add_option( - "--print-hour", help="Print only HH of result", - action="store_true", default=False, dest="print_hour") - - (options, args) = parser.parse_args() - - if len(args) == 0: - # input cycle point must be defined in the environment. - if 'CYLC_TASK_CYCLE_POINT' not in os.environ: - parser.error("Provide CYCLE arg, or define $CYLC_TASK_CYCLE_POINT") - cycle_point_string = os.environ['CYLC_TASK_CYCLE_POINT'] - - elif len(args) == 1: - # must be cycle point - cycle_point_string = args[0] - - else: - parser.error("Wrong number of arguments!") - - # template string - template = None - if options.template: - if (options.print_month or options.print_year or options.print_day or - options.print_hour): - parser.error( - '"print only" options are incompatible with templating') - tmp = options.template - if tmp in os.environ: - # name of a variable that contains a template - template = os.environ[tmp] - else: - # or a raw template string - template = tmp - else: - n_chosen = 0 - - if options.print_year: - n_chosen += 1 - if options.num_expanded_year_digits: - template = u"±XCCYY" - else: - template = "CCYY" - - if options.print_month: - n_chosen += 1 - template = "MM" - - if options.print_day: - n_chosen += 1 - template = "DD" - - if options.print_hour: - n_chosen += 1 - template = "%H" - - if n_chosen != 0 and n_chosen != 1: - parser.error("Choose NONE or ONE of print_(year|month|day|hour)") - - if re.match("\d{10}$", cycle_point_string): - # Auto-detect prev Cylc format. - if template is None: - template = "%Y%m%d%H" - else: - template = template.replace("YYYY", "%Y") - template = template.replace("MM", "%m") - template = template.replace("DD", "%d") - template = template.replace("HH", "%H") - - cylc.cycling.iso8601.init( - num_expanded_year_digits=options.num_expanded_year_digits, - time_zone=options.time_zone - ) - iso_point_parser = isodatetime.parsers.TimePointParser( - num_expanded_year_digits=options.num_expanded_year_digits - ) - iso_point_dumper = isodatetime.dumpers.TimePointDumper( - num_expanded_year_digits=options.num_expanded_year_digits - ) - try: - cycle_point = iso_point_parser.parse( - cycle_point_string, dump_as_parsed=(template is None)) - except ValueError: - # May be in prev Cylc format. - try: - cycle_point = cylc.cycling.iso8601.point_parse(cycle_point_string) - except ValueError as exc: - parser.error('ERROR: invalid cycle: %s' % exc) - - offset_props = {} - - if options.offsethours: - try: - offset_props["hours"] = int(options.offsethours) - except ValueError: - parser.error('ERROR: offset must be integer') - - if options.offsetdays: - try: - offset_props["days"] = int(options.offsetdays) - except ValueError: - parser.error('ERROR: offset must be integer') - - if options.offsetmonths: - try: - offset_props["months"] = int(options.offsetmonths) - except ValueError: - parser.error('ERROR: offset must be integer') - - if options.offsetyears: - try: - offset_props["years"] = int(options.offsetyears) - except ValueError: - parser.error('ERROR: offset must be integer') - - offset = isodatetime.data.Duration(**offset_props) - - if options.offset: - opt_offset = options.offset - sign_factor = 1 - if options.offset.startswith("-"): - opt_offset = options.offset[1:] - sign_factor = -1 - try: - offset += isodatetime.parsers.DurationParser().parse( - opt_offset) * sign_factor - except ValueError as exc: - parser.error('ERROR: offset not valid: %s' % exc) - cycle_point += offset - if template is None: - print cycle_point - else: - print iso_point_dumper.dump(cycle_point, template) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-diff b/bin/cylc-diff deleted file mode 100755 index bc36e23dc7d..00000000000 --- a/bin/cylc-diff +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [prep] diff|compare [OPTIONS] SUITE1 SUITE2 - -Compare two suite definitions and display any differences. - -Differencing is done after parsing the suite.rc files so it takes -account of default values that are not explicitly defined, it disregards -the order of configuration items, and it sees any include-file content -after inlining has occurred. - -Note that seemingly identical suites normally differ due to inherited -default configuration values (e.g. the default job submission log -directory. - -Files in the suite bin directory and other sub-directories of the -suite definition directory are not currently differenced.""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.config import SuiteConfig - -n_oone = 0 -n_otwo = 0 -n_diff = 0 - - -def diffdict(one, two, oone, otwo, diff): - global n_oone, n_otwo, n_diff - # Recursively difference two dictionaries in which any element - # may be another dictionary, keeping items that appear only - # in one or the other, and items that appear in both but differ. - for key in one: - if key not in two: - oone[key] = one[key] - n_oone += 1 - elif one[key] != two[key]: - if isinstance(one[key], dict): - for item in oone, otwo, diff: - if key not in item: - item[key] = {} - diffdict(one[key], two[key], oone[key], otwo[key], diff[key]) - else: - diff[key] = (one[key], two[key]) - n_diff += 1 - - for key in two: - if key not in one: - otwo[key] = two[key] - n_otwo += 1 - - -def prdict(dct, arrow='<', section='', level=0, diff=False, nested=False): - # Recursively print, in pseudo 'diff' format, the contents of - # one of the three dictionaries populated by the diffdict() function - # above (any element may itself be a dictionary). - - count = 0 - - if section != '': - prfx = section + ' ' - else: - prfx = '' - - if section == '': - sctn = '(top)' - else: - sctn = section - - foo = False - - for key in dct: - if isinstance(dct[key], dict): - lvl = level + 1 - if nested: - pre = prfx + '\n' + ' '*lvl - else: - pre = prfx - prdict(dct[key], arrow, - pre + '['*lvl + str(key) + ']'*lvl, lvl, - diff, nested) - else: - if not foo: - if nested: - print ' ', sctn - else: - print '\n ', sctn - foo = True - - if diff: - print ' < ', key, '=', dct[key][0] - print ' > ', key, '=', dct[key][1] - else: - print ' ' + arrow + ' ', key, '=', dct[key] - - -def main(): - parser = cop( - __doc__, jset=True, prep=True, twosuites=True, - argdoc=[('SUITE1', 'Suite name or path'), - ('SUITE2', 'Suite name or path')]) - - parser.add_option( - "-n", "--nested", - help="print suite.rc section headings in nested form.", - action="store_true", default=False, dest="nested") - - (options, args) = parser.parse_args() - - suite1, suite1rc, junk = parser.get_suite() - suite2, suite2rc, junk = parser.get_suite(index=1) - if suite1 == suite2: - parser.error("You can't diff a single suite.") - print "Parsing %s (%s)" % (suite1, suite1rc) - config1 = SuiteConfig.get_inst( - suite1, suite1rc, - template_vars=options.templatevars, - template_vars_file=options.templatevars_file).cfg - SuiteConfig._FORCE = True - print "Parsing %s (%s)" % (suite2, suite2rc) - config2 = SuiteConfig.get_inst( - suite2, suite2rc, - template_vars=options.templatevars, - template_vars_file=options.templatevars_file).cfg - - if config1 == config2: - print "Suite definitions %s and %s are identical" % (suite1, suite2) - sys.exit(0) - - print "Suite definitions %s and %s differ" % (suite1, suite2) - - suite1_only = {} - suite2_only = {} - diff_1_2 = {} - - diffdict(config1, config2, suite1_only, suite2_only, diff_1_2) - - if n_oone > 0: - print - msg = str(n_oone) + ' items only in ' + suite1 + ' (<)' - print msg - prdict(suite1_only, '<', nested=options.nested) - - if n_otwo > 0: - print - msg = str(n_otwo) + ' items only in ' + suite2 + ' (>)' - print msg - prdict(suite2_only, '>', nested=options.nested) - - if n_diff > 0: - print - msg = (str(n_diff) + ' common items differ ' + - suite1 + '(<) ' + suite2 + '(>)') - print msg - prdict(diff_1_2, '', diff=True, nested=options.nested) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-documentation b/bin/cylc-documentation deleted file mode 100755 index cb69c353128..00000000000 --- a/bin/cylc-documentation +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] documentation|browse [OPTIONS] [SUITE] - -Open cylc or suite documentation in your browser or PDF viewer (as defined -in cylc global config files). - -% cylc doc [OPTIONS] - Open local or internet [--www] cylc documentation (locations must be -specified in cylc global config files). - -% cylc doc -u [-t TASK] SUITE - Open suite or task documentation if corresponding URL items are specified -in the suite definition. - -Arguments: - [TARGET] File, URL, or suite name""" - -import sys -for arg in sys.argv[1:]: - if arg.startswith('--host=') or arg.startswith('--user='): - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import os -import re -import subprocess -from optparse import OptionParser - -import cylc.flags -from cylc.cfgspec.globalcfg import GLOBAL_CFG -from cylc.run_get_stdout import run_get_stdout -from cylc.suite_host import get_hostname -from cylc.owner import user - - -def main(): - parser = OptionParser(__doc__) - - parser.add_option( - "-p", "--pdf", help="Open the PDF User Guide directly.", - action="store_true", default=False, dest="pdf") - - parser.add_option( - "-w", "--www", help="Open the cylc internet homepage", - action="store_true", default=False, dest="www") - - parser.add_option( - "-t", "--task", help="Browse task documentation URLs.", - metavar="TASK_NAME", action="store", default=None, dest="task_name") - - parser.add_option( - "-s", "--stdout", help="Just print the URL to stdout.", - action="store_true", default=False, dest="stdout") - - parser.add_option( - "--user", - help="Other user account name. This results in " - "command reinvocation on the remote account.", - metavar="USER", default=user, action="store", dest="owner") - - parser.add_option( - "--host", - help="Other host name. This results in " - "command reinvocation on the remote account.", - metavar="HOST", action="store", default=get_hostname(), dest="host") - - (options, args) = parser.parse_args() - - intranet_url = GLOBAL_CFG.get(['documentation', 'urls', 'local index']) - internet_url = GLOBAL_CFG.get(['documentation', 'urls', - 'internet homepage']) - html_file = GLOBAL_CFG.get(['documentation', 'files', 'html index']) - html_viewer = GLOBAL_CFG.get(['document viewers', 'html']) - pdf_file = GLOBAL_CFG.get(['documentation', 'files', 'pdf user guide']) - pdf_viewer = GLOBAL_CFG.get(['document viewers', 'pdf']) - if len(args) == 0: - # Cylc documentation. - if options.pdf: - # Force PDF. - viewer = pdf_viewer - target = pdf_file - else: - # HTML documentation index. - viewer = html_viewer - if options.www: - # Force internet. - if internet_url is not None: - target = internet_url - else: - sys.exit("ERROR: cylc internet URL not defined.") - elif intranet_url is not None: - # Intranet. - target = intranet_url - else: - # Open in file:// mode as a last resort. - print >> sys.stderr, ("WARNING: cylc intranet URL not " - "defined, trying file mode.") - target = html_file - - elif len(args) == 1: - # Suite or task documentation. - if options.pdf or options.www: - print >> sys.stderr, ( - "(Note: --pdf and --www are ignored for suite documentation).") - suite = args[0] - if options.task_name: - # Task documentation. - res, stdout = run_get_stdout( - "cylc get-suite-config -i [runtime][%s]URL %s" % ( - options.task_name, suite)) - else: - # Suite documentation. - res, stdout = run_get_stdout( - "cylc get-suite-config -i URL %s" % suite) - if not res: - # (Illegal config item) - sys.exit(stdout) - elif len(stdout) == 0: - if options.task_name is not None: - sys.exit("ERROR: No URL defined for %s in %s." % ( - options.task_name, suite)) - else: - sys.exit("ERROR: No URL defined for %s." % suite) - target = stdout[0] - viewer = html_viewer - else: - parser.error("Too many arguments.") - - if target in [pdf_file, html_file] and not os.path.isfile(target): - sys.exit("ERROR, file not found: %s (see your cylc admin)" % target) - - # viewer may have spaces (e.g. 'firefox --no-remote'): - command = '%s %s' % (viewer, target) - command_list = re.split(' ', command) - - if options.stdout: - print target - sys.exit(0) - - retcode = subprocess.call(command_list) - if retcode != 0: - print >> sys.stderr, 'ERROR, command failed: %s' % command - sys.exit(retcode) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-dump b/bin/cylc-dump deleted file mode 100755 index 797a928721f..00000000000 --- a/bin/cylc-dump +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] dump [OPTIONS] ARGS - -Print state information (e.g. the state of each task) from a running -suite. For small suites 'watch cylc [info] dump SUITE' is an effective -non-GUI real time monitor (but see also 'cylc monitor'). - -For more information about a specific task, such as the current state of -its prerequisites and outputs, see 'cylc [info] show'. - -Examples: - Display the state of all running tasks, sorted by cycle point: - % cylc [info] dump --tasks --sort SUITE | grep running - - Display the state of all tasks in a particular cycle point: - % cylc [info] dump -t SUITE | grep 2010082406""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.network.suite_state import StateSummaryClient -from cylc.CylcOptionParsers import cop -from cylc.dump import dump_to_stdout - - -def main(): - parser = cop(__doc__, pyro=True, noforce=True) - - parser.add_option( - "-g", "--global", help="Global information only.", - action="store_true", default=False, dest="global_only") - - parser.add_option( - "-t", "--tasks", help="Task states only.", - action="store_true", default=False, dest="tasks_only") - - parser.add_option( - "-s", "--sort", - help="Task states only; sort by cycle point instead of name.", - action="store_true", default=False, dest="sort_by_cycle") - - (options, args) = parser.parse_args() - suite = args[0] - - # default: display all information - display_tasks = True - display_global = True - # check for restricted output - if options.global_only and options.tasks_only: - parser.error('--tasks and --global are incompatible') - - if options.global_only: - display_tasks = False - if options.tasks_only: - display_global = False - - try: - pclient = StateSummaryClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - # get state summary, task names, cycle points - glbl, states, fam_states = pclient.get_suite_state_summary() - except Exception, x: - if cylc.flags.debug: - raise - sys.exit(x) - - if display_global: - for item in glbl: - print item, '=', glbl[item] - - if display_tasks: - dump_to_stdout(states, options.sort_by_cycle) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-edit b/bin/cylc-edit deleted file mode 100755 index 3789eb5b59e..00000000000 --- a/bin/cylc-edit +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [prep] edit [OPTIONS] ARGS - -Edit suite definitions without having to move to their directory -locations, and with optional reversible inlining of include-files. Note -that Jinja2 suites can only be edited in raw form but the processed -version can be viewed with 'cylc [prep] view -p'. - -1/cylc [prep] edit SUITE -Change to the suite definition directory and edit the suite.rc file. - -2/ cylc [prep] edit -i,--inline SUITE -Edit the suite with include-files inlined between special markers. The -original suite.rc file is temporarily replaced so that the inlined -version is "live" during editing (i.e. you can run suites during -editing and cylc will pick up changes to the suite definition). The -inlined file is then split into its constituent include-files -again when you exit the editor. Include-files can be nested or -multiply-included; in the latter case only the first inclusion is -inlined (this prevents conflicting changes made to the same file). - -3/ cylc [prep] edit --cleanup SUITE -Remove backup files left by previous INLINED edit sessions. - -INLINED EDITING SAFETY: The suite.rc file and its include-files are -automatically backed up prior to an inlined editing session. If the -editor dies mid-session just invoke 'cylc edit -i' again to recover from -the last saved inlined file. On exiting the editor, if any of the -original include-files are found to have changed due to external -intervention during editing you will be warned and the affected files -will be written to new backups instead of overwriting the originals. -Finally, the inlined suite.rc file is also backed up on exiting -the editor, to allow recovery in case of accidental corruption of the -include-file boundary markers in the inlined file. - -The edit process is spawned in the foreground as follows: - % suite.rc -Where is defined in the cylc site/user config files. - -See also 'cylc [prep] view'.""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import os -import re -import datetime -import subprocess -from shutil import copy - -import cylc.flags -from cylc.cfgspec.globalcfg import GLOBAL_CFG -from cylc.CylcOptionParsers import cop -from parsec.include import inline, \ - split_file, backup, backups, newfiles, cleanup, modtimes -from cylc.wallclock import get_current_time_string - - -def main(): - parser = cop(__doc__, prep=True) - - parser.add_option( - "--inline", "-i", - help="Edit with include-files inlined as described above.", - action="store_true", default=False, dest="inline") - - parser.add_option( - "--cleanup", - help="Remove backup files left by previous inlined edit sessions.", - action="store_true", default=False, dest="cleanup") - - parser.add_option( - "--gui", "-g", help="Force use of the configured GUI editor.", - action="store_true", default=False, dest="geditor") - - (options, args) = parser.parse_args() - - suite, suiterc, junk = parser.get_suite() - - if options.geditor: - editor = GLOBAL_CFG.get(['editors', 'gui']) - else: - editor = GLOBAL_CFG.get(['editors', 'terminal']) - - suitedir = os.path.dirname(suiterc) - - if options.cleanup: - # remove backup files left by inlined editing sessions - cleanup(suitedir) - sys.exit(0) - - if not options.inline: - # plain old editing. - # move to suite def dir - os.chdir(suitedir) - - # edit the suite.rc file - if not os.path.isfile(suiterc): - print >> sys.stderr, 'ERROR, file not found: ', suiterc - sys.exit(1) - - # in case editor has options, e.g. 'emacs -nw': - command_list = re.split(' ', editor) - command_list.append(suiterc) - command = ' '.join(command_list) - # THIS BLOCKS UNTIL THE COMMAND COMPLETES - retcode = subprocess.call(command_list) - if retcode != 0: - # the command returned non-zero exist status - print >> sys.stderr, command, 'failed:', retcode - sys.exit(1) - - # !!!EDITING FINISHED!!! - sys.exit(0) - - # read the suite.rc file - if os.path.isfile(suiterc): - # back up the original - backup(suiterc) - # record original modtime - modtimes[suiterc] = os.stat(suiterc).st_mtime - # read the file - h = open(suiterc, 'rb') - lines0 = h.readlines() - h.close() - if lines0[0].startswith('# !WARNING! CYLC EDIT INLINED'): - print 'WARNING: RECOVERING A PREVIOUSLY INLINED FILE' - recovery = True - lines = lines0 - else: - recovery = False - lines = inline(lines0, suitedir, suiterc, for_edit=True) - else: - parser.error("File not found: " + suiterc) - - lines = [i.rstrip() for i in lines] - - # overwrite the (now backed up) original with the inlined file: - h = open(suiterc, 'wb') - for line in lines: - h.write(line + '\n') - h.close() - - print 'PRE-EDIT BACKUPS:' - for file in backups: - src = re.sub(suitedir + '/', '', file) - dst = re.sub(suitedir + '/', '', backups[file]) - print ' + ' + src + ' ---> ' + dst - - # in case editor has options, e.g. 'emacs -nw': - command_list = re.split(' ', editor) - command_list.append(suiterc) - command = ' '.join(command_list) - # THIS BLOCKS UNTIL THE COMMAND COMPLETES - retcode = subprocess.call(command_list) - if retcode != 0: - # the command returned non-zero exist status - print >> sys.stderr, command, 'failed:', retcode - sys.exit(1) - print 'EDITING DONE' - - # Now back up the inlined file in case of absolute disaster, so as the - # user or his editor corrupting the inlined-include-file marker lines. - inlined_suiterc_backup = ( - suitedir + '/suite.rc.INLINED.EDIT.' + - get_current_time_string(override_use_utc=True, use_basic_format=True) - ) - copy(suiterc, inlined_suiterc_backup) - - # read in the edited inlined file - h = open(suiterc, 'rb') - lines = h.readlines() - h.close() - - # split it back into separate files - split_file(suitedir, lines, suiterc, recovery) - - print ' + edited:', suiterc - print ' + backup:', inlined_suiterc_backup - print 'INCLUDE-FILES WRITTEN:' - for file in newfiles: - f = re.sub(suitedir + '/', '', file) - if re.search('\.EDIT\.NEW\.', f): - print ' + ' + f + ' (!!! WARNING: original changed on disk !!!)' - else: - print ' + ' + f - # DONE - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-email-suite b/bin/cylc-email-suite deleted file mode 100755 index 45cf056a6d1..00000000000 --- a/bin/cylc-email-suite +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -usage() { - echo "Usage: cylc [hook] email-suite EVENT SUITE MESSAGE" - echo "" - echo "This is a simple suite event hook script that sends an email." - echo "The command line arguments are supplied automatically by cylc." - echo "" - echo "For example, to get an email alert when a suite shuts down:" - echo "" - echo "# SUITE.RC" - echo "[cylc]" - echo " [[environment]]" - echo " MAIL_ADDRESS = foo@bar.baz.waz" - echo " [[event hooks]]" - echo " shutdown handler = cylc email-suite" - echo "" - echo "See the Suite.rc Reference (Cylc User Guide) for more information" - echo "on suite and task event hooks and event handler scripts." -} - -if [[ $# = 1 ]]; then - if [[ $1 = '--help' ]]; then - usage - exit 0 - fi -fi - -if [[ $# < 3 ]]; then - usage - exit 1 -fi - -EVENT=$1 # e.g. "shutdown" -SUITE=$2 # suite name -MESSAGE=$3 - -MAIL_SUBJECT="!!cylc alert!! suite $SUITE $EVENT" -MAIL_ADDRESS=${MAIL_ADDRESS:-$USER@$HOSTNAME} -MAIL_BODY="SUITE: $SUITE -MESSAGE: $MESSAGE" - -echo "cylc hook email-suite: Sending email to $MAIL_ADDRESS" -echo "$MAIL_BODY" | mail -s "$MAIL_SUBJECT" $MAIL_ADDRESS diff --git a/bin/cylc-email-task b/bin/cylc-email-task deleted file mode 100755 index 297186e2ee8..00000000000 --- a/bin/cylc-email-task +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -usage() { - echo "Usage: cylc [hook] email-task EVENT SUITE TASKID MESSAGE" - echo "" - echo "This is a simple task event hook handler script that sends an email." - echo "The command line arguments are supplied automatically by cylc." - echo "" - echo "For example, to get an email alert whenever any task fails:" - echo "" - echo "# SUITE.RC" - echo "[cylc]" - echo " [[environment]]" - echo " MAIL_ADDRESS = foo@bar.baz.waz" - echo "[runtime]" - echo " [[root]]" - echo " [[[event hooks]]]" - echo " failed handler = cylc email-task" - echo "" - echo "See the Suite.rc Reference (Cylc User Guide) for more information" - echo "on suite and task event hooks and event handler scripts." -} - -if [[ $# = 1 ]]; then - if [[ $1 = '--help' ]]; then - usage - exit 0 - fi -fi - -if [[ $# < 4 ]]; then - usage - exit 1 -fi - -EVENT=$1 # e.g. "failed" -SUITE=$2 # suite name -TASKID=$3 # task ID -MESSAGE=$4 - -MAIL_SUBJECT="!!cylc alert!! suite $SUITE task $TASKID $EVENT" -MAIL_ADDRESS=${MAIL_ADDRESS:-$USER@$HOSTNAME} -MAIL_BODY="SUITE: $SUITE -TASK: $TASKID -MESSAGE: $MESSAGE" - -echo "cylc hook email-task: Sending email to $MAIL_ADDRESS" -echo "$MAIL_BODY" | mail -s "$MAIL_SUBJECT" $MAIL_ADDRESS diff --git a/bin/cylc-ext-trigger b/bin/cylc-ext-trigger deleted file mode 100755 index 975e6a73f20..00000000000 --- a/bin/cylc-ext-trigger +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] ext-trigger [OPTIONS] ARGS - -Report an external event message to a suite daemon. It is expected that a -task in the suite has registered the same message as an external trigger - a -special prerequisite to be satisifed by an external system, via this command, -rather than by triggering off other tasks. - -The ID argument should uniquely distinguish one external trigger event from the -next. When a task's external trigger is satisfied by an incoming message, the -message ID is broadcast to all downstream tasks in the cycle point as -$CYLC_EXT_TRIGGER_ID so that they can use it - e.g. to identify a new data file -that the external triggering system is responding to. - -Use the retry options in case the target suite is down or out of contact. - -The suite passphrase must be installed in $HOME/.cylc//. - -Note: to manually trigger a task use 'cylc trigger', not this command.""" - -import os -import sys - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.network.ext_trigger import ExtTriggerClient - - -def main(): - parser = cop( - __doc__, pyro=True, - argdoc=[("REG", "Suite name"), - ("MSG", "External trigger message"), - ("ID", "Unique trigger ID")]) - - parser.add_option( - "--max-tries", help="Maximum number of send attempts " - "(default %s)." % ExtTriggerClient.MAX_N_TRIES, metavar="INT", - action="store", default=None, dest="max_n_tries") - - parser.add_option( - "--retry-interval", help="Delay in seconds before retrying " - "(default %s)." % ExtTriggerClient.RETRY_INTVL_SECS, metavar="SEC", - action="store", default=None, dest="retry_intvl_secs") - - (options, args) = parser.parse_args() - suite = args[0] - - cylc.flags.verbose = options.verbose - event_msg = args[1] - event_id = args[2] - - print 'Send to suite %s: "%s" (%s)' % (suite, event_msg, event_id) - - pclient = ExtTriggerClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - pclient.send_retry( - event_msg, event_id, options.max_n_tries, options.retry_intvl_secs) - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-get-directory b/bin/cylc-get-directory deleted file mode 100755 index 2a3988ae4b9..00000000000 --- a/bin/cylc-get-directory +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [db] get-directory REG - -Retrieve and print the directory location of suite REG. -Here's an easy way to move to a suite directory: - $ cd $(cylc get-dir REG).""" - -import sys -from cylc.CylcOptionParsers import cop -from cylc.registration import localdb -import cylc.flags - -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - - -def main(): - parser = cop(__doc__) - - (options, args) = parser.parse_args() - - reg = args[0] - db = localdb(file=options.db) - print db.get_suitedir(reg) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-get-gui-config b/bin/cylc-get-gui-config deleted file mode 100755 index 9a17da3ac9c..00000000000 --- a/bin/cylc-get-gui-config +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [admin] get-gui-config [OPTIONS] - -Print gcylc configuration settings. - -By default all settings are printed. For specific sections or items -use -i/--item and wrap parent sections in square brackets: - cylc get-gui-config --item '[themes][default]succeeded' -Multiple items can be specified at once.""" - -import sys -from optparse import OptionParser -import cylc.flags -from parsec.util import itemstr - - -def main(): - parser = OptionParser(__doc__) - - parser.add_option( - "-v", "--verbose", help="Print extra information.", - action="store_true", default=False, dest="verbose") - - parser.add_option( - "--debug", help="Show exception tracebacks.", - action="store_true", default=False, dest="debug") - - parser.add_option( - "-i", "--item", metavar="[SEC...]ITEM", - help="Item or section to print (multiple use allowed).", - action="append", dest="item", default=[]) - - parser.add_option( - "--sparse", - help="Only print items explicitly set in the config files.", - action="store_true", default=False, dest="sparse") - - parser.add_option( - "-p", "--python", help="Print native Python format.", - action="store_true", default=False, dest="pnative") - - (options, args) = parser.parse_args() - cylc.flags.verbose = options.verbose - cylc.flags.debug = options.debug - - if len(args) != 0: - parser.error("ERROR: wrong number of arguments") - - # Import gcfg here to avoid aborting before command help is printed. - from cylc.cfgspec.gcylc import gcfg - gcfg.idump(options.item, sparse=options.sparse, pnative=options.pnative) - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-get-site-config b/bin/cylc-get-site-config deleted file mode 100755 index d1dbc7b5e38..00000000000 --- a/bin/cylc-get-site-config +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [admin] get-site-config [OPTIONS] - -Print cylc site/user configuration settings. - -By default all settings are printed. For specific sections or items -use -i/--item and wrap parent sections in square brackets: - cylc get-site-config --item '[editors]terminal' -Multiple items can be specified at once.""" - -import sys -from optparse import OptionParser -import cylc.flags -from parsec.util import itemstr - - -def main(): - parser = OptionParser(__doc__) - - parser.add_option( - "-i", "--item", metavar="[SEC...]ITEM", - help="Item or section to print (multiple use allowed).", - action="append", dest="item", default=[]) - - parser.add_option( - "--sparse", - help="Only print items explicitly set in the config files.", - action="store_true", default=False, dest="sparse") - - parser.add_option( - "-p", "--python", - help="Print native Python format.", - action="store_true", default=False, dest="pnative") - - parser.add_option( - "--print-run-dir", - help="Print the configured cylc run directory.", - action="store_true", default=False, dest="run_dir") - - parser.add_option( - "-v", "--verbose", help="Print extra information.", - action="store_true", default=False, dest="verbose") - - parser.add_option( - "--debug", - help="Show exception tracebacks.", - action="store_true", default=False, dest="debug") - - (options, args) = parser.parse_args() - cylc.flags.verbose = options.verbose - cylc.flags.debug = options.debug - - if len(args) != 0: - parser.error("ERROR: wrong number of arguments") - - # import GLOBAL_CFG here to avoid aborting before command help is printed - from cylc.cfgspec.globalcfg import GLOBAL_CFG - if options.run_dir: - print GLOBAL_CFG.get_host_item('run directory') - else: - GLOBAL_CFG.idump( - options.item, sparse=options.sparse, pnative=options.pnative) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-get-suite-config b/bin/cylc-get-suite-config deleted file mode 100755 index 64de958025b..00000000000 --- a/bin/cylc-get-suite-config +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] get-suite-config [OPTIONS] ARGS - -Print parsed suite configuration items, after runtime inheritance. - -By default all settings are printed. For specific sections or items -use -i/--item and wrap sections in square brackets, e.g.: - cylc get-suite-config --item '[scheduling]initial cycle point' -Multiple items can be retrieved at once. - -By default, unset values are printed as an empty string, or (for -historical reasons) as "None" with -o/--one-line. These defaults -can be changed with the -n/--null-value option. - -Example: - |# SUITE.RC - |[runtime] - | [[modelX]] - | [[[environment]]] - | FOO = foo - | BAR = bar - -$ cylc get-suite-config --item=[runtime][modelX][environment]FOO SUITE -foo - -$ cylc get-suite-config --item=[runtime][modelX][environment] SUITE -FOO = foo -BAR = bar - -$ cylc get-suite-config --item=[runtime][modelX] SUITE -... -[[[environment]]] - FOO = foo - BAR = bar -...""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import re -from cylc.config import SuiteConfig, SuiteConfigError -from cylc.CylcOptionParsers import cop -import cylc.flags - - -def main(): - parser = cop(__doc__, jset=True, prep=True) - - parser.add_option( - "-i", "--item", metavar="[SEC...]ITEM", - help="Item or section to print (multiple use allowed).", - action="append", dest="item", default=[]) - - parser.add_option( - "-r", "--sparse", - help="Only print items explicitly set in the config files.", - action="store_true", default=False, dest="sparse") - - parser.add_option( - "-p", "--python", - help="Print native Python format.", - action="store_true", default=False, dest="pnative") - - parser.add_option( - "-a", "--all-tasks", - help="For [runtime] items (e.g. --item='script') report " - "values for all tasks prefixed by task name.", - action="store_true", default=False, dest="alltasks") - - parser.add_option( - "-n", "--null-value", - help="The string to print for unset values (default nothing).", - metavar="STRING", action="store", default='', dest="none_str") - - parser.add_option( - "-m", "--mark-up", - help="Prefix each line with '!cylc!'.", - action="store_true", default=False, dest="markup") - - parser.add_option( - "-o", "--one-line", - help="Print multiple single-value items at once.", - action="store_true", default=False, dest="oneline") - - parser.add_option( - "-t", "--tasks", - help="Print the suite task list " - "[DEPRECATED: use 'cylc list SUITE'].", - action="store_true", default=False, dest="tasks") - - (options, args) = parser.parse_args() - owner = options.db_owner - suite, suiterc, junk = parser.get_suite() - - if options.markup: - prefix = '!cylc!' - else: - prefix = '' - - config = SuiteConfig( - suite, suiterc, - template_vars=options.templatevars, - template_vars_file=options.templatevars_file, - owner=owner) - if options.tasks: - for task in config.get_task_name_list(): - print prefix + task - elif options.alltasks: - for task in config.get_task_name_list(): - items = ['[runtime]['+task+']'+i for i in options.item] - print prefix+task, - config.pcfg.idump( - items, options.sparse, options.pnative, prefix, - options.oneline, none_str=options.none_str) - else: - config.pcfg.idump( - options.item, options.sparse, options.pnative, prefix, - options.oneline, none_str=options.none_str) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-get-suite-version b/bin/cylc-get-suite-version deleted file mode 100755 index 3f1ef61d009..00000000000 --- a/bin/cylc-get-suite-version +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info] get-suite-version [OPTIONS] ARGS - -Interrogate running suite REG to find what version of cylc is running it. - -To find the version you've invoked at the command line see "cylc version".""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.task_id import TaskID -from cylc.network.suite_info import SuiteInfoClient -from cylc.cfgspec.globalcfg import GLOBAL_CFG - - -def main(): - parser = cop(__doc__, pyro=True, argdoc=[('REG', 'Suite name')]) - - (options, args) = parser.parse_args() - suite = args[0] - - pclient = SuiteInfoClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - print pclient.get_info('get_cylc_version') - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-gpanel b/bin/cylc-gpanel deleted file mode 100755 index a5f0680c197..00000000000 --- a/bin/cylc-gpanel +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc gpanel [OPTIONS] - -This is a cylc scan panel applet for monitoring running suites on a set of -hosts in GNOME 2. - -To install this applet, run "cylc gpanel --install" -and follow the instructions that it gives you. - -This applet can be tested using the --test option. - -To customize themes, copy $CYLC_DIR/conf/gcylcrc/gcylc.rc.eg to -$HOME/.cylc/gcylc.rc and follow the instructions in the file. - -To configure default suite hosts, edit the -[suite host scanning]hosts entry in your global.rc file.""" - -from optparse import OptionParser -import os -import sys - -sys.path.append( - os.path.dirname(os.path.realpath(os.path.abspath(__file__))) + '/../lib/') -cylc_dir = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) -if cylc_dir != os.getenv('CYLC_DIR', ''): - os.environ['CYLC_DIR'] = cylc_dir - -parser = OptionParser(__doc__) - -parser.add_option("--compact", - help="Switch on compact mode at runtime.", - action="store_true", dest="compact") - -parser.add_option("--install", - help="Install the panel applet.", - default=False, - action="store_true", dest="install") - -parser.add_option("--test", - help="Run in a standalone window.", - default=False, - action="store_true", dest="test") - -arglist = [a for a in sys.argv[1:] if not a.startswith("--oaf")] -(options, args) = parser.parse_args(arglist) - -import gtk -import warnings -warnings.filterwarnings('ignore', 'use the new', Warning) - -import cylc.gui - -cylc_dir = os.path.dirname( - os.path.dirname(os.path.realpath(os.path.abspath(__file__)))) -if os.path.basename(cylc_dir).startswith("cylc-"): - # If using the wrapper, reference 'cylc' rather than 'cylc-6.3.1'. - cylc_alt_dir = os.path.join(os.path.dirname(cylc_dir), "cylc") - if os.path.realpath(cylc_alt_dir) == os.path.realpath(cylc_dir): - cylc_dir = cylc_alt_dir - -if cylc_dir != os.getenv('CYLC_DIR', ''): - os.environ['CYLC_DIR'] = cylc_dir - -from cylc.gui.gpanel import ScanPanelApplet, run_in_window - - -def install_panel_applet(): - home = os.path.expanduser("~") - dest_dir = os.path.join(home, ".bonobo") - if not os.path.isdir(dest_dir): - os.mkdir(dest_dir) - for server_filename in ["cylc.server", "cylc-compact.server"]: - source_file = os.path.join(cylc_dir, "conf", "gpanel", - server_filename) - dest_file = os.path.join(dest_dir, server_filename) - f = open(source_file, "r") - text = f.read().replace("$CYLC_DIR", cylc_dir) - f.close() - open(dest_file, "w").write(text) - print dest_file, "created." - print - print """Further Instructions: - * normal users - place this line in your ~/.profile file: -export BONOBO_ACTIVATION_PATH=$HOME/.bonobo - * (sys)admin users - move the new cylc.server and cylc-compact.server -files into your /usr/lib/bonobo/servers/ directory, or equivalent. - * After logging in and out, right-click within the GNOME 2 panel and -select "Add to Panel", then choose a Cylc Applet.""" - - -def panel_applet_factory_compact(applet, iid): - my_panel_app = ScanPanelApplet(is_compact=True) - applet.add(my_panel_app.get_widget()) - applet.show_all() - return True - - -def panel_applet_factory(applet, iid): - my_panel_app = ScanPanelApplet() - applet.add(my_panel_app.get_widget()) - applet.show_all() - return True - - -if __name__ == "__main__": - if options.test: - run_in_window(options.compact) - elif options.install: - install_panel_applet() - elif options.compact: - import gnomeapplet - gnomeapplet.bonobo_factory( - "OAFIID:GNOME_CylcCompactMonitorFactory", - gnomeapplet.Applet.__gtype__, - "cylc gpanel compact", "0", panel_applet_factory_compact) - else: - import gnomeapplet - gnomeapplet.bonobo_factory( - "OAFIID:GNOME_CylcMonitorFactory", - gnomeapplet.Applet.__gtype__, - "cylc gpanel", "0", panel_applet_factory) diff --git a/bin/cylc-graph b/bin/cylc-graph deleted file mode 100755 index cbda41ce203..00000000000 --- a/bin/cylc-graph +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""1/ cylc [prep] graph [OPTIONS] SUITE [START[STOP]] - Plot the suite.rc dependency graph for SUITE. - 2/ cylc [prep] graph [OPTIONS] -f,--file FILE - Plot the specified dot-language graph file. - 3/ cylc [prep] graph [OPTIONS] --reference SUITE [START[STOP]] - Print out a reference format for the dependencies in SUITE. - 4/ cylc [prep] graph [OPTIONS] --output-file FILE SUITE - Plot SUITE dependencies to a file FILE with a extension-derived format. - If FILE endswith ".png", output in PNG format, etc. - -Plot suite dependency graphs in an interactive graph viewer. - -If START is given it overrides "[visualization] initial cycle point" to -determine the start point of the graph, which defaults to the suite initial -cycle point. If STOP is given it overrides "[visualization] final cycle point" -to determine the end point of the graph, which defaults to the graph start -point plus "[visualization] number of cycle points" (which defaults to 3). -The graph start and end points are adjusted up and down to the suite initial -and final cycle points, respectively, if necessary. - -The "Save" button generates an image of the current view, of format (e.g. png, -svg, jpg, eps) determined by the filename extension. If the chosen format is -not available a dialog box will show those that are available. - -If the optional output filename is specified, the viewer will not open and a -graph will be written directly to the file. - -GRAPH VIEWER CONTROLS: - * Center on a node: left-click. - * Pan view: left-drag. - * Zoom: +/- buttons, mouse-wheel, or ctrl-left-drag. - * Box zoom: shift-left-drag. - * "Best Fit" and "Normal Size" buttons. - * Left-to-right graphing mode toggle button. - * "Ignore suicide triggers" button. - * "Ignore cold-start tasks" button. - * "Save" button: save an image of the view. - Family (namespace) grouping controls: - Toolbar: - * "group" - group all families up to root. - * "ungroup" - recursively ungroup all families. - Right-click menu: - * "group" - close this node's parent family. - * "ungroup" - open this family node. - * "recursive ungroup" - ungroup all families below this node.""" - -import sys -import gtk -import StringIO - -import cylc.flags -from cylc.remote import remrun -from cylc.task_id import TaskID - -if remrun().execute(): - sys.exit(0) - -from cylc.CylcOptionParsers import cop - -# DEVELOPER NOTE: family grouping controls via the viewer toolbar and -# right-click menu have been rather hastily stuck on to the original -# viewer, via changes to this file and to lib/cylc/cylc_xdot.py - all -# of which could stand some refactoring to streamline the code a bit. - -# TODO - clarify what it means to choose visualization boundaries (by CLI -# or in-suite) outside of the defined suite initial and final cycle times. - - -def on_url_clicked(widget, url, event, window): - if event.button != 3: - return False - # URL is node ID - right_click_menu(event, url, type='live task', window=window) - - -def right_click_menu(event, task_id, type='live task', window=None): - name, point_string = TaskID.split(task_id) - - menu = gtk.Menu() - menu_root = gtk.MenuItem(task_id) - menu_root.set_submenu(menu) - - group_item = gtk.MenuItem('Group') - group_item.connect('activate', grouping, name, window, False, False) - ungroup_item = gtk.MenuItem('UnGroup') - ungroup_item.connect('activate', grouping, name, window, True, False) - ungroup_rec_item = gtk.MenuItem('Recursive UnGroup') - ungroup_rec_item.connect('activate', grouping, name, window, True, True) - - title_item = gtk.MenuItem(task_id) - title_item.set_sensitive(False) - menu.append(title_item) - - menu.append(gtk.SeparatorMenuItem()) - - menu.append(group_item) - menu.append(ungroup_item) - menu.append(ungroup_rec_item) - - menu.show_all() - menu.popup(None, None, None, event.button, event.time) - - # TODO - popup menus are not automatically destroyed and can be - # reused if saved; however, we need to reconstruct or at least - # alter ours dynamically => should destroy after each use to - # prevent a memory leak? But I'm not sure how to do this as yet.) - - return True - - -def grouping(w, name, window, un, recursive): - if not un: - window.get_graph(group_nodes=[name]) - else: - if recursive: - window.get_graph(ungroup_nodes=[name], ungroup_recursive=True) - else: - window.get_graph(ungroup_nodes=[name], ungroup_recursive=False) - - -def main(): - parser = cop( - __doc__, jset=True, prep=True, - argdoc=[ - ('[SUITE]', 'Suite name or path'), - ('[START]', 'Initial cycle point to plot (default=2999010100)'), - ('[STOP]', 'Final cycle point to plot (default=2999010123)')]) - - parser.add_option( - "-n", "--namespaces", - help="Plot the suite namespace inheritance hierarchy " - "(task run time properties).", - action="store_true", default=False, dest="namespaces") - - parser.add_option( - "-f", "--file", - help="View a specific dot-language graphfile.", - metavar="FILE", action="store", default=None, dest="filename") - - parser.add_option( - "--filter", help="Filter out one or many nodes.", - metavar="NODE_NAME_PATTERN", action="append", dest="filter_patterns") - - parser.add_option( - "-O", "--output-file", - help="Output to a specific file, with a format given by " - "--output-format or extrapolated from the extension. " - "'-' implies stdout in plain format.", - metavar="FILE", action="store", default=None, dest="output_filename") - - parser.add_option( - "--output-format", - help="Specify a format for writing out the graph to --output-file " - "e.g. png, svg, jpg, eps, dot. 'ref' is a special sorted plain " - "text format for comparison and reference purposes.", - metavar="FORMAT", action="store", default=None, dest="output_format") - - parser.add_option( - "-r", "--reference", - help="Output in a sorted plain text format for comparison purposes. " - "If not given, assume --output-file=-.", - action="store_true", default=False, dest="reference") - - (options, args) = parser.parse_args() - - # import modules that require gtk now, so that a display is not needed - # just to get command help (e.g. when running make on a post-commit hook - # on a remote repository). - import gtk - import gobject - from xdot import DotWindow - from cylc.cylc_xdot import ( - MyDotWindow, MyDotWindow2, get_reference_from_plain_format) - - if options.filename: - if len(args) != 0: - parser.error( - "file graphing arguments: '-f FILE' or '--file=FILE'") - sys.exit(1) - file = options.filename - from xdot import DotWindow - if options.output_filename: - raise SystemExit("ERROR: output-file not supported for " - "dot files. Use 'dot' command instead.") - window = DotWindow() - window.update(file) - window.connect('destroy', gtk.main_quit) - # checking periodically for file changed - gobject.timeout_add(1000, window.update, file) - gtk.main() - sys.exit(0) - - should_hide_gtk_window = (options.output_filename is not None) - - suite, suiterc, junk = parser.get_suite() - - start_point_string = stop_point_string = None - if len(args) >= 2: - start_point_string = args[1] - if len(args) == 3: - stop_point_string = args[2] - - if options.namespaces: - window = MyDotWindow2(suite, suiterc, options.templatevars, - options.templatevars_file, - should_hide=should_hide_gtk_window) - else: - window = MyDotWindow(suite, suiterc, start_point_string, - stop_point_string, options.templatevars, - options.templatevars_file, - should_hide=should_hide_gtk_window) - - window.widget.connect('clicked', on_url_clicked, window) - if options.filter_patterns: - filter_patterns = set(options.filter_patterns) - window.set_filter_graph_patterns(options.filter_patterns) - window.get_graph() - - if options.reference and options.output_filename is None: - options.output_filename = "-" - - if options.output_filename: - if (options.reference or options.output_filename.endswith(".ref") or - options.output_format == "ref"): - dest = StringIO.StringIO() - window.graph.draw(dest, format="plain", prog="dot") - output_text = get_reference_from_plain_format(dest.getvalue()) - if options.output_filename == "-": - sys.stdout.write(output_text) - else: - open(options.output_filename).write(output_text) - else: - if options.output_filename == "-": - window.graph.draw(sys.stdout, format="plain", prog="dot") - elif options.output_format: - window.graph.draw( - options.output_filename, format=options.output_format, - prog="dot" - ) - else: - window.graph.draw(options.output_filename, prog="dot") - sys.exit(0) - - window.connect('destroy', gtk.main_quit) - gtk.main() - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-graph-diff b/bin/cylc-graph-diff deleted file mode 100755 index 75662e680ae..00000000000 --- a/bin/cylc-graph-diff +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -set -eu - -usage() { - cat <<'__USAGE__' -Usage: cylc graph-diff [OPTIONS] SUITE1 SUITE2 -- [GRAPH_OPTIONS_ARGS] - -Difference 'cylc graph --reference' output for SUITE1 and SUITE2. - -OPTIONS: Use '-g' to launch a graphical diff utility. - Use '--diff-cmd=MY_DIFF_CMD' to use a custom diff tool. - -SUITE1, SUITE2: Suite names to compare. -GRAPH_OPTIONS_ARGS: Options and arguments passed directly to cylc graph. -__USAGE__ -} - -DIFF_CMD="diff -u" -GRAPHICAL_DIFF=false -SUITES="" -NUM_SUITES=0 -for ARG in "$@"; do - shift - if [[ "$ARG" == '--' ]]; then - break - elif [[ "$ARG" == '--help' ]]; then - usage - exit 0 - elif [[ "$ARG" == "-g" ]]; then - GRAPHICAL_DIFF=true - elif [[ "$ARG" == --diff-cmd=* ]]; then - DIFF_CMD=${ARG#*=} - else - # A suite - check it's registered. - if ! cylc db print --fail "$ARG$" >/dev/null 2>&1; then - echo "Suite not found: "$ARG >&2 - exit 1 - fi - SUITES="$SUITES $ARG" - NUM_SUITES=$((NUM_SUITES+1)) - fi -done -if (( NUM_SUITES != 2 )); then - usage >&2 - exit 1 -fi - -DIFF_FILES="" -for SUITE in $SUITES; do - FILE=$(mktemp -t $SUITE.graph.ref.XXXX) - cylc graph --reference "$@" $SUITE >$FILE - DIFF_FILES="$DIFF_FILES $FILE" -done - -if $GRAPHICAL_DIFF; then - for DIFFTOOL in xxdiff diffuse kdiff3 kompare gvimdiff2 meld tkdiff; do - if type -P $DIFFTOOL >/dev/null; then - "$DIFFTOOL" $DIFF_FILES - exit 0 - fi - done - echo "Error: no graphical diff tool found." >&2 - exit 1 -fi - -$DIFF_CMD $DIFF_FILES diff --git a/bin/cylc-gscan b/bin/cylc-gscan deleted file mode 100755 index 115ee38a6d0..00000000000 --- a/bin/cylc-gscan +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc gscan [OPTIONS] - -This is the cylc scan gui for monitoring running suites on a set of -hosts. - -To customize themes copy $CYLC_DIR/conf/gcylcrc/gcylc.rc.eg to -$HOME/.cylc/gcylc.rc and follow the instructions in the file.""" - -from cylc.CylcOptionParsers import cop - -parser = cop(__doc__, argdoc=[]) - -parser.add_option("--host", - help="Host names to monitor (override site default).", - metavar="HOST", action="append", - dest="hosts") -parser.add_option("--poll-interval", - help="Polling interval (time between updates) in seconds", - type="int", metavar="SECONDS", dest="interval") - -options, args = parser.parse_args() - -import gtk -import warnings -warnings.filterwarnings('ignore', 'use the new', Warning) - -from cylc.gui.gscan import ScanApp - -ScanApp(hosts=options.hosts, owner=options.owner, - poll_interval=options.interval) -gtk.main() diff --git a/bin/cylc-gsummary b/bin/cylc-gsummary deleted file mode 100755 index 2d5c0322f5c..00000000000 --- a/bin/cylc-gsummary +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -exec $(dirname $0)/cylc scan "$@" diff --git a/bin/cylc-gui b/bin/cylc-gui deleted file mode 100755 index 82a65025eb3..00000000000 --- a/bin/cylc-gui +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc gui [OPTIONS] [REG] -gcylc [OPTIONS] [REG] - -This is the cylc Graphical User Interface. - -Local suites can be opened and switched between from within gcylc. To connect -to running remote suites (whose passphrase you have installed) you must -currently use --host and/or --user on the gcylc command line. - -Available task state color themes are shown under the View menu. To customize -themes copy $CYLC_DIR/conf/gcylcrc/gcylc.rc.eg to $HOME/.cylc/gcylc.rc and -follow the instructions in the file. - -To see current configuration settings use "cylc get-gui-config". - -In the graph view, View -> Options -> "Write Graph Frames" writes .dot graph -files to the suite share directory (locally, for a remote suite). These can -be processed into a movie by \$CYLC_DIR/dev/bin/live-graph-movie.sh=.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(): - sys.exit(0) - -import os -from cylc.CylcOptionParsers import cop -import cylc.flags - - -def main(): - sys.path.append( - os.path.dirname( - os.path.realpath(os.path.abspath(__file__))) + '/../lib') - sys.path.append( - os.path.dirname(os.path.realpath(os.path.abspath(__file__))) + '/../') - - parser = cop(__doc__, pyro=True, noforce=True, jset=True, - argdoc=[('[REG]', 'Suite name')]) - - parser.add_option( - "-r", "--restricted", - help="Restrict display to 'active' task states: submitted, " - "submit-failed, submit-retrying, running, failed, retrying; " - "and disable the graph view. This may be needed for very large " - "suites. The state summary icons in the status bar still " - "represent all task proxies.", - action="store_true", default=False, dest="restricted") - - (options, args) = parser.parse_args() - - # import modules that require gtk now, so that a display is not needed - # just to get command help (e.g. when running make on a post-commit hook - # on a remote repository). - - import gtk - import warnings - warnings.filterwarnings('ignore', 'use the new', Warning) - from cylc.gui.app_gcylc import ControlApp - - # Make current working directory be $HOME. Otherwise (1) if the user - # attempts to start gcylc from a CWD that has been removed, Pyro will - # not be importable below; and (2) if the CWD gets removed later while - # gcylc is running, subprocesses spawned by gcylc will fail when they - # attempt to determine their CWD. - os.chdir(os.environ['HOME']) - - gtk.settings_get_default().set_long_property( - "gtk-toolbar-icon-size", gtk.ICON_SIZE_SMALL_TOOLBAR, "main") - gtk.settings_get_default().set_long_property( - "gtk-button-images", True, "main") - gtk.settings_get_default().set_long_property( - "gtk-menu-images", True, "main") - - if len(args) == 1: - suite = args[0] - else: - suite = None - app = ControlApp(suite, options.db, options.owner, options.host, - options.port, options.pyro_timeout, options.templatevars, - options.templatevars_file, options.restricted) - gtk.main() - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-hold b/bin/cylc-hold deleted file mode 100755 index 9b980e7d5b9..00000000000 --- a/bin/cylc-hold +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] hold [OPTIONS] ARGS - -Hold one or more waiting tasks (cylc hold REG MATCH POINT), or -a whole suite (cylc hold REG). - -Held tasks do not submit even if they are ready to run.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage - - -def main(): - parser = cop( - __doc__ + multitask_usage + """ -See also 'cylc [control] release'.""", pyro=True, multitask=True, - argdoc=[ - ("REG", "Suite name"), - ('[MATCH]', 'Task or family name matching regular expression'), - ('[POINT]', 'Task cycle point (e.g. date-time or integer)')]) - - parser.add_option( - "--after", - help="Hold whole suite AFTER this cycle point.", - metavar="CYCLE_POINT", action="store", dest="hold_point_string") - - (options, args) = parser.parse_args() - suite = args[0] - - whole_suite = True - if len(args) == 3: - whole_suite = False - name = args[1] - point_string = args[2] - prompt('Hold task(s) %s at %s in %s' % (name, point_string, suite), - options.force) - elif options.hold_point_string: - prompt('Hold suite after %s' % options.hold_point_string, - options.force) - elif len(args) == 1: - prompt('Hold suite %s' % suite, options.force) - else: - parser.error("Wrong number of arguments") - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - if options.hold_point_string: - pclient.put_command('hold_after_point_string', - options.hold_point_string) - elif whole_suite: - pclient.put_command('hold_suite') - else: - pclient.put_command('hold_task', name, point_string, options.is_family) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-import-examples b/bin/cylc-import-examples deleted file mode 100755 index cace729f46c..00000000000 --- a/bin/cylc-import-examples +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -set -e - -usage() { - echo "" - echo "Usage: cylc [admin] import-examples DIR [GROUP]" - echo "" - echo "Copy the cylc example suites to DIR/GROUP and register" - echo "them for use under the GROUP suite name group." - echo "" - echo "Arguments:" - echo " DIR destination directory" - echo " GROUP suite name group (default: cylc-)" -} - -if [[ $1 == '-h' ]] || [[ $1 == '--help' ]]; then - usage - exit 0 -fi - -if [[ -z $CYLC_DIR ]]; then - echo "ERROR: \$CYLC_DIR is not defined. Run this script via" >&2 - echo "the main command interface: 'cylc admin import-examples'" >&2 - exit 1 -fi - -if [[ $# > 2 ]] || [[ $# < 1 ]]; then - usage >&2 - exit 1 -fi - -DIR=$1 - -if [[ $# == 2 ]]; then - TOPGRP=$2 -else - TOPGRP=cylc-$( cylc -v | tr '.' '-' ) -fi - -if $( cylc db print --fail $TOPGRP > /dev/null 2>&1 ); then - echo "ERROR: the $TOPGRP name group already exists." >&2 - echo " Reregister it, or choose another name group." >&2 - exit 1 -fi - -TOPDIR=$DIR/$TOPGRP - -if [[ -d $TOPDIR ]]; then - echo "ERROR: $TOPDIR already exists." >&2 - echo "Remove it or choose another DIR." >&2 - exit 1 -fi - -echo " + Copying example suites" -mkdir -p $TOPDIR -cp -r $CYLC_DIR/examples/* $TOPDIR - -echo " + Registering example suites" -cd $TOPDIR -SUITE_RCS=$( find . -name suite.rc | sed -e 's@./@@' ) -for SUITE_RC in $SUITE_RCS; do - SUITE_DEF_DIR=$( dirname $SUITE_RC ) - SUITE_REG_NAME=${TOPGRP}.$( echo $SUITE_DEF_DIR | tr '/' '.' ) - cylc db register $SUITE_REG_NAME $SUITE_DEF_DIR -done - -echo "DONE" diff --git a/bin/cylc-insert b/bin/cylc-insert deleted file mode 100755 index a618935dac8..00000000000 --- a/bin/cylc-insert +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] insert [OPTIONS] ARGS - -Insert task proxies into a running suite. Uses of insertion include: - 1) insert a task that was excluded by the suite definition at start-up. - 2) reinstate a task that was previously removed from a running suite. - 3) re-run an old task that cannot be retriggered because its task proxy - is no longer live in the a suite. - -Be aware that inserted cycling tasks keep on cycling as normal, even if -another instance of the same task exists at a later cycle (instances of -the same task at different cycles can coexist, but a newly spawned task -will not be added to the pool if it catches up to another task with the -same ID). - -See also 'cylc submit', for running tasks without the scheduler. -""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage - - -def main(): - parser = cop( - __doc__ + multitask_usage, - pyro=True, multitask=True, - argdoc=[ - ("REG", "Suite name"), - ('MATCH', 'Task or family name matching regular expression'), - ('CYCLE_POINT', 'Cycle point (e.g. date-time or integer)'), - ('[STOP_POINT]', 'Optional stop cycle point for inserted task.')]) - - (options, args) = parser.parse_args() - suite = args[0] - - name = args[1] - point_string = args[2] - if len(args) == 4: - stop_point_string = args[3] - else: - stop_point_string = None - - prompt('Insert %s at %s in %s' % (name, point_string, suite), - options.force) - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - pclient.put_command('insert_task', name, point_string, options.is_family, - stop_point_string) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-job-kill b/bin/cylc-job-kill deleted file mode 100755 index 530eca85a13..00000000000 --- a/bin/cylc-job-kill +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [control] job-kill ST-FILE - -(This command is for internal use. Users should use "cylc kill".) Read the job -status file to obtain the name of the batch system and the job ID in the -system. Invoke the relevant batch system command to ask the batch system to -remove and terminate the job. - -""" - - -from cylc.CylcOptionParsers import cop -from cylc.batch_sys_manager import BATCH_SYS_MANAGER -import sys - - -def main(): - """CLI main.""" - parser = cop(__doc__, argdoc=[("ST-FILE", "the task status file")]) - sys.exit(BATCH_SYS_MANAGER.job_kill(parser.parse_args()[1][0])) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-job-logs-retrieve b/bin/cylc-job-logs-retrieve deleted file mode 100755 index 022b0003388..00000000000 --- a/bin/cylc-job-logs-retrieve +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [hook] job-logs-retrieve [OPTIONS] HOST:HOST-PATH LOCALHOST-PATH - -(This command is for internal use.) -Retrieve logs from a remote host for a task job. - -""" - - -import os -from subprocess import check_call -import shlex -import sys -import traceback - -from cylc.cfgspec.globalcfg import GLOBAL_CFG -from cylc.CylcOptionParsers import cop -from cylc.wallclock import get_time_string_from_unix_time - - -def main(): - """CLI main.""" - # Options and arguments - opt_parser = cop(__doc__, argdoc=[ - ("HOST:HOST-PATH", "Path to remote job logs directory"), - ("LOCALHOST-PATH", "Path to local job logs directory"), - ]) - opt_parser.add_option("--max-size", - help="Don't transfer any file larger than SIZE.", - action="store", default="10M", dest="max_size", metavar="SIZE") - opts, args = opt_parser.parse_args() - - # Determine the remote shell template to use - source, target = args - source_auth, source_path = source.split(":", 1) - if "@" in source_auth: - source_owner, source_host = source_auth.split("@", 1) - else: - source_owner, source_host = (None, source_auth) - ssh_tmpl = str(GLOBAL_CFG.get_host_item( - "remote shell template", source_host, source_owner)).replace(" %s", "") - - # Retrieve remote job logs - # N.B. "scp" does not have a "max-size" option. - check_call([ - "rsync", "-a", "--rsh=" + ssh_tmpl, "--max-size=" + opts.max_size, - source + "/", target]) - - filenames = os.listdir(target) - if "job.out" not in filenames: - sys.exit("ERROR: job.out: file not found") - sys.stdout.write("%s:\n" % os.path.basename(sys.argv[0])) - for filename in filenames: - stat = os.stat(os.path.join(target, filename)) - sys.stdout.write("%s\t%s\t%s\n" % ( - get_time_string_from_unix_time(stat.st_mtime), - stat.st_size, - filename)) - - os.listdir(target) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-job-poll b/bin/cylc-job-poll deleted file mode 100755 index 6ebb6a3361b..00000000000 --- a/bin/cylc-job-poll +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [control] job-poll ST-FILE - -(This command is for internal use. Users should use "cylc poll".) Poll a -submitted or running job by inspecting its status file and possibly querying -the batch system to see if the job is still alive or not. - -""" - - -import sys -from cylc.CylcOptionParsers import cop -from cylc.batch_sys_manager import BATCH_SYS_MANAGER - - -def main(): - """CLI main.""" - parser = cop(__doc__, argdoc=[("ST-FILE", "the task status file")]) - args = parser.parse_args()[1] - return sys.stdout.write(BATCH_SYS_MANAGER.job_poll(args[0])) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-job-submit b/bin/cylc-job-submit deleted file mode 100755 index baa98ccc637..00000000000 --- a/bin/cylc-job-submit +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [task] job-submit [--remote-mode] JOB-FILE-PATH - -(This command is for internal use. Users should use "cylc submit".) -Submit a job file. - -""" - - -import sys -from cylc.remote import remrun - - -def main(): - """CLI main.""" - parser = cop( - __doc__, - argdoc=[("JOB-FILE-PATH", "the path of the job file")]) - parser.add_option( - "--remote-mode", - help="Is this being run on a remote job host?", - action="store_true", dest="remote_mode", default=False) - opts, args = parser.parse_args() - ret_code, out, err, job_id = BATCH_SYS_MANAGER.job_submit( - args[0], opts.remote_mode) - if err: - sys.stderr.write(err) - if out: - sys.stdout.write(out) - if job_id: - sys.stdout.write( - "%s=%s\n" % (BATCH_SYS_MANAGER.CYLC_BATCH_SYS_JOB_ID, job_id)) - sys.exit(ret_code) - - -if __name__ == "__main__" and not remrun().execute(): - from cylc.CylcOptionParsers import cop - from cylc.batch_sys_manager import BATCH_SYS_MANAGER - main() diff --git a/bin/cylc-jobs-kill b/bin/cylc-jobs-kill deleted file mode 100755 index 817870767e7..00000000000 --- a/bin/cylc-jobs-kill +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [control] jobs-kill JOB-LOG-ROOT [JOB-LOG-DIR ...] - -(This command is for internal use. Users should use "cylc kill".) Read job -status files to obtain the names of the batch systems and the job IDs in the -systems. Invoke the relevant batch system commands to ask the batch systems to -terminate the jobs. - -""" - - -import sys -from cylc.remote import remrun - - -def main(): - """CLI main.""" - parser = cop(__doc__, argdoc=[ - ("JOB-LOG-ROOT", "The log/job sub-directory for the suite"), - ("[JOB-LOG-DIR ...]", "A point/name/submit_num sub-directory")]) - args = parser.parse_args()[1] - BATCH_SYS_MANAGER.jobs_kill(args[0], args[1:]) - - -if __name__ == "__main__" and not remrun().execute(): - from cylc.CylcOptionParsers import cop - from cylc.batch_sys_manager import BATCH_SYS_MANAGER - main() diff --git a/bin/cylc-jobs-poll b/bin/cylc-jobs-poll deleted file mode 100755 index 309e72093ec..00000000000 --- a/bin/cylc-jobs-poll +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [control] jobs-poll JOB-LOG-ROOT [JOB-LOG-DIR ...] - -(This command is for internal use. Users should use "cylc poll".) Read job -status files to obtain the statuses of the jobs. If necessary, Invoke the -relevant batch system commands to ask the batch systems for more statuses. - -""" - - -import sys -from cylc.remote import remrun - - -def main(): - """CLI main.""" - parser = cop(__doc__, argdoc=[ - ("JOB-LOG-ROOT", "The log/job sub-directory for the suite"), - ("[JOB-LOG-DIR ...]", "A point/name/submit_num sub-directory")]) - args = parser.parse_args()[1] - BATCH_SYS_MANAGER.jobs_poll(args[0], args[1:]) - - -if __name__ == "__main__" and not remrun().execute(): - from cylc.CylcOptionParsers import cop - from cylc.batch_sys_manager import BATCH_SYS_MANAGER - main() diff --git a/bin/cylc-jobs-submit b/bin/cylc-jobs-submit deleted file mode 100755 index b746a055f78..00000000000 --- a/bin/cylc-jobs-submit +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [control] jobs-submit JOB-LOG-ROOT [JOB-LOG-DIR ...] - -(This command is for internal use. Users should use "cylc submit".) Submit task -jobs to relevant batch systems. On a remote job host, this command reads the -job files from STDIN. - -""" - - -import sys -from cylc.remote import remrun - - -def main(): - """CLI main.""" - parser = cop(__doc__, argdoc=[ - ("JOB-LOG-ROOT", "The log/job sub-directory for the suite"), - ("[JOB-LOG-DIR ...]", "A point/name/submit_num sub-directory")]) - parser.add_option( - "--remote-mode", - help="Is this being run on a remote job host?", - action="store_true", dest="remote_mode", default=False) - opts, args = parser.parse_args() - BATCH_SYS_MANAGER.jobs_submit( - args[0], args[1:], remote_mode=opts.remote_mode) - - -if __name__ == "__main__" and not remrun().execute(): - from cylc.CylcOptionParsers import cop - from cylc.batch_sys_manager import BATCH_SYS_MANAGER - main() diff --git a/bin/cylc-jobscript b/bin/cylc-jobscript deleted file mode 100755 index 1affc926589..00000000000 --- a/bin/cylc-jobscript +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -set -e - -usage() { - cat <<__END__ -Usage: cylc [prep] jobscript [OPTIONS] REG TASK - -Generate a task job script and print it to stdout. - -Here's how to capture the script in the vim editor: - % cylc jobscript REG TASK | vim - -Emacs unfortunately cannot read from stdin: - % cylc jobscript REG TASK > tmp.sh; emacs tmp.sh - -This command wraps 'cylc [control] submit --dry-run'. -Other options (e.g. for suite host and owner) are passed -through to the submit command. - -Options: - -h,--help - print this usage message. - (see also 'cylc submit --help') - -Arguments: - REG - Registered suite name. - TASK - Task ID (NAME.CYCLE_POINT) -__END__ -} - -for arg in "${@}"; do - if [[ "${arg}" == '-h' ]] || [[ "${arg}" == '--help' ]]; then - usage - exit 0 - fi -done - -JOBSCRIPT=$(cylc submit --dry-run "${@}" | awk -F= '$1 == "JOB SCRIPT" {print $2}') -if [[ -z "${JOBSCRIPT}" ]]; then - echo "ERROR: no jobscript generated" >&2 - exit 1 -fi -# (only send script contents to stdout) -echo "Task Job Script Generated: ${JOBSCRIPT}" >&2 -exec less "${JOBSCRIPT}" diff --git a/bin/cylc-kill b/bin/cylc-kill deleted file mode 100755 index 0946b3adbb3..00000000000 --- a/bin/cylc-kill +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] kill [OPTIONS] ARGS - -Kill jobs of active tasks (those in the 'submitted' or 'running' states) and -update their statuses accordingly. - -To kill one or more tasks, "cylc kill REG MATCH POINT"; to kill all active -tasks: "cylc kill REG". - -Kill a 'submitted' or 'running' task and update the suite state accordingly. -""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage - - -def main(): - parser = cop( - __doc__ + multitask_usage, - pyro=True, multitask=True, - argdoc=[('REG', 'Suite name'), - ('[MATCH]', 'Task or family name matching regular expression'), - ('[POINT]', 'Task cycle point (e.g. date-time or integer)')]) - - (options, args) = parser.parse_args() - suite = args[0] - - if len(args) == 3: - name = args[1] - point_string = args[2] - elif len(args) == 1: - name = None - point_string = None - else: - parser.error("Wrong number of arguments.") - - if name and point_string: - prompt('Kill task %s at %s in %s' % (name, point_string, suite), - options.force) - else: - prompt('Kill ALL task in %s' % (suite), options.force) - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - pclient.put_command('kill_tasks', name, point_string, options.is_family) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-list b/bin/cylc-list deleted file mode 100755 index 5948faf51a4..00000000000 --- a/bin/cylc-list +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [info|prep] list|ls [OPTIONS] ARGS - -Print runtime namespace names (tasks and families), the first-parent -inheritance graph, or actual tasks for a given cycle range. - -The first-parent inheritance graph determines the primary task family -groupings that are collapsible in gcylc suite views and the graph -viewer tool. To visualize the full multiple inheritance hierarchy use: - 'cylc graph -n'.""" - -import os -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.config import SuiteConfig - - -def main(): - - parser = cop(__doc__, jset=True, prep=True) - - parser.add_option( - "-a", "--all-tasks", - help="Print all tasks, not just those used in the graph.", - action="store_true", default=False, dest="all_tasks") - - parser.add_option( - "-n", "--all-namespaces", - help="Print all runtime namespaces, not just tasks.", - action="store_true", default=False, dest="all_namespaces") - - parser.add_option( - "-m", "--mro", - help="Print the linear \"method resolution order\" for each namespace " - "(the multiple-inheritance precedence order as determined by the " - "C3 linearization algorithm).", - action="store_true", default=False, dest="mro") - - parser.add_option( - "-t", "--tree", - help="Print the first-parent inheritance hierarchy in tree form.", - action="store_true", default=False, dest="tree") - - parser.add_option( - "-b", "--box", - help="With -t/--tree, using unicode box characters. Your terminal " - "must be able to display unicode characters.", - action="store_true", default=False, dest="box") - - parser.add_option( - "-w", "--with-titles", help="Print namespaces titles too.", - action="store_true", default=False, dest="titles") - - parser.add_option( - "-p", "--points", - help="Print actual task IDs from the " - "START [through STOP] cycle points.", - metavar="START[,STOP]", action="store", default=None, dest="crange") - - parser.add_option( - "-c", "--cycles", help="(deprecated: use -p/--points).", - metavar="START[,STOP]", action="store", default=None, dest="crange") - - (options, args) = parser.parse_args() - suite, suiterc, junk = parser.get_suite() - - if options.all_tasks and options.all_namespaces: - parser.error("Choose either -a or -n") - if options.all_tasks: - which = "all tasks" - elif options.all_namespaces: - which = "all namespaces" - elif options.crange: - which = "crange" - try: - tr_start, tr_stop = options.crange.split(',') - except ValueError: - tr_start = tr_stop = options.crange - else: - which = "graphed tasks" - - if options.tree: - if os.environ['LANG'] == 'C' and options.box: - print >> sys.stderr, "WARNING, ignoring -t/--tree: $LANG=C" - options.tree = False - - if options.titles and options.mro: - parser.error("Please choose --mro or --title, not both") - - if options.tree and any( - [options.all_tasks, options.all_namespaces, options.mro]): - print >> sys.stderr, "WARNING: -t chosen, ignoring non-tree options." - - config = SuiteConfig( - suite, suiterc, - template_vars=options.templatevars, - template_vars_file=options.templatevars_file) - if options.tree: - config.print_first_parent_tree( - pretty=options.box, titles=options.titles) - elif options.crange: - node_labels = config.get_node_labels(tr_start, tr_stop) - node_labels.sort() - for nl in node_labels: - print nl - else: - result = config.get_namespace_list(which) - namespaces = result.keys() - namespaces.sort() - - if (options.mro or options.titles): - # compute padding - maxlen = 0 - for ns in namespaces: - if len(ns) > maxlen: - maxlen = len(ns) - padding = maxlen*' ' - - for ns in namespaces: - if options.mro: - print ns, padding[0:len(padding)-len(ns)], - for i in config.get_mro(ns): - print i, - print - elif options.titles: - print ns, padding[0:len(padding)-len(ns)], - print result[ns] - else: - print ns - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-message b/bin/cylc-message deleted file mode 100755 index 4664ae9b377..00000000000 --- a/bin/cylc-message +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -"""cylc [task] message [OPTIONS] MESSAGE ... - -This command is part of the cylc task messaging interface, used by -running tasks to communicate progress to their parent suite. - -The message command can be used to report "message outputs" completed. -Other messages received by the suite daemon will just be logged. - -Suite and task identity are determined from the task execution -environment supplied by the suite (or by the single task 'submit' -command, in which case case the message is just printed to stdout).""" - - -import os -import sys -from optparse import OptionParser -import cylc.flags -from cylc.task_message import TaskMessage - - -def main(): - """CLI.""" - parser = OptionParser(__doc__) - - parser.add_option( - "-p", "--priority", metavar="PRIORITY", type="choice", - choices=['NORMAL', 'WARNING', 'CRITICAL'], - help="message priority: NORMAL, WARNING, or CRITICAL; default NORMAL.", - action="store", dest="priority", default="NORMAL") - - parser.add_option( - "-v", "--verbose", help="Verbose output mode.", action="store_true", - default=False, dest="verbose") - - options, args = parser.parse_args() - if not args: - parser.error("No task message supplied") - - try: - TaskMessage(priority=options.priority).send(args) - except Exception, exc: - print >> sys.stderr, 'ERROR: task messaging failure.' - if os.getenv("CYLC_DEBUG") in ["True", "true"]: - import traceback - traceback.print_exc(exc) - raise SystemExit(exc) - - -if __name__ == "__main__": - main() diff --git a/bin/cylc-monitor b/bin/cylc-monitor deleted file mode 100755 index ba2cd3455f2..00000000000 --- a/bin/cylc-monitor +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -Display the state of live task proxies in a running suite. - -For color terminal ASCII escape codes, see -http://ascii-table.com/ansi-escape-sequences.php -""" - -import sys -if '--use-ssh' in sys.argv[1:]: - # requires local terminal - sys.exit("No '--use-ssh': this command requires a local terminal.") - -import os -import re -from time import sleep - -from cylc.CylcOptionParsers import cop -from cylc.task_state import task_state -from cylc.registration import localdb -from cylc.network.suite_state import ( - StateSummaryClient, SuiteStillInitialisingError) -from cylc.wallclock import get_time_string_from_unix_time - - -class SuiteMonitor(object): - def __init__(self): - self.parser = cop( - """cylc [info] monitor [OPTIONS] ARGS - -A terminal-based live suite monitor. Exit with 'Ctrl-C'.""", - pyro=True, noforce=True) - - self.parser.add_option( - "-a", "--align", - help="Align task names. Only useful for small suites.", - action="store_true", default=False, dest="align_columns") - - self.parser.add_option( - "-r", "--restricted", - help="Restrict display to 'active' task states: submitted, " - "submit-failed, submit-retrying, running, failed, retrying. " - "This may be needed for very large suites. The state summary " - "line still represents all task proxies.", - action="store_true", default=False, dest="restricted") - - self.parser.add_option( - "-o", "--once", - help="Show a single view then exit.", - action="store_true", default=False, dest="once") - - self.parser.add_option( - "-u", "--runahead", - help="Display task proxies in the runahead pool (off by default).", - action="store_true", default=False, dest="display_runahead") - - self.parser.add_option( - "-i", "--interval", - help="Interval between suite state retrievals, " - "in seconds (default 1).", - metavar="SECONDS", action="store", default=1, - dest="update_interval") - - def run(self): - (options, args) = self.parser.parse_args() - suite = args[0] - - client_name = os.path.basename(sys.argv[0]) - if options.restricted: - client_name += " -r" - - legend = '' - for state in task_state.legal: - legend += "%s%s%s" % ( - task_state.ctrl[state], state, task_state.ctrl_end) - legend = legend.rstrip() - len_header = sum(len(s) for s in task_state.legal) - - self.pclient = StateSummaryClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db) - - while True: - try: - glbl, task_summaries, fam_summaries = ( - self.pclient.get_suite_state_summary()) - except SuiteStillInitialisingError as exc: - print str(exc) - except Exception as exc: - print >> sys.stderr, "\033[1;37;41mERROR%s" % ( - task_state.ctrl_end), str(exc) - self.pclient.reset() - else: - states = [t["state"] for t in task_summaries.values() if ( - "state" in t)] - n_tasks_total = len(states) - if options.restricted: - task_summaries = dict( - (i, j) for i, j in task_summaries.items() if ( - j['state'] in - task_state.legal_for_restricted_monitoring)) - if not options.display_runahead: - task_summaries = dict( - (i, j) for i, j in task_summaries.items() if ( - j['state'] != 'runahead')) - try: - updated_at = get_time_string_from_unix_time( - glbl['last_updated']) - except (TypeError, ValueError): - # Older suite. - updated_at = glbl['last_updated'].isoformat() - - run_mode = glbl['run_mode'] - paused = glbl['paused'] - stopping = glbl['stopping'] - will_pause_at = glbl['will_pause_at'] - will_stop_at = glbl['will_stop_at'] - - task_info = {} - name_list = set() - task_ids = task_summaries.keys() - for task_id in task_ids: - name = task_summaries[task_id]['name'] - point_string = task_summaries[task_id]['label'] - state = task_summaries[task_id]['state'] - name_list.add(name) - if point_string not in task_info: - task_info[point_string] = {} - task_info[point_string][name] = "%s%s%s" % ( - task_state.ctrl[state], name, task_state.ctrl_end) - - # Construct lines to blit to the screen. - blit = [] - - suite_name = suite - if run_mode != "live": - suite_name += " (%s)" % run_mode - prefix = "%s - %d tasks" % (suite_name, int(n_tasks_total)) - suffix = "%s %s" % (client_name, self.pclient.my_uuid) - title_str = ' ' * len_header - title_str = prefix + title_str[len(prefix):] - title_str = '\033[1;37;44m%s%s%s' % ( - title_str[:-len(suffix)], suffix, task_state.ctrl_end) - blit.append(title_str) - blit.append(legend) - - updated_str = "updated: %s%s%s" % ( - '\033[1;38m', updated_at, task_state.ctrl_end) - blit.append(updated_str) - - summary = 'state summary:' - try: - state_totals = glbl['state totals'] - except KeyError: - # Back-compat for suite daemons <= 6.4.1. - state_totals = {} - for state in states: - state_totals.setdefault(state, 0) - state_totals[state] += 1 - for state, tot in state_totals.items(): - summary += '%s %d %s' % ( - task_state.ctrl[state], tot, task_state.ctrl_end) - blit.append(summary) - - if stopping: - suffix = 'S_T_O_P_P_I_N_G' - elif paused: - suffix = 'P_A_U_S_E_D' - elif will_pause_at: - suffix = 'P_A_U_S_I_N_G__A_T__' + will_pause_at - elif will_stop_at: - suffix = 'S_T_O_P_P_I_N_G__A_T__' + will_stop_at - else: - suffix = 'R_U_N_N_I_N_G' - divider_str = '_'*len_header - divider_str = "\033[1;31m%s%s%s" % ( - divider_str[:-len(suffix)], suffix, task_state.ctrl_end) - blit.append(divider_str) - - blitlines = {} - for point_str, val in task_info.items(): - indx = point_str - line = "%s%s%s" % ( - '\033[1;34m', point_str, task_state.ctrl_end) - if options.align_columns: - for name in sorted(name_list): - if name in val: - line += " %s" % val[name] - else: - line += " %s" % (' '*len(name)) - else: - for name, info in val.items(): - line += " %s" % info - blitlines[indx] = line - - if not options.once: - os.system("clear") - print '\n'.join(blit) - indxs = blitlines.keys() - try: - int(indxs[1]) - except: - indxs.sort() - else: - indxs.sort(key=int) - for ix in indxs: - print blitlines[ix] - - if options.once: - break - else: - sleep(float(options.update_interval)) - - -if __name__ == "__main__": - monitor = SuiteMonitor() - try: - monitor.run() - except KeyboardInterrupt: - monitor.pclient.signout() diff --git a/bin/cylc-nudge b/bin/cylc-nudge deleted file mode 100755 index ce731b389eb..00000000000 --- a/bin/cylc-nudge +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] nudge [OPTIONS] ARGS - -Cause the cylc task processing loop to be invoked in a running suite. - -This happens automatically when the state of any task changes such that -task processing (dependency negotation etc.) is required, or if a -clock-trigger task is ready to run. - -The main reason to use this command is to update the "estimated time till -completion" intervals shown in the tree-view suite control GUI, during -periods when nothing else is happening. -""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop - - -def main(): - parser = cop(__doc__, pyro=True) - - (options, args) = parser.parse_args() - suite = args[0] - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - success, msg = pclient.put_command('nudge') - if success: - print msg - else: - sys.exit(msg) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-ping b/bin/cylc-ping deleted file mode 100755 index 12e42c1718e..00000000000 --- a/bin/cylc-ping +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [discovery] ping [OPTIONS] ARGS - -If suite REG is running or TASK in suite REG is currently in the 'running' -state exit with success status, else exit with error status.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.task_id import TaskID -from cylc.network.suite_info import SuiteInfoClient -from cylc.cfgspec.globalcfg import GLOBAL_CFG - - -def main(): - parser = cop( - __doc__, pyro=True, - argdoc=[('REG', 'Suite name'), ('[TASK]', 'Task ' + TaskID.SYNTAX)]) - - parser.add_option( - "--print-ports", - help="Print the port range from the cylc site config file.", - action="store_true", default=False, dest="print_ports") - - (options, args) = parser.parse_args() - - if options.print_ports: - base = GLOBAL_CFG.get(['pyro', 'base port']) - range = GLOBAL_CFG.get(['pyro', 'maximum number of ports']) - print base, '<= port <=', base + range - sys.exit(0) - - suite = args[0] - - pclient = SuiteInfoClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - # cylc ping SUITE - pclient.get_info('ping_suite') # (no need to check the result) - if len(args) == 1: - sys.exit(0) - - # cylc ping SUITE TASKID - task_id = args[1] - if not TaskID.is_valid_id(task_id): - sys.exit("Invalid task ID: " + task_id) - success, msg = pclient.get_info('ping_task', task_id) - if not success: - sys.exit('ERROR: ' + msg) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-poll b/bin/cylc-poll deleted file mode 100755 index 07a74753ac9..00000000000 --- a/bin/cylc-poll +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] poll [OPTIONS] ARGS - -Poll jobs of active tasks (those in the 'submitted' or 'running' states) to -verify or update their statuses - even if they have suffered an external hard -kill. - -To poll one or more tasks, "cylc poll REG MATCH POINT"; to poll all active -tasks: "cylc poll REG". - -Note that automatic job polling can used to track task status on task hosts -that do not allow any communication by RPC (pyro) or ssh back to the suite host -- see site/user config file documentation. - -Polling is also done automatically on restarting a suite, for any tasks that -were recorded as submitted or running when the suite went down. -""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage - - -def main(): - parser = cop( - __doc__ + multitask_usage, - pyro=True, multitask=True, - argdoc=[('REG', 'Suite name'), - ('[MATCH]', 'Task or family name matching regular expression'), - ('[POINT]', 'Task cycle point (e.g. date-time or integer)')]) - - (options, args) = parser.parse_args() - suite = args[0] - - if len(args) == 3: - name = args[1] - point_string = args[2] - elif len(args) == 1: - name = None - point_string = None - else: - parser.error("Wrong number of arguments.") - - if name and point_string: - prompt('Poll task %s at %s in %s' % (name, point_string, suite), - options.force) - else: - prompt('Poll ALL task in %s' % (suite), options.force) - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - pclient.put_command('poll_tasks', name, point_string, options.is_family) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-print b/bin/cylc-print deleted file mode 100755 index 239130fe335..00000000000 --- a/bin/cylc-print +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [db] print [OPTIONS] [REGEX] - -Print suite database registrations. - -Note on result filtering: - (a) The filter patterns are Regular Expressions, not shell globs, so -the general wildcard is '.*' (match zero or more of anything), NOT '*'. - (b) For printing purposes there is an implicit wildcard at the end of -each pattern ('foo' is the same as 'foo.*'); use the string end marker -to prevent this ('foo$' matches only literal 'foo').""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import os -import re - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.registration import localdb -from cylc.print_tree import print_tree -from cylc.regpath import RegPath - - -def get_padding(reglist): - maxlen = 0 - for reg in reglist: - items = RegPath(reg[0]).get_list() - for i in range(0, len(items)): - if i == 0: - tmp = len(items[i]) - else: - tmp = 2*i + 1 + len(items[i]) - if tmp > maxlen: - maxlen = tmp - return maxlen * ' ' - - -def main(): - parser = cop(__doc__, - argdoc=[('[REGEX]', 'Suite name regular expression pattern')]) - - parser.add_option( - "-t", "--tree", help="Print registrations in nested tree form.", - action="store_true", default=False, dest="tree") - - parser.add_option( - "-b", "--box", - help="Use unicode box drawing characters in tree views.", - action="store_true", default=False, dest="unicode") - - parser.add_option( - "-a", "--align", help="Align columns.", - action="store_true", default=False, dest="align") - - parser.add_option( - "-x", help="don't print suite definition directory paths.", - action="store_true", default=False, dest="x") - - parser.add_option( - "-y", help="Don't print suite titles.", - action="store_true", default=False, dest="y") - - parser.add_option( - "--fail", help="Fail (exit 1) if no matching suites are found.", - action="store_true", default=False, dest="fail") - - (options, args) = parser.parse_args() - - if len(args) == 0: - regfilter = None - elif len(args) == 1: - regfilter = args[0] - else: - parser.error("Wrong number of arguments.") - - db = localdb(file=options.db) - allsuites = db.get_list(regfilter) - if options.fail and len(allsuites) == 0: - raise SystemExit('ERROR: no suites matched.') - if not options.tree: - if options.align: - maxlen_suite = 0 - maxlen_title = 0 - for suite, dir, title in allsuites: - if len(suite) > maxlen_suite: - maxlen_suite = len(suite) - if len(title) > maxlen_title: - maxlen_title = len(title) - spacer_suite = maxlen_suite * ' ' - spacer_title = maxlen_title * ' ' - for suite, dir, title in allsuites: - dir = re.sub('^' + os.environ['HOME'], '~', dir) - if options.align: - suite = suite + spacer_suite[len(suite):] - title = title + spacer_title[len(title):] - if not options.x and not options.y: - line = suite + ' | ' + title + ' | ' + dir - elif not options.y: - line = suite + ' | ' + title - elif not options.x: - line = suite + ' | ' + dir - else: - line = suite - print line - else: - tree = {} - if options.align: - maxlen_title = 0 - for suite, dir, title in allsuites: - if len(title) > maxlen_title: - maxlen_title = len(title) - spacer_title = maxlen_title * ' ' - - for suite, dir, title in allsuites: - dir = re.sub('^' + os.environ['HOME'], '~', dir) - if options.align: - title = title + spacer_title[len(title):] - regpath = RegPath(suite).get_list() - sub = tree - for key in regpath[:-1]: - if key not in sub: - sub[key] = {} - sub = sub[key] - if not options.x and not options.y: - line = title + ' | ' + dir - elif not options.y: - line = ' ' + title - elif not options.x: - line = ' ' + dir - else: - line = '' - sub[regpath[-1]] = line - - pad = get_padding(allsuites) - print_tree(tree, pad, options.unicode) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-random b/bin/cylc-random deleted file mode 100755 index 9fbc3d11ae2..00000000000 --- a/bin/cylc-random +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [util] random A B - -Generate a random integer in the range [A,B). This is just a command -interface to Python's random.randrange() function. - -Arguments: - A start of the range interval (inclusive) - B end of the random range (exclusive, so must be > A)""" - -import os -import sys -import random -from optparse import OptionParser - -import cylc.flags - - -def main(): - parser = OptionParser(__doc__) - (options, args) = parser.parse_args() - - if len(args) != 2: - parser.error("Two integer arguments required") - - start = int(args[0]) - end = int(args[1]) - print random.randrange(start, end) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-refresh b/bin/cylc-refresh deleted file mode 100755 index 482f0da75e7..00000000000 --- a/bin/cylc-refresh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [db] refresh [OPTIONS] ARGS - -Check a suite database for invalid registrations (no suite definition -directory or suite.rc file) and refresh suite titles in case they have -changed since the suite was registered. Explicit wildcards must be -used in the match pattern (e.g. 'f' will not match 'foo.bar' unless -you use 'f.*').""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.CylcOptionParsers import cop -from cylc.registration import localdb, RegistrationError -from cylc.config import SuiteConfigError -from cylc.regpath import RegPath - - -def main(): - parser = cop(__doc__, - argdoc=[('[REGEX]', 'Suite name match pattern')]) - - parser.add_option( - "-u", "--unregister", - help="Automatically unregister invalid registrations.", - action="store_true", default=False, dest="unregister") - - (options, args) = parser.parse_args() - - db = localdb(file=options.db) - - if len(args) == 0: - pattern = '.*' - else: - pattern = args[0] - # force explicit wildcards - if not pattern.startswith('^'): - pattern = '^' + pattern - if not pattern.endswith('$'): - pattern += '$' - - invalid = [] # no suite.rc file - readerror = [] # can't read title (suite.rc parse error) - - # check validity - invalid = db.get_invalid() - # refresh titles - changed = [] - items = db.get_list(pattern) - if len(items) == 0: - if pattern: - print 'No suites found to match', pattern - else: - print 'No suites found' - for suite, dir, title in items: - if suite in invalid: - continue - try: - db.refresh_suite_title(suite) - except (RegistrationError, SuiteConfigError), x: - print >> sys.stderr, x - readerror.append(suite) - if len(invalid) > 0: - print ("ERROR, %d invalid registrations " - "(no suite.rc file):" % len(invalid)) - for i in invalid: - if options.unregister: - db.unregister(i) - else: - print ' -', i - if len(readerror) > 0: - print ("ERROR, %d title parse failures " - "(bad suite.rc file):" % len(readerror)) - for i in readerror: - print ' -', i - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-register b/bin/cylc-register deleted file mode 100755 index 349c0af7da5..00000000000 --- a/bin/cylc-register +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [db] register [OPTIONS] ARGS - -Register the suite definition located in PATH as REG. - -Suite names are hierarchical, delimited by '.' (foo.bar.baz); they -may contain letters, digits, underscore, and hyphens. Colons are not -allowed because directory paths incorporating the suite name are -sometimes needed in PATH variables. - -EXAMPLES: - -For suite definition directories /home/bob/(one,two,three,four): - -% cylc db reg bob /home/bob/one -% cylc db reg foo.bag /home/bob/two -% cylc db reg foo.bar.baz /home/bob/three -% cylc db reg foo.bar.waz /home/bob/four - -% cylc db pr '^foo' # print in flat form - bob | 'Test Suite One' | /home/bob/one - foo.bag | 'Test Suite Two' | /home/bob/two - foo.bar.baz | 'Test Suite Four' | /home/bob/three - foo.bar.waz | 'Test Suite Three' | /home/bob/four - -% cylc db pr -t '^foo' # print in tree form - bob 'Test Suite One' | /home/bob/one - foo - |-bag 'Test Suite Two' | /home/bob/two - `-bar - |-baz 'Test Suite Three' | /home/bob/three - `-waz 'Test Suite Four' | /home/bob/four""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import os - -from cylc.CylcOptionParsers import cop -from cylc.registration import localdb -import cylc.flags - - -def main(): - parser = cop( - __doc__, - argdoc=[("REG", "Suite name"), - ("PATH", "Suite definition directory")]) - - (options, args) = parser.parse_args() - suite = args[0] - - if args[1].endswith('suite.rc'): - suiterc = args[1] - rdir = os.path.dirname(suiterc) - else: - rdir = args[1] - suiterc = os.path.join(rdir, 'suite.rc') - - db = localdb(file=options.db) - db.register(suite, rdir) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-release b/bin/cylc-release deleted file mode 100755 index b0ed3443e17..00000000000 --- a/bin/cylc-release +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] release|unhold [OPTIONS] ARGS - -Release one or more held tasks (cylc release REG MATCH POINT) -or the whole suite (cylc release REG). Held tasks do not -submit even if they are ready to run.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage - - -def main(): - parser = cop( - __doc__ + multitask_usage + "\nSee also 'cylc [control] hold'.", - pyro=True, multitask=True, - argdoc=[ - ("REG", 'Suite name'), - ('[MATCH]', 'Task or family name matching regular expression'), - ('[POINT]', 'Task cycle point (e.g. date-time or integer)')]) - - (options, args) = parser.parse_args() - suite = args[0] - - if len(args) == 3: - whole_suite = False - name = args[1] - point_string = args[2] - prompt('Release task(s) %s at %s in %s' % (name, point_string, suite), - options.force) - - elif len(args) == 1: - whole_suite = True - prompt('Release suite %s' % suite, options.force) - else: - parser.error("Wrong number of arguments.") - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - if whole_suite: - pclient.put_command('release_suite') - else: - pclient.put_command('release_task', name, point_string, - options.is_family) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-reload b/bin/cylc-reload deleted file mode 100755 index 6e7709e2378..00000000000 --- a/bin/cylc-reload +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] reload [OPTIONS] ARGS - -Tell a suite to reload its definition at run time. All settings -including task definitions, with the exception of suite log -configuration, can be changed on reload. Note that defined tasks can be -be added to or removed from a running suite with the 'cylc insert' and -'cylc remove' commands, without reloading. This command also allows -addition and removal of actual task definitions, and therefore insertion -of tasks that were not defined at all when the suite started (you will -still need to manually insert a particular instance of a newly defined -task). Live task proxies that are orphaned by a reload (i.e. their task -definitions have been removed) will be removed from the task pool if -they have not started running yet. Changes to task definitions take -effect immediately, unless a task is already running at reload time. - -If the suite was started with Jinja2 template variables set on the -command line (cylc run --set FOO=bar REG) the same template settings -apply to the reload (only changes to the suite.rc file itself are -reloaded). - -If the modified suite definition does not parse, failure to reload will -be reported but no harm will be done to the running suite.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(): - sys.exit(0) - -import cylc.flags -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop -from cylc.prompt import prompt - - -def main(): - parser = cop(__doc__, pyro=True) - (options, args) = parser.parse_args() - suite = args[0] - - prompt('Reload %s' % suite, options.force) - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - pclient.put_command('reload_suite') - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-remove b/bin/cylc-remove deleted file mode 100755 index f7d85302104..00000000000 --- a/bin/cylc-remove +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] remove [OPTIONS] ARGS - -Remove one or more tasks (cylc remove REG MATCH POINT), or all tasks with a -given cycle point (cylc remove REG POINT) from a running suite. - -Tasks will spawn successors first if they have not done so already. -""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage - - -def main(): - parser = cop( - __doc__ + multitask_usage, pyro=True, multitask=True, - argdoc=[ - ("REG", "Suite name"), - ('[MATCH]', 'Task or family name matching regular expression'), - ('[POINT]', 'Task cycle point (e.g. date-time or integer)')]) - - parser.add_option( - "--no-spawn", - help="Do not spawn successors before removal.", - action="store_true", default=False, dest="no_spawn") - - (options, args) = parser.parse_args() - suite = args[0] - - if len(args) == 3: - name = args[1] - point_string = args[2] - remove_point = False - elif len(args) == 2: - point_string = args[1] - remove_point = True - else: - parser.error("Wrong number of arguments.") - - spawn = not options.no_spawn - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - if remove_point: - prompt('remove ALL tasks at %s in %s' % (point_string, suite), - options.force) - pclient.put_command('remove_cycle', point_string, spawn) - else: - prompt('remove task(s) %s at %s in %s' % (name, point_string, suite), - options.force) - pclient.put_command('remove_task', name, point_string, - options.is_family, spawn) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-reregister b/bin/cylc-reregister deleted file mode 100755 index f5bb87bfb87..00000000000 --- a/bin/cylc-reregister +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [db] reregister|rename [OPTIONS] ARGS - -Change the name of a suite (or group of suites) from REG1 to REG2. -Example: - cylc db rereg foo.bar.baz test.baz""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -from cylc.CylcOptionParsers import cop -from cylc.registration import localdb -import cylc.flags - - -def main(): - parser = cop(__doc__, argdoc=[("REG1", "original name"), - ("REG2", "new name")]) - (options, args) = parser.parse_args() - arg_from = args[0] - arg_to = args[1] - - db = localdb(file=options.db) - db.reregister(arg_from, arg_to) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-reset b/bin/cylc-reset deleted file mode 100755 index 1bd49ec7908..00000000000 --- a/bin/cylc-reset +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] reset [OPTIONS] ARGS - -Force one or more task proxies in a running suite to change state and modify -their prerequisites and outputs accordingly. For example, the 'waiting' state -means "prerequisites not satisfied, outputs not completed"; 'ready' means -"prerequisites satisfied, outputs not completed". Setting a task to 'ready' -generally has the same effect as using the "cylc trigger" command. - -See the documentation for the -s/--state option for legal reset states.""" - -import sys -if '--use-ssh' in sys.argv[1:]: - sys.argv.remove('--use-ssh') - from cylc.remote import remrun - if remrun().execute(force_required=True): - sys.exit(0) - -import cylc.flags -from cylc.prompt import prompt -from cylc.network.suite_command import SuiteCommandClient -from cylc.CylcOptionParsers import cop, multitask_usage -from cylc.task_state import task_state - - -def main(): - parser = cop( - __doc__ + multitask_usage, pyro=True, multitask=True, - argdoc=[ - ('REG', 'Suite name'), - ('MATCH', 'Task or family name matching regular expression'), - ('POINT', 'Task cycle point (e.g. date-time or integer)')]) - - parser.add_option( - "-s", "--state", metavar="STATE", - help="Reset task state to STATE to on of %s" % ( - ', '.join(task_state.legal_for_reset)), - action="store", default=None, dest="state") - - (options, args) = parser.parse_args() - suite = args[0] - - if options.state not in task_state.legal_for_reset: - parser.error("Illegal STATE value: " + options.state) - - name = args[1] - point_string = args[2] - - prompt('Reset task(s) ' + name + ' at ' + point_string + ' in ' + suite, - options.force) - - pclient = SuiteCommandClient( - suite, options.owner, options.host, options.pyro_timeout, - options.port, options.db, my_uuid=options.set_uuid, - print_uuid=options.print_uuid) - - pclient.put_command('reset_task_state', name, point_string, options.state, - options.is_family) - - -if __name__ == "__main__": - try: - main() - except Exception as exc: - if cylc.flags.debug: - raise - sys.exit(exc) diff --git a/bin/cylc-restart b/bin/cylc-restart deleted file mode 100755 index 826be8643f5..00000000000 --- a/bin/cylc-restart +++ /dev/null @@ -1,488 +0,0 @@ -#!/usr/bin/env python - -# THIS FILE IS PART OF THE CYLC SUITE ENGINE. -# Copyright (C) 2008-2015 NIWA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc [control] restart [OPTIONS] ARGS - -Start a suite run from a previous state. To start from scratch (cold or warm -start) see the 'cylc run' command. - -The scheduler runs in daemon mode unless you specify n/--no-detach or --debug. - -The most recent previous suite state is loaded by default, but earlier state -files in the suite state directory can be specified on the command line. - -Tasks recorded as 'submitted' or 'running' will be polled at start-up to -determine what happened to them while the suite was down.""" - -import sys -from cylc.remote import remrun -if remrun().execute(): - sys.exit(0) - -import os -import re -from datetime import datetime - -import cylc.flags -from cylc.config import SuiteConfig, TaskNotDefinedError -from cylc.CylcOptionParsers import cop -from cylc.scheduler import scheduler -from cylc.suite_state_dumping import SuiteStateDumper -from cylc.task_state import task_state -from cylc.run import main -from cylc.get_task_proxy import get_task_proxy -from cylc.registration import localdb -from cylc.task_id import TaskID -from cylc.cycling.loader import get_point, DefaultCycler, ISO8601_CYCLING_TYPE -from cylc.wallclock import get_current_time_string -from cylc.network.suite_broadcast import BroadcastServer - - -class restart(scheduler): - def __init__(self): - - self.parser = cop( - __doc__, jset=True, - argdoc=[ - ("REG", "Suite name"), - ("[FILE]", """Optional state dump, assumed to reside in the - suite state dump directory unless an absolute path - is given. Defaults to the most recent suite state.""") - ] - ) - - self.parser.add_option( - "--non-daemon", help="(deprecated: use --no-detach)", - action="store_true", default=False, dest="no_detach") - - self.parser.add_option( - "-n", "--no-detach", help="Do not daemonize the suite", - action="store_true", default=False, dest="no_detach") - - self.parser.add_option( - "--profile", help="Output profiling (performance) information", - action="store_true", default=False, dest="profile_mode") - - self.parser.add_option( - "--ignore-final-cycle-point", - help="Ignore the final cycle point in the state dump. If one is" - "specified in the suite definition it will be used, however.", - action="store_true", default=False, dest="ignore_stop_point") - - self.parser.add_option( - "--ignore-initial-cycle-point", - help="Ignore the initial cycle point in the state dump. If one is " - "specified in the suite definition it will be used, however.", - action="store_true", default=False, dest="ignore_start_point") - - scheduler.__init__(self, is_restart=True) - - def parse_commandline(self): - (self.options, self.args) = self.parser.parse_args() - self.suite = self.args[0] - self.suiterc = localdb(self.options.db).get_suiterc(self.suite) - self.suite_dir = os.path.dirname(self.suiterc) - - # For user-defined job submission methods: - sys.path.append(os.path.join(self.suite_dir, 'python')) - - self.restart_from = None - if len(self.args) == 2: - self.restart_from = self.args[1] - - scheduler.parse_commandline(self) - - def get_state_initial_point_string(self): - """Return the initial point string from the state file, if any.""" - return self.get_state_file_info()[0] - - def get_state_file_path(self): - """Return the state file path that we are restarting from.""" - my_dumper = self.state_dumper - if my_dumper is None: - my_dumper = SuiteStateDumper(self.suite) - base_name = my_dumper.BASE_NAME - file_name = base_name - dir_name = my_dumper.dir_name - if self.restart_from and os.path.isabs(self.restart_from): - file_name = self.restart_from - elif self.restart_from: - file_name = os.path.join(dir_name, self.restart_from) - else: - file_name = os.path.join(dir_name, file_name) - if not os.path.isfile(file_name): - raise Exception("state dump file not found: " + file_name) - return os.path.realpath(file_name) - - def get_state_file_info(self): - """Return the state file start & stop strings, broadcast, tasks. - - The state dump file format is: - run mode : - time :