From 7ea6151300550463b7bae4ae5e96f4056d6db94d Mon Sep 17 00:00:00 2001 From: Min RK Date: Tue, 19 Feb 2019 16:00:30 +0100 Subject: [PATCH 1/9] use git describe for better prerelease chart versions - preserves prerelease category (alpha, beta, etc.) - use '0' if no prerelease category is set - chart version has the form `x.y.z-{alpha|beta|rc|0}.n.sha` where `n` is commits since tag, sha is current commit --- chartpress.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/chartpress.py b/chartpress.py index 60e43e8..2c039d5 100755 --- a/chartpress.py +++ b/chartpress.py @@ -310,11 +310,19 @@ def build_chart(name, version=None, paths=None, reset=False): if paths is None: paths = ['.'] commit = last_modified_commit(*paths) - - if reset: - version = chart['version'].split('-')[0] - else: - version = chart['version'].split('-')[0] + '-' + commit + # parse prerelease version, preserving only the first part + # which will be the prerelease category (alpha, beta, etc.) + # if nothing is found, use `0`, to sort before alpha + base_version, *rest = chart['version'].split('-', 1) + prerelease = 0 + if rest: + prerelease = rest[0].split('.')[0] + if not prerelease: + prerelease = '0' + + describe = check_output(['git', 'describe', '--tags', '--long', commit]).decode('utf8').strip() + _tag, n_commits, sha = describe.rsplit('-', 2) + version = "{base_version}-{prerelease}.{n_commits}.{sha}".format(**locals()) chart['version'] = version From b09ff8bd1a9f6a79828fa0b467efdccecd39c1fe Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 18:15:23 +0200 Subject: [PATCH 2/9] Chart versioning with suffix: +. --- chartpress.py | 70 +++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/chartpress.py b/chartpress.py index 2c039d5..d95d3ff 100755 --- a/chartpress.py +++ b/chartpress.py @@ -195,12 +195,12 @@ def image_needs_building(image): return image_needs_pushing(image) -def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_version=None, skip_build=False): +def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_tag=None, skip_build=False): """Build a collection of docker images Args: - prefix (str): the prefix to add to images - images (dict): dict of image-specs from chartpress.yml + prefix (str): the prefix to add to image names + images (dict): dict of image-specs from chartpress.yaml tag (str): Specific tag to use instead of the last modified commit. If unspecified the tag for each image will be the hash of the last commit @@ -211,8 +211,8 @@ def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_ it will not be rebuilt. push (bool): Whether to push the resulting images (default: False). - chart_version (str): - The chart version, included as a prefix on image tags + chart_tag (str): + The latest chart tag, included as a prefix on image tags if `tag` is not specified. skip_build (bool): Whether to skip the actual image build (only updates tags). @@ -224,12 +224,12 @@ def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_ # include chartpress.yaml itself as it can contain build args and # similar that influence the image that would be built paths = list(options.get('paths', [])) + [image_path, 'chartpress.yaml'] - last_commit = last_modified_commit(*paths) + last_image_commit = last_modified_commit(*paths) if tag is None: - if chart_version: - image_tag = "{}-{}".format(chart_version, last_commit) + if chart_tag: + image_tag = f"{chart_tag}-{last_image_commit}" else: - image_tag = last_commit + image_tag = last_image_commit image_name = prefix + name image_spec = '{}:{}'.format(image_name, image_tag) @@ -241,12 +241,11 @@ def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_ if skip_build: continue - if tag or image_needs_building(image_spec): build_args = render_build_args( options, { - 'LAST_COMMIT': last_commit, + 'LAST_COMMIT': last_image_commit, 'TAG': image_tag, }, ) @@ -300,29 +299,19 @@ def build_values(name, values_mods): yaml.dump(values, f) -def build_chart(name, version=None, paths=None, reset=False): - """Update chart with specified version or last-modified commit in path(s)""" +def build_chart(name, version=None, paths=None): + """Update Chart.yaml with specified version or last-modified commit in path(s)""" chart_file = os.path.join(name, 'Chart.yaml') with open(chart_file) as f: chart = yaml.load(f) + last_chart_commit = last_modified_commit(*paths) + if version is None: - if paths is None: - paths = ['.'] - commit = last_modified_commit(*paths) - # parse prerelease version, preserving only the first part - # which will be the prerelease category (alpha, beta, etc.) - # if nothing is found, use `0`, to sort before alpha - base_version, *rest = chart['version'].split('-', 1) - prerelease = 0 - if rest: - prerelease = rest[0].split('.')[0] - if not prerelease: - prerelease = '0' - - describe = check_output(['git', 'describe', '--tags', '--long', commit]).decode('utf8').strip() - _tag, n_commits, sha = describe.rsplit('-', 2) - version = "{base_version}-{prerelease}.{n_commits}.{sha}".format(**locals()) + git_describe = check_output(['git', 'describe', '--tags', '--long', last_commit]).decode('utf8').strip() + latest_tag_in_branch, n_commits, sha = git_describe.rsplit('-', maxsplit=2) + + version = f"{latest_tag_in_branch}+{n_commits}.{sha}" chart['version'] = version @@ -398,7 +387,7 @@ def main(): argparser.add_argument('--image-prefix', default=None, help='Override image prefix with this value') argparser.add_argument('--reset', action='store_true', - help='Reset image tags') + help="Skip image build and reset Chart.yaml's version field and values.yaml's image tags") argparser.add_argument('--skip-build', action='store_true', help='Skip image build, only render the charts') argparser.add_argument('--version', action='store_true', @@ -416,12 +405,15 @@ def main(): for chart in config['charts']: chart_paths = ['.'] + list(chart.get('paths', [])) - version = args.tag - if version: - # version of the chart shouldn't have leading 'v' prefix - # if tag is of the form 'v1.2.3' - version = version.lstrip('v') - chart_version = build_chart(chart['name'], paths=chart_paths, version=version, reset=args.reset) + chart_version = args.tag + if chart_version: + # The chart's version shouldn't have leading 'v' prefix if tag is of + # the form 'v1.2.3', as that would break Chart.yaml's SemVer 2 + # requirement on the version field. + chart_version = chart_version.lstrip('v') + if args.reset: + chart_version = chart.get('resetTag', 'set-by-chartpress') + chart_version = build_chart(chart['name'], paths=chart_paths, version=chart_version, reset=args.reset) if 'images' in chart: image_prefix = args.image_prefix if args.image_prefix is not None else chart['imagePrefix'] @@ -431,8 +423,10 @@ def main(): tag=args.tag if not args.reset else chart.get('resetTag', 'set-by-chartpress'), commit_range=args.commit_range, push=args.push, - # exclude `-` from chart_version prefix for images - chart_version=chart_version.split('-', 1)[0], + # chart_tag will act as a image tag prefix, we can get it from + # the chart_version by stripping away the build part of the + # SemVer 2 compliant chart_version. + chart_tag=chart_version.split('+')[0], skip_build=args.skip_build or args.reset, ) build_values(chart['name'], value_mods) From bf9dcb360eccefff4c578d95f7ba89958be1e141 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 20:35:52 +0200 Subject: [PATCH 3/9] 03d formatting and allow for no available git tags --- chartpress.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/chartpress.py b/chartpress.py index d95d3ff..b8cb642 100755 --- a/chartpress.py +++ b/chartpress.py @@ -308,10 +308,18 @@ def build_chart(name, version=None, paths=None): last_chart_commit = last_modified_commit(*paths) if version is None: - git_describe = check_output(['git', 'describe', '--tags', '--long', last_commit]).decode('utf8').strip() - latest_tag_in_branch, n_commits, sha = git_describe.rsplit('-', maxsplit=2) - - version = f"{latest_tag_in_branch}+{n_commits}.{sha}" + try: + git_describe = check_output(['git', 'describe', '--tags', '--long', last_commit]).decode('utf8').strip() + latest_tag_in_branch, n_commits, sha = git_describe.rsplit('-', maxsplit=2) + version = f"{latest_tag_in_branch}+{int(n_commits):03d}.{sha}" + except CalledProcessError: + # no tags on branch: fallback to the SemVer 2 compliant version + # 0.0.1+. + n_commits = check_output( + ['git', 'rev-list', '--count', 'HEAD'], + echo=False, + ).decode('utf-8') + version = f"0.0.1+{int(n_commits):03d}.{last_chart_commit}" chart['version'] = version From 08001bce83da88496402565f68a9b5918d434be2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 21:12:43 +0200 Subject: [PATCH 4/9] Image tags revised Image tags are now set like the chart versions are set, but where + has been replaced with _ as is required to be a valid docker tag. ``` _- ``` Note that `git rev-list --count ..` would return 0 while the reverse ordering would return 5. --- chartpress.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/chartpress.py b/chartpress.py index b8cb642..922d0ce 100755 --- a/chartpress.py +++ b/chartpress.py @@ -226,10 +226,14 @@ def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_ paths = list(options.get('paths', [])) + [image_path, 'chartpress.yaml'] last_image_commit = last_modified_commit(*paths) if tag is None: - if chart_tag: - image_tag = f"{chart_tag}-{last_image_commit}" - else: - image_tag = last_image_commit + n_commits = check_output( + [ + 'git', 'rev-list', '--count', + f'{chart_tag + ".." if chart_tag != "0.0.1" else ""}{last_chart_commit}', + ], + echo=False, + ).decode('utf-8') + image_tag = f"{chart_tag}_{n_commits}-{last_image_commit}" image_name = prefix + name image_spec = '{}:{}'.format(image_name, image_tag) @@ -316,7 +320,7 @@ def build_chart(name, version=None, paths=None): # no tags on branch: fallback to the SemVer 2 compliant version # 0.0.1+. n_commits = check_output( - ['git', 'rev-list', '--count', 'HEAD'], + ['git', 'rev-list', '--count', last_chart_commit], echo=False, ).decode('utf-8') version = f"0.0.1+{int(n_commits):03d}.{last_chart_commit}" From 0c3db3434bd0100a7e4dee5fb18843f9dd0b3767 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 21:19:18 +0200 Subject: [PATCH 5/9] Stop passing unused reset argument --- chartpress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chartpress.py b/chartpress.py index 922d0ce..47f454a 100755 --- a/chartpress.py +++ b/chartpress.py @@ -425,7 +425,7 @@ def main(): chart_version = chart_version.lstrip('v') if args.reset: chart_version = chart.get('resetTag', 'set-by-chartpress') - chart_version = build_chart(chart['name'], paths=chart_paths, version=chart_version, reset=args.reset) + chart_version = build_chart(chart['name'], paths=chart_paths, version=chart_version) if 'images' in chart: image_prefix = args.image_prefix if args.image_prefix is not None else chart['imagePrefix'] From 80fe1f5d10ad60a0d6201d237c6d273dd176c43a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 21:27:27 +0200 Subject: [PATCH 6/9] Apply pyflake's excellent suggestions --- chartpress.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/chartpress.py b/chartpress.py index 47f454a..b4ff27d 100755 --- a/chartpress.py +++ b/chartpress.py @@ -229,7 +229,12 @@ def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_ n_commits = check_output( [ 'git', 'rev-list', '--count', - f'{chart_tag + ".." if chart_tag != "0.0.1" else ""}{last_chart_commit}', + # Note that the 0.0.1 chart_tag may not exist as it was a + # workaround to handle git histories with no tags in the + # current branch. Also, if the chart_tag is a later git + # reference than the last_image_commit, this command will + # return 0. + f'{chart_tag + ".." if chart_tag != "0.0.1" else ""}{last_image_commit}', ], echo=False, ).decode('utf-8') @@ -313,10 +318,10 @@ def build_chart(name, version=None, paths=None): if version is None: try: - git_describe = check_output(['git', 'describe', '--tags', '--long', last_commit]).decode('utf8').strip() + git_describe = check_output(['git', 'describe', '--tags', '--long', last_chart_commit]).decode('utf8').strip() latest_tag_in_branch, n_commits, sha = git_describe.rsplit('-', maxsplit=2) version = f"{latest_tag_in_branch}+{int(n_commits):03d}.{sha}" - except CalledProcessError: + except subprocess.CalledProcessError: # no tags on branch: fallback to the SemVer 2 compliant version # 0.0.1+. n_commits = check_output( From 97ebe4bd4b6be3101088d236feee0bcd7ba9fc6a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 21:35:17 +0200 Subject: [PATCH 7/9] Fix failure to fail in .travis.yml Here is a build that should have failed, that this commit ensures will fail in the future: https://travis-ci.org/jupyterhub/chartpress/builds/598821781 --- .travis.yml | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6bcbc43..169dc9f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,21 +9,24 @@ script: - chartpress --version - chartpress --help - pyflakes . - - | - # This is a workaround to an issue caused by the existence of a docker registry - # mirror in our CI environment. Without this fix that removes the mirror, - # chartpress fails to realize the existence of already built images and rebuilds - # them. - # - # ref: https://github.com/moby/moby/issues/39120 - echo '{"mtu": 1460}' | sudo dd of=/etc/docker/daemon.json + + # This is a workaround to an issue caused by the existence of a docker registry + # mirror in our CI environment. Without this fix that removes the mirror, + # chartpress fails to realize the existence of already built images and rebuilds + # them. + # + # ref: https://github.com/moby/moby/issues/39120 + - |- + echo '{"mtu": 1460}' | sudo dd of=/etc/docker/daemon.json && sudo systemctl restart docker - - | - # run chartpress on zero-to-jupyterhub-k8s - git clone https://github.com/jupyterhub/zero-to-jupyterhub-k8s - cd zero-to-jupyterhub-k8s - chartpress + + # run chartpress on zero-to-jupyterhub-k8s + - |- + git clone https://github.com/jupyterhub/zero-to-jupyterhub-k8s && + cd zero-to-jupyterhub-k8s && + chartpress && git --no-pager diff - - | - chartpress --reset + + - |- + chartpress --reset && git --no-pager diff From 582d4295de431362c05d2219216e0bd98911da27 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 22:00:26 +0200 Subject: [PATCH 8/9] Fix lack of .strip() related bug --- .travis.yml | 9 ++++----- chartpress.py | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 169dc9f..04b6a0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,9 +24,8 @@ script: - |- git clone https://github.com/jupyterhub/zero-to-jupyterhub-k8s && cd zero-to-jupyterhub-k8s && - chartpress && - git --no-pager diff + chartpress + - git --no-pager diff - - |- - chartpress --reset && - git --no-pager diff + - chartpress --reset + - git --no-pager diff diff --git a/chartpress.py b/chartpress.py index b4ff27d..a725318 100755 --- a/chartpress.py +++ b/chartpress.py @@ -64,7 +64,7 @@ def last_modified_commit(*paths, **kwargs): '--pretty=format:%h', '--', *paths - ], **kwargs).decode('utf-8') + ], **kwargs).decode('utf-8').strip() def last_modified_date(*paths, **kwargs): @@ -77,7 +77,7 @@ def last_modified_date(*paths, **kwargs): '--date=iso', '--', *paths - ], **kwargs).decode('utf-8') + ], **kwargs).decode('utf-8').strip() def path_touched(*paths, commit_range): @@ -237,7 +237,7 @@ def build_images(prefix, images, tag=None, commit_range=None, push=False, chart_ f'{chart_tag + ".." if chart_tag != "0.0.1" else ""}{last_image_commit}', ], echo=False, - ).decode('utf-8') + ).decode('utf-8').strip() image_tag = f"{chart_tag}_{n_commits}-{last_image_commit}" image_name = prefix + name image_spec = '{}:{}'.format(image_name, image_tag) @@ -327,7 +327,7 @@ def build_chart(name, version=None, paths=None): n_commits = check_output( ['git', 'rev-list', '--count', last_chart_commit], echo=False, - ).decode('utf-8') + ).decode('utf-8').strip() version = f"0.0.1+{int(n_commits):03d}.{last_chart_commit}" chart['version'] = version From 67d05601831ec6593d0af3aeca29a92189a8ef52 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 16 Oct 2019 23:18:21 +0200 Subject: [PATCH 9/9] Add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19dc0bf..51563f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +Chart and image versioning, and Chart.yaml's --reset interaction [#52](https://github.com/jupyterhub/chartpress/pull/52) ([@consideRatio](https://github.com/consideRatio)) + Add --version flag [#45](https://github.com/jupyterhub/chartpress/pull/45) ([@consideRatio](https://github.com/consideRatio)) ## 0.3