scripts/release
#!/usr/bin/env python
import os
import re
import sys
import json
import glob
import shutil
import tempfile
import subprocess
import contextlib
from os.path import join
from github import Github, UnknownObjectException
from pkg_resources import parse_version
from pkg_resources.extern.packaging.version import LegacyVersion
import click
import requests
HERE = os.path.abspath(os.path.dirname(__file__))
BASE_DIR = os.path.normpath(join(HERE, '..'))
VERSION_PAT = re.compile(
r'^(?P<full>(?P<prefix>v)?(?P<noprefix>(?P<breaking>\d+)\.(?P<feature>\d+)\.(?P<fix>\d+)'
r'(?P<rc>rc\d+)?(?P<dev>\.dev0)?))$'
)
# matches things like v0.2.x or v1.x
STABLE_VERSION_PAT = re.compile(
r'^v\d+\.(\d+|x)(\.(\d+|x))?$'
)
GH_REPO_NAME = "LiberTEM/LiberTEM"
PYTHON_PKG_NAME = 'libertem'
def get_travis_env():
return {
'log_url': os.environ.get("TRAVIS_BUILD_WEB_URL", ""),
'commit': os.environ["TRAVIS_COMMIT"],
'branch': os.environ["TRAVIS_BRANCH"],
'is_pr': os.environ['TRAVIS_PULL_REQUEST'] != "false",
}
def get_azure_env():
return {
'log_url': None,
'commit': os.environ.get('BUILD_SOURCEVERSION'),
'branch': os.environ.get('BUILD_SOURCEBRANCHNAME'),
'is_pr': os.environ.get('BUILD_REASON') == 'PullRequest',
}
def get_github_actions_env():
branch = get_current_branch()
return {
'log_url': None,
'commit': os.environ.get('GITHUB_SHA'),
'branch': branch,
'is_pr': os.environ.get('GITHUB_EVENT_NAME') == 'pull_request',
}
def get_all_branches():
"""
Returns a dict from branch name to revision
"""
cmd = ["git", "for-each-ref", "--format=%(objectname) %(refname:short)",
"refs/heads"]
lines = subprocess.check_output(cmd, text="utf-8").strip().split("\n")
lines = [line for line in lines if line]
if not lines:
return {}
return {
line.split()[1]: line.split()[0]
for line in lines
}
def get_current_rev():
cmd = ["git", "rev-parse", "HEAD"]
return subprocess.check_output(cmd, text="utf-8").strip()
def get_current_branch():
# tag builds are in detached HEAD state for github actions, so we need to
# put some effort into finding the matching branch:
all_branches = get_all_branches()
branch_by_rev = {
rev: branch
for (branch, rev) in all_branches.items()
}
return branch_by_rev.get(get_current_rev(), "HEAD")
env = get_github_actions_env()
def render_version(version_file, new_version):
version_stm = f'__version__ = "{new_version}"\n'
with open(version_file, 'wb') as f:
f.write(version_stm.encode("utf8"))
def read_version(version_file):
res = {}
with open(version_file) as f:
exec(f.read(), res)
return res['__version__']
def get_version_fn():
return join(BASE_DIR, 'src', PYTHON_PKG_NAME, '__version__.py')
def do_git_commit(old_version, new_version, version_file):
cmd = ["git", "commit", version_file, "-m",
f"bump version: {old_version} → {new_version}"]
subprocess.check_call(cmd)
def do_git_tag(tag):
cmd = ["git", "tag", tag]
subprocess.check_call(cmd)
def validate_version_tag():
"""
Validate the current version tag. rules:
1) There must be exactly one version tag, and it has to conform to
to the version pattern.
2) The tag needs to have a prefix of "v".
"""
tags = get_current_tags()
matches = tag_matches(tags)
if len(matches) != 1:
raise click.ClickException(
"can only have a single version tag for HEAD, aborting"
)
version_tag = matches[0]
if version_tag['prefix'] != 'v':
raise click.ClickException(
"version tags need to have a 'v' prefix"
)
v = parse_version(version_tag['full'])
if isinstance(v, LegacyVersion):
raise click.ClickException(
"The version tag %s could not be parsed as valid version" % v
)
version_from_file = parse_version(read_version(get_version_fn()))
if isinstance(version_from_file, LegacyVersion):
raise click.ClickException(
"__version__ %s could not be parsed as valid version" % version_from_file
)
if version_from_file != v:
raise click.ClickException(
"version tag {} and __version__ {} do not match".format(
v, version_from_file
)
)
return True
def get_current_tags():
"""
returns list of tags that point to HEAD
"""
cmd = ["git", "tag", "--points-at", "HEAD"]
return subprocess.check_output(cmd, text="utf-8").strip().split("\n")
def get_all_tags():
"""
returns list of all tags in the repository
"""
cmd = ["git", "tag"]
return subprocess.check_output(cmd, text="utf-8").strip().split("\n")
def get_release_tag():
"""
A commit can have more than one tag (but should only have one version tag).
This function returns the matching version tag as a match dictionary.
"""
tags = get_current_tags()
matches = tag_matches(tags)
if len(matches) == 0:
return None
elif len(matches) == 1:
return matches[0]
else:
raise Exception(
"cannot have more than one version tags per commit"
)
def get_latest_tag():
"""
return the latest tags, looking "back" from HEAD
"""
try:
cmd = ["git", "describe", "--abbrev=0", "--tags"]
return subprocess.check_output(cmd, text="utf-8").strip()
except subprocess.CalledProcessError:
return ""
def get_latest_stable_tag():
tags = get_all_tags()
sorted_stable_tags = list(
sorted(
filter(STABLE_VERSION_PAT.match, tags),
key=parse_version
)
)
if len(sorted_stable_tags) == 0:
return None
return sorted_stable_tags[-1]
def tag_matches(tags):
"""
"""
matches = [
VERSION_PAT.match(tag)
for tag in tags
]
return [m.groupdict()
for m in matches
if m is not None]
def current_version_tag_is_rc():
return get_release_kind() == "rc"
def current_version_tag_is_release():
return get_release_kind() == "release"
def get_release_kind():
match = get_release_tag()
if match is None:
return "dev"
if match['rc'] is not None:
return "rc"
elif match['rc'] is None and match['dev'] is None:
return "release"
else:
return "dev"
def get_wheel():
wheels = glob.glob("%s/dist/*.whl" % BASE_DIR)
assert len(wheels) == 1, "expected only one wheel, have: %s" % wheels
return wheels[0]
def get_sdist():
sdists = glob.glob("%s/dist/*.tar.gz" % BASE_DIR)
assert len(sdists) == 1, "expected only one tarball, have: %s" % sdists
return sdists[0]
def get_py_release_files():
"""
get wheel and sdist files (basically dist/*)
"""
return glob.glob("%s/dist/*" % BASE_DIR)
def stream_download(dest_fn, url):
with open(dest_fn, "wb") as f, requests.get(url, stream=True) as resp:
resp.raise_for_status()
for chunk in resp.iter_content(chunk_size=1024*1024):
f.write(chunk)
@contextlib.contextmanager
def get_archive(tag):
"""
Download repository archive from github releases page.
Parameters
----------
tag : re.Match
Release tag to download the archive for, as
matched by VERSION_PAT
"""
filename = "libertem-repo-archive-%s.tar.gz" % tag['full']
with tempfile.TemporaryDirectory() as tempdir:
full_fn = os.path.join(tempdir, filename)
url = f"https://github.com/{GH_REPO_NAME}/archive/{tag['full']}.tar.gz"
stream_download(dest_fn=full_fn, url=url)
yield full_fn
def upload_to_zenodo(verbose, parent, token, sandbox_token):
"""
Upload wheel, sdist and repo archive to zenodo. Only works if
there is a matching GitHub release already created.
"""
if verbose:
print("uploading to zenodo")
wheel = get_wheel()
sdist = get_sdist()
release = get_release_kind()
if release == "dev":
raise click.ClickException("don't want to upload to zenodo from dev release")
elif release == "rc":
url = "https://sandbox.zenodo.org/api/"
os.environ['ZENODO_OAUTH_TOKEN'] = sandbox_token
elif release == "release":
url = "https://zenodo.org/api/"
os.environ['ZENODO_OAUTH_TOKEN'] = token
tag = get_release_tag()
with get_archive(tag) as repo_archive:
out = subprocess.check_output([
"python", join(HERE, "zenodo_upload"),
"--url=%s" % url,
"--parent=%s" % parent,
"--mask-zenodo-exception",
wheel,
sdist,
repo_archive,
])
if verbose and out:
print(out.decode("utf-8"))
def get_release_msg():
parts = []
if env['log_url']:
parts.append("CI build log: %s" % env['log_url'])
return "\n\n".join(parts)
def upload_to_github(verbose, files, token, release):
if verbose:
print("uploading files to github:", files)
msg = get_release_msg()
if verbose:
print("release message:", msg)
g = Github(token)
repo = g.get_repo(GH_REPO_NAME)
release_data = {
"message": msg,
"target_commitish": env['commit'],
}
if release == "dev":
release_data.update({
"tag": "continuous",
"name": "Continuous build",
"draft": False,
"prerelease": True,
})
# first, delete existing continuous tag and release(s):
try:
continuous_releases = [
r
for r in repo.get_releases()
if r.tag_name == "continuous"
]
for cont_release in continuous_releases:
cont_release.delete_release()
if verbose:
print("deleted existing continuous release")
except UnknownObjectException:
pass
try:
cont_tag = repo.get_git_ref("tags/continuous")
cont_tag.delete()
if verbose:
print("deleted existing continuous tag")
except UnknownObjectException:
pass
if verbose:
print("continuous release")
elif release == "rc":
tag = get_release_tag()
release_data.update({
"tag": tag['full'],
"name": "Release Candidate %s" % tag['full'],
"draft": True,
"prerelease": True,
})
if verbose:
print("release candidate, tag:", tag['full'])
elif release == "release":
tag = get_release_tag()
release_data.update({
"tag": tag['full'],
"name": "Release %s" % tag['full'],
"draft": True,
"prerelease": False,
})
if verbose:
print("release, tag:", tag['full'])
else:
raise ValueError("unknown release kind: %s" % release)
if verbose:
print("release data: %r" % release_data)
rel = repo.create_git_release(**release_data)
for path in files:
rel.upload_asset(path=path, content_type='application/octet-stream')
def build_the_appimage(verbose):
APPIMAGE_DIR = join(BASE_DIR, 'packaging', 'appimage')
if verbose:
print("building AppImage in %s" % APPIMAGE_DIR)
# cleanup existing AppDir
APPDIR = join(APPIMAGE_DIR, 'AppDir')
if os.path.exists(APPDIR):
shutil.rmtree(APPDIR)
cmd = [
join(APPIMAGE_DIR, 'make_app_image.sh')
]
output = subprocess.check_output(cmd, text="utf-8", cwd=APPIMAGE_DIR).strip()
if verbose:
print(output)
# FIXME: where does the zsync file come from?
appimage_files = glob.glob(join(APPIMAGE_DIR, 'LiberTEM*.AppImage*'))
return appimage_files
def prepare_upload(verbose):
"""
Prepare for upload. This includes:
* building sdist and wheel
"""
if verbose:
print("preparing for upload")
if verbose:
print("building wheel and sdist")
out = subprocess.check_output([
"python", "-m", "build",
], cwd=BASE_DIR)
if verbose and out:
print(out.decode("utf-8"))
def run_twine_check(verbose):
cmd = ["twine", "check"]
cmd.extend(get_py_release_files())
out = subprocess.check_output(cmd)
if verbose:
print(out)
def upload_to_pypi(verbose, user, password):
files = get_py_release_files()
cmd = ["twine", "upload"]
release = get_release_kind()
cmd.extend(["-u", user])
cmd.extend(["-p", password])
# upload command for test.pypi.org:
# twine upload --repository-url https://test.pypi.org/legacy/ dist/*
if release == "dev":
raise Exception("won't upload dev release to pypi")
elif release == "rc":
cmd.extend(["--repository-url", "https://test.pypi.org/legacy/"])
elif release == "release":
pass
else:
raise Exception("unknown release kind %s" % release)
cmd.extend(files)
out = subprocess.check_output(cmd)
if verbose:
print(out)
@click.group()
@click.option('--verbose/--no-verbose', default=True)
@click.pass_context
def cli(ctx, verbose):
ctx.obj['verbose'] = verbose
@cli.command()
@click.pass_context
def is_rc(ctx):
res = current_version_tag_is_rc()
if ctx.obj['verbose']:
print("is rc?", res)
sys.exit(int(not res))
@cli.command()
@click.option('--dry-run/--no-dry-run', default=True,
help="don't actually upload, only prepare files etc.")
@click.option('--build-appimage/--no-build-appimage', default=False,
help="Build an AppImage")
@click.option('--pypi-user', type=str, show_envvar=True)
@click.option('--pypi-password', type=str, show_envvar=True)
@click.option('--pypi-test-user', type=str, show_envvar=True)
@click.option('--pypi-test-password', type=str, show_envvar=True)
@click.option('--token', type=str, show_envvar=True,
help='Github token for creating releases')
@click.option('--zenodo-sandbox-token', type=str, show_envvar=True,
help='Zenodo sandbox token for RC upload testing')
@click.option('--zenodo-token', type=str, show_envvar=True,
help='Zenodo production token for final release upload')
@click.option('--zenodo-parent', type=str, show_envvar=True,
help='Zenodo production parent deposition id')
@click.option('--zenodo-sandbox-parent', type=str, show_envvar=True,
help='Zenodo sandbox parent deposition id')
@click.pass_context
def upload(ctx, dry_run, build_appimage, pypi_user, pypi_password,
pypi_test_user, pypi_test_password, token, zenodo_sandbox_token,
zenodo_token, zenodo_parent, zenodo_sandbox_parent):
"""
prepare, build and upload to github, zenodo and pypi
(if release candidate, to the test instance(s))
"""
if dry_run:
print("NOTE: running in dry-run mode, specify --no-dry-run to really upload!")
print("environment: %s" % env)
is_rc = current_version_tag_is_rc()
is_release = current_version_tag_is_release()
if is_rc or is_release:
validate_version_tag()
if is_rc:
# we want to upload release candidates to test.pypi.org:
assert (pypi_test_user is not None and pypi_test_password is not None)
if is_release:
# releases need the "real" pypi credentials:
assert (pypi_user is not None and pypi_password is not None)
prepare_upload(verbose=ctx.obj['verbose'])
# validate zenodo-upload.json:
with open(join(BASE_DIR, 'packaging/zenodo-upload.json')) as f:
json.load(f)
run_twine_check(ctx.obj['verbose'])
if not dry_run:
if is_release:
upload_to_pypi(ctx.obj['verbose'], user=pypi_user, password=pypi_password)
elif is_rc:
upload_to_pypi(ctx.obj['verbose'], user=pypi_test_user, password=pypi_test_password)
release = get_release_kind()
if build_appimage:
appimage_files = build_the_appimage(verbose=ctx.obj['verbose'])
else:
appimage_files = []
github_files = get_py_release_files()
github_files.extend(appimage_files)
branch = env['branch']
is_master = branch == "master"
is_stable_branch = STABLE_VERSION_PAT.match(branch) is not None
is_pull_request = env['is_pr']
if (is_master or is_stable_branch or is_release or is_rc) and not is_pull_request:
if dry_run:
print("would upload the following files to github:")
for f in github_files:
print(f)
else:
upload_to_github(ctx.obj['verbose'], github_files, token=token, release=release)
else:
print(
"branch is not master or stable and not tagged (or CI run is for PR)"
", not uploading to github (files=%s)" % github_files
)
if not dry_run and (is_release or is_rc):
upload_to_zenodo(verbose=ctx.obj['verbose'],
parent=zenodo_sandbox_parent if is_rc else zenodo_parent,
token=zenodo_token, sandbox_token=zenodo_sandbox_token)
@cli.command()
@click.pass_context
def status(ctx):
version_file = get_version_fn()
current_version = read_version(version_file)
print(f"env: {env}")
print("current version: %s" % current_version)
print("version tags for HEAD: %s" % [m['full'] for m in tag_matches(get_current_tags())])
print("is release candidate? %s" % current_version_tag_is_rc())
kind = get_release_kind()
if kind in ["rc", "release"]:
print("validating version tag...")
validate_version_tag()
print("done.")
@cli.command()
@click.pass_context
def do_build_appimage(ctx):
appimage_files = build_the_appimage(verbose=ctx.obj['verbose'])
print("success, files=", appimage_files)
def get_docker_tags() -> list[str]:
version = get_release_tag()
is_latest_stable = version and (version["full"] == get_latest_stable_tag())
tags = []
if version is not None:
tags.append(version["full"])
if is_latest_stable:
tags.append("latest")
tags.append("continuous")
return tags
@cli.command()
@click.argument('source_image', type=str)
@click.argument('base_image_name', type=str)
@click.option('--dry-run/--no-dry-run', default=False,
help="don't actually push, only prints the command that "
"would be executed")
def docker_retag(source_image, base_image_name, dry_run):
"""
Re-tag and push the `source_image`, with the following template:
{base_image_name}:{tag}
For all tags that are applicable to the current version.
"""
tags = get_docker_tags()
cmd = ['docker', 'buildx', 'imagetools', 'create']
for tag in tags:
cmd.extend(['--tag', f'{base_image_name}:{tag}'])
cmd.append(source_image)
if dry_run:
print(" ".join(cmd))
else:
subprocess.check_call(cmd)
@cli.command()
def docker_tags():
"""
Prints the tags that should be used for any docker images that are built,
separated by whitespace.
"""
tags = get_docker_tags()
print(" ".join(tags))
@cli.command()
@click.argument('new_version', type=str)
@click.option('--tag/--no-tag', help='create a git tag after bumping (implies --commit)',
default=False)
@click.option('--commit/--no-commit', help='create a git commit after bumping',
default=True)
@click.option('--force/--no-force', help='force operation, even if it doesn\'t fit our conventions',
default=False)
def bump(new_version, tag, commit, force):
"""
bump the version in {PYTHON_PKG_NAME}.__version__
NEW_VERSION should be pep440 compatible and conform to our version conventions
"""
if tag and not commit:
commit = True
print("NOTE: implicitly enabling --commit")
match = VERSION_PAT.match(new_version)
if match is None:
raise click.UsageError("could not parse version, may not conform to our scheme")
new_version = match['noprefix']
if tag and match.groupdict()['dev'] is not None:
if not force:
raise click.ClickException("dev releases should not be tagged, use --force to override")
else:
print("NOTE: tagging a dev release because of --force")
version_file = get_version_fn()
old_version = read_version(version_file)
if parse_version(old_version) > parse_version(new_version):
print(f'old version {old_version} is larger than new version {new_version}')
if not force:
raise click.ClickException("new version should be larger than old version")
else:
print("NOTE: bumping version by force")
render_version(version_file, new_version)
version_tag = f"v{new_version}"
if commit:
do_git_commit(old_version=old_version, new_version=new_version,
version_file=version_file)
if tag:
do_git_tag(tag=version_tag)
print(f"version bumped from {old_version} to {new_version}")
repo = f"git@github.com:{GH_REPO_NAME}.git"
if tag:
print("now, push the new version: $ git push {} && git push {} {}".format(
repo, repo, version_tag)
)
else:
if commit:
print("now, tag as needed and push")
else:
print("WARNING: not committed! commit the new version, tag as needed and push")
if __name__ == "__main__":
cli(obj={}, auto_envvar_prefix="LT_RELEASE")