diff --git a/.spin/cmds.py b/.spin/cmds.py new file mode 100644 index 000000000..a97286c87 --- /dev/null +++ b/.spin/cmds.py @@ -0,0 +1,159 @@ +import os +import sys + +import click +import spin +from spin.cmds import meson + + +@click.command() +@click.option( + '--tests', '-t', + default=None, metavar='TESTS', multiple=True, + help="Which tests to run" +) +@click.option( + '--compare', '-c', + is_flag=True, + default=False, + help="Compare benchmarks between the current branch and main " + "(unless other branches specified). " + "The benchmarks are each executed in a new isolated " + "environment." +) +@click.option( + '--verbose', '-v', is_flag=True, default=False +) +@click.option( + '--quick', '-q', is_flag=True, default=False, + help="Run each benchmark only once (timings won't be accurate)" +) +@click.option( + '--factor', '-f', default=1.05, + help="The factor above or below which a benchmark result is " + "considered reportable. This is passed on to the asv command." +) +@click.argument( + 'commits', metavar='', + required=False, + nargs=-1 +) +@meson.build_dir_option +@click.pass_context +def bench(ctx, tests, compare, verbose, quick, factor, commits, build_dir): + """🏋 Run benchmarks. + + \b + Examples: + + \b + $ spin bench -t dwt_benchmarks + $ spin bench -t swt_benchmarks.Swt2TimeSuite.time_swt2 + + Two benchmark runs can be compared. + By default, `HEAD` is compared to `main`. + You can also specify the branches/commits to compare: + + \b + $ spin bench --compare + $ spin bench --compare main + $ spin bench --compare main HEAD + + You can also choose which benchmarks to run in comparison mode: + + $ spin bench -t Swt2TimeSuite --compare + + For a quicker but less accurate check to see if benchmarks work: + + $ spin bench --quick + + """ + if not commits: + commits = ('main', 'HEAD') + elif len(commits) == 1: + commits = commits + ('HEAD',) + elif len(commits) > 2: + raise click.ClickException( + 'Need a maximum of two revisions to compare' + ) + + bench_args = [] + for t in tests: + bench_args += ['--bench', t] + + if verbose: + bench_args = ['-v'] + bench_args + + if quick: + bench_args = ['--quick'] + bench_args + + if not compare: + # No comparison requested; we build and benchmark the current version + + click.secho( + "Invoking `build` prior to running benchmarks:", + bold=True, fg="bright_green" + ) + ctx.invoke(meson.build) + + meson._set_pythonpath(build_dir) + # Some weird bug, not sure what's going on here, but it seems necessary + # on Python 3.14 + if not os.environ['PYTHONPATH'].endswith(os.sep): + os.environ['PYTHONPATH'] += os.sep + + p = spin.util.run( + [sys.executable, '-c', 'import pywt; print(pywt.__version__)'], + cwd='benchmarks', + echo=False, + output=False + ) + pywt_ver = p.stdout.strip().decode('ascii') + + click.secho( + f'Running benchmarks on PyWavelets {pywt_ver}', + bold=True, fg="bright_green" + ) + cmd = [ + 'asv', 'run', '--dry-run', '--show-stderr', '--python=same' + ] + bench_args + + os.chdir('..') + spin.util.run(cmd, cwd='benchmarks', env=os.environ) + else: + # Ensure that we don't have uncommitted changes + commit_a, commit_b = (_commit_to_sha(c) for c in commits) + + if commit_b == 'HEAD' and _dirty_git_working_dir(): + click.secho( + "WARNING: you have uncommitted changes --- " + "these will NOT be benchmarked!", + fg="red" + ) + + cmd_compare = [ + 'asv', 'continuous', '--factor', str(factor), + ] + bench_args + [commit_a, commit_b] + spin.util.run(cmd_compare, cwd='benchmarks') + + +def _commit_to_sha(commit): + p = spin.util.run(['git', 'rev-parse', commit], output=False, echo=False) + if p.returncode != 0: + raise ( + click.ClickException( + f'Could not find SHA matching commit `{commit}`' + ) + ) + + return p.stdout.decode('ascii').strip() + + +def _dirty_git_working_dir(): + # Changes to the working directory + p0 = spin.util.run(['git', 'diff-files', '--quiet']) + + # Staged changes + p1 = spin.util.run(['git', 'diff-index', '--quiet', '--cached', 'HEAD']) + + return (p0.returncode != 0 or p1.returncode != 0) diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 0dd074b46..3f4b2f3eb 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -19,6 +19,13 @@ "branches": ["main"], // for git // "branches": ["tip"], // for mercurial + // This is a frequent source of failures for `--compare`. Check the + // asv docs for what `build_command` does. + // Avoid build isolation, that will cause problems. + "build_command": [ + "python -m build -wnx -o {build_cache_dir} {build_dir}" + ], + // The DVCS being used. If not set, it will be automatically // determined from "repo" by looking at the protocol in the URL // (if remote), or by looking for special directories, such as @@ -30,7 +37,7 @@ // If missing or the empty string, the tool will be automatically // determined by looking for tools on the PATH environment // variable. - // "environment_type": "virtualenv", + "environment_type": "virtualenv", // the base URL to show a commit for the project. "show_commit_url": "http://github.com/PyWavelets/pywt/commit/", @@ -46,6 +53,7 @@ "matrix": { "numpy": [], "Cython": [], + "meson-python": [], }, // The directory (relative to the current directory) that benchmarks are @@ -71,5 +79,5 @@ // `asv` will cache wheels of the recent builds in each // environment, making them faster to install next time. This is // number of builds to keep, per environment. - // "wheel_cache_size": 0 + "wheel_cache_size": 20 } diff --git a/pyproject.toml b/pyproject.toml index f39919053..6aca36801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,3 +98,4 @@ Environments = [ 'spin.cmds.meson.python', ] Documentation = ["spin.cmds.meson.docs"] +Metrics = [".spin/cmds.py:bench"]