Skip to content

Commit

Permalink
test: benchmark for small local files: bug1339.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@nedbat
nedbat committed May 30, 2022
1 parent 53061ed commit 59ce1f4
Showing 1 changed file with 107 additions and 41 deletions.
148 changes: 107 additions & 41 deletions lab/benchmark.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import time
from pathlib import Path

from typing import Iterator, List, Optional, Tuple, Union
from typing import Iterator, List, Optional, Tuple


class ShellSession:
Expand All @@ -21,6 +21,7 @@ class ShellSession:
def __init__(self, output_filename: str):
self.output_filename = output_filename
self.last_duration: float = 0
self.foutput = None

def __enter__(self):
self.foutput = open(self.output_filename, "a", encoding="utf-8")
Expand All @@ -31,6 +32,7 @@ def __exit__(self, exc_type, exc_value, traceback):
self.foutput.close()

def print(self, *args, **kwargs):
"""Print a message to this shell's log."""
print(*args, **kwargs, file=self.foutput)

def run_command(self, cmd: str) -> str:
Expand Down Expand Up @@ -114,8 +116,9 @@ class ProjectToTest:
git_url: Optional[str] = None

def __init__(self):
self.slug = self.git_url.split("/")[-1]
self.dir = Path(self.slug)
if self.git_url:
self.slug = self.git_url.split("/")[-1]
self.dir = Path(self.slug)

def get_source(self, shell):
"""Get the source of the project."""
Expand Down Expand Up @@ -147,6 +150,7 @@ def prep_environment(self, env):
self.run_tox(env, env.pyver.toxenv, "--notest")

def run_tox(self, env, toxenv, toxargs=""):
"""Run a tox command. Return the duration."""
env.shell.run_command(f"{env.python} -m tox -e {toxenv} {toxargs}")
return env.shell.last_duration

Expand All @@ -155,7 +159,9 @@ def run_no_coverage(self, env):

def run_with_coverage(self, env, pip_args, cov_options):
self.run_tox(env, env.pyver.toxenv, "--notest")
env.shell.run_command(f".tox/{env.pyver.toxenv}/bin/python -m pip install {pip_args}")
env.shell.run_command(
f".tox/{env.pyver.toxenv}/bin/python -m pip install {pip_args}"
)
return self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install")


Expand All @@ -177,6 +183,7 @@ def run_with_coverage(self, env, pip_args, cov_options):
env.shell.run_command(f".tox/{covenv}/bin/python -m coverage debug sys")
return self.run_tox(env, covenv, "--skip-pkg-install")


class ProjectDateutil(ToxProject):
"""dateutil/dateutil"""

Expand All @@ -190,11 +197,45 @@ def run_no_coverage(self, env):
env.shell.run_command("echo No option to run without coverage")
return 0


class ProjectAttrs(ToxProject):
"""python-attrs/attrs"""

git_url = "https://github.com/python-attrs/attrs"


class AdHocProject(ProjectToTest):
"""A standalone program to run locally."""

def __init__(self, python_file, pip_args=None):
super().__init__()
self.python_file = Path(python_file)
self.pip_args = pip_args
self.slug = self.python_file.name

def get_source(self, shell):
pass

def prep_environment(self, env):
env.shell.run_command(f"{env.python} -m pip install {self.pip_args}")

def run_no_coverage(self, env):
with change_dir(self.python_file.parent):
env.shell.run_command(f"{env.python} {self.python_file.name}")
return env.shell.last_duration

def run_with_coverage(self, env, pip_args, cov_options):
env.shell.run_command(f"{env.python} -m pip install {pip_args}")
with change_dir(self.python_file.parent):
env.shell.run_command(
f"{env.python} -m coverage run {self.python_file.name}"
)
return env.shell.last_duration


class PyVersion:
"""A version of Python to use."""

# The command to run this Python
command: str
# The tox environment to run this Python
Expand Down Expand Up @@ -236,7 +277,7 @@ def run_experiments(

results = []
for proj in projects:
print(f"Testing with {proj.git_url}")
print(f"Testing with {proj.slug}")
with ShellSession(f"output_{proj.slug}.log") as shell:
proj.get_source(shell)

Expand Down Expand Up @@ -264,7 +305,10 @@ def run_experiments(
print(f"Tests took {dur:.3f}s")
durations.append(dur)
med = statistics.median(durations)
result = f"Median for {proj.slug}, {pyver.command}, cov={cov_slug}: {med:.3f}s"
result = (
f"Median for {proj.slug}, {pyver.command}, "
+ f"cov={cov_slug}: {med:.3f}s"
)
print(f"## {result}")
results.append(result)

Expand All @@ -281,38 +325,60 @@ def run_experiments(

with change_dir(PERF_DIR):

run_experiments(
py_versions=[
Python(3, 7),
Python(3, 10),
],
cov_versions=[
#("none", None, None),
("6.4 timid", "coverage==6.4", "timid=True"),
("tip timid", "git+https://github.com/nedbat/coveragepy.git@master", "timid=True"),
],
projects=[
ProjectPytestHtml(),
ProjectAttrs(),
],
num_runs=3,
)

# run_experiments(
# py_versions=[
# PyPy(3, 9),
# ],
# cov_versions=[
# ("none", None, None),
# ("6.4", "coverage==6.4", ""),
# (
# "PR 1381",
# "git+https://github.com/cfbolz/coveragepy.git@f_trace_lines",
# "",
# ),
# ],
# projects=[
# ProjectPytestHtml(),
# ],
# num_runs=3,
# )
if 1:
run_experiments(
py_versions=[
Python(3, 7),
Python(3, 10),
],
cov_versions=[
# ("none", None, None),
("6.4 timid", "coverage==6.4", "timid=True"),
(
"tip timid",
"git+https://github.com/nedbat/coveragepy.git@master",
"timid=True",
),
],
projects=[
ProjectPytestHtml(),
ProjectAttrs(),
],
num_runs=3,
)

if 0:
run_experiments(
py_versions=[
PyPy(3, 9),
],
cov_versions=[
("none", None, None),
("6.4", "coverage==6.4", ""),
(
"PR 1381",
"git+https://github.com/cfbolz/coveragepy.git@f_trace_lines",
"",
),
],
projects=[
ProjectPytestHtml(),
],
num_runs=3,
)

if 0:
run_experiments(
py_versions=[
PyPy(3, 9),
],
cov_versions=[
("none", None, None),
("6.4", "coverage", ""),
("tip", "git+https://github.com/nedbat/coveragepy.git@master", ""),
],
projects=[
AdHocProject("/src/bugs/bug1339/bug1339.py"),
],
num_runs=7,
)

0 comments on commit 59ce1f4

Please sign in to comment.