josch/mmdebstrap
2
3
Fork 6
Code Issues 8 Pull requests 1 Projects Releases Wiki Activity

coverage.py: add timing for tests

Browse source
This commit is contained in:
Johannes Schauer Marin Rodrigues 2023-01-16 13:21:05 +01:00
parent 1dbea10f6b
commit 95c79d0ba3
Signed by: josch
GPG key ID: F2CBA5C78FBD83E1
1 changed files with 34 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
coverage.py
View file

@ -119,7 +119,7 @@ def parse_config(confname):
return config_order, config_dict return config_order, config_dict
def format_failed(num, total, name, dist, mode, variant, fmt, config_dict): def format_test(num, total, name, dist, mode, variant, fmt, config_dict):
ret = f"({num}/{total}) {name}" ret = f"({num}/{total}) {name}"
if len(config_dict[name].get("Dists", [])) > 1: if len(config_dict[name].get("Dists", [])) > 1:
ret += f" --dist={dist}" ret += f" --dist={dist}"
@ -287,6 +287,7 @@ def main():
failed = [] failed = []
num_success = 0 num_success = 0
num_finished = 0 num_finished = 0
time_per_test = {}
for i, (test, name, dist, mode, variant, fmt) in enumerate(tests): for i, (test, name, dist, mode, variant, fmt) in enumerate(tests):
if torun and i not in torun: if torun and i not in torun:
continue continue
@ -365,6 +366,7 @@ def main():
if args.format and args.format != fmt: if args.format and args.format != fmt:
print(f"skipping because of --format={args.format}", file=sys.stderr) print(f"skipping because of --format={args.format}", file=sys.stderr)
continue continue
before = time.time()
proc = subprocess.Popen(argv) proc = subprocess.Popen(argv)
try: try:
proc.wait() proc.wait()
@ -372,21 +374,25 @@ def main():
proc.terminate() proc.terminate()
proc.wait() proc.wait()
break break
after = time.time()
walltime = timedelta(seconds=int(after - before))
formated_test_name = format_test(
i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
)
time_per_test[formated_test_name] = walltime
print(separator, file=sys.stderr) print(separator, file=sys.stderr)
print(f"duration: {walltime}", file=sys.stderr)
if proc.returncode != 0 or shellcheck != "": if proc.returncode != 0 or shellcheck != "":
if shellcheck != "": if shellcheck != "":
print(shellcheck) print(shellcheck)
failed.append( failed.append(formated_test_name)
format_failed(
i + 1, len(tests), name, dist, mode, variant, fmt, config_dict
)
)
print("result: FAILURE", file=sys.stderr) print("result: FAILURE", file=sys.stderr)
else: else:
print("result: SUCCESS", file=sys.stderr) print("result: SUCCESS", file=sys.stderr)
num_success += 1 num_success += 1
if args.maxfail and len(failed) >= args.maxfail: if args.maxfail and len(failed) >= args.maxfail:
break break
print(separator, file=sys.stderr)
print( print(
"successfully ran %d tests" % num_success, "successfully ran %d tests" % num_success,
file=sys.stderr, file=sys.stderr,
@ -402,6 +408,28 @@ def main():
for f in failed: for f in failed:
print(f, file=sys.stderr) print(f, file=sys.stderr)
exit(1) exit(1)
if time_per_test:
print(
"average time per test:",
sum(time_per_test.values(), start=timedelta()) / len(time_per_test),
file=sys.stderr,
)
print(
"median time per test:",
sorted(time_per_test.values())[len(time_per_test) // 2],
file=sys.stderr,
)
head_tail_num = 10
print(f"{head_tail_num} fastests tests:", file=sys.stderr)
for k, v in sorted(time_per_test.items(), key=lambda i: i[1])[
: min(head_tail_num, len(time_per_test))
]:
print(f" {k}: {v}", file=sys.stderr)
print(f"{head_tail_num} slowest tests:", file=sys.stderr)
for k, v in sorted(time_per_test.items(), key=lambda i: i[1], reverse=True)[
: min(head_tail_num, len(time_per_test))
]:
print(f" {k}: {v}", file=sys.stderr)
if __name__ == "__main__": if __name__ == "__main__":