diff --git a/metrics/__init__.py b/metrics/__init__.py index bb87be0..63f99dd 100644 --- a/metrics/__init__.py +++ b/metrics/__init__.py @@ -1 +1 @@ -from .benchmarks.pre_cycle_setup import * \ No newline at end of file +from .benchmarks import * \ No newline at end of file diff --git a/metrics/__main__.py b/metrics/__main__.py index 6e3bee0..477965a 100644 --- a/metrics/__main__.py +++ b/metrics/__main__.py @@ -1,20 +1,36 @@ -from metrics.utils import attempts_to_average +from concurrent.futures import ProcessPoolExecutor +import os + +from rich import Console + +from metrics.utils import run_benchmark, BenchmarkResult from .registry import Benchmarks, Benchmark def main(): + console = Console() all_benchmarks: list[Benchmark] = Benchmarks.get_benchmarks() - - for benchmark in all_benchmarks: - bench_attempts: list[float] = [] - for _ in range(benchmark.iterations): - bench_attempts.append(benchmark.run()) - - print(f'Name: {benchmark.name}\n' - f'Description: {benchmark.description}\n' - f'Iterations: {benchmark.iterations}\n' - f'Average time per iteration: {attempts_to_average(bench_attempts, benchmark.iterations)} ms\n') - - -if __name__ == '__main__': - main() \ No newline at end of file + + workers = os.cpu_count() or 1 + with ProcessPoolExecutor(max_workers=workers) as executor: + results = executor.map(run_benchmark, all_benchmarks) + + type_paired_benchmarks: dict[str, list[BenchmarkResult]] = {} + + for result in results: + type_paired_benchmarks.setdefault(result.type_, []).append(result) + + for type_, benchmarks in type_paired_benchmarks.items(): + console.print('\n' + ('='*(len(type_)+14))) + console.print(f' TYPE: {type_.upper()}') + console.print('='*(len(type_)+14) + '\n') + + for benchmark in benchmarks: + console.print(f'Name: {benchmark.name}\n' + f'Description: {benchmark.description}\n' + f'Iterations: {benchmark.iterations}\n' + f'Average time per iteration: {benchmark.avg_time} ms\n') + + +if __name__ == "__main__": + main() diff --git a/metrics/benchmarks/__init__.py b/metrics/benchmarks/__init__.py new file mode 100644 index 0000000..64424de --- /dev/null +++ b/metrics/benchmarks/__init__.py @@ -0,0 +1 @@ +from .pre_cycle_setup import * \ No newline at end of file diff --git a/metrics/benchmarks/pre_cycle_setup.py b/metrics/benchmarks/pre_cycle_setup.py index 8db6d17..4d0e386 100644 --- a/metrics/benchmarks/pre_cycle_setup.py +++ b/metrics/benchmarks/pre_cycle_setup.py @@ -15,7 +15,7 @@ from ..utils import get_time_of_pre_cycle_setup from ..registry import benchmark -@benchmark(name="Time of pre_cycle_setup", description="With no aliases") +@benchmark(type_="pre_cycle_setup", description="With no aliases") def benchmark_no_aliases() -> float: app = App(override_system_messages=True) router = Router() @@ -37,7 +37,7 @@ def benchmark_no_aliases() -> float: return execution_time -@benchmark(name="Time of pre_cycle_setup", description="With few aliases (6 total)") +@benchmark(type_="pre_cycle_setup", description="With few aliases (6 total)") def benchmark_few_aliases() -> float: app = App(override_system_messages=True) router = Router() @@ -59,7 +59,7 @@ def benchmark_few_aliases() -> float: return execution_time -@benchmark(name="Time of pre_cycle_setup", description="With many aliases (15 total)") +@benchmark(type_="pre_cycle_setup", description="With many aliases (15 total)") def benchmark_many_aliases() -> float: app = App(override_system_messages=True) router = Router() @@ -81,7 +81,7 @@ def benchmark_many_aliases() -> float: return execution_time -@benchmark(name="Time of pre_cycle_setup", description="With very many aliases (60 total)") +@benchmark(type_="pre_cycle_setup", description="With very many aliases (60 total)") def benchmark_very_many_aliases() -> float: app = App(override_system_messages=True) router = Router() @@ -103,7 +103,7 @@ def benchmark_very_many_aliases() -> float: return execution_time -@benchmark(name="Time of pre_cycle_setup", description="With extreme aliases (300 total)") +@benchmark(type_="pre_cycle_setup", description="With extreme aliases (300 total)") def benchmark_extreme_aliases() -> float: app = App(override_system_messages=True) router = Router() diff --git a/metrics/registry.py b/metrics/registry.py index db7ca06..40c9537 100644 --- a/metrics/registry.py +++ b/metrics/registry.py @@ -6,34 +6,36 @@ __all__ = [ from typing import Callable, ClassVar, overload, override - BenchmarkAsFunc = Callable[[], float] + class Benchmark: def __init__( - self, - func: BenchmarkAsFunc, - *, - name: str, - description: str, - iterations: int + self, + func: BenchmarkAsFunc, + *, + type_: str, + name: str, + description: str, + iterations: int ) -> None: self.func = func + self.type_ = type_ self.name = name self.description = description self.iterations = iterations - + def run(self) -> float: return self.func() - + @override def __repr__(self) -> str: - return f'Benchmark<{self.name=}, {self.description=}, {self.iterations=}>' - + return f'Benchmark<{self.type_=}, {self.name=}, {self.description=}, {self.iterations=}>' + @override def __str__(self) -> str: - return f'Benchmark({self.name=}, {self.description=}, {self.iterations=})' - + return f'Benchmark({self.type_=}, {self.name=}, {self.description=}, {self.iterations=})' + class Benchmarks: _benchmarks: ClassVar[list[Benchmark]] = [] @@ -41,41 +43,44 @@ class Benchmarks: @overload @classmethod def register( - cls, - call: BenchmarkAsFunc, - *, - name: str = "", - description: str = "", - iterations: int = 100, - ) -> BenchmarkAsFunc: ... + cls, + call: BenchmarkAsFunc, + *, + type_: str = "", + description: str = "", + iterations: int = 100, + ) -> BenchmarkAsFunc: + ... @overload @classmethod def register( - cls, - call: None = None, - *, - name: str = "", - description: str = "", - iterations: int = 100, - ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]: ... + cls, + call: None = None, + *, + type_: str = "", + description: str = "", + iterations: int = 100, + ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]: + ... @classmethod def register( - cls, - call: BenchmarkAsFunc | None = None, - *, - name: str = "", - description: str = "", - iterations: int = 100, + cls, + call: BenchmarkAsFunc | None = None, + *, + type_: str = "", + description: str = "", + iterations: int = 100, ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc] | BenchmarkAsFunc: def decorator(func: BenchmarkAsFunc) -> BenchmarkAsFunc: cls._benchmarks.append( Benchmark( func, - name = name or func.__name__, - description = description or f'description for {name or func.__name__} with {iterations} iterations', - iterations = iterations + type_=type_, + name=func.__name__, + description=description or f'description for {func.__name__} with {iterations} iterations', + iterations=iterations ) ) return func @@ -89,4 +94,5 @@ class Benchmarks: def get_benchmarks(cls) -> list[Benchmark]: return cls._benchmarks + benchmark = Benchmarks.register diff --git a/metrics/utils.py b/metrics/utils.py index 2c3f83e..315431f 100644 --- a/metrics/utils.py +++ b/metrics/utils.py @@ -1,14 +1,18 @@ __all__ = [ "get_time_of_pre_cycle_setup", - "attempts_to_average" + "attempts_to_average", + "run_benchmark", + "BenchmarkResult" ] import io from contextlib import redirect_stdout import time +from dataclasses import dataclass from decimal import Decimal, ROUND_HALF_UP from argenta import App +from metrics.registry import Benchmark def get_time_of_pre_cycle_setup(app: App) -> float: @@ -21,3 +25,20 @@ def get_time_of_pre_cycle_setup(app: App) -> float: def attempts_to_average(bench_attempts: list[float], iterations: int) -> Decimal: return Decimal(sum(bench_attempts) / iterations).quantize(Decimal("0.00001"), rounding=ROUND_HALF_UP) + + +@dataclass(frozen=True) +class BenchmarkResult: + type_: str + name: str + description: str + iterations: int + avg_time: Decimal + + +def run_benchmark(benchmark: Benchmark) -> BenchmarkResult: + bench_attempts: list[float] = [] + for _ in range(benchmark.iterations): + bench_attempts.append(benchmark.run()) + avg = attempts_to_average(bench_attempts, benchmark.iterations) + return BenchmarkResult(benchmark.type_, benchmark.name, benchmark.description, benchmark.iterations, avg)