benchs

metrics/__init__.py

@@ -1 +1 @@
-from .benchmarks.pre_cycle_setup import *
+from .benchmarks import *

metrics/__main__.py

@@ -1,20 +1,36 @@
-from metrics.utils import attempts_to_average
+from concurrent.futures import ProcessPoolExecutor
+import os
+
+from rich import Console
+
+from metrics.utils import run_benchmark, BenchmarkResult
 from .registry import Benchmarks, Benchmark
 
 
 def main():
+    console = Console()
     all_benchmarks: list[Benchmark] = Benchmarks.get_benchmarks()
 
-    for benchmark in all_benchmarks:
-        bench_attempts: list[float] = []
-        for _ in range(benchmark.iterations):
-            bench_attempts.append(benchmark.run())
+    workers = os.cpu_count() or 1
+    with ProcessPoolExecutor(max_workers=workers) as executor:
+        results = executor.map(run_benchmark, all_benchmarks)
 
-        print(f'Name: {benchmark.name}\n' 
+    type_paired_benchmarks: dict[str, list[BenchmarkResult]] = {}
+
+    for result in results:
+        type_paired_benchmarks.setdefault(result.type_, []).append(result)
+
+    for type_, benchmarks in type_paired_benchmarks.items():
+        console.print('\n' + ('='*(len(type_)+14)))
+        console.print(f'    TYPE: {type_.upper()}')
+        console.print('='*(len(type_)+14) + '\n')
+
+        for benchmark in benchmarks:
+            console.print(f'Name: {benchmark.name}\n'
                           f'Description: {benchmark.description}\n'
                           f'Iterations: {benchmark.iterations}\n'
-              f'Average time per iteration: {attempts_to_average(bench_attempts, benchmark.iterations)} ms\n')
+                          f'Average time per iteration: {benchmark.avg_time} ms\n')
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

metrics/benchmarks/__init__.py

@@ -0,0 +1 @@
+from .pre_cycle_setup import *

metrics/benchmarks/pre_cycle_setup.py

@@ -15,7 +15,7 @@ from ..utils import get_time_of_pre_cycle_setup
 from ..registry import benchmark
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With no aliases")
+@benchmark(type_="pre_cycle_setup", description="With no aliases")
 def benchmark_no_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -37,7 +37,7 @@ def benchmark_no_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With few aliases (6 total)")
+@benchmark(type_="pre_cycle_setup", description="With few aliases (6 total)")
 def benchmark_few_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -59,7 +59,7 @@ def benchmark_few_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With many aliases (15 total)")
+@benchmark(type_="pre_cycle_setup", description="With many aliases (15 total)")
 def benchmark_many_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -81,7 +81,7 @@ def benchmark_many_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With very many aliases (60 total)")
+@benchmark(type_="pre_cycle_setup", description="With very many aliases (60 total)")
 def benchmark_very_many_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -103,7 +103,7 @@ def benchmark_very_many_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With extreme aliases (300 total)")
+@benchmark(type_="pre_cycle_setup", description="With extreme aliases (300 total)")
 def benchmark_extreme_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()

metrics/registry.py

@@ -6,19 +6,21 @@ __all__ = [
 
 from typing import Callable, ClassVar, overload, override
 
-
 BenchmarkAsFunc = Callable[[], float]
 
+
 class Benchmark:
     def __init__(
             self,
             func: BenchmarkAsFunc,
             *,
+            type_: str,
             name: str,
             description: str,
             iterations: int
     ) -> None:
         self.func = func
+        self.type_ = type_
         self.name = name
         self.description = description
         self.iterations = iterations
@@ -28,11 +30,11 @@ class Benchmark:
 
     @override
     def __repr__(self) -> str:
-        return f'Benchmark<{self.name=}, {self.description=}, {self.iterations=}>'
+        return f'Benchmark<{self.type_=}, {self.name=}, {self.description=}, {self.iterations=}>'
 
     @override
     def __str__(self) -> str:
-        return f'Benchmark({self.name=}, {self.description=}, {self.iterations=})'
+        return f'Benchmark({self.type_=}, {self.name=}, {self.description=}, {self.iterations=})'
 
 
 class Benchmarks:
@@ -44,10 +46,11 @@ class Benchmarks:
             cls,
             call: BenchmarkAsFunc,
             *,
-        name: str = "",
+            type_: str = "",
             description: str = "",
             iterations: int = 100,
-    ) -> BenchmarkAsFunc: ...
+    ) -> BenchmarkAsFunc:
+        ...
 
     @overload
     @classmethod
@@ -55,17 +58,18 @@ class Benchmarks:
             cls,
             call: None = None,
             *,
-        name: str = "",
+            type_: str = "",
             description: str = "",
             iterations: int = 100,
-    ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]: ...
+    ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]:
+        ...
 
     @classmethod
     def register(
             cls,
             call: BenchmarkAsFunc | None = None,
             *,
-        name: str = "",
+            type_: str = "",
             description: str = "",
             iterations: int = 100,
     ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc] | BenchmarkAsFunc:
@@ -73,9 +77,10 @@ class Benchmarks:
             cls._benchmarks.append(
                 Benchmark(
                     func,
-                    name = name or func.__name__,
-                    description = description or f'description for {name or func.__name__} with {iterations} iterations',
-                    iterations = iterations
+                    type_=type_,
+                    name=func.__name__,
+                    description=description or f'description for {func.__name__} with {iterations} iterations',
+                    iterations=iterations
                 )
             )
             return func
@@ -89,4 +94,5 @@ class Benchmarks:
     def get_benchmarks(cls) -> list[Benchmark]:
         return cls._benchmarks
 
+
 benchmark = Benchmarks.register

metrics/utils.py

@@ -1,14 +1,18 @@
 __all__ = [
     "get_time_of_pre_cycle_setup",
-    "attempts_to_average"
+    "attempts_to_average",
+    "run_benchmark",
+    "BenchmarkResult"
 ]
 
 import io
 from contextlib import redirect_stdout
 import time
+from dataclasses import dataclass
 from decimal import Decimal, ROUND_HALF_UP
 
 from argenta import App
+from metrics.registry import Benchmark
 
 
 def get_time_of_pre_cycle_setup(app: App) -> float:
@@ -21,3 +25,20 @@ def get_time_of_pre_cycle_setup(app: App) -> float:
 
 def attempts_to_average(bench_attempts: list[float], iterations: int) -> Decimal:
     return Decimal(sum(bench_attempts) / iterations).quantize(Decimal("0.00001"), rounding=ROUND_HALF_UP)
+
+
+@dataclass(frozen=True)
+class BenchmarkResult:
+    type_: str
+    name: str
+    description: str
+    iterations: int
+    avg_time: Decimal
+
+
+def run_benchmark(benchmark: Benchmark) -> BenchmarkResult:
+    bench_attempts: list[float] = []
+    for _ in range(benchmark.iterations):
+        bench_attempts.append(benchmark.run())
+    avg = attempts_to_average(bench_attempts, benchmark.iterations)
+    return BenchmarkResult(benchmark.type_, benchmark.name, benchmark.description, benchmark.iterations, avg)