benchs

metrics/__init__.py

@@ -1 +1 @@
-from .benchmarks.pre_cycle_setup import *
+from .benchmarks import *

metrics/__main__.py

@@ -1,20 +1,36 @@
-from metrics.utils import attempts_to_average
+from concurrent.futures import ProcessPoolExecutor
+import os
+
+from rich import Console
+
+from metrics.utils import run_benchmark, BenchmarkResult
 from .registry import Benchmarks, Benchmark
 
 
 def main():
+    console = Console()
     all_benchmarks: list[Benchmark] = Benchmarks.get_benchmarks()
-    
+
-    for benchmark in all_benchmarks:
+    workers = os.cpu_count() or 1
-        bench_attempts: list[float] = []
+    with ProcessPoolExecutor(max_workers=workers) as executor:
-        for _ in range(benchmark.iterations):
+        results = executor.map(run_benchmark, all_benchmarks)
-            bench_attempts.append(benchmark.run())
+
-            
+    type_paired_benchmarks: dict[str, list[BenchmarkResult]] = {}
-        print(f'Name: {benchmark.name}\n' 
+
-              f'Description: {benchmark.description}\n' 
+    for result in results:
-              f'Iterations: {benchmark.iterations}\n' 
+        type_paired_benchmarks.setdefault(result.type_, []).append(result)
-              f'Average time per iteration: {attempts_to_average(bench_attempts, benchmark.iterations)} ms\n')
+
-        
+    for type_, benchmarks in type_paired_benchmarks.items():
-    
+        console.print('\n' + ('='*(len(type_)+14)))
-if __name__ == '__main__':
+        console.print(f'    TYPE: {type_.upper()}')
-    main()
+        console.print('='*(len(type_)+14) + '\n')
+
+        for benchmark in benchmarks:
+            console.print(f'Name: {benchmark.name}\n'
+                          f'Description: {benchmark.description}\n'
+                          f'Iterations: {benchmark.iterations}\n'
+                          f'Average time per iteration: {benchmark.avg_time} ms\n')
+
+
+if __name__ == "__main__":
+    main()

metrics/benchmarks/__init__.py

@@ -0,0 +1 @@
+from .pre_cycle_setup import *

metrics/benchmarks/pre_cycle_setup.py

@@ -15,7 +15,7 @@ from ..utils import get_time_of_pre_cycle_setup
 from ..registry import benchmark
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With no aliases")
+@benchmark(type_="pre_cycle_setup", description="With no aliases")
 def benchmark_no_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -37,7 +37,7 @@ def benchmark_no_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With few aliases (6 total)")
+@benchmark(type_="pre_cycle_setup", description="With few aliases (6 total)")
 def benchmark_few_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -59,7 +59,7 @@ def benchmark_few_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With many aliases (15 total)")
+@benchmark(type_="pre_cycle_setup", description="With many aliases (15 total)")
 def benchmark_many_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -81,7 +81,7 @@ def benchmark_many_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With very many aliases (60 total)")
+@benchmark(type_="pre_cycle_setup", description="With very many aliases (60 total)")
 def benchmark_very_many_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()
@@ -103,7 +103,7 @@ def benchmark_very_many_aliases() -> float:
     return execution_time
 
 
-@benchmark(name="Time of pre_cycle_setup", description="With extreme aliases (300 total)")
+@benchmark(type_="pre_cycle_setup", description="With extreme aliases (300 total)")
 def benchmark_extreme_aliases() -> float:
     app = App(override_system_messages=True)
     router = Router()

metrics/registry.py

@@ -6,34 +6,36 @@ __all__ = [
 
 from typing import Callable, ClassVar, overload, override
 
-
 BenchmarkAsFunc = Callable[[], float]
 
+
 class Benchmark:
     def __init__(
-        self, 
+            self,
-        func: BenchmarkAsFunc, 
+            func: BenchmarkAsFunc,
-        *,
+            *,
-        name: str, 
+            type_: str,
-        description: str, 
+            name: str,
-        iterations: int
+            description: str,
+            iterations: int
     ) -> None:
         self.func = func
+        self.type_ = type_
         self.name = name
         self.description = description
         self.iterations = iterations
-        
+
     def run(self) -> float:
         return self.func()
-    
+
     @override
     def __repr__(self) -> str:
-        return f'Benchmark<{self.name=}, {self.description=}, {self.iterations=}>'
+        return f'Benchmark<{self.type_=}, {self.name=}, {self.description=}, {self.iterations=}>'
-        
+
     @override
     def __str__(self) -> str:
-        return f'Benchmark({self.name=}, {self.description=}, {self.iterations=})'
+        return f'Benchmark({self.type_=}, {self.name=}, {self.description=}, {self.iterations=})'
-    
+
 
 class Benchmarks:
     _benchmarks: ClassVar[list[Benchmark]] = []
@@ -41,41 +43,44 @@ class Benchmarks:
     @overload
     @classmethod
     def register(
-        cls,
+            cls,
-        call: BenchmarkAsFunc,
+            call: BenchmarkAsFunc,
-        *,
+            *,
-        name: str = "",
+            type_: str = "",
-        description: str = "",
+            description: str = "",
-        iterations: int = 100,
+            iterations: int = 100,
-    ) -> BenchmarkAsFunc: ...
+    ) -> BenchmarkAsFunc:
+        ...
 
     @overload
     @classmethod
     def register(
-        cls,
+            cls,
-        call: None = None,
+            call: None = None,
-        *,
+            *,
-        name: str = "",
+            type_: str = "",
-        description: str = "",
+            description: str = "",
-        iterations: int = 100,
+            iterations: int = 100,
-    ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]: ...
+    ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]:
+        ...
 
     @classmethod
     def register(
-        cls,
+            cls,
-        call: BenchmarkAsFunc | None = None,
+            call: BenchmarkAsFunc | None = None,
-        *,
+            *,
-        name: str = "",
+            type_: str = "",
-        description: str = "",
+            description: str = "",
-        iterations: int = 100,
+            iterations: int = 100,
     ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc] | BenchmarkAsFunc:
         def decorator(func: BenchmarkAsFunc) -> BenchmarkAsFunc:
             cls._benchmarks.append(
                 Benchmark(
                     func,
-                    name = name or func.__name__,
+                    type_=type_,
-                    description = description or f'description for {name or func.__name__} with {iterations} iterations',
+                    name=func.__name__,
-                    iterations = iterations
+                    description=description or f'description for {func.__name__} with {iterations} iterations',
+                    iterations=iterations
                 )
             )
             return func
@@ -89,4 +94,5 @@ class Benchmarks:
     def get_benchmarks(cls) -> list[Benchmark]:
         return cls._benchmarks
 
+
 benchmark = Benchmarks.register

metrics/utils.py

@@ -1,14 +1,18 @@
 __all__ = [
     "get_time_of_pre_cycle_setup",
-    "attempts_to_average"
+    "attempts_to_average",
+    "run_benchmark",
+    "BenchmarkResult"
 ]
 
 import io
 from contextlib import redirect_stdout
 import time
+from dataclasses import dataclass
 from decimal import Decimal, ROUND_HALF_UP
 
 from argenta import App
+from metrics.registry import Benchmark
 
 
 def get_time_of_pre_cycle_setup(app: App) -> float:
@@ -21,3 +25,20 @@ def get_time_of_pre_cycle_setup(app: App) -> float:
 
 def attempts_to_average(bench_attempts: list[float], iterations: int) -> Decimal:
     return Decimal(sum(bench_attempts) / iterations).quantize(Decimal("0.00001"), rounding=ROUND_HALF_UP)
+
+
+@dataclass(frozen=True)
+class BenchmarkResult:
+    type_: str
+    name: str
+    description: str
+    iterations: int
+    avg_time: Decimal
+
+
+def run_benchmark(benchmark: Benchmark) -> BenchmarkResult:
+    bench_attempts: list[float] = []
+    for _ in range(benchmark.iterations):
+        bench_attempts.append(benchmark.run())
+    avg = attempts_to_average(bench_attempts, benchmark.iterations)
+    return BenchmarkResult(benchmark.type_, benchmark.name, benchmark.description, benchmark.iterations, avg)