From 1648a8206a6440c9f46438884187de757158b723 Mon Sep 17 00:00:00 2001 From: kolo Date: Sat, 17 Jan 2026 23:43:59 +0300 Subject: [PATCH] bbbbbbenchh --- metrics/benchmarks/exceptions.py | 14 +++ metrics/benchmarks/models.py | 163 +++++++++++++++++++------------ metrics/benchmarks/utils.py | 46 ++++----- metrics/handlers.py | 6 +- pyproject.toml | 1 + uv.lock | 15 ++- 6 files changed, 153 insertions(+), 92 deletions(-) create mode 100644 metrics/benchmarks/exceptions.py diff --git a/metrics/benchmarks/exceptions.py b/metrics/benchmarks/exceptions.py new file mode 100644 index 0000000..497ee02 --- /dev/null +++ b/metrics/benchmarks/exceptions.py @@ -0,0 +1,14 @@ +class BenchmarkNotFound(Exception): + def __init__(self, benchmark_name: str): + self.benchmark_name = benchmark_name + + def __str__(self): + return f"Benchmark with name '{self.benchmark_name}' not found" + + +class BenchmarksNotFound(Exception): + def __init__(self, type_: str): + self.type_ = type_ + + def __str__(self): + return f"Benchmarks with type '{self.type_}' not found" diff --git a/metrics/benchmarks/models.py b/metrics/benchmarks/models.py index efe7ead..9726d9c 100644 --- a/metrics/benchmarks/models.py +++ b/metrics/benchmarks/models.py @@ -1,26 +1,39 @@ __all__ = [ "Benchmark", "Benchmarks", - "BenchmarkResult", - "benchmark" + "BenchmarkResult" ] from dataclasses import dataclass -from decimal import Decimal -from typing import Callable, ClassVar, overload, override +import time +import gc +import statistics +from typing import Callable, override + +from .exceptions import BenchmarkNotFound, BenchmarksNotFound + BenchmarkAsFunc = Callable[[], float] -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class BenchmarkResult: type_: str name: str description: str iterations: int - avg_time: Decimal + is_gc_disabled: bool + avg_time: float + median_time: float + std_dev: float + + +@dataclass(frozen=True, slots=True) +class BenchmarkGroupResult: + type_: str + benchmark_results: list[BenchmarkResult] + - class Benchmark: def __init__( self, @@ -28,83 +41,109 @@ class Benchmark: *, type_: str, name: str, - description: str, - iterations: int + description: str ) -> None: self.func = func self.type_ = type_ self.name = name self.description = description - self.iterations = iterations - def run(self) -> float: - return self.func() + def single_run(self, is_gc_disabled: bool = False) -> float: + if is_gc_disabled: + was_gc_enabled = gc.isenabled() + gc.disable() + + start = time.perf_counter() + self.func() + end = time.perf_counter() + + if was_gc_enabled: + gc.enable() + gc.collect() + + return end - start + else: + start = time.perf_counter() + self.func() + end = time.perf_counter() + return end - start + + def multiple_runs(self, iterations: int, is_gc_disabled: bool = False) -> tuple[float]: + run_attempts: list[float] = [] + for _ in range(iterations): + run_attempts.append(self.single_run(is_gc_disabled)) + return tuple(*run_attempts) @override def __repr__(self) -> str: - return f'Benchmark<{self.type_=}, {self.name=}, {self.description=}, {self.iterations=}>' + return f'Benchmark<{self.type_=}, {self.name=}, {self.description=}>' @override def __str__(self) -> str: - return f'Benchmark({self.type_=}, {self.name=}, {self.description=}, {self.iterations=})' + return f'benchmark {self.name} with type {self.type_}' class Benchmarks: - _benchmarks: ClassVar[list[Benchmark]] = [] + def __init__(self, *benchmarks: Benchmark) -> None: + self._benchmarks: list[Benchmark] = list(benchmarks) + self._benchmarks_grouped_by_type: dict[str, list[Benchmark]] = {} + self._benchmarks_paired_by_name: dict[str, Benchmark] = {} - @overload - @classmethod def register( - cls, - call: BenchmarkAsFunc, - *, - type_: str = "", - description: str = "", - iterations: int = 100, - ) -> BenchmarkAsFunc: - ... - - @overload - @classmethod - def register( - cls, - call: None = None, - *, - type_: str = "", - description: str = "", - iterations: int = 100, + self, + type_: str, + description: str = "" ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc]: - ... - - @classmethod - def register( - cls, - call: BenchmarkAsFunc | None = None, - *, - type_: str = "", - description: str = "", - iterations: int = 100, - ) -> Callable[[BenchmarkAsFunc], BenchmarkAsFunc] | BenchmarkAsFunc: def decorator(func: BenchmarkAsFunc) -> BenchmarkAsFunc: - cls._benchmarks.append( - Benchmark( - func, - type_=type_, - name=func.__name__, - description=description or f'description for {func.__name__} with {iterations} iterations', - iterations=iterations - ) + benchmark = Benchmark( + func, + type_=type_, + name=func.__name__, + description=description or f'description for {func.__name__} with type {type_}', ) + self._benchmarks.append(benchmark) + self._benchmarks_paired_by_name[type_] = benchmark + self._benchmarks_grouped_by_type.setdefault(type_, []).append(benchmark) return func + return decorator - if call is None: - return decorator - else: - return decorator(call) + def run_benchmark_by_name(self, name: str, iterations: int = 100, is_gc_disables: bool = False) -> BenchmarkResult: + benchmark = self.get_benchmark_by_name(name) + if not benchmark: + raise BenchmarkNotFound(name) + run_attempts: tuple[float] = benchmark.multiple_runs(iterations, is_gc_disables) - @classmethod - def get_benchmarks(cls) -> list[Benchmark]: - return cls._benchmarks + avg = statistics.mean(run_attempts) + median = statistics.median(run_attempts) + std_dev = statistics.stdev(run_attempts) if len(run_attempts) > 1 else 0 + return BenchmarkResult( + type_=benchmark.type_, + name=benchmark.name, + description=benchmark.description, + iterations=iterations, + is_gc_disabled=is_gc_disables, + avg_time=avg, + median_time=median, + std_dev=std_dev + ) -benchmark = Benchmarks.register + def run_benchmarks_by_type(self, type_: str, iterations: int = 100, is_gc_disabled: bool = False) -> BenchmarkGroupResult: + benchmarks = self.get_benchmarks_by_type(type_) + if not benchmarks: + raise BenchmarksNotFound(type_) + benchmark_results: list[BenchmarkResult] = [] + + for benchmark in benchmarks: + benchmark_results.append(self.run_benchmark_by_name(benchmark.name, iterations, is_gc_disabled)) + + return BenchmarkGroupResult( + type_=type_, + benchmark_results=benchmark_results + ) + + def get_benchmarks_by_type(self, type_: str) -> list[Benchmark]: + return self._benchmarks_grouped_by_type.get(type_, []) + + def get_benchmark_by_name(self, name: str) -> Benchmark | None: + return self._benchmarks_paired_by_name.get(name) diff --git a/metrics/benchmarks/utils.py b/metrics/benchmarks/utils.py index bb10000..95af193 100644 --- a/metrics/benchmarks/utils.py +++ b/metrics/benchmarks/utils.py @@ -3,10 +3,8 @@ __all__ = [ "get_time_of_validate_routers_for_collisions", "get_time_of_most_similar_command", "get_time_of_finds_appropriate_handler", - "attempts_to_average", - "run_benchmark", - "run_all_benchmarks", - "get_kernel_version" + "get_kernel_version", + "get_gpu_info" ] import io @@ -18,6 +16,8 @@ from concurrent.futures import ProcessPoolExecutor from contextlib import redirect_stdout from decimal import ROUND_HALF_UP, Decimal +import pynvml + from argenta import App from argenta.router import Router from argenta.command.models import InputCommand @@ -48,7 +48,7 @@ def get_time_of_most_similar_command(app: App, unknown_command: str) -> float: return (end - start) * 1000 -def get_time_of_finds_appropriate_handler(router: "Router", input_command: "InputCommand") -> float: +def get_time_of_finds_appropriate_handler(router: Router, input_command: InputCommand) -> float: start = time.perf_counter() with redirect_stdout(io.StringIO()): router.finds_appropriate_handler(input_command) @@ -90,29 +90,21 @@ def get_kernel_version() -> dict[str, str]: 'product_name': platform.system(), } +def get_gpu_info() -> str: + try: + pynvml.nvmlInit() + device_count = pynvml.nvmlDeviceGetCount() + if device_count == 0: + return "N/A" -def attempts_to_average(bench_attempts: list[float], iterations: int) -> Decimal: - return Decimal(sum(bench_attempts) / iterations).quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP) + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + name = pynvml.nvmlDeviceGetName(handle) + if isinstance(name, bytes): + name = name.decode("utf-8") -def run_all_benchmarks() -> dict[str, list[BenchmarkResult]]: - all_benchmarks: list[Benchmark] = Benchmarks.get_benchmarks() + pynvml.nvmlShutdown() + return name + except pynvml.NVMLError: + return "N/A" - workers = os.cpu_count() or 1 - with ProcessPoolExecutor(max_workers=workers) as executor: - results = executor.map(run_benchmark, all_benchmarks) - - type_paired_benchmarks: dict[str, list[BenchmarkResult]] = {} - - for result in results: - type_paired_benchmarks.setdefault(result.type_, []).append(result) - - return type_paired_benchmarks - - -def run_benchmark(benchmark: Benchmark) -> BenchmarkResult: - bench_attempts: list[float] = [] - for _ in range(benchmark.iterations): - bench_attempts.append(benchmark.run()) - avg = attempts_to_average(bench_attempts, benchmark.iterations) - return BenchmarkResult(benchmark.type_, benchmark.name, benchmark.description, benchmark.iterations, avg) diff --git a/metrics/handlers.py b/metrics/handlers.py index aed1984..9516bc0 100644 --- a/metrics/handlers.py +++ b/metrics/handlers.py @@ -10,7 +10,7 @@ from argenta.command.models import Command from argenta.response import Response from argenta.router import Router from .benchmarks.models import BenchmarkResult -from .benchmarks.utils import run_all_benchmarks, get_kernel_version +from .benchmarks.utils import run_all_benchmarks, get_kernel_version, get_gpu_info console = Console() router = Router(title="Metrics commands:") @@ -19,6 +19,7 @@ router = Router(title="Metrics commands:") @router.command(Command("all-print", description="Print all benchmarks results")) def all_print_handler(_: Response) -> None: cpu_info = cpuinfo.get_cpu_info() + gpu_info = get_gpu_info() os_info = get_kernel_version() table = Table(show_header=True, header_style="bold cyan", border_style="blue", show_lines=True) @@ -29,7 +30,8 @@ def all_print_handler(_: Response) -> None: table.add_row("OS Name", os_info['product_name']) table.add_row("OS Kernel Version", os_info['kernel_version']) table.add_row("Architecture", cpu_info['arch']) - table.add_row("Processor", cpu_info['brand_raw']) + table.add_row("CPU", cpu_info['brand_raw']) + table.add_row("GPU", gpu_info) table.add_row("Python Version", cpu_info['python_version']) table.add_row("Python Implementation", platform.python_implementation()) diff --git a/pyproject.toml b/pyproject.toml index 560c6d9..d92d3b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ tests = [ "pytest-mock>=3.15.1", ] metrics = [ + "nvidia-ml-py>=13.590.44", "py-cpuinfo>=9.0.0", ] diff --git a/uv.lock b/uv.lock index f3ad6ca..5fceb6e 100644 --- a/uv.lock +++ b/uv.lock @@ -62,6 +62,7 @@ linters = [ { name = "wemake-python-styleguide" }, ] metrics = [ + { name = "nvidia-ml-py" }, { name = "py-cpuinfo" }, ] tests = [ @@ -95,7 +96,10 @@ linters = [ { name = "ruff", specifier = ">=0.12.12" }, { name = "wemake-python-styleguide", specifier = ">=0.17.0" }, ] -metrics = [{ name = "py-cpuinfo", specifier = ">=9.0.0" }] +metrics = [ + { name = "nvidia-ml-py", specifier = ">=13.590.44" }, + { name = "py-cpuinfo", specifier = ">=9.0.0" }, +] tests = [ { name = "pyfakefs", specifier = ">=5.5.0" }, { name = "pytest", specifier = ">=8.3.2" }, @@ -557,6 +561,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "nvidia-ml-py" +version = "13.590.44" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/23/3871537f204aee823c574ba25cbeb08cae779979d4d43c01adddda00bab9/nvidia_ml_py-13.590.44.tar.gz", hash = "sha256:b358c7614b0fdeea4b95f046f1c90123bfe25d148ab93bb1c00248b834703373", size = 49737, upload-time = "2025-12-08T14:41:10.872Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/47/4c822bd37a008e72fd5a0eae33524ae3ac97b13f7030f63bae1728b8957e/nvidia_ml_py-13.590.44-py3-none-any.whl", hash = "sha256:18feb54eca7d0e3cdc8d1a040a771eda72d9ec3148e5443087970dbfd7377ecc", size = 50683, upload-time = "2025-12-08T14:41:09.597Z" }, +] + [[package]] name = "packaging" version = "25.0"