From e7d064908fb60b241486fa36c727e7d69dfd0839 Mon Sep 17 00:00:00 2001 From: kolo Date: Sun, 18 Jan 2026 18:41:03 +0300 Subject: [PATCH] bench --- metrics/__main__.py | 2 +- metrics/benchmarks/core/exceptions.py | 6 ++ metrics/benchmarks/core/models.py | 12 ++- metrics/benchmarks/most_similar_command.py | 7 +- metrics/handlers.py | 13 +-- metrics/services/__init__.py | 0 metrics/services/report_generator.py | 0 metrics/services/system_info_reader.py | 114 +++++++++++++++++++++ metrics/utils.py | 63 ------------ pyproject.toml | 2 +- uv.lock | 41 ++++++-- 11 files changed, 167 insertions(+), 93 deletions(-) create mode 100644 metrics/services/__init__.py create mode 100644 metrics/services/report_generator.py create mode 100644 metrics/services/system_info_reader.py delete mode 100644 metrics/utils.py diff --git a/metrics/__main__.py b/metrics/__main__.py index a775226..78f64ea 100644 --- a/metrics/__main__.py +++ b/metrics/__main__.py @@ -3,7 +3,7 @@ from argenta.app import StaticDividingLine from .handlers import router -app = App(initial_message="metrics", prompt=">>> ", dividing_line=StaticDividingLine('~', length=120)) +app = App(initial_message="metrics", prompt=">>> ", dividing_line=StaticDividingLine('~', length=70)) orchestrator = Orchestrator() diff --git a/metrics/benchmarks/core/exceptions.py b/metrics/benchmarks/core/exceptions.py index 497ee02..b849974 100644 --- a/metrics/benchmarks/core/exceptions.py +++ b/metrics/benchmarks/core/exceptions.py @@ -12,3 +12,9 @@ class BenchmarksNotFound(Exception): def __str__(self): return f"Benchmarks with type '{self.type_}' not found" + +class BenchmarksWithSameNameAlreadyExists(Exception): + def __init__(self, benchmark_name: str): + self.benchmark_name = benchmark_name + def __str__(self): + return f"Benchmarks with name '{self.benchmark_name}' already exists" diff --git a/metrics/benchmarks/core/models.py b/metrics/benchmarks/core/models.py index 4be6969..386dea1 100644 --- a/metrics/benchmarks/core/models.py +++ b/metrics/benchmarks/core/models.py @@ -13,8 +13,7 @@ import gc import statistics from typing import Callable, override -from .exceptions import BenchmarkNotFound, BenchmarksNotFound - +from .exceptions import BenchmarkNotFound, BenchmarksNotFound, BenchmarksWithSameNameAlreadyExists FuncForBenchmark = Callable[[], None] MILLISECONDS_IN_SECONDS = 1000 @@ -35,6 +34,8 @@ class BenchmarkResult: @dataclass(frozen=True, slots=True) class BenchmarkGroupResult: type_: str + iterations: int + is_gc_disabled: bool benchmark_results: list[BenchmarkResult] @@ -107,8 +108,11 @@ class Benchmarks: name=func.__name__, description=description or f'description for {func.__name__} with type {type_}', ) - self._benchmarks.append(benchmark) + if self._benchmarks_paired_by_name.get(func.__name__): + raise BenchmarksWithSameNameAlreadyExists(func.__name__) + self._benchmarks_paired_by_name[func.__name__] = benchmark + self._benchmarks.append(benchmark) self._benchmarks_grouped_by_type.setdefault(type_, []).append(benchmark) return func return decorator @@ -145,6 +149,8 @@ class Benchmarks: return BenchmarkGroupResult( type_=type_, + iterations=iterations, + is_gc_disabled=is_gc_disabled, benchmark_results=benchmark_results ) diff --git a/metrics/benchmarks/most_similar_command.py b/metrics/benchmarks/most_similar_command.py index 7455999..bcc1d3b 100644 --- a/metrics/benchmarks/most_similar_command.py +++ b/metrics/benchmarks/most_similar_command.py @@ -1,14 +1,11 @@ __all__ = [ "benchmark_few_commands", - "benchmark_many_commands", + "benchmark_many_commands_most_similar", "benchmark_many_aliases", "benchmark_partial_match", "benchmark_extreme_commands" ] -import io -from contextlib import redirect_stdout - from argenta import App from argenta.command.models import Command from argenta.response import Response @@ -39,7 +36,7 @@ def benchmark_few_commands() -> None: @benchmarks.register(type_="most_similar_command", description="Many commands (50 commands, no match)") -def benchmark_many_commands() -> None: +def benchmark_many_commands_most_similar() -> None: app = setup_app_with_commands(50) app._most_similar_command("unknown") diff --git a/metrics/handlers.py b/metrics/handlers.py index 6ab2484..108bb65 100644 --- a/metrics/handlers.py +++ b/metrics/handlers.py @@ -11,7 +11,6 @@ from argenta.response import Response from argenta.router import Router from .benchmarks.core.models import BenchmarkGroupResult from .benchmarks.entity import benchmarks as registered_benchmarks -from .utils import get_kernel_version, get_gpu_info console = Console() router = Router(title="Metrics commands:") @@ -43,22 +42,18 @@ def all_print_handler(_: Response) -> None: type_grouped_benchmarks: list[BenchmarkGroupResult] = registered_benchmarks.run_benchmarks_grouped_by_type() for results in type_grouped_benchmarks: - header_text = Text(f"TYPE: {results.type_.upper()}", style="bold magenta") + header_text = Text(f"TYPE: {results.type_.upper()} ; ITERATIONS: {results.iterations} ; ALL TIME IN MS", style="bold magenta") console.print(Panel(header_text, expand=False, border_style="magenta")) table = Table(show_header=True, header_style="bold cyan", border_style="blue", show_lines=True) - table.add_column("Name", style="green") table.add_column("Description", style="dim") - table.add_column("Iterations", justify="right") - table.add_column("Avg Time (ms)", justify="right", style="bold yellow") - table.add_column("Median Time (ms)", justify="right", style="bold yellow") - table.add_column("Stdev (ms)", justify="right", style="bold yellow") + table.add_column("Avg Time", justify="right", style="bold yellow") + table.add_column("Median Time", justify="right", style="bold yellow") + table.add_column("Stdev", justify="right", style="bold yellow") for benchmark in results.benchmark_results: table.add_row( - benchmark.name, benchmark.description, - str(benchmark.iterations), str(benchmark.avg_time), str(benchmark.median_time), str(benchmark.std_dev), diff --git a/metrics/services/__init__.py b/metrics/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metrics/services/report_generator.py b/metrics/services/report_generator.py new file mode 100644 index 0000000..e69de29 diff --git a/metrics/services/system_info_reader.py b/metrics/services/system_info_reader.py new file mode 100644 index 0000000..47d17e6 --- /dev/null +++ b/metrics/services/system_info_reader.py @@ -0,0 +1,114 @@ +__all__ = [ + "SystemInfo", + "SystemInfoGetter", + "get_system_info" +] + +from dataclasses import dataclass +import platform +import sys +from typing import Protocol + +import cpuinfo +import psutil + + +@dataclass(frozen=True, slots=True) +class SystemInfo: + os_info: OSInfo + cpu_info: CPUInfo + memory_info: MemoryInfo + python_version: str + python_implementation: str + + +@dataclass(frozen=True, slots=True) +class OSInfo: + os_name: str + kernel_version: str + + +@dataclass(frozen=True, slots=True) +class CPUInfo: + cpu_name: str + cpu_architecture: str + cpu_physical_cores: int + cpu_logical_cores: int + cpu_max_frequency: float + cpu_base_frequency: float + + +@dataclass(frozen=True, slots=True) +class MemoryInfo: + total_ram: float # in GB + available_ram: float # in GB + l1_cache: float + l2_cache: float + l3_cache: float + + +@dataclass(frozen=True, slots=True) +class PythonInfo: + python_version: str + python_implementation: str + python_compiler: str + + +class SystemInfoGetter(Protocol): + def __call__(self) -> SystemInfo: + raise NotImplementedError + + +def get_system_info() -> SystemInfo: + os_info = get_os_info() + os_name = os_info.os_name + os_kernel_version = os_info.kernel_version + + cpu_info = cpuinfo.get_cpu_info() + cpu_architecture = cpu_info["arch"] + cpu_name = cpu_info["brand_raw"] + + gpu_name = get_gpu_name() + + total_ram = psutil.virtual_memory().total / (1024 ** 3) + + python_version = platform.python_version() + python_implementation = platform.python_implementation() + + return SystemInfo( + os_name=os_name, + kernel_version=os_kernel_version, + cpu_architecture=cpu_architecture, + cpu_name=cpu_name, + gpu_name=gpu_name, + total_ram=total_ram, + python_version=python_version, + python_implementation=python_implementation, + ) + +def get_os_info() -> OSInfo: + system = platform.system() + + if system == "Windows": + ver = sys.getwindowsversion() + kernel_version = f"{ver.major}.{ver.minor}.{ver.build}" + + if ver.build >= 22000: + product_name = "Windows 11" + else: + product_name = "Windows 10" + + return OSInfo( + os_name=product_name, + kernel_version=kernel_version, + ) + elif system == "Darwin": + return OSInfo( + kernel_version=platform.release(), + os_name=f"macOS {platform.mac_ver()[0]}" + ) + else: + return OSInfo( + kernel_version=platform.release(), + os_name=platform.system() + ) diff --git a/metrics/utils.py b/metrics/utils.py deleted file mode 100644 index ba951e3..0000000 --- a/metrics/utils.py +++ /dev/null @@ -1,63 +0,0 @@ -__all__ = [ - "get_kernel_version", - "get_gpu_info" -] - -import platform -import sys - -import pynvml - - -def get_kernel_version() -> dict[str, str]: - system = platform.system() - - if system == "Windows": - ver = sys.getwindowsversion() - kernel_version = f"{ver.major}.{ver.minor}.{ver.build}" - - if ver.build >= 22000: - product_name = "Windows 11" - else: - product_name = "Windows 10" - - return { - 'kernel_version': kernel_version, - 'product_name': product_name - } - - elif system == "Linux": - return { - 'kernel_version': platform.release(), - 'product_name': platform.system() - } - - elif system == "Darwin": - return { - 'kernel_version': platform.release(), - 'product_name': f"macOS {platform.mac_ver()[0]}" - } - else: - return { - 'kernel_version': platform.release(), - 'product_name': platform.system(), - } - -def get_gpu_info() -> str: - try: - pynvml.nvmlInit() - device_count = pynvml.nvmlDeviceGetCount() - if device_count == 0: - return "N/A" - - handle = pynvml.nvmlDeviceGetHandleByIndex(0) - name = pynvml.nvmlDeviceGetName(handle) - - if isinstance(name, bytes): - name = name.decode("utf-8") - - pynvml.nvmlShutdown() - return name - except pynvml.NVMLError: - return "N/A" - diff --git a/pyproject.toml b/pyproject.toml index d92d3b9..ed0413d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ tests = [ "pytest-mock>=3.15.1", ] metrics = [ - "nvidia-ml-py>=13.590.44", + "psutil>=7.2.1", "py-cpuinfo>=9.0.0", ] diff --git a/uv.lock b/uv.lock index 5fceb6e..63462f5 100644 --- a/uv.lock +++ b/uv.lock @@ -62,7 +62,7 @@ linters = [ { name = "wemake-python-styleguide" }, ] metrics = [ - { name = "nvidia-ml-py" }, + { name = "psutil" }, { name = "py-cpuinfo" }, ] tests = [ @@ -97,7 +97,7 @@ linters = [ { name = "wemake-python-styleguide", specifier = ">=0.17.0" }, ] metrics = [ - { name = "nvidia-ml-py", specifier = ">=13.590.44" }, + { name = "psutil", specifier = ">=7.2.1" }, { name = "py-cpuinfo", specifier = ">=9.0.0" }, ] tests = [ @@ -561,15 +561,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] -[[package]] -name = "nvidia-ml-py" -version = "13.590.44" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/23/3871537f204aee823c574ba25cbeb08cae779979d4d43c01adddda00bab9/nvidia_ml_py-13.590.44.tar.gz", hash = "sha256:b358c7614b0fdeea4b95f046f1c90123bfe25d148ab93bb1c00248b834703373", size = 49737, upload-time = "2025-12-08T14:41:10.872Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/47/4c822bd37a008e72fd5a0eae33524ae3ac97b13f7030f63bae1728b8957e/nvidia_ml_py-13.590.44-py3-none-any.whl", hash = "sha256:18feb54eca7d0e3cdc8d1a040a771eda72d9ec3148e5443087970dbfd7377ecc", size = 50683, upload-time = "2025-12-08T14:41:09.597Z" }, -] - [[package]] name = "packaging" version = "25.0" @@ -606,6 +597,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "psutil" +version = "7.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/cb/09e5184fb5fc0358d110fc3ca7f6b1d033800734d34cac10f4136cfac10e/psutil-7.2.1.tar.gz", hash = "sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3", size = 490253, upload-time = "2025-12-29T08:26:00.169Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/8e/f0c242053a368c2aa89584ecd1b054a18683f13d6e5a318fc9ec36582c94/psutil-7.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d", size = 129624, upload-time = "2025-12-29T08:26:04.255Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a58a4968f8990617decee234258a2b4fc7cd9e35668387646c1963e69f26/psutil-7.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49", size = 130132, upload-time = "2025-12-29T08:26:06.228Z" }, + { url = "https://files.pythonhosted.org/packages/db/6d/ed44901e830739af5f72a85fa7ec5ff1edea7f81bfbf4875e409007149bd/psutil-7.2.1-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc", size = 180612, upload-time = "2025-12-29T08:26:08.276Z" }, + { url = "https://files.pythonhosted.org/packages/c7/65/b628f8459bca4efbfae50d4bf3feaab803de9a160b9d5f3bd9295a33f0c2/psutil-7.2.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf", size = 183201, upload-time = "2025-12-29T08:26:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/fb/23/851cadc9764edcc18f0effe7d0bf69f727d4cf2442deb4a9f78d4e4f30f2/psutil-7.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f", size = 139081, upload-time = "2025-12-29T08:26:12.483Z" }, + { url = "https://files.pythonhosted.org/packages/59/82/d63e8494ec5758029f31c6cb06d7d161175d8281e91d011a4a441c8a43b5/psutil-7.2.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672", size = 134767, upload-time = "2025-12-29T08:26:14.528Z" }, + { url = "https://files.pythonhosted.org/packages/05/c2/5fb764bd61e40e1fe756a44bd4c21827228394c17414ade348e28f83cd79/psutil-7.2.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679", size = 129716, upload-time = "2025-12-29T08:26:16.017Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d2/935039c20e06f615d9ca6ca0ab756cf8408a19d298ffaa08666bc18dc805/psutil-7.2.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f", size = 130133, upload-time = "2025-12-29T08:26:18.009Z" }, + { url = "https://files.pythonhosted.org/packages/77/69/19f1eb0e01d24c2b3eacbc2f78d3b5add8a89bf0bb69465bc8d563cc33de/psutil-7.2.1-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129", size = 181518, upload-time = "2025-12-29T08:26:20.241Z" }, + { url = "https://files.pythonhosted.org/packages/e1/6d/7e18b1b4fa13ad370787626c95887b027656ad4829c156bb6569d02f3262/psutil-7.2.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a", size = 184348, upload-time = "2025-12-29T08:26:22.215Z" }, + { url = "https://files.pythonhosted.org/packages/98/60/1672114392dd879586d60dd97896325df47d9a130ac7401318005aab28ec/psutil-7.2.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79", size = 140400, upload-time = "2025-12-29T08:26:23.993Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7b/d0e9d4513c46e46897b46bcfc410d51fc65735837ea57a25170f298326e6/psutil-7.2.1-cp314-cp314t-win_arm64.whl", hash = "sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266", size = 135430, upload-time = "2025-12-29T08:26:25.999Z" }, + { url = "https://files.pythonhosted.org/packages/c5/cf/5180eb8c8bdf6a503c6919f1da28328bd1e6b3b1b5b9d5b01ae64f019616/psutil-7.2.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42", size = 128137, upload-time = "2025-12-29T08:26:27.759Z" }, + { url = "https://files.pythonhosted.org/packages/c5/2c/78e4a789306a92ade5000da4f5de3255202c534acdadc3aac7b5458fadef/psutil-7.2.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1", size = 128947, upload-time = "2025-12-29T08:26:29.548Z" }, + { url = "https://files.pythonhosted.org/packages/29/f8/40e01c350ad9a2b3cb4e6adbcc8a83b17ee50dd5792102b6142385937db5/psutil-7.2.1-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8", size = 154694, upload-time = "2025-12-29T08:26:32.147Z" }, + { url = "https://files.pythonhosted.org/packages/06/e4/b751cdf839c011a9714a783f120e6a86b7494eb70044d7d81a25a5cd295f/psutil-7.2.1-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6", size = 156136, upload-time = "2025-12-29T08:26:34.079Z" }, + { url = "https://files.pythonhosted.org/packages/44/ad/bbf6595a8134ee1e94a4487af3f132cef7fce43aef4a93b49912a48c3af7/psutil-7.2.1-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8", size = 148108, upload-time = "2025-12-29T08:26:36.225Z" }, + { url = "https://files.pythonhosted.org/packages/1c/15/dd6fd869753ce82ff64dcbc18356093471a5a5adf4f77ed1f805d473d859/psutil-7.2.1-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67", size = 147402, upload-time = "2025-12-29T08:26:39.21Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/d9317542e3f2b180c4306e3f45d3c922d7e86d8ce39f941bb9e2e9d8599e/psutil-7.2.1-cp37-abi3-win_amd64.whl", hash = "sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17", size = 136938, upload-time = "2025-12-29T08:26:41.036Z" }, + { url = "https://files.pythonhosted.org/packages/3e/73/2ce007f4198c80fcf2cb24c169884f833fe93fbc03d55d302627b094ee91/psutil-7.2.1-cp37-abi3-win_arm64.whl", hash = "sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442", size = 133836, upload-time = "2025-12-29T08:26:43.086Z" }, +] + [[package]] name = "py-cpuinfo" version = "9.0.0"