mirror of
https://github.com/koloideal/Argenta.git
synced 2026-06-10 18:15:28 +03:00
bbbbbbenchh
This commit is contained in:
@@ -1,4 +0,0 @@
|
||||
from .pre_cycle_setup import *
|
||||
from .validate_routers_for_collisions import *
|
||||
from .most_similar_command import *
|
||||
from .finds_appropriate_handler import *
|
||||
@@ -0,0 +1,14 @@
|
||||
class BenchmarkNotFound(Exception):
|
||||
def __init__(self, benchmark_name: str):
|
||||
self.benchmark_name = benchmark_name
|
||||
|
||||
def __str__(self):
|
||||
return f"Benchmark with name '{self.benchmark_name}' not found"
|
||||
|
||||
|
||||
class BenchmarksNotFound(Exception):
|
||||
def __init__(self, type_: str):
|
||||
self.type_ = type_
|
||||
|
||||
def __str__(self):
|
||||
return f"Benchmarks with type '{self.type_}' not found"
|
||||
@@ -1,81 +0,0 @@
|
||||
__all__ = [
|
||||
"benchmark_simple_command",
|
||||
"benchmark_command_with_flags",
|
||||
"benchmark_many_commands",
|
||||
"benchmark_command_with_many_flags",
|
||||
"benchmark_extreme_router"
|
||||
]
|
||||
|
||||
from argenta.command.models import Command, InputCommand
|
||||
from argenta.command import Flag, Flags
|
||||
from argenta.response import Response
|
||||
from argenta.router import Router
|
||||
|
||||
from ..models import benchmark
|
||||
from ..utils import get_time_of_finds_appropriate_handler
|
||||
|
||||
|
||||
@benchmark(type_="finds_appropriate_handler", description="Simple command (no flags)")
|
||||
def benchmark_simple_command() -> float:
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('test'))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
input_cmd = InputCommand.parse('test')
|
||||
return get_time_of_finds_appropriate_handler(router, input_cmd)
|
||||
|
||||
|
||||
@benchmark(type_="finds_appropriate_handler", description="Command with flags (3 flags)")
|
||||
def benchmark_command_with_flags() -> float:
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('test', flags=Flags([Flag('a'), Flag('b'), Flag('c')])))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
input_cmd = InputCommand.parse('test -a -b -c')
|
||||
return get_time_of_finds_appropriate_handler(router, input_cmd)
|
||||
|
||||
|
||||
@benchmark(type_="finds_appropriate_handler", description="Many commands (50 commands)")
|
||||
def benchmark_many_commands() -> float:
|
||||
router = Router()
|
||||
|
||||
for i in range(50):
|
||||
@router.command(Command(f'cmd{i}'))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
input_cmd = InputCommand.parse('cmd25')
|
||||
return get_time_of_finds_appropriate_handler(router, input_cmd)
|
||||
|
||||
|
||||
@benchmark(type_="finds_appropriate_handler", description="Command with many flags (20 flags)")
|
||||
def benchmark_command_with_many_flags() -> float:
|
||||
router = Router()
|
||||
|
||||
flags = Flags([Flag(f'flag{i}') for i in range(20)])
|
||||
|
||||
@router.command(Command('test', flags=flags))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
input_cmd = InputCommand.parse('test ' + ' '.join(f'-flag{i}' for i in range(10)))
|
||||
return get_time_of_finds_appropriate_handler(router, input_cmd)
|
||||
|
||||
|
||||
@benchmark(type_="finds_appropriate_handler", description="Extreme (100 commands, 10 flags each)")
|
||||
def benchmark_extreme_router() -> float:
|
||||
router = Router()
|
||||
|
||||
for i in range(100):
|
||||
flags = Flags([Flag(f'f{i}_{j}') for j in range(10)])
|
||||
|
||||
@router.command(Command(f'cmd{i}', flags=flags))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
input_cmd = InputCommand.parse('cmd50 -f50_0 -f50_1 -f50_2')
|
||||
return get_time_of_finds_appropriate_handler(router, input_cmd)
|
||||
@@ -0,0 +1,161 @@
|
||||
__all__ = [
|
||||
"Benchmark",
|
||||
"Benchmarks",
|
||||
"BenchmarkResult",
|
||||
"BenchmarkGroupResult"
|
||||
]
|
||||
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
from dataclasses import dataclass
|
||||
import time
|
||||
import gc
|
||||
import statistics
|
||||
from typing import Callable, override
|
||||
|
||||
from .exceptions import BenchmarkNotFound, BenchmarksNotFound
|
||||
|
||||
|
||||
FuncForBenchmark = Callable[[], None]
|
||||
MILLISECONDS_IN_SECONDS = 1000
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BenchmarkResult:
|
||||
type_: str
|
||||
name: str
|
||||
description: str
|
||||
iterations: int
|
||||
is_gc_disabled: bool
|
||||
avg_time: float
|
||||
median_time: float
|
||||
std_dev: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BenchmarkGroupResult:
|
||||
type_: str
|
||||
benchmark_results: list[BenchmarkResult]
|
||||
|
||||
|
||||
class Benchmark:
|
||||
def __init__(
|
||||
self,
|
||||
func: FuncForBenchmark,
|
||||
*,
|
||||
type_: str,
|
||||
name: str,
|
||||
description: str
|
||||
) -> None:
|
||||
self.func = func
|
||||
self.type_ = type_
|
||||
self.name = name
|
||||
self.description = description
|
||||
|
||||
def single_run(self, is_gc_disabled: bool = False) -> float:
|
||||
if is_gc_disabled:
|
||||
was_gc_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
|
||||
with redirect_stdout(io.StringIO()):
|
||||
start = time.perf_counter()
|
||||
self.func()
|
||||
end = time.perf_counter()
|
||||
|
||||
if was_gc_enabled:
|
||||
gc.enable()
|
||||
gc.collect()
|
||||
|
||||
return (end - start) * MILLISECONDS_IN_SECONDS
|
||||
else:
|
||||
with redirect_stdout(io.StringIO()):
|
||||
start = time.perf_counter()
|
||||
self.func()
|
||||
end = time.perf_counter()
|
||||
return (end - start) * MILLISECONDS_IN_SECONDS
|
||||
|
||||
def multiple_runs(self, iterations: int, is_gc_disabled: bool = False) -> tuple[float, ...]:
|
||||
run_attempts: list[float] = []
|
||||
for _ in range(iterations):
|
||||
run_attempts.append(self.single_run(is_gc_disabled))
|
||||
return tuple(run_attempts)
|
||||
|
||||
@override
|
||||
def __repr__(self) -> str:
|
||||
return f'Benchmark<{self.type_=}, {self.name=}, {self.description=}>'
|
||||
|
||||
@override
|
||||
def __str__(self) -> str:
|
||||
return f'benchmark {self.name} with type {self.type_}'
|
||||
|
||||
|
||||
class Benchmarks:
|
||||
def __init__(self, *benchmarks: Benchmark) -> None:
|
||||
self._benchmarks: list[Benchmark] = list(benchmarks)
|
||||
self._benchmarks_grouped_by_type: dict[str, list[Benchmark]] = {}
|
||||
self._benchmarks_paired_by_name: dict[str, Benchmark] = {}
|
||||
|
||||
def register(
|
||||
self,
|
||||
type_: str,
|
||||
description: str = ""
|
||||
) -> Callable[[FuncForBenchmark], FuncForBenchmark]:
|
||||
def decorator(func: FuncForBenchmark) -> FuncForBenchmark:
|
||||
benchmark = Benchmark(
|
||||
func,
|
||||
type_=type_,
|
||||
name=func.__name__,
|
||||
description=description or f'description for {func.__name__} with type {type_}',
|
||||
)
|
||||
self._benchmarks.append(benchmark)
|
||||
self._benchmarks_paired_by_name[func.__name__] = benchmark
|
||||
self._benchmarks_grouped_by_type.setdefault(type_, []).append(benchmark)
|
||||
return func
|
||||
return decorator
|
||||
|
||||
def run_benchmark_by_name(self, name: str, iterations: int = 100, is_gc_disables: bool = False) -> BenchmarkResult:
|
||||
benchmark = self.get_benchmark_by_name(name)
|
||||
if not benchmark:
|
||||
raise BenchmarkNotFound(name)
|
||||
run_attempts: tuple[float, ...] = benchmark.multiple_runs(iterations, is_gc_disables)
|
||||
|
||||
avg = round(statistics.mean(run_attempts), 4)
|
||||
median = round(statistics.median(run_attempts), 4)
|
||||
std_dev = round(statistics.stdev(run_attempts) if len(run_attempts) > 1 else 0, 4)
|
||||
|
||||
return BenchmarkResult(
|
||||
type_=benchmark.type_,
|
||||
name=benchmark.name,
|
||||
description=benchmark.description,
|
||||
iterations=iterations,
|
||||
is_gc_disabled=is_gc_disables,
|
||||
avg_time=avg,
|
||||
median_time=median,
|
||||
std_dev=std_dev
|
||||
)
|
||||
|
||||
def run_benchmarks_by_type(self, type_: str, iterations: int = 100, is_gc_disabled: bool = False) -> BenchmarkGroupResult:
|
||||
benchmarks = self.get_benchmarks_by_type(type_)
|
||||
if not benchmarks:
|
||||
raise BenchmarksNotFound(type_)
|
||||
benchmark_results: list[BenchmarkResult] = []
|
||||
|
||||
for benchmark in benchmarks:
|
||||
benchmark_results.append(self.run_benchmark_by_name(benchmark.name, iterations, is_gc_disabled))
|
||||
|
||||
return BenchmarkGroupResult(
|
||||
type_=type_,
|
||||
benchmark_results=benchmark_results
|
||||
)
|
||||
|
||||
def run_benchmarks_grouped_by_type(self) -> list[BenchmarkGroupResult]:
|
||||
results: list[BenchmarkGroupResult] = []
|
||||
for type_, benchmarks in self._benchmarks_grouped_by_type.items():
|
||||
results.append(self.run_benchmarks_by_type(type_))
|
||||
return results
|
||||
|
||||
def get_benchmarks_by_type(self, type_: str) -> list[Benchmark]:
|
||||
return self._benchmarks_grouped_by_type.get(type_, [])
|
||||
|
||||
def get_benchmark_by_name(self, name: str) -> Benchmark | None:
|
||||
return self._benchmarks_paired_by_name.get(name)
|
||||
@@ -1,65 +0,0 @@
|
||||
__all__ = [
|
||||
"benchmark_few_commands",
|
||||
"benchmark_many_commands",
|
||||
"benchmark_many_aliases",
|
||||
"benchmark_partial_match",
|
||||
"benchmark_extreme_commands"
|
||||
]
|
||||
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
from argenta import App
|
||||
from argenta.command.models import Command
|
||||
from argenta.response import Response
|
||||
from argenta.router import Router
|
||||
|
||||
from ..models import benchmark
|
||||
from ..utils import get_time_of_most_similar_command
|
||||
|
||||
|
||||
def setup_app_with_commands(command_count: int, aliases_per_command: int = 0) -> App:
|
||||
app = App(override_system_messages=True)
|
||||
router = Router()
|
||||
|
||||
for i in range(command_count):
|
||||
aliases = {f'alias{i}_{j}' for j in range(aliases_per_command)} if aliases_per_command else set()
|
||||
|
||||
@router.command(Command(f'command{i}', aliases=aliases))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
with redirect_stdout(io.StringIO()):
|
||||
app._pre_cycle_setup() # pyright: ignore[reportPrivateUsage]
|
||||
return app
|
||||
|
||||
|
||||
@benchmark(type_="most_similar_command", description="Few commands (10 commands, no match)")
|
||||
def benchmark_few_commands() -> float:
|
||||
app = setup_app_with_commands(10)
|
||||
return get_time_of_most_similar_command(app, "unknown")
|
||||
|
||||
|
||||
@benchmark(type_="most_similar_command", description="Many commands (50 commands, no match)")
|
||||
def benchmark_many_commands() -> float:
|
||||
app = setup_app_with_commands(50)
|
||||
return get_time_of_most_similar_command(app, "unknown")
|
||||
|
||||
|
||||
@benchmark(type_="most_similar_command", description="Many aliases (20 commands, 10 aliases each)")
|
||||
def benchmark_many_aliases() -> float:
|
||||
app = setup_app_with_commands(20, aliases_per_command=10)
|
||||
return get_time_of_most_similar_command(app, "unknown")
|
||||
|
||||
|
||||
@benchmark(type_="most_similar_command", description="Partial match (50 commands, prefix match)")
|
||||
def benchmark_partial_match() -> float:
|
||||
app = setup_app_with_commands(50)
|
||||
return get_time_of_most_similar_command(app, "comm")
|
||||
|
||||
|
||||
@benchmark(type_="most_similar_command", description="Extreme (100 commands, 20 aliases each)")
|
||||
def benchmark_extreme_commands() -> float:
|
||||
app = setup_app_with_commands(100, aliases_per_command=20)
|
||||
return get_time_of_most_similar_command(app, "comm")
|
||||
@@ -1,125 +0,0 @@
|
||||
__all__ = [
|
||||
"benchmark_no_aliases",
|
||||
"benchmark_many_aliases",
|
||||
"benchmark_few_aliases",
|
||||
"benchmark_extreme_aliases",
|
||||
"benchmark_very_many_aliases"
|
||||
]
|
||||
|
||||
from argenta import App
|
||||
from argenta.command.models import Command
|
||||
from argenta.response import Response
|
||||
from argenta.router import Router
|
||||
|
||||
from ..models import benchmark
|
||||
from ..utils import get_time_of_pre_cycle_setup
|
||||
|
||||
|
||||
@benchmark(type_="pre_cycle_setup", description="With no aliases")
|
||||
def benchmark_no_aliases() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('command1'))
|
||||
def handler1(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command2'))
|
||||
def handler2(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command3'))
|
||||
def handler3(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
execution_time = get_time_of_pre_cycle_setup(app)
|
||||
return execution_time
|
||||
|
||||
|
||||
@benchmark(type_="pre_cycle_setup", description="With few aliases (6 total)")
|
||||
def benchmark_few_aliases() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('command1', aliases={'c1', 'cmd1'}))
|
||||
def handler1(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command2', aliases={'c2', 'cmd2'}))
|
||||
def handler2(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command3', aliases={'c3', 'cmd3'}))
|
||||
def handler3(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
execution_time = get_time_of_pre_cycle_setup(app)
|
||||
return execution_time
|
||||
|
||||
|
||||
@benchmark(type_="pre_cycle_setup", description="With many aliases (15 total)")
|
||||
def benchmark_many_aliases() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('command1', aliases={'c1', 'cmd1', 'com1', 'first', 'one'}))
|
||||
def handler1(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command2', aliases={'c2', 'cmd2', 'com2', 'second', 'two'}))
|
||||
def handler2(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command3', aliases={'c3', 'cmd3', 'com3', 'third', 'three'}))
|
||||
def handler3(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
execution_time = get_time_of_pre_cycle_setup(app)
|
||||
return execution_time
|
||||
|
||||
|
||||
@benchmark(type_="pre_cycle_setup", description="With very many aliases (60 total)")
|
||||
def benchmark_very_many_aliases() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('command1', aliases={f'alias1_{i}' for i in range(20)}))
|
||||
def handler1(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command2', aliases={f'alias2_{i}' for i in range(20)}))
|
||||
def handler2(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command3', aliases={f'alias3_{i}' for i in range(20)}))
|
||||
def handler3(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
execution_time = get_time_of_pre_cycle_setup(app)
|
||||
return execution_time
|
||||
|
||||
|
||||
@benchmark(type_="pre_cycle_setup", description="With extreme aliases (300 total)")
|
||||
def benchmark_extreme_aliases() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
router = Router()
|
||||
|
||||
@router.command(Command('command1', aliases={f'alias1_{i}' for i in range(100)}))
|
||||
def handler1(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command2', aliases={f'alias2_{i}' for i in range(100)}))
|
||||
def handler2(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
@router.command(Command('command3', aliases={f'alias3_{i}' for i in range(100)}))
|
||||
def handler3(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
execution_time = get_time_of_pre_cycle_setup(app)
|
||||
return execution_time
|
||||
@@ -1,99 +0,0 @@
|
||||
__all__ = [
|
||||
"benchmark_few_routers",
|
||||
"benchmark_many_routers",
|
||||
"benchmark_many_commands_per_router",
|
||||
"benchmark_many_aliases_per_command",
|
||||
"benchmark_extreme_routers"
|
||||
]
|
||||
|
||||
from argenta import App
|
||||
from argenta.command.models import Command
|
||||
from argenta.response import Response
|
||||
from argenta.router import Router
|
||||
|
||||
from ..utils import get_time_of_validate_routers_for_collisions
|
||||
|
||||
from ..models import benchmark
|
||||
|
||||
|
||||
@benchmark(type_="validate_routers_for_collisions", description="With few routers (3 routers, 1 command each)")
|
||||
def benchmark_few_routers() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
|
||||
for i in range(3):
|
||||
router = Router()
|
||||
|
||||
@router.command(Command(f'cmd{i}'))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
return get_time_of_validate_routers_for_collisions(app)
|
||||
|
||||
|
||||
@benchmark(type_="validate_routers_for_collisions", description="With many routers (10 routers, 1 command each)")
|
||||
def benchmark_many_routers() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
|
||||
for i in range(10):
|
||||
router = Router()
|
||||
|
||||
@router.command(Command(f'cmd{i}'))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
return get_time_of_validate_routers_for_collisions(app)
|
||||
|
||||
|
||||
@benchmark(type_="validate_routers_for_collisions", description="With many commands per router (3 routers, 10 commands each)")
|
||||
def benchmark_many_commands_per_router() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
|
||||
for i in range(3):
|
||||
router = Router()
|
||||
|
||||
for j in range(10):
|
||||
@router.command(Command(f'cmd{i}_{j}'))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
return get_time_of_validate_routers_for_collisions(app)
|
||||
|
||||
|
||||
@benchmark(type_="validate_routers_for_collisions", description="With many aliases (3 routers, 5 commands, 10 aliases each)")
|
||||
def benchmark_many_aliases_per_command() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
|
||||
for i in range(3):
|
||||
router = Router()
|
||||
|
||||
for j in range(5):
|
||||
@router.command(Command(f'cmd{i}_{j}', aliases={f'alias{i}_{j}_{k}' for k in range(10)}))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
return get_time_of_validate_routers_for_collisions(app)
|
||||
|
||||
|
||||
@benchmark(type_="validate_routers_for_collisions", description="Extreme (20 routers, 10 commands, 20 aliases each)")
|
||||
def benchmark_extreme_routers() -> float:
|
||||
app = App(override_system_messages=True)
|
||||
|
||||
for i in range(20):
|
||||
router = Router()
|
||||
|
||||
for j in range(10):
|
||||
@router.command(Command(f'cmd{i}_{j}', aliases={f'alias{i}_{j}_{k}' for k in range(20)}))
|
||||
def handler(_res: Response) -> None:
|
||||
pass
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
return get_time_of_validate_routers_for_collisions(app)
|
||||
Reference in New Issue
Block a user