diff --git a/docs/index.rst b/docs/index.rst index 0f3d9ec..2033b34 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -64,6 +64,7 @@ Argenta предназначена для создания приложений, root/contributing root/code_of_conduct + root/metrics .. toctree:: :hidden: diff --git a/docs/root/metrics.rst b/docs/root/metrics.rst new file mode 100644 index 0000000..24dcead --- /dev/null +++ b/docs/root/metrics.rst @@ -0,0 +1,181 @@ +Метрики +======= + +Система метрик ``Argenta`` предоставляет инструменты для измерения производительности ключевых компонентов библиотеки. Это позволяет отслеживать регрессию/прогрессию производительности между релизами и оптимизировать критические участки кода. + +----- + +Запуск метрик +------------- + +Для работы с метриками необходимо склонировать репозиторий и установить зависимости: + +.. code-block:: bash + + git clone https://github.com/koloideal/Argenta.git + cd Argenta + uv sync --group metrics + +Запуск системы метрик: + +.. code-block:: bash + + python -m metrics + +После запуска откроется интерактивная сессия с доступными командами для работы с бенчмарками. + +----- + +Доступные команды +----------------- + +run-all +~~~~~~~ + +Запускает все зарегистрированные бенчмарки и выводит результаты в виде таблиц. + +**Синтаксис:** + +.. code-block:: shell + + run-all [--without-gc] [--without-system-info] + +**Флаги:** + +- ``--without-gc`` — отключает сборщик мусора во время выполнения бенчмарков для более стабильных результатов +- ``--without-system-info`` — скрывает информацию о системе в выводе + +----- + +list-types +~~~~~~~~~~ + +Выводит список всех доступных типов бенчмарков с количеством тестов в каждой категории. + +**Синтаксис:** + +.. code-block:: shell + + list-types + +**Пример вывода:** + +.. code-block:: text + + Available benchmark types: + + • flag_validation (9 benchmarks) + • input_command_parse (7 benchmarks) + • finds_appropriate_handler (5 benchmarks) + +----- + +run-type +~~~~~~~~ + +Запускает бенчмарки определённого типа. + +**Синтаксис:** + +.. code-block:: shell + + run-type --type [--without-gc] [--without-system-info] + +**Флаги:** + +- ``--type`` — тип бенчмарков для запуска (обязательный) +- ``--without-gc`` — отключает сборщик мусора +- ``--without-system-info`` — скрывает информацию о системе + +----- + +diagrams-generate +~~~~~~~~~~~~~~~~~ + +Генерирует визуальные диаграммы сравнения производительности для всех бенчмарков. + +**Синтаксис:** + +.. code-block:: shell + + diagrams-generate [--iterations ] [--without-gc] + +**Флаги:** + +- ``--iterations`` — количество итераций для каждого бенчмарка (по умолчанию 100) +- ``--without-gc`` — отключает сборщик мусора + +Диаграммы сохраняются в директорию ``metrics/reports/diagrams//``. + +----- + +release-generate +~~~~~~~~~~~~~~~~ + +Генерирует полный отчёт о производительности для текущей версии библиотеки. Используется при подготовке релизов. + +**Синтаксис:** + +.. code-block:: shell + + release-generate + +Команда автоматически: + +1. Определяет текущую версию библиотеки +2. Запускает все бенчмарки с 1000 итераций и отключённым GC +3. Генерирует JSON-отчёты и диаграммы сравнения +4. Сохраняет результаты в ``metrics/reports/releases//`` + +----- + +Интерпретация результатов +------------------------- + +Результаты бенчмарков включают следующие метрики: + +**Среднее время (mean)** + Среднее время выполнения операции. Основная метрика для сравнения производительности. + +**Медиана (median)** + Медианное значение времени выполнения. Менее чувствительна к выбросам, чем среднее. + +**Стандартное отклонение (std)** + Показывает стабильность измерений. Меньшее значение означает более предсказуемую производительность. + +----- + +Рекомендации по использованию +------------------------------ + +**Для оптимизации** + Используйте ``run-type`` для фокусировки на конкретной области и ``--without-gc`` для более точных измерений. + +**Для визуализации** + Команда ``diagrams-generate`` создаёт наглядные графики, удобные для презентаций и документации. + +**Для стабильных результатов** + Закройте ресурсоёмкие приложения, используйте флаг ``--without-gc`` и увеличивайте количество итераций через ``--iterations``. + +----- + +Добавление новых бенчмарков +---------------------------- + +Вы можете реализовать свои бенчмарки для тестирования специфичных юнитов библиотеки. Новые бенчмарки добавляются через декоратор ``@benchmarks.register``: + +.. code-block:: python + + from metrics.benchmarks.entity import benchmarks + + @benchmarks.register( + type_="my_category", + description="Description of what is being measured" + ) + def benchmark_my_operation() -> None: + # Код, производительность которого измеряется + pass + +.. important:: + + Бенчмарк должен быть импортирован в ``metrics/benchmarks/__init__.py`` для автоматической регистрации. diff --git a/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler.json b/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler.json index 8a7ab02..e5ee8f1 100644 --- a/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler.json +++ b/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler.json @@ -6,37 +6,37 @@ { "name": "benchmark_simple_command", "description": "Simple command (no flags)", - "avg_time": 0.0354, - "median_time": 0.0345, - "std_dev": 0.0056 + "avg_time": 0.036, + "median_time": 0.0354, + "std_dev": 0.0087 }, { "name": "benchmark_command_with_flags", "description": "Command with flags (3 flags)", - "avg_time": 0.0544, - "median_time": 0.0536, - "std_dev": 0.009 + "avg_time": 0.0557, + "median_time": 0.0545, + "std_dev": 0.0171 }, { "name": "benchmark_many_commands", "description": "Many commands (50 commands)", - "avg_time": 1.009, - "median_time": 1.0022, - "std_dev": 0.039 + "avg_time": 1.0453, + "median_time": 1.0388, + "std_dev": 0.0322 }, { "name": "benchmark_command_with_many_flags", "description": "Command with many flags (20 flags)", - "avg_time": 0.131, - "median_time": 0.1298, - "std_dev": 0.0054 + "avg_time": 0.1322, + "median_time": 0.131, + "std_dev": 0.0045 }, { "name": "benchmark_extreme_router", "description": "Extreme (100 commands, 10 flags each)", - "avg_time": 3.1818, - "median_time": 3.1619, - "std_dev": 0.1673 + "avg_time": 3.2471, + "median_time": 3.235, + "std_dev": 0.0814 } ] } \ No newline at end of file diff --git a/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler_comparison.png b/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler_comparison.png index 404d202..3e4bbcc 100644 Binary files a/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler_comparison.png and b/metrics/reports/releases/1.1.2/finds_appropriate_handler/finds_appropriate_handler_comparison.png differ diff --git a/metrics/reports/releases/1.1.2/flag_validation/flag_validation.json b/metrics/reports/releases/1.1.2/flag_validation/flag_validation.json index b7174ae..d1e5ca3 100644 --- a/metrics/reports/releases/1.1.2/flag_validation/flag_validation.json +++ b/metrics/reports/releases/1.1.2/flag_validation/flag_validation.json @@ -6,9 +6,9 @@ { "name": "benchmark_validate_all_single_flag", "description": "Single flag with PossibleValues.ALL", - "avg_time": 0.0009, - "median_time": 0.0009, - "std_dev": 0.0003 + "avg_time": 0.0008, + "median_time": 0.0008, + "std_dev": 0.0002 }, { "name": "benchmark_validate_neither_single_flag", @@ -21,50 +21,50 @@ "name": "benchmark_validate_list_small", "description": "List validation (5 possible values)", "avg_time": 0.001, - "median_time": 0.001, - "std_dev": 0.0003 + "median_time": 0.0009, + "std_dev": 0.0007 }, { "name": "benchmark_validate_list_large", "description": "List validation (50 possible values)", "avg_time": 0.0079, "median_time": 0.0078, - "std_dev": 0.0007 + "std_dev": 0.0021 }, { "name": "benchmark_validate_regex_simple", "description": "Regex validation (simple pattern)", - "avg_time": 0.0019, + "avg_time": 0.0017, "median_time": 0.0016, - "std_dev": 0.0073 + "std_dev": 0.0028 }, { "name": "benchmark_validate_regex_complex", "description": "Regex validation (complex pattern)", "avg_time": 0.0018, - "median_time": 0.0017, + "median_time": 0.0016, "std_dev": 0.0051 }, { "name": "benchmark_validate_multiple_flags_10", "description": "Multiple flags validation (10 flags)", - "avg_time": 0.0147, - "median_time": 0.0146, + "avg_time": 0.0145, + "median_time": 0.0144, "std_dev": 0.0013 }, { "name": "benchmark_validate_multiple_flags_50", "description": "Multiple flags validation (50 flags)", - "avg_time": 0.0669, - "median_time": 0.0661, - "std_dev": 0.0025 + "avg_time": 0.0661, + "median_time": 0.0658, + "std_dev": 0.0024 }, { "name": "benchmark_validate_extreme_100_flags", "description": "Extreme (100 flags with regex validation)", - "avg_time": 0.1626, - "median_time": 0.1609, - "std_dev": 0.007 + "avg_time": 0.1599, + "median_time": 0.1589, + "std_dev": 0.0065 } ] } \ No newline at end of file diff --git a/metrics/reports/releases/1.1.2/flag_validation/flag_validation_comparison.png b/metrics/reports/releases/1.1.2/flag_validation/flag_validation_comparison.png index 6008aee..2d558d8 100644 Binary files a/metrics/reports/releases/1.1.2/flag_validation/flag_validation_comparison.png and b/metrics/reports/releases/1.1.2/flag_validation/flag_validation_comparison.png differ diff --git a/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse.json b/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse.json index 6c03ccc..d7d3a03 100644 --- a/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse.json +++ b/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse.json @@ -6,51 +6,51 @@ { "name": "benchmark_parse_simple_command", "description": "Simple command (no flags)", - "avg_time": 0.0099, - "median_time": 0.0098, - "std_dev": 0.0014 + "avg_time": 0.0096, + "median_time": 0.0095, + "std_dev": 0.0012 }, { "name": "benchmark_command_with_few_flags", "description": "Command with few flags (3 flags)", - "avg_time": 0.0214, - "median_time": 0.0211, - "std_dev": 0.003 + "avg_time": 0.0216, + "median_time": 0.0213, + "std_dev": 0.0021 }, { "name": "benchmark_command_with_flags_and_values", "description": "Command with flags and values (5 flags)", - "avg_time": 0.0598, - "median_time": 0.0591, - "std_dev": 0.0041 + "avg_time": 0.06, + "median_time": 0.0595, + "std_dev": 0.0025 }, { "name": "benchmark_command_with_mixed_prefixes", "description": "Command with mixed prefixes (-, --, ---)", - "avg_time": 0.0545, - "median_time": 0.0541, - "std_dev": 0.0024 + "avg_time": 0.0542, + "median_time": 0.0538, + "std_dev": 0.0028 }, { "name": "benchmark_command_with_long_values", "description": "Command with long values (10 flags)", - "avg_time": 0.2114, - "median_time": 0.2103, - "std_dev": 0.0068 + "avg_time": 0.2092, + "median_time": 0.2082, + "std_dev": 0.0067 }, { "name": "benchmark_command_with_quoted_values", "description": "Command with quoted values (5 flags)", - "avg_time": 0.0489, - "median_time": 0.0488, - "std_dev": 0.0019 + "avg_time": 0.0481, + "median_time": 0.0477, + "std_dev": 0.0023 }, { "name": "benchmark_extreme_many_flags", "description": "Extreme (50 flags with values)", - "avg_time": 0.8057, - "median_time": 0.7997, - "std_dev": 0.024 + "avg_time": 0.7907, + "median_time": 0.7884, + "std_dev": 0.0417 } ] } \ No newline at end of file diff --git a/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse_comparison.png b/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse_comparison.png index e55a824..45ac191 100644 Binary files a/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse_comparison.png and b/metrics/reports/releases/1.1.2/input_command_parse/input_command_parse_comparison.png differ diff --git a/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command.json b/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command.json index 24b0ffc..7d51637 100644 --- a/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command.json +++ b/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command.json @@ -6,37 +6,37 @@ { "name": "benchmark_few_commands", "description": "Few commands (10 commands, no match)", - "avg_time": 0.207, - "median_time": 0.2051, - "std_dev": 0.01 + "avg_time": 0.251, + "median_time": 0.2488, + "std_dev": 0.012 }, { "name": "benchmark_many_commands_most_similar", "description": "Many commands (50 commands, no match)", - "avg_time": 1.0116, - "median_time": 1.0074, - "std_dev": 0.0306 + "avg_time": 1.1933, + "median_time": 1.1878, + "std_dev": 0.0305 }, { "name": "benchmark_many_aliases", "description": "Many aliases (20 commands, 10 aliases each)", - "avg_time": 0.6857, - "median_time": 0.6672, - "std_dev": 0.0811 + "avg_time": 1.2151, + "median_time": 1.2124, + "std_dev": 0.0282 }, { "name": "benchmark_partial_match", "description": "Partial match (50 commands, prefix match)", - "avg_time": 1.0218, - "median_time": 1.0192, - "std_dev": 0.0232 + "avg_time": 1.6781, + "median_time": 1.6689, + "std_dev": 0.0573 }, { "name": "benchmark_extreme_commands", "description": "Extreme (100 commands, 20 aliases each)", - "avg_time": 6.4118, - "median_time": 6.4225, - "std_dev": 0.2008 + "avg_time": 10.5539, + "median_time": 10.5288, + "std_dev": 0.1603 } ] } \ No newline at end of file diff --git a/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command_comparison.png b/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command_comparison.png index 65e80bf..ec07f2c 100644 Binary files a/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command_comparison.png and b/metrics/reports/releases/1.1.2/most_similar_command/most_similar_command_comparison.png differ diff --git a/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup.json b/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup.json index 7d50f26..4b95e2d 100644 --- a/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup.json +++ b/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup.json @@ -6,37 +6,37 @@ { "name": "benchmark_no_aliases", "description": "With no aliases", - "avg_time": 8.9994, - "median_time": 8.9559, - "std_dev": 0.242 + "avg_time": 7.4799, + "median_time": 7.4576, + "std_dev": 0.1645 }, { "name": "benchmark_few_aliases", "description": "With few aliases (6 total)", - "avg_time": 8.957, - "median_time": 8.936, - "std_dev": 0.2296 + "avg_time": 7.4135, + "median_time": 7.4061, + "std_dev": 0.1709 }, { "name": "benchmark_with_many_aliases", "description": "With many aliases (15 total)", - "avg_time": 8.9953, - "median_time": 8.9721, - "std_dev": 0.2243 + "avg_time": 7.4018, + "median_time": 7.3943, + "std_dev": 0.1589 }, { "name": "benchmark_very_many_aliases", "description": "With very many aliases (60 total)", - "avg_time": 9.0155, - "median_time": 8.9992, - "std_dev": 0.2433 + "avg_time": 7.476, + "median_time": 7.4575, + "std_dev": 0.2156 }, { "name": "benchmark_extreme_aliases", "description": "With extreme aliases (300 total)", - "avg_time": 9.2977, - "median_time": 9.255, - "std_dev": 0.3884 + "avg_time": 7.7167, + "median_time": 7.706, + "std_dev": 0.2052 } ] } \ No newline at end of file diff --git a/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup_comparison.png b/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup_comparison.png index 29386c2..1d7be18 100644 Binary files a/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup_comparison.png and b/metrics/reports/releases/1.1.2/pre_cycle_setup/pre_cycle_setup_comparison.png differ diff --git a/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions.json b/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions.json index 5ff7ecc..279cda5 100644 --- a/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions.json +++ b/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions.json @@ -6,37 +6,37 @@ { "name": "benchmark_few_routers", "description": "With few routers (3 routers, 1 command each)", - "avg_time": 0.0912, - "median_time": 0.0901, - "std_dev": 0.0051 + "avg_time": 0.0959, + "median_time": 0.0944, + "std_dev": 0.0097 }, { "name": "benchmark_many_routers", "description": "With many routers (10 routers, 1 command each)", - "avg_time": 0.2433, - "median_time": 0.2382, - "std_dev": 0.0292 + "avg_time": 0.2488, + "median_time": 0.2467, + "std_dev": 0.0081 }, { "name": "benchmark_many_commands_per_router", "description": "With many commands per router (3 routers, 10 commands each)", - "avg_time": 0.6157, - "median_time": 0.6132, - "std_dev": 0.0199 + "avg_time": 0.6474, + "median_time": 0.6401, + "std_dev": 0.0304 }, { "name": "benchmark_many_aliases_per_command", "description": "With many aliases (3 routers, 5 commands, 10 aliases each)", - "avg_time": 0.516, - "median_time": 0.5082, - "std_dev": 0.0403 + "avg_time": 0.5261, + "median_time": 0.5156, + "std_dev": 0.0475 }, { "name": "benchmark_extreme_routers", "description": "Extreme (20 routers, 10 commands, 20 aliases each)", - "avg_time": 10.8103, - "median_time": 10.7871, - "std_dev": 0.2788 + "avg_time": 9.9128, + "median_time": 9.9518, + "std_dev": 0.2373 } ] } \ No newline at end of file diff --git a/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions_comparison.png b/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions_comparison.png index 9f93ee0..662f7ed 100644 Binary files a/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions_comparison.png and b/metrics/reports/releases/1.1.2/validate_routers_for_collisions/validate_routers_for_collisions_comparison.png differ diff --git a/src/argenta/app/models.py b/src/argenta/app/models.py index 26ba97a..b251e8c 100644 --- a/src/argenta/app/models.py +++ b/src/argenta/app/models.py @@ -119,7 +119,6 @@ class BaseApp(BehaviorHandlersSettersMixin): def _most_similar_command(self, unknown_command: str) -> str | None: all_commands = self.registered_routers.get_triggers() - print(all_commands) matches = difflib.get_close_matches(unknown_command, all_commands, n=1) return matches[0] if matches else None