From 1f2533872de73a14119b6900c17a30f162c05aa6 Mon Sep 17 00:00:00 2001 From: emil Date: Fri, 26 Jun 2026 23:36:58 +0000 Subject: [PATCH] DEVX-66: fix: calibrate molecule weights from actual CI execution times --- src/devx/molecule/distribute_molecule.py | 70 ++++++++++++++++++++---- tests/unit/test_distribute_molecule.py | 25 ++++++++- 2 files changed, 82 insertions(+), 13 deletions(-) diff --git a/src/devx/molecule/distribute_molecule.py b/src/devx/molecule/distribute_molecule.py index 2838663..ba11e8d 100644 --- a/src/devx/molecule/distribute_molecule.py +++ b/src/devx/molecule/distribute_molecule.py @@ -135,24 +135,74 @@ def build_multi_role_pairs( # Heuristic weights for known heavy molecule scenarios. # These are estimated from CI run times — scenarios that pull large Docker # images or run complex Ansible playbooks take longer. +# +# Weights are calibrated from actual CI execution times (converge→destroy): +# nextcloud: ~7.7m → 15 +# restore/default: ~5.7m → 11 +# customer-apps: ~5.5m → 11 +# zitadel/default: ~5.0m → 10 +# docker_base/default: ~3.9m → 8 +# app_hardening/default: ~1.9m → 4 +# app_container/default: ~1.8m → 3 +# postgres-upgrade: ~1.7m → 3 +# observability/default: ~1.7m → 3 +# storage/default: ~1.5m → 3 +# storage/object-storage: ~1.0m → 2 +# vaultwarden: ~0.7m → 2 +# simple-app: ~0.7m → 2 +# +# Role-specific weights take priority over scenario-name weights. +# The (role, scenario) tuple is checked first, then the scenario name +# alone, then the default weight. +_ROLE_SCENARIO_WEIGHTS: dict[tuple[str, str], int] = { + ("app_container", "nextcloud"): 15, + ("app_container", "customer-apps"): 11, + ("app_container", "vaultwarden"): 2, + ("app_container", "simple-app"): 2, + ("app_container", "postgres-upgrade"): 3, + ("app_container", "default"): 3, + ("restore", "default"): 11, + ("zitadel", "default"): 10, + ("docker_base", "default"): 8, + ("observability", "default"): 3, + ("app_hardening", "default"): 4, + ("storage", "default"): 3, + ("storage", "object-storage"): 2, +} + +# Fallback weights by scenario name only (for single-role projects or +# scenarios not in the role-specific table). _SCENARIO_WEIGHTS: dict[str, int] = { - "nextcloud": 10, + "nextcloud": 15, + "customer-apps": 11, + "restore": 11, + "zitadel": 10, + "docker-base": 8, + "postgresql": 3, + "postgres-upgrade": 3, "gitea": 8, - "vaultwarden": 7, - "zitadel": 7, - "postgresql": 6, "redis": 5, "backup": 5, - "docker-base": 4, + "vaultwarden": 2, + "simple-app": 2, + "object-storage": 2, "default": 3, "binary": 2, } _DEFAULT_SCENARIO_WEIGHT = 3 -def _scenario_weight(scenario: str) -> int: - """Estimate a weight for a scenario based on its name.""" +def _scenario_weight(scenario: str, role: str | None = None) -> int: + """Estimate a weight for a scenario based on its name and optionally its role. + + Role-specific weights take priority over scenario-name-only weights. + """ s = scenario.lower() + if role is not None: + r = role.lower() + key = (r, s) + if key in _ROLE_SCENARIO_WEIGHTS: + return _ROLE_SCENARIO_WEIGHTS[key] for key, weight in _SCENARIO_WEIGHTS.items(): if key in s: return weight @@ -181,11 +231,11 @@ def _lpt_distribute[T](items: list[T], weights: list[int], max_runners: int) -> def distribute_multi_role(pairs: list[MultiRoleTestPair], max_runners: int) -> list[list[MultiRoleTestPair]]: """Split *pairs* into *max_runners* balanced groups using LPT scheduling. - Each pair is weighted by scenario name heuristics (e.g. ``nextcloud`` is - heavier than ``binary``). Pairs are sorted by weight descending and + Each pair is weighted by role+scenario heuristics (e.g. ``nextcloud`` is + heavier than ``simple-app``). Pairs are sorted by weight descending and assigned to the runner with the least total weight. """ - weights = [_scenario_weight(p.scenario) for p in pairs] + weights = [_scenario_weight(p.scenario, p.role) for p in pairs] return _lpt_distribute(pairs, weights, max_runners) diff --git a/tests/unit/test_distribute_molecule.py b/tests/unit/test_distribute_molecule.py index a56c889..d225465 100644 --- a/tests/unit/test_distribute_molecule.py +++ b/tests/unit/test_distribute_molecule.py @@ -483,7 +483,7 @@ class TestCliMultiRole: class TestScenarioWeight: def test_known_heavy_scenario(self) -> None: - assert _scenario_weight("nextcloud") == 10 + assert _scenario_weight("nextcloud") == 15 assert _scenario_weight("gitea") == 8 def test_known_light_scenario(self) -> None: @@ -493,13 +493,32 @@ class TestScenarioWeight: assert _scenario_weight("unknown-scenario") == 3 def test_case_insensitive(self) -> None: - assert _scenario_weight("NextCloud") == 10 + assert _scenario_weight("NextCloud") == 15 assert _scenario_weight("GITEA") == 8 def test_substring_match(self) -> None: - assert _scenario_weight("nextcloud-with-redis") == 10 + assert _scenario_weight("nextcloud-with-redis") == 15 assert _scenario_weight("custom-gitea-setup") == 8 + def test_role_specific_weight(self) -> None: + """Role+scenario pairs take priority over scenario-name-only weights.""" + assert _scenario_weight("default", "restore") == 11 + assert _scenario_weight("default", "zitadel") == 10 + assert _scenario_weight("default", "docker_base") == 8 + assert _scenario_weight("default", "app_hardening") == 4 + assert _scenario_weight("default", "app_container") == 3 + assert _scenario_weight("default", "storage") == 3 + assert _scenario_weight("default", "observability") == 3 + + def test_role_specific_overrides_scenario_name(self) -> None: + """vaultwarden has a scenario-name weight of 2, but role-specific is also 2.""" + assert _scenario_weight("vaultwarden", "app_container") == 2 + assert _scenario_weight("vaultwarden") == 2 + + def test_customer_apps_weight(self) -> None: + assert _scenario_weight("customer-apps", "app_container") == 11 + assert _scenario_weight("customer-apps") == 11 + class TestLptDistribute: def test_equal_weights_produce_even_split(self) -> None: