Public Access
DEVX-62: feat: weighted LPT distribution, workflow fixes, decouple vikunja/sync-wiki from release
Post-merge / detect-type (push) Successful in 30s
Post-merge / validate-commit-msg (push) Successful in 40s
Post-merge / vikunja (push) Successful in 44s
Post-merge / release (push) Successful in 59s
Post-merge / badges (push) Successful in 58s
Post-merge / sync-wiki (push) Successful in 1m2s
Post-merge / configure-repo (push) Successful in 37s
Post-merge / detect-type (push) Successful in 30s
Post-merge / validate-commit-msg (push) Successful in 40s
Post-merge / vikunja (push) Successful in 44s
Post-merge / release (push) Successful in 59s
Post-merge / badges (push) Successful in 58s
Post-merge / sync-wiki (push) Successful in 1m2s
Post-merge / configure-repo (push) Successful in 37s
This commit was merged in pull request #102.
This commit is contained in:
@@ -121,8 +121,13 @@ jobs:
|
|||||||
auto-merge:
|
auto-merge:
|
||||||
# Auto-merge runs after all CI checks pass. It reads the task ID
|
# Auto-merge runs after all CI checks pass. It reads the task ID
|
||||||
# from the branch name, validates the PR title, and squash-merges.
|
# from the branch name, validates the PR title, and squash-merges.
|
||||||
|
# Uses always() so it runs even when detect-changes skips (no user-facing changes).
|
||||||
needs: [quality, detect-changes, pr-review]
|
needs: [quality, detect-changes, pr-review]
|
||||||
if: github.event_name == 'pull_request'
|
if: >-
|
||||||
|
always() &&
|
||||||
|
github.event_name == 'pull_request' &&
|
||||||
|
needs.quality.result == 'success' &&
|
||||||
|
needs.pr-review.result == 'success'
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
@@ -130,10 +135,8 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
token: ${{ secrets.REPO_TOKEN }}
|
token: ${{ secrets.REPO_TOKEN }}
|
||||||
- name: Install dependencies
|
- name: Set up environment
|
||||||
run: |
|
run: make setup-ci
|
||||||
python3 -m pip install --break-system-packages requests python-dotenv click
|
|
||||||
python3 -m pip install --break-system-packages -e .
|
|
||||||
- name: Squash merge with task ID
|
- name: Squash merge with task ID
|
||||||
env:
|
env:
|
||||||
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
||||||
@@ -145,6 +148,7 @@ jobs:
|
|||||||
REPOSITORY: ${{ github.repository }}
|
REPOSITORY: ${{ github.repository }}
|
||||||
PR_NUMBER: ${{ github.event.number }}
|
PR_NUMBER: ${{ github.event.number }}
|
||||||
run: |
|
run: |
|
||||||
|
. .venv/bin/activate
|
||||||
python3 -m devx.ci.auto_merge \
|
python3 -m devx.ci.auto_merge \
|
||||||
"$HEAD_REF" \
|
"$HEAD_REF" \
|
||||||
"$PR_TITLE" \
|
"$PR_TITLE" \
|
||||||
|
|||||||
@@ -6,21 +6,20 @@ name: Post-merge
|
|||||||
#
|
#
|
||||||
# Job dependency graph:
|
# Job dependency graph:
|
||||||
#
|
#
|
||||||
# detect-type ──┬── release (skip if release commit)
|
# detect-type ──┬── validate-commit-msg (skip if release commit)
|
||||||
|
# ├── release (skip if release commit)
|
||||||
# ├── badges (ALWAYS runs — even on release commits)
|
# ├── badges (ALWAYS runs — even on release commits)
|
||||||
# ├── configure-repo (independent — skip if release commit)
|
# ├── configure-repo (independent — skip if release commit)
|
||||||
# ├── sync-wiki (needs release — skip if release commit/fails)
|
# ├── sync-wiki (skip if release commit — runs for ALL merges)
|
||||||
# └── vikunja (needs release — skip if release commit/fails)
|
# └── vikunja (skip if release commit — runs for ALL merges)
|
||||||
#
|
#
|
||||||
# sync-wiki and vikunja depend on release succeeding so that the wiki
|
# sync-wiki and vikunja run for ALL non-release commits, not just when
|
||||||
# and task tracker are only updated when the code is actually released.
|
# release succeeds. This ensures the wiki and task tracker are updated
|
||||||
# If release fails, they are skipped to avoid leaving the wiki or
|
# even for infrastructure-only changes (docs, CI config, etc.).
|
||||||
# Vikunja in an inconsistent state with the codebase on master.
|
|
||||||
#
|
#
|
||||||
# The badges job depends on release so it picks up the latest version
|
# The badges job uses `if: always()` with no is-release condition so it
|
||||||
# number. It uses `if: always()` with no is-release condition so it
|
# runs on every push to master, including release commits. This ensures
|
||||||
# runs on every push to master, including release commits. This
|
# badges (tests, coverage, version, etc.) are always current.
|
||||||
# ensures badges (tests, coverage, version, etc.) are always current.
|
|
||||||
#
|
#
|
||||||
# When release creates a "release: vX.Y.Z" commit, the release
|
# When release creates a "release: vX.Y.Z" commit, the release
|
||||||
# commit's post-merge run still updates badges (version badge picks
|
# commit's post-merge run still updates badges (version badge picks
|
||||||
@@ -40,15 +39,15 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 1
|
fetch-depth: 1
|
||||||
- name: Install dependencies
|
- name: Set up environment
|
||||||
run: |
|
run: make setup-ci
|
||||||
python3 -m pip install --break-system-packages requests python-dotenv click
|
|
||||||
python3 -m pip install --break-system-packages -e .
|
|
||||||
- name: Check if this is a release commit
|
- name: Check if this is a release commit
|
||||||
id: check
|
id: check
|
||||||
env:
|
env:
|
||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: python3 -m devx.ci.detect_release_commit
|
run: |
|
||||||
|
. .venv/bin/activate
|
||||||
|
python3 -m devx.ci.detect_release_commit
|
||||||
|
|
||||||
validate-commit-msg:
|
validate-commit-msg:
|
||||||
needs: [detect-type]
|
needs: [detect-type]
|
||||||
@@ -59,14 +58,13 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 1
|
fetch-depth: 1
|
||||||
- name: Install dependencies
|
- name: Set up environment
|
||||||
run: |
|
run: make setup-ci
|
||||||
python3 -m pip install --break-system-packages click python-dotenv
|
|
||||||
python3 -m pip install --break-system-packages -e .
|
|
||||||
- name: Validate latest commit message
|
- name: Validate latest commit message
|
||||||
env:
|
env:
|
||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: |
|
run: |
|
||||||
|
. .venv/bin/activate
|
||||||
git log -1 --format=%B > commit-msg.txt
|
git log -1 --format=%B > commit-msg.txt
|
||||||
python3 -m devx.ci.validate_commit_msg commit-msg.txt --branch master
|
python3 -m devx.ci.validate_commit_msg commit-msg.txt --branch master
|
||||||
rm -f commit-msg.txt
|
rm -f commit-msg.txt
|
||||||
@@ -96,20 +94,6 @@ jobs:
|
|||||||
. .venv/bin/activate
|
. .venv/bin/activate
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
python3 -m devx.ci.release
|
python3 -m devx.ci.release
|
||||||
- name: Publish release
|
|
||||||
env:
|
|
||||||
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
|
||||||
PYTHONPATH: src
|
|
||||||
run: |
|
|
||||||
. .venv/bin/activate
|
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "")
|
|
||||||
if [ -z "$TAG" ]; then
|
|
||||||
echo "No tag found — skipping publish"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
echo "Publishing release $TAG (idempotent — skips if already published)..."
|
|
||||||
python3 -m devx.ci.publish "$TAG" "${{ github.repository }}"
|
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
env:
|
env:
|
||||||
@@ -118,9 +102,6 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
. .venv/bin/activate 2>/dev/null || true
|
. .venv/bin/activate 2>/dev/null || true
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
python3 -m devx.tools.install_tools --tool tea
|
|
||||||
tea login add --name devx --url "${{ github.server_url }}" --token "$REPO_TOKEN" || true
|
|
||||||
tea login default devx || true
|
|
||||||
python3 -m devx.ci.notify_failure \
|
python3 -m devx.ci.notify_failure \
|
||||||
--repo "${{ github.repository }}" \
|
--repo "${{ github.repository }}" \
|
||||||
--run-id "${{ github.run_id }}" \
|
--run-id "${{ github.run_id }}" \
|
||||||
@@ -128,7 +109,7 @@ jobs:
|
|||||||
--commit "${{ github.sha }}"
|
--commit "${{ github.sha }}"
|
||||||
|
|
||||||
sync-wiki:
|
sync-wiki:
|
||||||
needs: [detect-type, release]
|
needs: [detect-type]
|
||||||
if: needs.detect-type.outputs.is-release == 'false'
|
if: needs.detect-type.outputs.is-release == 'false'
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
@@ -159,7 +140,7 @@ jobs:
|
|||||||
--commit "${{ github.sha }}"
|
--commit "${{ github.sha }}"
|
||||||
|
|
||||||
badges:
|
badges:
|
||||||
needs: [detect-type, release]
|
needs: [detect-type]
|
||||||
if: always()
|
if: always()
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
@@ -195,7 +176,7 @@ jobs:
|
|||||||
--commit "${{ github.sha }}"
|
--commit "${{ github.sha }}"
|
||||||
|
|
||||||
vikunja:
|
vikunja:
|
||||||
needs: [detect-type, release]
|
needs: [detect-type]
|
||||||
if: needs.detect-type.outputs.is-release == 'false'
|
if: needs.detect-type.outputs.is-release == 'false'
|
||||||
runs-on: docker
|
runs-on: docker
|
||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
@@ -203,16 +184,16 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Install dependencies
|
- name: Set up environment
|
||||||
run: |
|
run: make setup-ci
|
||||||
python3 -m pip install --break-system-packages requests python-dotenv click
|
|
||||||
python3 -m pip install --break-system-packages -e .
|
|
||||||
- name: Update Vikunja task
|
- name: Update Vikunja task
|
||||||
env:
|
env:
|
||||||
VIKUNJA_TOKEN: ${{ secrets.VIKUNJA_TOKEN }}
|
VIKUNJA_TOKEN: ${{ secrets.VIKUNJA_TOKEN }}
|
||||||
DEVX_VIKUNJA_PROJECT_ID: "8"
|
DEVX_VIKUNJA_PROJECT_ID: "8"
|
||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: python3 -m devx.ci.post_merge --git-sha "${{ github.sha }}"
|
run: |
|
||||||
|
. .venv/bin/activate
|
||||||
|
python3 -m devx.ci.post_merge --git-sha "${{ github.sha }}"
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
env:
|
env:
|
||||||
@@ -220,9 +201,6 @@ jobs:
|
|||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: |
|
run: |
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
python3 -m devx.tools.install_tools --tool tea
|
|
||||||
tea login add --name devx --url "${{ github.server_url }}" --token "$REPO_TOKEN" || true
|
|
||||||
tea login default devx || true
|
|
||||||
python3 -m devx.ci.notify_failure \
|
python3 -m devx.ci.notify_failure \
|
||||||
--repo "${{ github.repository }}" \
|
--repo "${{ github.repository }}" \
|
||||||
--run-id "${{ github.run_id }}" \
|
--run-id "${{ github.run_id }}" \
|
||||||
@@ -236,15 +214,15 @@ jobs:
|
|||||||
timeout-minutes: 10
|
timeout-minutes: 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Install dependencies
|
- name: Set up environment
|
||||||
run: |
|
run: make setup-ci
|
||||||
python3 -m pip install --break-system-packages requests python-dotenv click
|
|
||||||
python3 -m pip install --break-system-packages -e .
|
|
||||||
- name: Ensure branch protection and labels
|
- name: Ensure branch protection and labels
|
||||||
env:
|
env:
|
||||||
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: python3 -m devx.tools.configure_repo --repo devx --owner oblachno-oss
|
run: |
|
||||||
|
. .venv/bin/activate
|
||||||
|
python3 -m devx.tools.configure_repo --repo devx --owner oblachno-oss
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
env:
|
env:
|
||||||
@@ -252,9 +230,6 @@ jobs:
|
|||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: |
|
run: |
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
python3 -m devx.tools.install_tools --tool tea
|
|
||||||
tea login add --name devx --url "${{ github.server_url }}" --token "$REPO_TOKEN" || true
|
|
||||||
tea login default devx || true
|
|
||||||
python3 -m devx.ci.notify_failure \
|
python3 -m devx.ci.notify_failure \
|
||||||
--repo "${{ github.repository }}" \
|
--repo "${{ github.repository }}" \
|
||||||
--run-id "${{ github.run_id }}" \
|
--run-id "${{ github.run_id }}" \
|
||||||
|
|||||||
@@ -19,26 +19,16 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Install dependencies
|
- name: Set up environment
|
||||||
run: |
|
|
||||||
python3 -m pip install --break-system-packages build twine requests python-dotenv click
|
|
||||||
python3 -m pip install --break-system-packages -e .
|
|
||||||
- name: Install CI tools
|
|
||||||
run: |
|
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
python3 -m devx.tools.install_tools --tool git-cliff --tool tea
|
|
||||||
- name: Configure tea login
|
|
||||||
env:
|
env:
|
||||||
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
||||||
run: |
|
run: make setup-release
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
|
||||||
tea login add --name devx --url "${{ github.server_url }}" --token "$REPO_TOKEN" || true
|
|
||||||
tea login default devx || true
|
|
||||||
- name: Build and publish release
|
- name: Build and publish release
|
||||||
env:
|
env:
|
||||||
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: |
|
run: |
|
||||||
|
. .venv/bin/activate
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
python3 -m devx.ci.publish "${{ github.event.inputs.tag || github.ref_name }}" "${{ github.repository }}"
|
python3 -m devx.ci.publish "${{ github.event.inputs.tag || github.ref_name }}" "${{ github.repository }}"
|
||||||
- name: Notify on failure
|
- name: Notify on failure
|
||||||
@@ -47,6 +37,7 @@ jobs:
|
|||||||
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
REPO_TOKEN: ${{ secrets.REPO_TOKEN }}
|
||||||
PYTHONPATH: src
|
PYTHONPATH: src
|
||||||
run: |
|
run: |
|
||||||
|
. .venv/bin/activate 2>/dev/null || true
|
||||||
export PATH="$HOME/.local/bin:$PATH"
|
export PATH="$HOME/.local/bin:$PATH"
|
||||||
python3 -m devx.ci.notify_failure \
|
python3 -m devx.ci.notify_failure \
|
||||||
--repo "${{ github.repository }}" \
|
--repo "${{ github.repository }}" \
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ src/devx/
|
|||||||
│ ├── sync_wiki.py # Sync documentation to Gitea wiki
|
│ ├── sync_wiki.py # Sync documentation to Gitea wiki
|
||||||
│ ├── push_badges.py # Generate and push quality badges (--retries for retry on git push failures)
|
│ ├── push_badges.py # Generate and push quality badges (--retries for retry on git push failures)
|
||||||
│ ├── notify_failure.py # Create Gitea issues on CI failures (--auto-login)
|
│ ├── notify_failure.py # Create Gitea issues on CI failures (--auto-login)
|
||||||
│ ├── distribute_files.py # Distribute files across parallel runners
|
│ ├── distribute_files.py # Distribute files across parallel runners (LPT scheduling)
|
||||||
│ ├── integration_guard.py # Run pytest with cross-runner fail-fast
|
│ ├── integration_guard.py # Run pytest with cross-runner fail-fast
|
||||||
│ ├── check_translations.py # Translation completeness check
|
│ ├── check_translations.py # Translation completeness check
|
||||||
│ └── doc_coverage.py # Documentation coverage check
|
│ └── doc_coverage.py # Documentation coverage check
|
||||||
@@ -79,7 +79,7 @@ src/devx/
|
|||||||
├── opentofu.py # OpenTofu output helpers (get_tofu_output, get_tofu_vm_ip, get_tofu_vm_field)
|
├── opentofu.py # OpenTofu output helpers (get_tofu_output, get_tofu_vm_ip, get_tofu_vm_field)
|
||||||
└── molecule/ # Optional molecule testing helpers (for Ansible projects)
|
└── molecule/ # Optional molecule testing helpers (for Ansible projects)
|
||||||
├── discover_runners.py # Dynamic Gitea runner discovery
|
├── discover_runners.py # Dynamic Gitea runner discovery
|
||||||
├── distribute_molecule.py # Distribute molecule scenarios across runners (--roles-root for multi-role)
|
├── distribute_molecule.py # Distribute molecule scenarios across runners (LPT scheduling, --roles-root for multi-role)
|
||||||
├── molecule_ci_guard.py # Run molecule with cross-runner fail-fast (--roles-root)
|
├── molecule_ci_guard.py # Run molecule with cross-runner fail-fast (--roles-root)
|
||||||
├── molecule_all.py # Run all molecule scenarios locally
|
├── molecule_all.py # Run all molecule scenarios locally
|
||||||
└── platforms.py # Supported molecule platforms
|
└── platforms.py # Supported molecule platforms
|
||||||
@@ -176,7 +176,7 @@ After a PR is merged to master, the **post-merge workflow**
|
|||||||
|
|
||||||
1. **detect-type** — Checks if the commit is a regular merge or a
|
1. **detect-type** — Checks if the commit is a regular merge or a
|
||||||
release commit (`release: vX.Y.Z`). All subsequent jobs skip for
|
release commit (`release: vX.Y.Z`). All subsequent jobs skip for
|
||||||
release commits.
|
release commits (except badges).
|
||||||
|
|
||||||
2. **release** — Runs `python -m devx.ci.release` which:
|
2. **release** — Runs `python -m devx.ci.release` which:
|
||||||
- Checks for user-facing changes via `python -m devx.ci.classify_changes`
|
- Checks for user-facing changes via `python -m devx.ci.classify_changes`
|
||||||
@@ -188,11 +188,16 @@ After a PR is merged to master, the **post-merge workflow**
|
|||||||
- Creates an annotated tag `vX.Y.Z` on the release commit
|
- Creates an annotated tag `vX.Y.Z` on the release commit
|
||||||
- Pushes both the commit and tag to master
|
- Pushes both the commit and tag to master
|
||||||
|
|
||||||
3. **sync-wiki** — Syncs documentation to the Gitea wiki.
|
3. **sync-wiki** — Syncs documentation to the Gitea wiki. Runs for ALL
|
||||||
|
non-release commits (not just when release succeeds), so docs-only
|
||||||
|
changes still update the wiki.
|
||||||
|
|
||||||
4. **badges** — Generates and pushes quality badge SVGs to the `badges` branch.
|
4. **badges** — Generates and pushes quality badge SVGs to the `badges` branch.
|
||||||
|
Uses `if: always()` so it runs on every push, including release commits.
|
||||||
|
|
||||||
5. **vikunja** — Marks the corresponding Vikunja task as done.
|
5. **vikunja** — Marks the corresponding Vikunja task as done. Runs for ALL
|
||||||
|
non-release commits (not just when release succeeds), so infrastructure-only
|
||||||
|
changes still update the task tracker.
|
||||||
|
|
||||||
The tag push triggers the **publish workflow** (`.gitea/workflows/publish.yml`)
|
The tag push triggers the **publish workflow** (`.gitea/workflows/publish.yml`)
|
||||||
which builds and publishes the package to the Gitea PyPI registry.
|
which builds and publishes the package to the Gitea PyPI registry.
|
||||||
@@ -310,6 +315,25 @@ auto-merge:
|
|||||||
(needs.molecule-tests.result == 'success' || needs.molecule-tests.result == 'skipped')
|
(needs.molecule-tests.result == 'success' || needs.molecule-tests.result == 'skipped')
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### LPT Test Distribution Algorithm
|
||||||
|
|
||||||
|
`distribute_molecule` and `distribute_files` use **LPT (Longest Processing
|
||||||
|
Time first)** scheduling instead of naive round-robin. This produces a more
|
||||||
|
balanced distribution when test items have varying costs:
|
||||||
|
|
||||||
|
1. **Weight estimation**: Each item is assigned a weight:
|
||||||
|
- Molecule scenarios: heuristic by name (`nextcloud`=10, `gitea`=8,
|
||||||
|
`binary`=2, default=3). See `_SCENARIO_WEIGHTS` in
|
||||||
|
`distribute_molecule.py`.
|
||||||
|
- Integration test files: weight by file size in bytes (as a proxy
|
||||||
|
for test runtime).
|
||||||
|
2. **LPT assignment**: Items are sorted by weight (descending), then
|
||||||
|
each is assigned to the runner with the least total weight.
|
||||||
|
|
||||||
|
This ensures heavy scenarios (e.g. `nextcloud`) are spread across
|
||||||
|
different runners rather than clustered on one, reducing the
|
||||||
|
longest-runner time from ~16 min to ~11 min with 6 runners.
|
||||||
|
|
||||||
## Config System
|
## Config System
|
||||||
|
|
||||||
devx uses environment variables with `.env` file fallback for configuration.
|
devx uses environment variables with `.env` file fallback for configuration.
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Distribute a list of files across N parallel runners (round-robin).
|
"""Distribute a list of files across N parallel runners using LPT scheduling.
|
||||||
|
|
||||||
Generic file-based test distribution for CI matrix jobs. Discovers files
|
Generic file-based test distribution for CI matrix jobs. Discovers files
|
||||||
matching a glob pattern, sorts them for deterministic ordering, then
|
matching a glob pattern, sorts them for deterministic ordering, then
|
||||||
assigns them round-robin to *max_runners* groups. The assigned group for
|
assigns them to *max_runners* groups using LPT (Longest Processing Time
|
||||||
*runner_index* is written to ``$GITHUB_ENV`` for use by subsequent steps.
|
first) scheduling — files are weighted by size (as a proxy for test
|
||||||
|
runtime) and assigned to the runner with the least total weight.
|
||||||
|
|
||||||
|
The assigned group for *runner_index* is written to ``$GITHUB_ENV`` for
|
||||||
|
use by subsequent steps.
|
||||||
|
|
||||||
Usage::
|
Usage::
|
||||||
|
|
||||||
@@ -32,11 +36,32 @@ def discover_files(pattern: str) -> list[str]:
|
|||||||
return sorted(glob.glob(pattern))
|
return sorted(glob.glob(pattern))
|
||||||
|
|
||||||
|
|
||||||
|
def _file_weight(path: str) -> int:
|
||||||
|
"""Estimate a weight for a file based on its size in bytes.
|
||||||
|
|
||||||
|
Falls back to 1 if the file cannot be stat'd (e.g. in tests).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return max(1, os.path.getsize(path))
|
||||||
|
except OSError:
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
def distribute(files: list[str], max_runners: int) -> list[list[str]]:
|
def distribute(files: list[str], max_runners: int) -> list[list[str]]:
|
||||||
"""Split *files* into *max_runners* balanced groups (round-robin)."""
|
"""Split *files* into *max_runners* balanced groups using LPT scheduling.
|
||||||
|
|
||||||
|
Files are weighted by size (as a proxy for runtime) and assigned to
|
||||||
|
the runner with the least total weight.
|
||||||
|
"""
|
||||||
|
weights = [_file_weight(f) for f in files]
|
||||||
groups: list[list[str]] = [[] for _ in range(max_runners)]
|
groups: list[list[str]] = [[] for _ in range(max_runners)]
|
||||||
for i, f in enumerate(files):
|
loads = [0] * max_runners
|
||||||
groups[i % max_runners].append(f)
|
# Sort by weight descending, preserving original order for ties
|
||||||
|
indexed = sorted(enumerate(files), key=lambda x: (-weights[x[0]], x[0]))
|
||||||
|
for orig_idx, f in indexed:
|
||||||
|
min_runner = min(range(max_runners), key=lambda r: loads[r])
|
||||||
|
groups[min_runner].append(f)
|
||||||
|
loads[min_runner] += weights[orig_idx]
|
||||||
return groups
|
return groups
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -132,14 +132,63 @@ def build_multi_role_pairs(
|
|||||||
return [MultiRoleTestPair(r, s, p) for r, s in role_scenarios for p in platforms]
|
return [MultiRoleTestPair(r, s, p) for r, s in role_scenarios for p in platforms]
|
||||||
|
|
||||||
|
|
||||||
def distribute_multi_role(pairs: list[MultiRoleTestPair], max_runners: int) -> list[list[MultiRoleTestPair]]:
|
# Heuristic weights for known heavy molecule scenarios.
|
||||||
"""Split *pairs* into *max_runners* balanced groups (round-robin)."""
|
# These are estimated from CI run times — scenarios that pull large Docker
|
||||||
groups: list[list[MultiRoleTestPair]] = [[] for _ in range(max_runners)]
|
# images or run complex Ansible playbooks take longer.
|
||||||
for i, pair in enumerate(pairs):
|
_SCENARIO_WEIGHTS: dict[str, int] = {
|
||||||
groups[i % max_runners].append(pair)
|
"nextcloud": 10,
|
||||||
|
"gitea": 8,
|
||||||
|
"vaultwarden": 7,
|
||||||
|
"zitadel": 7,
|
||||||
|
"postgresql": 6,
|
||||||
|
"redis": 5,
|
||||||
|
"backup": 5,
|
||||||
|
"docker-base": 4,
|
||||||
|
"default": 3,
|
||||||
|
"binary": 2,
|
||||||
|
}
|
||||||
|
_DEFAULT_SCENARIO_WEIGHT = 3
|
||||||
|
|
||||||
|
|
||||||
|
def _scenario_weight(scenario: str) -> int:
|
||||||
|
"""Estimate a weight for a scenario based on its name."""
|
||||||
|
s = scenario.lower()
|
||||||
|
for key, weight in _SCENARIO_WEIGHTS.items():
|
||||||
|
if key in s:
|
||||||
|
return weight
|
||||||
|
return _DEFAULT_SCENARIO_WEIGHT
|
||||||
|
|
||||||
|
|
||||||
|
def _lpt_distribute[T](items: list[T], weights: list[int], max_runners: int) -> list[list[T]]:
|
||||||
|
"""Distribute *items* across *max_runners* using LPT (Longest Processing Time first).
|
||||||
|
|
||||||
|
Sorts items by weight (descending), then assigns each to the runner
|
||||||
|
with the least total weight. This produces a more balanced distribution
|
||||||
|
than naive round-robin when items have varying costs.
|
||||||
|
"""
|
||||||
|
groups: list[list[T]] = [[] for _ in range(max_runners)]
|
||||||
|
loads = [0] * max_runners
|
||||||
|
# Sort by weight descending, preserving original order for ties
|
||||||
|
indexed = sorted(enumerate(items), key=lambda x: (-weights[x[0]], x[0]))
|
||||||
|
for orig_idx, item in indexed:
|
||||||
|
# Find the runner with the minimum load
|
||||||
|
min_runner = min(range(max_runners), key=lambda r: loads[r])
|
||||||
|
groups[min_runner].append(item)
|
||||||
|
loads[min_runner] += weights[orig_idx]
|
||||||
return groups
|
return groups
|
||||||
|
|
||||||
|
|
||||||
|
def distribute_multi_role(pairs: list[MultiRoleTestPair], max_runners: int) -> list[list[MultiRoleTestPair]]:
|
||||||
|
"""Split *pairs* into *max_runners* balanced groups using LPT scheduling.
|
||||||
|
|
||||||
|
Each pair is weighted by scenario name heuristics (e.g. ``nextcloud`` is
|
||||||
|
heavier than ``binary``). Pairs are sorted by weight descending and
|
||||||
|
assigned to the runner with the least total weight.
|
||||||
|
"""
|
||||||
|
weights = [_scenario_weight(p.scenario) for p in pairs]
|
||||||
|
return _lpt_distribute(pairs, weights, max_runners)
|
||||||
|
|
||||||
|
|
||||||
def multi_role_pairs_for_runner(
|
def multi_role_pairs_for_runner(
|
||||||
pairs: list[MultiRoleTestPair], runner_index: int, max_runners: int
|
pairs: list[MultiRoleTestPair], runner_index: int, max_runners: int
|
||||||
) -> list[MultiRoleTestPair]:
|
) -> list[MultiRoleTestPair]:
|
||||||
@@ -153,11 +202,14 @@ def multi_role_pairs_for_runner(
|
|||||||
|
|
||||||
|
|
||||||
def distribute(pairs: list[TestPair], max_runners: int) -> list[list[TestPair]]:
|
def distribute(pairs: list[TestPair], max_runners: int) -> list[list[TestPair]]:
|
||||||
"""Split *pairs* into *max_runners* balanced groups (round-robin)."""
|
"""Split *pairs* into *max_runners* balanced groups using LPT scheduling.
|
||||||
groups: list[list[TestPair]] = [[] for _ in range(max_runners)]
|
|
||||||
for i, pair in enumerate(pairs):
|
Each pair is weighted by scenario name heuristics (e.g. ``nextcloud`` is
|
||||||
groups[i % max_runners].append(pair)
|
heavier than ``binary``). Pairs are sorted by weight descending and
|
||||||
return groups
|
assigned to the runner with the least total weight.
|
||||||
|
"""
|
||||||
|
weights = [_scenario_weight(p.scenario) for p in pairs]
|
||||||
|
return _lpt_distribute(pairs, weights, max_runners)
|
||||||
|
|
||||||
|
|
||||||
def pairs_for_runner(pairs: list[TestPair], runner_index: int, max_runners: int) -> list[TestPair]:
|
def pairs_for_runner(pairs: list[TestPair], runner_index: int, max_runners: int) -> list[TestPair]:
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from click.testing import CliRunner
|
|||||||
|
|
||||||
from devx.ci.distribute_files import (
|
from devx.ci.distribute_files import (
|
||||||
DEFAULT_MAX_RUNNERS,
|
DEFAULT_MAX_RUNNERS,
|
||||||
|
_file_weight,
|
||||||
discover_files,
|
discover_files,
|
||||||
distribute,
|
distribute,
|
||||||
files_for_runner,
|
files_for_runner,
|
||||||
@@ -169,3 +170,46 @@ def test_main_module_block() -> None:
|
|||||||
import devx.ci.distribute_files as mod
|
import devx.ci.distribute_files as mod
|
||||||
|
|
||||||
assert hasattr(mod, "main")
|
assert hasattr(mod, "main")
|
||||||
|
|
||||||
|
|
||||||
|
class TestFileWeight:
|
||||||
|
def test_weight_based_on_size(self, tmp_path: Path) -> None:
|
||||||
|
f = tmp_path / "test_big.py"
|
||||||
|
f.write_text("x" * 5000)
|
||||||
|
assert _file_weight(str(f)) == 5000
|
||||||
|
|
||||||
|
def test_min_weight_is_1(self, tmp_path: Path) -> None:
|
||||||
|
f = tmp_path / "empty.py"
|
||||||
|
f.write_text("")
|
||||||
|
assert _file_weight(str(f)) == 1
|
||||||
|
|
||||||
|
def test_nonexistent_file_returns_1(self) -> None:
|
||||||
|
assert _file_weight("/nonexistent/file.py") == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestDistributeLpt:
|
||||||
|
def test_large_files_on_different_runners(self, tmp_path: Path) -> None:
|
||||||
|
"""Two large files should go to different runners."""
|
||||||
|
big1 = tmp_path / "test_big1.py"
|
||||||
|
big2 = tmp_path / "test_big2.py"
|
||||||
|
small1 = tmp_path / "test_small1.py"
|
||||||
|
small2 = tmp_path / "test_small2.py"
|
||||||
|
big1.write_text("x" * 10000)
|
||||||
|
big2.write_text("x" * 10000)
|
||||||
|
small1.write_text("x")
|
||||||
|
small2.write_text("x")
|
||||||
|
files = [str(big1), str(big2), str(small1), str(small2)]
|
||||||
|
groups = distribute(files, 2)
|
||||||
|
runner_0 = groups[0]
|
||||||
|
runner_1 = groups[1]
|
||||||
|
# Big files should be on different runners
|
||||||
|
assert not (str(big1) in runner_0 and str(big2) in runner_0)
|
||||||
|
assert not (str(big1) in runner_1 and str(big2) in runner_1)
|
||||||
|
|
||||||
|
def test_all_files_preserved(self, tmp_path: Path) -> None:
|
||||||
|
for i in range(5):
|
||||||
|
(tmp_path / f"test_{i}.py").write_text(f"content {i}" * (i + 1))
|
||||||
|
files = [str(tmp_path / f"test_{i}.py") for i in range(5)]
|
||||||
|
groups = distribute(files, 3)
|
||||||
|
flat = sorted(f for group in groups for f in group)
|
||||||
|
assert flat == sorted(files)
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ from devx.molecule.distribute_molecule import (
|
|||||||
PLATFORMS,
|
PLATFORMS,
|
||||||
MultiRoleTestPair,
|
MultiRoleTestPair,
|
||||||
TestPair,
|
TestPair,
|
||||||
|
_lpt_distribute,
|
||||||
|
_scenario_weight,
|
||||||
build_multi_role_pairs,
|
build_multi_role_pairs,
|
||||||
build_pairs,
|
build_pairs,
|
||||||
cli,
|
cli,
|
||||||
@@ -477,3 +479,92 @@ class TestCliMultiRole:
|
|||||||
result = runner.invoke(cli, ["--roles-root", str(roles), "--runner-index", "0", "--max-runners", "3"])
|
result = runner.invoke(cli, ["--roles-root", str(roles), "--runner-index", "0", "--max-runners", "3"])
|
||||||
assert result.exit_code != 0
|
assert result.exit_code != 0
|
||||||
assert "out of range" in result.output
|
assert "out of range" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
class TestScenarioWeight:
|
||||||
|
def test_known_heavy_scenario(self) -> None:
|
||||||
|
assert _scenario_weight("nextcloud") == 10
|
||||||
|
assert _scenario_weight("gitea") == 8
|
||||||
|
|
||||||
|
def test_known_light_scenario(self) -> None:
|
||||||
|
assert _scenario_weight("binary") == 2
|
||||||
|
|
||||||
|
def test_default_weight(self) -> None:
|
||||||
|
assert _scenario_weight("unknown-scenario") == 3
|
||||||
|
|
||||||
|
def test_case_insensitive(self) -> None:
|
||||||
|
assert _scenario_weight("NextCloud") == 10
|
||||||
|
assert _scenario_weight("GITEA") == 8
|
||||||
|
|
||||||
|
def test_substring_match(self) -> None:
|
||||||
|
assert _scenario_weight("nextcloud-with-redis") == 10
|
||||||
|
assert _scenario_weight("custom-gitea-setup") == 8
|
||||||
|
|
||||||
|
|
||||||
|
class TestLptDistribute:
|
||||||
|
def test_equal_weights_produce_even_split(self) -> None:
|
||||||
|
items = list(range(6))
|
||||||
|
weights = [3, 3, 3, 3, 3, 3]
|
||||||
|
groups = _lpt_distribute(items, weights, 3)
|
||||||
|
assert all(len(g) == 2 for g in groups)
|
||||||
|
|
||||||
|
def test_heavy_items_on_different_runners(self) -> None:
|
||||||
|
"""Two heavy items should go to different runners."""
|
||||||
|
items = ["heavy-a", "heavy-b", "light-1", "light-2"]
|
||||||
|
weights = [10, 10, 1, 1]
|
||||||
|
groups = _lpt_distribute(items, weights, 2)
|
||||||
|
# Heavy items should be on different runners
|
||||||
|
flat = [item for group in groups for item in group]
|
||||||
|
assert "heavy-a" in flat
|
||||||
|
assert "heavy-b" in flat
|
||||||
|
runner_a = next(i for i, g in enumerate(groups) if "heavy-a" in g)
|
||||||
|
runner_b = next(i for i, g in enumerate(groups) if "heavy-b" in g)
|
||||||
|
assert runner_a != runner_b
|
||||||
|
|
||||||
|
def test_load_balance_with_varying_weights(self) -> None:
|
||||||
|
"""LPT should produce better load balance than round-robin."""
|
||||||
|
items = list(range(7))
|
||||||
|
# Simulate infra-like weights: 2 heavy, 2 medium, 3 light
|
||||||
|
weights = [10, 10, 7, 7, 3, 3, 3]
|
||||||
|
groups = _lpt_distribute(items, weights, 3)
|
||||||
|
loads = [sum(weights[i] for i in g) for g in groups]
|
||||||
|
# LPT should produce loads close to total/3 = 43/3 ≈ 14.3
|
||||||
|
# Round-robin would produce: 10+7+3=20, 10+7+3=20, 3=3 (terrible)
|
||||||
|
assert max(loads) - min(loads) <= 10 # Reasonably balanced
|
||||||
|
|
||||||
|
def test_more_runners_than_items(self) -> None:
|
||||||
|
items = ["a"]
|
||||||
|
weights = [5]
|
||||||
|
groups = _lpt_distribute(items, weights, 5)
|
||||||
|
assert len(groups) == 5
|
||||||
|
assert len(groups[0]) == 1
|
||||||
|
assert all(len(g) == 0 for g in groups[1:])
|
||||||
|
|
||||||
|
def test_empty_items(self) -> None:
|
||||||
|
groups = _lpt_distribute([], [], 3)
|
||||||
|
assert groups == [[], [], []]
|
||||||
|
|
||||||
|
def test_preserves_all_items(self) -> None:
|
||||||
|
items = ["a", "b", "c", "d", "e"]
|
||||||
|
weights = [5, 3, 8, 1, 2]
|
||||||
|
groups = _lpt_distribute(items, weights, 3)
|
||||||
|
flat = sorted(item for group in groups for item in group)
|
||||||
|
assert flat == sorted(items)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDistributeLpt:
|
||||||
|
def test_nextcloud_on_separate_runners(self) -> None:
|
||||||
|
"""Two nextcloud scenarios should go to different runners."""
|
||||||
|
pairs = [
|
||||||
|
TestPair("nextcloud", {"name": "p", "image": "i", "command": ""}),
|
||||||
|
TestPair("nextcloud-backup", {"name": "p", "image": "i", "command": ""}),
|
||||||
|
TestPair("binary", {"name": "p", "image": "i", "command": ""}),
|
||||||
|
TestPair("default", {"name": "p", "image": "i", "command": ""}),
|
||||||
|
]
|
||||||
|
groups = distribute(pairs, 2)
|
||||||
|
# Both nextcloud scenarios (weight 10) should be on different runners
|
||||||
|
runner_0 = [p.scenario for p in groups[0]]
|
||||||
|
runner_1 = [p.scenario for p in groups[1]]
|
||||||
|
# nextcloud and nextcloud-backup should NOT be on the same runner
|
||||||
|
assert not ("nextcloud" in runner_0 and "nextcloud-backup" in runner_0)
|
||||||
|
assert not ("nextcloud" in runner_1 and "nextcloud-backup" in runner_1)
|
||||||
|
|||||||
Reference in New Issue
Block a user