Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 core-devops
|
65831c839e | ||
|
core-devops
|
d342149646 | ||
|
core-devops
|
54907ee852 | ||
|
core-devops
|
99453c6a71 |
@@ -23,6 +23,7 @@ import dataclasses
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -326,6 +327,43 @@ def update_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
)
|
||||
|
||||
|
||||
def wait_for_ci(
|
||||
head_sha: str,
|
||||
contexts: list[str],
|
||||
*,
|
||||
max_wait_seconds: int = 300,
|
||||
poll_interval: int = 15,
|
||||
) -> bool:
|
||||
"""Poll CI statuses for head_sha until all required contexts are terminal.
|
||||
|
||||
Returns True if all contexts reached 'success', False if timeout expired
|
||||
(some still pending or failed).
|
||||
|
||||
Background: after a queue-triggered PR update, CI re-runs on the new head.
|
||||
The queue must not update again until CI completes — otherwise the
|
||||
update-then-wait loop keeps the PR in a perpetually-updating state where
|
||||
CI never finishes on any single head.
|
||||
"""
|
||||
deadline = time.time() + max_wait_seconds
|
||||
while time.time() < deadline:
|
||||
time.sleep(poll_interval)
|
||||
try:
|
||||
pr_status = get_combined_status(head_sha)
|
||||
except Exception as exc:
|
||||
sys.stderr.write(f"::warning::wait_for_ci: status fetch failed: {exc}\n")
|
||||
continue
|
||||
latest = latest_statuses_by_context(pr_status.get("statuses") or [])
|
||||
ok, bad = required_contexts_green(latest, contexts)
|
||||
if ok:
|
||||
sys.stderr.write(f"::notice::wait_for_ci: all contexts green after {int(time.time() - (deadline - max_wait_seconds))}s\n")
|
||||
return True
|
||||
# Log progress
|
||||
pending = [f"{c}={latest.get(c, {}).get('status', 'missing')}" for c in contexts if latest.get(c, {}).get('status') != 'success']
|
||||
sys.stderr.write(f"::notice::wait_for_ci: still waiting ({int(deadline - time.time())}s left): {', '.join(pending[:3])}\n")
|
||||
sys.stderr.write(f"::warning::wait_for_ci: timeout after {max_wait_seconds}s; proceeding with merge check\n")
|
||||
return False
|
||||
|
||||
|
||||
def merge_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
payload = {
|
||||
"Do": "merge",
|
||||
@@ -338,7 +376,24 @@ def merge_pull(pr_number: int, *, dry_run: bool) -> None:
|
||||
print(f"::notice::merging PR #{pr_number}")
|
||||
if dry_run:
|
||||
return
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
|
||||
# Gitea's merge endpoint returns HTTP 200 with an empty body on success.
|
||||
# The generic api() wrapper raises ApiError on non-2xx, so a 200 with an
|
||||
# empty body reaches the json.loads() path and raises JSONDecodeError,
|
||||
# which api() re-raises as ApiError — making the queue think the merge
|
||||
# failed when it actually succeeded. Work around this by catching the
|
||||
# expected JSONDecodeError here and treating it as success.
|
||||
try:
|
||||
api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
|
||||
except ApiError as exc:
|
||||
# Surface non-merge errors (5xx server errors, 403 forbidden, etc.)
|
||||
if "merge" in str(exc).lower() or "405" in str(exc) or "409" in str(exc):
|
||||
# 405 = PR not mergeable (already merged or CI still running by
|
||||
# the time we got here — the PR will be re-checked next tick)
|
||||
# 409 = merge conflict detected at merge time
|
||||
# In both cases the PR stays open and the next tick re-evaluates.
|
||||
sys.stderr.write(f"::warning::merge call returned: {exc}\n")
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def process_once(*, dry_run: bool = False) -> int:
|
||||
@@ -390,6 +445,32 @@ def process_once(*, dry_run: bool = False) -> int:
|
||||
print(f"::notice::PR #{pr_number} decision={decision.action}: {decision.reason}")
|
||||
if decision.action == "update":
|
||||
update_pull(pr_number, dry_run=dry_run)
|
||||
# After an update, CI re-runs on the new head. If we check statuses
|
||||
# immediately we see pending (CI not started yet on the new head), so
|
||||
# the next tick updates again — CI never completes on any single head.
|
||||
# Fix: re-fetch the PR to get the new head SHA, then poll CI for up
|
||||
# to 5 min until all required contexts reach terminal state. If CI
|
||||
# finishes in time, proceed to merge on the same tick.
|
||||
if not dry_run:
|
||||
updated_pr = get_pull(pr_number)
|
||||
new_head = updated_pr.get("head", {}).get("sha", "")
|
||||
if new_head and new_head != head_sha:
|
||||
sys.stderr.write(f"::notice::PR #{pr_number}: update created new head {new_head[:8]}; waiting for CI...\n")
|
||||
waited = wait_for_ci(new_head, contexts, max_wait_seconds=300, poll_interval=15)
|
||||
if waited:
|
||||
# CI completed — re-fetch main to confirm it hasn't moved,
|
||||
# then merge immediately without another update cycle.
|
||||
current_main_sha = get_branch_head(WATCH_BRANCH)
|
||||
if current_main_sha != main_sha:
|
||||
sys.stderr.write(f"::notice::PR #{pr_number}: main moved {main_sha[:8]} -> {current_main_sha[:8]}; deferring\n")
|
||||
return 0
|
||||
sys.stderr.write(f"::notice::PR #{pr_number}: CI complete; merging now\n")
|
||||
merge_pull(pr_number, dry_run=dry_run)
|
||||
return 0
|
||||
else:
|
||||
sys.stderr.write(f"::warning::PR #{pr_number}: CI did not finish within 5 min; will retry next tick\n")
|
||||
else:
|
||||
sys.stderr.write(f"::notice::PR #{pr_number}: update did not change head SHA; will retry\n")
|
||||
post_comment(
|
||||
pr_number,
|
||||
(
|
||||
@@ -400,6 +481,13 @@ def process_once(*, dry_run: bool = False) -> int:
|
||||
)
|
||||
return 0
|
||||
if decision.ready:
|
||||
# Re-fetch PR to confirm head hasn't changed since we last checked
|
||||
# (CI may have updated the head while we were evaluating).
|
||||
current_pr = get_pull(pr_number)
|
||||
current_head = current_pr.get("head", {}).get("sha", "")
|
||||
if current_head != head_sha:
|
||||
print(f"::notice::PR #{pr_number} head changed {head_sha[:8]} -> {current_head[:8]}; re-evaluating")
|
||||
return 0
|
||||
latest_main_sha = get_branch_head(WATCH_BRANCH)
|
||||
if latest_main_sha != main_sha:
|
||||
print(
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
# ci-arm64-advisory — Mac arm64 self-hosted ADVISORY fast-check lane.
|
||||
#
|
||||
# === WHY ===
|
||||
#
|
||||
# The amd64 Gitea runner pool (molecule-runner-1..20) is queue-contended
|
||||
# (internal#418). This lane offloads the *genuinely container-independent*
|
||||
# fast checks (Go build/vet/lint, shellcheck, Python lint) onto the Mac
|
||||
# arm64 self-hosted runner so developers get a fast arm64 signal WITHOUT
|
||||
# adding load to the starved amd64 pool — capability-honestly, as an
|
||||
# additive pilot. Pilot ② of the Mac-CI strategy (CTO-delegated 2026-05-17).
|
||||
#
|
||||
# === NON-NEGOTIABLE SAFETY CONTRACT (the prime directive) ===
|
||||
#
|
||||
# This lane is **ADVISORY ONLY**. It is provably incapable of hanging a
|
||||
# merge. Concretely:
|
||||
#
|
||||
# 1. It is a SEPARATE workflow file. `ci.yml` is byte-for-byte
|
||||
# untouched by this PR. The `CI / all-required` aggregator sentinel
|
||||
# and the five contexts it polls
|
||||
# (`CI / Detect changes|Platform (Go)|Canvas (Next.js)|
|
||||
# Shellcheck (E2E scripts)|Python Lint & Test (pull_request)`)
|
||||
# are unchanged. The canonical required gate stays 100% on the
|
||||
# existing amd64 pool.
|
||||
#
|
||||
# 2. The context this workflow emits is
|
||||
# `ci-arm64-advisory / fast-checks (pull_request)`. That string is
|
||||
# DELIBERATELY NOT present in, and this PR does NOT add it to:
|
||||
# - branch_protections/{main,staging}.status_check_contexts
|
||||
# (DB-verified pb 86/75 = exactly
|
||||
# ["CI / all-required (pull_request)",
|
||||
# "sop-checklist / all-items-acked (pull_request)"])
|
||||
# - audit-force-merge.yml REQUIRED_CHECKS env
|
||||
# - ci.yml `all-required` sentinel's hardcoded `required[]` list
|
||||
# Branch protection therefore never waits on this context. If the
|
||||
# Mac runner is absent / offline / removed, this workflow's status
|
||||
# simply never appears — and because nothing requires it, every
|
||||
# merge proceeds exactly as it does today. There is no path by
|
||||
# which a missing/red arm64 status blocks a merge.
|
||||
#
|
||||
# 3. `continue-on-error: true` on the job — even a genuine arm64-only
|
||||
# failure (toolchain drift, arch-specific test flake) is surfaced
|
||||
# as information, never as a merge blocker, for the duration of
|
||||
# the pilot.
|
||||
#
|
||||
# 4. The job carries a `github.event_name` `if:` gate. Beyond its
|
||||
# functional purpose this also keeps the job OUT of
|
||||
# `ci-required-drift.py:ci_job_names()` (which excludes
|
||||
# `github.event_name`/`github.ref`-gated jobs), so the hourly
|
||||
# ci-required-drift sentinel's F1 ("job not under sentinel needs")
|
||||
# cannot ever flag this advisory job. F2/F3 are untouched because
|
||||
# this context is absent from BP and from REQUIRED_CHECKS.
|
||||
# `lint-bp-context-emit-match` only fails on BP→emitter gaps; an
|
||||
# emitter without a BP context is explicitly informational there.
|
||||
#
|
||||
# === RUNNER TARGETING ===
|
||||
#
|
||||
# The Mac runner is `hongming-pc-runner-1`. The bare `self-hosted`
|
||||
# label is POLLUTED in this Gitea instance: molecule-runner-1..20
|
||||
# (the contended amd64 pool) also advertise `self-hosted`. Targeting
|
||||
# bare `self-hosted` would route back onto the very pool we are trying
|
||||
# to relieve — and onto amd64 hardware. We therefore require an
|
||||
# AND-set of labels that ONLY the Mac satisfies. `macos-self-hosted`
|
||||
# is Mac-exclusive (the amd64 pool does not carry it). Until the
|
||||
# label-install burst (a10862b2) lands `self-hosted`+`macos-self-hosted`
|
||||
# on the Mac, the runner's current unique label `hongming-pc-laptop`
|
||||
# is also listed; AND-semantics over the labels a runner advertises
|
||||
# means a job requiring [self-hosted, macos-self-hosted] can ONLY be
|
||||
# claimed once the Mac advertises both. If neither label set is yet
|
||||
# present on the Mac, the workflow stays queued harmlessly and is
|
||||
# garbage-collected by the normal stale-run reaper — it blocks nothing
|
||||
# (see safety contract point 2).
|
||||
#
|
||||
# === ROLLBACK ===
|
||||
#
|
||||
# Delete this single file (`git rm .gitea/workflows/ci-arm64-advisory.yml`)
|
||||
# and merge. No branch-protection edit, no ci.yml edit, no
|
||||
# REQUIRED_CHECKS edit is required to roll back, because none were made
|
||||
# to roll forward. Zero blast radius either direction.
|
||||
|
||||
name: ci-arm64-advisory
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
# Per-ref cancel: a newer commit on the same ref supersedes the older
|
||||
# advisory run. Distinct from ci.yml's `ci-${ref}` group so this lane
|
||||
# never cancels (or is cancelled by) the canonical required CI.
|
||||
concurrency:
|
||||
group: ci-arm64-advisory-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
fast-checks:
|
||||
name: fast-checks
|
||||
# AND-set: only the Mac arm64 runner advertises macos-self-hosted.
|
||||
# See "RUNNER TARGETING" header note for why bare self-hosted is unsafe.
|
||||
runs-on: [self-hosted, macos-self-hosted]
|
||||
# ADVISORY: never blocks. See safety contract point 3.
|
||||
continue-on-error: true
|
||||
# event_name gate: functional (only meaningful on push/PR) AND keeps
|
||||
# this job out of ci-required-drift.py:ci_job_names() so F1 can never
|
||||
# flag it. See safety contract point 4.
|
||||
if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: Provenance — advisory lane, non-gating
|
||||
run: |
|
||||
echo "This is the arm64 ADVISORY fast-check lane."
|
||||
echo "It does NOT gate merges. Canonical required CI is ci.yml"
|
||||
echo "on the amd64 pool. Arch: $(uname -m) on $(uname -s)."
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# ---- Go: build + vet + lint (container-independent: needs only the
|
||||
# Go toolchain; no amd64 ECR image, no docker-in-job). Race-detector
|
||||
# unit-test + coverage gates are deliberately NOT duplicated here —
|
||||
# those stay authoritative on amd64 ci.yml `Platform (Go)`. This lane
|
||||
# is fast-feedback for the compile/vet/lint surface only. ----
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- name: Go build + vet (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go mod download
|
||||
go build ./cmd/server
|
||||
go vet ./...
|
||||
- name: golangci-lint (workspace-server)
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
"$(go env GOPATH)/bin/golangci-lint" run --timeout 3m ./...
|
||||
|
||||
# ---- Shellcheck (container-independent: shellcheck binary only).
|
||||
# Mirrors ci.yml `Shellcheck (E2E scripts)` bulk pass scope. ----
|
||||
- name: Install shellcheck (arm64)
|
||||
run: |
|
||||
if ! command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "shellcheck not preinstalled on this self-hosted runner."
|
||||
echo "Attempting Homebrew install (Mac arm64)."
|
||||
brew install shellcheck || {
|
||||
echo "::warning::shellcheck unavailable on runner; advisory shellcheck skipped."
|
||||
exit 0
|
||||
}
|
||||
fi
|
||||
shellcheck --version
|
||||
- name: Shellcheck tests/e2e + infra/scripts
|
||||
run: |
|
||||
command -v shellcheck >/dev/null 2>&1 || { echo "skip"; exit 0; }
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
# ---- Python lint/compile (container-independent: CPython only).
|
||||
# Lint + import-compile surface; the authoritative pytest + coverage
|
||||
# floors stay on amd64 ci.yml `Python Lint & Test`. ----
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Python byte-compile (workspace)
|
||||
working-directory: workspace
|
||||
run: |
|
||||
python -m pip install --quiet ruff || true
|
||||
python -m compileall -q .
|
||||
if command -v ruff >/dev/null 2>&1; then
|
||||
ruff check . || echo "::warning::ruff findings (advisory only)"
|
||||
fi
|
||||
|
||||
- name: Advisory summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "## arm64 advisory fast-checks complete"
|
||||
echo ""
|
||||
echo "This lane is **advisory** — it does not gate merges."
|
||||
echo "Authoritative required CI remains \`CI / all-required\`"
|
||||
echo "on the amd64 pool (\`ci.yml\`, unchanged by this PR)."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -32,6 +32,12 @@ on:
|
||||
# iterating all open PRs when PR_NUMBER is empty.
|
||||
workflow_dispatch:
|
||||
|
||||
# Cancel stale runs so the 8-runner pool stays available for PR jobs.
|
||||
# Per-SHA group ensures push and cron runs at different SHAs don't cancel each other.
|
||||
concurrency:
|
||||
group: gate-check-v3-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
# read: contents — for checkout (base ref, not PR head for security)
|
||||
# read: pull-requests — for reading PR info via API
|
||||
|
||||
@@ -162,7 +162,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
python -m twine upload \
|
||||
--verbose \
|
||||
--repository pypi \
|
||||
--username __token__ \
|
||||
--password "$PYPI_TOKEN" \
|
||||
|
||||
@@ -44,6 +44,12 @@ on:
|
||||
- ".github/scripts/lint_secret_pattern_drift.py"
|
||||
- ".githooks/pre-commit"
|
||||
|
||||
# Cancel stale runs to keep the 8-runner pool available for PR jobs.
|
||||
# Per-SHA group ensures push and scheduled runs at different SHAs don't cancel each other.
|
||||
concurrency:
|
||||
group: secret-pattern-drift-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
|
||||
@@ -22,6 +22,11 @@ on:
|
||||
- cron: '17 4 * * 1' # Mondays at 04:17 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
# Cancel stale runs to keep the 8-runner pool available for PR jobs.
|
||||
concurrency:
|
||||
group: weekly-platform-go-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
statuses: write
|
||||
|
||||
Reference in New Issue
Block a user