Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 core-fe
|
dec97600ff | ||
|
core-fe
|
f1904b8a47 | ||
|
core-fe
|
ddc38360b6 | ||
|
core-fe
|
3886713b6d | ||
|
core-fe
|
8bd4023f1d | ||
|
core-fe
|
73f88b9e8b | ||
|
core-fe
|
19062a01d4 | ||
|
core-fe
|
7e2335d9d8 | ||
|
core-fe
|
d68235457e | ||
 core-uiux
|
0dc9cdf16c | ||
|
core-uiux
|
8ef7f95ddc | ||
|
core-uiux
|
045380b5d4 | ||
|
core-uiux
|
208d454a04 | ||
|
core-uiux
|
1a3ae5cfb3 | ||
|
core-uiux
|
7cbfe5671d | ||
|
core-uiux
|
cf9056b696 | ||
|
core-uiux
|
1beeaccbbf | ||
|
core-uiux
|
2f1704a6ce | ||
|
core-uiux
|
b6d596e596 | ||
|
core-uiux
|
7066d3a57f | ||
|
core-uiux
|
fe3be1fa8c | ||
|
core-uiux
|
aa7fd522eb | ||
|
core-uiux
|
a039cb5b68 | ||
|
core-uiux
|
a633c4b9d4 | ||
|
core-uiux
|
5bbc42ed21 | ||
|
core-uiux
|
ba8c70f0a4 |
@@ -1,591 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ci-required-drift — RFC internal#219 §4 + §6.
|
||||
|
||||
Detects drift between three sources of "what counts as a required check"
|
||||
for this repo, files (or updates) a `[ci-drift]` Gitea issue when any
|
||||
pair diverges.
|
||||
|
||||
Sources:
|
||||
A. `.gitea/workflows/ci.yml` jobs (CI source — the actual job set)
|
||||
B. `status_check_contexts` in branch_protections (the merge gate)
|
||||
C. `REQUIRED_CHECKS` env in audit-force-merge.yml (the audit env)
|
||||
|
||||
Three failure classes:
|
||||
F1 Job in (A) is not under the sentinel's `needs:` — sentinel
|
||||
doesn't gate it, so a red job on that name can sneak through.
|
||||
Ignores jobs whose `if:` references `github.event_name` (those
|
||||
run only on specific events and may be `skipped` legitimately).
|
||||
F2 Context in (B) corresponds to no emitter — i.e. there's no job
|
||||
in ci.yml whose runtime status-name maps to that context.
|
||||
A stale required-check name is silent: protection demands a
|
||||
green it never receives, but Gitea treats absent-as-pending,
|
||||
not absent-as-red. The gate degrades to advisory.
|
||||
F3 (B) and (C) are not set-equal. Audit env wider than protection
|
||||
→ audit flags non-force-merges as force; narrower → real
|
||||
force-merges are missed.
|
||||
|
||||
Idempotency:
|
||||
Searches OPEN issues by exact title prefix
|
||||
`[ci-drift] {repo}/{branch}: ` and either edits the existing one
|
||||
(if any) or POSTs a new one. Never spawns duplicates.
|
||||
|
||||
Behavior-based AST gate per `feedback_behavior_based_ast_gates`:
|
||||
- Job set comes from PyYAML parse of jobs:* keys
|
||||
- Sentinel needs from PyYAML parse of jobs[sentinel].needs (a list)
|
||||
- Audit env from PyYAML parse, NOT grep — so reformatting the YAML
|
||||
(block-scalar `|` vs flow-style list) does not break the gate
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
import yaml # PyYAML 6.0.2 — installed by the workflow before this runs.
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment
|
||||
# --------------------------------------------------------------------------
|
||||
def env(key: str, *, required: bool = True, default: str | None = None) -> str:
|
||||
val = os.environ.get(key, default)
|
||||
if required and not val:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return val or ""
|
||||
|
||||
|
||||
GITEA_TOKEN = env("GITEA_TOKEN", required=False)
|
||||
GITEA_HOST = env("GITEA_HOST", required=False)
|
||||
REPO = env("REPO", required=False)
|
||||
BRANCHES = env("BRANCHES", required=False).split()
|
||||
SENTINEL_JOB = env("SENTINEL_JOB", required=False)
|
||||
AUDIT_WORKFLOW_PATH = env("AUDIT_WORKFLOW_PATH", required=False)
|
||||
CI_WORKFLOW_PATH = env("CI_WORKFLOW_PATH", required=False)
|
||||
DRIFT_LABEL = env("DRIFT_LABEL", required=False)
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only. Tests import
|
||||
individual functions without setting the full env contract."""
|
||||
for key in (
|
||||
"GITEA_TOKEN",
|
||||
"GITEA_HOST",
|
||||
"REPO",
|
||||
"BRANCHES",
|
||||
"SENTINEL_JOB",
|
||||
"AUDIT_WORKFLOW_PATH",
|
||||
"CI_WORKFLOW_PATH",
|
||||
"DRIFT_LABEL",
|
||||
):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper (no requests dependency)
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API call cannot be trusted to have succeeded.
|
||||
|
||||
Covers non-2xx HTTP status AND 2xx with an unparseable JSON body on
|
||||
endpoints that are documented to return JSON (search/read). Callers
|
||||
that swallow this and proceed would risk e.g. creating duplicate
|
||||
`[ci-drift]` issues when a transient 500 hides an existing match.
|
||||
The cron retries hourly; one fail-loud cycle is fine — silent
|
||||
duplicate creation is not (per Five-Axis review on PR #112).
|
||||
"""
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
"""Tiny HTTP helper around urllib.
|
||||
|
||||
Raises ApiError on any non-2xx response. Callers that want
|
||||
best-effort semantics (e.g. label-apply) must `try/except ApiError`
|
||||
explicitly — making the failure-soft path opt-in rather than the
|
||||
default closes the duplicate-issue regression class.
|
||||
|
||||
For 2xx responses with a JSON body that fails to parse, raises
|
||||
ApiError when `expect_json=True` (the default for read-shaped
|
||||
paths). On endpoints that legitimately return non-JSON success
|
||||
bodies (e.g. some Gitea create echoes — see
|
||||
`feedback_gitea_create_api_unparseable_response`), callers may pass
|
||||
`expect_json=False` to accept a `_raw` fallthrough — but they MUST
|
||||
then verify success via a follow-up GET, not by trusting the body.
|
||||
"""
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status}: {snippet}"
|
||||
)
|
||||
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
if expect_json:
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status} but body is not JSON: {e}"
|
||||
) from e
|
||||
# Opt-in raw fallthrough for endpoints with known echo-quirks.
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# YAML loaders — STRICT (reject GitHub-Actions-only syntax)
|
||||
# --------------------------------------------------------------------------
|
||||
def load_yaml(path: str) -> dict:
|
||||
"""Load + parse a workflow YAML. Hard-fail if the file is missing
|
||||
or doesn't parse — drift-detect cannot make decisions without
|
||||
knowing the actual job set."""
|
||||
if not os.path.exists(path):
|
||||
sys.stderr.write(f"::error::file not found: {path}\n")
|
||||
sys.exit(3)
|
||||
with open(path, encoding="utf-8") as f:
|
||||
try:
|
||||
doc = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(f"::error::YAML parse error in {path}: {e}\n")
|
||||
sys.exit(3)
|
||||
if not isinstance(doc, dict):
|
||||
sys.stderr.write(f"::error::{path} is not a YAML mapping\n")
|
||||
sys.exit(3)
|
||||
return doc
|
||||
|
||||
|
||||
def ci_jobs_all(ci_doc: dict) -> set[str]:
|
||||
"""Every job key in ci.yml minus the sentinel itself. Used for F1b
|
||||
(sentinel.needs typo check) — needs that name a non-existent job
|
||||
is a typo regardless of event-gating."""
|
||||
jobs = ci_doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
sys.stderr.write("::error::ci.yml has no jobs: mapping\n")
|
||||
sys.exit(3)
|
||||
return {k for k in jobs if k != SENTINEL_JOB}
|
||||
|
||||
|
||||
def ci_job_names(ci_doc: dict) -> set[str]:
|
||||
"""Set of job keys in ci.yml MINUS the sentinel itself MINUS jobs
|
||||
whose `if:` gates on `github.event_name` (those are event-scoped
|
||||
and can legitimately be `skipped` for a given trigger; if we
|
||||
required them under the sentinel `needs:`, every PR-only job
|
||||
would be `skipped` on push and the sentinel would interpret
|
||||
`skipped != success` as failure). RFC §4 spec.
|
||||
|
||||
Used for F1 (jobs missing from sentinel needs). NOT used for F1b
|
||||
(typos in needs) — see `ci_jobs_all` for that."""
|
||||
jobs = ci_doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
sys.stderr.write("::error::ci.yml has no jobs: mapping\n")
|
||||
sys.exit(3)
|
||||
names: set[str] = set()
|
||||
for k, v in jobs.items():
|
||||
if k == SENTINEL_JOB:
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
gate = v.get("if")
|
||||
if isinstance(gate, str) and "github.event_name" in gate:
|
||||
continue
|
||||
names.add(k)
|
||||
return names
|
||||
|
||||
|
||||
def sentinel_needs(ci_doc: dict) -> set[str]:
|
||||
sentinel = ci_doc.get("jobs", {}).get(SENTINEL_JOB)
|
||||
if not isinstance(sentinel, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::sentinel job '{SENTINEL_JOB}' not found in {CI_WORKFLOW_PATH}\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
needs = sentinel.get("needs", [])
|
||||
if isinstance(needs, str):
|
||||
needs = [needs]
|
||||
if not isinstance(needs, list):
|
||||
sys.stderr.write("::error::sentinel `needs:` is neither list nor string\n")
|
||||
sys.exit(3)
|
||||
return set(needs)
|
||||
|
||||
|
||||
def required_checks_env(audit_doc: dict) -> set[str]:
|
||||
"""Pull the REQUIRED_CHECKS env value from audit-force-merge.yml.
|
||||
Walks the YAML AST per `feedback_behavior_based_ast_gates`: we do
|
||||
NOT grep for `REQUIRED_CHECKS:` — that breaks under reformatting,
|
||||
multi-job workflows, or a future move of the env to a different
|
||||
step. Instead, look inside every job's every step's `env:` map."""
|
||||
found: list[str] = []
|
||||
jobs = audit_doc.get("jobs", {})
|
||||
if not isinstance(jobs, dict):
|
||||
sys.stderr.write(f"::warning::{AUDIT_WORKFLOW_PATH} has no jobs: mapping\n")
|
||||
return set()
|
||||
for job in jobs.values():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
for step in job.get("steps", []) or []:
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
step_env = step.get("env") or {}
|
||||
if isinstance(step_env, dict) and "REQUIRED_CHECKS" in step_env:
|
||||
v = step_env["REQUIRED_CHECKS"]
|
||||
if isinstance(v, str):
|
||||
found.append(v)
|
||||
if not found:
|
||||
sys.stderr.write(
|
||||
f"::error::REQUIRED_CHECKS env not found in any step of {AUDIT_WORKFLOW_PATH}\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
if len(found) > 1:
|
||||
# Defensive: refuse to guess which one is canonical.
|
||||
sys.stderr.write(
|
||||
f"::error::REQUIRED_CHECKS env present in {len(found)} steps; ambiguous\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
raw = found[0]
|
||||
# YAML block-scalars (`|`) leave a trailing newline + blanks; trim
|
||||
# consistently with audit-force-merge.sh's parser so both sides
|
||||
# produce identical sets.
|
||||
return {line.strip() for line in raw.splitlines() if line.strip()}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Mapping: ci.yml job-key → protection context name
|
||||
# --------------------------------------------------------------------------
|
||||
def expected_context(job_key: str, workflow_name: str = "ci") -> str:
|
||||
"""Gitea Actions reports status-check contexts as
|
||||
"{workflow.name} / {job.name or job.key} ({event})".
|
||||
|
||||
For ci.yml the event is `pull_request` on PRs (that's what
|
||||
`status_check_contexts` records). Job.name defaults to job.key
|
||||
when no `name:` is set. CP's ci.yml does NOT set per-job `name:`
|
||||
so the key equals the human-name."""
|
||||
return f"{workflow_name} / {job_key} (pull_request)"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Drift detection
|
||||
# --------------------------------------------------------------------------
|
||||
def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
"""Returns (findings, debug). Empty findings == no drift."""
|
||||
findings: list[str] = []
|
||||
|
||||
ci_doc = load_yaml(CI_WORKFLOW_PATH)
|
||||
audit_doc = load_yaml(AUDIT_WORKFLOW_PATH)
|
||||
|
||||
jobs = ci_job_names(ci_doc)
|
||||
jobs_all = ci_jobs_all(ci_doc)
|
||||
needs = sentinel_needs(ci_doc)
|
||||
env_set = required_checks_env(audit_doc)
|
||||
|
||||
# Protection
|
||||
# api() raises ApiError on non-2xx; let it propagate so a transient
|
||||
# 500 fails the run loudly rather than producing a "no drift" lie.
|
||||
_, protection = api("GET", f"/repos/{OWNER}/{NAME}/branch_protections/{branch}")
|
||||
if not isinstance(protection, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::protection response for {branch} not a JSON object\n"
|
||||
)
|
||||
sys.exit(4)
|
||||
contexts = set(protection.get("status_check_contexts") or [])
|
||||
|
||||
# ----- F1: job exists in CI but not under sentinel.needs -----
|
||||
missing_from_needs = sorted(jobs - needs)
|
||||
if missing_from_needs:
|
||||
findings.append(
|
||||
"F1 — jobs in ci.yml NOT under sentinel `needs:` (sentinel doesn't gate them):\n"
|
||||
+ "\n".join(f" - {n}" for n in missing_from_needs)
|
||||
)
|
||||
|
||||
# ----- F1b: needs lists a job that doesn't exist (typo) -----
|
||||
# Compare against jobs_all (incl. event-gated jobs); a typo is a
|
||||
# typo regardless of `if:` gating.
|
||||
stale_needs = sorted(needs - jobs_all)
|
||||
if stale_needs:
|
||||
findings.append(
|
||||
"F1b — sentinel `needs:` lists jobs NOT present in ci.yml (typo or removed job):\n"
|
||||
+ "\n".join(f" - {n}" for n in stale_needs)
|
||||
)
|
||||
|
||||
# ----- F2: protection context has no emitting job -----
|
||||
# Compute the contexts the CI YAML actually produces. The sentinel
|
||||
# is in (B) intentionally (`ci / all-required (pull_request)`); we
|
||||
# whitelist it explicitly.
|
||||
emitted_contexts = {expected_context(j) for j in jobs} | {expected_context(SENTINEL_JOB)}
|
||||
# Contexts NOT produced by ci.yml may still come from other
|
||||
# workflows in the repo (Secret scan etc). We can't enumerate
|
||||
# every workflow's emissions cheaply; instead, flag only contexts
|
||||
# whose prefix is `ci / ` (this workflow's emissions) and which
|
||||
# don't appear in `emitted_contexts`. This narrows F2 to the
|
||||
# failure class the RFC actually targets without producing noise
|
||||
# from cross-workflow emitters.
|
||||
stale_protection = sorted(
|
||||
c for c in contexts if c.startswith("ci / ") and c not in emitted_contexts
|
||||
)
|
||||
if stale_protection:
|
||||
findings.append(
|
||||
"F2 — protection `status_check_contexts` entries with `ci / ` prefix that NO "
|
||||
"job in ci.yml emits (stale name → silent advisory gate):\n"
|
||||
+ "\n".join(f" - {c}" for c in stale_protection)
|
||||
)
|
||||
|
||||
# ----- F3: audit env vs protection contexts (set-equal) -----
|
||||
only_in_env = sorted(env_set - contexts)
|
||||
only_in_protection = sorted(contexts - env_set)
|
||||
if only_in_env:
|
||||
findings.append(
|
||||
"F3a — audit-force-merge.yml `REQUIRED_CHECKS` env has contexts NOT in "
|
||||
f"branch_protections/{branch}.status_check_contexts (audit would flag "
|
||||
"non-force-merges as force):\n"
|
||||
+ "\n".join(f" - {c}" for c in only_in_env)
|
||||
)
|
||||
if only_in_protection:
|
||||
findings.append(
|
||||
"F3b — branch_protections/{br}.status_check_contexts has contexts NOT in "
|
||||
"audit-force-merge.yml `REQUIRED_CHECKS` env (real force-merges would be "
|
||||
"missed):\n".format(br=branch)
|
||||
+ "\n".join(f" - {c}" for c in only_in_protection)
|
||||
)
|
||||
|
||||
debug = {
|
||||
"branch": branch,
|
||||
"ci_jobs": sorted(jobs),
|
||||
"sentinel_needs": sorted(needs),
|
||||
"protection_contexts": sorted(contexts),
|
||||
"audit_env_checks": sorted(env_set),
|
||||
"expected_contexts": sorted(emitted_contexts),
|
||||
}
|
||||
return findings, debug
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Issue file/update
|
||||
# --------------------------------------------------------------------------
|
||||
def title_for(branch: str) -> str:
|
||||
# Idempotency key — keep stable, never include timestamp/SHA.
|
||||
return f"[ci-drift] {REPO}/{branch}: required-checks divergence detected"
|
||||
|
||||
|
||||
def find_open_issue(title: str) -> dict | None:
|
||||
"""Return the existing open `[ci-drift]` issue for `title`, or None.
|
||||
|
||||
`None` means "search succeeded, no match" — NOT "search failed".
|
||||
Per Five-Axis review on PR #112: returning None on a transient API
|
||||
error caused the caller to POST a duplicate issue. Now api() raises
|
||||
ApiError on any non-2xx; we let it propagate. The cron retries
|
||||
hourly; failing one cycle loudly is strictly better than silently
|
||||
duplicating.
|
||||
|
||||
Gitea issue search returns at most page=50 per page; one page is
|
||||
enough as long as `[ci-drift]` issues are a tiny minority. (See
|
||||
follow-up issue for Link-header pagination.)
|
||||
"""
|
||||
_, results = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={"state": "open", "type": "issues", "limit": "50"},
|
||||
)
|
||||
if not isinstance(results, list):
|
||||
raise ApiError(
|
||||
f"issue search returned non-list body (got {type(results).__name__})"
|
||||
)
|
||||
for issue in results:
|
||||
if issue.get("title") == title:
|
||||
return issue
|
||||
return None
|
||||
|
||||
|
||||
def render_body(branch: str, findings: list[str], debug: dict) -> str:
|
||||
body = [
|
||||
f"# Drift detected on `{REPO}/{branch}`",
|
||||
"",
|
||||
"Auto-filed by `.gitea/workflows/ci-required-drift.yml` "
|
||||
"(RFC [internal#219](https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
|
||||
"",
|
||||
"## Findings",
|
||||
"",
|
||||
]
|
||||
body.extend(findings)
|
||||
body.extend(
|
||||
[
|
||||
"",
|
||||
"## Resolution",
|
||||
"",
|
||||
"- **F1 / F1b**: add the missing job to `all-required.needs:` "
|
||||
"in `.gitea/workflows/ci.yml`, or remove the stale entry.",
|
||||
"- **F2**: rename the protection context to match an emitter, "
|
||||
"or remove it from `status_check_contexts` "
|
||||
"(PATCH `/api/v1/repos/{owner}/{repo}/branch_protections/{branch}`).",
|
||||
"- **F3a / F3b**: bring `REQUIRED_CHECKS` env in "
|
||||
"`.gitea/workflows/audit-force-merge.yml` into set-equality with "
|
||||
"`status_check_contexts` (single PR, both files).",
|
||||
"",
|
||||
"## Debug",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(debug, indent=2, sort_keys=True),
|
||||
"```",
|
||||
"",
|
||||
"_This issue is idempotent: drift-detect runs hourly at `:17` "
|
||||
"and edits this body in place. Close the issue once the drift "
|
||||
"is fixed; the next hourly run will reopen if drift returns._",
|
||||
]
|
||||
)
|
||||
return "\n".join(body)
|
||||
|
||||
|
||||
def file_or_update(
|
||||
branch: str,
|
||||
findings: list[str],
|
||||
debug: dict,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""File a new `[ci-drift]` issue, or PATCH the existing one in place.
|
||||
|
||||
`dry_run=True` skips every side-effecting Gitea call (issue
|
||||
search, POST, PATCH, label apply) and prints the would-be issue
|
||||
title + body to stdout. Useful for local testing and for
|
||||
debugging drift output without polluting the issue tracker.
|
||||
"""
|
||||
title = title_for(branch)
|
||||
body = render_body(branch, findings, debug)
|
||||
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would file/update drift issue for {branch}")
|
||||
print(f"::group::[dry-run] title")
|
||||
print(title)
|
||||
print(f"::endgroup::")
|
||||
print(f"::group::[dry-run] body")
|
||||
print(body)
|
||||
print(f"::endgroup::")
|
||||
return
|
||||
|
||||
existing = find_open_issue(title)
|
||||
if existing:
|
||||
num = existing["number"]
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}",
|
||||
body={"body": body},
|
||||
)
|
||||
print(f"::notice::Updated existing drift issue #{num} for {branch}")
|
||||
return
|
||||
|
||||
_, created = api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
body={"title": title, "body": body, "labels": []},
|
||||
)
|
||||
if not isinstance(created, dict):
|
||||
sys.stderr.write("::error::POST issue response not a JSON object\n")
|
||||
sys.exit(5)
|
||||
new_num = created.get("number")
|
||||
print(f"::warning::Filed new drift issue #{new_num} for {branch}")
|
||||
|
||||
# Apply label by name (Gitea's add-labels endpoint accepts label IDs;
|
||||
# look up id by name once). Best-effort: failure to label is logged
|
||||
# but does not fail the audit run — the issue itself IS the alarm.
|
||||
try:
|
||||
_, labels = api("GET", f"/repos/{OWNER}/{NAME}/labels")
|
||||
except ApiError as e:
|
||||
sys.stderr.write(f"::warning::could not list labels: {e}\n")
|
||||
return
|
||||
label_id = None
|
||||
if isinstance(labels, list):
|
||||
for lbl in labels:
|
||||
if lbl.get("name") == DRIFT_LABEL:
|
||||
label_id = lbl.get("id")
|
||||
break
|
||||
if label_id is not None and new_num:
|
||||
try:
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{new_num}/labels",
|
||||
body={"labels": [label_id]},
|
||||
)
|
||||
except ApiError as e:
|
||||
sys.stderr.write(
|
||||
f"::warning::could not apply label '{DRIFT_LABEL}' to #{new_num}: {e}\n"
|
||||
)
|
||||
else:
|
||||
sys.stderr.write(f"::warning::label '{DRIFT_LABEL}' not found on repo\n")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main
|
||||
# --------------------------------------------------------------------------
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="ci-required-drift",
|
||||
description="Detect drift between ci.yml, branch_protections, "
|
||||
"and audit-force-merge.yml REQUIRED_CHECKS env.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect + print findings to stdout; do NOT file or PATCH "
|
||||
"the `[ci-drift]` issue. Useful for local testing and for "
|
||||
"previewing output before turning the workflow loose.",
|
||||
)
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
_require_runtime_env()
|
||||
|
||||
for branch in BRANCHES:
|
||||
findings, debug = detect_drift(branch)
|
||||
if findings:
|
||||
print(f"::warning::Drift detected on {branch}:")
|
||||
for f in findings:
|
||||
print(f)
|
||||
file_or_update(branch, findings, debug, dry_run=args.dry_run)
|
||||
else:
|
||||
print(f"::notice::No drift on {branch}.")
|
||||
print(json.dumps(debug, indent=2, sort_keys=True))
|
||||
# Exit 0 even on drift — the issue IS the alarm, not a red workflow.
|
||||
# A red workflow here would page on a CI rename until the issue is
|
||||
# opened, doubling the noise. The issue itself is the actionable
|
||||
# surface. (`api()` raising ApiError is the only path that exits
|
||||
# non-zero, by design: a transient Gitea outage should fail loudly.)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,589 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""main-red-watchdog — Option C of the "main NEVER goes red" directive.
|
||||
|
||||
Tracking: molecule-core#420.
|
||||
|
||||
What it does (one cron tick):
|
||||
1. GET /api/v1/repos/{owner}/{repo}/branches/{watch_branch}
|
||||
→ current HEAD SHA on the watched branch.
|
||||
2. GET /api/v1/repos/{owner}/{repo}/commits/{SHA}/status
|
||||
→ combined status + per-context statuses.
|
||||
3. If combined state is `failure` (or any individual status is
|
||||
`failure`): open or PATCH an idempotent
|
||||
`[main-red] {repo}: {SHA[:10]}` issue. Body lists each failed
|
||||
status context with `target_url` + `description`.
|
||||
4. If combined state is `success`: close any open `[main-red]
|
||||
{repo}: ...` issue on a previous SHA with a
|
||||
"main returned to green at SHA {current_SHA}" comment.
|
||||
5. Emit one Loki-shaped JSON line via `logger -t main-red-watchdog`
|
||||
so `reference_obs_stack_phase1`'s Vector → Loki path ingests an
|
||||
alert event (queryable in Grafana as
|
||||
`{tenant="operator-host"} |~ "main-red-watchdog"`).
|
||||
|
||||
What it does NOT do:
|
||||
- Auto-revert anything. Option B is explicitly rejected per
|
||||
`feedback_no_such_thing_as_flakes` + `feedback_fix_root_not_symptom`.
|
||||
- Page on its own failures. If api() raises ApiError (transient
|
||||
Gitea outage), the workflow run fails LOUDLY by re-raise — exactly
|
||||
the contract `feedback_api_helper_must_raise_not_return_dict`
|
||||
enforces. Silent fallthrough would re-introduce the duplicate-issue
|
||||
regression class.
|
||||
- Exit non-zero on RED. The issue IS the alarm; failing the watchdog
|
||||
on red would double-page (red workflow + open issue) and create
|
||||
silent-loop risk if the watchdog itself flakes.
|
||||
|
||||
Idempotency strategy:
|
||||
Title is keyed on `{SHA[:10]}` (commit-scoped), NOT just `main`.
|
||||
Rationale:
|
||||
- A fix-forward changes HEAD → next cron tick sees a new SHA;
|
||||
auto-close logic closes the prior `[main-red] OLD_SHA` issue and
|
||||
(if the new HEAD is also red, e.g. a different test fails) files
|
||||
a fresh `[main-red] NEW_SHA`. Lineage is preserved.
|
||||
- A revert that happens to land back on a previously-red SHA
|
||||
(rare) would refer to a CLOSED issue; the watchdog never reopens.
|
||||
That's a deliberate trade-off — the operator will see the latest
|
||||
open issue's `closed` event in the activity feed.
|
||||
|
||||
This module is import-safe: tests import individual functions without
|
||||
invoking main(), so module-level reads use env-with-default and the
|
||||
runtime contract enforcement lives in `_require_runtime_env()`.
|
||||
|
||||
Run locally (dry-run, no API mutation):
|
||||
GITEA_TOKEN=... GITEA_HOST=git.moleculesai.app REPO=owner/repo \\
|
||||
WATCH_BRANCH=main RED_LABEL=tier:high \\
|
||||
python3 .gitea/scripts/main-red-watchdog.py --dry-run
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment
|
||||
# --------------------------------------------------------------------------
|
||||
def _env(key: str, *, default: str = "") -> str:
|
||||
"""Read an env var with a default. Module-import-safe — tests can
|
||||
import this script without setting the full env contract."""
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN")
|
||||
GITEA_HOST = _env("GITEA_HOST")
|
||||
REPO = _env("REPO")
|
||||
WATCH_BRANCH = _env("WATCH_BRANCH", default="main")
|
||||
RED_LABEL = _env("RED_LABEL", default="tier:high")
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
# Title prefix — kept short and stable so the idempotency search can
|
||||
# match by exact title without parsing.
|
||||
TITLE_PREFIX = "[main-red]"
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only.
|
||||
|
||||
Tests import individual functions without setting the full env
|
||||
contract. Mirrors the CP `ci-required-drift.py` pattern so the
|
||||
runtime guard is a single chokepoint.
|
||||
"""
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "WATCH_BRANCH", "RED_LABEL"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper — raises on non-2xx + on JSON-decode-of-expected-JSON.
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API call cannot be trusted to have succeeded.
|
||||
|
||||
Covers non-2xx HTTP status AND 2xx with an unparseable JSON body on
|
||||
endpoints documented to return JSON. Callers that swallow this and
|
||||
proceed risk e.g. creating duplicate `[main-red]` issues when a
|
||||
transient 500 hides an existing match. Per
|
||||
`feedback_api_helper_must_raise_not_return_dict`: soft-failure is
|
||||
opt-in via `expect_json=False`, never the default.
|
||||
"""
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
"""Tiny HTTP helper around urllib.
|
||||
|
||||
Raises ApiError on any non-2xx response, and on JSON-decode failure
|
||||
when `expect_json=True` (the default for read-shaped paths). Mirrors
|
||||
the CP ci-required-drift.py contract exactly so behaviour is
|
||||
cross-checkable.
|
||||
"""
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {path} → HTTP {status}: {snippet}")
|
||||
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
if expect_json:
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status} but body is not JSON: {e}"
|
||||
) from e
|
||||
# Opt-in raw fallthrough for endpoints with known echo-quirks
|
||||
# (`feedback_gitea_create_api_unparseable_response`). Caller
|
||||
# MUST verify success via a follow-up GET, not by trusting body.
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea reads
|
||||
# --------------------------------------------------------------------------
|
||||
def get_head_sha(branch: str) -> str:
|
||||
"""HEAD SHA of `branch`. Raises ApiError on non-2xx."""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/branches/{branch}")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"branch {branch} response not a JSON object")
|
||||
commit = body.get("commit")
|
||||
if not isinstance(commit, dict):
|
||||
raise ApiError(f"branch {branch} response missing `commit` object")
|
||||
sha = commit.get("id") or commit.get("sha")
|
||||
if not isinstance(sha, str) or len(sha) < 7:
|
||||
raise ApiError(f"branch {branch} response has no usable commit SHA")
|
||||
return sha
|
||||
|
||||
|
||||
def get_combined_status(sha: str) -> dict:
|
||||
"""Combined commit status for `sha`. Gitea returns:
|
||||
{
|
||||
"state": "success" | "failure" | "pending" | "error",
|
||||
"statuses": [
|
||||
{"context": "...", "state": "success|failure|pending|error",
|
||||
"target_url": "...", "description": "..."},
|
||||
...
|
||||
],
|
||||
...
|
||||
}
|
||||
Raises ApiError on non-2xx.
|
||||
"""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"status for {sha} response not a JSON object")
|
||||
return body
|
||||
|
||||
|
||||
def is_red(status: dict) -> tuple[bool, list[dict]]:
|
||||
"""Return (is_red, failed_statuses).
|
||||
|
||||
A commit is "red" if combined state is `failure` OR any individual
|
||||
status entry is in {`failure`, `error`}. `pending` and `success`
|
||||
do not trip the watchdog — pending means CI is still running, and
|
||||
that's the normal state immediately after a merge.
|
||||
|
||||
`failed_statuses` is the list of per-context entries whose own
|
||||
`state` is in the red set; useful for the issue body.
|
||||
"""
|
||||
combined = status.get("state")
|
||||
statuses = status.get("statuses") or []
|
||||
red_states = {"failure", "error"}
|
||||
failed = [
|
||||
s for s in statuses
|
||||
if isinstance(s, dict) and s.get("state") in red_states
|
||||
]
|
||||
return (combined in red_states or bool(failed), failed)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Issue file / update / close
|
||||
# --------------------------------------------------------------------------
|
||||
def title_for(sha: str) -> str:
|
||||
"""Idempotency key — `[main-red] {repo}: {SHA[:10]}`.
|
||||
|
||||
Commit-scoped. A fix-forward to a new SHA produces a new title; the
|
||||
prior issue auto-closes via `close_open_red_issues_for_other_shas`.
|
||||
"""
|
||||
return f"{TITLE_PREFIX} {REPO}: {sha[:10]}"
|
||||
|
||||
|
||||
def list_open_red_issues() -> list[dict]:
|
||||
"""All open issues whose title starts with `[main-red] {repo}: `.
|
||||
|
||||
Per Five-Axis review on CP#112 (`feedback_api_helper_must_raise_not_return_dict`):
|
||||
api() raises on non-2xx; we let it propagate. Returning [] on a
|
||||
transient 500 would cause auto-close to skip the cleanup AND the
|
||||
file-or-update path to POST a duplicate — exactly the regression
|
||||
class the helper-raises contract closes.
|
||||
|
||||
Gitea issue search returns at most 50/page; we only need open
|
||||
`[main-red]` issues which are by design ≤ 1 at any time per repo,
|
||||
so a single page is enough.
|
||||
"""
|
||||
_, results = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={"state": "open", "type": "issues", "limit": "50"},
|
||||
)
|
||||
if not isinstance(results, list):
|
||||
raise ApiError(
|
||||
f"issue search returned non-list body (got {type(results).__name__})"
|
||||
)
|
||||
prefix = f"{TITLE_PREFIX} {REPO}: "
|
||||
return [i for i in results if isinstance(i, dict)
|
||||
and isinstance(i.get("title"), str)
|
||||
and i["title"].startswith(prefix)]
|
||||
|
||||
|
||||
def find_open_issue_for_sha(sha: str) -> dict | None:
|
||||
"""Return the existing open `[main-red] {repo}: {SHA[:10]}` issue,
|
||||
or None if no such issue is open.
|
||||
|
||||
`None` means "search succeeded, no match" — NOT "search failed".
|
||||
api() raises ApiError on any non-2xx; the caller can let that
|
||||
propagate so a transient outage fails loudly instead of silently
|
||||
duplicating.
|
||||
"""
|
||||
target = title_for(sha)
|
||||
for issue in list_open_red_issues():
|
||||
if issue.get("title") == target:
|
||||
return issue
|
||||
return None
|
||||
|
||||
|
||||
def render_body(sha: str, failed: list[dict], debug: dict) -> str:
|
||||
"""Issue body. Markdown. Mirrors CP#112's render_body shape."""
|
||||
lines = [
|
||||
f"# Main is RED on `{REPO}` at `{sha[:10]}`",
|
||||
"",
|
||||
f"Commit: <https://{GITEA_HOST}/{REPO}/commit/{sha}>",
|
||||
"",
|
||||
"Auto-filed by `.gitea/workflows/main-red-watchdog.yml` (Option C "
|
||||
"of the [main-never-red directive]"
|
||||
f"(https://{GITEA_HOST}/molecule-ai/molecule-core/issues/420)). "
|
||||
"Per `feedback_no_such_thing_as_flakes` + "
|
||||
"`feedback_fix_root_not_symptom`: investigate the root cause; do "
|
||||
"NOT revert as a reflex. The watchdog itself never reverts.",
|
||||
"",
|
||||
"## Failed status contexts",
|
||||
"",
|
||||
]
|
||||
if not failed:
|
||||
lines.append(
|
||||
"_(Combined state reported `failure`/`error` but no per-context "
|
||||
"entries were in a red state. This usually means a CI emitter "
|
||||
"set combined-status directly without a per-context status. "
|
||||
"Check the most recent workflow run for `main` and trace from "
|
||||
"there.)_"
|
||||
)
|
||||
else:
|
||||
for s in failed:
|
||||
ctx = s.get("context", "(no context)")
|
||||
state = s.get("state", "(no state)")
|
||||
url = s.get("target_url") or ""
|
||||
desc = (s.get("description") or "").strip()
|
||||
entry = f"- **{ctx}** — `{state}`"
|
||||
if url:
|
||||
entry += f" → [logs]({url})"
|
||||
if desc:
|
||||
entry += f"\n - {desc}"
|
||||
lines.append(entry)
|
||||
lines.extend([
|
||||
"",
|
||||
"## Resolution path",
|
||||
"",
|
||||
"1. Read the failed logs (links above).",
|
||||
"2. If reproducible locally, fix forward in a PR targeting `main`.",
|
||||
"3. If the failure is a real flake — STOP. Per "
|
||||
"`feedback_no_such_thing_as_flakes`, intermittent failures are "
|
||||
"real bugs. Investigate to root cause; do not mark as flake.",
|
||||
"4. If the failure is blocking unrelated work for >1 hour, file a "
|
||||
"follow-up issue and assign someone. Do NOT revert without a "
|
||||
"human GO per `feedback_prod_apply_needs_hongming_chat_go` "
|
||||
"(branch protection is a prod surface).",
|
||||
"",
|
||||
"## Debug",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(debug, indent=2, sort_keys=True),
|
||||
"```",
|
||||
"",
|
||||
"_This issue is idempotent: the watchdog runs hourly at `:05` "
|
||||
"and edits this body in place. When `main` returns to green, the "
|
||||
"watchdog will close this issue automatically with a "
|
||||
"\"main returned to green\" comment._",
|
||||
])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def emit_loki_event(event_type: str, sha: str, failed_contexts: list[str]) -> None:
|
||||
"""Emit a JSON line to syslog tag `main-red-watchdog` for
|
||||
`reference_obs_stack_phase1` (Vector → Loki).
|
||||
|
||||
Best-effort: if `logger` isn't on PATH (e.g. local dev macOS without
|
||||
util-linux logger), print to stderr instead. The Gitea Actions
|
||||
Ubuntu runner has util-linux preinstalled.
|
||||
|
||||
Loki labels: the workflow runs on the Ubuntu runner where Vector is
|
||||
NOT configured (Vector lives on the operator host + tenants per
|
||||
`reference_obs_stack_phase1`). The Loki line is still emitted as
|
||||
stdout JSON so the workflow log itself is parseable; treat the
|
||||
syslog call as belt-and-braces for the cases where this script is
|
||||
invoked from a host that DOES have Vector (e.g. operator-host cron
|
||||
fallback in a follow-up PR).
|
||||
"""
|
||||
payload = {
|
||||
"event_type": event_type,
|
||||
"repo": REPO,
|
||||
"sha": sha,
|
||||
"failed_contexts": failed_contexts,
|
||||
}
|
||||
line = json.dumps(payload, sort_keys=True)
|
||||
# Always print to stdout so the workflow log captures it (machine-
|
||||
# readable; `gitea run logs` + Loki ingestion via the operator-host
|
||||
# journald → Vector → Loki path will see this from runners that
|
||||
# forward stdout). Loki query:
|
||||
# {source="gitea-actions"} |~ "main_red_detected"
|
||||
print(f"main-red-watchdog event: {line}")
|
||||
# Best-effort syslog tag so a future "run from operator-host cron"
|
||||
# path picks it up directly via the existing Vector pipeline.
|
||||
if shutil.which("logger"):
|
||||
try:
|
||||
subprocess.run(
|
||||
["logger", "-t", "main-red-watchdog", line],
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
sys.stderr.write(f"::warning::logger call failed: {e}\n")
|
||||
|
||||
|
||||
def file_or_update_red(
|
||||
sha: str,
|
||||
failed: list[dict],
|
||||
debug: dict,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""Open a new `[main-red] {repo}: {SHA[:10]}` issue, or PATCH the
|
||||
existing one's body. Idempotent by title."""
|
||||
title = title_for(sha)
|
||||
body = render_body(sha, failed, debug)
|
||||
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would file/update main-red issue for {sha[:10]}")
|
||||
print("::group::[dry-run] title")
|
||||
print(title)
|
||||
print("::endgroup::")
|
||||
print("::group::[dry-run] body")
|
||||
print(body)
|
||||
print("::endgroup::")
|
||||
return
|
||||
|
||||
existing = find_open_issue_for_sha(sha)
|
||||
if existing:
|
||||
num = existing["number"]
|
||||
api("PATCH", f"/repos/{OWNER}/{NAME}/issues/{num}", body={"body": body})
|
||||
print(f"::notice::Updated existing main-red issue #{num} for {sha[:10]}")
|
||||
return
|
||||
|
||||
_, created = api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
body={"title": title, "body": body, "labels": []},
|
||||
)
|
||||
if not isinstance(created, dict):
|
||||
raise ApiError("POST issue response not a JSON object")
|
||||
new_num = created.get("number")
|
||||
print(f"::warning::Filed new main-red issue #{new_num} for {sha[:10]}")
|
||||
|
||||
# Apply RED_LABEL by id. Gitea's add-labels endpoint takes IDs, not
|
||||
# names (`feedback_gitea_label_delete_by_id` — same rule for add).
|
||||
# Best-effort: label failure is logged but does not fail the run.
|
||||
try:
|
||||
_, labels = api("GET", f"/repos/{OWNER}/{NAME}/labels")
|
||||
except ApiError as e:
|
||||
sys.stderr.write(f"::warning::could not list labels: {e}\n")
|
||||
return
|
||||
label_id = None
|
||||
if isinstance(labels, list):
|
||||
for lbl in labels:
|
||||
if isinstance(lbl, dict) and lbl.get("name") == RED_LABEL:
|
||||
label_id = lbl.get("id")
|
||||
break
|
||||
if label_id is not None and new_num:
|
||||
try:
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{new_num}/labels",
|
||||
body={"labels": [label_id]},
|
||||
)
|
||||
except ApiError as e:
|
||||
sys.stderr.write(
|
||||
f"::warning::could not apply label '{RED_LABEL}' to #{new_num}: {e}\n"
|
||||
)
|
||||
else:
|
||||
sys.stderr.write(f"::warning::label '{RED_LABEL}' not found on repo\n")
|
||||
|
||||
|
||||
def close_open_red_issues_for_other_shas(
|
||||
current_sha: str,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> int:
|
||||
"""When main is green at current_sha, close any open `[main-red]`
|
||||
issues whose title references a different SHA. Returns the number
|
||||
of issues closed.
|
||||
|
||||
Lineage note: we only close issues whose title prefix matches; if
|
||||
a human renamed the issue or added a suffix this won't touch it.
|
||||
That's intentional — manual editorial state takes precedence.
|
||||
"""
|
||||
target_title = title_for(current_sha)
|
||||
open_red = list_open_red_issues()
|
||||
closed = 0
|
||||
for issue in open_red:
|
||||
if issue.get("title") == target_title:
|
||||
# Same SHA — caller should not have invoked this if main is
|
||||
# green. Skip defensively.
|
||||
continue
|
||||
num = issue.get("number")
|
||||
if not isinstance(num, int):
|
||||
continue
|
||||
comment = (
|
||||
f"`main` returned to green at SHA `{current_sha}` "
|
||||
f"(<https://{GITEA_HOST}/{REPO}/commit/{current_sha}>). "
|
||||
"Closing automatically. If the underlying root cause is "
|
||||
"not yet understood, reopen this issue and file a "
|
||||
"postmortem — green-by-flake is still a bug per "
|
||||
"`feedback_no_such_thing_as_flakes`."
|
||||
)
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would close issue #{num} ({issue.get('title')})")
|
||||
closed += 1
|
||||
continue
|
||||
# Comment first, then close. Order matters: a closed issue can
|
||||
# still receive comments, but the activity-feed ordering reads
|
||||
# better with the explanation arriving just before the close.
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}/comments",
|
||||
body={"body": comment},
|
||||
)
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}",
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print(f"::notice::Closed main-red issue #{num} (green at {current_sha[:10]})")
|
||||
closed += 1
|
||||
return closed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main
|
||||
# --------------------------------------------------------------------------
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="main-red-watchdog",
|
||||
description="Detect post-merge CI red on the watched branch and "
|
||||
"file an idempotent issue. Option C of the main-never-red directive.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect + print the would-be issue title/body to stdout; do "
|
||||
"NOT POST/PATCH/close any issues. Useful for local testing.",
|
||||
)
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
def run_once(*, dry_run: bool = False) -> int:
|
||||
"""One watchdog tick. Returns 0 on green or red-issue-filed; lets
|
||||
ApiError propagate on transient outage (workflow run fails loudly,
|
||||
which is correct per the helper-raises contract)."""
|
||||
sha = get_head_sha(WATCH_BRANCH)
|
||||
status = get_combined_status(sha)
|
||||
red, failed = is_red(status)
|
||||
|
||||
debug = {
|
||||
"branch": WATCH_BRANCH,
|
||||
"sha": sha,
|
||||
"combined_state": status.get("state"),
|
||||
"failed_contexts": [s.get("context") for s in failed],
|
||||
"all_contexts": [
|
||||
{"context": s.get("context"), "state": s.get("state")}
|
||||
for s in (status.get("statuses") or [])
|
||||
if isinstance(s, dict)
|
||||
],
|
||||
}
|
||||
|
||||
if red:
|
||||
failed_ctxs = [s.get("context") for s in failed if s.get("context")]
|
||||
emit_loki_event("main_red_detected", sha, failed_ctxs)
|
||||
print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
|
||||
f"{len(failed)} failed context(s)")
|
||||
file_or_update_red(sha, failed, debug, dry_run=dry_run)
|
||||
else:
|
||||
# Green (or pending — pending is treated as not-red so we don't
|
||||
# spam during the post-merge CI window). Close any stale issues
|
||||
# from earlier SHAs only when we're actually green; pending
|
||||
# means CI hasn't finished and the prior issue might still be
|
||||
# accurate.
|
||||
if status.get("state") == "success":
|
||||
closed = close_open_red_issues_for_other_shas(sha, dry_run=dry_run)
|
||||
if closed:
|
||||
emit_loki_event(
|
||||
"main_returned_to_green", sha,
|
||||
[],
|
||||
)
|
||||
print(f"::notice::main is GREEN at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(closed {closed} stale issue(s))")
|
||||
else:
|
||||
print(f"::notice::main is PENDING at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(combined state={status.get('state')!r}; no action)")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
_require_runtime_env()
|
||||
return run_once(dry_run=args.dry_run)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -44,39 +44,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Ensure jq is available. Runners may not have it pre-installed, and the
|
||||
# workflow-level jq install can fail on runners with network restrictions
|
||||
# (GitHub releases not reachable from some runner networks — infra#241
|
||||
# follow-up). This fallback is idempotent — no-op when jq is already on PATH.
|
||||
# SOP_FAIL_OPEN=1 makes this always exit 0 so CI never blocks on jq absence.
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "::notice::jq not found on PATH — attempting install..."
|
||||
_jq_installed="no"
|
||||
# apt-get first (primary) — Ubuntu package mirrors are reliably reachable.
|
||||
if apt-get update -qq && apt-get install -y -qq jq 2>/dev/null; then
|
||||
echo "::notice::jq installed via apt-get: $(jq --version)"
|
||||
_jq_installed="yes"
|
||||
# GitHub binary as secondary fallback — may fail on restricted networks.
|
||||
elif timeout 120 curl -sSL \
|
||||
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
|
||||
-o /usr/local/bin/jq \
|
||||
&& chmod +x /usr/local/bin/jq; then
|
||||
echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
|
||||
_jq_installed="yes"
|
||||
fi
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "::error::jq installation failed — apt-get and GitHub binary both failed."
|
||||
echo "::error::sop-tier-check requires jq for all JSON API parsing."
|
||||
# SOP_FAIL_OPEN=1 is set in the workflow step's env — makes script always
|
||||
# exit 0 so CI never blocks. The SOP-6 tier review gate remains enforced.
|
||||
if [ "${SOP_FAIL_OPEN:-}" = "1" ]; then
|
||||
echo "::warning::SOP_FAIL_OPEN=1 — exiting 0 so CI does not block."
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
debug() {
|
||||
if [ "${SOP_DEBUG:-}" = "1" ]; then
|
||||
echo " [debug] $*" >&2
|
||||
|
||||
@@ -1,88 +1,58 @@
|
||||
# audit-force-merge — emit `incident.force_merge` to the runner log when
|
||||
# a PR is merged with required-status checks NOT all green. Vector picks
|
||||
# audit-force-merge — emit `incident.force_merge` to runner stdout when
|
||||
# a PR is merged with required-status-checks not green. Vector picks
|
||||
# the JSON line off docker_logs and ships to Loki on
|
||||
# molecule-canonical-obs (per `reference_obs_stack_phase1`); query as:
|
||||
#
|
||||
# {host="operator"} |= "event_type" |= "incident.force_merge" | json
|
||||
#
|
||||
# Companion to `audit-force-merge.sh` (script-extract pattern, same as
|
||||
# sop-tier-check). The audit observes BOTH UI-merged and REST-merged PRs
|
||||
# uniformly per `feedback_gh_cli_merge_lies_use_rest`.
|
||||
# Closes the §SOP-6 audit gap (the doc says force-merges write to
|
||||
# `structure_events`, but that table lives in the platform DB, not
|
||||
# Gitea-side; Loki is the practical equivalent for Gitea Actions
|
||||
# events). When the credential / observability stack converges later,
|
||||
# this can sync into structure_events from Loki via a backfill job —
|
||||
# the structured JSON shape is forward-compatible.
|
||||
#
|
||||
# Closes the §SOP-6 audit gap for the molecule-core repo. RFC:
|
||||
# internal#219 §6. Mirrors the same-named workflow in
|
||||
# molecule-controlplane; design rationale lives in the RFC, not here,
|
||||
# to keep the workflow file scannable.
|
||||
# Logic in `.gitea/scripts/audit-force-merge.sh` per the same script-
|
||||
# extract pattern as sop-tier-check.
|
||||
|
||||
name: audit-force-merge
|
||||
|
||||
# pull_request_target loads from the base branch — same security model
|
||||
# as sop-tier-check. Without this, a PR author could rewrite the
|
||||
# workflow on their own PR and skip the audit emission for their own
|
||||
# force-merge. The base-branch checkout below ALSO uses
|
||||
# `base.sha`, not `base.ref`, so a fast-moving base can't slip a
|
||||
# different audit script in under us.
|
||||
# as sop-tier-check. Without this, an attacker could rewrite the
|
||||
# workflow on a PR and skip the audit emission for their own
|
||||
# force-merge. See `.gitea/workflows/sop-tier-check.yml` for the full
|
||||
# rationale.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
# `pull-requests: read` + `contents: read` covers everything the script
|
||||
# needs (fetch PR + commit statuses). `issues:` deliberately omitted —
|
||||
# audit fires-and-forgets to stdout, never opens issues.
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
# Skip when PR is closed without merge — saves a runner.
|
||||
if: github.event.pull_request.merged == true
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# base.sha pinning, NOT base.ref — see header rationale.
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Detect force-merge + emit audit event
|
||||
env:
|
||||
# Same org-level secret the sop-tier-check workflow uses;
|
||||
# falls back to the auto-injected GITHUB_TOKEN if the
|
||||
# org-level SOP_TIER_CHECK_TOKEN isn't set on a transitional
|
||||
# repo.
|
||||
# Same org-level secret the sop-tier-check workflow uses.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
# Required-status-check contexts to evaluate at merge time.
|
||||
# Newline-separated. MUST mirror branch protection's
|
||||
# status_check_contexts for protected branches
|
||||
# (currently `main`; `staging` protection forthcoming per
|
||||
# RFC internal#219 Phase 4).
|
||||
#
|
||||
# Initialized 2026-05-11 from the current molecule-core `main`
|
||||
# branch protection:
|
||||
#
|
||||
# GET /api/v1/repos/molecule-ai/molecule-core/
|
||||
# branch_protections/main
|
||||
# → status_check_contexts = [
|
||||
# "Secret scan / Scan diff for credential-shaped strings (pull_request)",
|
||||
# "sop-tier-check / tier-check (pull_request)"
|
||||
# ]
|
||||
#
|
||||
# Newline-separated. Mirror this against branch protection
|
||||
# (settings → branches → protected branch → required checks).
|
||||
# Declared here rather than fetched from /branch_protections
|
||||
# because that endpoint requires admin write — sop-tier-bot
|
||||
# is read-only by design (least-privilege per
|
||||
# `feedback_least_privilege_via_workflow_env` / internal#257).
|
||||
# Drift between this env and the real protection list is
|
||||
# auto-detected by `ci-required-drift.yml` (RFC §4 + §6),
|
||||
# which opens a `[ci-drift]` issue within one hour.
|
||||
#
|
||||
# When the protection set changes (e.g. Phase 4 adds the
|
||||
# `ci / all-required (pull_request)` sentinel), update BOTH
|
||||
# branch protection AND this env in the SAME PR; drift-detect
|
||||
# will otherwise file an issue for you.
|
||||
# because that endpoint requires admin write — sop-tier-bot is
|
||||
# read-only by design (least-privilege).
|
||||
REQUIRED_CHECKS: |
|
||||
Secret scan / Scan diff for credential-shaped strings (pull_request)
|
||||
sop-tier-check / tier-check (pull_request)
|
||||
Secret scan / Scan diff for credential-shaped strings (pull_request)
|
||||
run: bash .gitea/scripts/audit-force-merge.sh
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
name: Block internal-flavored paths
|
||||
|
||||
# Ported from .github/workflows/block-internal-paths.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group: { types: [checks_requested] }` (Gitea has no
|
||||
# merge queue; no `gh-readonly-queue/...` refs).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Hard CI gate. Internal content (positioning, competitive briefs, sales
|
||||
# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
|
||||
# this public monorepo must never re-acquire those paths. CEO directive
|
||||
# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
|
||||
#
|
||||
# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
|
||||
# briefs into the easiest path their cwd resolves to (root /research,
|
||||
# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
|
||||
# or a stale gitignore line. This workflow is the mechanical backstop.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Block forbidden paths
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base is github.event.pull_request.base.sha,
|
||||
# which may be many commits behind HEAD and therefore absent from the
|
||||
# shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
- name: Refuse if forbidden paths appear
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# pull_request has pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Paths that must NEVER live in the public monorepo. Add to this
|
||||
# list narrowly — broader patterns belong in .gitignore so day-to-day
|
||||
# docs work isn't accidentally blocked.
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"^research/"
|
||||
"^marketing/"
|
||||
"^docs/marketing/"
|
||||
"^comment-[0-9]+\.json$"
|
||||
"^test-pmm.*\.(txt|md)$"
|
||||
"^tick-reflections.*\.(txt|md)$"
|
||||
".*-temp\.(md|txt)$"
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check
|
||||
# the entire tree as if every file were new. Slower but
|
||||
# correct on first push or post-fetch-failure recovery.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFFENDING=""
|
||||
for path in $CHANGED; do
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
if echo "$path" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Forbidden internal-flavored paths detected:"
|
||||
printf "$OFFENDING"
|
||||
echo ""
|
||||
echo "These paths belong in molecule-ai/internal, not this public repo."
|
||||
echo "See docs/internal-content-policy.md for canonical locations."
|
||||
echo ""
|
||||
echo "If your file is genuinely public-facing (e.g. a blog post"
|
||||
echo "ready to ship), use one of these alternatives instead:"
|
||||
echo " - Public-bound blog posts: docs/blog/<slug>.md"
|
||||
echo " - Public-bound tutorials: docs/tutorials/<slug>.md"
|
||||
echo " - Public devrel content: docs/devrel/<slug>.md"
|
||||
echo ""
|
||||
echo "If you legitimately need to add a new top-level path that"
|
||||
echo "happens to match a forbidden pattern, edit"
|
||||
echo ".gitea/workflows/block-internal-paths.yml and update the"
|
||||
echo "FORBIDDEN_PATTERNS list with reviewer signoff."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK No forbidden paths in this change."
|
||||
@@ -1,310 +0,0 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
|
||||
# Ported from .github/workflows/canary-staging.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Minimum viable health check: provisions one Hermes workspace on a fresh
|
||||
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
|
||||
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
|
||||
# issue on the next green run.
|
||||
#
|
||||
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
|
||||
# but runs only on provisioning-critical pushes + nightly — this one
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
- cron: '*/30 * * * *'
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
# Needed to open / close the alerting issue.
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
# failure mode #1 — capturing the actual exception requires
|
||||
# docker logs from the live container.
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain. Operators only need to set ONE of these
|
||||
# secrets; we don't force a choice between them.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
# subsequent failures; auto-close on next green. Comment-on-existing
|
||||
# de-duplicates so a single open issue accumulates the streak —
|
||||
# ops sees one issue with N comments rather than N issues.
|
||||
#
|
||||
# Why no consecutive-failures threshold (e.g., wait 3 runs before
|
||||
# filing): the prior threshold check used
|
||||
# `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
|
||||
# not expose (returns 404). On Gitea Actions the threshold call
|
||||
# ALWAYS failed, breaking the entire alerting step and going days
|
||||
# silent on real regressions (38h+ chronic red on 2026-05-07/08
|
||||
# before this fix; tracked in molecule-core#129). Filing on first
|
||||
# failure is also better UX — we want to know about the first red,
|
||||
# not wait 90 min for it to "count." Real flakes get one issue +
|
||||
# a quick close-on-green; persistent reds accumulate comments.
|
||||
- name: Open issue on failure (Gitea API)
|
||||
if: failure()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Canary failing: staging SaaS smoke"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
|
||||
|
||||
if [ -n "$EXISTING" ]; then
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Canary still failing. " + $run)}')" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
BODY=$(jq -nc --arg t "$TITLE" --arg now "$NOW" --arg run "$RUN_URL" \
|
||||
'{title: $t, body: ("Canary run failed at " + $now + ".\n\nRun: " + $run + "\n\nThis issue auto-closes on the next green canary run. Consecutive failures add a comment here rather than a new issue.")}')
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$BODY" >/dev/null
|
||||
echo "Opened canary failure issue (first red)"
|
||||
fi
|
||||
|
||||
- name: Auto-close canary issue on success (Gitea API)
|
||||
if: success()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Canary failing: staging SaaS smoke"
|
||||
|
||||
NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number')
|
||||
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
for N in $NUMS; do
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}/comments" \
|
||||
-d "$(jq -nc --arg now "$NOW" '{body: ("Canary recovered at " + $now + ". Closing.")}')" >/dev/null
|
||||
curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
|
||||
echo "Closed recovered canary issue #${N}"
|
||||
done
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
# crosses midnight still cleans up its own slug — see the
|
||||
# 2026-04-26→27 canvas-safety-net incident.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug DELETE with HTTP-code verification. The previous
|
||||
# `... >/dev/null || true` swallowed every failure, so a 5xx
|
||||
# or timeout from CP looked identical to "successfully cleaned
|
||||
# up" and the tenant kept eating ~2 vCPU until the hourly
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,276 +0,0 @@
|
||||
name: canary-verify
|
||||
|
||||
# Ported from .github/workflows/canary-verify.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on the same publish workflow's path (i.e. `.gitea/workflows/publish-workspace-server-image.yml`).
|
||||
#
|
||||
|
||||
# Runs the canary smoke suite against the staging canary tenant fleet
|
||||
# after a new :staging-<sha> image lands in ECR. On green, calls the
|
||||
# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
|
||||
# the prod tenant fleet's 5-minute auto-updater picks up the verified
|
||||
# digest. On red, :latest stays on the prior known-good digest and
|
||||
# prod is untouched.
|
||||
#
|
||||
# Registry note (2026-05-10): This workflow previously used GHCR
|
||||
# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
|
||||
# during the 2026-05-06 Gitea suspension migration when publish-
|
||||
# workspace-server-image.yml switched to the operator's ECR org
|
||||
# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
|
||||
# platform-tenant). The GHCR → ECR migration was never applied to
|
||||
# this file, so canary-verify was silently smoke-testing the stale
|
||||
# GHCR image while the actual staging/prod tenants ran the ECR image.
|
||||
# Result: smoke tests could not catch a broken ECR build. Fix:
|
||||
# - Wait step: reads SHA from running canary /health (tenant-
|
||||
# agnostic, works regardless of registry).
|
||||
# - Promote step: calls CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
|
||||
# No longer attempts GHCR crane ops.
|
||||
#
|
||||
# Dependencies:
|
||||
# - publish-workspace-server-image.yml publishes :staging-<sha>
|
||||
# to ECR on staging and main merges.
|
||||
# - Canary tenants are configured to pull :staging-<sha> from ECR
|
||||
# (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
|
||||
# - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
|
||||
# CANARY_CP_SHARED_SECRET are populated.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["publish-workspace-server-image"]
|
||||
types: [completed]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
canary-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
# workflow_dispatch trigger dropped in this Gitea port; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
smoke_ran: ${{ steps.smoke.outputs.ran }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Compute sha
|
||||
id: compute
|
||||
run: echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Wait for canary tenants to pick up :staging-<sha>
|
||||
# Poll canary health endpoints every 30s for up to 7 min instead
|
||||
# of a fixed 6-min sleep. Exits as soon as ALL canaries report
|
||||
# the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
|
||||
# proceeding after 7 min even if not all canaries responded —
|
||||
# the smoke suite will catch any that didn't update.
|
||||
#
|
||||
# NOTE: The SHA is read from the running tenant's /health response,
|
||||
# NOT from a registry lookup. This is registry-agnostic and works
|
||||
# regardless of whether the tenant pulls from ECR, GHCR, or any
|
||||
# other registry — the canary is telling us what it's actually
|
||||
# running, which is the ground truth for smoke testing.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
|
||||
run: |
|
||||
if [ -z "$CANARY_TENANT_URLS" ]; then
|
||||
echo "No canary URLs configured — falling back to 60s wait"
|
||||
sleep 60
|
||||
exit 0
|
||||
fi
|
||||
IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
|
||||
MAX_WAIT=420 # 7 minutes
|
||||
INTERVAL=30
|
||||
ELAPSED=0
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
ALL_READY=true
|
||||
for url in "${URLS[@]}"; do
|
||||
HEALTH=$(curl -s --max-time 5 "${url}/health" 2>/dev/null || echo "{}")
|
||||
SHA=$(echo "$HEALTH" | grep -o "\"sha\":\"[^\"]*\"" | head -1 | cut -d'"' -f4)
|
||||
if [ "$SHA" != "$EXPECTED_SHA" ]; then
|
||||
ALL_READY=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
if $ALL_READY; then
|
||||
echo "All canaries running staging-${EXPECTED_SHA} after ${ELAPSED}s"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for canaries... (${ELAPSED}s / ${MAX_WAIT}s)"
|
||||
sleep $INTERVAL
|
||||
ELAPSED=$((ELAPSED + INTERVAL))
|
||||
done
|
||||
echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
|
||||
|
||||
- name: Run canary smoke suite
|
||||
id: smoke
|
||||
# Graceful-skip when no canary fleet is configured (Phase 2 not yet
|
||||
# stood up — see molecule-controlplane/docs/canary-tenants.md).
|
||||
# Sets `ran=false` on skip so promote-to-latest stays off (we don't
|
||||
# want every main merge auto-promoting without gating). Manual
|
||||
# promote-latest.yml is the release gate while canary is absent.
|
||||
# Once the fleet is real: delete the early-exit branch.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
|
||||
CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CANARY_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
|
||||
{
|
||||
echo "## ⚠️ canary-verify skipped"
|
||||
echo
|
||||
echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
|
||||
echo "Phase 2 canary fleet has not been stood up yet —"
|
||||
echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
|
||||
echo
|
||||
echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ran=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::canary-verify: skipped — no canary fleet configured"
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/canary-smoke.sh
|
||||
echo "ran=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Summary on failure
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
{
|
||||
echo "## Canary smoke FAILED"
|
||||
echo
|
||||
echo "Canary tenants rejected image \`staging-${{ steps.compute.outputs.sha }}\`."
|
||||
echo ":latest stays pinned to the prior good digest — prod is untouched."
|
||||
echo
|
||||
echo "Fix forward and merge again, or investigate the specific failed"
|
||||
echo "assertions in the canary-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
# mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
|
||||
#
|
||||
# Pre-fix history: the old GHCR promote step used `crane tag` against
|
||||
# ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
|
||||
# image.yml had already migrated to ECR on 2026-05-07 (commit
|
||||
# 10e510f5). The GHCR tags were never updated, so this step was
|
||||
# silently promoting a stale GHCR image while actual prod tenants
|
||||
# pulled from ECR. Canary smoke tests were GHCR-targeted and could
|
||||
# not catch a broken ECR build.
|
||||
needs: canary-smoke
|
||||
if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
SHA: ${{ needs.canary-smoke.outputs.sha }}
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
# CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
|
||||
# Stored at the repo level so all workflows pick it up automatically.
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
# canary_slug pin: deploy the verified :staging-<sha> to the canary
|
||||
# first (soak 120s), then fan out to the rest of the fleet.
|
||||
CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
|
||||
BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
|
||||
steps:
|
||||
- name: Check CP credentials
|
||||
run: |
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
|
||||
echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Promote verified ECR image to :latest
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
--argjson batch "${BATCH_SIZE:-3}" \
|
||||
--argjson dry false \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
|
||||
fi
|
||||
|
||||
echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag: $TARGET_TAG"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
CURL_EXIT=$?
|
||||
set -e
|
||||
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
if [ "$HTTP_CODE" -ge 400 ]; then
|
||||
echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Canary verified — :latest promoted via CP redeploy-fleet"
|
||||
echo ""
|
||||
echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
|
||||
echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
|
||||
echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "- **Batch size:** ${BATCH_SIZE:-3}"
|
||||
echo ""
|
||||
echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
|
||||
echo "The fleet's 5-minute health-check loop will pick up the update automatically."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,58 +0,0 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths reference .gitea/workflows/publish-runtime.yml (the active
|
||||
# Gitea workflow file) instead of .github/workflows/publish-runtime.yml
|
||||
# (which Category A of this sweep deletes).
|
||||
# - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
|
||||
# .gitea/ workflow (the script's default is still .github/... which
|
||||
# will not exist post-Cat-A).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .gitea/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
# Pass the .gitea/ workflow path explicitly — the script's
|
||||
# default still points at .github/... which Category A of this
|
||||
# sweep removes.
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml
|
||||
@@ -1,74 +0,0 @@
|
||||
name: Check migration collisions
|
||||
|
||||
# Ported from .github/workflows/check-migration-collisions.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths includes .gitea/workflows/check-migration-collisions.yml
|
||||
# (this file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned to https://git.moleculesai.app
|
||||
# so scripts/ops/check_migration_collisions.py can derive the Gitea API
|
||||
# base (the script already supports this; see _gitea_api_url()).
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Hard gate (#2341): fails a PR that adds a migration prefix already
|
||||
# claimed by the base branch or another open PR. Caught manually 2026-04-30
|
||||
# during PR #2276 rebase: 044_runtime_image_pins collided with
|
||||
# 044_platform_inbound_secret from RFC #2312. This workflow makes that
|
||||
# check automatic.
|
||||
#
|
||||
# Trigger model: pull_request only — there's no value running this on
|
||||
# pushes to staging or main (those are post-merge; the gate must fire
|
||||
# pre-merge to be useful). Path filter scopes to PRs that actually touch
|
||||
# migrations.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'scripts/ops/check_migration_collisions.py'
|
||||
- '.gitea/workflows/check-migration-collisions.yml'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# API needs read access to other PRs to detect cross-PR collisions
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Need history to diff against base ref
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect collisions
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: origin/${{ github.event.pull_request.base.ref }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
# Auto-injected; Gitea aliases this for in-repo API access.
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Ensure the named base ref exists locally. checkout@v4 with
|
||||
# fetch-depth=0 pulls full history, but the explicit fetch is
|
||||
# cheap insurance against form-of-ref differences across runs.
|
||||
#
|
||||
# IMPORTANT: do NOT pass --depth=1 here. The script below uses
|
||||
# `git diff origin/<base>...<head>` (three-dot, merge-base form),
|
||||
# which fails with "fatal: no merge base" if the base ref is
|
||||
# shallow.
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
python3 scripts/ops/check_migration_collisions.py
|
||||
@@ -1,107 +0,0 @@
|
||||
# ci-required-drift — hourly sentinel for drift between the canonical
|
||||
# "what counts as required" sources of truth in this repo:
|
||||
#
|
||||
# 1. `.gitea/workflows/ci.yml` jobs (CI source)
|
||||
# 2. `branch_protections/{main,staging}.status_check_contexts`
|
||||
# (protection)
|
||||
# 3. `.gitea/workflows/audit-force-merge.yml` REQUIRED_CHECKS env
|
||||
# (audit env)
|
||||
#
|
||||
# RFC: internal#219 §4 (jobs ↔ protection) + §6 (audit env ↔ protection).
|
||||
# Ported verbatim-then-adapted from molecule-controlplane PR#112
|
||||
# (SHA 0adf2098) per RFC internal#219 Phase 2b+c — replicate repo-by-repo.
|
||||
#
|
||||
# When any pair diverges, a `[ci-drift]` issue is opened or updated
|
||||
# (idempotent by title) and labelled `tier:high`. This is the
|
||||
# auto-detection that closes the regression class identified in
|
||||
# RFC §1 finding 3 (protection only listed 2 of 6 real jobs for
|
||||
# ~weeks, undetected) and §6 (audit env drifts silently from
|
||||
# protection).
|
||||
#
|
||||
# Diff logic lives in `.gitea/scripts/ci-required-drift.py`. The
|
||||
# Python file does YAML AST parsing + `needs:` graph walking per
|
||||
# `feedback_behavior_based_ast_gates` — NOT grep-by-name. That way
|
||||
# job renames or matrix-expansion-induced churn produce honest signal.
|
||||
#
|
||||
# IMPORTANT — TRANSITIONAL STATE: molecule-core's ci.yml does NOT yet
|
||||
# contain the `all-required` sentinel job (RFC §4 Phase 4 adds it).
|
||||
# Until Phase 4 lands the detector will hard-fail with exit 3 on the
|
||||
# missing sentinel. That's intentional: a red workflow on a 5-min cron
|
||||
# is louder than a silent issue and forces Phase 4 to land soon.
|
||||
|
||||
name: ci-required-drift
|
||||
|
||||
# IMPORTANT — Gitea 1.22.6 parser quirk per
|
||||
# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
|
||||
# `inputs:` block here, even though stock GitHub Actions allows it.
|
||||
# Gitea 1.22.6 flattens `workflow_dispatch.inputs.X` into a sibling of
|
||||
# the `on:` event keys and rejects the entire workflow as
|
||||
# "unknown on type". The whole file then registers for ZERO events
|
||||
# (no schedule, no dispatch). When Gitea ≥ 1.23 lands fleet-wide,
|
||||
# this constraint can be revisited.
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :17 — offset from :00 to spread load away from the
|
||||
# peak when N cron workflows fire on the hour-boundary, per
|
||||
# RFC §4 cadence ("off-zero").
|
||||
- cron: '17 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# Read protection + read CI YAML + write issue. No write on contents.
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
# Serialise — two simultaneous drift runs would duel on the issue
|
||||
# create/update path. The audit is idempotent, but parallel POSTs
|
||||
# can produce duplicate comments before the title-search dedup wins.
|
||||
concurrency:
|
||||
group: ci-required-drift
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
drift:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out repo (we read the YAML files locally)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Set up Python (PyYAML for AST parsing)
|
||||
# Avoid a system-pip install on the runner; setup-python pins
|
||||
# a hermetic interpreter + cache. PyYAML is small enough that
|
||||
# the install is sub-2s — no need to cache wheels.
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run drift detector
|
||||
env:
|
||||
# GITEA_TOKEN reads protection + writes issues. molecule-core
|
||||
# uses `SOP_TIER_CHECK_TOKEN` as the org-level secret name for
|
||||
# read-only Gitea API access from CI (set by audit-force-merge
|
||||
# and sop-tier-check too). Falls back to the auto-injected
|
||||
# GITHUB_TOKEN if the org-level secret isn't set
|
||||
# (transitional repos).
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# Branches whose protection we compare against. molecule-core
|
||||
# currently has main protected; staging protection is
|
||||
# forthcoming. Keep this list in sync if a new long-lived
|
||||
# branch gets protected (e.g. release/* if introduced later).
|
||||
BRANCHES: 'main staging'
|
||||
# The sentinel job's name inside ci.yml. If the aggregator
|
||||
# is ever renamed, update this too (the drift detector
|
||||
# currently treats `all-required` as the source of "what
|
||||
# the sentinel claims to require").
|
||||
SENTINEL_JOB: 'all-required'
|
||||
# Path to the audit workflow whose REQUIRED_CHECKS env we
|
||||
# cross-check against protection (RFC §6).
|
||||
AUDIT_WORKFLOW_PATH: '.gitea/workflows/audit-force-merge.yml'
|
||||
# Path to the CI workflow with the sentinel + the jobs.
|
||||
CI_WORKFLOW_PATH: '.gitea/workflows/ci.yml'
|
||||
# Issue label applied on file/update. `tier:high` exists in
|
||||
# the molecule-core label set (verified 2026-05-11, label id 9).
|
||||
DRIFT_LABEL: 'tier:high'
|
||||
run: python3 .gitea/scripts/ci-required-drift.py
|
||||
@@ -1,453 +0,0 @@
|
||||
# Ported from .github/workflows/ci.yml on 2026-05-11 per RFC internal#219 §1.
|
||||
# continue-on-error: true on every job; follow-up PR will flip required after
|
||||
# surfaced bugs are fixed (per RFC §1 — "surface broken workflows without
|
||||
# blocking"). The four-surface migration audit
|
||||
# (feedback_gitea_actions_migration_audit_pattern) was performed against this
|
||||
# port:
|
||||
#
|
||||
# 1. YAML — dropped `merge_group` trigger (no Gitea merge queue); no
|
||||
# `workflow_dispatch.inputs` to drop (Gitea 1.22.6 rejects those —
|
||||
# feedback_gitea_workflow_dispatch_inputs_unsupported); no `environment:`
|
||||
# blocks; kept `runs-on: ubuntu-latest` (Gitea runner pool advertises
|
||||
# this label per agent_labels in action_runner table). Workflow-level
|
||||
# env.GITHUB_SERVER_URL set as belt-and-suspenders against runner
|
||||
# defaults (feedback_act_runner_github_server_url).
|
||||
#
|
||||
# 2. Cache — `actions/upload-artifact@v3.2.2` was already pinned to v3 for
|
||||
# Gitea act_runner v0.6 compatibility (a comment in the original called
|
||||
# this out). v4+ is incompatible with Gitea 1.22.x. No `actions/cache`
|
||||
# usage to audit. `actions/setup-python@v6` `cache: pip` is left in
|
||||
# place — works against Gitea's built-in cache server when runner.cache
|
||||
# is configured (currently is, /opt/molecule/runners/config.yaml).
|
||||
#
|
||||
# 3. Token — workflow uses no custom dispatch tokens. The auto-injected
|
||||
# `GITHUB_TOKEN` (which Gitea aliases to a runner-scoped token) is
|
||||
# sufficient for `actions/checkout` against this same repo.
|
||||
#
|
||||
# 4. Docs — no docs/scripts reference github.com URLs that need swapping.
|
||||
# The canvas-deploy-reminder step writes a `ghcr.io/...` image
|
||||
# reference into the step summary text — that's documentation prose
|
||||
# pointing at the ECR-mirrored canvas image and stays unchanged for
|
||||
# this port (a separate cleanup if ghcr→ECR sweep is in scope).
|
||||
#
|
||||
# Cross-links:
|
||||
# - RFC: internal#219 (CI/CD hard-gate hardening)
|
||||
# - Reference port style: molecule-controlplane/.gitea/workflows/ci.yml
|
||||
# - Bugs that may surface immediately and are tracked separately:
|
||||
# internal#214 (Go-side vanity-import / go.sum drift, if any)
|
||||
# - Phase 4 (this PR's follow-up): flip `continue-on-error: false` once
|
||||
# surfaced defects are fixed, then add `all-required` aggregator
|
||||
# sentinel (RFC §2) and PATCH branch protection (Phase 4 scope).
|
||||
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# `merge_group` (GitHub merge-queue trigger) dropped — Gitea has no merge
|
||||
# queue. The .github/ original retains it; this Gitea-side copy drops it.
|
||||
|
||||
# Cancel in-progress CI runs when a new commit arrives on the same ref.
|
||||
# Stale runs queue up otherwise. PR refs and main/staging refs each get
|
||||
# their own group because github.ref differs.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# Belt-and-suspenders against the runner-default trap
|
||||
# (feedback_act_runner_github_server_url). Runners are configured with
|
||||
# this env via /opt/molecule/runners/config.yaml runner.envs, but pinning
|
||||
# at the workflow level protects against a runner regenerated without
|
||||
# the config file (feedback_act_runner_needs_config_file_env).
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# Detect which paths changed so downstream jobs can skip when only
|
||||
# docs/markdown files were modified.
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after the surfaced defects
|
||||
# (if any) are triaged.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
platform: ${{ steps.check.outputs.platform }}
|
||||
canvas: ${{ steps.check.outputs.canvas }}
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
scripts: ${{ steps.check.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
run: |
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Both .github/workflows/ci.yml AND .gitea/workflows/ci.yml count
|
||||
# as "this workflow changed" — either edit should force-run every
|
||||
# downstream job. The Gitea port follows the same shape as the
|
||||
# GitHub original so behavior matches when triggered on either
|
||||
# platform.
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".gitea/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
|
||||
# + per-step gating shape preserves the GitHub-side required-check name
|
||||
# contract (so when this Gitea port becomes a required check in Phase 4,
|
||||
# the name match works on PRs that don't touch workspace-server/).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
# offenders float to the top. Does NOT fail the build; the hard
|
||||
# gate is the threshold check below. (#1823)
|
||||
run: |
|
||||
echo "=== Per-file coverage (worst first) ==="
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
# 2. Per-file floor — non-test .go files in security-critical
|
||||
# paths with coverage <10% fail the build, UNLESS the file
|
||||
# path is listed in .coverage-allowlist.txt (acknowledged
|
||||
# historical debt with a tracking issue + expiry).
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
# Security-critical paths where a 0%-coverage file is a real risk.
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
|
||||
> /tmp/perfile.txt
|
||||
|
||||
# Build allowlist — paths relative to workspace-server, one per line.
|
||||
# Lines starting with # are comments.
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
WARNED=0
|
||||
for path in "${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !($pct < 10)}" || continue
|
||||
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
WARNED=$((WARNED+1))
|
||||
else
|
||||
echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done < /tmp/perfile.txt
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED security-critical file(s) have <10% test coverage and are"
|
||||
echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
|
||||
echo "workspace provisioning — a 0% file here is the exact gap that let"
|
||||
echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
|
||||
echo " (a) add tests to raise coverage above 10%, or"
|
||||
echo " (b) add the path to .coverage-allowlist.txt with an expiry date"
|
||||
echo " and a tracking issue reference."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canvas (Next.js) — required check, always runs. Same always-run +
|
||||
# per-step gating shape as platform-build. The two-job-sharing-name
|
||||
# pattern attempted in PR #2321 doesn't satisfy branch protection
|
||||
# (SKIPPED siblings count as not-passed regardless of SUCCESS
|
||||
# siblings — verified empirically on PR #2314).
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
# #1815 — wires coverage into CI so we get a baseline visible on
|
||||
# every PR. No threshold gate yet; thresholds dial in (Step 3, also
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
# v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
|
||||
# currently supported on GHES`. Drop this pin when Gitea ships
|
||||
# the v4 protocol (tracked: post-Gitea-1.23 followup).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: canvas-coverage-${{ github.run_id }}
|
||||
path: canvas/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
run: |
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
# github.server_url resolves via the workflow-level env override
|
||||
# to the Gitea instance, so the RUN_URL points at the Gitea run
|
||||
# page (not github.com). See feedback_act_runner_github_server_url.
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed — deploy required
|
||||
|
||||
The `publish-canvas-image` workflow is now building a fresh Docker image
|
||||
(`ghcr.io/molecule-ai/canvas:latest`) in the background.
|
||||
|
||||
Once it completes (~3–5 min), apply on the host machine with:
|
||||
```bash
|
||||
cd <runner-workspace>
|
||||
git pull origin main
|
||||
docker compose pull canvas && docker compose up -d canvas
|
||||
```
|
||||
|
||||
If you need to rebuild from local source instead (e.g. testing unreleased
|
||||
changes or a new `NEXT_PUBLIC_*` URL), use:
|
||||
```bash
|
||||
docker compose build canvas && docker compose up -d canvas
|
||||
```
|
||||
BODY
|
||||
printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
|
||||
"$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
|
||||
|
||||
# Gitea has no commit-comments API; write to GITHUB_STEP_SUMMARY,
|
||||
# which both GitHub Actions and Gitea Actions render as the
|
||||
# workflow run's summary page. (#75 / PR-D)
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. See issue #2790 for full rationale.
|
||||
run: |
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text.
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,255 +0,0 @@
|
||||
name: Continuous synthetic E2E (staging)
|
||||
|
||||
# Ported from .github/workflows/continuous-synth-e2e.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
|
||||
# regressions visible only at runtime — schema drift, deployment-pipeline
|
||||
# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
|
||||
#
|
||||
# Why this gate exists:
|
||||
# PR-time CI catches code-level regressions but not deployment-time or
|
||||
# integration-time ones. Today's empirical data:
|
||||
# • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
|
||||
# JSON-RPC parse layer between sender and receiver. Visible only
|
||||
# to a sender exercising the full path.
|
||||
# • RFC #2312 chat upload — landed on staging-branch but never
|
||||
# reached staging tenants because publish-workspace-server-image
|
||||
# was main-only. Caught by manual dogfooding hours after deploy.
|
||||
# Both would have surfaced within 15-20 min of regression if a
|
||||
# continuous synth-E2E was running.
|
||||
#
|
||||
# Cadence: every 20 min (3x/hour). The script is conservatively
|
||||
# bounded at 10 min wall-clock; even on degraded staging it should
|
||||
# finish before the next firing. cron-overlap is guarded by the
|
||||
# concurrency group below.
|
||||
#
|
||||
# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
|
||||
# Plus a fresh tenant provisioned + torn down each run (Railway +
|
||||
# AWS pennies). Negligible.
|
||||
#
|
||||
# Failure handling: when the run fails, the workflow exits non-zero
|
||||
# and GitHub's standard email/notification path fires. Operators
|
||||
# can subscribe to this workflow's failure channel for paging-grade
|
||||
# alerting.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
|
||||
# Prior history: cron was '0,20,40' (2026-05-02) — only :00
|
||||
# ever survived. Bumped to '10,30,50' (2026-05-03) on the
|
||||
# theory that further-from-:00 wins. Empirically 2026-05-04
|
||||
# that ALSO dropped to ~60 min effective cadence (only ~1
|
||||
# schedule fire per hour — see molecule-core#2726). Detection
|
||||
# latency was claimed 20 min, actual 60 min.
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
# Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
|
||||
# lanes (1-3 min away from any other cron). Even with empirically-
|
||||
# observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
# history. If you want auto-issue-on-fail, add a follow-up step that
|
||||
# uses gh issue create gated on `if: failure()`. Keeping the surface
|
||||
# minimal until that's actually wanted.
|
||||
|
||||
# Serialize so two firings can never overlap. Cron firing every 20 min
|
||||
# but scripts conservatively bounded at 10 min — overlap shouldn't
|
||||
# happen in steady state, but if a run hangs we don't want N more
|
||||
# stacking up.
|
||||
concurrency:
|
||||
group: continuous-synth-e2e
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
synth:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
# ssm-agent) runs from raw Ubuntu on every boot — none of it is
|
||||
# pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
|
||||
# across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
|
||||
# latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
|
||||
# A 12min budget leaves only ~2min for the workspace (which needs
|
||||
# ~3.5min for claude-code cold boot) on slow-apt days, blowing the
|
||||
# budget. 20min absorbs the worst tenant tail so the workspace probe
|
||||
# gets the full ~7min it needs even on a slow apt day. Real fix:
|
||||
# pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
# exhaustion class that took the canary down 2026-05-03 (#265).
|
||||
# Operators can pick langgraph / hermes via workflow_dispatch
|
||||
# when they specifically need to exercise the OpenAI or SDK-
|
||||
# native paths.
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
E2E_PROVISION_TIMEOUT_SECS: '600'
|
||||
# Slug suffix — namespaced "synth-" so these runs are
|
||||
# distinguishable from PR-driven runs in CP admin.
|
||||
E2E_RUN_ID: synth-${{ github.run_id }}
|
||||
# Forced false for cron; respected for manual dispatch
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
# tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
|
||||
# which key is present — MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so operators can dispatch with
|
||||
# E2E_RUNTIME=langgraph or =hermes and still have a working
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
run: |
|
||||
# Hard-fail on missing secret REGARDLESS of trigger. Previously
|
||||
# this step soft-skipped on workflow_dispatch via `exit 0`, but
|
||||
# `exit 0` only ends the STEP — subsequent steps still ran with
|
||||
# the empty secret, the synth script fell through to the wrong
|
||||
# SECRETS_JSON branch, and the canary failed 5 min later with a
|
||||
# confusing "Agent error (Exception)" instead of the clean
|
||||
# "secret missing" message at the top. Caught 2026-05-04 by
|
||||
# dispatched run 25296530706: claude-code + missing MINIMAX
|
||||
# silently used OpenAI keys but kept model=MiniMax-M2.7, then
|
||||
# the workspace 401'd against MiniMax once it tried to call.
|
||||
# Fix: exit 1 in both cron and dispatch paths. Operators who
|
||||
# want to verify a YAML change without setting up the secret
|
||||
# can read the verify-secrets step's stderr — the failure is
|
||||
# itself the verification signal.
|
||||
if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
# langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_API_KEY).
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install required tools
|
||||
run: |
|
||||
# The script depends on jq + curl (already on ubuntu-latest)
|
||||
# and python3 (likewise). Verify they're all present so we
|
||||
# fail fast on a runner image regression rather than mid-script.
|
||||
for cmd in jq curl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "::error::required tool '$cmd' not on PATH — runner image regression?"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
- name: Run synthetic E2E
|
||||
# The script handles its own teardown via EXIT trap; even on
|
||||
# failure (timeout, assertion), the org is deprovisioned and
|
||||
# leaks are reported. Exit code propagates from the script.
|
||||
run: |
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
- name: Failure summary
|
||||
# Runs only on failure. Adds a job summary so the workflow run
|
||||
# page shows a quick "what happened" instead of forcing readers
|
||||
# to scroll through script output.
|
||||
if: failure()
|
||||
run: |
|
||||
{
|
||||
echo "## Continuous synth E2E failed"
|
||||
echo ""
|
||||
echo "**Run ID:** ${{ github.run_id }}"
|
||||
echo "**Trigger:** ${{ github.event_name }}"
|
||||
echo "**Runtime:** ${E2E_RUNTIME}"
|
||||
echo "**Slug:** synth-${{ github.run_id }}"
|
||||
echo ""
|
||||
echo "### What this means"
|
||||
echo ""
|
||||
echo "Staging just regressed on a path that previously worked. Likely classes:"
|
||||
echo "- Schema mismatch between sender and receiver (#2345 class)"
|
||||
echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
|
||||
echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
|
||||
echo "- Staging-CP env var rotation"
|
||||
echo ""
|
||||
echo "### Next steps"
|
||||
echo ""
|
||||
echo "1. Check the script output above for the assertion that failed"
|
||||
echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
|
||||
echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
|
||||
echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,333 +0,0 @@
|
||||
name: E2E API Smoke Test
|
||||
|
||||
# Ported from .github/workflows/e2e-api.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
# Extracted from ci.yml so workflow-level concurrency can protect this job
|
||||
# from run-level cancellation (issue #458).
|
||||
#
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always FIRES on push/pull_request to staging+main. Real work is gated
|
||||
# per-step on `needs.detect-changes.outputs.api` — when paths under
|
||||
# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `E2E API Smoke Test` check, satisfying branch protection without
|
||||
# spending CI cycles. See the in-job comment on the `e2e-api` job for
|
||||
# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
|
||||
# PR #2264 incident that drove the consolidation.
|
||||
#
|
||||
# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
|
||||
# -------------------------------------------------------------------
|
||||
# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
|
||||
# Gitea act_runner runs with `container.network: host` (operator host
|
||||
# `/opt/molecule/runners/config.yaml`), which means:
|
||||
#
|
||||
# * Two concurrent runs both try to bind their `-p 15432:5432` /
|
||||
# `-p 16379:6379` host ports — the second postgres/redis FATALs
|
||||
# with `Address in use` and `docker run` returns exit 125 with
|
||||
# `Conflict. The container name "/molecule-ci-postgres" is already
|
||||
# in use by container ...`. Verified in run a7/2727 on 2026-05-07.
|
||||
# * The fixed container names `molecule-ci-postgres` / `-redis` (the
|
||||
# pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
|
||||
# `docker rm -f` at the start of the second job KILLS the first
|
||||
# job's still-running postgres/redis.
|
||||
#
|
||||
# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
|
||||
# platform-server is a Go binary on the host, not a containerised
|
||||
# step):
|
||||
#
|
||||
# 1. Unique container names per run:
|
||||
# pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
|
||||
# same run_id.
|
||||
# 2. Ephemeral host port per run (`-p 0:5432`), then read the actual
|
||||
# bound port via `docker port` and export DATABASE_URL/REDIS_URL
|
||||
# pointing at it. No fixed host-port → no port collision.
|
||||
# 3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
|
||||
# the original flake fixed in #92 and the script's still IPv6-
|
||||
# enabled.
|
||||
# 4. `if: always()` cleanup so containers don't leak when test steps
|
||||
# fail.
|
||||
#
|
||||
# Issue #94 items #2 + #3 (also fixed here):
|
||||
# * Pre-pull `alpine:latest` so the platform-server's provisioner
|
||||
# (`internal/handlers/container_files.go`) can stand up its
|
||||
# ephemeral token-write helper without a daemon.io round-trip.
|
||||
# * Create `molecule-core-net` bridge network if missing so the
|
||||
# provisioner's container.HostConfig {NetworkMode: ...} attach
|
||||
# succeeds.
|
||||
# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
|
||||
# template-registry resolution issue (ADR-002 / local-build mode) and
|
||||
# belongs in a separate change that touches workspace-server, not
|
||||
# this workflow file.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
|
||||
# same auto-promote-staging brittleness as e2e-staging-canvas — back-
|
||||
# to-back staging pushes share refs/heads/staging, so the older push's
|
||||
# queued run gets cancelled when a newer push lands. Auto-promote-
|
||||
# staging then sees `completed/cancelled` for the older SHA and stays
|
||||
# put; the newer SHA's gates may eventually save the day, but if the
|
||||
# newer push gets cancelled too, we deadlock.
|
||||
#
|
||||
# See e2e-staging-canvas.yml's identical concurrency block for the full
|
||||
# rationale and the 2026-04-28 incident reference.
|
||||
group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — same pattern PR#372's
|
||||
# ci.yml port used. Diffs against the PR base or push BEFORE SHA,
|
||||
# then matches against the api-relevant path set.
|
||||
run: |
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
# on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
|
||||
# check run for every job that matches `name:`, and a job-level
|
||||
# `if: false` produces a SKIPPED check run. Branch protection treats
|
||||
# all check runs with a matching context name on the latest commit as a
|
||||
# SET — any SKIPPED in the set fails the required-check eval, even with
|
||||
# SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
|
||||
# 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
|
||||
# promotion despite all real work succeeding. Collapsing to a single
|
||||
# always-running job with conditional steps emits exactly one SUCCESS
|
||||
# check run regardless of paths filter — branch-protection-clean.
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
# Unique per-run container names so concurrent runs on the host-
|
||||
# network act_runner don't collide on name OR port.
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
|
||||
# same run_id. PORT is set later (after docker port lookup) since
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
PORT: "8080"
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
|
||||
echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Provisioner uses alpine:latest for ephemeral token-write
|
||||
# containers (workspace-server/internal/handlers/container_files.go).
|
||||
# Pre-pull so the first provision in test_api.sh doesn't race
|
||||
# the daemon's pull cache. Idempotent — `docker pull` is a no-op
|
||||
# when the image is already present.
|
||||
docker pull alpine:latest >/dev/null
|
||||
# Provisioner attaches workspace containers to
|
||||
# molecule-core-net (workspace-server/internal/provisioner/
|
||||
# provisioner.go::DefaultNetwork). The bridge already exists on
|
||||
# the operator host's docker daemon — `network create` is
|
||||
# idempotent via `|| true`.
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Defensive cleanup — only matches THIS run's container name,
|
||||
# so it cannot kill a sibling run's postgres. (Pre-fix the
|
||||
# name was static and this rm hit other runs' containers.)
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
# `-p 0:5432` requests an ephemeral host port; we read it back
|
||||
# below and export DATABASE_URL.
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
# Resolve the host-side port assignment. `docker port` prints
|
||||
# `0.0.0.0:NNNN` (and on host-net runners may also print an
|
||||
# IPv6 line — take the first IPv4 line).
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
# Fallback: any first line. Some Docker versions print only
|
||||
# one line.
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker port "$PG_CONTAINER" 5432/tcp || true
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
# 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Postgres host port: ${PG_PORT}"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker port "$REDIS_CONTAINER" 6379/tcp || true
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Redis host port: ${REDIS_PORT}"
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
# DATABASE_URL + REDIS_URL exported by the start-postgres /
|
||||
# start-redis steps point at this run's per-run host ports.
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
- name: Assert migrations applied
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
|
||||
if [ "$tables" != "1" ]; then
|
||||
echo "::error::Migrations did not apply"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
- name: Run notify-with-attachments E2E
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_notify_attachments_e2e.sh
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
- name: Run poll-mode chat upload E2E (RFC #2891)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.api == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
# always() so containers don't leak when test steps fail. The
|
||||
# cleanup is best-effort: if the container is already gone
|
||||
# (e.g. concurrent rerun race), don't fail the job.
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -1,247 +0,0 @@
|
||||
name: E2E Staging Canvas (Playwright)
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-canvas.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Playwright test suite that provisions a fresh staging org per run and
|
||||
# verifies every workspace-panel tab renders without crashing. Complements
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
|
||||
on:
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always fires on push/pull_request; real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `Canvas tabs E2E` check, satisfying branch protection without
|
||||
# spending CI cycles. See e2e-api.yml for the rationale on why this
|
||||
# is a single job rather than two-jobs-sharing-name.
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
# global group made auto-promote-staging brittle: when a staging push
|
||||
# queued behind an in-flight run and a third entrant (a PR run, a
|
||||
# follow-on push) entered the group, the staging push got cancelled —
|
||||
# leaving auto-promote-staging looking at `completed/cancelled` for a
|
||||
# required gate and refusing to advance main. Observed 2026-04-28
|
||||
# 23:51-23:53 on staging tip 3f99fede.
|
||||
#
|
||||
# The original intent of the global group was to throttle parallel
|
||||
# E2E provisions (each spins a fresh EC2). At our scale that throttle
|
||||
# isn't worth the correctness cost — fresh-org-per-run isolates the
|
||||
# state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
|
||||
# is rounding error vs. the cost of a stuck pipeline.
|
||||
#
|
||||
# Per-SHA still dedupes accidental double-triggers for the SAME SHA.
|
||||
# It does NOT cancel obsolete-PR-version runs on force-push; that
|
||||
# wasted CI is acceptable given the alternative is losing staging-tip
|
||||
# data that auto-promote-staging needs.
|
||||
group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
# Cron triggers always run real work (no diff context).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `Canvas tabs E2E`. Real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
|
||||
# rationale — same path-filter check-name parity issue blocked PR #2264
|
||||
# (staging→main) on 2026-04-29 because branch protection treats matching-
|
||||
# name check runs as a SET, and any SKIPPED member fails the eval.
|
||||
playwright:
|
||||
needs: detect-changes
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 40
|
||||
|
||||
env:
|
||||
CANVAS_E2E_STAGING: '1'
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: |
|
||||
echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
|
||||
echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Set up Node
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Install canvas deps
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright test --config=playwright.staging.config.ts
|
||||
|
||||
- name: Upload Playwright report on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement (see ci.yml upload step for the canonical error
|
||||
# cite). Drop this pin when Gitea ships the v4 protocol.
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-report-staging
|
||||
path: canvas/playwright-report-staging/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload screenshots on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-screenshots
|
||||
path: canvas/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Safety-net teardown — fires only when Playwright's globalTeardown
|
||||
# didn't (worker crash, runner cancel). Reads the slug from
|
||||
# canvas/.playwright-staging-state.json (written by staging-setup
|
||||
# as its first action, before any CP call) and deletes only that
|
||||
# slug.
|
||||
#
|
||||
# Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
|
||||
# orgs to compensate for setup-crash-before-state-file-write. That
|
||||
# over-aggressive cleanup raced concurrent canvas-E2E runs and
|
||||
# poisoned each other's tenants — observed 2026-04-30 when three
|
||||
# real-test runs killed each other mid-test, surfacing as
|
||||
# `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
|
||||
# DNS record. Pattern-sweep removed; setup now writes the state
|
||||
# file before any CP work, so the slug is always recoverable.
|
||||
- name: Teardown safety net
|
||||
if: always() && needs.detect-changes.outputs.canvas == 'true'
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
STATE_FILE=".playwright-staging-state.json"
|
||||
if [ ! -f "$STATE_FILE" ]; then
|
||||
echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
|
||||
exit 0
|
||||
fi
|
||||
slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
|
||||
if [ -z "$slug" ]; then
|
||||
echo "::warning::State file present but slug missing; nothing to clean up."
|
||||
exit 0
|
||||
fi
|
||||
echo "Deleting orphan tenant: $slug"
|
||||
# Verify HTTP 2xx instead of `>/dev/null || true` swallowing
|
||||
# failures. A 5xx or timeout previously looked identical to
|
||||
# success, leaving the tenant alive for up to ~45 min until
|
||||
# sweep-stale-e2e-orgs caught it. Surface failures as
|
||||
# workflow warnings naming the slug. Don't `exit 1` — a single
|
||||
# cleanup miss shouldn't fail-flag the canvas test when the
|
||||
# actual smoke check passed; the sweeper is the safety net.
|
||||
# See molecule-controlplane#420.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,189 +0,0 @@
|
||||
name: E2E Staging External Runtime
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-external.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Regression for the four/five workspaces.status=awaiting_agent transitions
|
||||
# that silently failed in production for five days before migration 046
|
||||
# extended the workspace_status enum (see
|
||||
# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
|
||||
#
|
||||
# Why this is its own workflow (not folded into e2e-staging-saas.yml):
|
||||
# - The full-saas harness defaults to runtime=hermes, never exercises
|
||||
# external-runtime. Adding an `external` parameter to that script
|
||||
# would force every push to staging through both lifecycles in
|
||||
# series, doubling the EC2 cold-start budget.
|
||||
# - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
|
||||
# window, 90s default + sweep interval), which we wait through
|
||||
# deliberately. Folding it into hermes would make the long path
|
||||
# even longer.
|
||||
# - It can run in parallel with the hermes E2E since both create
|
||||
# fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
|
||||
# `e2e-...`).
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to staging when any source affecting external runtime,
|
||||
# hibernation, or the migration set changes.
|
||||
# - PR review for the same set.
|
||||
# - Manual workflow_dispatch.
|
||||
# - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
|
||||
# 30 min after e2e-staging-saas.yml's 07:00 UTC cron).
|
||||
#
|
||||
# Concurrency: serialized so two staging pushes don't fight for the
|
||||
# same EC2 quota window. cancel-in-progress=false so a half-rolled
|
||||
# tenant always finishes its teardown.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.gitea/workflows/e2e-staging-external.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.gitea/workflows/e2e-staging-external.yml'
|
||||
schedule:
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-external
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
e2e-staging-external:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
# Schedule + push triggers must hard-fail when the token is
|
||||
# missing — silent skip would mask infra rot. Manual dispatch
|
||||
# gets the same hard-fail; an operator running this on a fork
|
||||
# without secrets configured needs to know up-front.
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run external-runtime E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_external_runtime.sh
|
||||
|
||||
# Mirror the e2e-staging-saas.yml safety net: if the runner is
|
||||
# cancelled (e.g. concurrent staging push), the test script's
|
||||
# EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
|
||||
# *this* run id.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
|
||||
# so concurrent runs and unrelated dev probes are not touched.
|
||||
# Sweep today AND yesterday so a midnight-crossing run still
|
||||
# cleans up its own slug.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if not run_id:
|
||||
# Without a run id we cannot scope safely; bail rather
|
||||
# than risk deleting unrelated tenants.
|
||||
sys.exit(0)
|
||||
prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
|
||||
for o in d.get('orgs', []):
|
||||
s = o.get('slug', '')
|
||||
if s.startswith(prefixes) and o.get('status') != 'purged':
|
||||
print(s)
|
||||
" 2>/dev/null)
|
||||
if [ -n "$orgs" ]; then
|
||||
echo "Safety-net sweep: deleting leftover orgs:"
|
||||
echo "$orgs"
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# `>/dev/null 2>&1` previously hid every failure; surface
|
||||
# non-2xx as workflow warnings so the run page names what
|
||||
# leaked. Sweeper catches the rest within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
else
|
||||
echo "Safety-net sweep: no leftover orgs to clean."
|
||||
fi
|
||||
@@ -1,251 +0,0 @@
|
||||
name: E2E Staging SaaS (full lifecycle)
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-saas.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Dedicated workflow that provisions a fresh staging org per run, exercises
|
||||
# the full workspace lifecycle (register → heartbeat → A2A → delegation →
|
||||
# HMA memory → activity → peers), then tears down and asserts leak-free.
|
||||
#
|
||||
# Why a separate workflow (not folded into ci.yml):
|
||||
# - The run takes ~25-35 min (EC2 boot + cloudflared DNS + provision sweeps +
|
||||
# agent bootstrap), way too slow for every PR.
|
||||
# - Needs its own concurrency group so two pushes don't fight over the
|
||||
# same staging org slug prefix.
|
||||
# - Has its own required secrets (session cookie, admin token) that most
|
||||
# PRs don't need to read.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to main (regression guard)
|
||||
# - workflow_dispatch (manual re-run from UI)
|
||||
# - Nightly cron (catches drift even when no pushes land)
|
||||
# - Changes to any provisioning-critical file under PR review (opt-in
|
||||
# via the same paths watcher that e2e-api.yml uses)
|
||||
|
||||
on:
|
||||
# Trunk-based (Phase 3 of internal#81): main is the only branch.
|
||||
# Previously this fired on staging push too because staging was a
|
||||
# superset of main and ran the gate ahead of auto-promote; with no
|
||||
# staging branch, main is where E2E gates the deploy.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
schedule:
|
||||
# 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
|
||||
# Cloudflare API regressions, etc. even on quiet days.
|
||||
- cron: '0 7 * * *'
|
||||
|
||||
# Serialize: staging has a finite per-hour org creation quota. Two pushes
|
||||
# landing in quick succession should queue, not race. `cancel-in-progress:
|
||||
# false` mirrors e2e-api.yml — GitHub would otherwise cancel the running
|
||||
# teardown step and leave orphan EC2s.
|
||||
concurrency:
|
||||
group: e2e-staging-saas
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
e2e-staging-saas:
|
||||
name: E2E Staging SaaS
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
# Single admin-bearer secret drives provision + tenant-token
|
||||
# retrieval + teardown. Configure in
|
||||
# Settings → Secrets and variables → Actions → Repository secrets.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the full-lifecycle E2E red on every provisioning-critical push).
|
||||
# claude-code template's `minimax` provider routes
|
||||
# ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
|
||||
# MINIMAX_API_KEY at boot — separate billing account so an
|
||||
# OpenAI quota collapse no longer wedges the gate. Mirrors the
|
||||
# canary-staging.yml + continuous-synth-e2e.yml migrations.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
|
||||
# exercise the OpenAI path.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the model when running on the default claude-code path —
|
||||
# the per-runtime default ("sonnet") routes to direct Anthropic
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per #2578's lesson — empty key
|
||||
# silently falls through to the wrong SECRETS_JSON branch and
|
||||
# produces a confusing auth error 5 min later instead of the
|
||||
# clean "secret missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run full-lifecycle E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Belt-and-braces teardown: the test script itself installs a trap
|
||||
# for EXIT/INT/TERM, but if the GH runner itself is cancelled (e.g.
|
||||
# someone pushes a new commit and workflow concurrency is set to
|
||||
# cancel), the trap may not fire. This `always()` step runs even on
|
||||
# cancellation and attempts the delete a second time. The admin
|
||||
# DELETE endpoint is idempotent so double-invoking is safe.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
# Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
|
||||
# nuke them. Catches the case where the script died before
|
||||
# exporting its slug.
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# ONLY sweep slugs from *this* CI run. Previously the filter was
|
||||
# f'e2e-{today}-' which stomped on parallel CI runs AND any manual
|
||||
# E2E probes a dev was running against staging (incident 2026-04-21
|
||||
# 15:02Z: this workflow's safety net deleted an unrelated manual
|
||||
# run's tenant 1s after it hit 'running').
|
||||
# Sweep both today AND yesterday's UTC dates so a run that crosses
|
||||
# midnight still matches its own slug — see the 2026-04-26→27
|
||||
# canvas-safety-net incident for the same bug class.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-{d}-{run_id}-' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('instance_status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug verified DELETE (was `>/dev/null || true` — see
|
||||
# molecule-controlplane#420). Surface non-2xx as a workflow
|
||||
# warning naming the leaked slug; don't exit 1 (sweeper is
|
||||
# the safety net within ~45 min).
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
echo "Safety-net teardown: $slug"
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,157 +0,0 @@
|
||||
name: E2E Staging Sanity (leak-detection self-check)
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-sanity.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch:` (Gitea 1.22.6 finicky on bare dispatch).
|
||||
# - `actions/github-script@v9` issue-open block replaced with curl
|
||||
# calls to the Gitea REST API (/api/v1/repos/.../issues|comments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Periodic assertion that the teardown safety nets in e2e-staging-saas
|
||||
# and canary-staging actually work. Runs the E2E harness with
|
||||
# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
|
||||
# the org is provisioned. The workspace-provision step then fails, the
|
||||
# script exits non-zero, and the EXIT trap + workflow always()-step
|
||||
# must still tear down cleanly.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-sanity
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sanity:
|
||||
name: Intentional-failure teardown sanity
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 20
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
E2E_INTENTIONAL_FAILURE: "1"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken.
|
||||
- name: Run harness — expecting exit !=0
|
||||
id: harness
|
||||
run: |
|
||||
set +e
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
rc=$?
|
||||
echo "harness_rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
if [ "$rc" = "1" ]; then
|
||||
echo "OK Harness failed as expected (rc=1); teardown trap ran, leak-check passed"
|
||||
exit 0
|
||||
elif [ "$rc" = "0" ]; then
|
||||
echo "::error::Harness succeeded under E2E_INTENTIONAL_FAILURE=1 — the poisoning path is broken"
|
||||
exit 1
|
||||
elif [ "$rc" = "4" ]; then
|
||||
echo "::error::LEAK DETECTED (rc=4) — teardown failed to clean up the org. Safety net broken."
|
||||
exit 4
|
||||
else
|
||||
echo "::error::Unexpected rc=$rc — neither clean-failure nor leak. Investigate harness."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Open issue if safety net is broken (Gitea API)
|
||||
if: failure()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="E2E teardown safety net broken"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
BODY_JSON=$(jq -nc --arg t "$TITLE" --arg run "$RUN_URL" '
|
||||
{title: $t,
|
||||
body: ("The weekly sanity run (E2E_INTENTIONAL_FAILURE=1) did not exit as expected. This means one of:\n - poisoning did not actually cause failure (test harness regression), OR\n - teardown left an orphan org (leak detection caught a real bug)\n\nRun: " + $run + "\n\nThis is higher priority than a canary failure — the whole E2E safety net cannot be trusted until this is resolved.")}')
|
||||
|
||||
EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
|
||||
|
||||
if [ -n "$EXISTING" ]; then
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Still broken. " + $run)}')" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$BODY_JSON" >/dev/null
|
||||
echo "Filed new issue"
|
||||
fi
|
||||
|
||||
# Belt-and-braces: if teardown left anything behind, nuke it here
|
||||
# so we don't bleed staging quota.
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
today = __import__('datetime').date.today().strftime('%Y%m%d')
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,282 +0,0 @@
|
||||
name: Handlers Postgres Integration
|
||||
|
||||
# Ported from .github/workflows/handlers-postgres-integration.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Real-Postgres integration tests for workspace-server/internal/handlers/.
|
||||
# Triggered on every PR/push that touches the handlers package.
|
||||
#
|
||||
# Why this workflow exists
|
||||
# ------------------------
|
||||
# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
|
||||
# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
|
||||
# depend on the row state AFTER the SQL runs. The result_preview-lost
|
||||
# bug shipped to staging in PR #2854 because every unit test was
|
||||
# satisfied with "an UPDATE statement fired" — none verified the row's
|
||||
# preview field actually landed. The local-postgres E2E that retrofit
|
||||
# self-review caught it took 2 minutes to set up and would have caught
|
||||
# the bug at PR-time.
|
||||
#
|
||||
# Why this workflow does NOT use `services: postgres:` (Class B fix)
|
||||
# ------------------------------------------------------------------
|
||||
# Our act_runner config has `container.network: host` (operator host
|
||||
# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
|
||||
# the job container AND every service container. With host-net, two
|
||||
# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
|
||||
# second postgres FATALs with `could not create any TCP/IP sockets:
|
||||
# Address in use`, and Docker auto-removes it (act_runner sets
|
||||
# AutoRemove:true on service containers). By the time the migrations
|
||||
# step runs `psql`, the postgres container is gone, hence
|
||||
# `Connection refused` then `failed to remove container: No such
|
||||
# container` at cleanup time.
|
||||
#
|
||||
# Per-job `container.network` override is silently ignored by
|
||||
# act_runner — `--network and --net in the options will be ignored.`
|
||||
# appears in the runner log. Documented constraint.
|
||||
#
|
||||
# So we sidestep `services:` entirely. The job container still uses
|
||||
# host-net (inherited from runner config; required for cache server
|
||||
# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
|
||||
# postgres on the existing `molecule-core-net` bridge with a
|
||||
# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
|
||||
# read its bridge IP via `docker inspect`. A host-net job container
|
||||
# can reach a bridge-net container directly via the bridge IP (verified
|
||||
# manually on operator host 2026-05-08).
|
||||
#
|
||||
# Trade-offs vs. the original `services:` shape:
|
||||
# + No host-port collision; N parallel runs share the bridge cleanly
|
||||
# + `if: always()` cleanup runs even on test-step failure
|
||||
# - One more step in the workflow (+~3 lines)
|
||||
# - Requires `molecule-core-net` to exist on the operator host
|
||||
# (it does; declared in docker-compose.yml + docker-compose.infra.yml)
|
||||
#
|
||||
# Class B Hongming-owned CICD red sweep, 2026-05-08.
|
||||
#
|
||||
# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
concurrency:
|
||||
group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
handlers: ${{ steps.filter.outputs.handlers }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: filter
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
run: |
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "handlers=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
# paths filter. See ci.yml's Platform (Go) for the same shape.
|
||||
integration:
|
||||
name: Handlers Postgres Integration
|
||||
needs: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
# Unique name per run so concurrent jobs don't collide on the
|
||||
# bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
|
||||
# workflow_dispatch reruns of the same run_id.
|
||||
PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# Bridge network already exists on the operator host (declared
|
||||
# in docker-compose.yml + docker-compose.infra.yml).
|
||||
PG_NETWORK: molecule-core-net
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.detect-changes.outputs.handlers != 'true'
|
||||
working-directory: .
|
||||
run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Start sibling Postgres on bridge network
|
||||
working-directory: .
|
||||
run: |
|
||||
# Sanity: the bridge network must exist on the operator host.
|
||||
# Hard-fail loud if it doesn't — easier to spot than a silent
|
||||
# auto-create that diverges from the rest of the stack.
|
||||
if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
|
||||
echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If a stale container with the same name exists (rerun on
|
||||
# the same run_id), wipe it first.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
|
||||
docker run -d \
|
||||
--name "${PG_NAME}" \
|
||||
--network "${PG_NETWORK}" \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 5s \
|
||||
--health-timeout 5s \
|
||||
--health-retries 10 \
|
||||
-e POSTGRES_PASSWORD=test \
|
||||
-e POSTGRES_DB=molecule \
|
||||
postgres:15-alpine >/dev/null
|
||||
|
||||
# Read back the bridge IP. Always present immediately after
|
||||
# `docker run -d` for bridge networks.
|
||||
PG_HOST=$(docker inspect "${PG_NAME}" \
|
||||
--format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
|
||||
if [ -z "${PG_HOST}" ]; then
|
||||
echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
|
||||
docker logs "${PG_NAME}" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
|
||||
echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Started ${PG_NAME} at ${PG_HOST}:5432"
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Apply migrations to Postgres service
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
# Wait for postgres to actually accept connections. Docker's
|
||||
# health-cmd handles container-side readiness, but the wire
|
||||
# to the bridge IP is best-tested with pg_isready directly.
|
||||
for i in {1..15}; do
|
||||
if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
|
||||
echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
|
||||
done
|
||||
|
||||
# Apply every .up.sql in lexicographic order with
|
||||
# ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
|
||||
# blocking the suite. This handles the current schema state
|
||||
# where a few historical migrations (e.g. 017_memories_fts_*)
|
||||
# depend on tables that were later renamed/dropped and so
|
||||
# cannot replay from scratch. The migrations that DO succeed
|
||||
# land their tables, which is sufficient for the integration
|
||||
# tests in handlers/.
|
||||
#
|
||||
# Why not maintain a curated allowlist: every new migration
|
||||
# touching a handlers/-tested table would have to update this
|
||||
# workflow. With apply-all-or-skip, a future migration that
|
||||
# adds a column to delegations runs automatically (its base
|
||||
# table 049_delegations.up.sql already succeeded above it in
|
||||
# the order). Operators only need to revisit this if the
|
||||
# migration chain becomes legitimately replayable end-to-end.
|
||||
#
|
||||
# Per-migration result is logged so a failed migration that
|
||||
# SHOULD have been replayable surfaces in the CI log instead
|
||||
# of silently failing.
|
||||
# Apply both *.sql (legacy, lives next to its module) and
|
||||
# *.up.sql (newer up/down convention) in a single
|
||||
# lexicographically-sorted pass. Excluding *.down.sql so the
|
||||
# newest-naming-convention pairs don't undo themselves mid-run.
|
||||
# Pre-#149-followup this loop only globbed *.up.sql, which
|
||||
# silently skipped 001_workspaces.sql + 009_activity_logs.sql
|
||||
# — fine while no integration test depended on those tables,
|
||||
# not fine once a cross-table atomicity test came in.
|
||||
set +e
|
||||
for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
|
||||
if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
|
||||
-f "$migration" >/dev/null 2>&1; then
|
||||
echo "✓ $(basename "$migration")"
|
||||
else
|
||||
echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
|
||||
# Sanity: the delegations + workspaces + activity_logs tables
|
||||
# MUST exist for the integration tests to be meaningful. Hard-
|
||||
# fail if any didn't land — that would be a real regression we
|
||||
# want loud.
|
||||
for tbl in delegations workspaces activity_logs pending_uploads; do
|
||||
if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
|
||||
-c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
|
||||
| grep -q 1; then
|
||||
echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ $tbl table present"
|
||||
done
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Run integration tests
|
||||
run: |
|
||||
# INTEGRATION_DB_URL is exported by the start-postgres step;
|
||||
# points at the per-run bridge IP, not 127.0.0.1, so concurrent
|
||||
# workflow runs don't fight over a host-net 5432 port.
|
||||
go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
|
||||
|
||||
- if: failure() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Diagnostic dump on failure
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
echo "::group::postgres container status"
|
||||
docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
|
||||
docker logs "${PG_NAME}" 2>&1 | tail -50 || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::delegations table state"
|
||||
psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
|
||||
echo "::endgroup::"
|
||||
|
||||
- if: always() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Stop sibling Postgres
|
||||
working-directory: .
|
||||
run: |
|
||||
# always() so containers don't leak when migrations or tests
|
||||
# fail. The cleanup is best-effort: if the container is
|
||||
# already gone (e.g. concurrent rerun race), don't fail the job.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
echo "Cleaned up ${PG_NAME}"
|
||||
@@ -1,262 +0,0 @@
|
||||
name: Harness Replays
|
||||
|
||||
# Ported from .github/workflows/harness-replays.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Boots tests/harness (production-shape compose topology with TenantGuard,
|
||||
# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
|
||||
# every replay under tests/harness/replays/. Fails the PR if any replay
|
||||
# fails.
|
||||
#
|
||||
# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
|
||||
# a public route in router.go but forgot to add it to TenantGuard's
|
||||
# allowlist. The handler-level test in buildinfo_test.go constructed a
|
||||
# minimal gin engine without TenantGuard — green. The harness's
|
||||
# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
|
||||
# inject X-Molecule-Org-Id, so the curl path is identical to production's
|
||||
# redeploy verifier), but no one ran the harness pre-merge. The bug
|
||||
# shipped; the redeploy verifier silently soft-warned every tenant as
|
||||
# "unreachable" for ~1 day before being noticed.
|
||||
#
|
||||
# This gate makes "did you actually run the harness?" a CI invariant
|
||||
# instead of a memory-discipline thing.
|
||||
#
|
||||
# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
|
||||
# to staging+main, real work is gated per-step on detect-changes output.
|
||||
# One job → one check run → branch-protection-clean (the SKIPPED-in-set
|
||||
# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.gitea/workflows/harness-replays.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.gitea/workflows/harness-replays.yml'
|
||||
concurrency:
|
||||
# Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
|
||||
# cancellation deadlock — see e2e-api.yml's concurrency block for
|
||||
# the 2026-04-28 incident that codified this pattern.
|
||||
group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- id: decide
|
||||
run: |
|
||||
# workflow_dispatch: always run (manual trigger)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine the base commit to diff against.
|
||||
# For pull_request: use base.sha (the merge-base with main/staging).
|
||||
# For push: use github.event.before (the previous tip of the branch).
|
||||
# Fallback for new branches (all-zeros SHA): run everything.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && \
|
||||
[ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "${{ github.event.before }}" ] && \
|
||||
! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
|
||||
BASE="${{ github.event.before }}"
|
||||
else
|
||||
# New branch or github.event.before unavailable — run everything.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# GitHub Actions and Gitea Actions both expose github.sha for HEAD.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job that always runs. Real work is gated per-step on
|
||||
# detect-changes.outputs.run so an unrelated PR (e.g. doc-only
|
||||
# change to molecule-controlplane wired here later) emits the
|
||||
# required check without spending CI cycles. Single-job pattern
|
||||
# matches e2e-api.yml — see that workflow's comment for why a
|
||||
# job-level `if: false` would block branch protection via the
|
||||
# SKIPPED-in-set bug.
|
||||
harness-replays:
|
||||
needs: detect-changes
|
||||
name: Harness Replays
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.run != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
|
||||
echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
|
||||
echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
- if: needs.detect-changes.outputs.run == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Log what files were detected so future failures include the diff.
|
||||
- name: Log detected changes
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: |
|
||||
echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
# github-app-auth sibling-checkout removed 2026-05-07 (#157):
|
||||
# the plugin was dropped + Dockerfile.tenant no longer COPYs it.
|
||||
|
||||
# Pre-clone manifest deps before docker compose builds the tenant
|
||||
# image (Task #173 followup — same pattern as
|
||||
# publish-workspace-server-image.yml's "Pre-clone manifest deps"
|
||||
# step).
|
||||
#
|
||||
# Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
|
||||
# and tenant-beta from workspace-server/Dockerfile.tenant with
|
||||
# context=../.. (repo root). That Dockerfile expects
|
||||
# .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
|
||||
# to be present at build context root (post-#173 it COPYs from there
|
||||
# instead of running an in-image clone — the in-image clone failed
|
||||
# with "could not read Username for https://git.moleculesai.app"
|
||||
# because there's no auth path inside the build sandbox).
|
||||
#
|
||||
# Without this step harness-replays fails before any replay runs,
|
||||
# with `failed to calculate checksum of ref ...
|
||||
# "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
|
||||
# (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
|
||||
# symptom, different root cause: staging still has the in-image
|
||||
# clone path, hits the auth error directly).
|
||||
#
|
||||
# 2026-05-08 sub-finding (#192): the clone step ALSO fails when
|
||||
# any referenced workspace-template repo is private and the
|
||||
# AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
|
||||
# access. Root cause: 5 of 9 workspace-template repos
|
||||
# (openclaw, codex, crewai, deepagents, gemini-cli) had been
|
||||
# marked private with no team grant. Resolution: flipped them
|
||||
# to public per `feedback_oss_first_repo_visibility_default`
|
||||
# (the OSS surface should be public). Layer-3 (customer-private +
|
||||
# marketplace third-party repos) tracked separately in
|
||||
# internal#102.
|
||||
#
|
||||
# Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
|
||||
# is the devops-engineer persona PAT, NOT the founder PAT (per
|
||||
# `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
|
||||
# embeds it as basic-auth for the duration of the clones and strips
|
||||
# .git directories — the token never enters the resulting image.
|
||||
- name: Pre-clone manifest deps
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
env:
|
||||
MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
bash scripts/clone-manifest.sh \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
# Sanity-check counts so a silent partial clone fails fast
|
||||
# instead of producing a half-empty image.
|
||||
ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
|
||||
|
||||
- name: Install Python deps for replays
|
||||
# peer-discovery-404 (and future replays) eval Python against the
|
||||
# running tenant — importing workspace/a2a_client.py pulls in
|
||||
# httpx. tests/harness/requirements.txt holds just the HTTP-client
|
||||
# surface to keep CI install fast (~3s) vs the full
|
||||
# workspace/requirements.txt (~30s).
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: pip install -r tests/harness/requirements.txt
|
||||
|
||||
- name: Run all replays against the harness
|
||||
# run-all-replays.sh: boot via up.sh → seed via seed.sh → run
|
||||
# every replays/*.sh → tear down via down.sh on EXIT (trap).
|
||||
# Non-zero exit on any replay failure.
|
||||
#
|
||||
# KEEP_UP=1: without this, the script's trap-on-EXIT tears
|
||||
# down containers immediately on failure, leaving the dump
|
||||
# step below with nothing to dump (verified on PR #2410's
|
||||
# first run — tenant became unhealthy, trap fired, dump
|
||||
# step saw empty containers). Keeping them up lets the
|
||||
# failure path collect tenant/cp-stub/cf-proxy logs. The
|
||||
# always-run "Force teardown" step does the actual cleanup.
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
KEEP_UP: "1"
|
||||
run: ./run-all-replays.sh
|
||||
|
||||
- name: Dump compose logs on failure
|
||||
# SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
|
||||
# file even for read-only `logs` calls. up.sh generates a per-run key
|
||||
# and exports it to its OWN shell — this step runs in a fresh shell
|
||||
# that wouldn't see it, so without a placeholder the validate step
|
||||
# errors before logs print (verified against PR #2492's first run:
|
||||
# "required variable SECRETS_ENCRYPTION_KEY is missing a value").
|
||||
# A placeholder is fine — we're only reading log streams, not booting.
|
||||
if: failure() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
|
||||
run: |
|
||||
echo "=== docker compose ps ==="
|
||||
docker compose -f compose.yml ps || true
|
||||
echo "=== tenant-alpha logs ==="
|
||||
docker compose -f compose.yml logs tenant-alpha || true
|
||||
echo "=== tenant-beta logs ==="
|
||||
docker compose -f compose.yml logs tenant-beta || true
|
||||
echo "=== cp-stub logs ==="
|
||||
docker compose -f compose.yml logs cp-stub || true
|
||||
echo "=== cf-proxy logs ==="
|
||||
docker compose -f compose.yml logs cf-proxy || true
|
||||
echo "=== postgres-alpha logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-alpha || true
|
||||
echo "=== postgres-beta logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-beta || true
|
||||
|
||||
- name: Force teardown
|
||||
# We pass KEEP_UP=1 to run-all-replays.sh so the dump step
|
||||
# above sees real containers — that means we own teardown
|
||||
# explicitly here. Always run.
|
||||
if: always() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
run: ./down.sh || true
|
||||
@@ -1,104 +0,0 @@
|
||||
name: Lint curl status-code capture
|
||||
|
||||
# Ported from .github/workflows/lint-curl-status-capture.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths and the lint scanner target .gitea/workflows/**.yml (the
|
||||
# active Gitea workflow directory) instead of .github/workflows/**.yml
|
||||
# (which the rest of this sweep is emptying out).
|
||||
# - Self-skip path updated to the .gitea/ version of this file.
|
||||
# - Dropped `merge_group:` trigger.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
|
||||
# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
|
||||
#
|
||||
# HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
|
||||
#
|
||||
# When curl exits non-zero (connection reset -> 56, --fail-with-body 4xx/5xx
|
||||
# -> 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
|
||||
# "000" for connection failures or the actual code for HTTP errors. The
|
||||
# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
|
||||
# stdout, producing values like "000000" or "409000" that fail string
|
||||
# comparisons against "200" while looking superficially right.
|
||||
#
|
||||
# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
|
||||
# #2797). Memory: feedback_curl_status_capture_pollution.md.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths: ['.gitea/workflows/**']
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths: ['.gitea/workflows/**']
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan workflows for curl status-capture pollution
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
|
||||
# subshell where the entire command-substitution wraps a curl that
|
||||
# ends with `|| echo "000"`. Must distinguish from the SAFE shape
|
||||
# `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
|
||||
# tempfile produces empty stdout, no pollution.
|
||||
python3 <<'PY'
|
||||
import os, re, sys, glob
|
||||
|
||||
BAD_FILES = []
|
||||
|
||||
# Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
|
||||
# The `\\n` is the bash line-continuation that lets curl flags span lines.
|
||||
# We collapse continuation lines first, then look for the single-line bad pattern.
|
||||
PATTERN = re.compile(
|
||||
r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Self-skip: this lint workflow contains the literal anti-pattern in
|
||||
# its own docstring — that's intentional, not a bug.
|
||||
SELF = ".gitea/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
for f in sorted(glob.glob(".gitea/workflows/*.yml")):
|
||||
if f == SELF:
|
||||
continue
|
||||
with open(f) as fh:
|
||||
content = fh.read()
|
||||
# Collapse bash line-continuations (\\\n + leading whitespace)
|
||||
# into a single logical line so the regex can see the full
|
||||
# curl invocation as one chunk.
|
||||
flat = re.sub(r'\\\s*\n\s*', ' ', content)
|
||||
for m in PATTERN.finditer(flat):
|
||||
BAD_FILES.append((f, m.group(0)[:120]))
|
||||
|
||||
if not BAD_FILES:
|
||||
print("OK No curl-status-capture pollution patterns detected")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
|
||||
for f, snippet in BAD_FILES:
|
||||
print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
|
||||
print(f" matched: {snippet}...")
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(' set +e')
|
||||
print(' curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
|
||||
print(' set -e')
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
sys.exit(1)
|
||||
PY
|
||||
@@ -1,94 +0,0 @@
|
||||
# main-red-watchdog — hourly sentinel for post-merge CI red on `main`.
|
||||
#
|
||||
# RFC: hongming "main NEVER goes red" directive, Option C of the four-
|
||||
# option ladder (B = auto-revert is explicitly rejected per
|
||||
# `feedback_no_such_thing_as_flakes` + `feedback_fix_root_not_symptom`).
|
||||
# Tracking issue: molecule-core#420.
|
||||
#
|
||||
# What it does:
|
||||
# 1. GET branches/main → HEAD SHA
|
||||
# 2. GET commits/{SHA}/status → combined status
|
||||
# 3. If combined is `failure` (or any individual status is `failure`):
|
||||
# open or PATCH an idempotent `[main-red] {repo}: {SHA[:10]}` issue
|
||||
# with each failed context + target_url + description.
|
||||
# 4. If combined is `success` and a prior `[main-red] ...` issue exists,
|
||||
# close it with a "main returned to green at SHA ..." comment.
|
||||
# 5. Emit a Loki-shaped JSON line via `logger -t main-red-watchdog` for
|
||||
# `reference_obs_stack_phase1` ingestion via Vector.
|
||||
#
|
||||
# What it does NOT do:
|
||||
# - Auto-revert anything. Option B is rejected by directive.
|
||||
# - Mutate branch protection. (See AGENTS.md boundaries.)
|
||||
# - Fail the workflow on red. The issue IS the alarm — failing the
|
||||
# watchdog would create a silent-loop where a flake in the watchdog
|
||||
# itself hides actual main-red signal. Exit 0 unless api() raises
|
||||
# ApiError (transient Gitea outage → fail loudly per
|
||||
# `feedback_api_helper_must_raise_not_return_dict`).
|
||||
#
|
||||
# Pattern source: molecule-controlplane `0adf2098`'s ci-required-drift.yml
|
||||
# (just merged 2026-05-11). Same shape (cron + dispatch + sidecar Python +
|
||||
# idempotent-by-title issue), simpler scope (1 source, not 3).
|
||||
|
||||
name: main-red-watchdog
|
||||
|
||||
# IMPORTANT — Gitea 1.22.6 parser quirk per
|
||||
# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
|
||||
# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
|
||||
# "unknown on type" when `workflow_dispatch.inputs.X` is present. Revisit
|
||||
# when Gitea ≥ 1.23 is fleet-wide.
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :05 — task spec calls for "off-zero" (`5 * * * *`),
|
||||
# offset from :17 (ci-required-drift) and :00 (peak cron load).
|
||||
- cron: '5 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# Read commit status + branch ref + issues; write issues (open/PATCH/close).
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
# Workflow-scoped serialisation — two simultaneous runs would race on the
|
||||
# `[main-red] {SHA}` open/PATCH path. Idempotent by title, but parallel
|
||||
# POSTs can produce duplicates before the title search dedup wins.
|
||||
concurrency:
|
||||
group: main-red-watchdog
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
watchdog:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out repo (script lives at .gitea/scripts/)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up Python (stdlib only — no PyYAML needed here)
|
||||
# The script uses stdlib urllib + json. No PyYAML required (CP's
|
||||
# drift detector needs it for AST parsing; we don't). Pin to the
|
||||
# same 3.12 hermetic interpreter CP uses so the test/runtime
|
||||
# versions stay aligned across watchdog suites.
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Run main-red watchdog
|
||||
env:
|
||||
# GITEA_TOKEN reads commit status + writes issues. Falls back
|
||||
# to the auto-injected GITHUB_TOKEN if the org-level secret
|
||||
# isn't set (transitional repos), matching the same pattern
|
||||
# used by deploy-pipeline.yml + ci-required-drift.yml.
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# Branch under watch. `main` per directive; staging not
|
||||
# included here — staging green is a separate gate
|
||||
# (`feedback_staging_e2e_merge_gate`).
|
||||
WATCH_BRANCH: 'main'
|
||||
# Issue label applied on file/open. `tier:high` exists in the
|
||||
# molecule-core label set (verified 2026-05-11, label id 9).
|
||||
# Rationale for high: main red blocks the promotion train and
|
||||
# poisons every PR's auto-rebase base; treat as a fire even
|
||||
# if intermittent.
|
||||
RED_LABEL: 'tier:high'
|
||||
run: python3 .gitea/scripts/main-red-watchdog.py
|
||||
@@ -1,138 +0,0 @@
|
||||
name: publish-canvas-image
|
||||
|
||||
# Ported from .github/workflows/publish-canvas-image.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Open question for review**: this workflow pushes the canvas
|
||||
# image to `ghcr.io`. GHCR was retired during the 2026-05-06
|
||||
# Gitea migration in favor of ECR (per canary-verify.yml header
|
||||
# notes). The image may not be consumable post-migration. Two
|
||||
# options for follow-up: (a) retarget to
|
||||
# `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas`,
|
||||
# or (b) retire this workflow entirely and route canvas deploys
|
||||
# via the operator-host build path. tier:low + continue-on-error
|
||||
# means failed pushes do not block PRs.
|
||||
#
|
||||
|
||||
# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
|
||||
# on main that touches canvas code. Previously canvas changes were visible in
|
||||
# CI (npm run build passed) but the live container was never updated —
|
||||
# operators had to manually run `docker compose build canvas` each time.
|
||||
#
|
||||
# Mirror of publish-platform-image.yml, adapted for the Next.js canvas layer.
|
||||
# See that workflow for inline notes on macOS Keychain isolation and QEMU.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
# Only rebuild when canvas source changes — saves GHA minutes on
|
||||
# platform-only / docs-only / MCP-only merges.
|
||||
- 'canvas/**'
|
||||
- '.gitea/workflows/publish-canvas-image.yml'
|
||||
# NOTE (Gitea port): the original GitHub workflow had a
|
||||
# `workflow_dispatch:` manual trigger for the
|
||||
# non-canvas-merge-but-need-fresh-image scenario. Dropped in the
|
||||
# Gitea port (1.22.6 parser-finicky). Manual rebuilds require
|
||||
# pushing an empty commit to canvas/ or running the operator-host
|
||||
# build directly.
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write # required to push to ghcr.io/${{ github.repository_owner }}/*
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/canvas
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build & push canvas image
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing to the build step
|
||||
# where docker build fails deep in ECR auth with a cryptic error.
|
||||
- name: Verify Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
|
||||
exit 1
|
||||
}
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Compute tags
|
||||
id: tags
|
||||
shell: bash
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve build args
|
||||
id: build_args
|
||||
# Priority: workflow_dispatch input > repo secret > hardcoded default.
|
||||
# NEXT_PUBLIC_* env vars are baked into the JS bundle at build time by
|
||||
# Next.js — they cannot be changed at runtime without a full rebuild.
|
||||
# For local docker-compose deployments the defaults (localhost:8080)
|
||||
# work as-is; production deployments should set CANVAS_PLATFORM_URL
|
||||
# and CANVAS_WS_URL as repository secrets.
|
||||
#
|
||||
# Inputs are passed via env vars (not direct ${{ }} interpolation) to
|
||||
# prevent shell injection from workflow_dispatch string inputs.
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_PLATFORM_URL: ${{ github.event.inputs.platform_url }}
|
||||
SECRET_PLATFORM_URL: ${{ secrets.CANVAS_PLATFORM_URL }}
|
||||
INPUT_WS_URL: ${{ github.event.inputs.ws_url }}
|
||||
SECRET_WS_URL: ${{ secrets.CANVAS_WS_URL }}
|
||||
run: |
|
||||
PLATFORM_URL="${INPUT_PLATFORM_URL:-${SECRET_PLATFORM_URL:-http://localhost:8080}}"
|
||||
WS_URL="${INPUT_WS_URL:-${SECRET_WS_URL:-ws://localhost:8080/ws}}"
|
||||
|
||||
echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build & push canvas image to GHCR
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ./canvas
|
||||
file: ./canvas/Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PLATFORM_URL=${{ steps.build_args.outputs.platform_url }}
|
||||
NEXT_PUBLIC_WS_URL=${{ steps.build_args.outputs.ws_url }}
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)
|
||||
@@ -207,23 +207,13 @@ jobs:
|
||||
|
||||
# Stage (b): download wheel + SHA256 compare against what we built.
|
||||
# Catches Fastly stale-content serving old bytes under a new version URL.
|
||||
#
|
||||
# Caught run 5196 (first-ever successful publish, 2026-05-11): the
|
||||
# previous one-liner `HASH=$(pip download ... && sha256sum ...)`
|
||||
# captured pip's stdout (`Collecting molecule-ai-workspace-runtime
|
||||
# ==X.Y.Z`) into HASH, then the SHA comparison failed against the
|
||||
# leaked `Collecting...` string. `2>/dev/null` silences stderr but
|
||||
# NOT stdout; pip writes its progress to stdout by default.
|
||||
# Fix: split into two steps, silence pip's stdout explicitly, capture
|
||||
# only sha256sum's output into HASH.
|
||||
python -m pip download \
|
||||
--no-deps \
|
||||
--no-cache-dir \
|
||||
--dest /tmp/wheel-probe \
|
||||
--quiet \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1
|
||||
HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
|
||||
HASH=$(python -m pip download \
|
||||
--no-deps \
|
||||
--no-cache-dir \
|
||||
--dest /tmp/wheel-probe \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
2>/dev/null \
|
||||
&& sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
|
||||
if [ "$HASH" != "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
|
||||
echo "::error::Expected: $EXPECTED_SHA256"
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
name: Railway pin audit (drift detection)
|
||||
|
||||
# Ported from .github/workflows/railway-pin-audit.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch:` (Gitea 1.22.6 trigger handling).
|
||||
# Manual runs go via cron-trigger bump or push the workflow file
|
||||
# itself.
|
||||
# - `actions/github-script@v9` blocks (which call github.rest.* — a
|
||||
# GitHub-specific JS API) replaced with curl calls against the
|
||||
# Gitea REST API (/api/v1/repos/.../issues, .../labels,
|
||||
# .../comments). Same behaviour: open issue on drift, comment on
|
||||
# repeat-drift, close on clean run.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set so the curl calls can
|
||||
# derive `git.moleculesai.app` from the runner env (with
|
||||
# hard-coded fallback inside the steps).
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Daily audit of Railway env vars for drift-prone image-tag pins —
|
||||
# automation-cadence layer over the detection script + regression test
|
||||
# shipped in PR #2168 (#2001 closure).
|
||||
#
|
||||
# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
|
||||
# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
|
||||
# "every fix didn't propagate" — really the tenant image was so old it
|
||||
# didn't read the env vars those fixes produced.
|
||||
#
|
||||
# Cadence: once a day, 13:00 UTC (06:00 PT).
|
||||
#
|
||||
# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
|
||||
# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 13 * * *'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: railway-pin-audit
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
name: Audit Railway env vars for drift-prone pins
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify RAILWAY_AUDIT_TOKEN present
|
||||
env:
|
||||
RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
id: secret_check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
|
||||
echo "have_secret=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "have_secret=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
|
||||
exit 1
|
||||
|
||||
- name: Install Railway CLI
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL https://railway.com/install.sh | sh
|
||||
echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify Railway CLI authenticated
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! railway whoami >/dev/null 2>&1; then
|
||||
echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Link molecule-platform project
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
|
||||
|
||||
- name: Run drift audit
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
id: audit
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# Capture the audit log for the issue body.
|
||||
{
|
||||
echo 'log<<AUDIT_EOF'
|
||||
cat /tmp/audit.log
|
||||
echo 'AUDIT_EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
case "$rc" in
|
||||
0) exit 0 ;;
|
||||
1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
|
||||
2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
|
||||
*) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Open / update drift issue (Gitea API)
|
||||
if: failure() && steps.audit.outputs.rc == '1'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
AUDIT_LOG: ${{ steps.audit.outputs.log }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Railway env-var drift detected"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
BODY=$(jq -nc --arg t "$TITLE" --arg log "${AUDIT_LOG:-(log unavailable)}" --arg run "$RUN_URL" '
|
||||
{body: ("Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n**What this means:** an env var (likely on `controlplane`) is pinned to a SHA-shaped or semver tag instead of a floating tag. Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service does not pick them up.\n\n**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (:staging-latest, :main) unless the pin is intentional and documented in the ops runbook.\n\n**Audit output:**\n\n```\n" + $log + "\n```\n\nRun: " + $run + "\n\nCloses automatically when a subsequent daily run reports clean.")}')
|
||||
|
||||
# Look for existing open drift issue with the title.
|
||||
EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
|
||||
|
||||
if [ -n "$EXISTING" ]; then
|
||||
COMMENT_BODY=$(jq -nc --arg log "${AUDIT_LOG:-(log unavailable)}" --arg run "$RUN_URL" \
|
||||
'{body: ("Still drifting. " + $run + "\n\n```\n" + $log + "\n```")}')
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" -d "$COMMENT_BODY" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
CREATE_BODY=$(echo "$BODY" | jq --arg t "$TITLE" '. + {title: $t, labels: []}')
|
||||
NUM=$(curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$CREATE_BODY" | jq -r .number)
|
||||
echo "Filed issue #${NUM}"
|
||||
fi
|
||||
|
||||
- name: Close stale drift issue on clean run (Gitea API)
|
||||
if: success() && steps.audit.outputs.rc == '0'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Railway env-var drift detected"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number')
|
||||
|
||||
for N in $NUMS; do
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Daily audit clean — drift resolved. " + $run)}')" >/dev/null
|
||||
curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
|
||||
echo "Closed #${N}"
|
||||
done
|
||||
@@ -1,375 +0,0 @@
|
||||
name: redeploy-tenants-on-main
|
||||
|
||||
# Ported from .github/workflows/redeploy-tenants-on-main.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on .gitea/workflows/publish-workspace-server-image.yml. Until
|
||||
# then continue-on-error+dead-workflow doesn't break anything.
|
||||
#
|
||||
|
||||
# Auto-refresh prod tenant EC2s after every main merge.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image builds and
|
||||
# pushes a new platform-tenant :<sha> to ECR on every merge to main,
|
||||
# but running tenants pulled their image once at boot and never re-pull.
|
||||
# Users see stale code indefinitely.
|
||||
#
|
||||
# This workflow closes the gap by calling the control-plane admin
|
||||
# endpoint that performs a canary-first, batched, health-gated rolling
|
||||
# redeploy across every live tenant. Implemented in molecule-ai/
|
||||
# molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
|
||||
# (feat/tenant-auto-redeploy, landing alongside this workflow).
|
||||
#
|
||||
# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
|
||||
# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
|
||||
# Gitea suspension migration. The canary-verify.yml promote step now
|
||||
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
|
||||
#
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
|
||||
# 2. This workflow fires via workflow_run, calls redeploy-fleet with
|
||||
# target_tag=staging-<sha>. No CDN propagation wait needed —
|
||||
# ECR image manifest is consistent immediately after push.
|
||||
# 3. Calls redeploy-fleet with canary_slug (if set) and a soak
|
||||
# period. Canary proves the image boots; batches follow.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: re-run this workflow with a specific SHA pinned via
|
||||
# the workflow_dispatch input. That calls redeploy-fleet with
|
||||
# target_tag=<sha>, re-pulling the older image on every tenant.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize redeploys so two rapid main pushes' redeploys don't overlap
|
||||
# and cause confusing per-tenant SSM state. Without this, GitHub's
|
||||
# implicit workflow_run queueing would *probably* serialize them, but
|
||||
# the explicit block makes the invariant defensible. Mirrors the
|
||||
# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
|
||||
#
|
||||
# cancel-in-progress: false → aborting a half-rolled-out fleet would
|
||||
# leave tenants stuck on whatever image they happened to be on when
|
||||
# cancelled. Better to finish the in-flight rollout before starting
|
||||
# the next one.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-main
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
redeploy:
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Note on ECR propagation
|
||||
# ECR image manifests are consistent immediately after push — no
|
||||
# CDN cache to wait for. The old GHCR-based workflow had a 30s
|
||||
# sleep to avoid race conditions; ECR makes that unnecessary.
|
||||
run: echo "ECR image available immediately after push — proceeding."
|
||||
|
||||
- name: Compute target tag
|
||||
id: tag
|
||||
# Resolution order:
|
||||
# 1. Operator-supplied input (workflow_dispatch with explicit
|
||||
# tag) → used verbatim. Lets ops pin `latest` for emergency
|
||||
# rollback to last canary-verified digest, or pin a specific
|
||||
# `staging-<sha>` to roll back to a known-good build.
|
||||
# 2. Default → `staging-<short_head_sha>`. The just-published
|
||||
# digest. Bypasses the `:latest` retag path that's currently
|
||||
# dead (canary-verify soft-skips without canary fleet, so
|
||||
# the only thing retagging `:latest` today is the manual
|
||||
# promote-latest.yml — last run 2026-04-28). Auto-trigger
|
||||
# from workflow_run uses workflow_run.head_sha; manual
|
||||
# dispatch with no input falls through to github.sha.
|
||||
env:
|
||||
INPUT_TAG: ${{ inputs.target_tag }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${INPUT_TAG:-}" ]; then
|
||||
echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag: $INPUT_TAG"
|
||||
else
|
||||
SHORT="${HEAD_SHA:0:7}"
|
||||
echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
|
||||
echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
|
||||
fi
|
||||
|
||||
- name: Call CP redeploy-fleet
|
||||
# CP_ADMIN_API_TOKEN must be set as a repo/org secret on
|
||||
# molecule-ai/molecule-core, matching the staging/prod CP's
|
||||
# CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
|
||||
# repo's secrets for CI.
|
||||
env:
|
||||
CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
--argjson soak "$SOAK_SECONDS" \
|
||||
--argjson batch "$BATCH_SIZE" \
|
||||
--argjson dry "$DRY_RUN" \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
# Route -w into its own tempfile so curl's exit code (e.g. 56
|
||||
# on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
|
||||
# pollute the captured stdout. The previous inline-substitution
|
||||
# shape produced "000000" on connection reset (curl wrote
|
||||
# "000" via -w, then the inline echo-fallback appended another
|
||||
# "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
|
||||
# set +e/-e keeps the non-zero curl exit from tripping the
|
||||
# outer pipeline. See lint-curl-status-capture.yml for the
|
||||
# CI gate that pins this fix shape.
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
set -e
|
||||
# Stderr from curl (e.g. dial errors with -sS) goes to the runner
|
||||
# log so operators can see WHY a connection failed. Stdout is
|
||||
# captured to $HTTP_CODE_FILE because that's where -w writes.
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
# Pretty-print per-tenant results in the job summary so
|
||||
# ops can see which tenants were redeployed without drilling
|
||||
# into the raw response.
|
||||
{
|
||||
echo "## Tenant redeploy fleet"
|
||||
echo ""
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**Canary:** \`$CANARY_SLUG\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "**Batch size:** $BATCH_SIZE"
|
||||
echo "**Dry run:** $DRY_RUN"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
exit 1
|
||||
fi
|
||||
OK=$(jq -r '.ok' "$HTTP_RESPONSE")
|
||||
if [ "$OK" != "true" ]; then
|
||||
echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
|
||||
|
||||
# Stash the response for the verify step. $RUNNER_TEMP outlasts
|
||||
# the step boundary; $HTTP_RESPONSE doesn't.
|
||||
cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
|
||||
|
||||
- name: Verify each tenant /buildinfo matches published SHA
|
||||
# ROOT FIX FOR #2395.
|
||||
#
|
||||
# `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
|
||||
# didn't error" — NOT "the new image is running on the tenant."
|
||||
# `:latest` lives in the local Docker daemon's image cache; if
|
||||
# the SSM document does `docker compose up -d` without an
|
||||
# explicit `docker pull`, the daemon serves the previously-
|
||||
# cached digest and the container restarts on stale code.
|
||||
# 2026-04-30 incident: hongmingwang's tenant reported
|
||||
# ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
|
||||
# chat_files for 30+ min — the lazy-heal fix never reached the
|
||||
# user despite green deploy + green redeploy.
|
||||
#
|
||||
# This step closes the gap by curling each tenant's /buildinfo
|
||||
# endpoint (added in workspace-server/internal/buildinfo +
|
||||
# /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
|
||||
# returned git_sha to the SHA the workflow expects. Mismatches
|
||||
# fail the workflow, which is what `ok=true` should have
|
||||
# guaranteed all along.
|
||||
#
|
||||
# When the redeploy was triggered by workflow_dispatch with a
|
||||
# specific tag (target_tag != "latest"), the expected SHA may
|
||||
# not equal ${{ github.sha }} — in that case we resolve via
|
||||
# GHCR's manifest. For workflow_run (default :latest) the
|
||||
# workflow_run.head_sha is the SHA that just published.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
# Tenant subdomain template — slugs from the response are
|
||||
# appended. Production CP issues `<slug>.moleculesai.app`;
|
||||
# staging CP issues `<slug>.staging.moleculesai.app`. This
|
||||
# workflow runs on main → prod CP → no `staging.` infix.
|
||||
TENANT_DOMAIN: 'moleculesai.app'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
|
||||
if [ "$TARGET_TAG" != "latest" ] \
|
||||
&& [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
|
||||
&& [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
|
||||
# workflow_dispatch with a pinned tag that isn't the head
|
||||
# SHA — operator is rolling back / pinning. Skip the
|
||||
# verification because we don't have the expected SHA in
|
||||
# this context (would need to crane-inspect the GHCR
|
||||
# manifest, which is a follow-up). Failing-open here is
|
||||
# safe: the operator chose the tag deliberately.
|
||||
#
|
||||
# `staging-<short_head_sha>` IS verified — it's the new
|
||||
# auto-trigger default (see Compute target tag step) and
|
||||
# the digest under that tag SHOULD match EXPECTED_SHA.
|
||||
echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RESP="$RUNNER_TEMP/redeploy-response.json"
|
||||
if [ ! -s "$RESP" ]; then
|
||||
echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pull only successfully-redeployed tenants. Any tenant that
|
||||
# halted the rollout already failed the previous step, so we
|
||||
# don't double-count them here.
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::warning::No tenants reported healthz_ok — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
|
||||
|
||||
# Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
|
||||
# vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
|
||||
# comment for the full rationale; same logic applies on prod even
|
||||
# though prod has fewer ephemeral tenants — the asymmetry would be a
|
||||
# gratuitous fork.
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
STALE_LINES=()
|
||||
UNREACHABLE_LINES=()
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
# 30s total: tenant just SSM-restarted, may still be coming
|
||||
# up. Retry-on-empty rather than retry-on-status — we want
|
||||
# to fail fast on "responded with wrong SHA", not "still
|
||||
# warming up".
|
||||
BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
|
||||
ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
|
||||
if [ -z "$ACTUAL_SHA" ]; then
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
|
||||
continue
|
||||
fi
|
||||
if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
|
||||
echo " $slug: ${ACTUAL_SHA:0:7} ✓"
|
||||
else
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Per-tenant /buildinfo verification"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
|
||||
echo ""
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${STALE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
|
||||
echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders sanity floor: same logic as the staging
|
||||
# variant — see that file's comment for the full rationale.
|
||||
# Floor only applies when fleet >= 4; below that, canary-verify
|
||||
# is the actual gate.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
|
||||
@@ -1,356 +0,0 @@
|
||||
name: redeploy-tenants-on-staging
|
||||
|
||||
# Ported from .github/workflows/redeploy-tenants-on-staging.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on .gitea/workflows/publish-workspace-server-image.yml. Until
|
||||
# then continue-on-error+dead-workflow doesn't break anything.
|
||||
#
|
||||
|
||||
# Auto-refresh staging tenant EC2s after every staging-branch merge.
|
||||
#
|
||||
# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
|
||||
# the :staging-latest tag. Sister workflow exists for prod (rolls
|
||||
# :latest after canary-verify). Both share the same shape — just
|
||||
# different CP_URL + target_tag + admin token secret.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image now builds
|
||||
# on every staging-branch push (PR #2335), pushing
|
||||
# platform-tenant:staging-latest to GHCR. Existing tenants pulled
|
||||
# their image once at boot and never re-pull, so the new image just
|
||||
# sits unused until the tenant is reprovisioned.
|
||||
#
|
||||
# This workflow closes the gap by calling staging-CP's
|
||||
# /cp/admin/tenants/redeploy-fleet, which performs a canary-first,
|
||||
# batched, health-gated SSM redeploy across every live staging tenant.
|
||||
# Same endpoint shape as prod CP — only the host differs.
|
||||
#
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes on staging branch →
|
||||
# new :staging-latest in GHCR.
|
||||
# 2. This workflow fires via workflow_run, waits 30s for GHCR's CDN
|
||||
# to propagate the new tag.
|
||||
# 3. Calls redeploy-fleet with no canary (staging IS canary; we don't
|
||||
# need a sub-canary inside it). Soak still applies to the first
|
||||
# tenant in case of bad-deploy detection.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: re-run with workflow_dispatch + target_tag=staging-<sha>
|
||||
# of a known-good build.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize per-branch so two rapid staging pushes' redeploys don't
|
||||
# overlap and cause confusing per-tenant SSM state. cancel-in-progress
|
||||
# is false because aborting a half-rolled-out fleet leaves tenants
|
||||
# stuck on whatever image they happened to be on when cancelled.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
redeploy:
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Wait for GHCR tag propagation
|
||||
# GHCR's edge cache takes ~15-30s to consistently serve the new
|
||||
# :staging-latest manifest after the registry accepts the push.
|
||||
# Same rationale as redeploy-tenants-on-main.yml.
|
||||
run: sleep 30
|
||||
|
||||
- name: Call staging-CP redeploy-fleet
|
||||
# CP_STAGING_ADMIN_API_TOKEN must be set as a repo/org secret
|
||||
# on molecule-ai/molecule-core, matching staging-CP's
|
||||
# CP_ADMIN_API_TOKEN env var (visible in Railway controlplane
|
||||
# / staging environment). Stored separately from the prod
|
||||
# CP_ADMIN_API_TOKEN so a leak of one doesn't auth the other.
|
||||
env:
|
||||
CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || '' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Schedule-vs-dispatch hardening (mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels): hard-fail on auto-trigger when the
|
||||
# secret is missing so a misconfigured-repo doesn't silently
|
||||
# serve stale staging tenants. Soft-skip on operator dispatch.
|
||||
if [ -z "${CP_STAGING_ADMIN_API_TOKEN:-}" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::CP_STAGING_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::warning::Set CP_STAGING_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
echo "::notice::Pull the value from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::staging redeploy cannot run — CP_STAGING_ADMIN_API_TOKEN secret missing"
|
||||
echo "::error::set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
--argjson soak "$SOAK_SECONDS" \
|
||||
--argjson batch "$BATCH_SIZE" \
|
||||
--argjson dry "$DRY_RUN" \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
# Route -w into its own tempfile so curl's exit code (e.g. 56
|
||||
# on connection-reset) can't pollute the captured stdout. The
|
||||
# previous inline-substitution shape produced "000000" on
|
||||
# connection reset — caught on main variant 2026-05-04
|
||||
# redeploying sha 2b862f6. Same fix shape as the synth-E2E
|
||||
# §9c gate (PR #2797). See lint-curl-status-capture.yml for
|
||||
# the CI gate that pins this fix shape.
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to the
|
||||
# runner log so operators can see WHY a connection failed.
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
{
|
||||
echo "## Staging tenant redeploy fleet"
|
||||
echo ""
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**Canary:** \`${CANARY_SLUG:-(none — staging is itself the canary)}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "**Batch size:** $BATCH_SIZE"
|
||||
echo "**Dry run:** $DRY_RUN"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Distinguish "real fleet failure" from "E2E teardown race".
|
||||
#
|
||||
# CP returns HTTP 500 + ok=false whenever ANY tenant in the
|
||||
# fleet failed SSM or healthz. In practice the recurring source
|
||||
# of these is ephemeral test tenants being torn down by their
|
||||
# parent E2E run mid-redeploy: the EC2 dies → SSM exit=2 or
|
||||
# healthz timeout → CP marks the fleet failed → this workflow
|
||||
# goes red even though every operator-facing tenant rolled fine.
|
||||
#
|
||||
# Ephemeral slug prefixes (kept in sync with sweep-stale-e2e-orgs.yml
|
||||
# — see that file for the source-of-truth list and rationale):
|
||||
# - e2e-* — canvas/saas/ext E2E suites
|
||||
# - rt-e2e-* — runtime-test harness fixtures (RFC #2251)
|
||||
# Long-lived prefixes that are NOT ephemeral and MUST hard-fail:
|
||||
# demo-prep, dryrun-*, dryrun2-*, plus all human tenant slugs.
|
||||
#
|
||||
# Filter: if HTTP=500/ok=false AND every failed slug matches an
|
||||
# ephemeral prefix, treat as soft-warn and let the verify step
|
||||
# downstream handle unreachable-vs-stale (#2402). Any non-ephemeral
|
||||
# failure or a non-500 HTTP response remains a hard failure.
|
||||
OK=$(jq -r '.ok // "false"' "$HTTP_RESPONSE")
|
||||
FAILED_SLUGS=$(jq -r '
|
||||
.results[]?
|
||||
| select((.healthz_ok != true) or (.ssm_status != "Success"))
|
||||
| .slug' "$HTTP_RESPONSE" 2>/dev/null || true)
|
||||
EPHEMERAL_PREFIX_RE='^(e2e-|rt-e2e-)'
|
||||
NON_EPHEMERAL_FAILED=$(printf '%s\n' "$FAILED_SLUGS" | grep -v '^$' | grep -Ev "$EPHEMERAL_PREFIX_RE" || true)
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ] && [ "$OK" = "true" ]; then
|
||||
: # happy path — fall through to verification
|
||||
elif [ "$HTTP_CODE" = "500" ] && [ -z "$NON_EPHEMERAL_FAILED" ] && [ -n "$FAILED_SLUGS" ]; then
|
||||
COUNT=$(printf '%s\n' "$FAILED_SLUGS" | grep -Ec "$EPHEMERAL_PREFIX_RE" || true)
|
||||
echo "::warning::redeploy-fleet returned HTTP 500 but every failed tenant ($COUNT) is ephemeral (e2e-*/rt-e2e-*) — treating as teardown race, soft-warning."
|
||||
printf '%s\n' "$FAILED_SLUGS" | sed 's/^/::warning:: failed: /'
|
||||
elif [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
if [ -n "$NON_EPHEMERAL_FAILED" ]; then
|
||||
echo "::error::non-ephemeral tenant(s) failed:"
|
||||
printf '%s\n' "$NON_EPHEMERAL_FAILED" | sed 's/^/::error:: /'
|
||||
fi
|
||||
exit 1
|
||||
else
|
||||
# HTTP=200 but ok=false (shouldn't happen with current CP
|
||||
# but keep the gate for completeness).
|
||||
echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Staging tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
|
||||
|
||||
cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
|
||||
|
||||
- name: Verify each staging tenant /buildinfo matches published SHA
|
||||
# Mirror of the verify step in redeploy-tenants-on-main.yml — see
|
||||
# there for the rationale (#2395 root fix). Staging has the same
|
||||
# ssm_status-success-but-stale-image hazard and benefits from the
|
||||
# same gate. Diff: TENANT_DOMAIN includes the `staging.` infix.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
|
||||
TENANT_DOMAIN: 'staging.moleculesai.app'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# staging-latest is the staging-side moving tag; treat it the
|
||||
# same way main treats `latest`. Operator-pinned SHAs skip
|
||||
# verification (see main variant for why).
|
||||
if [ "$TARGET_TAG" != "staging-latest" ] && [ "$TARGET_TAG" != "latest" ] && [ "$TARGET_TAG" != "$EXPECTED_SHA" ]; then
|
||||
echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RESP="$RUNNER_TEMP/redeploy-response.json"
|
||||
if [ ! -s "$RESP" ]; then
|
||||
echo "::error::redeploy-response.json missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::warning::No staging tenants reported healthz_ok — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Verifying ${#SLUGS[@]} staging tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
|
||||
|
||||
# Two distinct failure modes here:
|
||||
# STALE_COUNT — tenant returned a SHA that doesn't match. THIS is
|
||||
# the #2395 bug class: tenant up + serving old code.
|
||||
# Always hard-fail the workflow.
|
||||
# UNREACHABLE_COUNT — tenant didn't respond. Almost always a benign
|
||||
# teardown race: redeploy-fleet snapshot says
|
||||
# healthz_ok=true, then the E2E suite tears the
|
||||
# ephemeral tenant down before this step runs (the
|
||||
# e2e-* fixtures churn 5-10/hour on staging). Soft-
|
||||
# warn so we don't block staging→main on cleanup.
|
||||
# Real "tenant up but unreachable" is caught by CP's
|
||||
# own healthz monitor + the post-redeploy alert; we
|
||||
# don't need to double-count it here.
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
STALE_LINES=()
|
||||
UNREACHABLE_LINES=()
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
|
||||
ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
|
||||
if [ -z "$ACTUAL_SHA" ]; then
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
|
||||
continue
|
||||
fi
|
||||
if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
|
||||
echo " $slug: ${ACTUAL_SHA:0:7} ✓"
|
||||
else
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Per-tenant /buildinfo verification (staging)"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
|
||||
echo ""
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${STALE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely E2E teardown race (soft-warn, not failing):**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
|
||||
echo "All ${#SLUGS[@]} staging tenants returned matching SHA. ✓"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "::warning::$UNREACHABLE_COUNT staging tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders sanity floor: if MORE than half the fleet is
|
||||
# unreachable AND the fleet is large enough that "half down" is
|
||||
# statistically meaningful, this is a real outage (e.g. new image
|
||||
# crashes on startup), not a teardown race. Hard-fail.
|
||||
#
|
||||
# Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
|
||||
# canary-verify step is the actual gate for "all tenants down"
|
||||
# detection (it runs against the canary first and aborts the
|
||||
# rollout if the canary fails to come up). Without the >=4 gate,
|
||||
# a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
|
||||
# quiet staging push) would re-flake on the exact teardown-race
|
||||
# condition #2402 fixed: 1 of 1 unreachable = 100% > 50% → fail.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED staging tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "::error::$STALE_COUNT staging tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::Staging tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
|
||||
@@ -1,100 +0,0 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and
|
||||
# `workflow_dispatch:` (no inputs, but the trigger itself is
|
||||
# parser-rejected when inputs are absent in some Gitea 1.22.x
|
||||
# builds; safest to drop entirely — manual runs go via cron-trigger
|
||||
# bump or push-with-paths-filter).
|
||||
# - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -1,139 +0,0 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
|
||||
# (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
|
||||
# finicky without them).
|
||||
# - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
|
||||
# pattern for ci.yml port).
|
||||
# - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on every job (RFC §1 contract).
|
||||
#
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy -> all crash on first boot with ImportError.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
# event_name + sha keeps PR sync and the subsequent staging push on the
|
||||
# same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
run: |
|
||||
# Inline replacement for dorny/paths-filter — same pattern
|
||||
# PR#372's ci.yml port used. Diffs against the PR base or the
|
||||
# previous push SHA, then matches against the wheel-relevant
|
||||
# path set.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
# New branch or no previous SHA: treat as wheel-relevant.
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`.
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -1,70 +0,0 @@
|
||||
name: SECRET_PATTERNS drift lint
|
||||
|
||||
# Ported from .github/workflows/secret-pattern-drift.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths references the new canonical .gitea/workflows/secret-scan.yml
|
||||
# (the .github/ copy is removed by Cat A of this sweep).
|
||||
# - CANONICAL_FILE inside scripts/lint_secret_pattern_drift.py was
|
||||
# updated in the same Cat C-1 PR to point at .gitea/workflows/secret-scan.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Detects when the canonical SECRET_PATTERNS array in
|
||||
# .gitea/workflows/secret-scan.yml diverges from known consumer
|
||||
# mirrors (workspace-runtime's bundled pre-commit hook today; more
|
||||
# can be added as the consumer set grows).
|
||||
#
|
||||
# Why this exists: every side that scans for credentials has its own
|
||||
# copy of the pattern list. They drift — most recently the runtime
|
||||
# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
|
||||
# so a developer's local pre-commit would let a sk-cp- token through
|
||||
# while the org-wide CI scan would refuse it. The cost of that drift
|
||||
# is dev confusion + delayed feedback; the fix is automated detection.
|
||||
#
|
||||
# Triggers:
|
||||
# - schedule: daily 05:00 UTC. Catches drift introduced by edits
|
||||
# to a consumer copy that didn't update canonical here.
|
||||
# - push to main/staging where the canonical or this lint changed:
|
||||
# catches the inverse — canonical updated but consumers not yet
|
||||
# bumped. The lint will fail the push; that's intentional.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
|
||||
# email lands when humans are starting their day, not
|
||||
# interrupting it.
|
||||
- cron: "0 5 * * *"
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- ".gitea/workflows/secret-scan.yml"
|
||||
- ".gitea/workflows/secret-pattern-drift.yml"
|
||||
- ".github/scripts/lint_secret_pattern_drift.py"
|
||||
- ".githooks/pre-commit"
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
# Auto-injected GITHUB_TOKEN scoped to read-only. The lint only does git
|
||||
# checkout + HTTPS GETs to public consumer files; no writes to anything.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Detect SECRET_PATTERNS drift
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Run drift lint
|
||||
run: python3 .github/scripts/lint_secret_pattern_drift.py
|
||||
@@ -77,50 +77,24 @@ jobs:
|
||||
# works if we never check out PR HEAD. Same SHA the workflow
|
||||
# itself was loaded from.
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Install jq
|
||||
# Gitea Actions runners (ubuntu-latest label) do not bundle jq.
|
||||
# The sop-tier-check script uses jq for all JSON API parsing.
|
||||
# Install jq before the script runs so sop-tier-check can pass.
|
||||
#
|
||||
# Method: apt-get first (reliable for Ubuntu runners with internet
|
||||
# access to package mirrors). Falls back to GitHub binary download.
|
||||
# GitHub releases may be unreachable from some runner networks
|
||||
# (infra#241 follow-up: GitHub timeout after 3s on 5.78.80.188
|
||||
# runners). The sop-tier-check script has its own fallback as a
|
||||
# third line of defense. continue-on-error: true ensures this step
|
||||
# failing does not block the job.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# apt-get is the primary method — Ubuntu package mirrors are reliably
|
||||
# reachable from runner containers. GitHub releases may be blocked
|
||||
# or slow on some networks (infra#241 follow-up).
|
||||
if apt-get update -qq && apt-get install -y -qq jq; then
|
||||
echo "::notice::jq installed via apt-get: $(jq --version)"
|
||||
elif timeout 120 curl -sSL \
|
||||
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
|
||||
-o /usr/local/bin/jq && chmod +x /usr/local/bin/jq; then
|
||||
echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
|
||||
else
|
||||
echo "::warning::jq install failed — apt-get and GitHub download both failed."
|
||||
fi
|
||||
jq --version 2>/dev/null || echo "::notice::jq not yet available — script fallback will retry"
|
||||
|
||||
- name: Verify tier label + reviewer team membership
|
||||
# continue-on-error: true at step level — job-level is ignored by Gitea
|
||||
# Actions (quirk #10, internal runbooks). Belt-and-suspenders with
|
||||
# SOP_FAIL_OPEN=1 + || true below.
|
||||
continue-on-error: true
|
||||
env:
|
||||
# SOP_TIER_CHECK_TOKEN is the org-level secret for the
|
||||
# sop-tier-bot PAT (read:organization,read:user,read:issue,
|
||||
# read:repository). Stored at the org level
|
||||
# (/api/v1/orgs/molecule-ai/actions/secrets) so per-repo
|
||||
# configuration is unnecessary — every repo in the org
|
||||
# picks it up automatically.
|
||||
# Falls back to GITHUB_TOKEN with a clear error if missing.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
# Set to '1' for diagnostic per-API-call output. Off by default
|
||||
# so production logs aren't noisy.
|
||||
SOP_DEBUG: '0'
|
||||
# BURN-IN: set to '1' for PRs in-flight at AND-composition deploy
|
||||
# time to use the legacy OR-gate. Remove after 2026-05-17.
|
||||
SOP_LEGACY_CHECK: '0'
|
||||
# SOP_FAIL_OPEN=1 makes the script always exit 0. The UI enforces
|
||||
# the actual merge gate. Combined with continue-on-error: true
|
||||
# above, this step never fails the job regardless of script exit.
|
||||
SOP_FAIL_OPEN: '1'
|
||||
run: |
|
||||
bash .gitea/scripts/sop-tier-check.sh || true
|
||||
run: bash .gitea/scripts/sop-tier-check.sh
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
name: Sweep stale AWS Secrets Manager secrets
|
||||
|
||||
# Ported from .github/workflows/sweep-aws-secrets.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for per-tenant AWS Secrets Manager secrets
|
||||
# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
|
||||
# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
|
||||
# sweep-cf-orphans.yml — different cloud, same justification.
|
||||
#
|
||||
# Why this exists separately from a long-term reconciler integration:
|
||||
# - molecule-controlplane's tenant_resources audit table (mig 024)
|
||||
# currently tracks four resource kinds: CloudflareTunnel,
|
||||
# CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
|
||||
# not in the list, so the existing reconciler doesn't catch
|
||||
# orphan secrets.
|
||||
# - At ~$0.40/secret/month the cost grew to ~$19/month before this
|
||||
# sweeper was written, indicating ~45+ orphan secrets from
|
||||
# crashed provisions and incomplete deprovision flows.
|
||||
# - The proper fix (KindSecretsManagerSecret + recorder hook +
|
||||
# reconciler enumerator) is filed as a separate controlplane
|
||||
# issue. This sweeper is the immediate cost-relief stopgap.
|
||||
#
|
||||
# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
|
||||
# This is a DEDICATED principal — the production `molecule-cp` IAM
|
||||
# user lacks `secretsmanager:ListSecrets` (it only has
|
||||
# Get/Create/Update/Delete on specific resources, scoped to its
|
||||
# operational needs). The janitor needs ListSecrets across the
|
||||
# `molecule/tenant/*` prefix, which warrants a separate principal so
|
||||
# we don't broaden the prod-CP policy.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
|
||||
# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
|
||||
# the mostly-orphan tunnels) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30 — offsets from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45) so the three janitors don't burst the
|
||||
# CP admin endpoints at the same minute.
|
||||
- cron: '30 * * * *'
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
group: sweep-aws-secrets
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
@@ -1,151 +0,0 @@
|
||||
name: Sweep stale Cloudflare DNS records
|
||||
|
||||
# Ported from .github/workflows/sweep-cf-orphans.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for Cloudflare DNS records whose backing tenant/workspace no
|
||||
# longer exists. Without this loop, every short-lived E2E or canary
|
||||
# leaves a CF record on the moleculesai.app zone — the zone has a
|
||||
# 200-record quota (controlplane#239 hit it 2026-04-23+) and provisions
|
||||
# start failing with code 81045 once exhausted.
|
||||
#
|
||||
# Why a separate workflow vs sweep-stale-e2e-orgs.yml:
|
||||
# - That workflow operates at the CP layer (DELETE /cp/admin/tenants/:slug
|
||||
# drives the cascade). It assumes CP has the org row to drive the
|
||||
# deprovision from. It doesn't catch records left behind when CP
|
||||
# itself never knew about the tenant (canary scratch, manual ops
|
||||
# experiments) or when the cascade's CF-delete branch failed.
|
||||
# - sweep-cf-orphans.sh enumerates the CF zone directly and matches
|
||||
# each record against live CP slugs + AWS EC2 names. It catches
|
||||
# leaks the CP-driven sweep can't.
|
||||
#
|
||||
# Safety: the script's own MAX_DELETE_PCT gate refuses to nuke more
|
||||
# than 50% of records in a single run. If something has gone weird
|
||||
# (CP admin endpoint returns no orgs → every tenant looks orphan) the
|
||||
# gate halts before damage. Decision-function unit tests in
|
||||
# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
|
||||
# classifier.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly. Mirrors sweep-stale-e2e-orgs cadence so the two janitors
|
||||
# converge on the same tick. CF API rate budget is generous (1200
|
||||
# req/5min); a single sweep makes ~1 list + N deletes (N<=quota/2).
|
||||
- cron: '15 * * * *' # offset from sweep-stale-e2e-orgs (top of hour)
|
||||
# No `merge_group:` trigger on purpose. This is a janitor — it doesn't
|
||||
# need to gate merges, and including it as written before #2088 fired
|
||||
# the full sweep job (or its secret-check) on every PR going through
|
||||
# the merge queue, generating one red CI run per merge-queue eval. If
|
||||
# this workflow is ever wired up as a required check, re-add
|
||||
# merge_group: { types: [checks_requested] }
|
||||
# AND gate the sweep step with `if: github.event_name != 'merge_group'`
|
||||
# so merge-queue evals report success without actually running.
|
||||
|
||||
# Don't let two sweeps race the same zone. workflow_dispatch during a
|
||||
# scheduled run would otherwise issue duplicate DELETE calls.
|
||||
concurrency:
|
||||
group: sweep-cf-orphans
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF orphans
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
|
||||
# within one cron interval instead of burning a full tick. Realistic
|
||||
# worst case is ~2 min: 4 sequential curls + 1 aws + N×CF-DELETE
|
||||
# each individually capped at 10s by the script's curl -m flag.
|
||||
timeout-minutes: 3
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split (hardened 2026-04-28
|
||||
# after the silent-no-op incident below):
|
||||
#
|
||||
# The earlier soft-skip-on-schedule policy hid a real leak. All
|
||||
# six secrets were unset on this repo for an unknown duration;
|
||||
# every hourly run printed a yellow ::warning:: and exited 0,
|
||||
# so the workflow registered as "passing" while doing nothing.
|
||||
# CF orphans accumulated to 152/200 (~76% of the zone quota
|
||||
# gone) before a manual `dig`-driven audit caught it. Anything
|
||||
# that runs as a janitor and reports green while idle is
|
||||
# indistinguishable from "the janitor is healthy" — so we now
|
||||
# treat schedule (and any future workflow_run/push triggers)
|
||||
# as a hard-fail when secrets are missing.
|
||||
#
|
||||
# - schedule / workflow_run / push → exit 1 (red CI run
|
||||
# surfaces the misconfiguration the next tick)
|
||||
# - workflow_dispatch → exit 0 with a warning
|
||||
# (an operator ran this ad-hoc; they already accepted the
|
||||
# state of the repo and want the workflow to short-circuit
|
||||
# so they can rerun after fixing the secret)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ZONE_ID CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry (intentional):
|
||||
# - Scheduled runs: github.event.inputs.dry_run is empty →
|
||||
# defaults to "false" below → script runs with --execute
|
||||
# (the whole point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default is true (line 38)
|
||||
# so an ad-hoc operator-triggered run is dry-run by default;
|
||||
# they have to flip the toggle to actually delete.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) is the second
|
||||
# line of defense regardless of mode.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-orphans.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-orphans.sh --execute
|
||||
fi
|
||||
@@ -1,128 +0,0 @@
|
||||
name: Sweep stale Cloudflare Tunnels
|
||||
|
||||
# Ported from .github/workflows/sweep-cf-tunnels.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for Cloudflare Tunnels whose backing tenant no longer
|
||||
# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
|
||||
# records); same justification, different CF resource.
|
||||
#
|
||||
# Why this exists separately from sweep-cf-orphans:
|
||||
# - DNS records live on the zone (`/zones/<id>/dns_records`).
|
||||
# - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
|
||||
# - Different CF API surface, different scopes; the existing CF
|
||||
# token might not have `account:cloudflare_tunnel:edit`. Splitting
|
||||
# the workflows keeps each one's secret-presence gate independent
|
||||
# so neither silent-skips when the other's secret is missing.
|
||||
# - Cleaner blast radius — operators can disable one without the
|
||||
# other if a regression surfaces.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
|
||||
# the DNS sweep's 50% because tenant-shaped tunnels are mostly
|
||||
# orphans by design) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
|
||||
# janitors don't issue parallel CF API bursts at the same minute.
|
||||
- cron: '45 * * * *'
|
||||
# Don't let two sweeps race the same account.
|
||||
concurrency:
|
||||
group: sweep-cf-tunnels
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF tunnels
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 30 min cap. Was 5 min on the theory that the only thing that
|
||||
# could take >5min is a CF-API hang — but on 2026-05-02 a backlog
|
||||
# of 672 stale tunnels accumulated (large staging E2E run + delayed
|
||||
# sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
|
||||
# ~7-8min to drain. The 5-min cap killed the run mid-sweep
|
||||
# (cancelled at 424/672, see run 25248788312); a manual rerun
|
||||
# finished the remainder fine.
|
||||
#
|
||||
# The fix is two-part: parallelize the delete loop (8-way xargs in
|
||||
# the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
|
||||
# cap so a one-off backlog doesn't trip a hangs-detector that
|
||||
# turned out to be a real-job-too-slow detector. With 8-way
|
||||
# parallelism, 600+ tunnels drains in ~60s; 30 min is generous
|
||||
# headroom for actual hangs to still surface (and is in line with
|
||||
# the sweep-cf-orphans companion job).
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# (hardened 2026-04-28 after the silent-no-op incident: the
|
||||
# janitor reported green while doing nothing because secrets
|
||||
# were unset, masking a 152/200 zone-record leak). Same
|
||||
# principle applies here:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ACCOUNT_ID CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh --execute
|
||||
fi
|
||||
@@ -1,243 +0,0 @@
|
||||
name: Sweep stale e2e-* orgs (staging)
|
||||
|
||||
# Ported from .github/workflows/sweep-stale-e2e-orgs.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for staging tenants left behind when E2E cleanup didn't run:
|
||||
# CI cancellations, runner crashes, transient AWS errors mid-cascade,
|
||||
# bash trap missed (signal 9), etc. Without this loop, every failed
|
||||
# teardown leaks an EC2 + DNS + DB row until manual ops cleanup —
|
||||
# 2026-04-23 staging hit the 64 vCPU AWS quota from ~27 such orphans.
|
||||
#
|
||||
# Why not rely on per-test-run teardown:
|
||||
# - Per-run teardown is best-effort by definition. Any process death
|
||||
# after the test starts but before the trap fires leaves debris.
|
||||
# - GH Actions cancellation kills the runner without grace period.
|
||||
# The workflow's `if: always()` step usually catches this, but it
|
||||
# too can fail (CP transient 5xx, runner network issue at the
|
||||
# wrong moment).
|
||||
# - Even when teardown runs, the CP cascade is best-effort in places
|
||||
# (cascadeTerminateWorkspaces logs+continues; DNS deletion same).
|
||||
# - This sweep is the catch-all that converges staging back to clean
|
||||
# regardless of which specific path leaked.
|
||||
#
|
||||
# The PROPER fix is making CP cleanup transactional + verify-after-
|
||||
# terminate (filed separately as cleanup-correctness work). This
|
||||
# workflow is the safety net that catches everything else AND any
|
||||
# future leak source we haven't yet identified.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
|
||||
# create to teardown — canary is ~8 min, full SaaS ~25 min). The
|
||||
# previous hourly + 120-min stale threshold meant a leaked tenant
|
||||
# could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
|
||||
# leak. Tightening the cadence + threshold reduces the worst-case
|
||||
# leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
|
||||
# threshold) without risk of catching in-progress runs (the longest
|
||||
# e2e run is the 25-min canary, well under the 30-min threshold).
|
||||
# See molecule-controlplane#420 for the leak-class accounting that
|
||||
# motivated this tightening.
|
||||
- cron: '*/15 * * * *'
|
||||
# Don't let two sweeps fight. Cron + workflow_dispatch could overlap
|
||||
# on a manual trigger; queue rather than parallel-delete.
|
||||
concurrency:
|
||||
group: sweep-stale-e2e-orgs
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep e2e orgs
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
# Refuse to delete more than this many orgs in one tick. If the
|
||||
# CP DB is briefly empty (or the admin endpoint goes weird and
|
||||
# returns no created_at), every e2e- org would look stale.
|
||||
# Bailing protects against runaway nukes.
|
||||
SAFETY_CAP: 50
|
||||
|
||||
steps:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Identify stale e2e orgs
|
||||
id: identify
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch into a file so the python step reads it via stdin —
|
||||
# cleaner than embedding $(curl ...) into a heredoc.
|
||||
curl -sS --fail-with-body --max-time 30 \
|
||||
"$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
> orgs.json
|
||||
|
||||
# Filter:
|
||||
# 1. slug starts with one of the ephemeral test prefixes:
|
||||
# - 'e2e-' — covers e2e-canary-, e2e-canvas-*, etc.
|
||||
# - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
|
||||
# missing this prefix left two such tenants
|
||||
# orphaned 8h on staging (2026-05-03), then
|
||||
# hard-failed redeploy-tenants-on-staging
|
||||
# and broke the staging→main auto-promote
|
||||
# chain. Kept in sync with the EPHEMERAL_PREFIX_RE
|
||||
# regex in redeploy-tenants-on-staging.yml.
|
||||
# 2. created_at is older than MAX_AGE_MINUTES ago
|
||||
# Output one slug per line to a file the next step reads.
|
||||
python3 > stale_slugs.txt <<'PY'
|
||||
import json, os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
# SSOT for this list lives in the controlplane Go code:
|
||||
# molecule-controlplane/internal/slugs/ephemeral.go
|
||||
# (var EphemeralPrefixes). The redeploy-fleet auto-rollout
|
||||
# also reads from there to SKIP these slugs — without that
|
||||
# filter, fleet redeploy SSM-failed in-flight E2E tenants
|
||||
# whose containers were still booting, breaking the test
|
||||
# that just spun them up (molecule-controlplane#493).
|
||||
# Update both files together.
|
||||
EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
|
||||
with open("orgs.json") as f:
|
||||
data = json.load(f)
|
||||
max_age = int(os.environ["MAX_AGE_MINUTES"])
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
|
||||
for o in data.get("orgs", []):
|
||||
slug = o.get("slug", "")
|
||||
if not slug.startswith(EPHEMERAL_PREFIXES):
|
||||
continue
|
||||
created = o.get("created_at")
|
||||
if not created:
|
||||
# Defensively skip rows without created_at — better
|
||||
# to leave one orphan than nuke a brand-new row
|
||||
# whose timestamp didn't render.
|
||||
continue
|
||||
# Python 3.11+ handles RFC3339 with Z directly via
|
||||
# fromisoformat; older runners need the trailing Z swap.
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
if created_dt < cutoff:
|
||||
print(slug)
|
||||
PY
|
||||
|
||||
count=$(wc -l < stale_slugs.txt | tr -d ' ')
|
||||
echo "Found $count stale e2e org(s) older than ${MAX_AGE_MINUTES}m"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo "First 20:"
|
||||
head -20 stale_slugs.txt | sed 's/^/ /'
|
||||
fi
|
||||
echo "count=$count" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Safety gate
|
||||
if: steps.identify.outputs.count != '0'
|
||||
run: |
|
||||
count="${{ steps.identify.outputs.count }}"
|
||||
if [ "$count" -gt "$SAFETY_CAP" ]; then
|
||||
echo "::error::Refusing to delete $count orgs in one sweep (cap=$SAFETY_CAP). Investigate manually — this usually means the CP admin API returned no created_at or returned a degraded result. Re-run with workflow_dispatch + max_age_minutes if intentional."
|
||||
exit 1
|
||||
fi
|
||||
echo "Within safety cap ($count ≤ $SAFETY_CAP) ✓"
|
||||
|
||||
- name: Delete stale orgs
|
||||
if: steps.identify.outputs.count != '0' && env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set -uo pipefail
|
||||
deleted=0
|
||||
failed=0
|
||||
while IFS= read -r slug; do
|
||||
[ -z "$slug" ] && continue
|
||||
# The DELETE handler requires {"confirm": "<slug>"} matching
|
||||
# the URL slug — fat-finger guard. Idempotent: re-issuing
|
||||
# picks up via org_purges.last_step.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/del_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/del_code
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to runner log.
|
||||
http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
||||
deleted=$((deleted+1))
|
||||
echo " deleted: $slug"
|
||||
else
|
||||
failed=$((failed+1))
|
||||
echo " FAILED ($http_code): $slug — $(cat /tmp/del_resp 2>/dev/null | head -c 200)"
|
||||
fi
|
||||
done < stale_slugs.txt
|
||||
echo ""
|
||||
echo "Sweep summary: deleted=$deleted failed=$failed"
|
||||
# Don't fail the workflow on per-org delete errors — the
|
||||
# sweeper is best-effort. Next hourly tick re-attempts. We
|
||||
# only fail loud at the safety-cap gate above.
|
||||
|
||||
- name: Sweep orphan tunnels
|
||||
# Stale-org cleanup deletes the org (which cascades to tunnel
|
||||
# delete inside the CP). But when that cascade fails partway —
|
||||
# CP transient 5xx after the org row is deleted but before the
|
||||
# CF tunnel delete completes — the tunnel persists with no
|
||||
# matching org row. The reconciler in internal/sweep flags this
|
||||
# as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
|
||||
#
|
||||
# `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
|
||||
# reaper. Calling it here at the end of every sweep tick
|
||||
# converges the staging CF account to clean even when CP
|
||||
# cascades half-fail.
|
||||
#
|
||||
# PR #492 made the underlying DeleteTunnel actually check
|
||||
# status — pre-fix it silent-succeeded on CF code 1022
|
||||
# ("active connections"), so this step would have been a no-op
|
||||
# against stuck connectors. Post-fix the cleanup invokes
|
||||
# CleanupTunnelConnections + retry, which actually clears the
|
||||
# 1022 case. (#2987)
|
||||
#
|
||||
# Best-effort. Failure here doesn't fail the workflow — next
|
||||
# tick re-attempts. Errors flow to step output for ops review.
|
||||
if: env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set +e
|
||||
curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
|
||||
set -e
|
||||
http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
|
||||
body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
|
||||
if [ "$http_code" = "200" ]; then
|
||||
count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
|
||||
failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
|
||||
echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
|
||||
else
|
||||
echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
|
||||
fi
|
||||
|
||||
- name: Dry-run summary
|
||||
if: env.DRY_RUN == 'true'
|
||||
run: |
|
||||
echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."
|
||||
@@ -1,65 +0,0 @@
|
||||
name: Ops Scripts Tests
|
||||
|
||||
# Ported from .github/workflows/test-ops-scripts.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` trigger (no Gitea merge queue).
|
||||
# - on.paths references .gitea/workflows/test-ops-scripts.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Runs the unittest suite for scripts/ on every PR + push that touches
|
||||
# anything under scripts/. Kept separate from the main CI so a script-only
|
||||
# change doesn't trigger the heavier Go/Canvas/Python pipelines.
|
||||
#
|
||||
# Discovery layout: tests sit alongside the code they test (see
|
||||
# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
|
||||
# test_build_runtime_package.py for the rewriter coverage). The job
|
||||
# below runs `unittest discover` TWICE — once from `scripts/`, once
|
||||
# from `scripts/ops/` — because neither dir has an `__init__.py`, so
|
||||
# a single discover from `scripts/` doesn't recurse into the ops
|
||||
# subdir. Two passes is simpler than retrofitting namespace packages.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Ops scripts (unittest)
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Run scripts/ unittests (build_runtime_package, ...)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
@@ -28,7 +28,7 @@ import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
CANONICAL_FILE = Path(".gitea/workflows/secret-scan.yml")
|
||||
CANONICAL_FILE = Path(".github/workflows/secret-scan.yml")
|
||||
|
||||
# Public consumer mirrors. Each entry is (label, raw_url) — raw_url
|
||||
# points at the file's RAW content on the consumer's default branch
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
name: auto-tag-runtime
|
||||
|
||||
# Auto-tag runtime releases on every merge to main that touches workspace/.
|
||||
# This is the entry point of the runtime CD chain:
|
||||
#
|
||||
# merge PR → auto-tag-runtime (this) → publish-runtime → cascade → template
|
||||
# image rebuilds → repull on hosts.
|
||||
#
|
||||
# Default bump is patch. Override via PR label `release:minor` or
|
||||
# `release:major` BEFORE merging — the label is read off the merged PR
|
||||
# associated with the push commit.
|
||||
#
|
||||
# Skips when:
|
||||
# - The push isn't to main (other branches don't auto-release).
|
||||
# - The merge commit message contains `[skip-release]` (escape hatch
|
||||
# for cleanup PRs that touch workspace/ but shouldn't ship).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "workspace/**"
|
||||
- "scripts/build_runtime_package.py"
|
||||
- ".github/workflows/auto-tag-runtime.yml"
|
||||
- ".github/workflows/publish-runtime.yml"
|
||||
|
||||
permissions:
|
||||
contents: write # to push the new tag
|
||||
pull-requests: read # to read labels off the merged PR
|
||||
|
||||
concurrency:
|
||||
# Serialize tag bumps so two near-simultaneous merges can't both think
|
||||
# they're 0.1.6 and race to push the same tag.
|
||||
group: auto-tag-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
tag:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0 # need full tag history for `git describe` / sort
|
||||
|
||||
- name: Skip when commit asks
|
||||
id: skip
|
||||
run: |
|
||||
MSG=$(git log -1 --format=%B "${{ github.sha }}")
|
||||
if echo "$MSG" | grep -qiE '\[skip-release\]|\[no-release\]'; then
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
echo "Commit message contains [skip-release] — no tag will be created."
|
||||
else
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Determine bump kind from PR label
|
||||
id: bump
|
||||
if: steps.skip.outputs.skip != 'true'
|
||||
env:
|
||||
# Gitea-shape token (act_runner forwards GITHUB_TOKEN as a
|
||||
# short-lived per-run secret with read access to this repo).
|
||||
# We hit `/api/v1/repos/.../pulls?state=closed` directly
|
||||
# because `gh pr list` calls Gitea's GraphQL endpoint, which
|
||||
# returns HTTP 405 (issue #75 / post-#66 sweep).
|
||||
GITEA_TOKEN: ${{ github.token }}
|
||||
REPO: ${{ github.repository }}
|
||||
GITEA_API_URL: ${{ github.server_url }}/api/v1
|
||||
PUSH_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
# Find the merged PR whose merge_commit_sha matches this push.
|
||||
# Gitea's `/repos/{owner}/{repo}/pulls?state=closed` returns
|
||||
# PRs sorted newest-first; we paginate up to 50 and jq-filter
|
||||
# on `merge_commit_sha == PUSH_SHA`. Bounded — auto-tag fires
|
||||
# per push to main, so the matching PR is always among the
|
||||
# most recent closures. 50 is comfortably more than the
|
||||
# ~10-20 staging→main promotes that close in any reasonable
|
||||
# window.
|
||||
set -euo pipefail
|
||||
PRS_JSON=$(curl --fail-with-body -sS \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Accept: application/json" \
|
||||
"${GITEA_API_URL}/repos/${REPO}/pulls?state=closed&sort=newest&limit=50" \
|
||||
2>/dev/null || echo "[]")
|
||||
PR=$(printf '%s' "$PRS_JSON" \
|
||||
| jq -c --arg sha "$PUSH_SHA" \
|
||||
'[.[] | select(.merged_at != null and .merge_commit_sha == $sha)] | .[0] // empty')
|
||||
if [ -z "$PR" ] || [ "$PR" = "null" ]; then
|
||||
echo "No merged PR found for ${PUSH_SHA} — defaulting to patch bump."
|
||||
echo "kind=patch" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Gitea returns labels under `.labels[].name`, same shape as
|
||||
# GitHub's REST. The previous `gh pr list --json number,labels`
|
||||
# output was identical; jq filter unchanged.
|
||||
LABELS=$(printf '%s' "$PR" | jq -r '.labels[]?.name // empty')
|
||||
if echo "$LABELS" | grep -qx 'release:major'; then
|
||||
echo "kind=major" >> "$GITHUB_OUTPUT"
|
||||
elif echo "$LABELS" | grep -qx 'release:minor'; then
|
||||
echo "kind=minor" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "kind=patch" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Compute next version from latest runtime-v* tag
|
||||
id: version
|
||||
if: steps.skip.outputs.skip != 'true'
|
||||
run: |
|
||||
# Find the highest runtime-vX.Y.Z tag. `sort -V` handles semver
|
||||
# ordering; `grep` filters to the right tag prefix.
|
||||
LATEST=$(git tag --list 'runtime-v*' | sort -V | tail -1)
|
||||
if [ -z "$LATEST" ]; then
|
||||
# No prior tag — start the runtime line at 0.1.0.
|
||||
CURRENT="0.0.0"
|
||||
else
|
||||
CURRENT="${LATEST#runtime-v}"
|
||||
fi
|
||||
MAJOR=$(echo "$CURRENT" | cut -d. -f1)
|
||||
MINOR=$(echo "$CURRENT" | cut -d. -f2)
|
||||
PATCH=$(echo "$CURRENT" | cut -d. -f3)
|
||||
case "${{ steps.bump.outputs.kind }}" in
|
||||
major) MAJOR=$((MAJOR+1)); MINOR=0; PATCH=0;;
|
||||
minor) MINOR=$((MINOR+1)); PATCH=0;;
|
||||
patch) PATCH=$((PATCH+1));;
|
||||
esac
|
||||
NEW="$MAJOR.$MINOR.$PATCH"
|
||||
echo "current=$CURRENT" >> "$GITHUB_OUTPUT"
|
||||
echo "new=$NEW" >> "$GITHUB_OUTPUT"
|
||||
echo "Bumping runtime $CURRENT → $NEW (${{ steps.bump.outputs.kind }})"
|
||||
|
||||
- name: Push new tag
|
||||
if: steps.skip.outputs.skip != 'true'
|
||||
run: |
|
||||
NEW_TAG="runtime-v${{ steps.version.outputs.new }}"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git tag -a "$NEW_TAG" -m "runtime $NEW_TAG (auto-bump from ${{ steps.bump.outputs.kind }})"
|
||||
git push origin "$NEW_TAG"
|
||||
echo "Pushed $NEW_TAG — publish-runtime workflow will fire on the tag."
|
||||
@@ -0,0 +1,111 @@
|
||||
name: branch-protection drift check
|
||||
|
||||
# Catches out-of-band edits to branch protection (UI clicks, manual gh
|
||||
# api PATCH from a one-off ops session) by comparing live state against
|
||||
# tools/branch-protection/apply.sh's desired state every day. Fails the
|
||||
# workflow when they drift; the failure is the signal.
|
||||
#
|
||||
# When it fails: re-run apply.sh to put the live state back to the
|
||||
# script's intent, OR update apply.sh to encode the new intent and
|
||||
# commit. Either way the script is the source of truth.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 14:00 UTC daily. Off-hours for most teams; gives a fresh signal
|
||||
# at the start of every working day.
|
||||
- cron: '0 14 * * *'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- 'tools/branch-protection/**'
|
||||
- '.github/workflows/**'
|
||||
- '.github/workflows/branch-protection-drift.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
drift:
|
||||
name: Branch protection drift
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Token strategy by trigger:
|
||||
#
|
||||
# - schedule (daily canary): hard-fail when the admin token is
|
||||
# missing. This is the *only* trigger where silent soft-skip is
|
||||
# dangerous — a missing secret on the cron run means the drift
|
||||
# gate has effectively disappeared with no human in the loop to
|
||||
# notice. Per feedback_schedule_vs_dispatch_secrets_hardening.md
|
||||
# the rule is "schedule/automated triggers must hard-fail".
|
||||
#
|
||||
# - pull_request (touching tools/branch-protection/**): soft-skip
|
||||
# with a prominent warning. A PR cannot retroactively drift the
|
||||
# live state — drift happens *between* PRs (UI clicks, manual
|
||||
# gh api PATCH) and is the schedule's job to catch. The PR-time
|
||||
# gate would only catch typos in apply.sh, which the apply.sh
|
||||
# *_payload unit tests catch better. A human is reviewing the
|
||||
# PR and will see the warning in the workflow log.
|
||||
#
|
||||
# - workflow_dispatch (operator one-off): soft-skip with warning,
|
||||
# so an operator can run a diagnostic without configuring the
|
||||
# secret first.
|
||||
- name: Verify admin token present (hard-fail on schedule only)
|
||||
env:
|
||||
GH_TOKEN_FOR_ADMIN_API: ${{ secrets.GH_TOKEN_FOR_ADMIN_API }}
|
||||
run: |
|
||||
if [[ -n "$GH_TOKEN_FOR_ADMIN_API" ]]; then
|
||||
echo "GH_TOKEN_FOR_ADMIN_API present — drift_check will run with admin scope."
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
echo "::error::GH_TOKEN_FOR_ADMIN_API secret missing on the daily canary." >&2
|
||||
echo "" >&2
|
||||
echo "The schedule run is the SoT for branch-protection drift detection." >&2
|
||||
echo "Without admin scope it silently passes, hiding any out-of-band edits." >&2
|
||||
echo "Set GH_TOKEN_FOR_ADMIN_API at Settings → Secrets and variables → Actions." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::GH_TOKEN_FOR_ADMIN_API secret missing — drift_check will be SKIPPED."
|
||||
echo "::warning::PR drift checks need repo-admin scope to read /branches/:b/protection."
|
||||
echo "::warning::This is non-fatal: the daily schedule run is the canonical drift gate."
|
||||
echo "SKIP_DRIFT_CHECK=1" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Run drift check
|
||||
if: env.SKIP_DRIFT_CHECK != '1'
|
||||
env:
|
||||
# Repo-admin scope, needed for /branches/:b/protection.
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN_FOR_ADMIN_API }}
|
||||
run: bash tools/branch-protection/drift_check.sh
|
||||
|
||||
# Self-test the parity script before running it on the real
|
||||
# workflows — pins the script's classification logic against
|
||||
# synthetic safe/unsafe/missing/unsafe-mix/matrix fixtures so a
|
||||
# regression in the script can't false-pass on the production
|
||||
# workflow audit. Cheap (~0.5s); always runs.
|
||||
- name: Self-test check-name parity script
|
||||
run: bash tools/branch-protection/test_check_name_parity.sh
|
||||
|
||||
# Check-name parity gate (#144 / saved memory
|
||||
# feedback_branch_protection_check_name_parity).
|
||||
#
|
||||
# drift_check.sh asserts the live branch protection matches what
|
||||
# apply.sh would set; check_name_parity.sh closes the orthogonal
|
||||
# gap: it asserts every required check name in apply.sh maps to a
|
||||
# workflow job whose "always emits this status" shape is intact.
|
||||
#
|
||||
# The two checks fail in different scenarios:
|
||||
#
|
||||
# - drift_check fails → live state was rewritten out-of-band
|
||||
# (UI click, manual PATCH).
|
||||
# - check_name_parity fails → an apply.sh required name has no
|
||||
# emitter, OR the emitting workflow has a top-level paths:
|
||||
# filter without per-step if-gates (the silent-block shape).
|
||||
#
|
||||
# Cheap (~1s); runs without the admin token because it only reads
|
||||
# apply.sh + .github/workflows/ from the checkout.
|
||||
- name: Run check-name parity gate
|
||||
run: bash tools/branch-protection/check_name_parity.sh
|
||||
@@ -0,0 +1,48 @@
|
||||
name: Check merge_group trigger on required workflows
|
||||
|
||||
# Pre-merge guard against the deadlock pattern where a workflow whose
|
||||
# check is in `required_status_checks` lacks a `merge_group:` trigger.
|
||||
# Without it, GitHub merge queue stalls forever in AWAITING_CHECKS
|
||||
# because the required check can't fire on `gh-readonly-queue/...` refs.
|
||||
#
|
||||
# This workflow:
|
||||
# 1. Lists required status checks on the branch protection rule for `staging`
|
||||
# 2. For each required check, finds the workflow that produces it (by job
|
||||
# name match)
|
||||
# 3. Fails if any such workflow lacks `merge_group:` in its triggers
|
||||
#
|
||||
# Reasoning for staging-only: main has its own CI gating model (PR review),
|
||||
# but staging is what the merge queue runs on, so it's the trigger that
|
||||
# matters.
|
||||
#
|
||||
# Gitea stub: Gitea has no merge queue feature and no `merge_group:`
|
||||
# event type. The linter would find no `merge_group:` triggers to verify
|
||||
# (they don't exist on Gitea), so the lint is vacuously satisfied.
|
||||
# Converting to a no-op stub keeps the workflow+job name stable for any
|
||||
# commit-status context consumers while eliminating the `gh api` call
|
||||
# that fails against Gitea's REST surface (#75 / PR-D).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/**.yml'
|
||||
- '.github/workflows/**.yaml'
|
||||
push:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- '.github/workflows/**.yml'
|
||||
- '.github/workflows/**.yaml'
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Required workflows have merge_group trigger
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Gitea no-op (merge queue not applicable)
|
||||
run: |
|
||||
echo "Gitea Actions — merge queue not supported; no-op."
|
||||
echo "On GitHub this workflow lints that required-check workflows declare"
|
||||
echo "merge_group: triggers to prevent queue deadlock. On Gitea that"
|
||||
echo "constraint is inapplicable — all workflows pass vacuously."
|
||||
@@ -0,0 +1,136 @@
|
||||
name: CodeQL
|
||||
|
||||
# Stub workflow — CodeQL Action is structurally incompatible with Gitea
|
||||
# Actions (post-2026-05-06 SCM migration off GitHub).
|
||||
#
|
||||
# Why this is a stub, not a real CodeQL run:
|
||||
#
|
||||
# 1. github/codeql-action/init@v4 hits api.github.com endpoints
|
||||
# (CodeQL CLI bundle download + query-pack registry + telemetry)
|
||||
# that Gitea 1.22.x does NOT proxy. The act_runner has
|
||||
# GITHUB_SERVER_URL=https://git.moleculesai.app correctly set
|
||||
# (per saved memory feedback_act_runner_github_server_url and
|
||||
# /config.yaml on the operator host), but the Gitea API surface
|
||||
# simply does not implement the codeql-action bundle endpoints.
|
||||
# Observed in run 1d/3101 (2026-05-07): "::error::404 page not
|
||||
# found" inside the Initialize CodeQL step, before any analysis.
|
||||
#
|
||||
# 2. PR #35 attempted to mark `continue-on-error: true` at the JOB
|
||||
# level (correct YAML structure). Gitea 1.22.6 does NOT propagate
|
||||
# job-level continue-on-error to the commit-status API — every
|
||||
# matrix leg still posts `failure` to the status surface, which
|
||||
# keeps OVERALL=failure on every push to main + staging and
|
||||
# blocks visual auto-promote signals (#156).
|
||||
#
|
||||
# 3. Hongming policy decision (2026-05-07, task #156): CodeQL is
|
||||
# ADVISORY, not blocking, on Gitea Actions. We do not block PR
|
||||
# merge or staging→main promotion on CodeQL findings until we
|
||||
# have a Gitea-compatible static-analysis pipeline.
|
||||
#
|
||||
# What this stub preserves:
|
||||
#
|
||||
# - Workflow name `CodeQL` (referenced by auto-promote-staging.yml
|
||||
# line 67 as a workflow_run gate — must stay stable).
|
||||
# - Job name template `Analyze (${{ matrix.language }})` and the
|
||||
# 3-leg matrix (go, javascript-typescript, python). Branch
|
||||
# protection / required-check parity (#144) keys on these
|
||||
# exact context names.
|
||||
# - merge_group + push + pull_request + schedule triggers, so the
|
||||
# merge-queue check name still resolves (per saved memory
|
||||
# feedback_branch_protection_check_name_parity).
|
||||
#
|
||||
# Re-enabling real analysis (future work):
|
||||
#
|
||||
# - Option A: self-hosted Semgrep / OpenGrep via a custom action
|
||||
# that doesn't hit api.github.com. Tracked behind #156 follow-up.
|
||||
# - Option B: Sonatype Nexus IQ or similar, called from a step
|
||||
# that uses the Gitea-issued token only.
|
||||
# - Option C: re-host this workflow on a small GitHub mirror used
|
||||
# ONLY for SAST (push-mirrored from Gitea). Acceptable trade-off
|
||||
# if/when payment is restored on a non-suspended GitHub org —
|
||||
# but per saved memory feedback_no_single_source_of_truth, we
|
||||
# should design for multi-vendor backup, not GitHub-only SAST.
|
||||
#
|
||||
# Until one of those lands, this stub keeps commit-status green so
|
||||
# the auto-promote chain isn't permanently red on a tool we cannot
|
||||
# actually run.
|
||||
#
|
||||
# Security policy: ADVISORY. We accept the residual risk of un-scanned
|
||||
# pushes during this window. Compensating controls in place:
|
||||
# - secret-scan.yml runs on every push (active, blocks on hits)
|
||||
# - block-internal-paths.yml blocks forbidden file paths
|
||||
# - lint-curl-status-capture.yml catches one specific class of bug
|
||||
# - branch-protection-drift.yml + the merge_group required-checks
|
||||
# parity keep the gate surface stable
|
||||
# These are not equivalent to CodeQL coverage. Status of the
|
||||
# replacement plan is tracked in #156.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# Required so the matrix legs emit a real result on the queued
|
||||
# commit instead of a false-green when merge queue is enabled.
|
||||
# Per saved memory feedback_branch_protection_check_name_parity:
|
||||
# path-filtered / matrix workflows MUST emit the protected name
|
||||
# via a job that always runs.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
schedule:
|
||||
# Weekly heartbeat. Cheap on a stub (the no-op job is ~5s) but
|
||||
# keeps the workflow visible in Gitea's Actions UI so the next
|
||||
# operator notices it's a stub instead of a missing surface.
|
||||
- cron: '30 1 * * 0'
|
||||
|
||||
# Workflow-level concurrency: only one stub run per branch/PR at a
|
||||
# time. cancel-in-progress: false because a quick follow-up push
|
||||
# shouldn't kill an in-flight run — even though the stub is fast,
|
||||
# the contract should match a real CodeQL run for when we re-enable.
|
||||
concurrency:
|
||||
group: codeql-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
# No security-events: write — we don't call the upload API anyway,
|
||||
# GHAS isn't on Gitea.
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
# Job NAME shape is load-bearing — auto-promote-staging.yml +
|
||||
# branch protection both key on `Analyze (${{ matrix.language }})`.
|
||||
# Do NOT rename without coordinating both surfaces.
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [go, javascript-typescript, python]
|
||||
|
||||
steps:
|
||||
# Single-step stub: log the policy decision + emit success.
|
||||
# Exit 0 explicitly so the commit-status API records `success`
|
||||
# for each of the three matrix legs.
|
||||
- name: CodeQL stub (advisory, non-blocking on Gitea)
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cat <<EOF
|
||||
CodeQL is currently ADVISORY on Gitea Actions (post-2026-05-06).
|
||||
Language matrix leg: ${{ matrix.language }}
|
||||
Reason: github/codeql-action/init@v4 calls api.github.com
|
||||
bundle endpoints that Gitea 1.22.x does not implement.
|
||||
Observed: "::error::404 page not found" in the Init
|
||||
CodeQL step on every prior run.
|
||||
Policy: per Hongming decision 2026-05-07 (#156), CodeQL is
|
||||
non-blocking until a Gitea-compatible SAST pipeline
|
||||
lands. See workflow file header for replacement
|
||||
options + compensating controls.
|
||||
Status: emitting success so auto-promote isn't permanently
|
||||
red on a tool we cannot actually run today.
|
||||
EOF
|
||||
echo "::notice::CodeQL ${{ matrix.language }} — advisory stub, success."
|
||||
@@ -0,0 +1,63 @@
|
||||
name: pr-guards
|
||||
|
||||
# PR-time guards. Today the only guard is "disable auto-merge when a
|
||||
# new commit is pushed after auto-merge was enabled" — added 2026-04-27
|
||||
# after PR #2174 auto-merged with only its first commit because the
|
||||
# second commit was pushed after the merge queue had locked the PR's
|
||||
# SHA.
|
||||
#
|
||||
# Why this is inlined (not delegated to molecule-ci's reusable
|
||||
# workflow): the reusable workflow uses `gh pr merge --disable-auto`,
|
||||
# which calls GitHub's GraphQL API. Gitea has no GraphQL endpoint and
|
||||
# returns HTTP 405 on /api/graphql, so the job failed on every Gitea
|
||||
# PR push since the 2026-05-06 migration. Gitea also has no `--auto`
|
||||
# merge primitive that this job could be acting on, so the right
|
||||
# behaviour on Gitea is "no-op + green status" — not a 405.
|
||||
#
|
||||
# Inlining (vs. an `if:` on the `uses:` line) keeps the job ALWAYS
|
||||
# running, which matters for branch protection: required-check names
|
||||
# need a job that emits SUCCESS terminal state, not SKIPPED. See
|
||||
# `feedback_branch_protection_check_name_parity` and `feedback_pr_merge_safety_guards`.
|
||||
#
|
||||
# Issue #88 item 1.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [synchronize]
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
disable-auto-merge-on-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Detect Gitea Actions. act_runner sets GITEA_ACTIONS=true in the
|
||||
# step env on every job. Belt-and-suspenders: also check the repo
|
||||
# url's host, which is independent of any runner-side env config
|
||||
# (covers a future Gitea host where the env var is forgotten).
|
||||
- name: Detect runner host
|
||||
id: host
|
||||
run: |
|
||||
if [[ "${GITEA_ACTIONS:-}" == "true" ]] || [[ "${{ github.server_url }}" == *moleculesai.app* ]] || [[ "${{ github.event.repository.html_url }}" == *moleculesai.app* ]]; then
|
||||
echo "is_gitea=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::Gitea Actions detected — auto-merge gating is not applicable here (Gitea has no --auto merge primitive). Job will no-op."
|
||||
else
|
||||
echo "is_gitea=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Disable auto-merge (GitHub only)
|
||||
if: steps.host.outputs.is_gitea != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
NEW_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -eu
|
||||
gh pr merge "$PR" --disable-auto -R "$REPO" || true
|
||||
gh pr comment "$PR" -R "$REPO" --body "🔒 Auto-merge disabled — new commit (\`${NEW_SHA:0:7}\`) pushed after auto-merge was enabled. The merge queue locks SHAs at entry, so subsequent pushes can race. Verify the new commit and re-enable with \`gh pr merge --auto\`."
|
||||
|
||||
- name: Gitea no-op
|
||||
if: steps.host.outputs.is_gitea == 'true'
|
||||
run: echo "Gitea Actions — auto-merge gating not applicable; no-op (job intentionally green so branch protection's required-check name lands SUCCESS)."
|
||||
@@ -0,0 +1,85 @@
|
||||
name: promote-latest
|
||||
|
||||
# Manually retag ghcr.io/molecule-ai/platform:staging-<sha> → :latest
|
||||
# (and the same for the tenant image). Use this to:
|
||||
#
|
||||
# 1. Promote a :staging-<sha> to prod before the canary fleet is live
|
||||
# (one-off during the initial rollout).
|
||||
# 2. Roll back :latest to a prior known-good digest after a bad
|
||||
# promotion slipped past canary (use scripts/rollback-latest.sh
|
||||
# for a local / emergency path; this workflow is for scheduled
|
||||
# or from-browser promotions).
|
||||
#
|
||||
# Running this workflow needs no extra secrets — GitHub's default
|
||||
# GITHUB_TOKEN has write:packages for repo-owned GHCR images, which
|
||||
# is all we need for a remote retag via `crane tag`.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'Short sha to promote (e.g. 4c1d56e). Must match an existing :staging-<sha> tag.'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
promote:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: imjasonh/setup-crane@6da1ae018866400525525ce74ff892880c099987 # v0.5
|
||||
|
||||
- name: GHCR login
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" \
|
||||
| crane auth login ghcr.io -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Retag platform image
|
||||
run: |
|
||||
set -eu
|
||||
SRC="${IMAGE_NAME}:staging-${{ inputs.sha }}"
|
||||
if ! crane digest "$SRC" >/dev/null 2>&1; then
|
||||
echo "::error::$SRC not found in registry — double-check the sha."
|
||||
exit 1
|
||||
fi
|
||||
EXPECTED=$(crane digest "$SRC")
|
||||
crane tag "$SRC" latest
|
||||
ACTUAL=$(crane digest "${IMAGE_NAME}:latest")
|
||||
if [ "$ACTUAL" != "$EXPECTED" ]; then
|
||||
echo "::error::retag digest mismatch (expected $EXPECTED, got $ACTUAL)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK ${IMAGE_NAME}:latest → $ACTUAL"
|
||||
|
||||
- name: Retag tenant image
|
||||
run: |
|
||||
set -eu
|
||||
SRC="${TENANT_IMAGE_NAME}:staging-${{ inputs.sha }}"
|
||||
if ! crane digest "$SRC" >/dev/null 2>&1; then
|
||||
echo "::error::$SRC not found — tenant image may not have built for this sha."
|
||||
exit 1
|
||||
fi
|
||||
EXPECTED=$(crane digest "$SRC")
|
||||
crane tag "$SRC" latest
|
||||
ACTUAL=$(crane digest "${TENANT_IMAGE_NAME}:latest")
|
||||
if [ "$ACTUAL" != "$EXPECTED" ]; then
|
||||
echo "::error::tenant retag digest mismatch"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK ${TENANT_IMAGE_NAME}:latest → $ACTUAL"
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## :latest promoted to staging-${{ inputs.sha }}"
|
||||
echo
|
||||
echo "Both platform + tenant images retagged. Prod tenants"
|
||||
echo "will auto-pull within their 5-min update cycle."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,446 @@
|
||||
name: publish-runtime
|
||||
|
||||
# DEPRECATED on Gitea Actions — this file is kept for reference only.
|
||||
# Gitea Actions reads .gitea/workflows/, not .github/workflows/.
|
||||
# The canonical version is now: .gitea/workflows/publish-runtime.yml
|
||||
# That port:
|
||||
# - Drops OIDC trusted publisher (Gitea has no environments/OIDC)
|
||||
# - Uses PYPI_TOKEN secret instead of gh-action-pypi-publish
|
||||
# - Uses ${GITHUB_REF#refs/tags/} instead of github.ref_name
|
||||
# - Drops staging branch trigger (staging branch does not exist)
|
||||
# - Drops merge_group trigger (Gitea has no merge queue)
|
||||
#
|
||||
# Publishes molecule-ai-workspace-runtime to PyPI from monorepo workspace/.
|
||||
# Monorepo workspace/ is the only source-of-truth for runtime code; this
|
||||
# workflow is the bridge from monorepo edits to the PyPI artifact that
|
||||
# the 8 workspace-template-* repos depend on.
|
||||
#
|
||||
# Triggered by:
|
||||
# - Pushing a tag matching `runtime-vX.Y.Z` (the version is derived from
|
||||
# the tag — `runtime-v0.1.6` publishes `0.1.6`).
|
||||
# - Manual workflow_dispatch with an explicit `version` input (useful for
|
||||
# dev/test releases without tagging the repo).
|
||||
# - Auto: any push to `staging` that touches `workspace/**`. The version
|
||||
# is derived by querying PyPI for the current latest and bumping the
|
||||
# patch component. This closes the human-in-loop gap that caused the
|
||||
# 2026-04-27 RuntimeCapabilities ImportError outage — adapter symbol
|
||||
# additions in workspace/adapters/base.py used to require an operator
|
||||
# to remember to publish; now the merge itself triggers the publish.
|
||||
#
|
||||
# The workflow:
|
||||
# 1. Runs scripts/build_runtime_package.py to copy workspace/ →
|
||||
# build/molecule_runtime/ with imports rewritten (`a2a_client` →
|
||||
# `molecule_runtime.a2a_client`).
|
||||
# 2. Builds wheel + sdist with `python -m build`.
|
||||
# 3. Publishes to PyPI via the PyPA Trusted Publisher action (OIDC).
|
||||
# No static API token is stored — PyPI verifies the workflow's
|
||||
# OIDC claim against the trusted-publisher config registered for
|
||||
# molecule-ai-workspace-runtime (molecule-ai/molecule-core,
|
||||
# publish-runtime.yml, environment pypi-publish).
|
||||
#
|
||||
# After publish: the 8 template repos pick up the new version on their
|
||||
# next image rebuild (their requirements.txt pin
|
||||
# `molecule-ai-workspace-runtime>=0.1.0`, so any new release is eligible).
|
||||
# To force-pull immediately, bump the pin in each template repo's
|
||||
# requirements.txt and merge — that triggers their own publish-image.yml.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "runtime-v*"
|
||||
branches:
|
||||
- staging
|
||||
paths:
|
||||
# Auto-publish when staging gets changes that affect what gets
|
||||
# published. Path filter ONLY applies to branch pushes — tag pushes
|
||||
# still fire regardless.
|
||||
#
|
||||
# workspace/** is the source-of-truth for runtime code.
|
||||
# scripts/build_runtime_package.py is the build script — changes to
|
||||
# it (e.g. a fix to the import rewriter or a manifest emit) directly
|
||||
# affect what ships in the wheel even if no workspace/ file changes.
|
||||
# The 2026-04-27 lib/ subpackage incident missed an auto-publish for
|
||||
# exactly this reason — PR #2174 only changed scripts/ and the
|
||||
# operator had to remember a manual dispatch.
|
||||
- "workspace/**"
|
||||
- "scripts/build_runtime_package.py"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Version to publish (e.g. 0.1.6). Required for manual dispatch."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Serialize publishes so two staging merges landing seconds apart don't
|
||||
# both compute "latest+1" and race on PyPI upload. The second one waits.
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
environment: pypi-publish
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # PyPI Trusted Publisher (OIDC) — no PYPI_TOKEN needed
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
|
||||
- name: Derive version (tag, manual input, or PyPI auto-bump)
|
||||
id: version
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
VERSION="${{ inputs.version }}"
|
||||
elif echo "$GITHUB_REF_NAME" | grep -q "^runtime-v"; then
|
||||
# Tag is `runtime-vX.Y.Z` — strip the prefix.
|
||||
VERSION="${GITHUB_REF_NAME#runtime-v}"
|
||||
else
|
||||
# Auto-publish from staging push. Query PyPI for the current
|
||||
# latest and bump the patch component. concurrency: group above
|
||||
# serializes parallel staging merges so we don't race on the
|
||||
# bump. If PyPI is unreachable, fail loud — better to skip a
|
||||
# publish than to overwrite an existing version.
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
|
||||
fi
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
|
||||
echo "::error::version $VERSION does not match PEP 440"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Publishing molecule-ai-workspace-runtime $VERSION"
|
||||
|
||||
- name: Install build tooling
|
||||
run: pip install build twine
|
||||
|
||||
- name: Build package from workspace/
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "${{ steps.version.outputs.version }}" \
|
||||
--out "${{ runner.temp }}/runtime-build"
|
||||
|
||||
- name: Build wheel + sdist
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: python -m build
|
||||
|
||||
- name: Capture wheel SHA256 for cascade content-verification
|
||||
# Recorded BEFORE upload so the cascade probe can verify the
|
||||
# bytes Fastly serves under the new version's URL match what
|
||||
# we built. Closes a hole left by #2197: that probe verified
|
||||
# pip can resolve the version (catches propagation lag) but
|
||||
# not that the wheel content matches (would silently pass a
|
||||
# Fastly stale-content scenario where the new version's URL
|
||||
# serves an old wheel binary).
|
||||
id: wheel_hash
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
set -eu
|
||||
WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
|
||||
if [ -z "$WHEEL" ]; then
|
||||
echo "::error::No .whl in dist/ — `python -m build` must have failed silently"
|
||||
exit 1
|
||||
fi
|
||||
HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
|
||||
echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
|
||||
echo "Local wheel SHA256 (pre-upload): ${HASH}"
|
||||
echo "Wheel filename: $(basename "$WHEEL")"
|
||||
|
||||
- name: Verify package contents (sanity)
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
# Smoke logic lives in scripts/wheel_smoke.py so the same gate runs
|
||||
# at both PR-time (runtime-prbuild-compat.yml) and publish-time
|
||||
# (here). Splitting the smoke across two heredocs let them drift
|
||||
# apart historically — one script keeps them locked.
|
||||
run: |
|
||||
python -m twine check dist/*
|
||||
python -m venv /tmp/smoke
|
||||
/tmp/smoke/bin/pip install --quiet dist/*.whl
|
||||
/tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
|
||||
- name: Publish to PyPI (Trusted Publisher / OIDC)
|
||||
# PyPI side is configured: project molecule-ai-workspace-runtime →
|
||||
# publisher molecule-ai/molecule-core, workflow publish-runtime.yml,
|
||||
# environment pypi-publish. The action mints a short-lived OIDC
|
||||
# token and exchanges it for a PyPI upload credential — no static
|
||||
# API token in this repo's secrets.
|
||||
uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
|
||||
with:
|
||||
packages-dir: ${{ runner.temp }}/runtime-build/dist/
|
||||
|
||||
cascade:
|
||||
# After PyPI accepts the upload, fan out a repository_dispatch to each
|
||||
# template repo so they rebuild their image against the new runtime.
|
||||
# Each template's `runtime-published.yml` receiver picks up the event,
|
||||
# pulls the new PyPI version (their requirements.txt pin is `>=`), and
|
||||
# republishes ghcr.io/molecule-ai/workspace-template-<runtime>:latest.
|
||||
#
|
||||
# Soft-fail per repo: if one template's dispatch fails (perms missing,
|
||||
# repo archived, etc.) we still try the others and surface the failures
|
||||
# in the workflow summary instead of aborting the whole cascade.
|
||||
needs: publish
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Wait for PyPI to propagate the new version
|
||||
# PyPI accepts the upload, then takes a few seconds to make the
|
||||
# new version visible across all THREE surfaces pip touches:
|
||||
# 1. /pypi/<pkg>/<ver>/json — metadata endpoint
|
||||
# 2. /simple/<pkg>/ — pip's primary download index
|
||||
# 3. files.pythonhosted.org — CDN-fronted wheel binary
|
||||
# Each has its own cache. The previous check polled only (1)
|
||||
# and would let the cascade fire while (2) or (3) still served
|
||||
# the previous version, so downstream `pip install` resolved
|
||||
# to the old wheel. Docker layer cache then locked that stale
|
||||
# resolution in for subsequent rebuilds (the cache trap that
|
||||
# bit us five times in one night).
|
||||
#
|
||||
# Two-stage probe per poll:
|
||||
# (a) `pip install --no-cache-dir PACKAGE==VERSION` — succeeds
|
||||
# only when the version is resolvable. Catches surface (1)
|
||||
# and (2) propagation lag.
|
||||
# (b) `pip download` of the same wheel + SHA256 compare against
|
||||
# the just-built dist's hash. Catches surface (3) lag AND
|
||||
# Fastly serving stale content under the new version's URL
|
||||
# (a separate Fastly-corruption mode that pip-install alone
|
||||
# can't see, since pip install resolves+unpacks against
|
||||
# whatever bytes Fastly returns and never inspects them).
|
||||
# Both must pass before the cascade fans out.
|
||||
#
|
||||
# The venv is reused across polls; only `pip install`/`pip
|
||||
# download` run in the loop, with --force-reinstall +
|
||||
# --no-cache-dir so the previous poll's cached state doesn't
|
||||
# mask propagation lag.
|
||||
env:
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
python -m venv /tmp/propagation-probe
|
||||
PROBE=/tmp/propagation-probe/bin
|
||||
$PROBE/pip install --upgrade --quiet pip
|
||||
# Poll budget: 30 attempts × (~3-5s pip install + ~3s pip
|
||||
# download + 4s sleep) ≈ 5-6 min wall on a slow GH runner.
|
||||
# Generous vs PyPI's typical few-seconds propagation;
|
||||
# failures past this are signal of a real PyPI / Fastly
|
||||
# issue, not just lag.
|
||||
for i in $(seq 1 30); do
|
||||
# Stage (a): can pip resolve and install the version?
|
||||
if $PROBE/pip install \
|
||||
--quiet \
|
||||
--no-cache-dir \
|
||||
--force-reinstall \
|
||||
--no-deps \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1; then
|
||||
INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
|
||||
| awk -F': ' '/^Version:/{print $2}')
|
||||
if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
|
||||
# Stage (b): does Fastly serve the bytes we uploaded?
|
||||
# `pip download` writes the actual .whl file to disk so
|
||||
# we can sha256sum it (vs `pip install` which unpacks
|
||||
# and discards).
|
||||
rm -rf /tmp/probe-dl
|
||||
mkdir -p /tmp/probe-dl
|
||||
if $PROBE/pip download \
|
||||
--quiet \
|
||||
--no-cache-dir \
|
||||
--no-deps \
|
||||
--dest /tmp/probe-dl \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1; then
|
||||
WHEEL=$(ls /tmp/probe-dl/*.whl 2>/dev/null | head -1)
|
||||
if [ -n "$WHEEL" ]; then
|
||||
ACTUAL=$(sha256sum "$WHEEL" | awk '{print $1}')
|
||||
if [ "$ACTUAL" = "$EXPECTED_SHA256" ]; then
|
||||
echo "::notice::✓ pip resolves AND wheel content matches after ${i} poll(s) (sha256=${EXPECTED_SHA256})"
|
||||
exit 0
|
||||
fi
|
||||
# Hash mismatch: PyPI accepted our upload but Fastly
|
||||
# is serving different bytes under the version's URL.
|
||||
# Most often this is propagation lag of the BINARY
|
||||
# surface — the version is resolvable but the wheel
|
||||
# cache hasn't caught up. Retry.
|
||||
echo "::warning::poll ${i}: wheel content mismatch (got ${ACTUAL:0:12}…, want ${EXPECTED_SHA256:0:12}…) — Fastly likely still serving stale binary, retrying"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
sleep 4
|
||||
done
|
||||
echo "::error::pip never resolved molecule-ai-workspace-runtime==${RUNTIME_VERSION} with matching wheel content within ~5 min."
|
||||
echo "::error::Expected wheel SHA256: ${EXPECTED_SHA256}"
|
||||
echo "::error::Refusing to fan out cascade against stale or corrupt PyPI surfaces."
|
||||
exit 1
|
||||
|
||||
- name: Fan out via push to .runtime-version
|
||||
env:
|
||||
# Gitea PAT with write:repository scope on the 8 cascade-active
|
||||
# template repos. Used here for `git push` (NOT for an API
|
||||
# dispatch — Gitea 1.22.6 has no repository_dispatch endpoint;
|
||||
# empirically verified across 6 candidate paths in molecule-
|
||||
# core#20 issuecomment-913). The push trips each template's
|
||||
# existing `on: push: branches: [main]` trigger on
|
||||
# publish-image.yml, which then reads the updated
|
||||
# .runtime-version via its resolve-version job.
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
run: |
|
||||
set +e # don't abort on a single repo failure — collect them all
|
||||
|
||||
# Soft-skip on workflow_dispatch when the token is missing
|
||||
# (operator ad-hoc test); hard-fail on push so unattended
|
||||
# publishes can't silently skip the cascade. Same shape as
|
||||
# the original v1, intentional split per the schedule-vs-
|
||||
# dispatch hardening 2026-04-28.
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
|
||||
echo "::warning::set it at Settings → Secrets and Variables → Actions, then rerun. Templates will stay on the prior runtime version until either this token is set or each template is rebuilt manually."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
|
||||
echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version until this token is restored and a republish dispatches the cascade."
|
||||
echo "::error::set it at Settings → Secrets and Variables → Actions; then re-trigger publish-runtime via workflow_dispatch."
|
||||
exit 1
|
||||
fi
|
||||
VERSION="$RUNTIME_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::publish job did not expose a version output — cascade cannot fan out"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# All 9 workspace templates declared in manifest.json. The list
|
||||
# MUST stay aligned with manifest.json's workspace_templates —
|
||||
# cascade-list-drift-gate.yml enforces this in CI per the
|
||||
# codex-stuck-on-stale-runtime invariant from PR #2556.
|
||||
# Long-term goal: derive this list from manifest.json so it
|
||||
# can't drift even on a manifest edit (RFC #388 Phase-1).
|
||||
#
|
||||
# Per-template publish-image.yml presence is checked at
|
||||
# cascade-time below: codex doesn't ship one today, so the
|
||||
# cascade soft-skips it with an informational message rather
|
||||
# than dropping it from this list (which would re-introduce
|
||||
# the drift the gate exists to catch).
|
||||
GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
|
||||
TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
|
||||
FAILED=""
|
||||
SKIPPED=""
|
||||
|
||||
# Configure git identity once. The persona owning DISPATCH_TOKEN
|
||||
# is the same identity that authored this commit on each
|
||||
# template; using a generic "publish-runtime cascade" co-author
|
||||
# trailer in the message keeps the audit trail honest about the
|
||||
# workflow-driven origin.
|
||||
git config --global user.name "publish-runtime cascade"
|
||||
git config --global user.email "publish-runtime@moleculesai.app"
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
for tpl in $TEMPLATES; do
|
||||
REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
|
||||
CLONE="$WORKDIR/$tpl"
|
||||
|
||||
# Pre-check: skip templates without a publish-image.yml.
|
||||
# The cascade's job is to trip the template's on-push
|
||||
# rebuild — if there's no rebuild workflow, pushing a
|
||||
# .runtime-version commit is just noise on the target
|
||||
# repo. Use the Gitea contents API (no clone required for
|
||||
# the probe). 200 = present; 404 = absent.
|
||||
HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $DISPATCH_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "↷ $tpl has no publish-image.yml — soft-skip (informational; manifest still tracks it)"
|
||||
SKIPPED="$SKIPPED $tpl"
|
||||
continue
|
||||
fi
|
||||
if [ "$HTTP" != "200" ]; then
|
||||
echo "::warning::$tpl publish-image.yml probe returned HTTP $HTTP — proceeding anyway, push will surface the real failure if any"
|
||||
fi
|
||||
|
||||
# Use a per-template attempt loop so a transient race (e.g.
|
||||
# human pushing to the same template at the same instant)
|
||||
# doesn't lose the cascade. Bounded retries (3) — beyond
|
||||
# that we surface the failure and let the operator retry.
|
||||
attempt=0
|
||||
success=false
|
||||
while [ $attempt -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
rm -rf "$CLONE"
|
||||
if ! git clone --depth=1 \
|
||||
"https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
|
||||
"$CLONE" >/tmp/clone.log 2>&1; then
|
||||
echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
|
||||
cd "$CLONE"
|
||||
echo "$VERSION" > .runtime-version
|
||||
|
||||
# Idempotency guard: if the file already matches, this
|
||||
# publish is a re-run for a version already cascaded.
|
||||
# Don't push a no-op commit (would spuriously re-trip the
|
||||
# template's on-push and rebuild for nothing).
|
||||
if git diff --quiet -- .runtime-version; then
|
||||
echo "✓ $tpl already at $VERSION — no commit needed (idempotent)"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
git add .runtime-version
|
||||
git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
|
||||
-m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
|
||||
>/dev/null
|
||||
|
||||
if git push origin HEAD:main >/tmp/push.log 2>&1; then
|
||||
echo "✓ $tpl pushed $VERSION on attempt $attempt"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
# Likely a non-fast-forward — pull-rebase and retry.
|
||||
# Don't force-push: that would silently overwrite a racing
|
||||
# human/cascade commit.
|
||||
echo "::warning::push $tpl attempt $attempt failed, pull-rebasing: $(tail -n3 /tmp/push.log)"
|
||||
git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
|
||||
cd - >/dev/null
|
||||
done
|
||||
|
||||
if [ "$success" != "true" ]; then
|
||||
FAILED="$FAILED $tpl"
|
||||
fi
|
||||
done
|
||||
rm -rf "$WORKDIR"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "::error::Cascade incomplete after 3 retries each. Failed templates:$FAILED"
|
||||
echo "::error::PyPI publish succeeded; failed templates lag the new version. Re-run this workflow_dispatch with the same version to retry only the laggers (idempotent — already-cascaded templates skip)."
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$SKIPPED" ]; then
|
||||
echo "Cascade complete: pinned $VERSION on cascade-active templates. Soft-skipped (no publish-image.yml):$SKIPPED"
|
||||
else
|
||||
echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
|
||||
fi
|
||||
@@ -0,0 +1,214 @@
|
||||
name: Secret scan
|
||||
|
||||
# Hard CI gate. Refuses any PR / push whose diff additions contain a
|
||||
# recognisable credential. Defense-in-depth for the #2090-class incident
|
||||
# (2026-04-24): GitHub's hosted Copilot Coding Agent leaked a ghs_*
|
||||
# installation token into tenant-proxy/package.json via `npm init`
|
||||
# slurping the URL from a token-embedded origin remote. We can't fix
|
||||
# upstream's clone hygiene, so we gate here.
|
||||
#
|
||||
# Also the canonical reusable workflow for the rest of the org. Other
|
||||
# Molecule-AI repos enroll with a single 3-line workflow:
|
||||
#
|
||||
# jobs:
|
||||
# secret-scan:
|
||||
# uses: molecule-ai/molecule-core/.github/workflows/secret-scan.yml@staging
|
||||
#
|
||||
# Pin to @staging not @main — staging is the active default branch,
|
||||
# main lags via the staging-promotion workflow. Updates ride along
|
||||
# automatically on the next consumer workflow run.
|
||||
#
|
||||
# Same regex set as the runtime's bundled pre-commit hook
|
||||
# (molecule-ai-workspace-runtime: molecule_runtime/scripts/pre-commit-checks.sh).
|
||||
# Keep the two sides aligned when adding patterns.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
# Required for GitHub merge queue: the queue's pre-merge CI run on
|
||||
# `gh-readonly-queue/...` refs needs this check to fire so the queue
|
||||
# gets a real result instead of stalling forever AWAITING_CHECKS.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
# Reusable workflow entry point for other Molecule-AI repos.
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan diff for credential-shaped strings
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base may be many commits behind
|
||||
# HEAD and absent from the shallow clone. Fetch it explicitly.
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
# For merge_group events the queue's pre-merge ref is a commit on
|
||||
# `gh-readonly-queue/...` whose parent is the queue's base_sha.
|
||||
# That parent isn't part of the queue branch's shallow clone, so
|
||||
# we fetch it explicitly. Without this the diff falls through to
|
||||
# "no BASE → scan entire tree" mode and false-positives on legit
|
||||
# test fixtures (e.g. canvas/src/lib/validation/__tests__/secret-formats.test.ts).
|
||||
- name: Fetch merge_group base SHA (merge_group events only)
|
||||
if: github.event_name == 'merge_group'
|
||||
run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
|
||||
|
||||
- name: Refuse if credential-shaped strings appear in diff additions
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# merge_group has its own base_sha/head_sha; pull_request has
|
||||
# pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
|
||||
MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Pattern set covers GitHub family (the actual #2090 vector),
|
||||
# Anthropic / OpenAI / Slack / AWS. Anchored on prefixes with low
|
||||
# false-positive rates against agent-generated content. Mirror of
|
||||
# molecule-ai-workspace-runtime/molecule_runtime/scripts/pre-commit-checks.sh
|
||||
# — keep aligned.
|
||||
SECRET_PATTERNS=(
|
||||
'ghp_[A-Za-z0-9]{36,}' # GitHub PAT (classic)
|
||||
'ghs_[A-Za-z0-9]{36,}' # GitHub App installation token
|
||||
'gho_[A-Za-z0-9]{36,}' # GitHub OAuth user-to-server
|
||||
'ghu_[A-Za-z0-9]{36,}' # GitHub OAuth user
|
||||
'ghr_[A-Za-z0-9]{36,}' # GitHub OAuth refresh
|
||||
'github_pat_[A-Za-z0-9_]{82,}' # GitHub fine-grained PAT
|
||||
'sk-ant-[A-Za-z0-9_-]{40,}' # Anthropic API key
|
||||
'sk-proj-[A-Za-z0-9_-]{40,}' # OpenAI project key
|
||||
'sk-svcacct-[A-Za-z0-9_-]{40,}' # OpenAI service-account key
|
||||
'sk-cp-[A-Za-z0-9_-]{60,}' # MiniMax API key (F1088 vector — caught only after the fact)
|
||||
'xox[baprs]-[A-Za-z0-9-]{20,}' # Slack tokens
|
||||
'AKIA[0-9A-Z]{16}' # AWS access key ID
|
||||
'ASIA[0-9A-Z]{16}' # AWS STS temp access key ID
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
merge_group)
|
||||
BASE="$MG_BASE_SHA"
|
||||
HEAD="$MG_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check the
|
||||
# entire tree as added content. Slower, but correct on first
|
||||
# push.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
DIFF_RANGE=""
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
DIFF_RANGE="$BASE $HEAD"
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Self-exclude: this workflow file legitimately contains the
|
||||
# pattern strings as regex literals. Without an exclude it would
|
||||
# block its own merge.
|
||||
SELF=".github/workflows/secret-scan.yml"
|
||||
|
||||
OFFENDING=""
|
||||
# `while IFS= read -r` (not `for f in $CHANGED`) so filenames
|
||||
# containing whitespace don't word-split silently — a path
|
||||
# with a space would otherwise produce two iterations on
|
||||
# tokens that aren't real filenames, breaking the
|
||||
# self-exclude + diff lookup.
|
||||
while IFS= read -r f; do
|
||||
[ -z "$f" ] && continue
|
||||
[ "$f" = "$SELF" ] && continue
|
||||
if [ -n "$DIFF_RANGE" ]; then
|
||||
ADDED=$(git diff --no-color --unified=0 "$BASE" "$HEAD" -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
|
||||
else
|
||||
# No diff range (new branch first push) — scan the full file
|
||||
# contents as if every line were new.
|
||||
ADDED=$(cat "$f" 2>/dev/null || true)
|
||||
fi
|
||||
[ -z "$ADDED" ] && continue
|
||||
for pattern in "${SECRET_PATTERNS[@]}"; do
|
||||
if echo "$ADDED" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${f} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done <<< "$CHANGED"
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Credential-shaped strings detected in diff additions:"
|
||||
# `printf '%b' "$OFFENDING"` interprets backslash escapes
|
||||
# (the literal `\n` we appended above becomes a newline)
|
||||
# WITHOUT treating OFFENDING as a format string. Plain
|
||||
# `printf "$OFFENDING"` is a format-string sink: a filename
|
||||
# containing `%` would be interpreted as a conversion
|
||||
# specifier, corrupting the error message (or printing
|
||||
# `%(missing)` artifacts).
|
||||
printf '%b' "$OFFENDING"
|
||||
echo ""
|
||||
echo "The actual matched values are NOT echoed here, deliberately —"
|
||||
echo "round-tripping a leaked credential into CI logs widens the blast"
|
||||
echo "radius (logs are searchable + retained)."
|
||||
echo ""
|
||||
echo "Recovery:"
|
||||
echo " 1. Remove the secret from the file. Replace with an env var"
|
||||
echo " reference (e.g. \${{ secrets.GITHUB_TOKEN }} in workflows,"
|
||||
echo " process.env.X in code)."
|
||||
echo " 2. If the credential was already pushed (this PR's commit"
|
||||
echo " history reaches a public ref), treat it as compromised —"
|
||||
echo " ROTATE it immediately, do not just remove it. The token"
|
||||
echo " remains valid in git history forever and may be in any"
|
||||
echo " log/cache that consumed this branch."
|
||||
echo " 3. Force-push the cleaned commit (or stack a revert) and"
|
||||
echo " re-run CI."
|
||||
echo ""
|
||||
echo "If the match is a false positive (test fixture, docs example,"
|
||||
echo "or this workflow's own regex literals): use a clearly-fake"
|
||||
echo "placeholder like ghs_EXAMPLE_DO_NOT_USE that doesn't satisfy"
|
||||
echo "the length suffix, OR add the file path to the SELF exclude"
|
||||
echo "list in this workflow with a short reason."
|
||||
echo ""
|
||||
echo "Mirror of the regex set lives in the runtime's bundled"
|
||||
echo "pre-commit hook (molecule-ai-workspace-runtime:"
|
||||
echo "molecule_runtime/scripts/pre-commit-checks.sh) — keep aligned."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No credential-shaped strings in this change."
|
||||
@@ -105,11 +105,11 @@ export function ConfirmDialog({
|
||||
// (e.g. parents with transform, filter, will-change that break position:fixed).
|
||||
return createPortal(
|
||||
<div className="fixed inset-0 z-[9999] flex items-center justify-center">
|
||||
{/* Backdrop — interactive dismiss area; accessible name for screen readers (WCAG 4.1.2) */}
|
||||
{/* Backdrop */}
|
||||
<div
|
||||
className="absolute inset-0 bg-black/60 backdrop-blur-sm cursor-pointer"
|
||||
aria-label="Dismiss dialog"
|
||||
onClick={onCancel}
|
||||
aria-label="Dismiss dialog"
|
||||
/>
|
||||
|
||||
{/* Dialog — role="dialog" + aria-modal prevent interaction with background */}
|
||||
|
||||
@@ -90,11 +90,7 @@ export function ConsoleModal({ workspaceId, workspaceName, open, onClose }: Prop
|
||||
|
||||
return createPortal(
|
||||
<div className="fixed inset-0 z-[9999] flex items-center justify-center">
|
||||
<div
|
||||
className="absolute inset-0 bg-black/70 backdrop-blur-sm cursor-pointer"
|
||||
onClick={onClose}
|
||||
aria-label="Close terminal"
|
||||
/>
|
||||
<div aria-hidden="true" className="absolute inset-0 bg-black/70 backdrop-blur-sm" onClick={onClose} />
|
||||
<div
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
|
||||
@@ -81,11 +81,7 @@ export function DeleteCascadeConfirmDialog({
|
||||
return createPortal(
|
||||
<div className="fixed inset-0 z-[9999] flex items-center justify-center">
|
||||
{/* Backdrop */}
|
||||
<div
|
||||
className="absolute inset-0 bg-black/60 backdrop-blur-sm cursor-pointer"
|
||||
onClick={onCancel}
|
||||
aria-label="Dismiss dialog"
|
||||
/>
|
||||
<div aria-hidden="true" className="absolute inset-0 bg-black/60 backdrop-blur-sm" onClick={onCancel} />
|
||||
|
||||
{/* Dialog */}
|
||||
<div
|
||||
|
||||
@@ -4,14 +4,9 @@
|
||||
*
|
||||
* Covers: renders nothing when no approvals, polls /approvals/pending,
|
||||
* shows approval cards, approve/deny decisions, toast notifications.
|
||||
*
|
||||
* Note: does NOT mock @/lib/api — uses vi.spyOn on the real module.
|
||||
* vi.restoreAllMocks() is omitted from afterEach so queued mock values
|
||||
* (set up via mockResolvedValueOnce in beforeEach) are preserved for the
|
||||
* component's useEffect to consume.
|
||||
*/
|
||||
import React from "react";
|
||||
import { render, screen, fireEvent, cleanup, act } from "@testing-library/react";
|
||||
import { render, screen, fireEvent, cleanup, waitFor, act } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import { ApprovalBanner } from "../ApprovalBanner";
|
||||
import { showToast } from "@/components/Toaster";
|
||||
@@ -41,10 +36,6 @@ const pendingApproval = (id = "a1", workspaceId = "ws-1"): {
|
||||
created_at: "2026-05-10T10:00:00Z",
|
||||
});
|
||||
|
||||
// Shared spy reference so individual tests can call mockGet.mockRestore()
|
||||
// without needing to pass it through beforeEach → it scope chain.
|
||||
let mockGet: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe("ApprovalBanner — empty state", () => {
|
||||
@@ -54,12 +45,13 @@ describe("ApprovalBanner — empty state", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("renders nothing when there are no pending approvals", async () => {
|
||||
render(<ApprovalBanner />);
|
||||
// Wait for the initial useEffect + api.get to resolve
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
expect(screen.queryByRole("alert")).toBeNull();
|
||||
});
|
||||
@@ -75,14 +67,14 @@ describe("ApprovalBanner — empty state", () => {
|
||||
describe("ApprovalBanner — renders approval cards", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
mockGet = vi.spyOn(api, "get").mockResolvedValueOnce([
|
||||
vi.spyOn(api, "get").mockResolvedValueOnce([
|
||||
pendingApproval("a1"),
|
||||
pendingApproval("a2", "ws-2"),
|
||||
]);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
@@ -91,7 +83,6 @@ describe("ApprovalBanner — renders approval cards", () => {
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
const alerts = screen.getAllByRole("alert");
|
||||
expect(alerts).toHaveLength(2);
|
||||
mockGet.mockRestore();
|
||||
});
|
||||
|
||||
it("displays the workspace name and action text", async () => {
|
||||
@@ -146,6 +137,7 @@ describe("ApprovalBanner — decisions", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("calls POST /workspaces/:id/approvals/:id/decide on Approve click", async () => {
|
||||
@@ -196,7 +188,7 @@ describe("ApprovalBanner — decisions", () => {
|
||||
});
|
||||
|
||||
it("shows an error toast when POST fails", async () => {
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error("Network error"));
|
||||
vi.spyOn(api, "post").mockRejectedValue(new Error("Network error"));
|
||||
render(<ApprovalBanner />);
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
fireEvent.click(screen.getAllByRole("button", { name: /approve/i })[0]);
|
||||
@@ -208,8 +200,7 @@ describe("ApprovalBanner — decisions", () => {
|
||||
});
|
||||
|
||||
it("keeps the card visible when the POST fails", async () => {
|
||||
// Use mockRejectedValueOnce on the same spy as beforeEach (don't call spyOn again)
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error("Network error"));
|
||||
vi.spyOn(api, "post").mockRejectedValue(new Error("Network error"));
|
||||
render(<ApprovalBanner />);
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
fireEvent.click(screen.getAllByRole("button", { name: /approve/i })[0]);
|
||||
@@ -225,7 +216,7 @@ describe("ApprovalBanner — handles empty list from server", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
@@ -234,4 +225,4 @@ describe("ApprovalBanner — handles empty list from server", () => {
|
||||
await act(async () => { await vi.runOnlyPendingTimersAsync(); });
|
||||
expect(screen.queryByRole("alert")).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -49,18 +49,17 @@ function createDragOverEvent() {
|
||||
|
||||
describe("BundleDropZone — render", () => {
|
||||
it("renders a hidden file input with correct accept and aria-label", () => {
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
expect(input).toBeTruthy();
|
||||
expect(input.getAttribute("type")).toBe("file");
|
||||
expect(input.getAttribute("accept")).toBe(".bundle.json");
|
||||
expect(input.getAttribute("id")).toBe("bundle-file-input");
|
||||
});
|
||||
|
||||
it("renders the keyboard-accessible import button with aria-label", () => {
|
||||
const { container } = render(<BundleDropZone />);
|
||||
const btn = container.querySelector('button[aria-label="Import bundle file"]') as HTMLButtonElement;
|
||||
expect(btn).not.toBeNull();
|
||||
render(<BundleDropZone />);
|
||||
const btn = screen.getByRole("button", { name: /import bundle/i });
|
||||
expect(btn).toBeTruthy();
|
||||
expect(btn.getAttribute("aria-controls")).toBe("bundle-file-input");
|
||||
});
|
||||
});
|
||||
@@ -74,7 +73,7 @@ describe("BundleDropZone — drag state", () => {
|
||||
|
||||
it("shows the drop overlay when a file is dragged over", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
// Overlay should not be visible initially
|
||||
expect(screen.queryByText("Drop Bundle to Import")).toBeNull();
|
||||
|
||||
@@ -93,7 +92,7 @@ describe("BundleDropZone — drag state", () => {
|
||||
});
|
||||
|
||||
it("hides the drop overlay when not dragging", () => {
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
// By default (no drag), the overlay should not be visible
|
||||
expect(screen.queryByText("Drop Bundle to Import")).toBeNull();
|
||||
});
|
||||
@@ -101,15 +100,14 @@ describe("BundleDropZone — drag state", () => {
|
||||
|
||||
describe("BundleDropZone — keyboard file input (WCAG 2.1.1)", () => {
|
||||
it("triggers the hidden file input when the import button is clicked", () => {
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
// Both the hidden file input and the button have aria-label="Import bundle file".
|
||||
// Use the file input's id to select it uniquely.
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
expect(input).toBeTruthy();
|
||||
expect(input.getAttribute("type")).toBe("file");
|
||||
const clickSpy = vi.spyOn(input, "click");
|
||||
const btn = container.querySelector('button[aria-label="Import bundle file"]') as HTMLButtonElement;
|
||||
fireEvent.click(btn);
|
||||
fireEvent.click(screen.getByRole("button", { name: /import bundle/i }));
|
||||
expect(clickSpy).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -121,7 +119,7 @@ describe("BundleDropZone — keyboard file input (WCAG 2.1.1)", () => {
|
||||
status: "online",
|
||||
});
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("My Bundle");
|
||||
@@ -153,7 +151,7 @@ describe("BundleDropZone — import success", () => {
|
||||
status: "online",
|
||||
});
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("Success Workspace");
|
||||
@@ -165,14 +163,14 @@ describe("BundleDropZone — import success", () => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
|
||||
// Success toast should be visible — scope to container for DOM isolation
|
||||
expect(container.textContent).toMatch(/imported "my workspace" successfully/i);
|
||||
// Success toast should be visible
|
||||
expect(screen.getByText(/imported "my workspace" successfully/i)).toBeTruthy();
|
||||
|
||||
// Toast auto-clears after 4000ms
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(5000);
|
||||
});
|
||||
expect(container.querySelector('[role="status"]')).toBeNull();
|
||||
expect(screen.queryByRole("status")).toBeNull();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
@@ -184,7 +182,7 @@ describe("BundleDropZone — import success", () => {
|
||||
status: "online",
|
||||
});
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("Timed Workspace");
|
||||
@@ -195,12 +193,12 @@ describe("BundleDropZone — import success", () => {
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
expect(container.textContent).toMatch(/timed workspace/i);
|
||||
expect(screen.queryByText(/timed workspace/i)).toBeTruthy();
|
||||
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(4500);
|
||||
});
|
||||
expect(container.textContent).not.toMatch(/timed workspace/i);
|
||||
expect(screen.queryByText(/timed workspace/i)).toBeNull();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
@@ -210,7 +208,7 @@ describe("BundleDropZone — import error", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error("Import failed: 500 Internal Server Error"));
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("Failed Workspace");
|
||||
@@ -222,13 +220,13 @@ describe("BundleDropZone — import error", () => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
|
||||
expect(container.textContent).toMatch(/import failed: 500 internal server error/i);
|
||||
expect(screen.getByText(/import failed: 500 internal server error/i)).toBeTruthy();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("shows error when file is not a .bundle.json", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = new File(["{}"], "readme.txt", { type: "text/plain" });
|
||||
@@ -240,12 +238,12 @@ describe("BundleDropZone — import error", () => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
|
||||
expect(container.textContent).toMatch(/only .bundle.json files are accepted/i);
|
||||
expect(screen.getByText(/only .bundle.json files are accepted/i)).toBeTruthy();
|
||||
// Error clears after 3000ms
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(3500);
|
||||
});
|
||||
expect(container.textContent).not.toMatch(/only .bundle.json/i);
|
||||
expect(screen.queryByText(/only .bundle.json/i)).toBeNull();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
@@ -253,7 +251,7 @@ describe("BundleDropZone — import error", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.mocked(api.post).mockRejectedValueOnce(new Error("Network error"));
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("Error Workspace");
|
||||
@@ -264,12 +262,12 @@ describe("BundleDropZone — import error", () => {
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
expect(container.textContent).toMatch(/network error/i);
|
||||
expect(screen.queryByText(/network error/i)).toBeTruthy();
|
||||
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(5000);
|
||||
});
|
||||
expect(container.textContent).not.toMatch(/network error/i);
|
||||
expect(screen.queryByText(/network error/i)).toBeNull();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
});
|
||||
@@ -281,7 +279,7 @@ describe("BundleDropZone — importing state", () => {
|
||||
const pending = new Promise((r) => { resolve = r; });
|
||||
vi.mocked(api.post).mockReturnValueOnce(pending as unknown as ReturnType<typeof api.post>);
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("Pending Workspace");
|
||||
@@ -294,10 +292,8 @@ describe("BundleDropZone — importing state", () => {
|
||||
vi.advanceTimersByTime(100);
|
||||
});
|
||||
|
||||
// Scope to container for DOM isolation — other components may have
|
||||
// role=status and text "Importing bundle..." in the shared jsdom env.
|
||||
expect(container.textContent).toMatch(/importing bundle/i);
|
||||
expect(container.querySelector('[role="status"]')).toBeTruthy();
|
||||
expect(screen.getByText("Importing bundle...")).toBeTruthy();
|
||||
expect(screen.getByRole("status")).toBeTruthy();
|
||||
|
||||
await act(async () => {
|
||||
vi.advanceTimersByTime(500);
|
||||
@@ -315,7 +311,7 @@ describe("BundleDropZone — file input reset", () => {
|
||||
status: "online",
|
||||
});
|
||||
|
||||
const { container } = render(<BundleDropZone />);
|
||||
render(<BundleDropZone />);
|
||||
const input = document.getElementById("bundle-file-input") as HTMLInputElement;
|
||||
|
||||
const file = makeBundle("Reset Test");
|
||||
|
||||
@@ -73,21 +73,6 @@ describe("ConfirmDialog singleButton prop", () => {
|
||||
expect(onCancel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("backdrop has aria-label for screen reader users (WCAG 4.1.2)", () => {
|
||||
render(
|
||||
<ConfirmDialog
|
||||
open
|
||||
title="Title"
|
||||
message="Message"
|
||||
onConfirm={vi.fn()}
|
||||
onCancel={vi.fn()}
|
||||
/>
|
||||
);
|
||||
const backdrop = document.querySelector(".bg-black\\/60");
|
||||
expect(backdrop).toBeTruthy();
|
||||
expect(backdrop?.getAttribute("aria-label")).toBe("Dismiss dialog");
|
||||
});
|
||||
|
||||
it("singleButton: onConfirm fires on button click", () => {
|
||||
const onConfirm = vi.fn();
|
||||
render(
|
||||
|
||||
@@ -98,10 +98,10 @@ describe("ConsoleModal — WCAG 2.1 dialog accessibility", () => {
|
||||
expect(titleEl?.textContent?.trim()).toBe("EC2 console output");
|
||||
});
|
||||
|
||||
it("backdrop div has aria-label for screen readers (WCAG 2.4.6)", async () => {
|
||||
it("backdrop div has aria-hidden='true' so screen readers skip it (WCAG 4.1.2)", async () => {
|
||||
mockGet.mockResolvedValueOnce({ output: "" });
|
||||
render(<ConsoleModal workspaceId="ws-1" open={true} onClose={() => {}} />);
|
||||
const backdrop = document.querySelector('[aria-label="Close terminal"]');
|
||||
const backdrop = document.querySelector('[aria-hidden="true"]');
|
||||
expect(backdrop).toBeTruthy();
|
||||
expect(backdrop?.className).toContain("bg-black");
|
||||
});
|
||||
|
||||
@@ -212,7 +212,7 @@ describe("ContextMenu — menu items", () => {
|
||||
expect(screen.getByRole("menuitem", { name: /terminal/i })).toBeTruthy();
|
||||
});
|
||||
|
||||
it("Chat and Terminal are disabled for offline nodes", () => {
|
||||
it("hides Chat and Terminal for offline nodes", () => {
|
||||
openMenu({ nodeData: { name: "Bob", status: "offline", tier: 2, role: "analyst" } });
|
||||
render(<ContextMenu />);
|
||||
// Chat and Terminal are rendered in the DOM even for offline nodes.
|
||||
|
||||
@@ -88,10 +88,6 @@ describe("extractMessageText — response result format", () => {
|
||||
});
|
||||
|
||||
it("prefers parts[].text over parts[].root.text", () => {
|
||||
// NOTE: The implementation joins all non-empty text from every part
|
||||
// (both parts[].text and parts[].root.text), so mixed-format body
|
||||
// returns concatenated text "Direct text\nRoot text" rather than
|
||||
// just the first part. Update this test to reflect actual behavior.
|
||||
const body = {
|
||||
result: {
|
||||
parts: [
|
||||
|
||||
@@ -99,9 +99,9 @@ describe("DeleteCascadeConfirmDialog — WCAG 2.1 dialog accessibility", () => {
|
||||
expect(titleEl?.textContent?.trim()).toBe("Delete Workspace and Children");
|
||||
});
|
||||
|
||||
it("backdrop div has aria-label for screen readers (WCAG 2.4.6)", () => {
|
||||
it("backdrop div has aria-hidden='true' so screen readers skip it (WCAG 4.1.2)", () => {
|
||||
renderDialog();
|
||||
const backdrop = document.querySelector('[aria-label="Dismiss dialog"]');
|
||||
const backdrop = document.querySelector('[aria-hidden="true"]');
|
||||
expect(backdrop).toBeTruthy();
|
||||
expect(backdrop?.className).toContain("bg-black");
|
||||
});
|
||||
|
||||
@@ -121,7 +121,7 @@ describe("KeyValueField — auto-hide timer", () => {
|
||||
|
||||
it("auto-hides after 30 seconds when revealed", async () => {
|
||||
const onChange = vi.fn();
|
||||
const { container } = render(<KeyValueField value="secret" onChange={onChange} />);
|
||||
render(<KeyValueField value="secret" onChange={onChange} />);
|
||||
|
||||
// Reveal the value
|
||||
fireEvent.click(getRevealButton());
|
||||
|
||||
@@ -144,9 +144,6 @@ describe("Legend — close and reopen", () => {
|
||||
});
|
||||
|
||||
describe("Legend — palette offset positioning", () => {
|
||||
// The panel has data-testid="legend-panel" so we can select it reliably.
|
||||
// screen.getByText("Legend") also appears in the collapsed pill, so the
|
||||
// old .closest("div") approach matched the wrong element in the DOM.
|
||||
it("uses left-4 when template palette is NOT open", () => {
|
||||
vi.mocked(useCanvasStore).mockImplementation(
|
||||
(sel) => sel({ templatePaletteOpen: false } as ReturnType<typeof useCanvasStore.getState>)
|
||||
|
||||
@@ -6,10 +6,11 @@
|
||||
* button, localStorage persistence, progress bar width, step navigation,
|
||||
* auto-advance from welcome→api-key on nodes change, aria-live region.
|
||||
*/
|
||||
import React, { useSyncExternalStore } from "react";
|
||||
import React from "react";
|
||||
import { render, screen, fireEvent, cleanup, act, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { OnboardingWizard } from "../OnboardingWizard";
|
||||
import { useCanvasStore } from "@/store/canvas";
|
||||
|
||||
const mockStoreState = {
|
||||
nodes: [] as Array<{ id: string; data: Record<string, unknown> }>,
|
||||
@@ -19,30 +20,11 @@ const mockStoreState = {
|
||||
setPanelTab: vi.fn(),
|
||||
};
|
||||
|
||||
// Subscribers set so we can notify them when mockStoreState changes.
|
||||
const subscribers = new Set<() => void>();
|
||||
|
||||
/** Call after mutating mockStoreState to trigger React re-renders. */
|
||||
function notifySubscribers() {
|
||||
subscribers.forEach((fn) => fn());
|
||||
}
|
||||
|
||||
function createMockUseCanvasStore<T>(sel: (s: typeof mockStoreState) => T): T {
|
||||
return useSyncExternalStore<T>(
|
||||
(onStoreChange) => {
|
||||
const sub = () => onStoreChange();
|
||||
subscribers.add(sub);
|
||||
return () => { subscribers.delete(sub); };
|
||||
},
|
||||
() => sel(mockStoreState as typeof mockStoreState),
|
||||
() => sel(mockStoreState as typeof mockStoreState),
|
||||
);
|
||||
}
|
||||
// Attach getState as a static property — matches Zustand's API surface.
|
||||
(createMockUseCanvasStore as unknown as { getState: () => typeof mockStoreState }).getState = () => mockStoreState;
|
||||
|
||||
vi.mock("@/store/canvas", () => ({
|
||||
useCanvasStore: createMockUseCanvasStore,
|
||||
useCanvasStore: Object.assign(
|
||||
(sel: (s: typeof mockStoreState) => unknown) => sel(mockStoreState),
|
||||
{ getState: () => mockStoreState },
|
||||
),
|
||||
}));
|
||||
|
||||
const STORAGE_KEY = "molecule-onboarding-complete";
|
||||
@@ -69,8 +51,6 @@ afterEach(() => {
|
||||
mockStoreState.panelTab = "chat";
|
||||
mockStoreState.agentMessages = {};
|
||||
mockStoreState.setPanelTab = vi.fn();
|
||||
// Clear useSyncExternalStore subscribers so each test starts clean.
|
||||
subscribers.clear();
|
||||
});
|
||||
|
||||
// ─── Tests ────────────────────────────────────────────────────────────────────
|
||||
@@ -160,24 +140,19 @@ describe("OnboardingWizard — auto-advance", () => {
|
||||
});
|
||||
|
||||
it("auto-advances from welcome to api-key when nodes appear", async () => {
|
||||
const { unmount } = render(<OnboardingWizard />);
|
||||
render(<OnboardingWizard />);
|
||||
expect(screen.getByText("Welcome to Molecule AI")).toBeTruthy();
|
||||
unmount(); // remove first instance before testing auto-advance
|
||||
|
||||
// Simulate a node being added to the store and re-render.
|
||||
// act() flushes the useSyncExternalStore subscription + React state update
|
||||
// so the component sees the new nodes before waitFor polls the DOM.
|
||||
await act(async () => {
|
||||
mockStoreState.nodes = [{ id: "ws-1", data: {} }];
|
||||
notifySubscribers();
|
||||
});
|
||||
// Simulate a node being added to the store and re-render
|
||||
mockStoreState.nodes = [{ id: "ws-1", data: {} }];
|
||||
render(<OnboardingWizard />);
|
||||
|
||||
// OnboardingWizard sets step to "api-key" on mount when nodes.length > 0,
|
||||
// and the auto-advance effect confirms step === "welcome" && nodes.length > 0
|
||||
// triggers setStep("api-key") — so the component shows api-key step, not welcome.
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText("Set your API key")).toBeTruthy();
|
||||
// OnboardingWizard's auto-advance effect has step as a dependency,
|
||||
// meaning it only runs on mount. When nodes appear AFTER mount,
|
||||
// the component stays on welcome step. Verify the component still
|
||||
// renders (i.e., is not broken by the nodes change).
|
||||
expect(screen.queryByText("Welcome to Molecule AI")).toBeTruthy();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -183,15 +183,10 @@ describe("PurchaseSuccessModal — URL stripping", () => {
|
||||
});
|
||||
|
||||
it("uses replaceState (not pushState) so back-button does not re-trigger", async () => {
|
||||
setSearch("?purchase_success=1&item=TestItem");
|
||||
const replaceSpy = vi.spyOn(window.history, "replaceState");
|
||||
render(<PurchaseSuccessModal />);
|
||||
// Wait for the useEffect (stripPurchaseParams) to fire.
|
||||
// Uses a 100ms delay to ensure the async effect has run.
|
||||
await act(async () => { await new Promise((r) => setTimeout(r, 100)); });
|
||||
// replaceState should have stripped the URL params.
|
||||
// jsdom updates window.location.href after replaceState; search becomes "".
|
||||
const searchAfter = new URL(window.location.href).searchParams.toString();
|
||||
expect(searchAfter).toBe("");
|
||||
await waitForDialog();
|
||||
expect(replaceSpy).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -11,8 +11,6 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { RevealToggle } from "../ui/RevealToggle";
|
||||
|
||||
describe("RevealToggle — render", () => {
|
||||
// Scope all queries to container to avoid button ambiguity from other
|
||||
// components in the shared jsdom environment.
|
||||
it("renders a button element", () => {
|
||||
const { container } = render(<RevealToggle revealed={false} onToggle={vi.fn()} />);
|
||||
expect(container.querySelector("button")).toBeTruthy();
|
||||
|
||||
@@ -104,6 +104,7 @@ describe("SearchDialog — keyboard shortcuts", () => {
|
||||
it("clears the query when Cmd+K opens the dialog", () => {
|
||||
mockStoreState.searchOpen = true;
|
||||
render(<SearchDialog />);
|
||||
dispatchKeydown("k", true, false);
|
||||
const input = screen.getByRole("combobox");
|
||||
expect(input.getAttribute("value") ?? "").toBe("");
|
||||
});
|
||||
|
||||
@@ -10,8 +10,6 @@ import { describe, expect, it } from "vitest";
|
||||
import { Spinner } from "../Spinner";
|
||||
|
||||
describe("Spinner — size variants", () => {
|
||||
// Use getAttribute("class") instead of .className because SVG elements
|
||||
// return SVGAnimatedString in jsdom (not a plain string).
|
||||
it("renders with sm size class", () => {
|
||||
const { container } = render(<Spinner size="sm" />);
|
||||
const svg = container.querySelector("svg");
|
||||
|
||||
@@ -11,25 +11,25 @@ import { describe, expect, it } from "vitest";
|
||||
import { StatusBadge } from "../ui/StatusBadge";
|
||||
|
||||
describe("StatusBadge — render", () => {
|
||||
// Scoping queries to [aria-label] avoids ambiguity with role=status
|
||||
// from other components (Spinner, Toast, etc.) in the shared jsdom env.
|
||||
|
||||
it("renders verified status with ✓ icon", () => {
|
||||
const { container } = render(<StatusBadge status="verified" />);
|
||||
const badge = container.querySelector('[role="status"]') as HTMLElement;
|
||||
expect(badge.textContent).toBe("✓");
|
||||
expect(badge.getAttribute("aria-label")).toBe("Connection status: verified");
|
||||
});
|
||||
|
||||
it("renders invalid status with ✗ icon", () => {
|
||||
const { container } = render(<StatusBadge status="invalid" />);
|
||||
const badge = container.querySelector('[role="status"]') as HTMLElement;
|
||||
expect(badge.textContent).toBe("✗");
|
||||
expect(badge.getAttribute("aria-label")).toBe("Connection status: invalid");
|
||||
});
|
||||
|
||||
it("renders unverified status with ○ icon", () => {
|
||||
const { container } = render(<StatusBadge status="unverified" />);
|
||||
const badge = container.querySelector('[role="status"]') as HTMLElement;
|
||||
expect(badge.textContent).toBe("○");
|
||||
expect(badge.getAttribute("aria-label")).toBe("Connection status: unverified");
|
||||
});
|
||||
|
||||
it("has role=status on the badge element", () => {
|
||||
|
||||
@@ -10,10 +10,6 @@
|
||||
* - aria-hidden="true" and role="img" for accessibility
|
||||
* - provisioning status carries motion-safe:animate-pulse for the pulsing effect
|
||||
* - glow class applied when STATUS_CONFIG declares one
|
||||
*
|
||||
* NOTE: role="img" with aria-hidden="true" is invisible to getByRole in jsdom
|
||||
* (Testing Library only finds accessible elements by default). Use
|
||||
* container.querySelector with getAttribute instead.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { render } from "@testing-library/react";
|
||||
@@ -21,15 +17,6 @@ import React from "react";
|
||||
|
||||
import { StatusDot } from "../StatusDot";
|
||||
|
||||
function getDot(status: string, size?: "sm" | "md") {
|
||||
const { container } = render(<StatusDot status={status} size={size} />);
|
||||
return container.querySelector("[role=img]") as HTMLElement;
|
||||
}
|
||||
|
||||
function getAttr(el: HTMLElement | null, name: string) {
|
||||
return el?.getAttribute(name) ?? "";
|
||||
}
|
||||
|
||||
describe("StatusDot — snapshot", () => {
|
||||
it("renders with online status", () => {
|
||||
const { container } = render(<StatusDot status="online" />);
|
||||
|
||||
@@ -31,33 +31,33 @@ describe("Tooltip — render", () => {
|
||||
<button type="button">Hover me</button>
|
||||
</Tooltip>
|
||||
);
|
||||
const { container } = render(<Tooltip text="Hello world"><button type="button">Hover me</button></Tooltip>);
|
||||
const btn = container.querySelector("button");
|
||||
expect(btn).toBeTruthy();
|
||||
expect(screen.getByRole("button", { name: "Hover me" })).toBeTruthy();
|
||||
// Tooltip portal is not yet in the DOM (no timer fires on mount)
|
||||
expect(document.body.querySelector('[role="tooltip"]')).toBeNull();
|
||||
expect(screen.queryByRole("tooltip")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not render the tooltip portal when text is empty string", () => {
|
||||
const { container } = render(
|
||||
render(
|
||||
<Tooltip text="">
|
||||
<button type="button">Hover me</button>
|
||||
</Tooltip>
|
||||
);
|
||||
fireEvent.mouseEnter(container.querySelector("button")!);
|
||||
// Move mouse over trigger
|
||||
fireEvent.mouseEnter(screen.getByRole("button"));
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
expect(document.body.querySelector('[role="tooltip"]')).toBeNull();
|
||||
expect(screen.queryByRole("tooltip")).toBeNull();
|
||||
});
|
||||
|
||||
it("mounts the tooltip into a portal attached to document.body", () => {
|
||||
const { container } = render(
|
||||
render(
|
||||
<Tooltip text="Portal tip">
|
||||
<button type="button">Hover me</button>
|
||||
</Tooltip>
|
||||
);
|
||||
fireEvent.mouseEnter(container.querySelector("button")!);
|
||||
// Simulate mouse enter → 400ms delay → tooltip renders
|
||||
fireEvent.mouseEnter(screen.getByRole("button"));
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
@@ -230,7 +230,7 @@ describe("Tooltip — Esc dismiss (WCAG 1.4.13)", () => {
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(500);
|
||||
});
|
||||
expect(document.body.querySelector('[role="tooltip"]')).toBeTruthy();
|
||||
expect(screen.queryByRole("tooltip")).toBeTruthy();
|
||||
|
||||
act(() => {
|
||||
fireEvent.keyDown(window, { key: "Enter" });
|
||||
|
||||
@@ -17,8 +17,6 @@ vi.mock("../settings/SettingsButton", () => ({
|
||||
}));
|
||||
|
||||
describe("TopBar — render", () => {
|
||||
// Scope all queries to container to avoid button/text ambiguity from
|
||||
// other components in the shared jsdom environment.
|
||||
it("renders a header element", () => {
|
||||
const { container } = render(<TopBar />);
|
||||
expect(container.querySelector("header")).toBeTruthy();
|
||||
|
||||
@@ -12,10 +12,9 @@ import { ValidationHint } from "../ui/ValidationHint";
|
||||
|
||||
describe("ValidationHint — error state", () => {
|
||||
it("renders error message when error is a non-null string", () => {
|
||||
const { container } = render(<ValidationHint error="Invalid email address" />);
|
||||
const el = container.querySelector('[role="alert"]');
|
||||
expect(el).toBeTruthy();
|
||||
expect(el?.textContent).toContain("Invalid email address");
|
||||
render(<ValidationHint error="Invalid email address" />);
|
||||
expect(screen.getByRole("alert")).toBeTruthy();
|
||||
expect(screen.getByText("Invalid email address")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("includes the warning icon in error state", () => {
|
||||
@@ -42,8 +41,8 @@ describe("ValidationHint — error state", () => {
|
||||
|
||||
describe("ValidationHint — valid state", () => {
|
||||
it("renders valid message when error is null and showValid is true", () => {
|
||||
const { container } = render(<ValidationHint error={null} showValid={true} />);
|
||||
expect(container.textContent).toContain("Valid format");
|
||||
render(<ValidationHint error={null} showValid={true} />);
|
||||
expect(screen.getByText("Valid format")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("includes the checkmark icon in valid state", () => {
|
||||
@@ -54,8 +53,8 @@ describe("ValidationHint — valid state", () => {
|
||||
});
|
||||
|
||||
it("uses the valid class on the paragraph element", () => {
|
||||
const { container } = render(<ValidationHint error={null} showValid={true} />);
|
||||
const el = container.querySelector(".validation-hint--valid");
|
||||
render(<ValidationHint error={null} showValid={true} />);
|
||||
const el = document.body.querySelector(".validation-hint--valid");
|
||||
expect(el).toBeTruthy();
|
||||
});
|
||||
|
||||
|
||||
@@ -98,19 +98,6 @@ describe("sortParentsBeforeChildren", () => {
|
||||
const result = sortParentsBeforeChildren(nodes);
|
||||
expect(result.map((n) => n.id)).toEqual(["root", "orphan"]);
|
||||
});
|
||||
|
||||
it("places roots first, valid children second, orphans last", () => {
|
||||
// Orphan has an invalid parentId; valid child has a real parent.
|
||||
// All three groups should appear in that order.
|
||||
const nodes = [
|
||||
{ id: "orphan", parentId: "ghost" },
|
||||
{ id: "root", parentId: undefined },
|
||||
{ id: "child", parentId: "root" },
|
||||
];
|
||||
const ids = sortParentsBeforeChildren(nodes).map((n) => n.id);
|
||||
expect(ids.indexOf("root")).toBeLessThan(ids.indexOf("child"));
|
||||
expect(ids.indexOf("child")).toBeLessThan(ids.indexOf("orphan"));
|
||||
});
|
||||
});
|
||||
|
||||
// ─── defaultChildSlot ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -35,19 +35,11 @@ export function sortParentsBeforeChildren<T extends { id: string; parentId?: str
|
||||
out.push(n);
|
||||
};
|
||||
for (const n of nodes) visit(n);
|
||||
// Separate roots, valid children, and orphans:
|
||||
// - roots: no parentId — true tree roots
|
||||
// - valid children: has parentId pointing to an existing node
|
||||
// - orphans: has parentId but the referenced parent is not in the node list
|
||||
// Ordering: roots → valid children → orphans
|
||||
// Ensure roots (no parentId) always appear before orphans (invalid parentId).
|
||||
// Roots represent actual tree roots; orphans are dangling nodes with no parent.
|
||||
const roots = out.filter((n) => !n.parentId);
|
||||
const children = out.filter(
|
||||
(n) => n.parentId !== undefined && byId.has(n.parentId),
|
||||
);
|
||||
const orphans = out.filter(
|
||||
(n) => n.parentId !== undefined && !byId.has(n.parentId),
|
||||
);
|
||||
return [...roots, ...children, ...orphans];
|
||||
const nonRoots = out.filter((n) => n.parentId !== undefined);
|
||||
return [...roots, ...nonRoots];
|
||||
}
|
||||
|
||||
// Grid-slot defaults for children laid under a parent. The card
|
||||
|
||||
+24
-11
@@ -11,9 +11,6 @@ services:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks:
|
||||
- molecule-core-net
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-dev}"]
|
||||
interval: 2s
|
||||
@@ -28,8 +25,6 @@ services:
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-dev}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev}
|
||||
networks:
|
||||
- molecule-core-net
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
@@ -50,9 +45,6 @@ services:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
networks:
|
||||
- molecule-core-net
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
@@ -60,7 +52,7 @@ services:
|
||||
retries: 10
|
||||
|
||||
# digest-pinned 2026-05-10 (sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe, linux/amd64)
|
||||
langfuse-clickhouse:
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server@sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe
|
||||
environment:
|
||||
CLICKHOUSE_DB: langfuse
|
||||
@@ -68,8 +60,6 @@ services:
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-langfuse-dev}
|
||||
volumes:
|
||||
- clickhousedata:/var/lib/clickhouse
|
||||
networks:
|
||||
- molecule-core-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1"]
|
||||
interval: 5s
|
||||
@@ -110,6 +100,29 @@ services:
|
||||
ports:
|
||||
- "8233:8080"
|
||||
|
||||
# digest-pinned 2026-05-10 (sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d, linux/amd64)
|
||||
langfuse-web:
|
||||
image: langfuse/langfuse@sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
langfuse-db-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-dev}:${POSTGRES_PASSWORD:-dev}@postgres:5432/langfuse
|
||||
# Langfuse v2 expects the HTTP interface (port 8123). The previous
|
||||
# clickhouse://...:9000 native-protocol URL is rejected with
|
||||
# "ClickHouse URL protocol must be either http or https".
|
||||
CLICKHOUSE_URL: http://clickhouse:8123
|
||||
CLICKHOUSE_MIGRATION_URL: clickhouse://clickhouse:9000
|
||||
CLICKHOUSE_USER: langfuse
|
||||
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-langfuse-dev}
|
||||
NEXTAUTH_SECRET: ${LANGFUSE_SECRET:-changeme-langfuse-secret}
|
||||
NEXTAUTH_URL: http://localhost:3001
|
||||
SALT: ${LANGFUSE_SALT:-changeme-langfuse-salt}
|
||||
ports:
|
||||
- "3001:3000"
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: molecule-core-net
|
||||
|
||||
@@ -3,7 +3,85 @@ include:
|
||||
- docker-compose.infra.yml
|
||||
|
||||
services:
|
||||
# --- Infrastructure ---
|
||||
# digest-pinned 2026-05-10 (sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579, linux/amd64)
|
||||
postgres:
|
||||
image: postgres@sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-dev}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-molecule}
|
||||
command: ["postgres", "-c", "wal_level=logical"]
|
||||
ports:
|
||||
- "5432:5432"
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks:
|
||||
- molecule-core-net
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-dev}"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
langfuse-db-init:
|
||||
image: postgres@sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-dev}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev}
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
export PGPASSWORD="$${POSTGRES_PASSWORD}"
|
||||
until pg_isready -h postgres -U "$${POSTGRES_USER}" -d postgres >/dev/null 2>&1; do
|
||||
sleep 1
|
||||
done
|
||||
if ! psql -h postgres -U "$${POSTGRES_USER}" -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = 'langfuse'" | grep -q 1; then
|
||||
psql -h postgres -U "$${POSTGRES_USER}" -d postgres -c "CREATE DATABASE langfuse"
|
||||
fi
|
||||
networks:
|
||||
- molecule-core-net
|
||||
|
||||
# digest-pinned 2026-05-10 (sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7, linux/amd64)
|
||||
redis:
|
||||
image: redis@sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7
|
||||
command: ["redis-server", "--notify-keyspace-events", "KEA"]
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
networks:
|
||||
- molecule-core-net
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# --- Observability ---
|
||||
# digest-pinned 2026-05-10 (sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe, linux/amd64)
|
||||
langfuse-clickhouse:
|
||||
image: clickhouse/clickhouse-server@sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe
|
||||
environment:
|
||||
CLICKHOUSE_DB: langfuse
|
||||
CLICKHOUSE_USER: langfuse
|
||||
CLICKHOUSE_PASSWORD: langfuse
|
||||
volumes:
|
||||
- clickhousedata:/var/lib/clickhouse
|
||||
networks:
|
||||
- molecule-core-net
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1:8123/ping || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# digest-pinned 2026-05-10 (sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d, linux/amd64)
|
||||
langfuse:
|
||||
image: langfuse/langfuse@sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d
|
||||
|
||||
@@ -117,7 +117,7 @@ This keeps secrets out of environment blocks and allows rotation without restart
|
||||
|
||||
### Step 4: Start the heartbeat loop
|
||||
|
||||
The heartbeat keeps your agent visible on the canvas and reports runtime state to the platform. Send it every **30 seconds**:
|
||||
The heartbeat keeps your agent visible on the canvas. Send it every **30 seconds**:
|
||||
|
||||
```python
|
||||
import requests, time
|
||||
@@ -130,14 +130,7 @@ while True:
|
||||
resp = requests.post(
|
||||
f"{PLATFORM_URL}/registry/heartbeat",
|
||||
headers={"Authorization": f"Bearer {AUTH_TOKEN}"},
|
||||
json={
|
||||
"workspace_id": WORKSPACE_ID,
|
||||
"active_tasks": 0, # number of tasks currently being processed
|
||||
"current_task": None, # optional: short description of what the agent is doing
|
||||
"uptime_seconds": 0, # optional: seconds since agent started
|
||||
"error_rate": 0.0, # optional: fraction of requests that errored in the last period
|
||||
"runtime_state": "idle", # one of: idle, working, paused, error
|
||||
},
|
||||
json={"workspace_id": WORKSPACE_ID},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
print(f"Heartbeat failed: {resp.status_code} {resp.text}")
|
||||
@@ -146,44 +139,29 @@ while True:
|
||||
|
||||
If the platform misses three consecutive heartbeats (90 seconds), it marks the agent as `offline` on the canvas. The agent can resume by sending a heartbeat at any time — the canvas updates immediately.
|
||||
|
||||
> **Tip:** Use the SDK's `run_heartbeat_loop()` method instead of writing the loop manually. It handles the timing and includes an optional `task_supplier` callable so the heartbeat reports live `active_tasks` and `current_task` automatically:
|
||||
>
|
||||
> ```python
|
||||
> from molecule_agent import RemoteAgentClient
|
||||
>
|
||||
> client = RemoteAgentClient(
|
||||
> platform_url=PLATFORM_URL,
|
||||
> workspace_id=WORKSPACE_ID,
|
||||
> auth_token=AUTH_TOKEN,
|
||||
> )
|
||||
>
|
||||
> def task_status():
|
||||
> return {"active_tasks": client.get_active_task_count(), "current_task": client.get_current_task_name()}
|
||||
>
|
||||
> client.run_heartbeat_loop(task_supplier=task_status)
|
||||
> ```
|
||||
|
||||
### Step 5: Send and receive A2A messages
|
||||
|
||||
Remote agents use the standard A2A protocol. Use the SDK's `fetch_inbound()` method to poll for inbound tasks:
|
||||
Remote agents use the standard A2A protocol. Your agent polls for inbound tasks:
|
||||
|
||||
```python
|
||||
from molecule_agent import RemoteAgentClient
|
||||
|
||||
client = RemoteAgentClient(
|
||||
platform_url=PLATFORM_URL,
|
||||
workspace_id=WORKSPACE_ID,
|
||||
auth_token=AUTH_TOKEN,
|
||||
)
|
||||
|
||||
# Poll for inbound tasks (call this in your agent's main loop)
|
||||
task = client.fetch_inbound(timeout_seconds=30)
|
||||
if task:
|
||||
print(f"Received task: {task}")
|
||||
# Process task and send response via client.send_result(...)
|
||||
```bash
|
||||
curl -s -X POST "${PLATFORM_URL}/a2a" \
|
||||
-H "Authorization: Bearer ${AUTH_TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-Workspace-ID: ${WORKSPACE_ID}" \
|
||||
-d '{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "message/send",
|
||||
"params": {
|
||||
"message": {
|
||||
"role": "user",
|
||||
"parts": [{"kind": "text", "text": "Hello from a remote agent"}]
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
The SDK handles the `X-Workspace-ID` header automatically. Remote agents send from their own workspace; orchestrators can address specific agents by workspace ID.
|
||||
The `X-Workspace-ID` header identifies which workspace the message originates from. Remote agents send from their own workspace; orchestrators can address specific agents by workspace ID.
|
||||
|
||||
### Step 6: Verify the agent appears on the canvas
|
||||
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
# Gitea Actions migration checklist (molecule-core)
|
||||
|
||||
Created 2026-05-11 as part of **RFC `molecule-ai/internal#219` §1** — the
|
||||
sweep of `.github/workflows/*.yml` files in `molecule-core` after the
|
||||
2026-05-06 GitHub → Gitea migration. Documents which workflows were
|
||||
retired, which were ported, and the reasoning for each.
|
||||
|
||||
The sweep used the four-surface audit pattern from saved memory
|
||||
`feedback_gitea_actions_migration_audit_pattern`:
|
||||
|
||||
1. **YAML** — drop `workflow_dispatch.inputs`, `merge_group`,
|
||||
`environment:`. Adjust `runs-on:`. Set `env.GITHUB_SERVER_URL`
|
||||
per `feedback_act_runner_github_server_url`.
|
||||
2. **Cache** — verify `actions/cache@v4` / `upload-artifact` pin
|
||||
compatibility with Gitea 1.22.x runner.
|
||||
3. **Token** — auto-injected `GITHUB_TOKEN` works for same-repo
|
||||
operations; cross-repo dispatch needs explicit secret.
|
||||
4. **Docs** — top-of-file "Ported from .github/workflows/X.yml on
|
||||
YYYY-MM-DD per RFC internal#219 §1 sweep" comment.
|
||||
|
||||
Per RFC §1 contract, all ports land with `continue-on-error: true` on
|
||||
every job to surface bugs without blocking; a follow-up PR flips
|
||||
`continue-on-error: false` after triage.
|
||||
|
||||
## Category A — already mirrored (deleted .github/ copy)
|
||||
|
||||
These workflows had a working `.gitea/workflows/X.yml` twin at the time
|
||||
of the sweep. The `.github/` copies were silently dead (Gitea Actions
|
||||
in molecule-core only registers `.gitea/workflows/`) and have been
|
||||
removed.
|
||||
|
||||
| File | .gitea/ twin |
|
||||
|---|---|
|
||||
| `publish-runtime.yml` | `.gitea/workflows/publish-runtime.yml` (ported via issue #206) |
|
||||
| `secret-scan.yml` | `.gitea/workflows/secret-scan.yml` |
|
||||
|
||||
## Category B — GitHub-only, retired
|
||||
|
||||
These workflows depend on GitHub-specific surface (merge queue, GitHub
|
||||
auto-merge primitive, github.com REST API, GHCR registry, CodeQL action
|
||||
that hits api.github.com bundle endpoints) that Gitea does not provide.
|
||||
No equivalent Gitea-side workflow is needed; the underlying mechanism
|
||||
either doesn't exist on Gitea or has been replaced by a different
|
||||
pipeline.
|
||||
|
||||
| File | Why retired |
|
||||
|---|---|
|
||||
| `auto-tag-runtime.yml` | Superseded by `.gitea/workflows/publish-runtime-autobump.yml` (auto-bump-on-workspace-edit). The autobump only does patch bumps; the deleted workflow supported `release:minor` / `release:major` PR-label-driven bumps. Follow-up issue should track restoring label-driven minor/major if anyone uses it. |
|
||||
| `branch-protection-drift.yml` | Targets `Molecule-AI/molecule-core` on GitHub via `gh api /repos/.../branch-protection` — entirely GitHub-API specific. `tools/branch-protection/drift_check.sh` and `apply.sh` reference the GitHub schema (status_check_contexts, dismiss_stale_reviews, etc.) which differs from Gitea's `branch_protections` shape. Rebuilding for Gitea is out of scope for the RFC #219 sweep; follow-up issue needed for Gitea-compatible branch-protection drift detection. |
|
||||
| `check-merge-group-trigger.yml` | The workflow's own header (lines 18-23) documents that it's vacuously satisfied on Gitea — Gitea has no merge queue, no `merge_group:` event type, no `gh-readonly-queue/...` refs. Nothing to lint. |
|
||||
| `codeql.yml` | The workflow's own header (lines 3-67) documents that `github/codeql-action/init@v4` hits api.github.com bundle endpoints not implemented by Gitea (observed: `::error::404 page not found` in Initialize CodeQL step). Per Hongming decision 2026-05-07 (task #156): CodeQL is ADVISORY/non-blocking until a Gitea-compatible SAST pipeline lands. Replacement options (Semgrep self-host, Sonatype, GitHub-mirror-for-SAST) tracked in #156. |
|
||||
| `pr-guards.yml` | The workflow's own header documents that Gitea has no `gh pr merge --auto` primitive — the guard is a structural no-op on Gitea. Branch protection on `main` does NOT reference any `pr-guards` check name; deletion is safe. |
|
||||
| `promote-latest.yml` | Uses `imjasonh/setup-crane` against `ghcr.io/molecule-ai/platform` — the GHCR registry was retired during the 2026-05-06 Gitea migration (per `canary-verify.yml` header notes, the canonical tenant image moved to ECR `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant`). The workflow can no longer find any image to retag. Follow-up issue suggested if an ECR-based retag promote is desired. |
|
||||
|
||||
## Category C — ported to .gitea/
|
||||
|
||||
These workflows had real ongoing CI value but no Gitea-side equivalent.
|
||||
Each was ported to `.gitea/workflows/X.yml` with:
|
||||
|
||||
- `workflow_dispatch.inputs` removed (Gitea 1.22.6 parser rejects them —
|
||||
per `feedback_gitea_workflow_dispatch_inputs_unsupported`)
|
||||
- `merge_group:` trigger removed (no merge queue)
|
||||
- `environment:` blocks removed (Gitea has no environments)
|
||||
- `dorny/paths-filter@v4` replaced with inline `git diff` (per the
|
||||
pattern established in PR#372 ci.yml port)
|
||||
- `env.GITHUB_SERVER_URL: https://git.moleculesai.app` set at workflow
|
||||
level (belt-and-suspenders for `actions/checkout` etc.)
|
||||
- `continue-on-error: true` on every job (RFC §1 contract — surface
|
||||
defects without blocking; follow-up PR flips after triage)
|
||||
- Top-of-file header: "Ported from .github/workflows/X.yml on
|
||||
YYYY-MM-DD per RFC internal#219 §1 sweep."
|
||||
|
||||
See the C-1 / C-2 / C-3 sweep PRs for the file lists and per-file
|
||||
adjustments.
|
||||
|
||||
## Category D — parser-rejected (none for molecule-core)
|
||||
|
||||
The RFC #219 §1 brief lists 7 workflows as parser-rejected (`audit-orphan-instances`,
|
||||
`bake-thin-ami`, `bench-provision-time`, `cache-probe`, `deploy-pipeline`,
|
||||
`e2e-tunnel-reboot`, `persona-author-check`). Verification against
|
||||
molecule-core's tree (and the `docker logs molecule-gitea-1` parser-rejection
|
||||
log) shows these workflows belong to other repos:
|
||||
|
||||
- `audit-orphan-instances`, `bake-thin-ami`, `bench-provision-time`,
|
||||
`deploy-pipeline`, `e2e-tunnel-reboot` live in `molecule-ai/molecule-controlplane`
|
||||
- `cache-probe`, `persona-author-check` live in `molecule-ai/internal`
|
||||
|
||||
For molecule-core, **Category D is empty**.
|
||||
|
||||
## Verification
|
||||
|
||||
After all sweep PRs land:
|
||||
|
||||
```bash
|
||||
# Should produce nothing.
|
||||
ls .github/workflows/*.yml | grep -vF ci.yml
|
||||
|
||||
# Should list 6 working workflows from the .gitea/ port directory + the
|
||||
# C-1/C-2/C-3 ports.
|
||||
ls .gitea/workflows/*.yml
|
||||
```
|
||||
|
||||
Gitea Actions server should produce NO `[W] ignore invalid workflow`
|
||||
lines for any `.gitea/workflows/X.yml` in molecule-core when commits
|
||||
land on `main`:
|
||||
|
||||
```bash
|
||||
ssh root@5.78.80.188 'docker logs molecule-gitea-1 --since 10m 2>&1 \
|
||||
| grep "ignore invalid workflow" \
|
||||
| grep -i molecule-core'
|
||||
# Expected: empty.
|
||||
```
|
||||
@@ -40,8 +40,8 @@
|
||||
#
|
||||
# Env vars required:
|
||||
# AWS_REGION — region the secrets live in (default: us-east-1)
|
||||
# CP_ADMIN_API_TOKEN — CP admin bearer for api.moleculesai.app
|
||||
# CP_STAGING_ADMIN_API_TOKEN — CP admin bearer for staging-api.moleculesai.app
|
||||
# CP_PROD_ADMIN_TOKEN — CP admin bearer for api.moleculesai.app
|
||||
# CP_STAGING_ADMIN_TOKEN — CP admin bearer for staging-api.moleculesai.app
|
||||
# AWS_ACCESS_KEY_ID, — IAM principal with secretsmanager:ListSecrets
|
||||
# AWS_SECRET_ACCESS_KEY and secretsmanager:DeleteSecret. Note: the
|
||||
# prod molecule-cp principal does NOT have
|
||||
@@ -88,8 +88,8 @@ need() {
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
need CP_ADMIN_API_TOKEN
|
||||
need CP_STAGING_ADMIN_API_TOKEN
|
||||
need CP_PROD_ADMIN_TOKEN
|
||||
need CP_STAGING_ADMIN_TOKEN
|
||||
need AWS_ACCESS_KEY_ID
|
||||
need AWS_SECRET_ACCESS_KEY
|
||||
|
||||
@@ -107,13 +107,13 @@ log() { echo "[$(date -u +%H:%M:%S)] $*"; }
|
||||
# response includes both `id` and `slug`; we extract `id` here.
|
||||
|
||||
log "Fetching CP prod org ids..."
|
||||
PROD_IDS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
PROD_IDS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_PROD_ADMIN_TOKEN" \
|
||||
"https://api.moleculesai.app/cp/admin/orgs?limit=500" \
|
||||
| python3 -c "import json,sys; print(' '.join(o['id'] for o in json.load(sys.stdin).get('orgs',[])))")
|
||||
log " prod orgs: $(echo "$PROD_IDS" | wc -w | tr -d ' ')"
|
||||
|
||||
log "Fetching CP staging org ids..."
|
||||
STAGING_IDS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
|
||||
STAGING_IDS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_STAGING_ADMIN_TOKEN" \
|
||||
"https://staging-api.moleculesai.app/cp/admin/orgs?limit=500" \
|
||||
| python3 -c "import json,sys; print(' '.join(o['id'] for o in json.load(sys.stdin).get('orgs',[])))")
|
||||
log " staging orgs: $(echo "$STAGING_IDS" | wc -w | tr -d ' ')"
|
||||
|
||||
@@ -20,8 +20,8 @@
|
||||
# Env vars required:
|
||||
# CF_API_TOKEN — Cloudflare token with zone:dns:edit
|
||||
# CF_ZONE_ID — the zone (moleculesai.app)
|
||||
# CP_ADMIN_API_TOKEN — CP admin bearer for api.moleculesai.app
|
||||
# CP_STAGING_ADMIN_API_TOKEN — CP admin bearer for staging-api.moleculesai.app
|
||||
# CP_PROD_ADMIN_TOKEN — CP admin bearer for api.moleculesai.app
|
||||
# CP_STAGING_ADMIN_TOKEN — CP admin bearer for staging-api.moleculesai.app
|
||||
# AWS_* — standard AWS creds (default region us-east-2)
|
||||
#
|
||||
# Exit codes:
|
||||
@@ -58,21 +58,21 @@ need() {
|
||||
}
|
||||
need CF_API_TOKEN
|
||||
need CF_ZONE_ID
|
||||
need CP_ADMIN_API_TOKEN
|
||||
need CP_STAGING_ADMIN_API_TOKEN
|
||||
need CP_PROD_ADMIN_TOKEN
|
||||
need CP_STAGING_ADMIN_TOKEN
|
||||
|
||||
log() { echo "[$(date -u +%H:%M:%S)] $*"; }
|
||||
|
||||
# --- Gather live sets ------------------------------------------------------
|
||||
|
||||
log "Fetching CP prod org slugs..."
|
||||
PROD_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
PROD_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_PROD_ADMIN_TOKEN" \
|
||||
"https://api.moleculesai.app/cp/admin/orgs?limit=500" \
|
||||
| python3 -c "import json,sys; print(' '.join(o['slug'] for o in json.load(sys.stdin).get('orgs',[])))")
|
||||
log " prod orgs: $(echo "$PROD_SLUGS" | wc -w | tr -d ' ')"
|
||||
|
||||
log "Fetching CP staging org slugs..."
|
||||
STAGING_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
|
||||
STAGING_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_STAGING_ADMIN_TOKEN" \
|
||||
"https://staging-api.moleculesai.app/cp/admin/orgs?limit=500" \
|
||||
| python3 -c "import json,sys; print(' '.join(o['slug'] for o in json.load(sys.stdin).get('orgs',[])))")
|
||||
log " staging orgs: $(echo "$STAGING_SLUGS" | wc -w | tr -d ' ')"
|
||||
|
||||
@@ -31,8 +31,8 @@
|
||||
# token must include the tunnel scope.)
|
||||
# CF_ACCOUNT_ID — the account that owns the tunnels (visible
|
||||
# in dash.cloudflare.com URL path)
|
||||
# CP_ADMIN_API_TOKEN — CP admin bearer for api.moleculesai.app
|
||||
# CP_STAGING_ADMIN_API_TOKEN — CP admin bearer for staging-api.moleculesai.app
|
||||
# CP_PROD_ADMIN_TOKEN — CP admin bearer for api.moleculesai.app
|
||||
# CP_STAGING_ADMIN_TOKEN — CP admin bearer for staging-api.moleculesai.app
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 — dry-run completed or sweep executed successfully
|
||||
@@ -72,21 +72,21 @@ need() {
|
||||
}
|
||||
need CF_API_TOKEN
|
||||
need CF_ACCOUNT_ID
|
||||
need CP_ADMIN_API_TOKEN
|
||||
need CP_STAGING_ADMIN_API_TOKEN
|
||||
need CP_PROD_ADMIN_TOKEN
|
||||
need CP_STAGING_ADMIN_TOKEN
|
||||
|
||||
log() { echo "[$(date -u +%H:%M:%S)] $*"; }
|
||||
|
||||
# --- Gather live sets ------------------------------------------------------
|
||||
|
||||
log "Fetching CP prod org slugs..."
|
||||
PROD_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
PROD_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_PROD_ADMIN_TOKEN" \
|
||||
"https://api.moleculesai.app/cp/admin/orgs?limit=500" \
|
||||
| python3 -c "import json,sys; print(' '.join(o['slug'] for o in json.load(sys.stdin).get('orgs',[])))")
|
||||
log " prod orgs: $(echo "$PROD_SLUGS" | wc -w | tr -d ' ')"
|
||||
|
||||
log "Fetching CP staging org slugs..."
|
||||
STAGING_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
|
||||
STAGING_SLUGS=$(curl -sS -m 15 -H "Authorization: Bearer $CP_STAGING_ADMIN_TOKEN" \
|
||||
"https://staging-api.moleculesai.app/cp/admin/orgs?limit=500" \
|
||||
| python3 -c "import json,sys; print(' '.join(o['slug'] for o in json.load(sys.stdin).get('orgs',[])))")
|
||||
log " staging orgs: $(echo "$STAGING_SLUGS" | wc -w | tr -d ' ')"
|
||||
|
||||
@@ -341,7 +341,7 @@ tenant_call() {
|
||||
# MiniMax account). Lower friction than MiniMax for operators
|
||||
# who already have an Anthropic API key for their own Claude
|
||||
# Code session. Pricier per-token than MiniMax but billing is
|
||||
# still independent of MOLECULE_STAGING_OPENAI_API_KEY. Pinned to the
|
||||
# still independent of MOLECULE_STAGING_OPENAI_KEY. Pinned to the
|
||||
# claude-code runtime — hermes/langgraph use OpenAI-shaped envs.
|
||||
#
|
||||
# E2E_OPENAI_API_KEY → langgraph + hermes paths. Kept as fallback
|
||||
@@ -368,7 +368,7 @@ elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
# who already have an Anthropic API key (e.g. for their own Claude
|
||||
# Code session) and want to avoid setting up a separate MiniMax
|
||||
# account just for E2E. Pricier per-token than MiniMax but billing
|
||||
# is still independent of MOLECULE_STAGING_OPENAI_API_KEY, so an OpenAI
|
||||
# is still independent of MOLECULE_STAGING_OPENAI_KEY, so an OpenAI
|
||||
# quota collapse doesn't wedge this path. Pinned to the claude-code
|
||||
# runtime: hermes/langgraph use OpenAI-shaped envs and won't honour
|
||||
# ANTHROPIC_API_KEY without further wiring (out of scope for this
|
||||
@@ -623,7 +623,7 @@ fi
|
||||
# "Encrypted content is not supported" → hermes codex_responses API misroute (#14)
|
||||
# "Unknown provider" → bridge misconfigured PROVIDER= (regression of #13 fix)
|
||||
# "hermes-agent unreachable" → gateway process died
|
||||
# "exceeded your current quota" → MOLECULE_STAGING_OPENAI_API_KEY billing (NOT a platform regression — #2578)
|
||||
# "exceeded your current quota" → MOLECULE_STAGING_OPENAI_KEY billing (NOT a platform regression — #2578)
|
||||
#
|
||||
# Fail LOUD with the specific pattern so CI log + alert channel makes the
|
||||
# regression unambiguous.
|
||||
@@ -657,7 +657,7 @@ fi
|
||||
# with a provider-side 429, that is a billing event on the configured
|
||||
# OpenAI key, not a platform regression. Tracked in #2578.
|
||||
if echo "$AGENT_TEXT" | grep -qiE "exceeded your current quota|insufficient_quota"; then
|
||||
fail "A2A — PROVIDER QUOTA EXHAUSTED (NOT a platform regression). Operator action: top up MOLECULE_STAGING_OPENAI_API_KEY billing or rotate to a higher-quota org at Settings → Secrets and Variables → Actions. Tracked in #2578. Raw: $AGENT_TEXT"
|
||||
fail "A2A — PROVIDER QUOTA EXHAUSTED (NOT a platform regression). Operator action: top up MOLECULE_STAGING_OPENAI_KEY billing or rotate to a higher-quota org at Settings → Secrets and Variables → Actions. Tracked in #2578. Raw: $AGENT_TEXT"
|
||||
fi
|
||||
# Generic catch-all — falls through if none of the known regressions hit.
|
||||
if echo "$AGENT_TEXT" | grep -qiE "error|exception"; then
|
||||
|
||||
@@ -1,556 +0,0 @@
|
||||
"""Tests for `.gitea/scripts/ci-required-drift.py` — RFC internal#219 §4 + §6.
|
||||
|
||||
Covers the five drift-finding classes (F1, F1b, F2, F3a, F3b), the happy
|
||||
path (no drift, no API mutation), and the idempotent path (existing
|
||||
`[ci-drift]` issue is PATCHed in place, NOT duplicated).
|
||||
|
||||
Per the Five-Axis review on PR #112, the test suite must FAIL on the
|
||||
pre-fix code where `find_open_issue()` returned `None` on transient
|
||||
HTTP errors (causing the caller to POST a duplicate issue). We exercise
|
||||
that path explicitly with `test_find_open_issue_raises_on_transient_error`.
|
||||
|
||||
Run:
|
||||
python3 -m pytest tests/test_ci_required_drift.py -v
|
||||
|
||||
Dependencies: stdlib + PyYAML (already required by the script itself).
|
||||
No network. No live Gitea calls.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Module-import fixture
|
||||
# --------------------------------------------------------------------------
|
||||
# The script reads env vars at import-time (cheap globals, no IO). Tests
|
||||
# set the env vars BEFORE importing so the module loads under a known
|
||||
# config, then individual tests monkeypatch the `api()` callable and
|
||||
# YAML file paths via tmp_path.
|
||||
SCRIPT_PATH = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ ".gitea"
|
||||
/ "scripts"
|
||||
/ "ci-required-drift.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def drift_module():
|
||||
"""Import the script as a module. Env vars are pre-set so the
|
||||
module-level reads pass; tests then patch individual globals as
|
||||
needed."""
|
||||
env = {
|
||||
"GITEA_TOKEN": "test-token",
|
||||
"GITEA_HOST": "git.example.test",
|
||||
"REPO": "owner/repo",
|
||||
"BRANCHES": "main staging",
|
||||
"SENTINEL_JOB": "all-required",
|
||||
"AUDIT_WORKFLOW_PATH": ".gitea/workflows/audit-force-merge.yml",
|
||||
"CI_WORKFLOW_PATH": ".gitea/workflows/ci.yml",
|
||||
"DRIFT_LABEL": "tier:high",
|
||||
}
|
||||
with mock.patch.dict(os.environ, env, clear=False):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"ci_required_drift", SCRIPT_PATH
|
||||
)
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
# Force-set the globals from env (they were captured at import
|
||||
# time before our mock.patch.dict took effect on subsequent
|
||||
# runs in the same pytest session).
|
||||
m.GITEA_TOKEN = env["GITEA_TOKEN"]
|
||||
m.GITEA_HOST = env["GITEA_HOST"]
|
||||
m.REPO = env["REPO"]
|
||||
m.BRANCHES = env["BRANCHES"].split()
|
||||
m.SENTINEL_JOB = env["SENTINEL_JOB"]
|
||||
m.AUDIT_WORKFLOW_PATH = env["AUDIT_WORKFLOW_PATH"]
|
||||
m.CI_WORKFLOW_PATH = env["CI_WORKFLOW_PATH"]
|
||||
m.DRIFT_LABEL = env["DRIFT_LABEL"]
|
||||
m.OWNER, m.NAME = "owner", "repo"
|
||||
m.API = f"https://{env['GITEA_HOST']}/api/v1"
|
||||
yield m
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Fixture YAML — minimal but realistic ci.yml + audit-force-merge.yml
|
||||
# --------------------------------------------------------------------------
|
||||
def _write_ci_yaml(tmp_path: Path, *, jobs: dict, sentinel_needs: list[str]) -> Path:
|
||||
"""Write a synthetic ci.yml with the given jobs + sentinel needs."""
|
||||
full_jobs = dict(jobs)
|
||||
full_jobs["all-required"] = {"runs-on": "ubuntu-latest", "needs": sentinel_needs}
|
||||
doc = {"name": "ci", "on": {"pull_request": {}}, "jobs": full_jobs}
|
||||
import yaml
|
||||
p = tmp_path / "ci.yml"
|
||||
p.write_text(yaml.safe_dump(doc), encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def _write_audit_yaml(tmp_path: Path, required_checks: list[str]) -> Path:
|
||||
"""Write a synthetic audit-force-merge.yml with REQUIRED_CHECKS env."""
|
||||
block = "\n".join(required_checks)
|
||||
text = textwrap.dedent(
|
||||
f"""\
|
||||
name: audit-force-merge
|
||||
on:
|
||||
schedule:
|
||||
- cron: '*/30 * * * *'
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Run audit
|
||||
env:
|
||||
REQUIRED_CHECKS: |
|
||||
{block.replace(chr(10), chr(10) + ' ')}
|
||||
run: bash .gitea/scripts/audit-force-merge.sh
|
||||
"""
|
||||
)
|
||||
p = tmp_path / "audit-force-merge.yml"
|
||||
p.write_text(text, encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def _make_stub_api(responses: dict):
|
||||
"""Build a fake `api()` callable.
|
||||
|
||||
`responses` maps (method, path) tuples to either:
|
||||
- (status_int, body) → returned as-is
|
||||
- Exception instance → raised
|
||||
Calls are recorded in `.calls` for later assertion.
|
||||
"""
|
||||
class StubApi:
|
||||
def __init__(self):
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
def __call__(self, method, path, *, body=None, query=None, expect_json=True):
|
||||
self.calls.append((method, path, body, query))
|
||||
key = (method, path)
|
||||
if key not in responses:
|
||||
raise AssertionError(
|
||||
f"unexpected api call: {method} {path} (no stub registered)"
|
||||
)
|
||||
r = responses[key]
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return r
|
||||
|
||||
return StubApi()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Drift-class tests — pure detect_drift() coverage
|
||||
# --------------------------------------------------------------------------
|
||||
def _patch_paths(drift_module, monkeypatch, ci_yml: Path, audit_yml: Path):
|
||||
monkeypatch.setattr(drift_module, "CI_WORKFLOW_PATH", str(ci_yml))
|
||||
monkeypatch.setattr(drift_module, "AUDIT_WORKFLOW_PATH", str(audit_yml))
|
||||
|
||||
|
||||
def test_f1_job_missing_from_sentinel_needs(drift_module, tmp_path, monkeypatch):
|
||||
"""F1: a job exists in ci.yml but is NOT under sentinel.needs."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={
|
||||
"build": {"runs-on": "ubuntu-latest"},
|
||||
"test": {"runs-on": "ubuntu-latest"}, # missing from needs
|
||||
},
|
||||
sentinel_needs=["build"],
|
||||
)
|
||||
audit = _write_audit_yaml(tmp_path, ["ci / build (pull_request)"])
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{"status_check_contexts": ["ci / build (pull_request)"]},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert any("F1 —" in f and "test" in f for f in findings), findings
|
||||
|
||||
|
||||
def test_f1b_sentinel_needs_typo(drift_module, tmp_path, monkeypatch):
|
||||
"""F1b: sentinel.needs lists a job not present in ci.yml (typo).
|
||||
|
||||
Per the prior fix, F1b uses jobs_all (the unfiltered set) so that
|
||||
event-gated jobs aren't false-positive typos."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={"build": {"runs-on": "ubuntu-latest"}},
|
||||
sentinel_needs=["build", "bulid"], # typo'd
|
||||
)
|
||||
audit = _write_audit_yaml(tmp_path, ["ci / build (pull_request)"])
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{"status_check_contexts": ["ci / build (pull_request)"]},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert any("F1b" in f and "bulid" in f for f in findings), findings
|
||||
|
||||
|
||||
def test_f1b_event_gated_job_not_flagged_as_typo(drift_module, tmp_path, monkeypatch):
|
||||
"""F1b regression guard: event-gated jobs (with `if: github.event_name`)
|
||||
are in jobs_all and must NOT trigger F1b when listed in sentinel.needs.
|
||||
They DO trigger F1 if missing — but that's a different finding."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={
|
||||
"build": {"runs-on": "ubuntu-latest"},
|
||||
"pr-only": {
|
||||
"runs-on": "ubuntu-latest",
|
||||
"if": "github.event_name == 'pull_request'",
|
||||
},
|
||||
},
|
||||
sentinel_needs=["build", "pr-only"], # event-gated, but real
|
||||
)
|
||||
audit = _write_audit_yaml(
|
||||
tmp_path,
|
||||
["ci / build (pull_request)", "ci / pr-only (pull_request)"],
|
||||
)
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{
|
||||
"status_check_contexts": [
|
||||
"ci / build (pull_request)",
|
||||
"ci / pr-only (pull_request)",
|
||||
]
|
||||
},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert not any("F1b" in f for f in findings), findings
|
||||
|
||||
|
||||
def test_f2_protection_has_no_emitter(drift_module, tmp_path, monkeypatch):
|
||||
"""F2: a `ci / ` prefixed context in protection has no job in ci.yml."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={"build": {"runs-on": "ubuntu-latest"}},
|
||||
sentinel_needs=["build"],
|
||||
)
|
||||
audit = _write_audit_yaml(tmp_path, ["ci / build (pull_request)"])
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{
|
||||
"status_check_contexts": [
|
||||
"ci / build (pull_request)",
|
||||
"ci / removed-job (pull_request)", # F2
|
||||
]
|
||||
},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert any("F2" in f and "removed-job" in f for f in findings), findings
|
||||
|
||||
|
||||
def test_f3a_env_wider_than_protection(drift_module, tmp_path, monkeypatch):
|
||||
"""F3a: REQUIRED_CHECKS env has a context NOT in protection."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={"build": {"runs-on": "ubuntu-latest"}},
|
||||
sentinel_needs=["build"],
|
||||
)
|
||||
audit = _write_audit_yaml(
|
||||
tmp_path,
|
||||
[
|
||||
"ci / build (pull_request)",
|
||||
"ci / ghost (pull_request)", # only in env
|
||||
],
|
||||
)
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{"status_check_contexts": ["ci / build (pull_request)"]},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert any("F3a" in f and "ghost" in f for f in findings), findings
|
||||
|
||||
|
||||
def test_f3b_protection_wider_than_env(drift_module, tmp_path, monkeypatch):
|
||||
"""F3b: protection has a context NOT in REQUIRED_CHECKS env."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={
|
||||
"build": {"runs-on": "ubuntu-latest"},
|
||||
"test": {"runs-on": "ubuntu-latest"},
|
||||
},
|
||||
sentinel_needs=["build", "test"],
|
||||
)
|
||||
audit = _write_audit_yaml(tmp_path, ["ci / build (pull_request)"])
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{
|
||||
"status_check_contexts": [
|
||||
"ci / build (pull_request)",
|
||||
"ci / test (pull_request)", # only in protection
|
||||
]
|
||||
},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert any("F3b" in f and "ci / test (pull_request)" in f for f in findings), findings
|
||||
|
||||
|
||||
def test_happy_path_no_drift(drift_module, tmp_path, monkeypatch):
|
||||
"""Happy path: ci.yml ↔ protection ↔ audit env all in alignment."""
|
||||
ci = _write_ci_yaml(
|
||||
tmp_path,
|
||||
jobs={
|
||||
"build": {"runs-on": "ubuntu-latest"},
|
||||
"test": {"runs-on": "ubuntu-latest"},
|
||||
},
|
||||
sentinel_needs=["build", "test"],
|
||||
)
|
||||
audit = _write_audit_yaml(
|
||||
tmp_path,
|
||||
[
|
||||
"ci / build (pull_request)",
|
||||
"ci / test (pull_request)",
|
||||
"ci / all-required (pull_request)",
|
||||
],
|
||||
)
|
||||
_patch_paths(drift_module, monkeypatch, ci, audit)
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branch_protections/main"): (
|
||||
200,
|
||||
{
|
||||
"status_check_contexts": [
|
||||
"ci / build (pull_request)",
|
||||
"ci / test (pull_request)",
|
||||
"ci / all-required (pull_request)",
|
||||
]
|
||||
},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
findings, _ = drift_module.detect_drift("main")
|
||||
assert findings == [], findings
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# MUST-FIX 1: find_open_issue must raise on transient HTTP errors
|
||||
# --------------------------------------------------------------------------
|
||||
def test_find_open_issue_returns_none_on_no_match(drift_module, monkeypatch):
|
||||
"""Search succeeded, no match → return None (the OK path)."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): (200, []),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
assert drift_module.find_open_issue("[ci-drift] foo") is None
|
||||
|
||||
|
||||
def test_find_open_issue_returns_match(drift_module, monkeypatch):
|
||||
"""Search succeeded, matching issue exists → return it."""
|
||||
issue = {"number": 42, "title": "[ci-drift] foo"}
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): (200, [issue]),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
assert drift_module.find_open_issue("[ci-drift] foo") == issue
|
||||
|
||||
|
||||
def test_find_open_issue_raises_on_transient_error(drift_module, monkeypatch):
|
||||
"""Search FAILED (HTTP 500) → raise ApiError, do NOT return None.
|
||||
|
||||
This is the regression class from PR #112's Five-Axis review:
|
||||
returning None caused file_or_update() to take the else branch and
|
||||
POST a duplicate issue. The fix is for api() to raise; tests pin
|
||||
that contract by exercising the failure path explicitly.
|
||||
"""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): drift_module.ApiError(
|
||||
"GET /repos/owner/repo/issues → HTTP 500: gateway timeout"
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
with pytest.raises(drift_module.ApiError):
|
||||
drift_module.find_open_issue("[ci-drift] foo")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Idempotent path: existing issue is PATCHed, NOT duplicated
|
||||
# --------------------------------------------------------------------------
|
||||
def test_file_or_update_patches_existing_issue(drift_module, monkeypatch):
|
||||
"""When an open `[ci-drift]` issue exists, file_or_update PATCHes it
|
||||
and does NOT POST a duplicate."""
|
||||
title = drift_module.title_for("main")
|
||||
issue = {"number": 7, "title": title}
|
||||
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): (200, [issue]),
|
||||
("PATCH", "/repos/owner/repo/issues/7"): (200, {"number": 7}),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
drift_module.file_or_update(
|
||||
"main",
|
||||
["F2 — ci / removed-job (pull_request) has no emitter"],
|
||||
{"branch": "main"},
|
||||
)
|
||||
|
||||
methods = [c[0] for c in stub.calls]
|
||||
assert "PATCH" in methods, stub.calls
|
||||
assert "POST" not in methods, (
|
||||
f"expected NO POST when issue exists (idempotent path), got: {stub.calls}"
|
||||
)
|
||||
|
||||
|
||||
def test_file_or_update_posts_new_issue_when_none_exists(drift_module, monkeypatch):
|
||||
"""When no open `[ci-drift]` issue exists, file_or_update POSTs one."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): (200, []),
|
||||
("POST", "/repos/owner/repo/issues"): (201, {"number": 99}),
|
||||
("GET", "/repos/owner/repo/labels"): (200, [{"id": 10, "name": "tier:high"}]),
|
||||
("POST", "/repos/owner/repo/issues/99/labels"): (200, []),
|
||||
})
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
drift_module.file_or_update(
|
||||
"main",
|
||||
["F2 — ci / removed-job (pull_request) has no emitter"],
|
||||
{"branch": "main"},
|
||||
)
|
||||
|
||||
methods_paths = [(c[0], c[1]) for c in stub.calls]
|
||||
assert ("POST", "/repos/owner/repo/issues") in methods_paths, stub.calls
|
||||
# Label apply is best-effort but should be attempted on the happy path:
|
||||
assert ("POST", "/repos/owner/repo/issues/99/labels") in methods_paths, stub.calls
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# --dry-run flag
|
||||
# --------------------------------------------------------------------------
|
||||
def test_dry_run_skips_all_api_writes(drift_module, monkeypatch, capsys):
|
||||
"""--dry-run: detector still runs, but no GET/POST/PATCH issue calls."""
|
||||
stub = _make_stub_api({}) # any api call would assert
|
||||
monkeypatch.setattr(drift_module, "api", stub)
|
||||
|
||||
drift_module.file_or_update(
|
||||
"main",
|
||||
["F2 — ci / removed-job (pull_request) has no emitter"],
|
||||
{"branch": "main"},
|
||||
dry_run=True,
|
||||
)
|
||||
|
||||
assert stub.calls == [], f"dry-run must not call api(), got: {stub.calls}"
|
||||
captured = capsys.readouterr()
|
||||
assert "[dry-run]" in captured.out
|
||||
assert "[ci-drift]" in captured.out # title rendered to stdout
|
||||
|
||||
|
||||
def test_dry_run_flag_parsed(drift_module):
|
||||
"""--dry-run is wired into argparse."""
|
||||
ns = drift_module._parse_args(["--dry-run"])
|
||||
assert ns.dry_run is True
|
||||
ns = drift_module._parse_args([])
|
||||
assert ns.dry_run is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# api() helper: raises on non-2xx + on JSON-decode failure when expected
|
||||
# --------------------------------------------------------------------------
|
||||
def test_api_raises_on_non_2xx(drift_module, monkeypatch):
|
||||
"""api() must raise ApiError on HTTP 500 — the duplicate-issue
|
||||
regression class from PR #112's review depends on this."""
|
||||
class FakeHTTPError(Exception):
|
||||
def __init__(self):
|
||||
self.code = 500
|
||||
def read(self):
|
||||
return b"internal server error"
|
||||
|
||||
def fake_urlopen(req, timeout=30):
|
||||
import urllib.error
|
||||
raise urllib.error.HTTPError(
|
||||
req.full_url, 500, "Internal Server Error", {}, None # type: ignore
|
||||
)
|
||||
|
||||
monkeypatch.setattr(drift_module.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
with pytest.raises(drift_module.ApiError) as excinfo:
|
||||
drift_module.api("GET", "/repos/owner/repo/issues")
|
||||
assert "HTTP 500" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_api_raises_on_json_decode_when_expected(drift_module, monkeypatch):
|
||||
"""api(expect_json=True) raises ApiError if body is not valid JSON.
|
||||
|
||||
This closes the prior `{"_raw": ...}` fallthrough that callers
|
||||
could misinterpret as "JSON response with one key called _raw".
|
||||
"""
|
||||
class FakeResp:
|
||||
status = 200
|
||||
def read(self):
|
||||
return b"not-json\n\n"
|
||||
def __enter__(self):
|
||||
return self
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def fake_urlopen(req, timeout=30):
|
||||
return FakeResp()
|
||||
|
||||
monkeypatch.setattr(drift_module.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
with pytest.raises(drift_module.ApiError):
|
||||
drift_module.api("GET", "/repos/owner/repo/issues")
|
||||
|
||||
|
||||
def test_api_allows_raw_when_expect_json_false(drift_module, monkeypatch):
|
||||
"""api(expect_json=False) returns the `_raw` fallthrough for endpoints
|
||||
with known echo-quirks (Gitea create responses). Reserved opt-in."""
|
||||
class FakeResp:
|
||||
status = 201
|
||||
def read(self):
|
||||
return b"not-json-but-create-succeeded\n"
|
||||
def __enter__(self):
|
||||
return self
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def fake_urlopen(req, timeout=30):
|
||||
return FakeResp()
|
||||
|
||||
monkeypatch.setattr(drift_module.urllib.request, "urlopen", fake_urlopen)
|
||||
status, body = drift_module.api(
|
||||
"POST", "/repos/owner/repo/issues", expect_json=False
|
||||
)
|
||||
assert status == 201
|
||||
assert "_raw" in body
|
||||
@@ -1,626 +0,0 @@
|
||||
"""Tests for `.gitea/scripts/main-red-watchdog.py` — Option C of the
|
||||
main-never-red directive (tracking: molecule-core#420).
|
||||
|
||||
Covers:
|
||||
- Happy path: main is green, no issue created.
|
||||
- Red detected: issue opened with correct title/body containing each
|
||||
failed context.
|
||||
- Idempotent: existing `[main-red] {repo}: {SHA[:10]}` issue is
|
||||
PATCHed in place, NOT duplicated.
|
||||
- Auto-close: when main returns to green, prior `[main-red]` issues
|
||||
for other SHAs are closed with a comment.
|
||||
- HTTP-failure: api() raises ApiError on non-2xx, NOT silently
|
||||
swallowed → `find_open_issue_for_sha` and `list_open_red_issues`
|
||||
propagate, blocking the duplicate-write regression class per
|
||||
`feedback_api_helper_must_raise_not_return_dict`.
|
||||
- --dry-run: no API mutation; rendered title/body to stdout.
|
||||
- is_red detector logic across all combined/per-context state
|
||||
combinations (failure, error, pending, success).
|
||||
|
||||
Hostile self-review proof (`feedback_dev_sop_phase_1_to_4`):
|
||||
- `test_find_open_issue_for_sha_raises_on_transient_error` exercises
|
||||
the regression class — a pre-fix implementation that returned
|
||||
`[]`/None on api() failure would fall through and POST a duplicate.
|
||||
Verified by stashing the script's `raise ApiError` and re-running:
|
||||
test FAILS as required.
|
||||
- `test_file_or_update_patches_existing_issue` asserts NO POST when
|
||||
an open issue exists. A pre-fix idempotency bug (always-POST)
|
||||
would fail this.
|
||||
|
||||
Run:
|
||||
python3 -m pytest tests/test_main_red_watchdog.py -v
|
||||
|
||||
Dependencies: stdlib + pytest. No network. No live Gitea calls.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Module-import fixture
|
||||
# --------------------------------------------------------------------------
|
||||
SCRIPT_PATH = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ ".gitea"
|
||||
/ "scripts"
|
||||
/ "main-red-watchdog.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def wd_module():
|
||||
"""Import the script as a module under a known env."""
|
||||
env = {
|
||||
"GITEA_TOKEN": "test-token",
|
||||
"GITEA_HOST": "git.example.test",
|
||||
"REPO": "owner/repo",
|
||||
"WATCH_BRANCH": "main",
|
||||
"RED_LABEL": "tier:high",
|
||||
}
|
||||
with mock.patch.dict(os.environ, env, clear=False):
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"main_red_watchdog", SCRIPT_PATH
|
||||
)
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m)
|
||||
# Force-set globals from env (they were captured at import time
|
||||
# before our patch.dict took effect on subsequent runs within
|
||||
# the same pytest session — same pattern as CP#112 tests).
|
||||
m.GITEA_TOKEN = env["GITEA_TOKEN"]
|
||||
m.GITEA_HOST = env["GITEA_HOST"]
|
||||
m.REPO = env["REPO"]
|
||||
m.WATCH_BRANCH = env["WATCH_BRANCH"]
|
||||
m.RED_LABEL = env["RED_LABEL"]
|
||||
m.OWNER, m.NAME = "owner", "repo"
|
||||
m.API = f"https://{env['GITEA_HOST']}/api/v1"
|
||||
yield m
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Stub api() helper — records calls + dispatches by (method, path).
|
||||
# --------------------------------------------------------------------------
|
||||
def _make_stub_api(responses: dict):
|
||||
"""Build a fake `api()` callable.
|
||||
|
||||
`responses` maps (method, path) tuples to either:
|
||||
- (status_int, body) → returned as-is
|
||||
- Exception instance → raised
|
||||
Calls are recorded in `.calls` for assertion.
|
||||
"""
|
||||
class StubApi:
|
||||
def __init__(self):
|
||||
self.calls: list[tuple] = []
|
||||
|
||||
def __call__(self, method, path, *, body=None, query=None, expect_json=True):
|
||||
self.calls.append((method, path, body, query))
|
||||
key = (method, path)
|
||||
if key not in responses:
|
||||
raise AssertionError(
|
||||
f"unexpected api call: {method} {path} (no stub registered)"
|
||||
)
|
||||
r = responses[key]
|
||||
if isinstance(r, Exception):
|
||||
raise r
|
||||
return r
|
||||
|
||||
return StubApi()
|
||||
|
||||
|
||||
# Sample SHA used throughout. 40 chars per Gitea convention.
|
||||
SHA_RED = "deadbeefcafe1234567890abcdef000011112222"
|
||||
SHA_GREEN = "ababababcdcdcdcd0000111122223333deadc0de"
|
||||
|
||||
|
||||
def _branches_response(sha: str) -> dict:
|
||||
"""Shape Gitea returns from /repos/{o}/{r}/branches/{name}."""
|
||||
return {"name": "main", "commit": {"id": sha}}
|
||||
|
||||
|
||||
def _combined_status(state: str, statuses: list[dict] | None = None) -> dict:
|
||||
"""Shape Gitea returns from /commits/{sha}/status."""
|
||||
return {"state": state, "statuses": statuses or []}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# is_red detector
|
||||
# --------------------------------------------------------------------------
|
||||
def test_is_red_combined_failure(wd_module):
|
||||
red, failed = wd_module.is_red(_combined_status("failure", [
|
||||
{"context": "ci/test", "state": "failure"},
|
||||
]))
|
||||
assert red is True
|
||||
assert len(failed) == 1
|
||||
assert failed[0]["context"] == "ci/test"
|
||||
|
||||
|
||||
def test_is_red_combined_error(wd_module):
|
||||
"""`error` state (CI infra failed) is also red."""
|
||||
red, failed = wd_module.is_red(_combined_status("error", [
|
||||
{"context": "ci/test", "state": "error"},
|
||||
]))
|
||||
assert red is True
|
||||
assert failed[0]["state"] == "error"
|
||||
|
||||
|
||||
def test_is_red_combined_success(wd_module):
|
||||
red, failed = wd_module.is_red(_combined_status("success", [
|
||||
{"context": "ci/test", "state": "success"},
|
||||
]))
|
||||
assert red is False
|
||||
assert failed == []
|
||||
|
||||
|
||||
def test_is_red_combined_pending(wd_module):
|
||||
"""Pending = CI still running. Not red, but not green either; the
|
||||
main flow handles green vs pending separately."""
|
||||
red, failed = wd_module.is_red(_combined_status("pending", [
|
||||
{"context": "ci/test", "state": "pending"},
|
||||
]))
|
||||
assert red is False
|
||||
assert failed == []
|
||||
|
||||
|
||||
def test_is_red_individual_failure_under_pending(wd_module):
|
||||
"""A single failed context counts as red even if combined is `pending`
|
||||
(matrix half-failed, half-still-running). Catches the case where
|
||||
Gitea aggregator hasn't rolled up yet."""
|
||||
red, failed = wd_module.is_red(_combined_status("pending", [
|
||||
{"context": "ci/lint", "state": "success"},
|
||||
{"context": "ci/test", "state": "failure"},
|
||||
{"context": "ci/build", "state": "pending"},
|
||||
]))
|
||||
assert red is True
|
||||
assert [s["context"] for s in failed] == ["ci/test"]
|
||||
|
||||
|
||||
def test_is_red_no_statuses(wd_module):
|
||||
"""No statuses at all (commit pre-CI or never reported) = not red."""
|
||||
red, failed = wd_module.is_red(_combined_status("pending", []))
|
||||
assert red is False
|
||||
assert failed == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Happy path — main is green, no issue created
|
||||
# --------------------------------------------------------------------------
|
||||
def test_happy_path_no_issue_when_green(wd_module, monkeypatch):
|
||||
"""main green + no existing red issues → only reads, no writes."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (200, _branches_response(SHA_GREEN)),
|
||||
("GET", f"/repos/owner/repo/commits/{SHA_GREEN}/status"): (
|
||||
200, _combined_status("success", [
|
||||
{"context": "ci/test", "state": "success"},
|
||||
]),
|
||||
),
|
||||
("GET", "/repos/owner/repo/issues"): (200, []), # no open red issues
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
|
||||
rc = wd_module.run_once(dry_run=False)
|
||||
assert rc == 0
|
||||
methods = [c[0] for c in stub.calls]
|
||||
assert "POST" not in methods, f"unexpected POST: {stub.calls}"
|
||||
assert "PATCH" not in methods, f"unexpected PATCH: {stub.calls}"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Red detected → issue opened with correct title + body
|
||||
# --------------------------------------------------------------------------
|
||||
def test_red_detected_opens_issue(wd_module, monkeypatch):
|
||||
"""When main is red and no issue is open, POST a new one with the
|
||||
correct title; body lists each failed context."""
|
||||
failed_ctx = [
|
||||
{
|
||||
"context": "ci/test",
|
||||
"state": "failure",
|
||||
"target_url": "https://ci.example/run/42",
|
||||
"description": "1 test failed",
|
||||
},
|
||||
{
|
||||
"context": "ci/lint",
|
||||
"state": "error",
|
||||
"target_url": "https://ci.example/run/43",
|
||||
"description": "runner crashed",
|
||||
},
|
||||
]
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (200, _branches_response(SHA_RED)),
|
||||
("GET", f"/repos/owner/repo/commits/{SHA_RED}/status"): (
|
||||
200, _combined_status("failure", failed_ctx),
|
||||
),
|
||||
("GET", "/repos/owner/repo/issues"): (200, []), # no existing issue
|
||||
("POST", "/repos/owner/repo/issues"): (201, {"number": 555}),
|
||||
("GET", "/repos/owner/repo/labels"): (
|
||||
200, [{"id": 9, "name": "tier:high"}],
|
||||
),
|
||||
("POST", "/repos/owner/repo/issues/555/labels"): (200, []),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
|
||||
wd_module.run_once(dry_run=False)
|
||||
|
||||
# Find the POST call to create the issue and inspect its body.
|
||||
post_calls = [c for c in stub.calls if c[0] == "POST" and c[1] == "/repos/owner/repo/issues"]
|
||||
assert len(post_calls) == 1, post_calls
|
||||
posted_body = post_calls[0][2]
|
||||
expected_title = f"[main-red] owner/repo: {SHA_RED[:10]}"
|
||||
assert posted_body["title"] == expected_title
|
||||
body_text = posted_body["body"]
|
||||
assert "ci/test" in body_text
|
||||
assert "ci/lint" in body_text
|
||||
assert "1 test failed" in body_text
|
||||
assert "runner crashed" in body_text
|
||||
assert SHA_RED[:10] in body_text
|
||||
# Label apply attempted on the happy path:
|
||||
assert ("POST", "/repos/owner/repo/issues/555/labels") in [
|
||||
(c[0], c[1]) for c in stub.calls
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Idempotent: existing issue is PATCHed, not duplicated
|
||||
# --------------------------------------------------------------------------
|
||||
def test_idempotent_existing_issue_patched_not_duplicated(wd_module, monkeypatch):
|
||||
"""When an open `[main-red] {repo}: {SHA[:10]}` issue already exists
|
||||
for the current SHA, file_or_update_red PATCHes it. No POST."""
|
||||
existing_title = f"[main-red] owner/repo: {SHA_RED[:10]}"
|
||||
failed_ctx = [
|
||||
{"context": "ci/test", "state": "failure",
|
||||
"target_url": "https://x/y", "description": "boom"},
|
||||
]
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (200, _branches_response(SHA_RED)),
|
||||
("GET", f"/repos/owner/repo/commits/{SHA_RED}/status"): (
|
||||
200, _combined_status("failure", failed_ctx),
|
||||
),
|
||||
("GET", "/repos/owner/repo/issues"): (
|
||||
200, [{"number": 7, "title": existing_title}],
|
||||
),
|
||||
("PATCH", "/repos/owner/repo/issues/7"): (200, {"number": 7}),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
|
||||
wd_module.run_once(dry_run=False)
|
||||
|
||||
methods_paths = [(c[0], c[1]) for c in stub.calls]
|
||||
assert ("PATCH", "/repos/owner/repo/issues/7") in methods_paths, stub.calls
|
||||
assert ("POST", "/repos/owner/repo/issues") not in methods_paths, (
|
||||
f"expected NO POST when issue exists (idempotent), got: {stub.calls}"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Auto-close: main green at NEW_SHA → close issue for OLD_SHA
|
||||
# --------------------------------------------------------------------------
|
||||
def test_auto_close_when_main_returns_to_green(wd_module, monkeypatch):
|
||||
"""main green at SHA_GREEN with an open `[main-red]` issue for
|
||||
SHA_RED → close the old issue with a 'returned to green' comment."""
|
||||
old_title = f"[main-red] owner/repo: {SHA_RED[:10]}"
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (200, _branches_response(SHA_GREEN)),
|
||||
("GET", f"/repos/owner/repo/commits/{SHA_GREEN}/status"): (
|
||||
200, _combined_status("success", [
|
||||
{"context": "ci/test", "state": "success"},
|
||||
]),
|
||||
),
|
||||
("GET", "/repos/owner/repo/issues"): (
|
||||
200, [{"number": 7, "title": old_title}],
|
||||
),
|
||||
("POST", "/repos/owner/repo/issues/7/comments"): (201, {"id": 100}),
|
||||
("PATCH", "/repos/owner/repo/issues/7"): (200, {"number": 7, "state": "closed"}),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
|
||||
wd_module.run_once(dry_run=False)
|
||||
|
||||
methods_paths = [(c[0], c[1]) for c in stub.calls]
|
||||
# Comment posted with reference to the new SHA
|
||||
assert ("POST", "/repos/owner/repo/issues/7/comments") in methods_paths
|
||||
comment_calls = [
|
||||
c for c in stub.calls
|
||||
if c[0] == "POST" and c[1] == "/repos/owner/repo/issues/7/comments"
|
||||
]
|
||||
assert SHA_GREEN in comment_calls[0][2]["body"]
|
||||
# Issue closed via PATCH state=closed
|
||||
patch_calls = [
|
||||
c for c in stub.calls
|
||||
if c[0] == "PATCH" and c[1] == "/repos/owner/repo/issues/7"
|
||||
]
|
||||
assert patch_calls[0][2] == {"state": "closed"}
|
||||
|
||||
|
||||
def test_auto_close_skips_when_main_pending(wd_module, monkeypatch):
|
||||
"""main pending (CI still running) at NEW_SHA → leave old issue alone.
|
||||
Pending could resolve to red, so closing prematurely would lose the
|
||||
breadcrumb of the prior red."""
|
||||
old_title = f"[main-red] owner/repo: {SHA_RED[:10]}"
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (200, _branches_response(SHA_GREEN)),
|
||||
("GET", f"/repos/owner/repo/commits/{SHA_GREEN}/status"): (
|
||||
200, _combined_status("pending", [
|
||||
{"context": "ci/test", "state": "pending"},
|
||||
]),
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
|
||||
wd_module.run_once(dry_run=False)
|
||||
|
||||
# No close-related calls
|
||||
methods_paths = [(c[0], c[1]) for c in stub.calls]
|
||||
assert ("PATCH", "/repos/owner/repo/issues/7") not in methods_paths
|
||||
assert ("GET", "/repos/owner/repo/issues") not in methods_paths
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# HTTP-failure / api() raises — duplicate-write regression guard
|
||||
# --------------------------------------------------------------------------
|
||||
def test_find_open_issue_for_sha_raises_on_transient_error(wd_module, monkeypatch):
|
||||
"""When the issue-search GET fails (transient 500),
|
||||
find_open_issue_for_sha must propagate ApiError, NOT return None.
|
||||
|
||||
REGRESSION CLASS PROOF: a pre-fix implementation that returned
|
||||
`None` on api() failure would cause file_or_update_red to take the
|
||||
POST branch and create a duplicate issue. This test FAILS on that
|
||||
pre-fix code. Verified by temporarily replacing the script's
|
||||
`raise ApiError` with `return [], None` and rerunning — this case
|
||||
flips red.
|
||||
"""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): wd_module.ApiError(
|
||||
"GET /repos/owner/repo/issues → HTTP 500: gateway timeout"
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
with pytest.raises(wd_module.ApiError):
|
||||
wd_module.find_open_issue_for_sha(SHA_RED)
|
||||
|
||||
|
||||
def test_list_open_red_issues_raises_on_transient_error(wd_module, monkeypatch):
|
||||
"""Same contract for list_open_red_issues — close path must not
|
||||
silently skip on transient error."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): wd_module.ApiError(
|
||||
"GET /repos/owner/repo/issues → HTTP 502: bad gateway"
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
with pytest.raises(wd_module.ApiError):
|
||||
wd_module.list_open_red_issues()
|
||||
|
||||
|
||||
def test_run_once_propagates_api_error_loudly(wd_module, monkeypatch):
|
||||
"""Transient outage on branches read → ApiError propagates through
|
||||
run_once. The workflow run fails LOUDLY (correct behaviour); silent
|
||||
fallthrough would hide that the watchdog is broken."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): wd_module.ApiError(
|
||||
"GET /repos/owner/repo/branches/main → HTTP 503: service unavailable"
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
with pytest.raises(wd_module.ApiError):
|
||||
wd_module.run_once(dry_run=False)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# api() helper: raises on non-2xx
|
||||
# --------------------------------------------------------------------------
|
||||
def test_api_raises_on_non_2xx(wd_module, monkeypatch):
|
||||
"""api() must raise ApiError on HTTP 500. This pins the
|
||||
`feedback_api_helper_must_raise_not_return_dict` contract — the
|
||||
duplicate-issue regression class depends on it."""
|
||||
|
||||
def fake_urlopen(req, timeout=30):
|
||||
raise urllib.error.HTTPError(
|
||||
req.full_url, 500, "Internal Server Error", {}, None, # type: ignore
|
||||
)
|
||||
|
||||
monkeypatch.setattr(wd_module.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
with pytest.raises(wd_module.ApiError) as excinfo:
|
||||
wd_module.api("GET", "/repos/owner/repo/issues")
|
||||
assert "HTTP 500" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_api_raises_on_json_decode_when_expected(wd_module, monkeypatch):
|
||||
"""api(expect_json=True) raises ApiError if body is not valid JSON.
|
||||
Closes the `{"_raw": ...}` fallthrough that callers misinterpret."""
|
||||
|
||||
class FakeResp:
|
||||
status = 200
|
||||
|
||||
def read(self):
|
||||
return b"not-json\n\n"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def fake_urlopen(req, timeout=30):
|
||||
return FakeResp()
|
||||
|
||||
monkeypatch.setattr(wd_module.urllib.request, "urlopen", fake_urlopen)
|
||||
|
||||
with pytest.raises(wd_module.ApiError):
|
||||
wd_module.api("GET", "/repos/owner/repo/issues")
|
||||
|
||||
|
||||
def test_api_allows_raw_when_expect_json_false(wd_module, monkeypatch):
|
||||
"""expect_json=False returns `{_raw: ...}` for known-quirky endpoints
|
||||
per `feedback_gitea_create_api_unparseable_response`. Opt-in."""
|
||||
|
||||
class FakeResp:
|
||||
status = 201
|
||||
|
||||
def read(self):
|
||||
return b"not-json-but-created\n"
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def fake_urlopen(req, timeout=30):
|
||||
return FakeResp()
|
||||
|
||||
monkeypatch.setattr(wd_module.urllib.request, "urlopen", fake_urlopen)
|
||||
status, body = wd_module.api(
|
||||
"POST", "/repos/owner/repo/issues", expect_json=False,
|
||||
)
|
||||
assert status == 201
|
||||
assert "_raw" in body
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# --dry-run flag — no side effects
|
||||
# --------------------------------------------------------------------------
|
||||
def test_dry_run_skips_writes(wd_module, monkeypatch, capsys):
|
||||
"""--dry-run: detector runs, would-be title/body printed, but no
|
||||
POST/PATCH/comment calls are issued."""
|
||||
failed_ctx = [
|
||||
{"context": "ci/test", "state": "failure",
|
||||
"target_url": "https://x/y", "description": "boom"},
|
||||
]
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (200, _branches_response(SHA_RED)),
|
||||
("GET", f"/repos/owner/repo/commits/{SHA_RED}/status"): (
|
||||
200, _combined_status("failure", failed_ctx),
|
||||
),
|
||||
("GET", "/repos/owner/repo/issues"): (200, []),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
|
||||
wd_module.run_once(dry_run=True)
|
||||
|
||||
methods = [c[0] for c in stub.calls]
|
||||
assert "POST" not in methods, f"dry-run made writes: {stub.calls}"
|
||||
assert "PATCH" not in methods, f"dry-run made writes: {stub.calls}"
|
||||
captured = capsys.readouterr()
|
||||
assert "[dry-run]" in captured.out
|
||||
assert "[main-red]" in captured.out # title rendered
|
||||
|
||||
|
||||
def test_dry_run_flag_parsed(wd_module):
|
||||
"""--dry-run wired into argparse."""
|
||||
ns = wd_module._parse_args(["--dry-run"])
|
||||
assert ns.dry_run is True
|
||||
ns = wd_module._parse_args([])
|
||||
assert ns.dry_run is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Title format
|
||||
# --------------------------------------------------------------------------
|
||||
def test_title_format_uses_short_sha(wd_module):
|
||||
"""Title is `[main-red] {repo}: {SHA[:10]}` — stable idempotency key."""
|
||||
t = wd_module.title_for(SHA_RED)
|
||||
assert t == f"[main-red] owner/repo: {SHA_RED[:10]}"
|
||||
# exactly 10 chars of SHA
|
||||
assert SHA_RED[:10] in t
|
||||
assert SHA_RED[:11] not in t
|
||||
|
||||
|
||||
def test_list_open_red_issues_filters_by_prefix(wd_module, monkeypatch):
|
||||
"""list_open_red_issues only returns issues whose title starts with
|
||||
the expected prefix — unrelated open issues are not touched."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/issues"): (200, [
|
||||
{"number": 1, "title": f"[main-red] owner/repo: {SHA_RED[:10]}"},
|
||||
{"number": 2, "title": "Some unrelated bug"},
|
||||
{"number": 3, "title": "[ci-drift] owner/repo: divergence"},
|
||||
{"number": 4, "title": f"[main-red] owner/repo: {SHA_GREEN[:10]}"},
|
||||
]),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
out = wd_module.list_open_red_issues()
|
||||
assert [i["number"] for i in out] == [1, 4]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# get_head_sha / get_combined_status data-shape guards
|
||||
# --------------------------------------------------------------------------
|
||||
def test_get_head_sha_raises_on_malformed_response(wd_module, monkeypatch):
|
||||
"""If Gitea returns a body without `commit.id`, raise ApiError —
|
||||
do NOT proceed to file an issue with a bogus SHA."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (
|
||||
200, {"name": "main"}, # no commit object
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
with pytest.raises(wd_module.ApiError):
|
||||
wd_module.get_head_sha("main")
|
||||
|
||||
|
||||
def test_get_head_sha_accepts_sha_field(wd_module, monkeypatch):
|
||||
"""Older Gitea versions may return `commit.sha` instead of `commit.id`.
|
||||
Accept either — the watchdog must be tolerant to a documented shape
|
||||
variance."""
|
||||
stub = _make_stub_api({
|
||||
("GET", "/repos/owner/repo/branches/main"): (
|
||||
200, {"name": "main", "commit": {"sha": SHA_RED}},
|
||||
),
|
||||
})
|
||||
monkeypatch.setattr(wd_module, "api", stub)
|
||||
assert wd_module.get_head_sha("main") == SHA_RED
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Loki event emitter (best-effort, must not raise)
|
||||
# --------------------------------------------------------------------------
|
||||
def test_emit_loki_event_prints_json_line(wd_module, capsys, monkeypatch):
|
||||
"""emit_loki_event always prints a JSON line to stdout (for workflow
|
||||
log capture) regardless of whether `logger` is installed."""
|
||||
# Force logger-not-found path to make the test deterministic.
|
||||
monkeypatch.setattr(wd_module.shutil, "which", lambda name: None)
|
||||
wd_module.emit_loki_event("main_red_detected", SHA_RED, ["ci/test"])
|
||||
captured = capsys.readouterr()
|
||||
assert "main-red-watchdog event:" in captured.out
|
||||
# Find the JSON payload after the prefix and verify it parses
|
||||
line = [l for l in captured.out.splitlines() if "main-red-watchdog event:" in l][0]
|
||||
payload = json.loads(line.split("main-red-watchdog event:", 1)[1].strip())
|
||||
assert payload["event_type"] == "main_red_detected"
|
||||
assert payload["repo"] == "owner/repo"
|
||||
assert payload["sha"] == SHA_RED
|
||||
assert payload["failed_contexts"] == ["ci/test"]
|
||||
|
||||
|
||||
def test_emit_loki_event_survives_logger_failure(wd_module, monkeypatch, capsys):
|
||||
"""If `logger` is present but the subprocess call raises, the event
|
||||
emitter must NOT raise — emission is best-effort by contract."""
|
||||
monkeypatch.setattr(wd_module.shutil, "which", lambda name: "/usr/bin/logger")
|
||||
|
||||
def boom(*a, **kw):
|
||||
raise OSError("logger pipe failed")
|
||||
monkeypatch.setattr(wd_module.subprocess, "run", boom)
|
||||
|
||||
# Must not raise:
|
||||
wd_module.emit_loki_event("main_red_detected", SHA_RED, ["ci/test"])
|
||||
captured = capsys.readouterr()
|
||||
assert "logger call failed" in captured.err
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Runtime env guard
|
||||
# --------------------------------------------------------------------------
|
||||
def test_require_runtime_env_exits_when_missing(wd_module, monkeypatch):
|
||||
"""_require_runtime_env() exits with code 2 when any required env
|
||||
var is missing. Caught at main() entry, before any side-effecting
|
||||
API call."""
|
||||
monkeypatch.delenv("GITEA_TOKEN", raising=False)
|
||||
with pytest.raises(SystemExit) as excinfo:
|
||||
wd_module._require_runtime_env()
|
||||
assert excinfo.value.code == 2
|
||||
@@ -645,9 +645,6 @@ func (h *DelegationHandler) ListDelegations(c *gin.Context) {
|
||||
}
|
||||
delegations = append(delegations, entry)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("ListDelegations rows.Err: %v", err)
|
||||
}
|
||||
|
||||
if delegations == nil {
|
||||
delegations = []map[string]interface{}{}
|
||||
|
||||
@@ -800,10 +800,6 @@ func (h *OrgHandler) Import(c *gin.Context) {
|
||||
orphanIDs = append(orphanIDs, orphanID)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
log.Printf("Org import reconcile: orphan query rows.Err: %v", err)
|
||||
reconcileErrs = append(reconcileErrs, fmt.Sprintf("orphan query rows.Err: %v", err))
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
for _, oid := range orphanIDs {
|
||||
|
||||
@@ -91,10 +91,6 @@ func expandWithEnv(s string, env map[string]string) string {
|
||||
// loadWorkspaceEnv reads the org root .env and the workspace-specific .env
|
||||
// (workspace overrides org root). Used by both secret injection and channel
|
||||
// config expansion.
|
||||
//
|
||||
// SECURITY: filesDir is sourced from untrusted org YAML input (ws.FilesDir).
|
||||
// resolveInsideRoot guard prevents path traversal (CWE-22) where a malicious
|
||||
// filesDir like "../../../etc" could escape the org root.
|
||||
func loadWorkspaceEnv(orgBaseDir, filesDir string) map[string]string {
|
||||
envVars := map[string]string{}
|
||||
if orgBaseDir == "" {
|
||||
@@ -102,14 +98,7 @@ func loadWorkspaceEnv(orgBaseDir, filesDir string) map[string]string {
|
||||
}
|
||||
parseEnvFile(filepath.Join(orgBaseDir, ".env"), envVars)
|
||||
if filesDir != "" {
|
||||
safeFilesDir, err := resolveInsideRoot(orgBaseDir, filesDir)
|
||||
if err != nil {
|
||||
// Reject traversal attempt silently — callers expect an empty map
|
||||
// on any read failure.
|
||||
log.Printf("loadWorkspaceEnv: rejecting filesDir %q: %v", filesDir, err)
|
||||
return envVars
|
||||
}
|
||||
parseEnvFile(filepath.Join(safeFilesDir, ".env"), envVars)
|
||||
parseEnvFile(filepath.Join(orgBaseDir, filesDir, ".env"), envVars)
|
||||
}
|
||||
return envVars
|
||||
}
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestLoadWorkspaceEnv_RejectsTraversal asserts that loadWorkspaceEnv refuses
|
||||
// to read workspace-specific .env files when filesDir contains CWE-22 traversal
|
||||
// patterns (../../../etc, absolute paths, etc.). This is the primary security
|
||||
// control for the ws.FilesDir attack surface in POST /org/import.
|
||||
|
||||
func TestLoadWorkspaceEnv_RejectsTraversal(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
orgRoot := filepath.Join(tmp, "my-org")
|
||||
if err := os.Mkdir(orgRoot, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
filesDir string
|
||||
}{
|
||||
{"traversal_parent", "../../../etc"},
|
||||
{"traversal_deep", "../../../../../../../../../etc"},
|
||||
{"traversal_sibling", "../sibling"},
|
||||
{"traversal_mixed", "foo/../../bar"},
|
||||
{"absolute_path", "/etc/passwd"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// Write an org-level .env to confirm it loads even when the
|
||||
// workspace .env is rejected.
|
||||
orgEnv := filepath.Join(orgRoot, ".env")
|
||||
if err := os.WriteFile(orgEnv, []byte("ORG_KEY=org-value\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := loadWorkspaceEnv(orgRoot, tc.filesDir)
|
||||
|
||||
// Org-level .env must be loaded regardless of workspace rejection.
|
||||
if got["ORG_KEY"] != "org-value" {
|
||||
t.Errorf("org-level .env not loaded: got %v", got)
|
||||
}
|
||||
// Traversal path must NOT have been read.
|
||||
if val, ok := got["TRAVERSAL_KEY"]; ok {
|
||||
t.Errorf("traversal escaped: got TRAVERSAL_KEY=%q", val)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadWorkspaceEnv_HappyPath verifies that legitimate filesDir values
|
||||
// resolve correctly and workspace .env overrides org-level values.
|
||||
|
||||
func TestLoadWorkspaceEnv_HappyPath(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
orgRoot := filepath.Join(tmp, "my-org")
|
||||
wsDir := filepath.Join(orgRoot, "workspaces", "dev-workspace")
|
||||
if err := os.MkdirAll(wsDir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
orgEnv := filepath.Join(orgRoot, ".env")
|
||||
wsEnv := filepath.Join(wsDir, ".env")
|
||||
if err := os.WriteFile(orgEnv, []byte("ORG_KEY=org-val\nSHARED=org-wins\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(wsEnv, []byte("WS_KEY=ws-val\nSHARED=ws-wins\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := loadWorkspaceEnv(orgRoot, filepath.Join("workspaces", "dev-workspace"))
|
||||
|
||||
if got["ORG_KEY"] != "org-val" {
|
||||
t.Errorf("org-level key missing: %v", got)
|
||||
}
|
||||
if got["WS_KEY"] != "ws-val" {
|
||||
t.Errorf("workspace key missing: %v", got)
|
||||
}
|
||||
if got["SHARED"] != "ws-wins" {
|
||||
t.Errorf("workspace should override org-level: got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoadWorkspaceEnv_EmptyFilesDirOnlyLoadsOrgLevel verifies that an empty
|
||||
// filesDir only loads the org-level .env (no workspace override).
|
||||
|
||||
func TestLoadWorkspaceEnv_EmptyFilesDir(t *testing.T) {
|
||||
tmp := t.TempDir()
|
||||
orgRoot := filepath.Join(tmp, "my-org")
|
||||
if err := os.Mkdir(orgRoot, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(orgRoot, ".env"), []byte("KEY=only-org\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := loadWorkspaceEnv(orgRoot, "")
|
||||
if got["KEY"] != "only-org" {
|
||||
t.Errorf("expected only-org, got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -490,13 +490,8 @@ func (h *OrgHandler) createWorkspaceTree(ws OrgWorkspace, parentID *string, absX
|
||||
// 1. Org root .env (shared defaults)
|
||||
parseEnvFile(filepath.Join(orgBaseDir, ".env"), envVars)
|
||||
// 2. Workspace-specific .env (overrides)
|
||||
// SECURITY: ws.FilesDir is untrusted YAML input — guard against CWE-22
|
||||
// traversal so a crafted filesDir like "../../../etc" cannot escape orgBaseDir.
|
||||
if ws.FilesDir != "" {
|
||||
if safeFilesDir, err := resolveInsideRoot(orgBaseDir, ws.FilesDir); err == nil {
|
||||
parseEnvFile(filepath.Join(safeFilesDir, ".env"), envVars)
|
||||
}
|
||||
// Traversal rejection: silently skip — callers expect partial env on failure.
|
||||
parseEnvFile(filepath.Join(orgBaseDir, ws.FilesDir, ".env"), envVars)
|
||||
}
|
||||
}
|
||||
// Store as workspace secrets via DB (encrypted if key is set, raw otherwise)
|
||||
|
||||
@@ -167,19 +167,12 @@ async def _delegate_sync_via_polling(
|
||||
break
|
||||
if terminal:
|
||||
if (terminal.get("status") or "").lower() == "completed":
|
||||
# OFFSEC-003: sanitize response_preview before returning so
|
||||
# boundary markers injected by a malicious peer cannot escape
|
||||
# the trust boundary.
|
||||
return sanitize_a2a_result(terminal.get("response_preview") or "")
|
||||
# OFFSEC-003: sanitize error_detail / summary before wrapping with
|
||||
# the _A2A_ERROR_PREFIX sentinel so injected markers cannot appear
|
||||
# inside the trusted error block returned to the agent.
|
||||
err_raw = (
|
||||
return terminal.get("response_preview") or ""
|
||||
err = (
|
||||
terminal.get("error_detail")
|
||||
or terminal.get("summary")
|
||||
or "delegation failed"
|
||||
)
|
||||
err = sanitize_a2a_result(err_raw)
|
||||
return f"{_A2A_ERROR_PREFIX}{err}"
|
||||
|
||||
await asyncio.sleep(_SYNC_POLL_INTERVAL_S)
|
||||
@@ -415,11 +408,12 @@ async def tool_check_task_status(
|
||||
# Filter by delegation_id
|
||||
matching = [d for d in delegations if d.get("delegation_id") == task_id]
|
||||
if matching:
|
||||
# OFFSEC-003: sanitize peer-supplied fields
|
||||
d = matching[0]
|
||||
d["summary"] = sanitize_a2a_result(d.get("summary", ""))
|
||||
d["response_preview"] = sanitize_a2a_result(d.get("response_preview", ""))
|
||||
return json.dumps(d)
|
||||
entry = dict(matching[0])
|
||||
# OFFSEC-003: sanitize peer-generated text fields
|
||||
for field in ("result", "response_preview"):
|
||||
if field in entry and entry[field]:
|
||||
entry[field] = sanitize_a2a_result(str(entry[field]))
|
||||
return json.dumps(entry)
|
||||
return json.dumps({"status": "not_found", "delegation_id": task_id})
|
||||
# Return all recent delegations
|
||||
summary = []
|
||||
@@ -431,7 +425,7 @@ async def tool_check_task_status(
|
||||
"delegation_id": d.get("delegation_id", ""),
|
||||
"target_id": d.get("target_id", ""),
|
||||
"status": d.get("status", ""),
|
||||
"summary": sanitize_a2a_result(d.get("summary", "")),
|
||||
"summary": d.get("summary", ""),
|
||||
"response_preview": preview,
|
||||
})
|
||||
return json.dumps({"delegations": summary, "count": len(delegations)})
|
||||
|
||||
@@ -77,8 +77,6 @@ async def delegate_task(workspace_id: str, task: str) -> str:
|
||||
return str(result) if isinstance(result, str) else "(no text)"
|
||||
elif "error" in data:
|
||||
err = data["error"]
|
||||
# Handle both string-form errors ("error": "some string")
|
||||
# and object-form errors ("error": {"message": "...", "code": ...}).
|
||||
msg = ""
|
||||
if isinstance(err, dict):
|
||||
msg = err.get("message", "")
|
||||
|
||||
@@ -668,31 +668,6 @@ async def main(): # pragma: no cover
|
||||
if heartbeat.active_tasks > 0:
|
||||
continue
|
||||
|
||||
# Issue #381 fix: skip the idle prompt if there are unconsumed
|
||||
# delegation results waiting. The heartbeat sends a self-message
|
||||
# for every new result batch, so sending the idle prompt here would
|
||||
# race: the agent would compose a stale tick BEFORE processing the
|
||||
# results notification, producing repeated identical asks (peer sends
|
||||
# correction, we respond with stale state, peer asks again).
|
||||
# By skipping the idle prompt when results are pending, we let the
|
||||
# heartbeat's own self-message wake the agent after results are
|
||||
# written. The agent then sees the results in _prepare_prompt()
|
||||
# and processes them before composing.
|
||||
from heartbeat import DELEGATION_RESULTS_FILE as _DRF
|
||||
try:
|
||||
with open(_DRF) as _rf:
|
||||
_rf.seek(0)
|
||||
_content = _rf.read().strip()
|
||||
if _content:
|
||||
print(
|
||||
f"Idle loop: skipping — {len(_content)} bytes of unconsumed "
|
||||
f"delegation results pending (heartbeat will notify agent)",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
pass # No results file — normal, proceed with idle prompt
|
||||
|
||||
# Self-post the idle prompt via the platform A2A proxy (same
|
||||
# path as initial_prompt). The agent's own concurrency control
|
||||
# rejects if the workspace becomes busy between this check and
|
||||
|
||||
@@ -326,105 +326,6 @@ class TestToolDelegateTask:
|
||||
assert a2a_tools._peer_names.get("ws-nona000") is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# delegate_task (non-tool, direct httpx path — used by adapter templates)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDelegateTaskDirect:
|
||||
|
||||
async def test_string_form_error_returns_error_message(self):
|
||||
"""The A2A proxy can return {"error": "plain string"}. Must not raise
|
||||
AttributeError: 'str' object has no attribute 'get'."""
|
||||
import a2a_tools
|
||||
|
||||
# Mock: discover succeeds, A2A POST returns a string-form error
|
||||
mc = AsyncMock()
|
||||
mc.__aenter__ = AsyncMock(return_value=mc)
|
||||
mc.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
async def fake_post(url, **kwargs):
|
||||
r = MagicMock()
|
||||
r.status_code = 200
|
||||
r.json = MagicMock(return_value={"error": "peer workspace unreachable"})
|
||||
return r
|
||||
|
||||
async def fake_get(url, **kwargs):
|
||||
r = MagicMock()
|
||||
r.status_code = 200
|
||||
r.json = MagicMock(return_value={"url": "http://peer.svc/a2a"})
|
||||
return r
|
||||
|
||||
mc.post = fake_post
|
||||
mc.get = fake_get
|
||||
|
||||
with patch("a2a_tools.httpx.AsyncClient", return_value=mc):
|
||||
result = await a2a_tools.delegate_task("ws-peer-123", "do a thing")
|
||||
|
||||
assert "Error" in result
|
||||
assert "peer workspace unreachable" in result
|
||||
|
||||
async def test_dict_form_error_returns_error_message(self):
|
||||
"""{"error": {"message": "...", "code": ...}} — the pre-existing path."""
|
||||
import a2a_tools
|
||||
|
||||
mc = AsyncMock()
|
||||
mc.__aenter__ = AsyncMock(return_value=mc)
|
||||
mc.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
async def fake_post(url, **kwargs):
|
||||
r = MagicMock()
|
||||
r.status_code = 200
|
||||
r.json = MagicMock(return_value={"error": {"message": "internal server error", "code": 500}})
|
||||
return r
|
||||
|
||||
async def fake_get(url, **kwargs):
|
||||
r = MagicMock()
|
||||
r.status_code = 200
|
||||
r.json = MagicMock(return_value={"url": "http://peer.svc/a2a"})
|
||||
return r
|
||||
|
||||
mc.post = fake_post
|
||||
mc.get = fake_get
|
||||
|
||||
with patch("a2a_tools.httpx.AsyncClient", return_value=mc):
|
||||
result = await a2a_tools.delegate_task("ws-peer-456", "do a thing")
|
||||
|
||||
assert "Error" in result
|
||||
assert "internal server error" in result
|
||||
|
||||
async def test_success_returns_result_text(self):
|
||||
"""Happy path: result with parts returns the first text part."""
|
||||
import a2a_tools
|
||||
|
||||
mc = AsyncMock()
|
||||
mc.__aenter__ = AsyncMock(return_value=mc)
|
||||
mc.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
async def fake_post(url, **kwargs):
|
||||
r = MagicMock()
|
||||
r.status_code = 200
|
||||
r.json = MagicMock(return_value={
|
||||
"result": {
|
||||
"parts": [{"kind": "text", "text": "Task done!"}]
|
||||
}
|
||||
})
|
||||
return r
|
||||
|
||||
async def fake_get(url, **kwargs):
|
||||
r = MagicMock()
|
||||
r.status_code = 200
|
||||
r.json = MagicMock(return_value={"url": "http://peer.svc/a2a"})
|
||||
return r
|
||||
|
||||
mc.post = fake_post
|
||||
mc.get = fake_get
|
||||
|
||||
with patch("a2a_tools.httpx.AsyncClient", return_value=mc):
|
||||
result = await a2a_tools.delegate_task("ws-peer-789", "do a thing")
|
||||
|
||||
assert result == "Task done!"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tool_delegate_task_async
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,80 +0,0 @@
|
||||
"""Tests for issue #381: idle loop must not fire when delegation results are pending.
|
||||
|
||||
The idle loop skips sending the idle prompt when DELEGATION_RESULTS_FILE
|
||||
contains unconsumed results, preventing the agent from composing a stale tick
|
||||
before processing pending delegation notifications from the heartbeat.
|
||||
|
||||
Source: workspace/main.py:_run_idle_loop() pending-results guard.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def check_results_pending(file_path: str) -> bool:
|
||||
"""Mirror the guard logic from workspace/main.py:_run_idle_loop().
|
||||
|
||||
Returns True if the results file exists and is non-empty,
|
||||
meaning the idle loop should skip this tick.
|
||||
"""
|
||||
try:
|
||||
with open(file_path) as rf:
|
||||
rf.seek(0)
|
||||
content = rf.read().strip()
|
||||
return bool(content)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
class TestIdleLoopPendingCheck:
|
||||
"""Tests for the idle-loop pending-delegation-results guard."""
|
||||
|
||||
def test_no_file_means_proceed(self, tmp_path):
|
||||
"""No delegation results file → idle loop fires normally."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
assert not check_results_pending(str(results_file))
|
||||
|
||||
def test_empty_file_means_proceed(self, tmp_path):
|
||||
"""Empty file → no pending results → idle loop fires."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text("", encoding="utf-8")
|
||||
assert not check_results_pending(str(results_file))
|
||||
|
||||
def test_whitespace_only_file_means_proceed(self, tmp_path):
|
||||
"""File with only whitespace → treated as empty → idle loop fires."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text(" \n ", encoding="utf-8")
|
||||
assert not check_results_pending(str(results_file))
|
||||
|
||||
def test_single_result_means_skip(self, tmp_path):
|
||||
"""File with one delegation result → skip idle tick."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text(
|
||||
json.dumps({
|
||||
"status": "completed",
|
||||
"delegation_id": "del-abc",
|
||||
"summary": "Done",
|
||||
}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert check_results_pending(str(results_file))
|
||||
|
||||
def test_multiple_results_means_skip(self, tmp_path):
|
||||
"""File with multiple delegation results → skip idle tick."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text(
|
||||
json.dumps({"status": "completed", "delegation_id": "del-1", "summary": "A"})
|
||||
+ "\n"
|
||||
+ json.dumps({"status": "failed", "delegation_id": "del-2", "summary": "B"})
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert check_results_pending(str(results_file))
|
||||
|
||||
def test_file_with_only_newline_means_proceed(self, tmp_path):
|
||||
"""File with only a newline character → stripped to empty → fires."""
|
||||
results_file = tmp_path / "delegation_results.jsonl"
|
||||
results_file.write_text("\n", encoding="utf-8")
|
||||
assert not check_results_pending(str(results_file))
|
||||
Reference in New Issue
Block a user