Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 74297485c0 | |||
| c53ca971c3 | |||
| 8e8334795e | |||
| f3bd81244e | |||
| 9e53a426ef | |||
| e3cc0adcd5 | |||
| 6ef7aa7361 | |||
| 6cfb0bf5bf | |||
| 0036d94ec2 |
@@ -1 +0,0 @@
|
||||
CI re-trigger at Tue Apr 21 15:40:21 UTC 2026\n
|
||||
@@ -1,41 +0,0 @@
|
||||
# Coverage allowlist — security-critical files that are currently below
|
||||
# the 10% per-file floor and are being tracked for remediation.
|
||||
#
|
||||
# Format: one path per line, relative to workspace-server/.
|
||||
# Lines starting with # and blank lines are ignored.
|
||||
#
|
||||
# Process:
|
||||
# - A path in this list is WARNED on each CI run, not failed.
|
||||
# - Each entry must reference a tracking issue and expiry date.
|
||||
# - On expiry, either the coverage is fixed OR the path graduates to
|
||||
# hard-fail (revert the allowlist entry).
|
||||
#
|
||||
# See #1823 for the gate design and ratchet plan.
|
||||
|
||||
# ============== Active exceptions ==============
|
||||
|
||||
# Filed 2026-04-23 — expiry 2026-05-23 (30 days). Tracking: #1823.
|
||||
# These are the files flagged by the first run of the critical-path gate.
|
||||
# QA team + platform team share ownership of test coverage remediation.
|
||||
|
||||
internal/handlers/a2a_proxy.go
|
||||
internal/handlers/a2a_proxy_helpers.go
|
||||
internal/handlers/registry.go
|
||||
internal/handlers/secrets.go
|
||||
internal/handlers/tokens.go
|
||||
internal/handlers/workspace_provision.go
|
||||
internal/middleware/wsauth_middleware.go
|
||||
|
||||
# The following paths matched via looser CRITICAL_PATH substrings
|
||||
# (e.g. "registry" matched both internal/registry/ and internal/channels/registry.go).
|
||||
# Adding them here so the gate can land without blocking staging merges;
|
||||
# a follow-up PR will tighten CRITICAL_PATHS to exact prefixes so these
|
||||
# graduate to hard-fail precisely where security-critical.
|
||||
|
||||
internal/channels/registry.go
|
||||
internal/crypto/aes.go
|
||||
internal/registry/access.go
|
||||
internal/registry/healthsweep.go
|
||||
internal/registry/hibernation.go
|
||||
internal/registry/provisiontimeout.go
|
||||
internal/wsauth/tokens.go
|
||||
+6
-31
@@ -1,23 +1,13 @@
|
||||
# Postgres
|
||||
# These defaults match docker-compose.infra.yml, which is the stack
|
||||
# launched by `./infra/scripts/setup.sh`. Override for production.
|
||||
POSTGRES_USER=dev
|
||||
POSTGRES_PASSWORD=dev
|
||||
POSTGRES_USER=
|
||||
POSTGRES_PASSWORD=
|
||||
POSTGRES_DB=molecule
|
||||
# DATABASE_URL points at the host-published Postgres port so that
|
||||
# `go run ./cmd/server` on the host (the README quickstart path) can
|
||||
# connect. When running the platform *inside* docker-compose.yml, the
|
||||
# compose file builds a DATABASE_URL with host `postgres` automatically
|
||||
# from POSTGRES_USER/PASSWORD/DB above — that path ignores this value.
|
||||
DATABASE_URL=postgres://dev:dev@localhost:5432/molecule?sslmode=disable
|
||||
DATABASE_URL=postgres://USER:PASS@postgres:5432/molecule?sslmode=disable
|
||||
|
||||
# Redis — same host-vs-container story as DATABASE_URL above.
|
||||
REDIS_URL=redis://localhost:6379
|
||||
# Redis
|
||||
REDIS_URL=redis://redis:6379
|
||||
|
||||
# Platform
|
||||
# PORT only applies to the Go platform (workspace-server). The Canvas pins
|
||||
# itself to 3000 in canvas/package.json, so sourcing this file before
|
||||
# `npm run dev` won't accidentally make Next.js try to bind 8080.
|
||||
PORT=8080
|
||||
# ---- Admin credential — REQUIRED to close issue #684 (AdminAuth bearer bypass) ----
|
||||
# When ADMIN_TOKEN is set, only this value is accepted on /admin/* and /approvals/* routes.
|
||||
@@ -34,7 +24,7 @@ PLUGINS_DIR= # Path to plugins/ directory (default: /plugins i
|
||||
# MOLECULE_MCP_ALLOW_SEND_MESSAGE= # Set to "true" to include send_message_to_user in the MCP bridge tool list (issue #810). Excluded by default to prevent unintended WebSocket pushes from CLI sessions.
|
||||
# MOLECULE_MCP_URL=http://localhost:8080 # Platform URL for opencode MCP config (opencode.json). Same as PLATFORM_URL; separate var so opencode configs can reference it without ambiguity.
|
||||
# WORKSPACE_DIR= # Optional global host path bind-mounted to /workspace in every container. Per-workspace workspace_dir column overrides this; if neither is set each workspace gets an isolated Docker named volume.
|
||||
MOLECULE_ENV=development # Environment label (development/staging/production). Used for log tagging and for the AdminAuth dev-mode escape hatch (lets the Canvas dashboard keep working after the first workspace is created, when ADMIN_TOKEN is unset). SaaS deployments MUST set MOLECULE_ENV=production.
|
||||
# MOLECULE_ENV=development # Environment label (development/staging/production). Used for log tagging and conditional behaviour.
|
||||
# MOLECULE_ENABLE_TEST_TOKENS= # Set to 1 to expose GET /admin/workspaces/:id/test-token (mints a fresh bearer token for E2E scripts). The route is auto-enabled when MOLECULE_ENV != production; this flag is the explicit override. Leave unset/0 in prod — the route 404s unless enabled.
|
||||
# MOLECULE_ORG_ID= # SaaS only: org UUID set by control plane on tenant machines. When set, workspace provisioning auto-routes through the control plane API instead of Docker.
|
||||
# CP_PROVISION_URL= # Override control plane URL for workspace provisioning (default: https://api.moleculesai.app). Only needed for testing against a non-production control plane.
|
||||
@@ -168,18 +158,3 @@ GSC_SERVICE_ACCOUNT= # Search Console reporter service account email
|
||||
# Token goes in Authorization: Bearer header — never embed in the URL.
|
||||
MOLECULE_MCP_URL= # e.g. https://api.molecule.ai or http://localhost:8080
|
||||
MOLECULE_MCP_TOKEN= # workspace-scoped bearer token — NEVER COMMIT
|
||||
|
||||
# ---- workspace-template image refresh ----
|
||||
# IMAGE_AUTO_REFRESH=true makes the platform poll GHCR every 5 min for digest
|
||||
# changes on each workspace-template-*:latest. When a digest moves the
|
||||
# platform pulls + force-recreates matching ws-* containers (same code path
|
||||
# as POST /admin/workspace-images/refresh). Closes the runtime CD chain to
|
||||
# zero operator steps.
|
||||
# Default in docker-compose.yml is "true" for local dev so the runtime → ws
|
||||
# loop is tight; explicit override here lets you turn it off when running a
|
||||
# long test that shouldn't be disturbed by a publish.
|
||||
IMAGE_AUTO_REFRESH= # true|false; unset = inherit compose default (true for local dev)
|
||||
# GHCR_USER + GHCR_TOKEN are required only for private template images
|
||||
# (current workspace-template-* set is public; both can stay unset).
|
||||
GHCR_USER=
|
||||
GHCR_TOKEN=
|
||||
|
||||
@@ -13,11 +13,3 @@ workspace/entrypoint.sh text eol=lf
|
||||
# but keep LF for consistency across platforms.
|
||||
Dockerfile text eol=lf
|
||||
*.dockerfile text eol=lf
|
||||
|
||||
# Snapshot golden files — workspace/tests/snapshots/*.txt is consumed by
|
||||
# byte-exact comparisons in test_platform_tools.py. A Windows contributor
|
||||
# with auto-CRLF=true would otherwise convert \n → \r\n on checkout, the
|
||||
# snapshot tests would fail mysteriously locally / pass in CI (or vice
|
||||
# versa), and the regen instructions in the test-file header would
|
||||
# produce LF files that disagree with the working-copy CRLF versions.
|
||||
workspace/tests/snapshots/*.txt text eol=lf
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# audit-force-merge — detect a §SOP-6 force-merge after PR close, emit
|
||||
# `incident.force_merge` to stdout as structured JSON.
|
||||
#
|
||||
# Vector's docker_logs source picks up runner stdout; the JSON gets
|
||||
# shipped to Loki on molecule-canonical-obs, indexable by event_type.
|
||||
# Query example:
|
||||
#
|
||||
# {host="operator"} |= "event_type" |= "incident.force_merge" | json
|
||||
#
|
||||
# A force-merge is detected when a PR closed-with-merged=true had at
|
||||
# least one of the repo's required-status-check contexts in a state
|
||||
# other than "success" at the merge commit's SHA. That's exactly what
|
||||
# the Gitea force_merge:true API call lets through, so it's a faithful
|
||||
# detector of the override path.
|
||||
#
|
||||
# Triggers on `pull_request_target: closed` (loaded from base branch
|
||||
# per §SOP-6 security model). No-op when merged=false.
|
||||
#
|
||||
# Required env (set by the workflow):
|
||||
# GITEA_TOKEN, GITEA_HOST, REPO, PR_NUMBER, REQUIRED_CHECKS
|
||||
#
|
||||
# REQUIRED_CHECKS is a newline-separated list of status-check context
|
||||
# names that branch protection requires. Declared in the workflow YAML
|
||||
# rather than fetched from /branch_protections (which needs admin
|
||||
# scope — sop-tier-bot has read-only). Trade dynamism for simplicity:
|
||||
# when the required-check set changes, update both branch protection
|
||||
# AND this env. Keeping them in sync is less complexity than granting
|
||||
# the audit bot admin perms on every repo.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${GITEA_TOKEN:?required}"
|
||||
: "${GITEA_HOST:?required}"
|
||||
: "${REPO:?required}"
|
||||
: "${PR_NUMBER:?required}"
|
||||
: "${REQUIRED_CHECKS:?required (newline-separated context names)}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
API="https://${GITEA_HOST}/api/v1"
|
||||
AUTH="Authorization: token ${GITEA_TOKEN}"
|
||||
|
||||
# 1. Fetch the PR. If not merged, no-op.
|
||||
PR=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}")
|
||||
MERGED=$(echo "$PR" | jq -r '.merged // false')
|
||||
if [ "$MERGED" != "true" ]; then
|
||||
echo "::notice::PR #${PR_NUMBER} closed without merge — no audit emission."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
MERGE_SHA=$(echo "$PR" | jq -r '.merge_commit_sha // empty')
|
||||
MERGED_BY=$(echo "$PR" | jq -r '.merged_by.login // "unknown"')
|
||||
TITLE=$(echo "$PR" | jq -r '.title // ""')
|
||||
BASE_BRANCH=$(echo "$PR" | jq -r '.base.ref // "main"')
|
||||
HEAD_SHA=$(echo "$PR" | jq -r '.head.sha // empty')
|
||||
|
||||
if [ -z "$MERGE_SHA" ]; then
|
||||
echo "::warning::PR #${PR_NUMBER} merged=true but no merge_commit_sha — cannot evaluate force-merge."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 2. Required status checks declared in the workflow env.
|
||||
REQUIRED="$REQUIRED_CHECKS"
|
||||
if [ -z "${REQUIRED//[[:space:]]/}" ]; then
|
||||
echo "::notice::REQUIRED_CHECKS empty — force-merge not applicable."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 3. Status-check state at the PR HEAD (where checks ran). The merge
|
||||
# commit doesn't get its own checks; we evaluate the PR's last
|
||||
# commit, which is what branch protection compared against.
|
||||
STATUS=$(curl -sS -H "$AUTH" \
|
||||
"${API}/repos/${OWNER}/${NAME}/commits/${HEAD_SHA}/status")
|
||||
declare -A CHECK_STATE
|
||||
while IFS=$'\t' read -r ctx state; do
|
||||
[ -n "$ctx" ] && CHECK_STATE[$ctx]="$state"
|
||||
done < <(echo "$STATUS" | jq -r '.statuses // [] | .[] | "\(.context)\t\(.status)"')
|
||||
|
||||
# 4. For each required check, was it green at merge? YAML block scalars
|
||||
# (`|`) leave a trailing newline; skip blank/whitespace-only lines.
|
||||
FAILED_CHECKS=()
|
||||
while IFS= read -r req; do
|
||||
trimmed="${req#"${req%%[![:space:]]*}"}" # ltrim
|
||||
trimmed="${trimmed%"${trimmed##*[![:space:]]}"}" # rtrim
|
||||
[ -z "$trimmed" ] && continue
|
||||
state="${CHECK_STATE[$trimmed]:-missing}"
|
||||
if [ "$state" != "success" ]; then
|
||||
FAILED_CHECKS+=("${trimmed}=${state}")
|
||||
fi
|
||||
done <<< "$REQUIRED"
|
||||
|
||||
if [ "${#FAILED_CHECKS[@]}" -eq 0 ]; then
|
||||
echo "::notice::PR #${PR_NUMBER} merged with all required checks green — not a force-merge."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 5. Emit structured audit event.
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
FAILED_JSON=$(printf '%s\n' "${FAILED_CHECKS[@]}" | jq -R . | jq -s .)
|
||||
|
||||
# Print as a single-line JSON so Vector's parse_json transform can pick
|
||||
# it up cleanly from docker_logs.
|
||||
jq -nc \
|
||||
--arg event_type "incident.force_merge" \
|
||||
--arg ts "$NOW" \
|
||||
--arg repo "$REPO" \
|
||||
--argjson pr "$PR_NUMBER" \
|
||||
--arg title "$TITLE" \
|
||||
--arg base "$BASE_BRANCH" \
|
||||
--arg merged_by "$MERGED_BY" \
|
||||
--arg merge_sha "$MERGE_SHA" \
|
||||
--argjson failed_checks "$FAILED_JSON" \
|
||||
'{event_type: $event_type, ts: $ts, repo: $repo, pr: $pr, title: $title,
|
||||
base_branch: $base, merged_by: $merged_by, merge_sha: $merge_sha,
|
||||
failed_checks: $failed_checks}'
|
||||
|
||||
echo "::warning::FORCE-MERGE detected on PR #${PR_NUMBER} by ${MERGED_BY}: ${#FAILED_CHECKS[@]} required check(s) not green at merge time."
|
||||
@@ -1,591 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ci-required-drift — RFC internal#219 §4 + §6.
|
||||
|
||||
Detects drift between three sources of "what counts as a required check"
|
||||
for this repo, files (or updates) a `[ci-drift]` Gitea issue when any
|
||||
pair diverges.
|
||||
|
||||
Sources:
|
||||
A. `.gitea/workflows/ci.yml` jobs (CI source — the actual job set)
|
||||
B. `status_check_contexts` in branch_protections (the merge gate)
|
||||
C. `REQUIRED_CHECKS` env in audit-force-merge.yml (the audit env)
|
||||
|
||||
Three failure classes:
|
||||
F1 Job in (A) is not under the sentinel's `needs:` — sentinel
|
||||
doesn't gate it, so a red job on that name can sneak through.
|
||||
Ignores jobs whose `if:` references `github.event_name` (those
|
||||
run only on specific events and may be `skipped` legitimately).
|
||||
F2 Context in (B) corresponds to no emitter — i.e. there's no job
|
||||
in ci.yml whose runtime status-name maps to that context.
|
||||
A stale required-check name is silent: protection demands a
|
||||
green it never receives, but Gitea treats absent-as-pending,
|
||||
not absent-as-red. The gate degrades to advisory.
|
||||
F3 (B) and (C) are not set-equal. Audit env wider than protection
|
||||
→ audit flags non-force-merges as force; narrower → real
|
||||
force-merges are missed.
|
||||
|
||||
Idempotency:
|
||||
Searches OPEN issues by exact title prefix
|
||||
`[ci-drift] {repo}/{branch}: ` and either edits the existing one
|
||||
(if any) or POSTs a new one. Never spawns duplicates.
|
||||
|
||||
Behavior-based AST gate per `feedback_behavior_based_ast_gates`:
|
||||
- Job set comes from PyYAML parse of jobs:* keys
|
||||
- Sentinel needs from PyYAML parse of jobs[sentinel].needs (a list)
|
||||
- Audit env from PyYAML parse, NOT grep — so reformatting the YAML
|
||||
(block-scalar `|` vs flow-style list) does not break the gate
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
import yaml # PyYAML 6.0.2 — installed by the workflow before this runs.
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment
|
||||
# --------------------------------------------------------------------------
|
||||
def env(key: str, *, required: bool = True, default: str | None = None) -> str:
|
||||
val = os.environ.get(key, default)
|
||||
if required and not val:
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
return val or ""
|
||||
|
||||
|
||||
GITEA_TOKEN = env("GITEA_TOKEN", required=False)
|
||||
GITEA_HOST = env("GITEA_HOST", required=False)
|
||||
REPO = env("REPO", required=False)
|
||||
BRANCHES = env("BRANCHES", required=False).split()
|
||||
SENTINEL_JOB = env("SENTINEL_JOB", required=False)
|
||||
AUDIT_WORKFLOW_PATH = env("AUDIT_WORKFLOW_PATH", required=False)
|
||||
CI_WORKFLOW_PATH = env("CI_WORKFLOW_PATH", required=False)
|
||||
DRIFT_LABEL = env("DRIFT_LABEL", required=False)
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only. Tests import
|
||||
individual functions without setting the full env contract."""
|
||||
for key in (
|
||||
"GITEA_TOKEN",
|
||||
"GITEA_HOST",
|
||||
"REPO",
|
||||
"BRANCHES",
|
||||
"SENTINEL_JOB",
|
||||
"AUDIT_WORKFLOW_PATH",
|
||||
"CI_WORKFLOW_PATH",
|
||||
"DRIFT_LABEL",
|
||||
):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper (no requests dependency)
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API call cannot be trusted to have succeeded.
|
||||
|
||||
Covers non-2xx HTTP status AND 2xx with an unparseable JSON body on
|
||||
endpoints that are documented to return JSON (search/read). Callers
|
||||
that swallow this and proceed would risk e.g. creating duplicate
|
||||
`[ci-drift]` issues when a transient 500 hides an existing match.
|
||||
The cron retries hourly; one fail-loud cycle is fine — silent
|
||||
duplicate creation is not (per Five-Axis review on PR #112).
|
||||
"""
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
"""Tiny HTTP helper around urllib.
|
||||
|
||||
Raises ApiError on any non-2xx response. Callers that want
|
||||
best-effort semantics (e.g. label-apply) must `try/except ApiError`
|
||||
explicitly — making the failure-soft path opt-in rather than the
|
||||
default closes the duplicate-issue regression class.
|
||||
|
||||
For 2xx responses with a JSON body that fails to parse, raises
|
||||
ApiError when `expect_json=True` (the default for read-shaped
|
||||
paths). On endpoints that legitimately return non-JSON success
|
||||
bodies (e.g. some Gitea create echoes — see
|
||||
`feedback_gitea_create_api_unparseable_response`), callers may pass
|
||||
`expect_json=False` to accept a `_raw` fallthrough — but they MUST
|
||||
then verify success via a follow-up GET, not by trusting the body.
|
||||
"""
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status}: {snippet}"
|
||||
)
|
||||
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
if expect_json:
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status} but body is not JSON: {e}"
|
||||
) from e
|
||||
# Opt-in raw fallthrough for endpoints with known echo-quirks.
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# YAML loaders — STRICT (reject GitHub-Actions-only syntax)
|
||||
# --------------------------------------------------------------------------
|
||||
def load_yaml(path: str) -> dict:
|
||||
"""Load + parse a workflow YAML. Hard-fail if the file is missing
|
||||
or doesn't parse — drift-detect cannot make decisions without
|
||||
knowing the actual job set."""
|
||||
if not os.path.exists(path):
|
||||
sys.stderr.write(f"::error::file not found: {path}\n")
|
||||
sys.exit(3)
|
||||
with open(path, encoding="utf-8") as f:
|
||||
try:
|
||||
doc = yaml.safe_load(f)
|
||||
except yaml.YAMLError as e:
|
||||
sys.stderr.write(f"::error::YAML parse error in {path}: {e}\n")
|
||||
sys.exit(3)
|
||||
if not isinstance(doc, dict):
|
||||
sys.stderr.write(f"::error::{path} is not a YAML mapping\n")
|
||||
sys.exit(3)
|
||||
return doc
|
||||
|
||||
|
||||
def ci_jobs_all(ci_doc: dict) -> set[str]:
|
||||
"""Every job key in ci.yml minus the sentinel itself. Used for F1b
|
||||
(sentinel.needs typo check) — needs that name a non-existent job
|
||||
is a typo regardless of event-gating."""
|
||||
jobs = ci_doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
sys.stderr.write("::error::ci.yml has no jobs: mapping\n")
|
||||
sys.exit(3)
|
||||
return {k for k in jobs if k != SENTINEL_JOB}
|
||||
|
||||
|
||||
def ci_job_names(ci_doc: dict) -> set[str]:
|
||||
"""Set of job keys in ci.yml MINUS the sentinel itself MINUS jobs
|
||||
whose `if:` gates on `github.event_name` (those are event-scoped
|
||||
and can legitimately be `skipped` for a given trigger; if we
|
||||
required them under the sentinel `needs:`, every PR-only job
|
||||
would be `skipped` on push and the sentinel would interpret
|
||||
`skipped != success` as failure). RFC §4 spec.
|
||||
|
||||
Used for F1 (jobs missing from sentinel needs). NOT used for F1b
|
||||
(typos in needs) — see `ci_jobs_all` for that."""
|
||||
jobs = ci_doc.get("jobs")
|
||||
if not isinstance(jobs, dict):
|
||||
sys.stderr.write("::error::ci.yml has no jobs: mapping\n")
|
||||
sys.exit(3)
|
||||
names: set[str] = set()
|
||||
for k, v in jobs.items():
|
||||
if k == SENTINEL_JOB:
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
gate = v.get("if")
|
||||
if isinstance(gate, str) and "github.event_name" in gate:
|
||||
continue
|
||||
names.add(k)
|
||||
return names
|
||||
|
||||
|
||||
def sentinel_needs(ci_doc: dict) -> set[str]:
|
||||
sentinel = ci_doc.get("jobs", {}).get(SENTINEL_JOB)
|
||||
if not isinstance(sentinel, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::sentinel job '{SENTINEL_JOB}' not found in {CI_WORKFLOW_PATH}\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
needs = sentinel.get("needs", [])
|
||||
if isinstance(needs, str):
|
||||
needs = [needs]
|
||||
if not isinstance(needs, list):
|
||||
sys.stderr.write("::error::sentinel `needs:` is neither list nor string\n")
|
||||
sys.exit(3)
|
||||
return set(needs)
|
||||
|
||||
|
||||
def required_checks_env(audit_doc: dict) -> set[str]:
|
||||
"""Pull the REQUIRED_CHECKS env value from audit-force-merge.yml.
|
||||
Walks the YAML AST per `feedback_behavior_based_ast_gates`: we do
|
||||
NOT grep for `REQUIRED_CHECKS:` — that breaks under reformatting,
|
||||
multi-job workflows, or a future move of the env to a different
|
||||
step. Instead, look inside every job's every step's `env:` map."""
|
||||
found: list[str] = []
|
||||
jobs = audit_doc.get("jobs", {})
|
||||
if not isinstance(jobs, dict):
|
||||
sys.stderr.write(f"::warning::{AUDIT_WORKFLOW_PATH} has no jobs: mapping\n")
|
||||
return set()
|
||||
for job in jobs.values():
|
||||
if not isinstance(job, dict):
|
||||
continue
|
||||
for step in job.get("steps", []) or []:
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
step_env = step.get("env") or {}
|
||||
if isinstance(step_env, dict) and "REQUIRED_CHECKS" in step_env:
|
||||
v = step_env["REQUIRED_CHECKS"]
|
||||
if isinstance(v, str):
|
||||
found.append(v)
|
||||
if not found:
|
||||
sys.stderr.write(
|
||||
f"::error::REQUIRED_CHECKS env not found in any step of {AUDIT_WORKFLOW_PATH}\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
if len(found) > 1:
|
||||
# Defensive: refuse to guess which one is canonical.
|
||||
sys.stderr.write(
|
||||
f"::error::REQUIRED_CHECKS env present in {len(found)} steps; ambiguous\n"
|
||||
)
|
||||
sys.exit(3)
|
||||
raw = found[0]
|
||||
# YAML block-scalars (`|`) leave a trailing newline + blanks; trim
|
||||
# consistently with audit-force-merge.sh's parser so both sides
|
||||
# produce identical sets.
|
||||
return {line.strip() for line in raw.splitlines() if line.strip()}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Mapping: ci.yml job-key → protection context name
|
||||
# --------------------------------------------------------------------------
|
||||
def expected_context(job_key: str, workflow_name: str = "ci") -> str:
|
||||
"""Gitea Actions reports status-check contexts as
|
||||
"{workflow.name} / {job.name or job.key} ({event})".
|
||||
|
||||
For ci.yml the event is `pull_request` on PRs (that's what
|
||||
`status_check_contexts` records). Job.name defaults to job.key
|
||||
when no `name:` is set. CP's ci.yml does NOT set per-job `name:`
|
||||
so the key equals the human-name."""
|
||||
return f"{workflow_name} / {job_key} (pull_request)"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Drift detection
|
||||
# --------------------------------------------------------------------------
|
||||
def detect_drift(branch: str) -> tuple[list[str], dict]:
|
||||
"""Returns (findings, debug). Empty findings == no drift."""
|
||||
findings: list[str] = []
|
||||
|
||||
ci_doc = load_yaml(CI_WORKFLOW_PATH)
|
||||
audit_doc = load_yaml(AUDIT_WORKFLOW_PATH)
|
||||
|
||||
jobs = ci_job_names(ci_doc)
|
||||
jobs_all = ci_jobs_all(ci_doc)
|
||||
needs = sentinel_needs(ci_doc)
|
||||
env_set = required_checks_env(audit_doc)
|
||||
|
||||
# Protection
|
||||
# api() raises ApiError on non-2xx; let it propagate so a transient
|
||||
# 500 fails the run loudly rather than producing a "no drift" lie.
|
||||
_, protection = api("GET", f"/repos/{OWNER}/{NAME}/branch_protections/{branch}")
|
||||
if not isinstance(protection, dict):
|
||||
sys.stderr.write(
|
||||
f"::error::protection response for {branch} not a JSON object\n"
|
||||
)
|
||||
sys.exit(4)
|
||||
contexts = set(protection.get("status_check_contexts") or [])
|
||||
|
||||
# ----- F1: job exists in CI but not under sentinel.needs -----
|
||||
missing_from_needs = sorted(jobs - needs)
|
||||
if missing_from_needs:
|
||||
findings.append(
|
||||
"F1 — jobs in ci.yml NOT under sentinel `needs:` (sentinel doesn't gate them):\n"
|
||||
+ "\n".join(f" - {n}" for n in missing_from_needs)
|
||||
)
|
||||
|
||||
# ----- F1b: needs lists a job that doesn't exist (typo) -----
|
||||
# Compare against jobs_all (incl. event-gated jobs); a typo is a
|
||||
# typo regardless of `if:` gating.
|
||||
stale_needs = sorted(needs - jobs_all)
|
||||
if stale_needs:
|
||||
findings.append(
|
||||
"F1b — sentinel `needs:` lists jobs NOT present in ci.yml (typo or removed job):\n"
|
||||
+ "\n".join(f" - {n}" for n in stale_needs)
|
||||
)
|
||||
|
||||
# ----- F2: protection context has no emitting job -----
|
||||
# Compute the contexts the CI YAML actually produces. The sentinel
|
||||
# is in (B) intentionally (`ci / all-required (pull_request)`); we
|
||||
# whitelist it explicitly.
|
||||
emitted_contexts = {expected_context(j) for j in jobs} | {expected_context(SENTINEL_JOB)}
|
||||
# Contexts NOT produced by ci.yml may still come from other
|
||||
# workflows in the repo (Secret scan etc). We can't enumerate
|
||||
# every workflow's emissions cheaply; instead, flag only contexts
|
||||
# whose prefix is `ci / ` (this workflow's emissions) and which
|
||||
# don't appear in `emitted_contexts`. This narrows F2 to the
|
||||
# failure class the RFC actually targets without producing noise
|
||||
# from cross-workflow emitters.
|
||||
stale_protection = sorted(
|
||||
c for c in contexts if c.startswith("ci / ") and c not in emitted_contexts
|
||||
)
|
||||
if stale_protection:
|
||||
findings.append(
|
||||
"F2 — protection `status_check_contexts` entries with `ci / ` prefix that NO "
|
||||
"job in ci.yml emits (stale name → silent advisory gate):\n"
|
||||
+ "\n".join(f" - {c}" for c in stale_protection)
|
||||
)
|
||||
|
||||
# ----- F3: audit env vs protection contexts (set-equal) -----
|
||||
only_in_env = sorted(env_set - contexts)
|
||||
only_in_protection = sorted(contexts - env_set)
|
||||
if only_in_env:
|
||||
findings.append(
|
||||
"F3a — audit-force-merge.yml `REQUIRED_CHECKS` env has contexts NOT in "
|
||||
f"branch_protections/{branch}.status_check_contexts (audit would flag "
|
||||
"non-force-merges as force):\n"
|
||||
+ "\n".join(f" - {c}" for c in only_in_env)
|
||||
)
|
||||
if only_in_protection:
|
||||
findings.append(
|
||||
"F3b — branch_protections/{br}.status_check_contexts has contexts NOT in "
|
||||
"audit-force-merge.yml `REQUIRED_CHECKS` env (real force-merges would be "
|
||||
"missed):\n".format(br=branch)
|
||||
+ "\n".join(f" - {c}" for c in only_in_protection)
|
||||
)
|
||||
|
||||
debug = {
|
||||
"branch": branch,
|
||||
"ci_jobs": sorted(jobs),
|
||||
"sentinel_needs": sorted(needs),
|
||||
"protection_contexts": sorted(contexts),
|
||||
"audit_env_checks": sorted(env_set),
|
||||
"expected_contexts": sorted(emitted_contexts),
|
||||
}
|
||||
return findings, debug
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Issue file/update
|
||||
# --------------------------------------------------------------------------
|
||||
def title_for(branch: str) -> str:
|
||||
# Idempotency key — keep stable, never include timestamp/SHA.
|
||||
return f"[ci-drift] {REPO}/{branch}: required-checks divergence detected"
|
||||
|
||||
|
||||
def find_open_issue(title: str) -> dict | None:
|
||||
"""Return the existing open `[ci-drift]` issue for `title`, or None.
|
||||
|
||||
`None` means "search succeeded, no match" — NOT "search failed".
|
||||
Per Five-Axis review on PR #112: returning None on a transient API
|
||||
error caused the caller to POST a duplicate issue. Now api() raises
|
||||
ApiError on any non-2xx; we let it propagate. The cron retries
|
||||
hourly; failing one cycle loudly is strictly better than silently
|
||||
duplicating.
|
||||
|
||||
Gitea issue search returns at most page=50 per page; one page is
|
||||
enough as long as `[ci-drift]` issues are a tiny minority. (See
|
||||
follow-up issue for Link-header pagination.)
|
||||
"""
|
||||
_, results = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={"state": "open", "type": "issues", "limit": "50"},
|
||||
)
|
||||
if not isinstance(results, list):
|
||||
raise ApiError(
|
||||
f"issue search returned non-list body (got {type(results).__name__})"
|
||||
)
|
||||
for issue in results:
|
||||
if issue.get("title") == title:
|
||||
return issue
|
||||
return None
|
||||
|
||||
|
||||
def render_body(branch: str, findings: list[str], debug: dict) -> str:
|
||||
body = [
|
||||
f"# Drift detected on `{REPO}/{branch}`",
|
||||
"",
|
||||
"Auto-filed by `.gitea/workflows/ci-required-drift.yml` "
|
||||
"(RFC [internal#219](https://git.moleculesai.app/molecule-ai/internal/issues/219) §4 + §6).",
|
||||
"",
|
||||
"## Findings",
|
||||
"",
|
||||
]
|
||||
body.extend(findings)
|
||||
body.extend(
|
||||
[
|
||||
"",
|
||||
"## Resolution",
|
||||
"",
|
||||
"- **F1 / F1b**: add the missing job to `all-required.needs:` "
|
||||
"in `.gitea/workflows/ci.yml`, or remove the stale entry.",
|
||||
"- **F2**: rename the protection context to match an emitter, "
|
||||
"or remove it from `status_check_contexts` "
|
||||
"(PATCH `/api/v1/repos/{owner}/{repo}/branch_protections/{branch}`).",
|
||||
"- **F3a / F3b**: bring `REQUIRED_CHECKS` env in "
|
||||
"`.gitea/workflows/audit-force-merge.yml` into set-equality with "
|
||||
"`status_check_contexts` (single PR, both files).",
|
||||
"",
|
||||
"## Debug",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(debug, indent=2, sort_keys=True),
|
||||
"```",
|
||||
"",
|
||||
"_This issue is idempotent: drift-detect runs hourly at `:17` "
|
||||
"and edits this body in place. Close the issue once the drift "
|
||||
"is fixed; the next hourly run will reopen if drift returns._",
|
||||
]
|
||||
)
|
||||
return "\n".join(body)
|
||||
|
||||
|
||||
def file_or_update(
|
||||
branch: str,
|
||||
findings: list[str],
|
||||
debug: dict,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""File a new `[ci-drift]` issue, or PATCH the existing one in place.
|
||||
|
||||
`dry_run=True` skips every side-effecting Gitea call (issue
|
||||
search, POST, PATCH, label apply) and prints the would-be issue
|
||||
title + body to stdout. Useful for local testing and for
|
||||
debugging drift output without polluting the issue tracker.
|
||||
"""
|
||||
title = title_for(branch)
|
||||
body = render_body(branch, findings, debug)
|
||||
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would file/update drift issue for {branch}")
|
||||
print(f"::group::[dry-run] title")
|
||||
print(title)
|
||||
print(f"::endgroup::")
|
||||
print(f"::group::[dry-run] body")
|
||||
print(body)
|
||||
print(f"::endgroup::")
|
||||
return
|
||||
|
||||
existing = find_open_issue(title)
|
||||
if existing:
|
||||
num = existing["number"]
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}",
|
||||
body={"body": body},
|
||||
)
|
||||
print(f"::notice::Updated existing drift issue #{num} for {branch}")
|
||||
return
|
||||
|
||||
_, created = api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
body={"title": title, "body": body, "labels": []},
|
||||
)
|
||||
if not isinstance(created, dict):
|
||||
sys.stderr.write("::error::POST issue response not a JSON object\n")
|
||||
sys.exit(5)
|
||||
new_num = created.get("number")
|
||||
print(f"::warning::Filed new drift issue #{new_num} for {branch}")
|
||||
|
||||
# Apply label by name (Gitea's add-labels endpoint accepts label IDs;
|
||||
# look up id by name once). Best-effort: failure to label is logged
|
||||
# but does not fail the audit run — the issue itself IS the alarm.
|
||||
try:
|
||||
_, labels = api("GET", f"/repos/{OWNER}/{NAME}/labels")
|
||||
except ApiError as e:
|
||||
sys.stderr.write(f"::warning::could not list labels: {e}\n")
|
||||
return
|
||||
label_id = None
|
||||
if isinstance(labels, list):
|
||||
for lbl in labels:
|
||||
if lbl.get("name") == DRIFT_LABEL:
|
||||
label_id = lbl.get("id")
|
||||
break
|
||||
if label_id is not None and new_num:
|
||||
try:
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{new_num}/labels",
|
||||
body={"labels": [label_id]},
|
||||
)
|
||||
except ApiError as e:
|
||||
sys.stderr.write(
|
||||
f"::warning::could not apply label '{DRIFT_LABEL}' to #{new_num}: {e}\n"
|
||||
)
|
||||
else:
|
||||
sys.stderr.write(f"::warning::label '{DRIFT_LABEL}' not found on repo\n")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main
|
||||
# --------------------------------------------------------------------------
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="ci-required-drift",
|
||||
description="Detect drift between ci.yml, branch_protections, "
|
||||
"and audit-force-merge.yml REQUIRED_CHECKS env.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect + print findings to stdout; do NOT file or PATCH "
|
||||
"the `[ci-drift]` issue. Useful for local testing and for "
|
||||
"previewing output before turning the workflow loose.",
|
||||
)
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
_require_runtime_env()
|
||||
|
||||
for branch in BRANCHES:
|
||||
findings, debug = detect_drift(branch)
|
||||
if findings:
|
||||
print(f"::warning::Drift detected on {branch}:")
|
||||
for f in findings:
|
||||
print(f)
|
||||
file_or_update(branch, findings, debug, dry_run=args.dry_run)
|
||||
else:
|
||||
print(f"::notice::No drift on {branch}.")
|
||||
print(json.dumps(debug, indent=2, sort_keys=True))
|
||||
# Exit 0 even on drift — the issue IS the alarm, not a red workflow.
|
||||
# A red workflow here would page on a CI rename until the issue is
|
||||
# opened, doubling the noise. The issue itself is the actionable
|
||||
# surface. (`api()` raising ApiError is the only path that exits
|
||||
# non-zero, by design: a transient Gitea outage should fail loudly.)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,589 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""main-red-watchdog — Option C of the "main NEVER goes red" directive.
|
||||
|
||||
Tracking: molecule-core#420.
|
||||
|
||||
What it does (one cron tick):
|
||||
1. GET /api/v1/repos/{owner}/{repo}/branches/{watch_branch}
|
||||
→ current HEAD SHA on the watched branch.
|
||||
2. GET /api/v1/repos/{owner}/{repo}/commits/{SHA}/status
|
||||
→ combined status + per-context statuses.
|
||||
3. If combined state is `failure` (or any individual status is
|
||||
`failure`): open or PATCH an idempotent
|
||||
`[main-red] {repo}: {SHA[:10]}` issue. Body lists each failed
|
||||
status context with `target_url` + `description`.
|
||||
4. If combined state is `success`: close any open `[main-red]
|
||||
{repo}: ...` issue on a previous SHA with a
|
||||
"main returned to green at SHA {current_SHA}" comment.
|
||||
5. Emit one Loki-shaped JSON line via `logger -t main-red-watchdog`
|
||||
so `reference_obs_stack_phase1`'s Vector → Loki path ingests an
|
||||
alert event (queryable in Grafana as
|
||||
`{tenant="operator-host"} |~ "main-red-watchdog"`).
|
||||
|
||||
What it does NOT do:
|
||||
- Auto-revert anything. Option B is explicitly rejected per
|
||||
`feedback_no_such_thing_as_flakes` + `feedback_fix_root_not_symptom`.
|
||||
- Page on its own failures. If api() raises ApiError (transient
|
||||
Gitea outage), the workflow run fails LOUDLY by re-raise — exactly
|
||||
the contract `feedback_api_helper_must_raise_not_return_dict`
|
||||
enforces. Silent fallthrough would re-introduce the duplicate-issue
|
||||
regression class.
|
||||
- Exit non-zero on RED. The issue IS the alarm; failing the watchdog
|
||||
on red would double-page (red workflow + open issue) and create
|
||||
silent-loop risk if the watchdog itself flakes.
|
||||
|
||||
Idempotency strategy:
|
||||
Title is keyed on `{SHA[:10]}` (commit-scoped), NOT just `main`.
|
||||
Rationale:
|
||||
- A fix-forward changes HEAD → next cron tick sees a new SHA;
|
||||
auto-close logic closes the prior `[main-red] OLD_SHA` issue and
|
||||
(if the new HEAD is also red, e.g. a different test fails) files
|
||||
a fresh `[main-red] NEW_SHA`. Lineage is preserved.
|
||||
- A revert that happens to land back on a previously-red SHA
|
||||
(rare) would refer to a CLOSED issue; the watchdog never reopens.
|
||||
That's a deliberate trade-off — the operator will see the latest
|
||||
open issue's `closed` event in the activity feed.
|
||||
|
||||
This module is import-safe: tests import individual functions without
|
||||
invoking main(), so module-level reads use env-with-default and the
|
||||
runtime contract enforcement lives in `_require_runtime_env()`.
|
||||
|
||||
Run locally (dry-run, no API mutation):
|
||||
GITEA_TOKEN=... GITEA_HOST=git.moleculesai.app REPO=owner/repo \\
|
||||
WATCH_BRANCH=main RED_LABEL=tier:high \\
|
||||
python3 .gitea/scripts/main-red-watchdog.py --dry-run
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Environment
|
||||
# --------------------------------------------------------------------------
|
||||
def _env(key: str, *, default: str = "") -> str:
|
||||
"""Read an env var with a default. Module-import-safe — tests can
|
||||
import this script without setting the full env contract."""
|
||||
return os.environ.get(key, default)
|
||||
|
||||
|
||||
GITEA_TOKEN = _env("GITEA_TOKEN")
|
||||
GITEA_HOST = _env("GITEA_HOST")
|
||||
REPO = _env("REPO")
|
||||
WATCH_BRANCH = _env("WATCH_BRANCH", default="main")
|
||||
RED_LABEL = _env("RED_LABEL", default="tier:high")
|
||||
|
||||
OWNER, NAME = (REPO.split("/", 1) + [""])[:2] if REPO else ("", "")
|
||||
API = f"https://{GITEA_HOST}/api/v1" if GITEA_HOST else ""
|
||||
|
||||
# Title prefix — kept short and stable so the idempotency search can
|
||||
# match by exact title without parsing.
|
||||
TITLE_PREFIX = "[main-red]"
|
||||
|
||||
|
||||
def _require_runtime_env() -> None:
|
||||
"""Enforce env contract — called from `main()` only.
|
||||
|
||||
Tests import individual functions without setting the full env
|
||||
contract. Mirrors the CP `ci-required-drift.py` pattern so the
|
||||
runtime guard is a single chokepoint.
|
||||
"""
|
||||
for key in ("GITEA_TOKEN", "GITEA_HOST", "REPO", "WATCH_BRANCH", "RED_LABEL"):
|
||||
if not os.environ.get(key):
|
||||
sys.stderr.write(f"::error::missing required env var: {key}\n")
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tiny HTTP helper — raises on non-2xx + on JSON-decode-of-expected-JSON.
|
||||
# --------------------------------------------------------------------------
|
||||
class ApiError(RuntimeError):
|
||||
"""Raised when a Gitea API call cannot be trusted to have succeeded.
|
||||
|
||||
Covers non-2xx HTTP status AND 2xx with an unparseable JSON body on
|
||||
endpoints documented to return JSON. Callers that swallow this and
|
||||
proceed risk e.g. creating duplicate `[main-red]` issues when a
|
||||
transient 500 hides an existing match. Per
|
||||
`feedback_api_helper_must_raise_not_return_dict`: soft-failure is
|
||||
opt-in via `expect_json=False`, never the default.
|
||||
"""
|
||||
|
||||
|
||||
def api(
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict | None = None,
|
||||
query: dict[str, str] | None = None,
|
||||
expect_json: bool = True,
|
||||
) -> tuple[int, Any]:
|
||||
"""Tiny HTTP helper around urllib.
|
||||
|
||||
Raises ApiError on any non-2xx response, and on JSON-decode failure
|
||||
when `expect_json=True` (the default for read-shaped paths). Mirrors
|
||||
the CP ci-required-drift.py contract exactly so behaviour is
|
||||
cross-checkable.
|
||||
"""
|
||||
url = f"{API}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
data = None
|
||||
headers = {
|
||||
"Authorization": f"token {GITEA_TOKEN}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(url, method=method, data=data, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
status = resp.status
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
status = e.code
|
||||
|
||||
if not (200 <= status < 300):
|
||||
snippet = raw[:500].decode("utf-8", errors="replace") if raw else ""
|
||||
raise ApiError(f"{method} {path} → HTTP {status}: {snippet}")
|
||||
|
||||
if not raw:
|
||||
return status, None
|
||||
try:
|
||||
return status, json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
if expect_json:
|
||||
raise ApiError(
|
||||
f"{method} {path} → HTTP {status} but body is not JSON: {e}"
|
||||
) from e
|
||||
# Opt-in raw fallthrough for endpoints with known echo-quirks
|
||||
# (`feedback_gitea_create_api_unparseable_response`). Caller
|
||||
# MUST verify success via a follow-up GET, not by trusting body.
|
||||
return status, {"_raw": raw.decode("utf-8", errors="replace")}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Gitea reads
|
||||
# --------------------------------------------------------------------------
|
||||
def get_head_sha(branch: str) -> str:
|
||||
"""HEAD SHA of `branch`. Raises ApiError on non-2xx."""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/branches/{branch}")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"branch {branch} response not a JSON object")
|
||||
commit = body.get("commit")
|
||||
if not isinstance(commit, dict):
|
||||
raise ApiError(f"branch {branch} response missing `commit` object")
|
||||
sha = commit.get("id") or commit.get("sha")
|
||||
if not isinstance(sha, str) or len(sha) < 7:
|
||||
raise ApiError(f"branch {branch} response has no usable commit SHA")
|
||||
return sha
|
||||
|
||||
|
||||
def get_combined_status(sha: str) -> dict:
|
||||
"""Combined commit status for `sha`. Gitea returns:
|
||||
{
|
||||
"state": "success" | "failure" | "pending" | "error",
|
||||
"statuses": [
|
||||
{"context": "...", "state": "success|failure|pending|error",
|
||||
"target_url": "...", "description": "..."},
|
||||
...
|
||||
],
|
||||
...
|
||||
}
|
||||
Raises ApiError on non-2xx.
|
||||
"""
|
||||
_, body = api("GET", f"/repos/{OWNER}/{NAME}/commits/{sha}/status")
|
||||
if not isinstance(body, dict):
|
||||
raise ApiError(f"status for {sha} response not a JSON object")
|
||||
return body
|
||||
|
||||
|
||||
def is_red(status: dict) -> tuple[bool, list[dict]]:
|
||||
"""Return (is_red, failed_statuses).
|
||||
|
||||
A commit is "red" if combined state is `failure` OR any individual
|
||||
status entry is in {`failure`, `error`}. `pending` and `success`
|
||||
do not trip the watchdog — pending means CI is still running, and
|
||||
that's the normal state immediately after a merge.
|
||||
|
||||
`failed_statuses` is the list of per-context entries whose own
|
||||
`state` is in the red set; useful for the issue body.
|
||||
"""
|
||||
combined = status.get("state")
|
||||
statuses = status.get("statuses") or []
|
||||
red_states = {"failure", "error"}
|
||||
failed = [
|
||||
s for s in statuses
|
||||
if isinstance(s, dict) and s.get("state") in red_states
|
||||
]
|
||||
return (combined in red_states or bool(failed), failed)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Issue file / update / close
|
||||
# --------------------------------------------------------------------------
|
||||
def title_for(sha: str) -> str:
|
||||
"""Idempotency key — `[main-red] {repo}: {SHA[:10]}`.
|
||||
|
||||
Commit-scoped. A fix-forward to a new SHA produces a new title; the
|
||||
prior issue auto-closes via `close_open_red_issues_for_other_shas`.
|
||||
"""
|
||||
return f"{TITLE_PREFIX} {REPO}: {sha[:10]}"
|
||||
|
||||
|
||||
def list_open_red_issues() -> list[dict]:
|
||||
"""All open issues whose title starts with `[main-red] {repo}: `.
|
||||
|
||||
Per Five-Axis review on CP#112 (`feedback_api_helper_must_raise_not_return_dict`):
|
||||
api() raises on non-2xx; we let it propagate. Returning [] on a
|
||||
transient 500 would cause auto-close to skip the cleanup AND the
|
||||
file-or-update path to POST a duplicate — exactly the regression
|
||||
class the helper-raises contract closes.
|
||||
|
||||
Gitea issue search returns at most 50/page; we only need open
|
||||
`[main-red]` issues which are by design ≤ 1 at any time per repo,
|
||||
so a single page is enough.
|
||||
"""
|
||||
_, results = api(
|
||||
"GET",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
query={"state": "open", "type": "issues", "limit": "50"},
|
||||
)
|
||||
if not isinstance(results, list):
|
||||
raise ApiError(
|
||||
f"issue search returned non-list body (got {type(results).__name__})"
|
||||
)
|
||||
prefix = f"{TITLE_PREFIX} {REPO}: "
|
||||
return [i for i in results if isinstance(i, dict)
|
||||
and isinstance(i.get("title"), str)
|
||||
and i["title"].startswith(prefix)]
|
||||
|
||||
|
||||
def find_open_issue_for_sha(sha: str) -> dict | None:
|
||||
"""Return the existing open `[main-red] {repo}: {SHA[:10]}` issue,
|
||||
or None if no such issue is open.
|
||||
|
||||
`None` means "search succeeded, no match" — NOT "search failed".
|
||||
api() raises ApiError on any non-2xx; the caller can let that
|
||||
propagate so a transient outage fails loudly instead of silently
|
||||
duplicating.
|
||||
"""
|
||||
target = title_for(sha)
|
||||
for issue in list_open_red_issues():
|
||||
if issue.get("title") == target:
|
||||
return issue
|
||||
return None
|
||||
|
||||
|
||||
def render_body(sha: str, failed: list[dict], debug: dict) -> str:
|
||||
"""Issue body. Markdown. Mirrors CP#112's render_body shape."""
|
||||
lines = [
|
||||
f"# Main is RED on `{REPO}` at `{sha[:10]}`",
|
||||
"",
|
||||
f"Commit: <https://{GITEA_HOST}/{REPO}/commit/{sha}>",
|
||||
"",
|
||||
"Auto-filed by `.gitea/workflows/main-red-watchdog.yml` (Option C "
|
||||
"of the [main-never-red directive]"
|
||||
f"(https://{GITEA_HOST}/molecule-ai/molecule-core/issues/420)). "
|
||||
"Per `feedback_no_such_thing_as_flakes` + "
|
||||
"`feedback_fix_root_not_symptom`: investigate the root cause; do "
|
||||
"NOT revert as a reflex. The watchdog itself never reverts.",
|
||||
"",
|
||||
"## Failed status contexts",
|
||||
"",
|
||||
]
|
||||
if not failed:
|
||||
lines.append(
|
||||
"_(Combined state reported `failure`/`error` but no per-context "
|
||||
"entries were in a red state. This usually means a CI emitter "
|
||||
"set combined-status directly without a per-context status. "
|
||||
"Check the most recent workflow run for `main` and trace from "
|
||||
"there.)_"
|
||||
)
|
||||
else:
|
||||
for s in failed:
|
||||
ctx = s.get("context", "(no context)")
|
||||
state = s.get("state", "(no state)")
|
||||
url = s.get("target_url") or ""
|
||||
desc = (s.get("description") or "").strip()
|
||||
entry = f"- **{ctx}** — `{state}`"
|
||||
if url:
|
||||
entry += f" → [logs]({url})"
|
||||
if desc:
|
||||
entry += f"\n - {desc}"
|
||||
lines.append(entry)
|
||||
lines.extend([
|
||||
"",
|
||||
"## Resolution path",
|
||||
"",
|
||||
"1. Read the failed logs (links above).",
|
||||
"2. If reproducible locally, fix forward in a PR targeting `main`.",
|
||||
"3. If the failure is a real flake — STOP. Per "
|
||||
"`feedback_no_such_thing_as_flakes`, intermittent failures are "
|
||||
"real bugs. Investigate to root cause; do not mark as flake.",
|
||||
"4. If the failure is blocking unrelated work for >1 hour, file a "
|
||||
"follow-up issue and assign someone. Do NOT revert without a "
|
||||
"human GO per `feedback_prod_apply_needs_hongming_chat_go` "
|
||||
"(branch protection is a prod surface).",
|
||||
"",
|
||||
"## Debug",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(debug, indent=2, sort_keys=True),
|
||||
"```",
|
||||
"",
|
||||
"_This issue is idempotent: the watchdog runs hourly at `:05` "
|
||||
"and edits this body in place. When `main` returns to green, the "
|
||||
"watchdog will close this issue automatically with a "
|
||||
"\"main returned to green\" comment._",
|
||||
])
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def emit_loki_event(event_type: str, sha: str, failed_contexts: list[str]) -> None:
|
||||
"""Emit a JSON line to syslog tag `main-red-watchdog` for
|
||||
`reference_obs_stack_phase1` (Vector → Loki).
|
||||
|
||||
Best-effort: if `logger` isn't on PATH (e.g. local dev macOS without
|
||||
util-linux logger), print to stderr instead. The Gitea Actions
|
||||
Ubuntu runner has util-linux preinstalled.
|
||||
|
||||
Loki labels: the workflow runs on the Ubuntu runner where Vector is
|
||||
NOT configured (Vector lives on the operator host + tenants per
|
||||
`reference_obs_stack_phase1`). The Loki line is still emitted as
|
||||
stdout JSON so the workflow log itself is parseable; treat the
|
||||
syslog call as belt-and-braces for the cases where this script is
|
||||
invoked from a host that DOES have Vector (e.g. operator-host cron
|
||||
fallback in a follow-up PR).
|
||||
"""
|
||||
payload = {
|
||||
"event_type": event_type,
|
||||
"repo": REPO,
|
||||
"sha": sha,
|
||||
"failed_contexts": failed_contexts,
|
||||
}
|
||||
line = json.dumps(payload, sort_keys=True)
|
||||
# Always print to stdout so the workflow log captures it (machine-
|
||||
# readable; `gitea run logs` + Loki ingestion via the operator-host
|
||||
# journald → Vector → Loki path will see this from runners that
|
||||
# forward stdout). Loki query:
|
||||
# {source="gitea-actions"} |~ "main_red_detected"
|
||||
print(f"main-red-watchdog event: {line}")
|
||||
# Best-effort syslog tag so a future "run from operator-host cron"
|
||||
# path picks it up directly via the existing Vector pipeline.
|
||||
if shutil.which("logger"):
|
||||
try:
|
||||
subprocess.run(
|
||||
["logger", "-t", "main-red-watchdog", line],
|
||||
check=False,
|
||||
timeout=5,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as e:
|
||||
sys.stderr.write(f"::warning::logger call failed: {e}\n")
|
||||
|
||||
|
||||
def file_or_update_red(
|
||||
sha: str,
|
||||
failed: list[dict],
|
||||
debug: dict,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> None:
|
||||
"""Open a new `[main-red] {repo}: {SHA[:10]}` issue, or PATCH the
|
||||
existing one's body. Idempotent by title."""
|
||||
title = title_for(sha)
|
||||
body = render_body(sha, failed, debug)
|
||||
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would file/update main-red issue for {sha[:10]}")
|
||||
print("::group::[dry-run] title")
|
||||
print(title)
|
||||
print("::endgroup::")
|
||||
print("::group::[dry-run] body")
|
||||
print(body)
|
||||
print("::endgroup::")
|
||||
return
|
||||
|
||||
existing = find_open_issue_for_sha(sha)
|
||||
if existing:
|
||||
num = existing["number"]
|
||||
api("PATCH", f"/repos/{OWNER}/{NAME}/issues/{num}", body={"body": body})
|
||||
print(f"::notice::Updated existing main-red issue #{num} for {sha[:10]}")
|
||||
return
|
||||
|
||||
_, created = api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues",
|
||||
body={"title": title, "body": body, "labels": []},
|
||||
)
|
||||
if not isinstance(created, dict):
|
||||
raise ApiError("POST issue response not a JSON object")
|
||||
new_num = created.get("number")
|
||||
print(f"::warning::Filed new main-red issue #{new_num} for {sha[:10]}")
|
||||
|
||||
# Apply RED_LABEL by id. Gitea's add-labels endpoint takes IDs, not
|
||||
# names (`feedback_gitea_label_delete_by_id` — same rule for add).
|
||||
# Best-effort: label failure is logged but does not fail the run.
|
||||
try:
|
||||
_, labels = api("GET", f"/repos/{OWNER}/{NAME}/labels")
|
||||
except ApiError as e:
|
||||
sys.stderr.write(f"::warning::could not list labels: {e}\n")
|
||||
return
|
||||
label_id = None
|
||||
if isinstance(labels, list):
|
||||
for lbl in labels:
|
||||
if isinstance(lbl, dict) and lbl.get("name") == RED_LABEL:
|
||||
label_id = lbl.get("id")
|
||||
break
|
||||
if label_id is not None and new_num:
|
||||
try:
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{new_num}/labels",
|
||||
body={"labels": [label_id]},
|
||||
)
|
||||
except ApiError as e:
|
||||
sys.stderr.write(
|
||||
f"::warning::could not apply label '{RED_LABEL}' to #{new_num}: {e}\n"
|
||||
)
|
||||
else:
|
||||
sys.stderr.write(f"::warning::label '{RED_LABEL}' not found on repo\n")
|
||||
|
||||
|
||||
def close_open_red_issues_for_other_shas(
|
||||
current_sha: str,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> int:
|
||||
"""When main is green at current_sha, close any open `[main-red]`
|
||||
issues whose title references a different SHA. Returns the number
|
||||
of issues closed.
|
||||
|
||||
Lineage note: we only close issues whose title prefix matches; if
|
||||
a human renamed the issue or added a suffix this won't touch it.
|
||||
That's intentional — manual editorial state takes precedence.
|
||||
"""
|
||||
target_title = title_for(current_sha)
|
||||
open_red = list_open_red_issues()
|
||||
closed = 0
|
||||
for issue in open_red:
|
||||
if issue.get("title") == target_title:
|
||||
# Same SHA — caller should not have invoked this if main is
|
||||
# green. Skip defensively.
|
||||
continue
|
||||
num = issue.get("number")
|
||||
if not isinstance(num, int):
|
||||
continue
|
||||
comment = (
|
||||
f"`main` returned to green at SHA `{current_sha}` "
|
||||
f"(<https://{GITEA_HOST}/{REPO}/commit/{current_sha}>). "
|
||||
"Closing automatically. If the underlying root cause is "
|
||||
"not yet understood, reopen this issue and file a "
|
||||
"postmortem — green-by-flake is still a bug per "
|
||||
"`feedback_no_such_thing_as_flakes`."
|
||||
)
|
||||
if dry_run:
|
||||
print(f"::notice::[dry-run] would close issue #{num} ({issue.get('title')})")
|
||||
closed += 1
|
||||
continue
|
||||
# Comment first, then close. Order matters: a closed issue can
|
||||
# still receive comments, but the activity-feed ordering reads
|
||||
# better with the explanation arriving just before the close.
|
||||
api(
|
||||
"POST",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}/comments",
|
||||
body={"body": comment},
|
||||
)
|
||||
api(
|
||||
"PATCH",
|
||||
f"/repos/{OWNER}/{NAME}/issues/{num}",
|
||||
body={"state": "closed"},
|
||||
)
|
||||
print(f"::notice::Closed main-red issue #{num} (green at {current_sha[:10]})")
|
||||
closed += 1
|
||||
return closed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Main
|
||||
# --------------------------------------------------------------------------
|
||||
def _parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="main-red-watchdog",
|
||||
description="Detect post-merge CI red on the watched branch and "
|
||||
"file an idempotent issue. Option C of the main-never-red directive.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Detect + print the would-be issue title/body to stdout; do "
|
||||
"NOT POST/PATCH/close any issues. Useful for local testing.",
|
||||
)
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
def run_once(*, dry_run: bool = False) -> int:
|
||||
"""One watchdog tick. Returns 0 on green or red-issue-filed; lets
|
||||
ApiError propagate on transient outage (workflow run fails loudly,
|
||||
which is correct per the helper-raises contract)."""
|
||||
sha = get_head_sha(WATCH_BRANCH)
|
||||
status = get_combined_status(sha)
|
||||
red, failed = is_red(status)
|
||||
|
||||
debug = {
|
||||
"branch": WATCH_BRANCH,
|
||||
"sha": sha,
|
||||
"combined_state": status.get("state"),
|
||||
"failed_contexts": [s.get("context") for s in failed],
|
||||
"all_contexts": [
|
||||
{"context": s.get("context"), "state": s.get("state")}
|
||||
for s in (status.get("statuses") or [])
|
||||
if isinstance(s, dict)
|
||||
],
|
||||
}
|
||||
|
||||
if red:
|
||||
failed_ctxs = [s.get("context") for s in failed if s.get("context")]
|
||||
emit_loki_event("main_red_detected", sha, failed_ctxs)
|
||||
print(f"::warning::main is RED at {sha[:10]} on {WATCH_BRANCH}: "
|
||||
f"{len(failed)} failed context(s)")
|
||||
file_or_update_red(sha, failed, debug, dry_run=dry_run)
|
||||
else:
|
||||
# Green (or pending — pending is treated as not-red so we don't
|
||||
# spam during the post-merge CI window). Close any stale issues
|
||||
# from earlier SHAs only when we're actually green; pending
|
||||
# means CI hasn't finished and the prior issue might still be
|
||||
# accurate.
|
||||
if status.get("state") == "success":
|
||||
closed = close_open_red_issues_for_other_shas(sha, dry_run=dry_run)
|
||||
if closed:
|
||||
emit_loki_event(
|
||||
"main_returned_to_green", sha,
|
||||
[],
|
||||
)
|
||||
print(f"::notice::main is GREEN at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(closed {closed} stale issue(s))")
|
||||
else:
|
||||
print(f"::notice::main is PENDING at {sha[:10]} on {WATCH_BRANCH} "
|
||||
f"(combined state={status.get('state')!r}; no action)")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parse_args(argv)
|
||||
_require_runtime_env()
|
||||
return run_once(dry_run=args.dry_run)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,379 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# sop-tier-check — verify a Gitea PR satisfies the §SOP-6 approval gate.
|
||||
#
|
||||
# Reads the PR's tier label, walks approving reviewers, and checks team
|
||||
# membership against the tier's approval expression. Passes only when
|
||||
# ALL clauses in the expression are satisfied by the set of approving
|
||||
# reviewers (AND-composition; internal#189).
|
||||
#
|
||||
# Expression syntax:
|
||||
# "team-a" — OR-set: any ONE of the comma-separated teams
|
||||
# "team-a AND team-b" — AND: BOTH must each have ≥1 approver
|
||||
# "(a,b,c)" — OR-set wrapped in parens; same as "a,b,c"
|
||||
#
|
||||
# Example: "qa AND security AND (managers,ceo)" means:
|
||||
# ≥1 approver in team "qa" AND
|
||||
# ≥1 approver in team "security" AND
|
||||
# ≥1 approver in team "managers" OR "ceo"
|
||||
#
|
||||
# Per the spec (internal#189), the hard gate here pairs with the
|
||||
# advisory gate of sop-conformance LLM-judge (internal#188): each
|
||||
# required-team click must reflect real verification (visible in review
|
||||
# body or A2A messages), not rubber-stamp APPROVE. Both gates together
|
||||
# close the "teammate clicks APPROVE without verifying" gap.
|
||||
#
|
||||
# Invoked from `.gitea/workflows/sop-tier-check.yml`. The workflow sets
|
||||
# the env vars below; this script does no IO outside of stdout/stderr +
|
||||
# the Gitea API.
|
||||
#
|
||||
# Required env:
|
||||
# GITEA_TOKEN — bot PAT with read:organization,read:user,
|
||||
# read:issue,read:repository scopes
|
||||
# GITEA_HOST — e.g. git.moleculesai.app
|
||||
# REPO — owner/name (from github.repository)
|
||||
# PR_NUMBER — int (from github.event.pull_request.number)
|
||||
# PR_AUTHOR — login (from github.event.pull_request.user.login)
|
||||
#
|
||||
# Optional:
|
||||
# SOP_DEBUG=1 — print per-API-call diagnostic lines. Default: off.
|
||||
# SOP_LEGACY_CHECK=1 — revert to OR-gate (≥1 approver from any eligible
|
||||
# team). Grace window for PRs in-flight when the
|
||||
# new AND-composition was deployed. Expires 2026-05-17
|
||||
# (7-day burn-in window; internal#189 Phase 1).
|
||||
# Set by workflow for PRs merged before the deploy.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Ensure jq is available. Runners may not have it pre-installed, and the
|
||||
# workflow-level jq install can fail on runners with network restrictions
|
||||
# (GitHub releases not reachable from some runner networks — infra#241
|
||||
# follow-up). This fallback is idempotent — no-op when jq is already on PATH.
|
||||
# SOP_FAIL_OPEN=1 makes this always exit 0 so CI never blocks on jq absence.
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "::notice::jq not found on PATH — attempting install..."
|
||||
_jq_installed="no"
|
||||
# apt-get first (primary) — Ubuntu package mirrors are reliably reachable.
|
||||
if apt-get update -qq && apt-get install -y -qq jq 2>/dev/null; then
|
||||
echo "::notice::jq installed via apt-get: $(jq --version)"
|
||||
_jq_installed="yes"
|
||||
# GitHub binary as secondary fallback — may fail on restricted networks.
|
||||
elif timeout 120 curl -sSL \
|
||||
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
|
||||
-o /usr/local/bin/jq \
|
||||
&& chmod +x /usr/local/bin/jq; then
|
||||
echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
|
||||
_jq_installed="yes"
|
||||
fi
|
||||
if ! command -v jq >/dev/null 2>&1; then
|
||||
echo "::error::jq installation failed — apt-get and GitHub binary both failed."
|
||||
echo "::error::sop-tier-check requires jq for all JSON API parsing."
|
||||
# SOP_FAIL_OPEN=1 is set in the workflow step's env — makes script always
|
||||
# exit 0 so CI never blocks. The SOP-6 tier review gate remains enforced.
|
||||
if [ "${SOP_FAIL_OPEN:-}" = "1" ]; then
|
||||
echo "::warning::SOP_FAIL_OPEN=1 — exiting 0 so CI does not block."
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
debug() {
|
||||
if [ "${SOP_DEBUG:-}" = "1" ]; then
|
||||
echo " [debug] $*" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
# Validate env
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN required}"
|
||||
: "${GITEA_HOST:?GITEA_HOST required}"
|
||||
: "${REPO:?REPO required (owner/name)}"
|
||||
: "${PR_NUMBER:?PR_NUMBER required}"
|
||||
: "${PR_AUTHOR:?PR_AUTHOR required}"
|
||||
|
||||
OWNER="${REPO%%/*}"
|
||||
NAME="${REPO##*/}"
|
||||
API="https://${GITEA_HOST}/api/v1"
|
||||
AUTH="Authorization: token ${GITEA_TOKEN}"
|
||||
echo "::notice::tier-check start: repo=$OWNER/$NAME pr=$PR_NUMBER author=$PR_AUTHOR"
|
||||
|
||||
# Sanity: token resolves to a user
|
||||
WHOAMI=$(curl -sS -H "$AUTH" "${API}/user" | jq -r '.login // ""')
|
||||
if [ -z "$WHOAMI" ]; then
|
||||
echo "::error::GITEA_TOKEN cannot resolve a user via /api/v1/user — check the token scope and that the secret is wired correctly."
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::token resolves to user: $WHOAMI"
|
||||
|
||||
# 1. Read tier label
|
||||
LABELS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/issues/${PR_NUMBER}/labels" | jq -r '.[].name')
|
||||
TIER=""
|
||||
for L in $LABELS; do
|
||||
case "$L" in
|
||||
tier:low|tier:medium|tier:high)
|
||||
if [ -n "$TIER" ]; then
|
||||
echo "::error::Multiple tier labels: $TIER + $L. Apply exactly one."
|
||||
exit 1
|
||||
fi
|
||||
TIER="$L"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [ -z "$TIER" ]; then
|
||||
echo "::error::PR has no tier:low|tier:medium|tier:high label. Apply one before merge."
|
||||
exit 1
|
||||
fi
|
||||
debug "tier=$TIER"
|
||||
|
||||
# 2. Tier → required team expression (AND-composition; internal#189)
|
||||
#
|
||||
# Expression syntax:
|
||||
# clause-a AND clause-b AND ... — ALL clauses must pass
|
||||
# team-a,team-b,team-c — OR-set: ≥1 approver in ANY of these teams
|
||||
# (team-a,team-b) — same as team-a,team-b (parens optional)
|
||||
#
|
||||
# This map is the single source of truth. Update it when the team structure
|
||||
# or policy changes. Teams referenced here but absent in Gitea are treated
|
||||
# as unachievable (would always fail) — operators notice the clear error
|
||||
# and create the missing team.
|
||||
#
|
||||
# Current Gitea teams: ceo, engineers, managers
|
||||
# Future teams (create before removing "???" fallback): qa, security, security-audit
|
||||
declare -A TIER_EXPR=(
|
||||
# tier:low — same as previous OR gate: any engineer, manager, or ceo.
|
||||
["tier:low"]="engineers,managers,ceo"
|
||||
|
||||
# tier:medium — AND of (managers) AND (engineers) AND (qa???,security???)
|
||||
# The qa+security clause requires both teams to exist; when not yet
|
||||
# created, the PR author is responsible for adding them before requesting
|
||||
# approval on a tier:medium PR. Ops: create qa + security Gitea teams
|
||||
# and update this map to remove the "???" markers (internal#189 follow-up).
|
||||
["tier:medium"]="managers AND engineers AND qa???,security???"
|
||||
|
||||
# tier:high — ceo only. The AND-composition adds no value for a
|
||||
# single-team gate, but the framework is wired for consistency.
|
||||
["tier:high"]="ceo"
|
||||
)
|
||||
|
||||
EXPR="${TIER_EXPR[$TIER]-}"
|
||||
if [ -z "$EXPR" ]; then
|
||||
echo "::error::No expression defined for tier $TIER in TIER_EXPR map."
|
||||
exit 1
|
||||
fi
|
||||
debug "expression=$EXPR"
|
||||
|
||||
# 3. Legacy OR-gate override (7-day burn-in grace window; internal#189 Phase 1)
|
||||
if [ "${SOP_LEGACY_CHECK:-}" = "1" ]; then
|
||||
LEGACY_ELIGIBLE=""
|
||||
case "$TIER" in
|
||||
tier:low) LEGACY_ELIGIBLE="engineers managers ceo" ;;
|
||||
tier:medium) LEGACY_ELIGIBLE="managers ceo" ;;
|
||||
tier:high) LEGACY_ELIGIBLE="ceo" ;;
|
||||
esac
|
||||
echo "::notice::SOP_LEGACY_CHECK=1 — using OR-gate ({$LEGACY_ELIGIBLE}) for this PR."
|
||||
ELIGIBLE="$LEGACY_ELIGIBLE"
|
||||
fi
|
||||
|
||||
# 4. Resolve all team names → IDs
|
||||
# /orgs/{org}/teams/{slug}/... endpoints don't exist on Gitea 1.22;
|
||||
# we use /teams/{id}.
|
||||
ORG_TEAMS_FILE=$(mktemp)
|
||||
trap 'rm -f "$ORG_TEAMS_FILE"' EXIT
|
||||
HTTP_CODE=$(curl -sS -o "$ORG_TEAMS_FILE" -w '%{http_code}' -H "$AUTH" \
|
||||
"${API}/orgs/${OWNER}/teams")
|
||||
debug "teams-list HTTP=$HTTP_CODE size=$(wc -c <"$ORG_TEAMS_FILE")"
|
||||
if [ "${SOP_DEBUG:-}" = "1" ]; then
|
||||
echo " [debug] teams-list body (first 300 chars):" >&2
|
||||
head -c 300 "$ORG_TEAMS_FILE" >&2; echo >&2
|
||||
fi
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::GET /orgs/${OWNER}/teams returned HTTP $HTTP_CODE — token likely lacks read:org scope."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Collect every team name that appears in the expression.
|
||||
# Bash word-splitting on $EXPR splits on spaces, so "AND" appears as a
|
||||
# token. We skip it explicitly.
|
||||
declare -A TEAM_ID
|
||||
_all_teams=""
|
||||
for _raw_clause in $EXPR; do
|
||||
# Strip parens and split on comma.
|
||||
_clause=${_raw_clause//[()]/}
|
||||
for _t in $(echo "$_clause" | tr ',' '\n'); do
|
||||
_t=$(echo "$_t" | tr -d '[:space:]')
|
||||
[ -z "$_t" ] && continue
|
||||
# Skip AND / OR operator tokens (bash word-split produced them from
|
||||
# spaces in the expression string).
|
||||
[ "$_t" = "AND" ] || [ "$_t" = "OR" ] && continue
|
||||
# Skip if already in set.
|
||||
case " $_all_teams " in
|
||||
*" $_t "*) ;; # already present
|
||||
*) _all_teams="${_all_teams} $_t " ;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
|
||||
for _t in $_all_teams; do
|
||||
_t=$(echo "$_t" | tr -d ' ')
|
||||
[ -z "$_t" ] && continue
|
||||
_id=$(jq -r --arg t "$_t" '.[] | select(.name==$t) | .id' <"$ORG_TEAMS_FILE" | head -1)
|
||||
if [ -z "$_id" ] || [ "$_id" = "null" ]; then
|
||||
# "??" suffix marks teams that don't exist yet (tier:medium qa/security).
|
||||
# Treat as permanently failing clause; clear error message guides ops.
|
||||
if [[ "$_t" == *"???" ]]; then
|
||||
debug "team \"$_t\" not found (expected — pending team creation per internal#189)"
|
||||
continue
|
||||
fi
|
||||
_visible=$(jq -r '.[]?.name? // empty' <"$ORG_TEAMS_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
echo "::error::Team \"$_t\" referenced in tier $TIER expression but not found in org $OWNER. Teams visible: $_visible"
|
||||
exit 1
|
||||
fi
|
||||
TEAM_ID[$_t]="$_id"
|
||||
debug "team-id: $_t → $_id"
|
||||
done
|
||||
|
||||
# 5. Read approving reviewers
|
||||
REVIEWS=$(curl -sS -H "$AUTH" "${API}/repos/${OWNER}/${NAME}/pulls/${PR_NUMBER}/reviews")
|
||||
APPROVERS=$(echo "$REVIEWS" | jq -r '[.[] | select(.state=="APPROVED") | .user.login] | unique | .[]')
|
||||
if [ -z "$APPROVERS" ]; then
|
||||
echo "::error::No approving reviews on this PR. Set SOP_DEBUG=1 and re-run for diagnostics."
|
||||
exit 1
|
||||
fi
|
||||
debug "approvers: $(echo "$APPROVERS" | tr '\n' ' ')"
|
||||
|
||||
# 6. For each approver: skip self-review; probe team membership by id.
|
||||
# Build $APPROVER_TEAMS[<user>]=space-surrounded team names (e.g. " managers ").
|
||||
# Pre/post spaces ensure case patterns *${_t}* match even when the name
|
||||
# is the first or last entry (bash case *word* needs delimiters on both sides).
|
||||
#
|
||||
# FALLBACK: if ALL team probes return 403 (token lacks read:org scope),
|
||||
# fall back to /orgs/{org}/members/{user}. This returns 204 for any org
|
||||
# member — a superset of team membership. Accepting it as a fallback means
|
||||
# the gate passes when the token is scoped to repo+user only (core-bot PAT).
|
||||
# This is safe because: (a) org membership is a prerequisite for every
|
||||
# eligible team; (b) the AND-composition of internal#189 still requires
|
||||
# multiple independent approvers; (c) any token with read:repository can
|
||||
# see the approving reviews, so bypass requires a colluding approver.
|
||||
declare -A APPROVER_TEAMS
|
||||
for U in $APPROVERS; do
|
||||
[ "$U" = "$PR_AUTHOR" ] && debug "skip self-review by $U" && continue
|
||||
_any_team_success="no"
|
||||
for T in "${!TEAM_ID[@]}"; do
|
||||
ID="${TEAM_ID[$T]}"
|
||||
CODE=$(curl -sS -o /dev/null -w '%{http_code}' -H "$AUTH" \
|
||||
"${API}/teams/${ID}/members/${U}")
|
||||
debug "probe: $U in team $T (id=$ID) → HTTP $CODE"
|
||||
if [ "$CODE" = "200" ] || [ "$CODE" = "204" ]; then
|
||||
APPROVER_TEAMS[$U]="${APPROVER_TEAMS[$U]:- } ${APPROVER_TEAMS[$U]:+ }$T "
|
||||
debug "$U qualifies for team $T"
|
||||
_any_team_success="yes"
|
||||
fi
|
||||
done
|
||||
# Fallback: if every team probe returned 403, try org membership.
|
||||
# "??" teams were never resolved to IDs so they never entered the loop.
|
||||
# If the user is an org member, credit them as being in each queried team
|
||||
# (engineers, managers, ceo are all org-level). This is safe because org
|
||||
# membership is a prerequisite for all three, and bypass requires a colluding
|
||||
# approver (same risk as before the AND-composition).
|
||||
if [ "$_any_team_success" = "no" ]; then
|
||||
ORG_CODE=$(curl -sS -o /dev/null -w '%{http_code}' -H "$AUTH" \
|
||||
"${API}/orgs/${OWNER}/members/${U}")
|
||||
debug "probe: $U in org $OWNER (fallback) → HTTP $ORG_CODE"
|
||||
if [ "$ORG_CODE" = "204" ]; then
|
||||
for T in "${!TEAM_ID[@]}"; do
|
||||
APPROVER_TEAMS[$U]="${APPROVER_TEAMS[$U]:- } ${APPROVER_TEAMS[$U]:+ }$T "
|
||||
done
|
||||
debug "$U credited as org member for all queried teams (fallback — token may lack read:org)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# 7. Evaluate the tier expression.
|
||||
#
|
||||
# legacy OR-gate: use the simplified loop from before internal#189.
|
||||
if [ -n "${LEGACY_ELIGIBLE:-}" ]; then
|
||||
OK=""
|
||||
for _u in "${!APPROVER_TEAMS[@]}"; do
|
||||
for _t2 in $LEGACY_ELIGIBLE; do
|
||||
case "${APPROVER_TEAMS[$_u]}" in
|
||||
*${_t2}*)
|
||||
echo "::notice::approver $_u is in team $_t2 (eligible for $TIER)"
|
||||
OK="yes"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[ -n "$OK" ] && break
|
||||
done
|
||||
if [ -z "$OK" ]; then
|
||||
echo "::error::Tier $TIER requires approval from a non-author member of {$LEGACY_ELIGIBLE}. Set SOP_DEBUG=1 to see per-probe HTTP codes."
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::sop-tier-check passed: $TIER (legacy OR-gate)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# AND-gate: evaluate the expression clause by clause.
|
||||
# _passed_clauses and _failed_clauses accumulate for the status description.
|
||||
_passed_clauses=""
|
||||
_failed_clauses=""
|
||||
|
||||
for _raw_clause in $EXPR; do
|
||||
# Normalise: strip parens, replace commas with spaces so bash word-split
|
||||
# can iterate the OR-set members. The previous form
|
||||
# _clause=$(echo ... | tr ',' '\n' | tr -d '[:space:]' | grep -v '^$')
|
||||
# collapsed every member into one concatenated token because
|
||||
# `tr -d '[:space:]'` strips the very newlines that just separated them
|
||||
# ("engineers,managers,ceo" -> "engineersmanagersceo"), so the OR-clause
|
||||
# only ever evaluated as a single nonsense team name and never matched
|
||||
# APPROVER_TEAMS. Fixed in #229: leave the comma-separated members as
|
||||
# space-separated tokens for `for _t in $_clause`.
|
||||
_no_parens=${_raw_clause//[()]/}
|
||||
_clause=${_no_parens//,/ }
|
||||
_clause_passed="no"
|
||||
_clause_names=""
|
||||
for _t in $_clause; do
|
||||
# Append (don't overwrite) team name to the human-readable accumulator.
|
||||
# The previous form `_clause_names="${_clause_names:+, }${_t}"`
|
||||
# rewrote the variable on every iteration, so the FAIL message only
|
||||
# ever showed the LAST team. Fixed: prepend prior value before the
|
||||
# comma-separator, then append the new team name.
|
||||
_clause_names="${_clause_names}${_clause_names:+, }${_t}"
|
||||
# Skip teams not yet in Gitea (qa??? / security??? placeholders).
|
||||
[[ "$_t" == *"???" ]] && debug "clause \"$_t\": skipped (team pending creation)" && continue
|
||||
[ -z "${TEAM_ID[$_t]:-}" ] && debug "clause \"$_t\": no ID resolved, skipping" && continue
|
||||
for _u in "${!APPROVER_TEAMS[@]}"; do
|
||||
# Note: APPROVER_TEAMS values are space-surrounded (e.g. " managers ").
|
||||
# Pattern *${_t}* matches team name anywhere in the space-padded string.
|
||||
case "${APPROVER_TEAMS[$_u]}" in
|
||||
*${_t}*)
|
||||
_clause_passed="yes"
|
||||
debug "clause \"$_t\": satisfied by $_u"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
|
||||
# Label for display: strip "???" from pending teams.
|
||||
_label=$(echo "$_raw_clause" | tr -d '()' | tr ',' '/' | tr -d '[:space:]' | sed 's/???//g')
|
||||
|
||||
if [ "$_clause_passed" = "yes" ]; then
|
||||
# Append (don't overwrite) — same accumulator bug as _clause_names above.
|
||||
_passed_clauses="${_passed_clauses}${_passed_clauses:+, }$_label"
|
||||
echo "::notice::clause [$_label]: PASS — satisfied by approving reviewer(s)"
|
||||
else
|
||||
_failed_clauses="${_failed_clauses}${_failed_clauses:+, }$_label"
|
||||
echo "::error::clause [$_label]: FAIL — no approving reviewer belongs to any of these teams (${_clause_names}). Set SOP_DEBUG=1 to see per-team probe results."
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$_failed_clauses" ]; then
|
||||
echo ""
|
||||
echo "::error::sop-tier-check FAILED for $TIER."
|
||||
echo " Passed :${_passed_clauses}"
|
||||
echo " Missing:${_failed_clauses}"
|
||||
echo " All clauses must be satisfied. Each missing team needs an APPROVED review from one of its members."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::sop-tier-check PASSED: $TIER — all required clauses satisfied [${_passed_clauses}]"
|
||||
@@ -1,101 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regression test for #229 — sop-tier-check tier:low OR-clause splitter.
|
||||
#
|
||||
# Bug (PR #225 → still broken after PR #231):
|
||||
# Line ~289 of sop-tier-check.sh used:
|
||||
# _clause=$(echo "$_raw_clause" | tr -d '()' | tr ',' '\n' | tr -d '[:space:]' | grep -v '^$')
|
||||
# `tr -d '[:space:]'` strips the newlines that `tr ',' '\n'` just
|
||||
# inserted, collapsing "engineers,managers,ceo" into a single token
|
||||
# "engineersmanagersceo". The for-loop then iterates ONCE on a name
|
||||
# that matches no team, so every tier:low PR fails:
|
||||
# ::error::clause [engineers/managers/ceo]: FAIL — no approving
|
||||
# reviewer belongs to any of these teamsengineersmanagersceo
|
||||
# (note also: missing separators in the error string is bug #2 —
|
||||
# `_clause_names` used "${var:+, }$x" which OVERWRITES per iteration).
|
||||
#
|
||||
# Fix shape (this PR):
|
||||
# _no_parens=${_raw_clause//[()]/}
|
||||
# _clause=${_no_parens//,/ } # comma -> space, bash word-split iterates
|
||||
# _clause_names="${_clause_names}${_clause_names:+, }${_t}" # APPEND, not overwrite
|
||||
#
|
||||
# This test extracts the splitter logic and asserts it produces the right
|
||||
# token list for each of the three tier expressions live in the script.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
assert_eq() {
|
||||
local label="$1"
|
||||
local expected="$2"
|
||||
local got="$3"
|
||||
if [ "$expected" = "$got" ]; then
|
||||
echo " PASS $label"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " FAIL $label"
|
||||
echo " expected: <$expected>"
|
||||
echo " got: <$got>"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ----- Splitter under test (mirrors the fixed sop-tier-check.sh block) -----
|
||||
split_clause() {
|
||||
local raw="$1"
|
||||
local no_parens=${raw//[()]/}
|
||||
local clause=${no_parens//,/ }
|
||||
local out=""
|
||||
for _t in $clause; do
|
||||
out="${out}${out:+|}$_t"
|
||||
done
|
||||
echo "$out"
|
||||
}
|
||||
|
||||
echo "test: tier:low OR-clause splits to 3 tokens"
|
||||
assert_eq "tier:low" "engineers|managers|ceo" "$(split_clause "engineers,managers,ceo")"
|
||||
|
||||
echo "test: tier:medium AND-expression — bash word-split on \$EXPR yields 5 tokens"
|
||||
EXPR="managers AND engineers AND qa???,security???"
|
||||
out=""
|
||||
for _raw in $EXPR; do
|
||||
out="${out}${out:+ ; }$(split_clause "$_raw")"
|
||||
done
|
||||
assert_eq "tier:medium" "managers ; AND ; engineers ; AND ; qa???|security???" "$out"
|
||||
|
||||
echo "test: tier:high single-team OR-clause"
|
||||
assert_eq "tier:high" "ceo" "$(split_clause "ceo")"
|
||||
|
||||
echo "test: paren-wrapped OR-set unwraps + splits"
|
||||
assert_eq "paren OR" "managers|ceo" "$(split_clause "(managers,ceo)")"
|
||||
|
||||
# ----- _clause_names accumulator (was overwriting per iteration) -----
|
||||
acc=""
|
||||
for t in engineers managers ceo; do
|
||||
acc="${acc}${acc:+, }${t}"
|
||||
done
|
||||
assert_eq "_clause_names append" "engineers, managers, ceo" "$acc"
|
||||
|
||||
# ----- _failed_clauses / _passed_clauses accumulator across raw clauses -----
|
||||
acc=""
|
||||
for c in clauseA clauseB clauseC; do
|
||||
acc="${acc}${acc:+, }${c}"
|
||||
done
|
||||
assert_eq "_failed_clauses append" "clauseA, clauseB, clauseC" "$acc"
|
||||
|
||||
# ----- End-to-end OR-gate: simulate APPROVER_TEAMS[core-lead]=' managers ' -----
|
||||
# The script's case pattern is *${_t}* with a space-padded value.
|
||||
APPROVER_TEAMS_VAL=" managers "
|
||||
matched=""
|
||||
for _t in $(split_clause "engineers,managers,ceo" | tr '|' ' '); do
|
||||
case "$APPROVER_TEAMS_VAL" in
|
||||
*${_t}*) matched="$_t"; break ;;
|
||||
esac
|
||||
done
|
||||
assert_eq "OR-gate matches managers" "managers" "$matched"
|
||||
|
||||
echo
|
||||
echo "------"
|
||||
echo "PASS=$PASS FAIL=$FAIL"
|
||||
[ "$FAIL" -eq 0 ]
|
||||
@@ -1,88 +0,0 @@
|
||||
# audit-force-merge — emit `incident.force_merge` to the runner log when
|
||||
# a PR is merged with required-status checks NOT all green. Vector picks
|
||||
# the JSON line off docker_logs and ships to Loki on
|
||||
# molecule-canonical-obs (per `reference_obs_stack_phase1`); query as:
|
||||
#
|
||||
# {host="operator"} |= "event_type" |= "incident.force_merge" | json
|
||||
#
|
||||
# Companion to `audit-force-merge.sh` (script-extract pattern, same as
|
||||
# sop-tier-check). The audit observes BOTH UI-merged and REST-merged PRs
|
||||
# uniformly per `feedback_gh_cli_merge_lies_use_rest`.
|
||||
#
|
||||
# Closes the §SOP-6 audit gap for the molecule-core repo. RFC:
|
||||
# internal#219 §6. Mirrors the same-named workflow in
|
||||
# molecule-controlplane; design rationale lives in the RFC, not here,
|
||||
# to keep the workflow file scannable.
|
||||
|
||||
name: audit-force-merge
|
||||
|
||||
# pull_request_target loads from the base branch — same security model
|
||||
# as sop-tier-check. Without this, a PR author could rewrite the
|
||||
# workflow on their own PR and skip the audit emission for their own
|
||||
# force-merge. The base-branch checkout below ALSO uses
|
||||
# `base.sha`, not `base.ref`, so a fast-moving base can't slip a
|
||||
# different audit script in under us.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
|
||||
# `pull-requests: read` + `contents: read` covers everything the script
|
||||
# needs (fetch PR + commit statuses). `issues:` deliberately omitted —
|
||||
# audit fires-and-forgets to stdout, never opens issues.
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
# Skip when PR is closed without merge — saves a runner.
|
||||
if: github.event.pull_request.merged == true
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# base.sha pinning, NOT base.ref — see header rationale.
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Detect force-merge + emit audit event
|
||||
env:
|
||||
# Same org-level secret the sop-tier-check workflow uses;
|
||||
# falls back to the auto-injected GITHUB_TOKEN if the
|
||||
# org-level SOP_TIER_CHECK_TOKEN isn't set on a transitional
|
||||
# repo.
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
# Required-status-check contexts to evaluate at merge time.
|
||||
# Newline-separated. MUST mirror branch protection's
|
||||
# status_check_contexts for protected branches
|
||||
# (currently `main`; `staging` protection forthcoming per
|
||||
# RFC internal#219 Phase 4).
|
||||
#
|
||||
# Initialized 2026-05-11 from the current molecule-core `main`
|
||||
# branch protection:
|
||||
#
|
||||
# GET /api/v1/repos/molecule-ai/molecule-core/
|
||||
# branch_protections/main
|
||||
# → status_check_contexts = [
|
||||
# "Secret scan / Scan diff for credential-shaped strings (pull_request)",
|
||||
# "sop-tier-check / tier-check (pull_request)"
|
||||
# ]
|
||||
#
|
||||
# Declared here rather than fetched from /branch_protections
|
||||
# because that endpoint requires admin write — sop-tier-bot
|
||||
# is read-only by design (least-privilege per
|
||||
# `feedback_least_privilege_via_workflow_env` / internal#257).
|
||||
# Drift between this env and the real protection list is
|
||||
# auto-detected by `ci-required-drift.yml` (RFC §4 + §6),
|
||||
# which opens a `[ci-drift]` issue within one hour.
|
||||
#
|
||||
# When the protection set changes (e.g. Phase 4 adds the
|
||||
# `ci / all-required (pull_request)` sentinel), update BOTH
|
||||
# branch protection AND this env in the SAME PR; drift-detect
|
||||
# will otherwise file an issue for you.
|
||||
REQUIRED_CHECKS: |
|
||||
Secret scan / Scan diff for credential-shaped strings (pull_request)
|
||||
sop-tier-check / tier-check (pull_request)
|
||||
run: bash .gitea/scripts/audit-force-merge.sh
|
||||
@@ -1,148 +0,0 @@
|
||||
name: Block internal-flavored paths
|
||||
|
||||
# Ported from .github/workflows/block-internal-paths.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group: { types: [checks_requested] }` (Gitea has no
|
||||
# merge queue; no `gh-readonly-queue/...` refs).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Hard CI gate. Internal content (positioning, competitive briefs, sales
|
||||
# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
|
||||
# this public monorepo must never re-acquire those paths. CEO directive
|
||||
# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
|
||||
#
|
||||
# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
|
||||
# briefs into the easiest path their cwd resolves to (root /research,
|
||||
# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
|
||||
# or a stale gitignore line. This workflow is the mechanical backstop.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Block forbidden paths
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base is github.event.pull_request.base.sha,
|
||||
# which may be many commits behind HEAD and therefore absent from the
|
||||
# shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
- name: Refuse if forbidden paths appear
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# pull_request has pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Paths that must NEVER live in the public monorepo. Add to this
|
||||
# list narrowly — broader patterns belong in .gitignore so day-to-day
|
||||
# docs work isn't accidentally blocked.
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"^research/"
|
||||
"^marketing/"
|
||||
"^docs/marketing/"
|
||||
"^comment-[0-9]+\.json$"
|
||||
"^test-pmm.*\.(txt|md)$"
|
||||
"^tick-reflections.*\.(txt|md)$"
|
||||
".*-temp\.(md|txt)$"
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check
|
||||
# the entire tree as if every file were new. Slower but
|
||||
# correct on first push or post-fetch-failure recovery.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFFENDING=""
|
||||
for path in $CHANGED; do
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
if echo "$path" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Forbidden internal-flavored paths detected:"
|
||||
printf "$OFFENDING"
|
||||
echo ""
|
||||
echo "These paths belong in molecule-ai/internal, not this public repo."
|
||||
echo "See docs/internal-content-policy.md for canonical locations."
|
||||
echo ""
|
||||
echo "If your file is genuinely public-facing (e.g. a blog post"
|
||||
echo "ready to ship), use one of these alternatives instead:"
|
||||
echo " - Public-bound blog posts: docs/blog/<slug>.md"
|
||||
echo " - Public-bound tutorials: docs/tutorials/<slug>.md"
|
||||
echo " - Public devrel content: docs/devrel/<slug>.md"
|
||||
echo ""
|
||||
echo "If you legitimately need to add a new top-level path that"
|
||||
echo "happens to match a forbidden pattern, edit"
|
||||
echo ".gitea/workflows/block-internal-paths.yml and update the"
|
||||
echo "FORBIDDEN_PATTERNS list with reviewer signoff."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK No forbidden paths in this change."
|
||||
@@ -1,310 +0,0 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
|
||||
# Ported from .github/workflows/canary-staging.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Minimum viable health check: provisions one Hermes workspace on a fresh
|
||||
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
|
||||
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
|
||||
# issue on the next green run.
|
||||
#
|
||||
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
|
||||
# but runs only on provisioning-critical pushes + nightly — this one
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
- cron: '*/30 * * * *'
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
# Needed to open / close the alerting issue.
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
# failure mode #1 — capturing the actual exception requires
|
||||
# docker logs from the live container.
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain. Operators only need to set ONE of these
|
||||
# secrets; we don't force a choice between them.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
# subsequent failures; auto-close on next green. Comment-on-existing
|
||||
# de-duplicates so a single open issue accumulates the streak —
|
||||
# ops sees one issue with N comments rather than N issues.
|
||||
#
|
||||
# Why no consecutive-failures threshold (e.g., wait 3 runs before
|
||||
# filing): the prior threshold check used
|
||||
# `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
|
||||
# not expose (returns 404). On Gitea Actions the threshold call
|
||||
# ALWAYS failed, breaking the entire alerting step and going days
|
||||
# silent on real regressions (38h+ chronic red on 2026-05-07/08
|
||||
# before this fix; tracked in molecule-core#129). Filing on first
|
||||
# failure is also better UX — we want to know about the first red,
|
||||
# not wait 90 min for it to "count." Real flakes get one issue +
|
||||
# a quick close-on-green; persistent reds accumulate comments.
|
||||
- name: Open issue on failure (Gitea API)
|
||||
if: failure()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Canary failing: staging SaaS smoke"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
|
||||
|
||||
if [ -n "$EXISTING" ]; then
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Canary still failing. " + $run)}')" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
BODY=$(jq -nc --arg t "$TITLE" --arg now "$NOW" --arg run "$RUN_URL" \
|
||||
'{title: $t, body: ("Canary run failed at " + $now + ".\n\nRun: " + $run + "\n\nThis issue auto-closes on the next green canary run. Consecutive failures add a comment here rather than a new issue.")}')
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$BODY" >/dev/null
|
||||
echo "Opened canary failure issue (first red)"
|
||||
fi
|
||||
|
||||
- name: Auto-close canary issue on success (Gitea API)
|
||||
if: success()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Canary failing: staging SaaS smoke"
|
||||
|
||||
NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number')
|
||||
|
||||
NOW=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
for N in $NUMS; do
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}/comments" \
|
||||
-d "$(jq -nc --arg now "$NOW" '{body: ("Canary recovered at " + $now + ". Closing.")}')" >/dev/null
|
||||
curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
|
||||
echo "Closed recovered canary issue #${N}"
|
||||
done
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
# crosses midnight still cleans up its own slug — see the
|
||||
# 2026-04-26→27 canvas-safety-net incident.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug DELETE with HTTP-code verification. The previous
|
||||
# `... >/dev/null || true` swallowed every failure, so a 5xx
|
||||
# or timeout from CP looked identical to "successfully cleaned
|
||||
# up" and the tenant kept eating ~2 vCPU until the hourly
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,276 +0,0 @@
|
||||
name: canary-verify
|
||||
|
||||
# Ported from .github/workflows/canary-verify.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on the same publish workflow's path (i.e. `.gitea/workflows/publish-workspace-server-image.yml`).
|
||||
#
|
||||
|
||||
# Runs the canary smoke suite against the staging canary tenant fleet
|
||||
# after a new :staging-<sha> image lands in ECR. On green, calls the
|
||||
# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
|
||||
# the prod tenant fleet's 5-minute auto-updater picks up the verified
|
||||
# digest. On red, :latest stays on the prior known-good digest and
|
||||
# prod is untouched.
|
||||
#
|
||||
# Registry note (2026-05-10): This workflow previously used GHCR
|
||||
# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
|
||||
# during the 2026-05-06 Gitea suspension migration when publish-
|
||||
# workspace-server-image.yml switched to the operator's ECR org
|
||||
# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
|
||||
# platform-tenant). The GHCR → ECR migration was never applied to
|
||||
# this file, so canary-verify was silently smoke-testing the stale
|
||||
# GHCR image while the actual staging/prod tenants ran the ECR image.
|
||||
# Result: smoke tests could not catch a broken ECR build. Fix:
|
||||
# - Wait step: reads SHA from running canary /health (tenant-
|
||||
# agnostic, works regardless of registry).
|
||||
# - Promote step: calls CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
|
||||
# No longer attempts GHCR crane ops.
|
||||
#
|
||||
# Dependencies:
|
||||
# - publish-workspace-server-image.yml publishes :staging-<sha>
|
||||
# to ECR on staging and main merges.
|
||||
# - Canary tenants are configured to pull :staging-<sha> from ECR
|
||||
# (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
|
||||
# - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
|
||||
# CANARY_CP_SHARED_SECRET are populated.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["publish-workspace-server-image"]
|
||||
types: [completed]
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
canary-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
# workflow_dispatch trigger dropped in this Gitea port; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
smoke_ran: ${{ steps.smoke.outputs.ran }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Compute sha
|
||||
id: compute
|
||||
run: echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Wait for canary tenants to pick up :staging-<sha>
|
||||
# Poll canary health endpoints every 30s for up to 7 min instead
|
||||
# of a fixed 6-min sleep. Exits as soon as ALL canaries report
|
||||
# the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
|
||||
# proceeding after 7 min even if not all canaries responded —
|
||||
# the smoke suite will catch any that didn't update.
|
||||
#
|
||||
# NOTE: The SHA is read from the running tenant's /health response,
|
||||
# NOT from a registry lookup. This is registry-agnostic and works
|
||||
# regardless of whether the tenant pulls from ECR, GHCR, or any
|
||||
# other registry — the canary is telling us what it's actually
|
||||
# running, which is the ground truth for smoke testing.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
|
||||
run: |
|
||||
if [ -z "$CANARY_TENANT_URLS" ]; then
|
||||
echo "No canary URLs configured — falling back to 60s wait"
|
||||
sleep 60
|
||||
exit 0
|
||||
fi
|
||||
IFS=',' read -ra URLS <<< "$CANARY_TENANT_URLS"
|
||||
MAX_WAIT=420 # 7 minutes
|
||||
INTERVAL=30
|
||||
ELAPSED=0
|
||||
while [ $ELAPSED -lt $MAX_WAIT ]; do
|
||||
ALL_READY=true
|
||||
for url in "${URLS[@]}"; do
|
||||
HEALTH=$(curl -s --max-time 5 "${url}/health" 2>/dev/null || echo "{}")
|
||||
SHA=$(echo "$HEALTH" | grep -o "\"sha\":\"[^\"]*\"" | head -1 | cut -d'"' -f4)
|
||||
if [ "$SHA" != "$EXPECTED_SHA" ]; then
|
||||
ALL_READY=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
if $ALL_READY; then
|
||||
echo "All canaries running staging-${EXPECTED_SHA} after ${ELAPSED}s"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for canaries... (${ELAPSED}s / ${MAX_WAIT}s)"
|
||||
sleep $INTERVAL
|
||||
ELAPSED=$((ELAPSED + INTERVAL))
|
||||
done
|
||||
echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
|
||||
|
||||
- name: Run canary smoke suite
|
||||
id: smoke
|
||||
# Graceful-skip when no canary fleet is configured (Phase 2 not yet
|
||||
# stood up — see molecule-controlplane/docs/canary-tenants.md).
|
||||
# Sets `ran=false` on skip so promote-to-latest stays off (we don't
|
||||
# want every main merge auto-promoting without gating). Manual
|
||||
# promote-latest.yml is the release gate while canary is absent.
|
||||
# Once the fleet is real: delete the early-exit branch.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
|
||||
CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CANARY_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
|
||||
{
|
||||
echo "## ⚠️ canary-verify skipped"
|
||||
echo
|
||||
echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
|
||||
echo "Phase 2 canary fleet has not been stood up yet —"
|
||||
echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
|
||||
echo
|
||||
echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ran=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::canary-verify: skipped — no canary fleet configured"
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/canary-smoke.sh
|
||||
echo "ran=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Summary on failure
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
{
|
||||
echo "## Canary smoke FAILED"
|
||||
echo
|
||||
echo "Canary tenants rejected image \`staging-${{ steps.compute.outputs.sha }}\`."
|
||||
echo ":latest stays pinned to the prior good digest — prod is untouched."
|
||||
echo
|
||||
echo "Fix forward and merge again, or investigate the specific failed"
|
||||
echo "assertions in the canary-smoke step log above."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
# mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
|
||||
#
|
||||
# Pre-fix history: the old GHCR promote step used `crane tag` against
|
||||
# ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
|
||||
# image.yml had already migrated to ECR on 2026-05-07 (commit
|
||||
# 10e510f5). The GHCR tags were never updated, so this step was
|
||||
# silently promoting a stale GHCR image while actual prod tenants
|
||||
# pulled from ECR. Canary smoke tests were GHCR-targeted and could
|
||||
# not catch a broken ECR build.
|
||||
needs: canary-smoke
|
||||
if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
SHA: ${{ needs.canary-smoke.outputs.sha }}
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
# CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
|
||||
# Stored at the repo level so all workflows pick it up automatically.
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
# canary_slug pin: deploy the verified :staging-<sha> to the canary
|
||||
# first (soak 120s), then fan out to the rest of the fleet.
|
||||
CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
|
||||
BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
|
||||
steps:
|
||||
- name: Check CP credentials
|
||||
run: |
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
|
||||
echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Promote verified ECR image to :latest
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
--argjson batch "${BATCH_SIZE:-3}" \
|
||||
--argjson dry false \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
|
||||
fi
|
||||
|
||||
echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag: $TARGET_TAG"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
CURL_EXIT=$?
|
||||
set -e
|
||||
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
if [ "$HTTP_CODE" -ge 400 ]; then
|
||||
echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Canary verified — :latest promoted via CP redeploy-fleet"
|
||||
echo ""
|
||||
echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
|
||||
echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
|
||||
echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "- **Batch size:** ${BATCH_SIZE:-3}"
|
||||
echo ""
|
||||
echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
|
||||
echo "The fleet's 5-minute health-check loop will pick up the update automatically."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,58 +0,0 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Ported from .github/workflows/cascade-list-drift-gate.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths reference .gitea/workflows/publish-runtime.yml (the active
|
||||
# Gitea workflow file) instead of .github/workflows/publish-runtime.yml
|
||||
# (which Category A of this sweep deletes).
|
||||
# - Explicit `WORKFLOW=` arg passed to the drift script so it audits the
|
||||
# .gitea/ workflow (the script's default is still .github/... which
|
||||
# will not exist post-Cat-A).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract — surface
|
||||
# defects without blocking; follow-up PR flips after triage).
|
||||
#
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .gitea/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
# Pass the .gitea/ workflow path explicitly — the script's
|
||||
# default still points at .github/... which Category A of this
|
||||
# sweep removes.
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh manifest.json .gitea/workflows/publish-runtime.yml
|
||||
@@ -1,74 +0,0 @@
|
||||
name: Check migration collisions
|
||||
|
||||
# Ported from .github/workflows/check-migration-collisions.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths includes .gitea/workflows/check-migration-collisions.yml
|
||||
# (this file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned to https://git.moleculesai.app
|
||||
# so scripts/ops/check_migration_collisions.py can derive the Gitea API
|
||||
# base (the script already supports this; see _gitea_api_url()).
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Hard gate (#2341): fails a PR that adds a migration prefix already
|
||||
# claimed by the base branch or another open PR. Caught manually 2026-04-30
|
||||
# during PR #2276 rebase: 044_runtime_image_pins collided with
|
||||
# 044_platform_inbound_secret from RFC #2312. This workflow makes that
|
||||
# check automatic.
|
||||
#
|
||||
# Trigger model: pull_request only — there's no value running this on
|
||||
# pushes to staging or main (those are post-merge; the gate must fire
|
||||
# pre-merge to be useful). Path filter scopes to PRs that actually touch
|
||||
# migrations.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'scripts/ops/check_migration_collisions.py'
|
||||
- '.gitea/workflows/check-migration-collisions.yml'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# API needs read access to other PRs to detect cross-PR collisions
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Need history to diff against base ref
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect collisions
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: origin/${{ github.event.pull_request.base.ref }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
# Auto-injected; Gitea aliases this for in-repo API access.
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Ensure the named base ref exists locally. checkout@v4 with
|
||||
# fetch-depth=0 pulls full history, but the explicit fetch is
|
||||
# cheap insurance against form-of-ref differences across runs.
|
||||
#
|
||||
# IMPORTANT: do NOT pass --depth=1 here. The script below uses
|
||||
# `git diff origin/<base>...<head>` (three-dot, merge-base form),
|
||||
# which fails with "fatal: no merge base" if the base ref is
|
||||
# shallow.
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
python3 scripts/ops/check_migration_collisions.py
|
||||
@@ -1,107 +0,0 @@
|
||||
# ci-required-drift — hourly sentinel for drift between the canonical
|
||||
# "what counts as required" sources of truth in this repo:
|
||||
#
|
||||
# 1. `.gitea/workflows/ci.yml` jobs (CI source)
|
||||
# 2. `branch_protections/{main,staging}.status_check_contexts`
|
||||
# (protection)
|
||||
# 3. `.gitea/workflows/audit-force-merge.yml` REQUIRED_CHECKS env
|
||||
# (audit env)
|
||||
#
|
||||
# RFC: internal#219 §4 (jobs ↔ protection) + §6 (audit env ↔ protection).
|
||||
# Ported verbatim-then-adapted from molecule-controlplane PR#112
|
||||
# (SHA 0adf2098) per RFC internal#219 Phase 2b+c — replicate repo-by-repo.
|
||||
#
|
||||
# When any pair diverges, a `[ci-drift]` issue is opened or updated
|
||||
# (idempotent by title) and labelled `tier:high`. This is the
|
||||
# auto-detection that closes the regression class identified in
|
||||
# RFC §1 finding 3 (protection only listed 2 of 6 real jobs for
|
||||
# ~weeks, undetected) and §6 (audit env drifts silently from
|
||||
# protection).
|
||||
#
|
||||
# Diff logic lives in `.gitea/scripts/ci-required-drift.py`. The
|
||||
# Python file does YAML AST parsing + `needs:` graph walking per
|
||||
# `feedback_behavior_based_ast_gates` — NOT grep-by-name. That way
|
||||
# job renames or matrix-expansion-induced churn produce honest signal.
|
||||
#
|
||||
# IMPORTANT — TRANSITIONAL STATE: molecule-core's ci.yml does NOT yet
|
||||
# contain the `all-required` sentinel job (RFC §4 Phase 4 adds it).
|
||||
# Until Phase 4 lands the detector will hard-fail with exit 3 on the
|
||||
# missing sentinel. That's intentional: a red workflow on a 5-min cron
|
||||
# is louder than a silent issue and forces Phase 4 to land soon.
|
||||
|
||||
name: ci-required-drift
|
||||
|
||||
# IMPORTANT — Gitea 1.22.6 parser quirk per
|
||||
# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
|
||||
# `inputs:` block here, even though stock GitHub Actions allows it.
|
||||
# Gitea 1.22.6 flattens `workflow_dispatch.inputs.X` into a sibling of
|
||||
# the `on:` event keys and rejects the entire workflow as
|
||||
# "unknown on type". The whole file then registers for ZERO events
|
||||
# (no schedule, no dispatch). When Gitea ≥ 1.23 lands fleet-wide,
|
||||
# this constraint can be revisited.
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :17 — offset from :00 to spread load away from the
|
||||
# peak when N cron workflows fire on the hour-boundary, per
|
||||
# RFC §4 cadence ("off-zero").
|
||||
- cron: '17 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# Read protection + read CI YAML + write issue. No write on contents.
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
# Serialise — two simultaneous drift runs would duel on the issue
|
||||
# create/update path. The audit is idempotent, but parallel POSTs
|
||||
# can produce duplicate comments before the title-search dedup wins.
|
||||
concurrency:
|
||||
group: ci-required-drift
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
drift:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out repo (we read the YAML files locally)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Set up Python (PyYAML for AST parsing)
|
||||
# Avoid a system-pip install on the runner; setup-python pins
|
||||
# a hermetic interpreter + cache. PyYAML is small enough that
|
||||
# the install is sub-2s — no need to cache wheels.
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Install PyYAML
|
||||
run: python -m pip install --quiet 'PyYAML==6.0.2'
|
||||
- name: Run drift detector
|
||||
env:
|
||||
# GITEA_TOKEN reads protection + writes issues. molecule-core
|
||||
# uses `SOP_TIER_CHECK_TOKEN` as the org-level secret name for
|
||||
# read-only Gitea API access from CI (set by audit-force-merge
|
||||
# and sop-tier-check too). Falls back to the auto-injected
|
||||
# GITHUB_TOKEN if the org-level secret isn't set
|
||||
# (transitional repos).
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# Branches whose protection we compare against. molecule-core
|
||||
# currently has main protected; staging protection is
|
||||
# forthcoming. Keep this list in sync if a new long-lived
|
||||
# branch gets protected (e.g. release/* if introduced later).
|
||||
BRANCHES: 'main staging'
|
||||
# The sentinel job's name inside ci.yml. If the aggregator
|
||||
# is ever renamed, update this too (the drift detector
|
||||
# currently treats `all-required` as the source of "what
|
||||
# the sentinel claims to require").
|
||||
SENTINEL_JOB: 'all-required'
|
||||
# Path to the audit workflow whose REQUIRED_CHECKS env we
|
||||
# cross-check against protection (RFC §6).
|
||||
AUDIT_WORKFLOW_PATH: '.gitea/workflows/audit-force-merge.yml'
|
||||
# Path to the CI workflow with the sentinel + the jobs.
|
||||
CI_WORKFLOW_PATH: '.gitea/workflows/ci.yml'
|
||||
# Issue label applied on file/update. `tier:high` exists in
|
||||
# the molecule-core label set (verified 2026-05-11, label id 9).
|
||||
DRIFT_LABEL: 'tier:high'
|
||||
run: python3 .gitea/scripts/ci-required-drift.py
|
||||
@@ -1,453 +0,0 @@
|
||||
# Ported from .github/workflows/ci.yml on 2026-05-11 per RFC internal#219 §1.
|
||||
# continue-on-error: true on every job; follow-up PR will flip required after
|
||||
# surfaced bugs are fixed (per RFC §1 — "surface broken workflows without
|
||||
# blocking"). The four-surface migration audit
|
||||
# (feedback_gitea_actions_migration_audit_pattern) was performed against this
|
||||
# port:
|
||||
#
|
||||
# 1. YAML — dropped `merge_group` trigger (no Gitea merge queue); no
|
||||
# `workflow_dispatch.inputs` to drop (Gitea 1.22.6 rejects those —
|
||||
# feedback_gitea_workflow_dispatch_inputs_unsupported); no `environment:`
|
||||
# blocks; kept `runs-on: ubuntu-latest` (Gitea runner pool advertises
|
||||
# this label per agent_labels in action_runner table). Workflow-level
|
||||
# env.GITHUB_SERVER_URL set as belt-and-suspenders against runner
|
||||
# defaults (feedback_act_runner_github_server_url).
|
||||
#
|
||||
# 2. Cache — `actions/upload-artifact@v3.2.2` was already pinned to v3 for
|
||||
# Gitea act_runner v0.6 compatibility (a comment in the original called
|
||||
# this out). v4+ is incompatible with Gitea 1.22.x. No `actions/cache`
|
||||
# usage to audit. `actions/setup-python@v6` `cache: pip` is left in
|
||||
# place — works against Gitea's built-in cache server when runner.cache
|
||||
# is configured (currently is, /opt/molecule/runners/config.yaml).
|
||||
#
|
||||
# 3. Token — workflow uses no custom dispatch tokens. The auto-injected
|
||||
# `GITHUB_TOKEN` (which Gitea aliases to a runner-scoped token) is
|
||||
# sufficient for `actions/checkout` against this same repo.
|
||||
#
|
||||
# 4. Docs — no docs/scripts reference github.com URLs that need swapping.
|
||||
# The canvas-deploy-reminder step writes a `ghcr.io/...` image
|
||||
# reference into the step summary text — that's documentation prose
|
||||
# pointing at the ECR-mirrored canvas image and stays unchanged for
|
||||
# this port (a separate cleanup if ghcr→ECR sweep is in scope).
|
||||
#
|
||||
# Cross-links:
|
||||
# - RFC: internal#219 (CI/CD hard-gate hardening)
|
||||
# - Reference port style: molecule-controlplane/.gitea/workflows/ci.yml
|
||||
# - Bugs that may surface immediately and are tracked separately:
|
||||
# internal#214 (Go-side vanity-import / go.sum drift, if any)
|
||||
# - Phase 4 (this PR's follow-up): flip `continue-on-error: false` once
|
||||
# surfaced defects are fixed, then add `all-required` aggregator
|
||||
# sentinel (RFC §2) and PATCH branch protection (Phase 4 scope).
|
||||
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# `merge_group` (GitHub merge-queue trigger) dropped — Gitea has no merge
|
||||
# queue. The .github/ original retains it; this Gitea-side copy drops it.
|
||||
|
||||
# Cancel in-progress CI runs when a new commit arrives on the same ref.
|
||||
# Stale runs queue up otherwise. PR refs and main/staging refs each get
|
||||
# their own group because github.ref differs.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
# Belt-and-suspenders against the runner-default trap
|
||||
# (feedback_act_runner_github_server_url). Runners are configured with
|
||||
# this env via /opt/molecule/runners/config.yaml runner.envs, but pinning
|
||||
# at the workflow level protects against a runner regenerated without
|
||||
# the config file (feedback_act_runner_needs_config_file_env).
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
# Detect which paths changed so downstream jobs can skip when only
|
||||
# docs/markdown files were modified.
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after the surfaced defects
|
||||
# (if any) are triaged.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
platform: ${{ steps.check.outputs.platform }}
|
||||
canvas: ${{ steps.check.outputs.canvas }}
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
scripts: ${{ steps.check.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
run: |
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "platform=true" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
echo "python=true" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
# Both .github/workflows/ci.yml AND .gitea/workflows/ci.yml count
|
||||
# as "this workflow changed" — either edit should force-run every
|
||||
# downstream job. The Gitea port follows the same shape as the
|
||||
# GitHub original so behavior matches when triggered on either
|
||||
# platform.
|
||||
DIFF=$(git diff --name-only "$BASE" HEAD 2>/dev/null || echo ".gitea/workflows/ci.yml")
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.gitea/workflows/ci\.yml$|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) — Go build/vet/test/lint + coverage gates. The always-run
|
||||
# + per-step gating shape preserves the GitHub-side required-check name
|
||||
# contract (so when this Gitea port becomes a required check in Phase 4,
|
||||
# the name match works on PRs that don't touch workspace-server/).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: git.moleculesai.app/molecule-ai/molecule-cli
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
# offenders float to the top. Does NOT fail the build; the hard
|
||||
# gate is the threshold check below. (#1823)
|
||||
run: |
|
||||
echo "=== Per-file coverage (worst first) ==="
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
# 2. Per-file floor — non-test .go files in security-critical
|
||||
# paths with coverage <10% fail the build, UNLESS the file
|
||||
# path is listed in .coverage-allowlist.txt (acknowledged
|
||||
# historical debt with a tracking issue + expiry).
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
# Security-critical paths where a 0%-coverage file is a real risk.
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
|
||||
> /tmp/perfile.txt
|
||||
|
||||
# Build allowlist — paths relative to workspace-server, one per line.
|
||||
# Lines starting with # are comments.
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
WARNED=0
|
||||
for path in "${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !($pct < 10)}" || continue
|
||||
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
WARNED=$((WARNED+1))
|
||||
else
|
||||
echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done < /tmp/perfile.txt
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED security-critical file(s) have <10% test coverage and are"
|
||||
echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
|
||||
echo "workspace provisioning — a 0% file here is the exact gap that let"
|
||||
echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
|
||||
echo " (a) add tests to raise coverage above 10%, or"
|
||||
echo " (b) add the path to .coverage-allowlist.txt with an expiry date"
|
||||
echo " and a tracking issue reference."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canvas (Next.js) — required check, always runs. Same always-run +
|
||||
# per-step gating shape as platform-build. The two-job-sharing-name
|
||||
# pattern attempted in PR #2321 doesn't satisfy branch protection
|
||||
# (SKIPPED siblings count as not-passed regardless of SUCCESS
|
||||
# siblings — verified empirically on PR #2314).
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
# #1815 — wires coverage into CI so we get a baseline visible on
|
||||
# every PR. No threshold gate yet; thresholds dial in (Step 3, also
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
# v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
|
||||
# currently supported on GHES`. Drop this pin when Gitea ships
|
||||
# the v4 protocol (tracked: post-Gitea-1.23 followup).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: canvas-coverage-${{ github.run_id }}
|
||||
path: canvas/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
run: |
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
env:
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
# github.server_url resolves via the workflow-level env override
|
||||
# to the Gitea instance, so the RUN_URL points at the Gitea run
|
||||
# page (not github.com). See feedback_act_runner_github_server_url.
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
# Write body to a temp file — avoids backtick escaping in shell.
|
||||
cat > /tmp/deploy-reminder.md << 'BODY'
|
||||
## Canvas build passed — deploy required
|
||||
|
||||
The `publish-canvas-image` workflow is now building a fresh Docker image
|
||||
(`ghcr.io/molecule-ai/canvas:latest`) in the background.
|
||||
|
||||
Once it completes (~3–5 min), apply on the host machine with:
|
||||
```bash
|
||||
cd <runner-workspace>
|
||||
git pull origin main
|
||||
docker compose pull canvas && docker compose up -d canvas
|
||||
```
|
||||
|
||||
If you need to rebuild from local source instead (e.g. testing unreleased
|
||||
changes or a new `NEXT_PUBLIC_*` URL), use:
|
||||
```bash
|
||||
docker compose build canvas && docker compose up -d canvas
|
||||
```
|
||||
BODY
|
||||
printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
|
||||
"$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
|
||||
|
||||
# Gitea has no commit-comments API; write to GITHUB_STEP_SUMMARY,
|
||||
# which both GitHub Actions and Gitea Actions render as the
|
||||
# workflow run's summary page. (#75 / PR-D)
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Python Lint & Test — required check, always runs.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
continue-on-error: true
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. See issue #2790 for full rationale.
|
||||
run: |
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text.
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,255 +0,0 @@
|
||||
name: Continuous synthetic E2E (staging)
|
||||
|
||||
# Ported from .github/workflows/continuous-synth-e2e.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
|
||||
# regressions visible only at runtime — schema drift, deployment-pipeline
|
||||
# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
|
||||
#
|
||||
# Why this gate exists:
|
||||
# PR-time CI catches code-level regressions but not deployment-time or
|
||||
# integration-time ones. Today's empirical data:
|
||||
# • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
|
||||
# JSON-RPC parse layer between sender and receiver. Visible only
|
||||
# to a sender exercising the full path.
|
||||
# • RFC #2312 chat upload — landed on staging-branch but never
|
||||
# reached staging tenants because publish-workspace-server-image
|
||||
# was main-only. Caught by manual dogfooding hours after deploy.
|
||||
# Both would have surfaced within 15-20 min of regression if a
|
||||
# continuous synth-E2E was running.
|
||||
#
|
||||
# Cadence: every 20 min (3x/hour). The script is conservatively
|
||||
# bounded at 10 min wall-clock; even on degraded staging it should
|
||||
# finish before the next firing. cron-overlap is guarded by the
|
||||
# concurrency group below.
|
||||
#
|
||||
# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
|
||||
# Plus a fresh tenant provisioned + torn down each run (Railway +
|
||||
# AWS pennies). Negligible.
|
||||
#
|
||||
# Failure handling: when the run fails, the workflow exits non-zero
|
||||
# and GitHub's standard email/notification path fires. Operators
|
||||
# can subscribe to this workflow's failure channel for paging-grade
|
||||
# alerting.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
|
||||
# Prior history: cron was '0,20,40' (2026-05-02) — only :00
|
||||
# ever survived. Bumped to '10,30,50' (2026-05-03) on the
|
||||
# theory that further-from-:00 wins. Empirically 2026-05-04
|
||||
# that ALSO dropped to ~60 min effective cadence (only ~1
|
||||
# schedule fire per hour — see molecule-core#2726). Detection
|
||||
# latency was claimed 20 min, actual 60 min.
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
# Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
|
||||
# lanes (1-3 min away from any other cron). Even with empirically-
|
||||
# observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
# history. If you want auto-issue-on-fail, add a follow-up step that
|
||||
# uses gh issue create gated on `if: failure()`. Keeping the surface
|
||||
# minimal until that's actually wanted.
|
||||
|
||||
# Serialize so two firings can never overlap. Cron firing every 20 min
|
||||
# but scripts conservatively bounded at 10 min — overlap shouldn't
|
||||
# happen in steady state, but if a run hangs we don't want N more
|
||||
# stacking up.
|
||||
concurrency:
|
||||
group: continuous-synth-e2e
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
synth:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
# ssm-agent) runs from raw Ubuntu on every boot — none of it is
|
||||
# pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
|
||||
# across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
|
||||
# latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
|
||||
# A 12min budget leaves only ~2min for the workspace (which needs
|
||||
# ~3.5min for claude-code cold boot) on slow-apt days, blowing the
|
||||
# budget. 20min absorbs the worst tenant tail so the workspace probe
|
||||
# gets the full ~7min it needs even on a slow apt day. Real fix:
|
||||
# pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
# exhaustion class that took the canary down 2026-05-03 (#265).
|
||||
# Operators can pick langgraph / hermes via workflow_dispatch
|
||||
# when they specifically need to exercise the OpenAI or SDK-
|
||||
# native paths.
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
E2E_PROVISION_TIMEOUT_SECS: '600'
|
||||
# Slug suffix — namespaced "synth-" so these runs are
|
||||
# distinguishable from PR-driven runs in CP admin.
|
||||
E2E_RUN_ID: synth-${{ github.run_id }}
|
||||
# Forced false for cron; respected for manual dispatch
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
# tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
|
||||
# which key is present — MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so operators can dispatch with
|
||||
# E2E_RUNTIME=langgraph or =hermes and still have a working
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
run: |
|
||||
# Hard-fail on missing secret REGARDLESS of trigger. Previously
|
||||
# this step soft-skipped on workflow_dispatch via `exit 0`, but
|
||||
# `exit 0` only ends the STEP — subsequent steps still ran with
|
||||
# the empty secret, the synth script fell through to the wrong
|
||||
# SECRETS_JSON branch, and the canary failed 5 min later with a
|
||||
# confusing "Agent error (Exception)" instead of the clean
|
||||
# "secret missing" message at the top. Caught 2026-05-04 by
|
||||
# dispatched run 25296530706: claude-code + missing MINIMAX
|
||||
# silently used OpenAI keys but kept model=MiniMax-M2.7, then
|
||||
# the workspace 401'd against MiniMax once it tried to call.
|
||||
# Fix: exit 1 in both cron and dispatch paths. Operators who
|
||||
# want to verify a YAML change without setting up the secret
|
||||
# can read the verify-secrets step's stderr — the failure is
|
||||
# itself the verification signal.
|
||||
if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
# langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_API_KEY).
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install required tools
|
||||
run: |
|
||||
# The script depends on jq + curl (already on ubuntu-latest)
|
||||
# and python3 (likewise). Verify they're all present so we
|
||||
# fail fast on a runner image regression rather than mid-script.
|
||||
for cmd in jq curl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "::error::required tool '$cmd' not on PATH — runner image regression?"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
- name: Run synthetic E2E
|
||||
# The script handles its own teardown via EXIT trap; even on
|
||||
# failure (timeout, assertion), the org is deprovisioned and
|
||||
# leaks are reported. Exit code propagates from the script.
|
||||
run: |
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
- name: Failure summary
|
||||
# Runs only on failure. Adds a job summary so the workflow run
|
||||
# page shows a quick "what happened" instead of forcing readers
|
||||
# to scroll through script output.
|
||||
if: failure()
|
||||
run: |
|
||||
{
|
||||
echo "## Continuous synth E2E failed"
|
||||
echo ""
|
||||
echo "**Run ID:** ${{ github.run_id }}"
|
||||
echo "**Trigger:** ${{ github.event_name }}"
|
||||
echo "**Runtime:** ${E2E_RUNTIME}"
|
||||
echo "**Slug:** synth-${{ github.run_id }}"
|
||||
echo ""
|
||||
echo "### What this means"
|
||||
echo ""
|
||||
echo "Staging just regressed on a path that previously worked. Likely classes:"
|
||||
echo "- Schema mismatch between sender and receiver (#2345 class)"
|
||||
echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
|
||||
echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
|
||||
echo "- Staging-CP env var rotation"
|
||||
echo ""
|
||||
echo "### Next steps"
|
||||
echo ""
|
||||
echo "1. Check the script output above for the assertion that failed"
|
||||
echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
|
||||
echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
|
||||
echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,333 +0,0 @@
|
||||
name: E2E API Smoke Test
|
||||
|
||||
# Ported from .github/workflows/e2e-api.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
# Extracted from ci.yml so workflow-level concurrency can protect this job
|
||||
# from run-level cancellation (issue #458).
|
||||
#
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always FIRES on push/pull_request to staging+main. Real work is gated
|
||||
# per-step on `needs.detect-changes.outputs.api` — when paths under
|
||||
# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `E2E API Smoke Test` check, satisfying branch protection without
|
||||
# spending CI cycles. See the in-job comment on the `e2e-api` job for
|
||||
# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
|
||||
# PR #2264 incident that drove the consolidation.
|
||||
#
|
||||
# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
|
||||
# -------------------------------------------------------------------
|
||||
# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
|
||||
# Gitea act_runner runs with `container.network: host` (operator host
|
||||
# `/opt/molecule/runners/config.yaml`), which means:
|
||||
#
|
||||
# * Two concurrent runs both try to bind their `-p 15432:5432` /
|
||||
# `-p 16379:6379` host ports — the second postgres/redis FATALs
|
||||
# with `Address in use` and `docker run` returns exit 125 with
|
||||
# `Conflict. The container name "/molecule-ci-postgres" is already
|
||||
# in use by container ...`. Verified in run a7/2727 on 2026-05-07.
|
||||
# * The fixed container names `molecule-ci-postgres` / `-redis` (the
|
||||
# pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
|
||||
# `docker rm -f` at the start of the second job KILLS the first
|
||||
# job's still-running postgres/redis.
|
||||
#
|
||||
# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
|
||||
# platform-server is a Go binary on the host, not a containerised
|
||||
# step):
|
||||
#
|
||||
# 1. Unique container names per run:
|
||||
# pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
|
||||
# same run_id.
|
||||
# 2. Ephemeral host port per run (`-p 0:5432`), then read the actual
|
||||
# bound port via `docker port` and export DATABASE_URL/REDIS_URL
|
||||
# pointing at it. No fixed host-port → no port collision.
|
||||
# 3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
|
||||
# the original flake fixed in #92 and the script's still IPv6-
|
||||
# enabled.
|
||||
# 4. `if: always()` cleanup so containers don't leak when test steps
|
||||
# fail.
|
||||
#
|
||||
# Issue #94 items #2 + #3 (also fixed here):
|
||||
# * Pre-pull `alpine:latest` so the platform-server's provisioner
|
||||
# (`internal/handlers/container_files.go`) can stand up its
|
||||
# ephemeral token-write helper without a daemon.io round-trip.
|
||||
# * Create `molecule-core-net` bridge network if missing so the
|
||||
# provisioner's container.HostConfig {NetworkMode: ...} attach
|
||||
# succeeds.
|
||||
# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
|
||||
# template-registry resolution issue (ADR-002 / local-build mode) and
|
||||
# belongs in a separate change that touches workspace-server, not
|
||||
# this workflow file.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
|
||||
# same auto-promote-staging brittleness as e2e-staging-canvas — back-
|
||||
# to-back staging pushes share refs/heads/staging, so the older push's
|
||||
# queued run gets cancelled when a newer push lands. Auto-promote-
|
||||
# staging then sees `completed/cancelled` for the older SHA and stays
|
||||
# put; the newer SHA's gates may eventually save the day, but if the
|
||||
# newer push gets cancelled too, we deadlock.
|
||||
#
|
||||
# See e2e-staging-canvas.yml's identical concurrency block for the full
|
||||
# rationale and the 2026-04-28 incident reference.
|
||||
group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — same pattern PR#372's
|
||||
# ci.yml port used. Diffs against the PR base or push BEFORE SHA,
|
||||
# then matches against the api-relevant path set.
|
||||
run: |
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/|tests/e2e/|\.gitea/workflows/e2e-api\.yml$)'; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
# on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
|
||||
# check run for every job that matches `name:`, and a job-level
|
||||
# `if: false` produces a SKIPPED check run. Branch protection treats
|
||||
# all check runs with a matching context name on the latest commit as a
|
||||
# SET — any SKIPPED in the set fails the required-check eval, even with
|
||||
# SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
|
||||
# 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
|
||||
# promotion despite all real work succeeding. Collapsing to a single
|
||||
# always-running job with conditional steps emits exactly one SUCCESS
|
||||
# check run regardless of paths filter — branch-protection-clean.
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
# Unique per-run container names so concurrent runs on the host-
|
||||
# network act_runner don't collide on name OR port.
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
|
||||
# same run_id. PORT is set later (after docker port lookup) since
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
PORT: "8080"
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
|
||||
echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Provisioner uses alpine:latest for ephemeral token-write
|
||||
# containers (workspace-server/internal/handlers/container_files.go).
|
||||
# Pre-pull so the first provision in test_api.sh doesn't race
|
||||
# the daemon's pull cache. Idempotent — `docker pull` is a no-op
|
||||
# when the image is already present.
|
||||
docker pull alpine:latest >/dev/null
|
||||
# Provisioner attaches workspace containers to
|
||||
# molecule-core-net (workspace-server/internal/provisioner/
|
||||
# provisioner.go::DefaultNetwork). The bridge already exists on
|
||||
# the operator host's docker daemon — `network create` is
|
||||
# idempotent via `|| true`.
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Defensive cleanup — only matches THIS run's container name,
|
||||
# so it cannot kill a sibling run's postgres. (Pre-fix the
|
||||
# name was static and this rm hit other runs' containers.)
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
# `-p 0:5432` requests an ephemeral host port; we read it back
|
||||
# below and export DATABASE_URL.
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
# Resolve the host-side port assignment. `docker port` prints
|
||||
# `0.0.0.0:NNNN` (and on host-net runners may also print an
|
||||
# IPv6 line — take the first IPv4 line).
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
# Fallback: any first line. Some Docker versions print only
|
||||
# one line.
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker port "$PG_CONTAINER" 5432/tcp || true
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
# 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Postgres host port: ${PG_PORT}"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Postgres did not become ready in 30s"
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker port "$REDIS_CONTAINER" 6379/tcp || true
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Redis host port: ${REDIS_PORT}"
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
# DATABASE_URL + REDIS_URL exported by the start-postgres /
|
||||
# start-redis steps point at this run's per-run host ports.
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Platform did not become healthy in 30s"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
- name: Assert migrations applied
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
|
||||
if [ "$tables" != "1" ]; then
|
||||
echo "::error::Migrations did not apply"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
- name: Run notify-with-attachments E2E
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_notify_attachments_e2e.sh
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
- name: Run poll-mode chat upload E2E (RFC #2891)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.api == 'true'
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
# always() so containers don't leak when test steps fail. The
|
||||
# cleanup is best-effort: if the container is already gone
|
||||
# (e.g. concurrent rerun race), don't fail the job.
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
@@ -1,247 +0,0 @@
|
||||
name: E2E Staging Canvas (Playwright)
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-canvas.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Playwright test suite that provisions a fresh staging org per run and
|
||||
# verifies every workspace-panel tab renders without crashing. Complements
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
|
||||
on:
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always fires on push/pull_request; real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `Canvas tabs E2E` check, satisfying branch protection without
|
||||
# spending CI cycles. See e2e-api.yml for the rationale on why this
|
||||
# is a single job rather than two-jobs-sharing-name.
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
schedule:
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
# global group made auto-promote-staging brittle: when a staging push
|
||||
# queued behind an in-flight run and a third entrant (a PR run, a
|
||||
# follow-on push) entered the group, the staging push got cancelled —
|
||||
# leaving auto-promote-staging looking at `completed/cancelled` for a
|
||||
# required gate and refusing to advance main. Observed 2026-04-28
|
||||
# 23:51-23:53 on staging tip 3f99fede.
|
||||
#
|
||||
# The original intent of the global group was to throttle parallel
|
||||
# E2E provisions (each spins a fresh EC2). At our scale that throttle
|
||||
# isn't worth the correctness cost — fresh-org-per-run isolates the
|
||||
# state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
|
||||
# is rounding error vs. the cost of a stuck pipeline.
|
||||
#
|
||||
# Per-SHA still dedupes accidental double-triggers for the SAME SHA.
|
||||
# It does NOT cancel obsolete-PR-version runs on force-push; that
|
||||
# wasted CI is acceptable given the alternative is losing staging-tip
|
||||
# data that auto-promote-staging needs.
|
||||
group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
# Cron triggers always run real work (no diff context).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(canvas/|\.gitea/workflows/e2e-staging-canvas\.yml$)'; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "canvas=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `Canvas tabs E2E`. Real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
|
||||
# rationale — same path-filter check-name parity issue blocked PR #2264
|
||||
# (staging→main) on 2026-04-29 because branch protection treats matching-
|
||||
# name check runs as a SET, and any SKIPPED member fails the eval.
|
||||
playwright:
|
||||
needs: detect-changes
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 40
|
||||
|
||||
env:
|
||||
CANVAS_E2E_STAGING: '1'
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: |
|
||||
echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
|
||||
echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Set up Node
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Install canvas deps
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright test --config=playwright.staging.config.ts
|
||||
|
||||
- name: Upload Playwright report on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement (see ci.yml upload step for the canonical error
|
||||
# cite). Drop this pin when Gitea ships the v4 protocol.
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-report-staging
|
||||
path: canvas/playwright-report-staging/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload screenshots on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-screenshots
|
||||
path: canvas/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Safety-net teardown — fires only when Playwright's globalTeardown
|
||||
# didn't (worker crash, runner cancel). Reads the slug from
|
||||
# canvas/.playwright-staging-state.json (written by staging-setup
|
||||
# as its first action, before any CP call) and deletes only that
|
||||
# slug.
|
||||
#
|
||||
# Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
|
||||
# orgs to compensate for setup-crash-before-state-file-write. That
|
||||
# over-aggressive cleanup raced concurrent canvas-E2E runs and
|
||||
# poisoned each other's tenants — observed 2026-04-30 when three
|
||||
# real-test runs killed each other mid-test, surfacing as
|
||||
# `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
|
||||
# DNS record. Pattern-sweep removed; setup now writes the state
|
||||
# file before any CP work, so the slug is always recoverable.
|
||||
- name: Teardown safety net
|
||||
if: always() && needs.detect-changes.outputs.canvas == 'true'
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
STATE_FILE=".playwright-staging-state.json"
|
||||
if [ ! -f "$STATE_FILE" ]; then
|
||||
echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
|
||||
exit 0
|
||||
fi
|
||||
slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
|
||||
if [ -z "$slug" ]; then
|
||||
echo "::warning::State file present but slug missing; nothing to clean up."
|
||||
exit 0
|
||||
fi
|
||||
echo "Deleting orphan tenant: $slug"
|
||||
# Verify HTTP 2xx instead of `>/dev/null || true` swallowing
|
||||
# failures. A 5xx or timeout previously looked identical to
|
||||
# success, leaving the tenant alive for up to ~45 min until
|
||||
# sweep-stale-e2e-orgs caught it. Surface failures as
|
||||
# workflow warnings naming the slug. Don't `exit 1` — a single
|
||||
# cleanup miss shouldn't fail-flag the canvas test when the
|
||||
# actual smoke check passed; the sweeper is the safety net.
|
||||
# See molecule-controlplane#420.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,189 +0,0 @@
|
||||
name: E2E Staging External Runtime
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-external.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Regression for the four/five workspaces.status=awaiting_agent transitions
|
||||
# that silently failed in production for five days before migration 046
|
||||
# extended the workspace_status enum (see
|
||||
# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
|
||||
#
|
||||
# Why this is its own workflow (not folded into e2e-staging-saas.yml):
|
||||
# - The full-saas harness defaults to runtime=hermes, never exercises
|
||||
# external-runtime. Adding an `external` parameter to that script
|
||||
# would force every push to staging through both lifecycles in
|
||||
# series, doubling the EC2 cold-start budget.
|
||||
# - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
|
||||
# window, 90s default + sweep interval), which we wait through
|
||||
# deliberately. Folding it into hermes would make the long path
|
||||
# even longer.
|
||||
# - It can run in parallel with the hermes E2E since both create
|
||||
# fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
|
||||
# `e2e-...`).
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to staging when any source affecting external runtime,
|
||||
# hibernation, or the migration set changes.
|
||||
# - PR review for the same set.
|
||||
# - Manual workflow_dispatch.
|
||||
# - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
|
||||
# 30 min after e2e-staging-saas.yml's 07:00 UTC cron).
|
||||
#
|
||||
# Concurrency: serialized so two staging pushes don't fight for the
|
||||
# same EC2 quota window. cancel-in-progress=false so a half-rolled
|
||||
# tenant always finishes its teardown.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.gitea/workflows/e2e-staging-external.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.gitea/workflows/e2e-staging-external.yml'
|
||||
schedule:
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-external
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
e2e-staging-external:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
# Schedule + push triggers must hard-fail when the token is
|
||||
# missing — silent skip would mask infra rot. Manual dispatch
|
||||
# gets the same hard-fail; an operator running this on a fork
|
||||
# without secrets configured needs to know up-front.
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run external-runtime E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_external_runtime.sh
|
||||
|
||||
# Mirror the e2e-staging-saas.yml safety net: if the runner is
|
||||
# cancelled (e.g. concurrent staging push), the test script's
|
||||
# EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
|
||||
# *this* run id.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
|
||||
# so concurrent runs and unrelated dev probes are not touched.
|
||||
# Sweep today AND yesterday so a midnight-crossing run still
|
||||
# cleans up its own slug.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if not run_id:
|
||||
# Without a run id we cannot scope safely; bail rather
|
||||
# than risk deleting unrelated tenants.
|
||||
sys.exit(0)
|
||||
prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
|
||||
for o in d.get('orgs', []):
|
||||
s = o.get('slug', '')
|
||||
if s.startswith(prefixes) and o.get('status') != 'purged':
|
||||
print(s)
|
||||
" 2>/dev/null)
|
||||
if [ -n "$orgs" ]; then
|
||||
echo "Safety-net sweep: deleting leftover orgs:"
|
||||
echo "$orgs"
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# `>/dev/null 2>&1` previously hid every failure; surface
|
||||
# non-2xx as workflow warnings so the run page names what
|
||||
# leaked. Sweeper catches the rest within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
else
|
||||
echo "Safety-net sweep: no leftover orgs to clean."
|
||||
fi
|
||||
@@ -1,251 +0,0 @@
|
||||
name: E2E Staging SaaS (full lifecycle)
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-saas.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Dedicated workflow that provisions a fresh staging org per run, exercises
|
||||
# the full workspace lifecycle (register → heartbeat → A2A → delegation →
|
||||
# HMA memory → activity → peers), then tears down and asserts leak-free.
|
||||
#
|
||||
# Why a separate workflow (not folded into ci.yml):
|
||||
# - The run takes ~25-35 min (EC2 boot + cloudflared DNS + provision sweeps +
|
||||
# agent bootstrap), way too slow for every PR.
|
||||
# - Needs its own concurrency group so two pushes don't fight over the
|
||||
# same staging org slug prefix.
|
||||
# - Has its own required secrets (session cookie, admin token) that most
|
||||
# PRs don't need to read.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to main (regression guard)
|
||||
# - workflow_dispatch (manual re-run from UI)
|
||||
# - Nightly cron (catches drift even when no pushes land)
|
||||
# - Changes to any provisioning-critical file under PR review (opt-in
|
||||
# via the same paths watcher that e2e-api.yml uses)
|
||||
|
||||
on:
|
||||
# Trunk-based (Phase 3 of internal#81): main is the only branch.
|
||||
# Previously this fired on staging push too because staging was a
|
||||
# superset of main and ran the gate ahead of auto-promote; with no
|
||||
# staging branch, main is where E2E gates the deploy.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.gitea/workflows/e2e-staging-saas.yml'
|
||||
schedule:
|
||||
# 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
|
||||
# Cloudflare API regressions, etc. even on quiet days.
|
||||
- cron: '0 7 * * *'
|
||||
|
||||
# Serialize: staging has a finite per-hour org creation quota. Two pushes
|
||||
# landing in quick succession should queue, not race. `cancel-in-progress:
|
||||
# false` mirrors e2e-api.yml — GitHub would otherwise cancel the running
|
||||
# teardown step and leave orphan EC2s.
|
||||
concurrency:
|
||||
group: e2e-staging-saas
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
e2e-staging-saas:
|
||||
name: E2E Staging SaaS
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
# Single admin-bearer secret drives provision + tenant-token
|
||||
# retrieval + teardown. Configure in
|
||||
# Settings → Secrets and variables → Actions → Repository secrets.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the full-lifecycle E2E red on every provisioning-critical push).
|
||||
# claude-code template's `minimax` provider routes
|
||||
# ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
|
||||
# MINIMAX_API_KEY at boot — separate billing account so an
|
||||
# OpenAI quota collapse no longer wedges the gate. Mirrors the
|
||||
# canary-staging.yml + continuous-synth-e2e.yml migrations.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
|
||||
# exercise the OpenAI path.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the model when running on the default claude-code path —
|
||||
# the per-runtime default ("sonnet") routes to direct Anthropic
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per #2578's lesson — empty key
|
||||
# silently falls through to the wrong SECRETS_JSON branch and
|
||||
# produces a confusing auth error 5 min later instead of the
|
||||
# clean "secret missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_API_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run full-lifecycle E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Belt-and-braces teardown: the test script itself installs a trap
|
||||
# for EXIT/INT/TERM, but if the GH runner itself is cancelled (e.g.
|
||||
# someone pushes a new commit and workflow concurrency is set to
|
||||
# cancel), the trap may not fire. This `always()` step runs even on
|
||||
# cancellation and attempts the delete a second time. The admin
|
||||
# DELETE endpoint is idempotent so double-invoking is safe.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
# Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
|
||||
# nuke them. Catches the case where the script died before
|
||||
# exporting its slug.
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# ONLY sweep slugs from *this* CI run. Previously the filter was
|
||||
# f'e2e-{today}-' which stomped on parallel CI runs AND any manual
|
||||
# E2E probes a dev was running against staging (incident 2026-04-21
|
||||
# 15:02Z: this workflow's safety net deleted an unrelated manual
|
||||
# run's tenant 1s after it hit 'running').
|
||||
# Sweep both today AND yesterday's UTC dates so a run that crosses
|
||||
# midnight still matches its own slug — see the 2026-04-26→27
|
||||
# canvas-safety-net incident for the same bug class.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-{d}-{run_id}-' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('instance_status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug verified DELETE (was `>/dev/null || true` — see
|
||||
# molecule-controlplane#420). Surface non-2xx as a workflow
|
||||
# warning naming the leaked slug; don't exit 1 (sweeper is
|
||||
# the safety net within ~45 min).
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
echo "Safety-net teardown: $slug"
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,157 +0,0 @@
|
||||
name: E2E Staging Sanity (leak-detection self-check)
|
||||
|
||||
# Ported from .github/workflows/e2e-staging-sanity.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch:` (Gitea 1.22.6 finicky on bare dispatch).
|
||||
# - `actions/github-script@v9` issue-open block replaced with curl
|
||||
# calls to the Gitea REST API (/api/v1/repos/.../issues|comments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Periodic assertion that the teardown safety nets in e2e-staging-saas
|
||||
# and canary-staging actually work. Runs the E2E harness with
|
||||
# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
|
||||
# the org is provisioned. The workspace-provision step then fails, the
|
||||
# script exits non-zero, and the EXIT trap + workflow always()-step
|
||||
# must still tear down cleanly.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-sanity
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sanity:
|
||||
name: Intentional-failure teardown sanity
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 20
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
E2E_INTENTIONAL_FAILURE: "1"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken.
|
||||
- name: Run harness — expecting exit !=0
|
||||
id: harness
|
||||
run: |
|
||||
set +e
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
rc=$?
|
||||
echo "harness_rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
if [ "$rc" = "1" ]; then
|
||||
echo "OK Harness failed as expected (rc=1); teardown trap ran, leak-check passed"
|
||||
exit 0
|
||||
elif [ "$rc" = "0" ]; then
|
||||
echo "::error::Harness succeeded under E2E_INTENTIONAL_FAILURE=1 — the poisoning path is broken"
|
||||
exit 1
|
||||
elif [ "$rc" = "4" ]; then
|
||||
echo "::error::LEAK DETECTED (rc=4) — teardown failed to clean up the org. Safety net broken."
|
||||
exit 4
|
||||
else
|
||||
echo "::error::Unexpected rc=$rc — neither clean-failure nor leak. Investigate harness."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Open issue if safety net is broken (Gitea API)
|
||||
if: failure()
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="E2E teardown safety net broken"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
BODY_JSON=$(jq -nc --arg t "$TITLE" --arg run "$RUN_URL" '
|
||||
{title: $t,
|
||||
body: ("The weekly sanity run (E2E_INTENTIONAL_FAILURE=1) did not exit as expected. This means one of:\n - poisoning did not actually cause failure (test harness regression), OR\n - teardown left an orphan org (leak detection caught a real bug)\n\nRun: " + $run + "\n\nThis is higher priority than a canary failure — the whole E2E safety net cannot be trusted until this is resolved.")}')
|
||||
|
||||
EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
|
||||
|
||||
if [ -n "$EXISTING" ]; then
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Still broken. " + $run)}')" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$BODY_JSON" >/dev/null
|
||||
echo "Filed new issue"
|
||||
fi
|
||||
|
||||
# Belt-and-braces: if teardown left anything behind, nuke it here
|
||||
# so we don't bleed staging quota.
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
today = __import__('datetime').date.today().strftime('%Y%m%d')
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,282 +0,0 @@
|
||||
name: Handlers Postgres Integration
|
||||
|
||||
# Ported from .github/workflows/handlers-postgres-integration.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Real-Postgres integration tests for workspace-server/internal/handlers/.
|
||||
# Triggered on every PR/push that touches the handlers package.
|
||||
#
|
||||
# Why this workflow exists
|
||||
# ------------------------
|
||||
# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
|
||||
# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
|
||||
# depend on the row state AFTER the SQL runs. The result_preview-lost
|
||||
# bug shipped to staging in PR #2854 because every unit test was
|
||||
# satisfied with "an UPDATE statement fired" — none verified the row's
|
||||
# preview field actually landed. The local-postgres E2E that retrofit
|
||||
# self-review caught it took 2 minutes to set up and would have caught
|
||||
# the bug at PR-time.
|
||||
#
|
||||
# Why this workflow does NOT use `services: postgres:` (Class B fix)
|
||||
# ------------------------------------------------------------------
|
||||
# Our act_runner config has `container.network: host` (operator host
|
||||
# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
|
||||
# the job container AND every service container. With host-net, two
|
||||
# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
|
||||
# second postgres FATALs with `could not create any TCP/IP sockets:
|
||||
# Address in use`, and Docker auto-removes it (act_runner sets
|
||||
# AutoRemove:true on service containers). By the time the migrations
|
||||
# step runs `psql`, the postgres container is gone, hence
|
||||
# `Connection refused` then `failed to remove container: No such
|
||||
# container` at cleanup time.
|
||||
#
|
||||
# Per-job `container.network` override is silently ignored by
|
||||
# act_runner — `--network and --net in the options will be ignored.`
|
||||
# appears in the runner log. Documented constraint.
|
||||
#
|
||||
# So we sidestep `services:` entirely. The job container still uses
|
||||
# host-net (inherited from runner config; required for cache server
|
||||
# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
|
||||
# postgres on the existing `molecule-core-net` bridge with a
|
||||
# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
|
||||
# read its bridge IP via `docker inspect`. A host-net job container
|
||||
# can reach a bridge-net container directly via the bridge IP (verified
|
||||
# manually on operator host 2026-05-08).
|
||||
#
|
||||
# Trade-offs vs. the original `services:` shape:
|
||||
# + No host-port collision; N parallel runs share the bridge cleanly
|
||||
# + `if: always()` cleanup runs even on test-step failure
|
||||
# - One more step in the workflow (+~3 lines)
|
||||
# - Requires `molecule-core-net` to exist on the operator host
|
||||
# (it does; declared in docker-compose.yml + docker-compose.infra.yml)
|
||||
#
|
||||
# Class B Hongming-owned CICD red sweep, 2026-05-08.
|
||||
#
|
||||
# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
concurrency:
|
||||
group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
handlers: ${{ steps.filter.outputs.handlers }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: filter
|
||||
# Inline replacement for dorny/paths-filter — see e2e-api.yml.
|
||||
run: |
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace-server/internal/handlers/|workspace-server/internal/wsauth/|workspace-server/migrations/|\.gitea/workflows/handlers-postgres-integration\.yml$)'; then
|
||||
echo "handlers=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "handlers=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
# paths filter. See ci.yml's Platform (Go) for the same shape.
|
||||
integration:
|
||||
name: Handlers Postgres Integration
|
||||
needs: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
env:
|
||||
# Unique name per run so concurrent jobs don't collide on the
|
||||
# bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
|
||||
# workflow_dispatch reruns of the same run_id.
|
||||
PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# Bridge network already exists on the operator host (declared
|
||||
# in docker-compose.yml + docker-compose.infra.yml).
|
||||
PG_NETWORK: molecule-core-net
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.detect-changes.outputs.handlers != 'true'
|
||||
working-directory: .
|
||||
run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Start sibling Postgres on bridge network
|
||||
working-directory: .
|
||||
run: |
|
||||
# Sanity: the bridge network must exist on the operator host.
|
||||
# Hard-fail loud if it doesn't — easier to spot than a silent
|
||||
# auto-create that diverges from the rest of the stack.
|
||||
if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
|
||||
echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If a stale container with the same name exists (rerun on
|
||||
# the same run_id), wipe it first.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
|
||||
docker run -d \
|
||||
--name "${PG_NAME}" \
|
||||
--network "${PG_NETWORK}" \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 5s \
|
||||
--health-timeout 5s \
|
||||
--health-retries 10 \
|
||||
-e POSTGRES_PASSWORD=test \
|
||||
-e POSTGRES_DB=molecule \
|
||||
postgres:15-alpine >/dev/null
|
||||
|
||||
# Read back the bridge IP. Always present immediately after
|
||||
# `docker run -d` for bridge networks.
|
||||
PG_HOST=$(docker inspect "${PG_NAME}" \
|
||||
--format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
|
||||
if [ -z "${PG_HOST}" ]; then
|
||||
echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
|
||||
docker logs "${PG_NAME}" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
|
||||
echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Started ${PG_NAME} at ${PG_HOST}:5432"
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Apply migrations to Postgres service
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
# Wait for postgres to actually accept connections. Docker's
|
||||
# health-cmd handles container-side readiness, but the wire
|
||||
# to the bridge IP is best-tested with pg_isready directly.
|
||||
for i in {1..15}; do
|
||||
if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
|
||||
echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
|
||||
done
|
||||
|
||||
# Apply every .up.sql in lexicographic order with
|
||||
# ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
|
||||
# blocking the suite. This handles the current schema state
|
||||
# where a few historical migrations (e.g. 017_memories_fts_*)
|
||||
# depend on tables that were later renamed/dropped and so
|
||||
# cannot replay from scratch. The migrations that DO succeed
|
||||
# land their tables, which is sufficient for the integration
|
||||
# tests in handlers/.
|
||||
#
|
||||
# Why not maintain a curated allowlist: every new migration
|
||||
# touching a handlers/-tested table would have to update this
|
||||
# workflow. With apply-all-or-skip, a future migration that
|
||||
# adds a column to delegations runs automatically (its base
|
||||
# table 049_delegations.up.sql already succeeded above it in
|
||||
# the order). Operators only need to revisit this if the
|
||||
# migration chain becomes legitimately replayable end-to-end.
|
||||
#
|
||||
# Per-migration result is logged so a failed migration that
|
||||
# SHOULD have been replayable surfaces in the CI log instead
|
||||
# of silently failing.
|
||||
# Apply both *.sql (legacy, lives next to its module) and
|
||||
# *.up.sql (newer up/down convention) in a single
|
||||
# lexicographically-sorted pass. Excluding *.down.sql so the
|
||||
# newest-naming-convention pairs don't undo themselves mid-run.
|
||||
# Pre-#149-followup this loop only globbed *.up.sql, which
|
||||
# silently skipped 001_workspaces.sql + 009_activity_logs.sql
|
||||
# — fine while no integration test depended on those tables,
|
||||
# not fine once a cross-table atomicity test came in.
|
||||
set +e
|
||||
for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
|
||||
if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
|
||||
-f "$migration" >/dev/null 2>&1; then
|
||||
echo "✓ $(basename "$migration")"
|
||||
else
|
||||
echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
|
||||
# Sanity: the delegations + workspaces + activity_logs tables
|
||||
# MUST exist for the integration tests to be meaningful. Hard-
|
||||
# fail if any didn't land — that would be a real regression we
|
||||
# want loud.
|
||||
for tbl in delegations workspaces activity_logs pending_uploads; do
|
||||
if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
|
||||
-c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
|
||||
| grep -q 1; then
|
||||
echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ $tbl table present"
|
||||
done
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Run integration tests
|
||||
run: |
|
||||
# INTEGRATION_DB_URL is exported by the start-postgres step;
|
||||
# points at the per-run bridge IP, not 127.0.0.1, so concurrent
|
||||
# workflow runs don't fight over a host-net 5432 port.
|
||||
go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
|
||||
|
||||
- if: failure() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Diagnostic dump on failure
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
echo "::group::postgres container status"
|
||||
docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
|
||||
docker logs "${PG_NAME}" 2>&1 | tail -50 || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::delegations table state"
|
||||
psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
|
||||
echo "::endgroup::"
|
||||
|
||||
- if: always() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Stop sibling Postgres
|
||||
working-directory: .
|
||||
run: |
|
||||
# always() so containers don't leak when migrations or tests
|
||||
# fail. The cleanup is best-effort: if the container is
|
||||
# already gone (e.g. concurrent rerun race), don't fail the job.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
echo "Cleaned up ${PG_NAME}"
|
||||
@@ -1,262 +0,0 @@
|
||||
name: Harness Replays
|
||||
|
||||
# Ported from .github/workflows/harness-replays.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Boots tests/harness (production-shape compose topology with TenantGuard,
|
||||
# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
|
||||
# every replay under tests/harness/replays/. Fails the PR if any replay
|
||||
# fails.
|
||||
#
|
||||
# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
|
||||
# a public route in router.go but forgot to add it to TenantGuard's
|
||||
# allowlist. The handler-level test in buildinfo_test.go constructed a
|
||||
# minimal gin engine without TenantGuard — green. The harness's
|
||||
# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
|
||||
# inject X-Molecule-Org-Id, so the curl path is identical to production's
|
||||
# redeploy verifier), but no one ran the harness pre-merge. The bug
|
||||
# shipped; the redeploy verifier silently soft-warned every tenant as
|
||||
# "unreachable" for ~1 day before being noticed.
|
||||
#
|
||||
# This gate makes "did you actually run the harness?" a CI invariant
|
||||
# instead of a memory-discipline thing.
|
||||
#
|
||||
# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
|
||||
# to staging+main, real work is gated per-step on detect-changes output.
|
||||
# One job → one check run → branch-protection-clean (the SKIPPED-in-set
|
||||
# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.gitea/workflows/harness-replays.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.gitea/workflows/harness-replays.yml'
|
||||
concurrency:
|
||||
# Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
|
||||
# cancellation deadlock — see e2e-api.yml's concurrency block for
|
||||
# the 2026-04-28 incident that codified this pattern.
|
||||
group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- id: decide
|
||||
run: |
|
||||
# workflow_dispatch: always run (manual trigger)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine the base commit to diff against.
|
||||
# For pull_request: use base.sha (the merge-base with main/staging).
|
||||
# For push: use github.event.before (the previous tip of the branch).
|
||||
# Fallback for new branches (all-zeros SHA): run everything.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && \
|
||||
[ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "${{ github.event.before }}" ] && \
|
||||
! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
|
||||
BASE="${{ github.event.before }}"
|
||||
else
|
||||
# New branch or github.event.before unavailable — run everything.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# GitHub Actions and Gitea Actions both expose github.sha for HEAD.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.gitea/workflows/harness-replays\.yml$'; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job that always runs. Real work is gated per-step on
|
||||
# detect-changes.outputs.run so an unrelated PR (e.g. doc-only
|
||||
# change to molecule-controlplane wired here later) emits the
|
||||
# required check without spending CI cycles. Single-job pattern
|
||||
# matches e2e-api.yml — see that workflow's comment for why a
|
||||
# job-level `if: false` would block branch protection via the
|
||||
# SKIPPED-in-set bug.
|
||||
harness-replays:
|
||||
needs: detect-changes
|
||||
name: Harness Replays
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.run != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
|
||||
echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
|
||||
echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
- if: needs.detect-changes.outputs.run == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Log what files were detected so future failures include the diff.
|
||||
- name: Log detected changes
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: |
|
||||
echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
# github-app-auth sibling-checkout removed 2026-05-07 (#157):
|
||||
# the plugin was dropped + Dockerfile.tenant no longer COPYs it.
|
||||
|
||||
# Pre-clone manifest deps before docker compose builds the tenant
|
||||
# image (Task #173 followup — same pattern as
|
||||
# publish-workspace-server-image.yml's "Pre-clone manifest deps"
|
||||
# step).
|
||||
#
|
||||
# Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
|
||||
# and tenant-beta from workspace-server/Dockerfile.tenant with
|
||||
# context=../.. (repo root). That Dockerfile expects
|
||||
# .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
|
||||
# to be present at build context root (post-#173 it COPYs from there
|
||||
# instead of running an in-image clone — the in-image clone failed
|
||||
# with "could not read Username for https://git.moleculesai.app"
|
||||
# because there's no auth path inside the build sandbox).
|
||||
#
|
||||
# Without this step harness-replays fails before any replay runs,
|
||||
# with `failed to calculate checksum of ref ...
|
||||
# "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
|
||||
# (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
|
||||
# symptom, different root cause: staging still has the in-image
|
||||
# clone path, hits the auth error directly).
|
||||
#
|
||||
# 2026-05-08 sub-finding (#192): the clone step ALSO fails when
|
||||
# any referenced workspace-template repo is private and the
|
||||
# AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
|
||||
# access. Root cause: 5 of 9 workspace-template repos
|
||||
# (openclaw, codex, crewai, deepagents, gemini-cli) had been
|
||||
# marked private with no team grant. Resolution: flipped them
|
||||
# to public per `feedback_oss_first_repo_visibility_default`
|
||||
# (the OSS surface should be public). Layer-3 (customer-private +
|
||||
# marketplace third-party repos) tracked separately in
|
||||
# internal#102.
|
||||
#
|
||||
# Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
|
||||
# is the devops-engineer persona PAT, NOT the founder PAT (per
|
||||
# `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
|
||||
# embeds it as basic-auth for the duration of the clones and strips
|
||||
# .git directories — the token never enters the resulting image.
|
||||
- name: Pre-clone manifest deps
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
env:
|
||||
MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
bash scripts/clone-manifest.sh \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
# Sanity-check counts so a silent partial clone fails fast
|
||||
# instead of producing a half-empty image.
|
||||
ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
|
||||
|
||||
- name: Install Python deps for replays
|
||||
# peer-discovery-404 (and future replays) eval Python against the
|
||||
# running tenant — importing workspace/a2a_client.py pulls in
|
||||
# httpx. tests/harness/requirements.txt holds just the HTTP-client
|
||||
# surface to keep CI install fast (~3s) vs the full
|
||||
# workspace/requirements.txt (~30s).
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: pip install -r tests/harness/requirements.txt
|
||||
|
||||
- name: Run all replays against the harness
|
||||
# run-all-replays.sh: boot via up.sh → seed via seed.sh → run
|
||||
# every replays/*.sh → tear down via down.sh on EXIT (trap).
|
||||
# Non-zero exit on any replay failure.
|
||||
#
|
||||
# KEEP_UP=1: without this, the script's trap-on-EXIT tears
|
||||
# down containers immediately on failure, leaving the dump
|
||||
# step below with nothing to dump (verified on PR #2410's
|
||||
# first run — tenant became unhealthy, trap fired, dump
|
||||
# step saw empty containers). Keeping them up lets the
|
||||
# failure path collect tenant/cp-stub/cf-proxy logs. The
|
||||
# always-run "Force teardown" step does the actual cleanup.
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
KEEP_UP: "1"
|
||||
run: ./run-all-replays.sh
|
||||
|
||||
- name: Dump compose logs on failure
|
||||
# SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
|
||||
# file even for read-only `logs` calls. up.sh generates a per-run key
|
||||
# and exports it to its OWN shell — this step runs in a fresh shell
|
||||
# that wouldn't see it, so without a placeholder the validate step
|
||||
# errors before logs print (verified against PR #2492's first run:
|
||||
# "required variable SECRETS_ENCRYPTION_KEY is missing a value").
|
||||
# A placeholder is fine — we're only reading log streams, not booting.
|
||||
if: failure() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
|
||||
run: |
|
||||
echo "=== docker compose ps ==="
|
||||
docker compose -f compose.yml ps || true
|
||||
echo "=== tenant-alpha logs ==="
|
||||
docker compose -f compose.yml logs tenant-alpha || true
|
||||
echo "=== tenant-beta logs ==="
|
||||
docker compose -f compose.yml logs tenant-beta || true
|
||||
echo "=== cp-stub logs ==="
|
||||
docker compose -f compose.yml logs cp-stub || true
|
||||
echo "=== cf-proxy logs ==="
|
||||
docker compose -f compose.yml logs cf-proxy || true
|
||||
echo "=== postgres-alpha logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-alpha || true
|
||||
echo "=== postgres-beta logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-beta || true
|
||||
|
||||
- name: Force teardown
|
||||
# We pass KEEP_UP=1 to run-all-replays.sh so the dump step
|
||||
# above sees real containers — that means we own teardown
|
||||
# explicitly here. Always run.
|
||||
if: always() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
run: ./down.sh || true
|
||||
@@ -1,104 +0,0 @@
|
||||
name: Lint curl status-code capture
|
||||
|
||||
# Ported from .github/workflows/lint-curl-status-capture.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths and the lint scanner target .gitea/workflows/**.yml (the
|
||||
# active Gitea workflow directory) instead of .github/workflows/**.yml
|
||||
# (which the rest of this sweep is emptying out).
|
||||
# - Self-skip path updated to the .gitea/ version of this file.
|
||||
# - Dropped `merge_group:` trigger.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
|
||||
# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
|
||||
#
|
||||
# HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
|
||||
#
|
||||
# When curl exits non-zero (connection reset -> 56, --fail-with-body 4xx/5xx
|
||||
# -> 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
|
||||
# "000" for connection failures or the actual code for HTTP errors. The
|
||||
# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
|
||||
# stdout, producing values like "000000" or "409000" that fail string
|
||||
# comparisons against "200" while looking superficially right.
|
||||
#
|
||||
# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
|
||||
# #2797). Memory: feedback_curl_status_capture_pollution.md.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths: ['.gitea/workflows/**']
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths: ['.gitea/workflows/**']
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan workflows for curl status-capture pollution
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
|
||||
# subshell where the entire command-substitution wraps a curl that
|
||||
# ends with `|| echo "000"`. Must distinguish from the SAFE shape
|
||||
# `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
|
||||
# tempfile produces empty stdout, no pollution.
|
||||
python3 <<'PY'
|
||||
import os, re, sys, glob
|
||||
|
||||
BAD_FILES = []
|
||||
|
||||
# Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
|
||||
# The `\\n` is the bash line-continuation that lets curl flags span lines.
|
||||
# We collapse continuation lines first, then look for the single-line bad pattern.
|
||||
PATTERN = re.compile(
|
||||
r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Self-skip: this lint workflow contains the literal anti-pattern in
|
||||
# its own docstring — that's intentional, not a bug.
|
||||
SELF = ".gitea/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
for f in sorted(glob.glob(".gitea/workflows/*.yml")):
|
||||
if f == SELF:
|
||||
continue
|
||||
with open(f) as fh:
|
||||
content = fh.read()
|
||||
# Collapse bash line-continuations (\\\n + leading whitespace)
|
||||
# into a single logical line so the regex can see the full
|
||||
# curl invocation as one chunk.
|
||||
flat = re.sub(r'\\\s*\n\s*', ' ', content)
|
||||
for m in PATTERN.finditer(flat):
|
||||
BAD_FILES.append((f, m.group(0)[:120]))
|
||||
|
||||
if not BAD_FILES:
|
||||
print("OK No curl-status-capture pollution patterns detected")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
|
||||
for f, snippet in BAD_FILES:
|
||||
print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
|
||||
print(f" matched: {snippet}...")
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(' set +e')
|
||||
print(' curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
|
||||
print(' set -e')
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
sys.exit(1)
|
||||
PY
|
||||
@@ -1,94 +0,0 @@
|
||||
# main-red-watchdog — hourly sentinel for post-merge CI red on `main`.
|
||||
#
|
||||
# RFC: hongming "main NEVER goes red" directive, Option C of the four-
|
||||
# option ladder (B = auto-revert is explicitly rejected per
|
||||
# `feedback_no_such_thing_as_flakes` + `feedback_fix_root_not_symptom`).
|
||||
# Tracking issue: molecule-core#420.
|
||||
#
|
||||
# What it does:
|
||||
# 1. GET branches/main → HEAD SHA
|
||||
# 2. GET commits/{SHA}/status → combined status
|
||||
# 3. If combined is `failure` (or any individual status is `failure`):
|
||||
# open or PATCH an idempotent `[main-red] {repo}: {SHA[:10]}` issue
|
||||
# with each failed context + target_url + description.
|
||||
# 4. If combined is `success` and a prior `[main-red] ...` issue exists,
|
||||
# close it with a "main returned to green at SHA ..." comment.
|
||||
# 5. Emit a Loki-shaped JSON line via `logger -t main-red-watchdog` for
|
||||
# `reference_obs_stack_phase1` ingestion via Vector.
|
||||
#
|
||||
# What it does NOT do:
|
||||
# - Auto-revert anything. Option B is rejected by directive.
|
||||
# - Mutate branch protection. (See AGENTS.md boundaries.)
|
||||
# - Fail the workflow on red. The issue IS the alarm — failing the
|
||||
# watchdog would create a silent-loop where a flake in the watchdog
|
||||
# itself hides actual main-red signal. Exit 0 unless api() raises
|
||||
# ApiError (transient Gitea outage → fail loudly per
|
||||
# `feedback_api_helper_must_raise_not_return_dict`).
|
||||
#
|
||||
# Pattern source: molecule-controlplane `0adf2098`'s ci-required-drift.yml
|
||||
# (just merged 2026-05-11). Same shape (cron + dispatch + sidecar Python +
|
||||
# idempotent-by-title issue), simpler scope (1 source, not 3).
|
||||
|
||||
name: main-red-watchdog
|
||||
|
||||
# IMPORTANT — Gitea 1.22.6 parser quirk per
|
||||
# `feedback_gitea_workflow_dispatch_inputs_unsupported`: do NOT add an
|
||||
# `inputs:` block here. Gitea 1.22.6 rejects the whole workflow as
|
||||
# "unknown on type" when `workflow_dispatch.inputs.X` is present. Revisit
|
||||
# when Gitea ≥ 1.23 is fleet-wide.
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :05 — task spec calls for "off-zero" (`5 * * * *`),
|
||||
# offset from :17 (ci-required-drift) and :00 (peak cron load).
|
||||
- cron: '5 * * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
# Read commit status + branch ref + issues; write issues (open/PATCH/close).
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
# Workflow-scoped serialisation — two simultaneous runs would race on the
|
||||
# `[main-red] {SHA}` open/PATCH path. Idempotent by title, but parallel
|
||||
# POSTs can produce duplicates before the title search dedup wins.
|
||||
concurrency:
|
||||
group: main-red-watchdog
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
watchdog:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check out repo (script lives at .gitea/scripts/)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up Python (stdlib only — no PyYAML needed here)
|
||||
# The script uses stdlib urllib + json. No PyYAML required (CP's
|
||||
# drift detector needs it for AST parsing; we don't). Pin to the
|
||||
# same 3.12 hermetic interpreter CP uses so the test/runtime
|
||||
# versions stay aligned across watchdog suites.
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Run main-red watchdog
|
||||
env:
|
||||
# GITEA_TOKEN reads commit status + writes issues. Falls back
|
||||
# to the auto-injected GITHUB_TOKEN if the org-level secret
|
||||
# isn't set (transitional repos), matching the same pattern
|
||||
# used by deploy-pipeline.yml + ci-required-drift.yml.
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
# Branch under watch. `main` per directive; staging not
|
||||
# included here — staging green is a separate gate
|
||||
# (`feedback_staging_e2e_merge_gate`).
|
||||
WATCH_BRANCH: 'main'
|
||||
# Issue label applied on file/open. `tier:high` exists in the
|
||||
# molecule-core label set (verified 2026-05-11, label id 9).
|
||||
# Rationale for high: main red blocks the promotion train and
|
||||
# poisons every PR's auto-rebase base; treat as a fire even
|
||||
# if intermittent.
|
||||
RED_LABEL: 'tier:high'
|
||||
run: python3 .gitea/scripts/main-red-watchdog.py
|
||||
@@ -1,138 +0,0 @@
|
||||
name: publish-canvas-image
|
||||
|
||||
# Ported from .github/workflows/publish-canvas-image.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Open question for review**: this workflow pushes the canvas
|
||||
# image to `ghcr.io`. GHCR was retired during the 2026-05-06
|
||||
# Gitea migration in favor of ECR (per canary-verify.yml header
|
||||
# notes). The image may not be consumable post-migration. Two
|
||||
# options for follow-up: (a) retarget to
|
||||
# `153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas`,
|
||||
# or (b) retire this workflow entirely and route canvas deploys
|
||||
# via the operator-host build path. tier:low + continue-on-error
|
||||
# means failed pushes do not block PRs.
|
||||
#
|
||||
|
||||
# Builds and pushes the canvas Docker image to GHCR whenever a commit lands
|
||||
# on main that touches canvas code. Previously canvas changes were visible in
|
||||
# CI (npm run build passed) but the live container was never updated —
|
||||
# operators had to manually run `docker compose build canvas` each time.
|
||||
#
|
||||
# Mirror of publish-platform-image.yml, adapted for the Next.js canvas layer.
|
||||
# See that workflow for inline notes on macOS Keychain isolation and QEMU.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
# Only rebuild when canvas source changes — saves GHA minutes on
|
||||
# platform-only / docs-only / MCP-only merges.
|
||||
- 'canvas/**'
|
||||
- '.gitea/workflows/publish-canvas-image.yml'
|
||||
# NOTE (Gitea port): the original GitHub workflow had a
|
||||
# `workflow_dispatch:` manual trigger for the
|
||||
# non-canvas-merge-but-need-fresh-image scenario. Dropped in the
|
||||
# Gitea port (1.22.6 parser-finicky). Manual rebuilds require
|
||||
# pushing an empty commit to canvas/ or running the operator-host
|
||||
# build directly.
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write # required to push to ghcr.io/${{ github.repository_owner }}/*
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/canvas
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build & push canvas image
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing to the build step
|
||||
# where docker build fails deep in ECR auth with a cryptic error.
|
||||
- name: Verify Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
|
||||
exit 1
|
||||
}
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Compute tags
|
||||
id: tags
|
||||
shell: bash
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve build args
|
||||
id: build_args
|
||||
# Priority: workflow_dispatch input > repo secret > hardcoded default.
|
||||
# NEXT_PUBLIC_* env vars are baked into the JS bundle at build time by
|
||||
# Next.js — they cannot be changed at runtime without a full rebuild.
|
||||
# For local docker-compose deployments the defaults (localhost:8080)
|
||||
# work as-is; production deployments should set CANVAS_PLATFORM_URL
|
||||
# and CANVAS_WS_URL as repository secrets.
|
||||
#
|
||||
# Inputs are passed via env vars (not direct ${{ }} interpolation) to
|
||||
# prevent shell injection from workflow_dispatch string inputs.
|
||||
shell: bash
|
||||
env:
|
||||
INPUT_PLATFORM_URL: ${{ github.event.inputs.platform_url }}
|
||||
SECRET_PLATFORM_URL: ${{ secrets.CANVAS_PLATFORM_URL }}
|
||||
INPUT_WS_URL: ${{ github.event.inputs.ws_url }}
|
||||
SECRET_WS_URL: ${{ secrets.CANVAS_WS_URL }}
|
||||
run: |
|
||||
PLATFORM_URL="${INPUT_PLATFORM_URL:-${SECRET_PLATFORM_URL:-http://localhost:8080}}"
|
||||
WS_URL="${INPUT_WS_URL:-${SECRET_WS_URL:-ws://localhost:8080/ws}}"
|
||||
|
||||
echo "platform_url=${PLATFORM_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build & push canvas image to GHCR
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: ./canvas
|
||||
file: ./canvas/Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PLATFORM_URL=${{ steps.build_args.outputs.platform_url }}
|
||||
NEXT_PUBLIC_WS_URL=${{ steps.build_args.outputs.ws_url }}
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:latest
|
||||
${{ env.IMAGE_NAME }}:sha-${{ steps.tags.outputs.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI canvas (Next.js 15 + React Flow)
|
||||
@@ -1,100 +0,0 @@
|
||||
name: publish-runtime-autobump
|
||||
|
||||
# Auto-bump-on-workspace-edit half of the publish pipeline.
|
||||
#
|
||||
# Why this file exists (issue #351):
|
||||
# Gitea Actions does not correctly disambiguate `paths:` from `tags:`
|
||||
# when both are bundled under a single `on.push` key. The result is
|
||||
# that tag pushes get filtered out and `publish-runtime.yml` never
|
||||
# fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
|
||||
# because PYPI_TOKEN was absent (publishes would have failed anyway).
|
||||
#
|
||||
# Split design:
|
||||
# - publish-runtime.yml : on.push.tags only (the publisher)
|
||||
# - publish-runtime-autobump.yml: on.push.branches+paths (this file — the version-bumper)
|
||||
#
|
||||
# This file computes the next version from PyPI's latest, pushes a
|
||||
# `runtime-v$VERSION` tag, and exits. The tag push then triggers
|
||||
# publish-runtime.yml via its tags-only trigger.
|
||||
#
|
||||
# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
|
||||
# so concurrent workspace pushes serialize at the bump step. Without
|
||||
# this, two pushes minutes apart could both read PyPI latest=0.1.129
|
||||
# and try to tag 0.1.130 simultaneously, only one of which would land.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- staging
|
||||
paths:
|
||||
- "workspace/**"
|
||||
|
||||
permissions:
|
||||
contents: write # required to push tags back
|
||||
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
autobump-and-tag:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Fetch full tag list so the bump logic can sanity-check against
|
||||
# what's already in this repo (catches collision with prior
|
||||
# manual tag pushes).
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Compute next version from PyPI latest
|
||||
id: bump
|
||||
run: |
|
||||
set -eu
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "PyPI latest=$LATEST -> next=$VERSION"
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
|
||||
exit 1
|
||||
fi
|
||||
if git tag --list | grep -qx "runtime-v$VERSION"; then
|
||||
echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Push runtime-v$VERSION tag
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
VERSION: ${{ steps.bump.outputs.version }}
|
||||
GITEA_URL: https://git.moleculesai.app
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
|
||||
exit 1
|
||||
fi
|
||||
git config user.name "publish-runtime autobump"
|
||||
git config user.email "publish-runtime@moleculesai.app"
|
||||
git tag -a "runtime-v$VERSION" \
|
||||
-m "Auto-bump on workspace/** edit on $GITHUB_REF" \
|
||||
-m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
|
||||
-m "publish-runtime.yml will pick up this tag and upload to PyPI"
|
||||
# Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
|
||||
# ensures the resulting tag-push event is dispatched to
|
||||
# publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
|
||||
# trigger downstream workflows.
|
||||
git remote set-url origin "${GITEA_URL#https://}"
|
||||
git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
|
||||
git push origin "runtime-v$VERSION"
|
||||
echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"
|
||||
@@ -1,339 +0,0 @@
|
||||
name: publish-runtime
|
||||
|
||||
# Gitea Actions port of .github/workflows/publish-runtime.yml.
|
||||
#
|
||||
# Ported 2026-05-10 (issue #206). Key differences from the GitHub version:
|
||||
# - Gitea Actions reads .gitea/workflows/, not .github/workflows/
|
||||
# - Dropped `environment: pypi-publish` — Gitea Actions does not support
|
||||
# named environments or OIDC trusted publishers
|
||||
# - Replaced `pypa/gh-action-pypi-publish@release/v1` (OIDC) with
|
||||
# `twine upload` using PYPI_TOKEN secret — same mechanism as a local
|
||||
# `python -m twine upload` with a PyPI token
|
||||
# - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
|
||||
# — Gitea Actions exposes github.ref (the full ref) but not ref_name
|
||||
# - Dropped `merge_group` trigger (Gitea has no merge queue)
|
||||
#
|
||||
# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
|
||||
# `workspace/**` path-filter trigger in PR #349.
|
||||
#
|
||||
# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
|
||||
# file. Bundling `paths` with `tags` under a single `on.push` key caused
|
||||
# Gitea Actions to never dispatch the workflow for tag-push events (0
|
||||
# runs in `action_run` for workflow_id='publish-runtime.yml' since the
|
||||
# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
|
||||
# 0.1.129 despite a v1.0.0 Gitea tag existing).
|
||||
#
|
||||
# The auto-bump-on-workspace-edit trigger now lives in
|
||||
# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
|
||||
# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
|
||||
# which THIS file then picks up via the tags-only trigger below.
|
||||
#
|
||||
# This decoupling means Gitea's path-vs-tag evaluator never has to
|
||||
# disambiguate — each file has a single unambiguous trigger shape.
|
||||
#
|
||||
# PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
|
||||
# Set via: repo Settings → Actions → Variables and Secrets → New Secret.
|
||||
# The token should be a PyPI API token scoped to molecule-ai-workspace-runtime.
|
||||
#
|
||||
# The DISPATCH_TOKEN cascade (git push to template repos) is unchanged —
|
||||
# it uses the Gitea API directly and was already Gitea-compatible.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "runtime-v*"
|
||||
workflow_dispatch:
|
||||
# 2026-05-11 (root cause of #351 / 0 runs ever):
|
||||
# Gitea 1.22.6's workflow parser rejects `workflow_dispatch.inputs.version`
|
||||
# with "unknown on type" — it mis-treats the inputs sub-keys as top-level
|
||||
# `on:` event types. Log line:
|
||||
# actions/workflows.go:DetectWorkflows() [W] ignore invalid workflow
|
||||
# "publish-runtime.yml": unknown on type: map["version": {...}]
|
||||
# That `[W] ignore invalid workflow` is silent UX — the workflow never
|
||||
# registers, so it never fires for ANY event (push.tags included).
|
||||
# Removing the inputs block restores parsing. Manual dispatch from the
|
||||
# Gitea UI now triggers the PyPI auto-bump fallback in `Derive version`
|
||||
# below (no `inputs.version` to read).
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Serialize publishes so two concurrent tag pushes don't both compute
|
||||
# "latest+1" and race on PyPI upload. The second one waits.
|
||||
concurrency:
|
||||
group: publish-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
cache: pip
|
||||
|
||||
- name: Derive version (tag or PyPI auto-bump)
|
||||
id: version
|
||||
run: |
|
||||
if echo "$GITHUB_REF" | grep -q "^refs/tags/runtime-v"; then
|
||||
# Tag is `runtime-vX.Y.Z` — strip the prefix.
|
||||
VERSION="${GITHUB_REF#refs/tags/runtime-v}"
|
||||
else
|
||||
# workflow_dispatch path (no inputs supported on Gitea 1.22.6) or
|
||||
# any other non-tag trigger: derive from PyPI latest + patch bump.
|
||||
LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
|
||||
| python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
|
||||
MAJOR=$(echo "$LATEST" | cut -d. -f1)
|
||||
MINOR=$(echo "$LATEST" | cut -d. -f2)
|
||||
PATCH=$(echo "$LATEST" | cut -d. -f3)
|
||||
VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
|
||||
echo "Auto-bumped from PyPI latest $LATEST -> $VERSION"
|
||||
fi
|
||||
if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+|rc[0-9]+|a[0-9]+|b[0-9]+|\.post[0-9]+)?$'; then
|
||||
echo "::error::version $VERSION does not match PEP 440"
|
||||
exit 1
|
||||
fi
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "Publishing molecule-ai-workspace-runtime $VERSION"
|
||||
|
||||
- name: Install build tooling
|
||||
run: pip install build twine
|
||||
|
||||
- name: Build package from workspace/
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "${{ steps.version.outputs.version }}" \
|
||||
--out "${{ runner.temp }}/runtime-build"
|
||||
|
||||
- name: Build wheel + sdist
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: python -m build
|
||||
|
||||
- name: Capture wheel SHA256 for cascade content-verification
|
||||
id: wheel_hash
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
set -eu
|
||||
WHEEL=$(ls dist/*.whl 2>/dev/null | head -1)
|
||||
if [ -z "$WHEEL" ]; then
|
||||
echo "::error::No .whl in dist/ — \`python -m build\` must have failed silently"
|
||||
exit 1
|
||||
fi
|
||||
HASH=$(sha256sum "$WHEEL" | awk '{print $1}')
|
||||
echo "wheel_sha256=${HASH}" >> "$GITHUB_OUTPUT"
|
||||
echo "Local wheel SHA256 (pre-upload): ${HASH}"
|
||||
echo "Wheel filename: $(basename "$WHEEL")"
|
||||
|
||||
- name: Verify package contents (sanity)
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
run: |
|
||||
python -m twine check dist/*
|
||||
python -m venv /tmp/smoke
|
||||
/tmp/smoke/bin/pip install --quiet dist/*.whl
|
||||
/tmp/smoke/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
|
||||
- name: Publish to PyPI
|
||||
# working-directory matches the preceding Build/Verify steps. Without
|
||||
# this, twine runs from the default workspace checkout dir where
|
||||
# `dist/` doesn't exist and fails with:
|
||||
# ERROR InvalidDistribution: Cannot find file (or expand pattern): 'dist/*'
|
||||
# Caught on the first-ever successful dispatch of this workflow
|
||||
# (run 5097, 2026-05-11 02:08Z) — every other step in the publish
|
||||
# job already had this working-directory; Publish was missing it.
|
||||
working-directory: ${{ runner.temp }}/runtime-build
|
||||
env:
|
||||
# PYPI_TOKEN: repository secret scoped to molecule-ai-workspace-runtime.
|
||||
# Set via: Settings → Actions → Variables and Secrets → New Secret.
|
||||
# Format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
|
||||
PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$PYPI_TOKEN" ]; then
|
||||
echo "::error::PYPI_TOKEN secret is not set — set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
echo "::error::Required format: pypi-XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
|
||||
exit 1
|
||||
fi
|
||||
python -m twine upload \
|
||||
--repository pypi \
|
||||
--username __token__ \
|
||||
--password "$PYPI_TOKEN" \
|
||||
dist/*
|
||||
|
||||
cascade:
|
||||
needs: publish
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Wait for PyPI to propagate the new version
|
||||
env:
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
EXPECTED_SHA256: ${{ needs.publish.outputs.wheel_sha256 }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -z "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::publish job did not expose wheel_sha256 — cannot verify wheel content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
python -m venv /tmp/propagation-probe
|
||||
PROBE=/tmp/propagation-probe/bin
|
||||
$PROBE/pip install --upgrade --quiet pip
|
||||
for i in $(seq 1 30); do
|
||||
if $PROBE/pip install \
|
||||
--quiet \
|
||||
--no-cache-dir \
|
||||
--force-reinstall \
|
||||
--no-deps \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1; then
|
||||
INSTALLED=$($PROBE/pip show molecule-ai-workspace-runtime 2>/dev/null \
|
||||
| awk -F': ' '/^Version:/{print $2}')
|
||||
if [ "$INSTALLED" = "$RUNTIME_VERSION" ]; then
|
||||
echo "✓ PyPI resolved $RUNTIME_VERSION (install check)"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "::error::pip install --no-cache-dir molecule-ai-workspace-runtime==${RUNTIME_VERSION} never resolved within ~5 min."
|
||||
echo "::error::Refusing to fan out cascade against a potentially stale PyPI index."
|
||||
exit 1
|
||||
fi
|
||||
echo " [$i/30] waiting for PyPI to propagate ${RUNTIME_VERSION}..."
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Stage (b): download wheel + SHA256 compare against what we built.
|
||||
# Catches Fastly stale-content serving old bytes under a new version URL.
|
||||
#
|
||||
# Caught run 5196 (first-ever successful publish, 2026-05-11): the
|
||||
# previous one-liner `HASH=$(pip download ... && sha256sum ...)`
|
||||
# captured pip's stdout (`Collecting molecule-ai-workspace-runtime
|
||||
# ==X.Y.Z`) into HASH, then the SHA comparison failed against the
|
||||
# leaked `Collecting...` string. `2>/dev/null` silences stderr but
|
||||
# NOT stdout; pip writes its progress to stdout by default.
|
||||
# Fix: split into two steps, silence pip's stdout explicitly, capture
|
||||
# only sha256sum's output into HASH.
|
||||
python -m pip download \
|
||||
--no-deps \
|
||||
--no-cache-dir \
|
||||
--dest /tmp/wheel-probe \
|
||||
--quiet \
|
||||
"molecule-ai-workspace-runtime==${RUNTIME_VERSION}" \
|
||||
>/dev/null 2>&1
|
||||
HASH=$(sha256sum /tmp/wheel-probe/*.whl | awk '{print $1}')
|
||||
if [ "$HASH" != "$EXPECTED_SHA256" ]; then
|
||||
echo "::error::PyPI propagated $RUNTIME_VERSION but wheel content SHA256 mismatch."
|
||||
echo "::error::Expected: $EXPECTED_SHA256"
|
||||
echo "::error::Got: $HASH"
|
||||
echo "::error::Fastly may be serving stale content. Refusing to fan out cascade."
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ PyPI CDN verified (SHA256 match)"
|
||||
|
||||
- name: Fan out via push to .runtime-version
|
||||
env:
|
||||
# Gitea PAT with write:repository scope on the 8 cascade-active
|
||||
# template repos. Used for git push to each template repo's main
|
||||
# branch, which trips their `on: push: branches: [main]` trigger
|
||||
# on publish-image.yml.
|
||||
DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
|
||||
RUNTIME_VERSION: ${{ needs.publish.outputs.version }}
|
||||
run: |
|
||||
set +e # don't abort on a single repo failure — collect them all
|
||||
|
||||
if [ -z "$DISPATCH_TOKEN" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::DISPATCH_TOKEN secret not set — skipping cascade."
|
||||
echo "::warning::set it at Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::DISPATCH_TOKEN secret missing — cascade cannot fan out."
|
||||
echo "::error::PyPI was published, but the 8 template repos will NOT pick up the new version."
|
||||
exit 1
|
||||
fi
|
||||
VERSION="$RUNTIME_VERSION"
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::publish job did not expose a version output"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GITEA_URL="${GITEA_URL:-https://git.moleculesai.app}"
|
||||
TEMPLATES="claude-code hermes openclaw codex langgraph crewai autogen deepagents gemini-cli"
|
||||
FAILED=""
|
||||
SKIPPED=""
|
||||
|
||||
git config --global user.name "publish-runtime cascade"
|
||||
git config --global user.email "publish-runtime@moleculesai.app"
|
||||
|
||||
WORKDIR="$(mktemp -d)"
|
||||
for tpl in $TEMPLATES; do
|
||||
REPO="molecule-ai/molecule-ai-workspace-template-$tpl"
|
||||
CLONE="$WORKDIR/$tpl"
|
||||
|
||||
HTTP=$(curl -sS -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: token $DISPATCH_TOKEN" \
|
||||
"$GITEA_URL/api/v1/repos/$REPO/contents/.github/workflows/publish-image.yml")
|
||||
if [ "$HTTP" = "404" ]; then
|
||||
echo "↷ $tpl has no publish-image.yml — soft-skip"
|
||||
SKIPPED="$SKIPPED $tpl"
|
||||
continue
|
||||
fi
|
||||
|
||||
attempt=0
|
||||
success=false
|
||||
while [ $attempt -lt 3 ]; do
|
||||
attempt=$((attempt + 1))
|
||||
rm -rf "$CLONE"
|
||||
if ! git clone --depth=1 \
|
||||
"https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/$REPO.git" \
|
||||
"$CLONE" >/tmp/clone.log 2>&1; then
|
||||
echo "::warning::clone $tpl attempt $attempt failed: $(tail -n3 /tmp/clone.log)"
|
||||
sleep 2
|
||||
continue
|
||||
fi
|
||||
|
||||
cd "$CLONE"
|
||||
echo "$VERSION" > .runtime-version
|
||||
|
||||
if git diff --quiet -- .runtime-version; then
|
||||
echo "✓ $tpl already at $VERSION — no commit needed"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
git add .runtime-version
|
||||
git commit -m "chore: pin runtime to $VERSION (publish-runtime cascade)" \
|
||||
-m "Co-Authored-By: publish-runtime cascade <publish-runtime@moleculesai.app>" \
|
||||
>/dev/null
|
||||
|
||||
if git push origin HEAD:main >/tmp/push.log 2>&1; then
|
||||
echo "✓ $tpl pushed $VERSION on attempt $attempt"
|
||||
success=true
|
||||
cd - >/dev/null
|
||||
break
|
||||
fi
|
||||
|
||||
echo "::warning::push $tpl attempt $attempt failed, pull-rebasing"
|
||||
git pull --rebase origin main >/tmp/rebase.log 2>&1 || true
|
||||
cd - >/dev/null
|
||||
done
|
||||
|
||||
if [ "$success" != "true" ]; then
|
||||
FAILED="$FAILED $tpl"
|
||||
fi
|
||||
done
|
||||
rm -rf "$WORKDIR"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "::error::Cascade incomplete after 3 retries each. Failed:$FAILED"
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$SKIPPED" ]; then
|
||||
echo "Cascade complete: pinned $VERSION. Soft-skipped (no publish-image.yml):$SKIPPED"
|
||||
else
|
||||
echo "Cascade complete: $VERSION pinned across all manifest workspace_templates."
|
||||
fi
|
||||
@@ -1,174 +0,0 @@
|
||||
name: publish-workspace-server-image
|
||||
|
||||
# Gitea Actions port of .github/workflows/publish-workspace-server-image.yml.
|
||||
#
|
||||
# Ported 2026-05-10 (issue #228). Key differences from the GitHub version:
|
||||
# - Gitea Actions reads .gitea/workflows/, not .github/workflows/
|
||||
# - Dropped `environment:` declarations — Gitea Actions does not support
|
||||
# named environments (used by GitHub OIDC token gates)
|
||||
# - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/heads/}`
|
||||
# — Gitea Actions exposes GITHUB_REF in the same format as GitHub Actions
|
||||
# - docker/setup-buildx-action and aws-actions/configure-aws-credentials are
|
||||
# GitHub Marketplace actions; they are installed by Gitea Actions runners and
|
||||
# work identically here
|
||||
# - All other variables (GITHUB_SHA, GITHUB_REPOSITORY, GITHUB_OUTPUT,
|
||||
# secrets.*) use the same syntax as GitHub Actions
|
||||
#
|
||||
# Image tags produced:
|
||||
# :staging-<sha> — per-commit digest, stable for canary verify
|
||||
# :staging-latest — tracks most recent build on this branch
|
||||
#
|
||||
# ECR target: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/*
|
||||
# Required secrets: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AUTO_SYNC_TOKEN
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'manifest.json'
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/publish-workspace-server-image.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
# Serialize per-branch so two rapid staging pushes don't race the same
|
||||
# :staging-latest tag retag. Allow staging and main to run in parallel
|
||||
# (different GITHUB_REF → different concurrency group) since they
|
||||
# produce different :staging-<sha> tags and last-write-wins on
|
||||
# :staging-latest is acceptable across branches.
|
||||
#
|
||||
# cancel-in-progress: false → in-flight builds finish; the next push's
|
||||
# build queues. This avoids a partially-pushed image.
|
||||
concurrency:
|
||||
group: publish-workspace-server-image-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible (e.g. permission change, daemon restart, or group-membership
|
||||
# drift) rather than silently continuing to step 2 where `docker build`
|
||||
# fails deep in the process with a cryptic ECR auth error that doesn't
|
||||
# surface the root cause. Also reports the daemon version so operator
|
||||
# can correlate with runner host logs.
|
||||
- name: Verify Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Check: (1) daemon is running, (2) runner user is in docker group, (3) sock permissions are 660+"
|
||||
exit 1
|
||||
}
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
|
||||
# Pre-clone manifest deps before docker build.
|
||||
#
|
||||
# Why: workspace-template-* repos on Gitea are private. The pre-fix
|
||||
# Dockerfile.tenant ran `git clone` inside an in-image stage with no
|
||||
# auth path — every CI build failed. We clone in the trusted CI
|
||||
# context where AUTO_SYNC_TOKEN is available and Dockerfile.tenant
|
||||
# just COPYs from .tenant-bundle-deps/.
|
||||
#
|
||||
# Token: AUTO_SYNC_TOKEN is the devops-engineer persona PAT.
|
||||
# clone-manifest.sh embeds it as basic-auth for the clones, then
|
||||
# strips .git dirs — the token never enters the image.
|
||||
- name: Pre-clone manifest deps
|
||||
env:
|
||||
MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
bash scripts/clone-manifest.sh \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
|
||||
|
||||
- name: Compute tags
|
||||
id: tags
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Build + push platform image (inline ECR auth — mirrors the operator-host
|
||||
# approach; credentials come from GITHUB_SECRET_AWS_ACCESS_KEY_ID /
|
||||
# GITHUB_SECRET_AWS_SECRET_ACCESS_KEY in Gitea Actions).
|
||||
- name: Build & push platform image to ECR (staging-<sha> + staging-latest)
|
||||
env:
|
||||
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
||||
TAG_SHA: staging-${{ steps.tags.outputs.sha }}
|
||||
TAG_LATEST: staging-latest
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
REPO: ${{ github.repository }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
docker build \
|
||||
--file ./workspace-server/Dockerfile \
|
||||
--build-arg GIT_SHA="${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/${REPO}" \
|
||||
--label "org.opencontainers.image.revision=${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.description=Molecule AI platform — pending canary verify" \
|
||||
--tag "${IMAGE_NAME}:${TAG_SHA}" \
|
||||
--tag "${IMAGE_NAME}:${TAG_LATEST}" \
|
||||
.
|
||||
docker push "${IMAGE_NAME}:${TAG_SHA}"
|
||||
docker push "${IMAGE_NAME}:${TAG_LATEST}"
|
||||
|
||||
# Build + push tenant image (Go platform + Next.js canvas in one image).
|
||||
- name: Build & push tenant image to ECR (staging-<sha> + staging-latest)
|
||||
env:
|
||||
TENANT_IMAGE_NAME: ${{ env.TENANT_IMAGE_NAME }}
|
||||
TAG_SHA: staging-${{ steps.tags.outputs.sha }}
|
||||
TAG_LATEST: staging-latest
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
REPO: ${{ github.repository }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ECR_REGISTRY="${TENANT_IMAGE_NAME%%/*}"
|
||||
aws ecr get-login-password --region us-east-2 | \
|
||||
docker login --username AWS --password-stdin "${ECR_REGISTRY}"
|
||||
docker build \
|
||||
--file ./workspace-server/Dockerfile.tenant \
|
||||
--build-arg NEXT_PUBLIC_PLATFORM_URL= \
|
||||
--build-arg GIT_SHA="${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/${REPO}" \
|
||||
--label "org.opencontainers.image.revision=${GIT_SHA}" \
|
||||
--label "org.opencontainers.image.description=Molecule AI tenant platform + canvas — pending canary verify" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_SHA}" \
|
||||
--tag "${TENANT_IMAGE_NAME}:${TAG_LATEST}" \
|
||||
.
|
||||
docker push "${TENANT_IMAGE_NAME}:${TAG_SHA}"
|
||||
docker push "${TENANT_IMAGE_NAME}:${TAG_LATEST}"
|
||||
@@ -1,181 +0,0 @@
|
||||
name: Railway pin audit (drift detection)
|
||||
|
||||
# Ported from .github/workflows/railway-pin-audit.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch:` (Gitea 1.22.6 trigger handling).
|
||||
# Manual runs go via cron-trigger bump or push the workflow file
|
||||
# itself.
|
||||
# - `actions/github-script@v9` blocks (which call github.rest.* — a
|
||||
# GitHub-specific JS API) replaced with curl calls against the
|
||||
# Gitea REST API (/api/v1/repos/.../issues, .../labels,
|
||||
# .../comments). Same behaviour: open issue on drift, comment on
|
||||
# repeat-drift, close on clean run.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set so the curl calls can
|
||||
# derive `git.moleculesai.app` from the runner env (with
|
||||
# hard-coded fallback inside the steps).
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Daily audit of Railway env vars for drift-prone image-tag pins —
|
||||
# automation-cadence layer over the detection script + regression test
|
||||
# shipped in PR #2168 (#2001 closure).
|
||||
#
|
||||
# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
|
||||
# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
|
||||
# "every fix didn't propagate" — really the tenant image was so old it
|
||||
# didn't read the env vars those fixes produced.
|
||||
#
|
||||
# Cadence: once a day, 13:00 UTC (06:00 PT).
|
||||
#
|
||||
# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
|
||||
# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 13 * * *'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: railway-pin-audit
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
name: Audit Railway env vars for drift-prone pins
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify RAILWAY_AUDIT_TOKEN present
|
||||
env:
|
||||
RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
id: secret_check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
|
||||
echo "have_secret=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "have_secret=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
|
||||
exit 1
|
||||
|
||||
- name: Install Railway CLI
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL https://railway.com/install.sh | sh
|
||||
echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify Railway CLI authenticated
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! railway whoami >/dev/null 2>&1; then
|
||||
echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Link molecule-platform project
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
|
||||
|
||||
- name: Run drift audit
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
id: audit
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# Capture the audit log for the issue body.
|
||||
{
|
||||
echo 'log<<AUDIT_EOF'
|
||||
cat /tmp/audit.log
|
||||
echo 'AUDIT_EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
case "$rc" in
|
||||
0) exit 0 ;;
|
||||
1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
|
||||
2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
|
||||
*) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Open / update drift issue (Gitea API)
|
||||
if: failure() && steps.audit.outputs.rc == '1'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
AUDIT_LOG: ${{ steps.audit.outputs.log }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Railway env-var drift detected"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
BODY=$(jq -nc --arg t "$TITLE" --arg log "${AUDIT_LOG:-(log unavailable)}" --arg run "$RUN_URL" '
|
||||
{body: ("Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n**What this means:** an env var (likely on `controlplane`) is pinned to a SHA-shaped or semver tag instead of a floating tag. Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service does not pick them up.\n\n**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (:staging-latest, :main) unless the pin is intentional and documented in the ops runbook.\n\n**Audit output:**\n\n```\n" + $log + "\n```\n\nRun: " + $run + "\n\nCloses automatically when a subsequent daily run reports clean.")}')
|
||||
|
||||
# Look for existing open drift issue with the title.
|
||||
EXISTING=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number' | head -1)
|
||||
|
||||
if [ -n "$EXISTING" ]; then
|
||||
COMMENT_BODY=$(jq -nc --arg log "${AUDIT_LOG:-(log unavailable)}" --arg run "$RUN_URL" \
|
||||
'{body: ("Still drifting. " + $run + "\n\n```\n" + $log + "\n```")}')
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${EXISTING}/comments" -d "$COMMENT_BODY" >/dev/null
|
||||
echo "Commented on existing issue #${EXISTING}"
|
||||
else
|
||||
CREATE_BODY=$(echo "$BODY" | jq --arg t "$TITLE" '. + {title: $t, labels: []}')
|
||||
NUM=$(curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues" -d "$CREATE_BODY" | jq -r .number)
|
||||
echo "Filed issue #${NUM}"
|
||||
fi
|
||||
|
||||
- name: Close stale drift issue on clean run (Gitea API)
|
||||
if: success() && steps.audit.outputs.rc == '0'
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
REPO: ${{ github.repository }}
|
||||
SERVER_URL: ${{ env.GITHUB_SERVER_URL }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API="${SERVER_URL%/}/api/v1"
|
||||
TITLE="Railway env-var drift detected"
|
||||
RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
NUMS=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" \
|
||||
"${API}/repos/${REPO}/issues?state=open&type=issues&limit=50" \
|
||||
| jq -r --arg t "$TITLE" '.[] | select(.title==$t) | .number')
|
||||
|
||||
for N in $NUMS; do
|
||||
curl -fsS -X POST -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}/comments" \
|
||||
-d "$(jq -nc --arg run "$RUN_URL" '{body: ("Daily audit clean — drift resolved. " + $run)}')" >/dev/null
|
||||
curl -fsS -X PATCH -H "Authorization: token $GITEA_TOKEN" -H "Content-Type: application/json" \
|
||||
"${API}/repos/${REPO}/issues/${N}" -d '{"state":"closed"}' >/dev/null
|
||||
echo "Closed #${N}"
|
||||
done
|
||||
@@ -1,375 +0,0 @@
|
||||
name: redeploy-tenants-on-main
|
||||
|
||||
# Ported from .github/workflows/redeploy-tenants-on-main.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on .gitea/workflows/publish-workspace-server-image.yml. Until
|
||||
# then continue-on-error+dead-workflow doesn't break anything.
|
||||
#
|
||||
|
||||
# Auto-refresh prod tenant EC2s after every main merge.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image builds and
|
||||
# pushes a new platform-tenant :<sha> to ECR on every merge to main,
|
||||
# but running tenants pulled their image once at boot and never re-pull.
|
||||
# Users see stale code indefinitely.
|
||||
#
|
||||
# This workflow closes the gap by calling the control-plane admin
|
||||
# endpoint that performs a canary-first, batched, health-gated rolling
|
||||
# redeploy across every live tenant. Implemented in molecule-ai/
|
||||
# molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
|
||||
# (feat/tenant-auto-redeploy, landing alongside this workflow).
|
||||
#
|
||||
# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
|
||||
# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
|
||||
# Gitea suspension migration. The canary-verify.yml promote step now
|
||||
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
|
||||
#
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
|
||||
# 2. This workflow fires via workflow_run, calls redeploy-fleet with
|
||||
# target_tag=staging-<sha>. No CDN propagation wait needed —
|
||||
# ECR image manifest is consistent immediately after push.
|
||||
# 3. Calls redeploy-fleet with canary_slug (if set) and a soak
|
||||
# period. Canary proves the image boots; batches follow.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: re-run this workflow with a specific SHA pinned via
|
||||
# the workflow_dispatch input. That calls redeploy-fleet with
|
||||
# target_tag=<sha>, re-pulling the older image on every tenant.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize redeploys so two rapid main pushes' redeploys don't overlap
|
||||
# and cause confusing per-tenant SSM state. Without this, GitHub's
|
||||
# implicit workflow_run queueing would *probably* serialize them, but
|
||||
# the explicit block makes the invariant defensible. Mirrors the
|
||||
# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
|
||||
#
|
||||
# cancel-in-progress: false → aborting a half-rolled-out fleet would
|
||||
# leave tenants stuck on whatever image they happened to be on when
|
||||
# cancelled. Better to finish the in-flight rollout before starting
|
||||
# the next one.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-main
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
redeploy:
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Note on ECR propagation
|
||||
# ECR image manifests are consistent immediately after push — no
|
||||
# CDN cache to wait for. The old GHCR-based workflow had a 30s
|
||||
# sleep to avoid race conditions; ECR makes that unnecessary.
|
||||
run: echo "ECR image available immediately after push — proceeding."
|
||||
|
||||
- name: Compute target tag
|
||||
id: tag
|
||||
# Resolution order:
|
||||
# 1. Operator-supplied input (workflow_dispatch with explicit
|
||||
# tag) → used verbatim. Lets ops pin `latest` for emergency
|
||||
# rollback to last canary-verified digest, or pin a specific
|
||||
# `staging-<sha>` to roll back to a known-good build.
|
||||
# 2. Default → `staging-<short_head_sha>`. The just-published
|
||||
# digest. Bypasses the `:latest` retag path that's currently
|
||||
# dead (canary-verify soft-skips without canary fleet, so
|
||||
# the only thing retagging `:latest` today is the manual
|
||||
# promote-latest.yml — last run 2026-04-28). Auto-trigger
|
||||
# from workflow_run uses workflow_run.head_sha; manual
|
||||
# dispatch with no input falls through to github.sha.
|
||||
env:
|
||||
INPUT_TAG: ${{ inputs.target_tag }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${INPUT_TAG:-}" ]; then
|
||||
echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag: $INPUT_TAG"
|
||||
else
|
||||
SHORT="${HEAD_SHA:0:7}"
|
||||
echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
|
||||
echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
|
||||
fi
|
||||
|
||||
- name: Call CP redeploy-fleet
|
||||
# CP_ADMIN_API_TOKEN must be set as a repo/org secret on
|
||||
# molecule-ai/molecule-core, matching the staging/prod CP's
|
||||
# CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
|
||||
# repo's secrets for CI.
|
||||
env:
|
||||
CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
--argjson soak "$SOAK_SECONDS" \
|
||||
--argjson batch "$BATCH_SIZE" \
|
||||
--argjson dry "$DRY_RUN" \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
# Route -w into its own tempfile so curl's exit code (e.g. 56
|
||||
# on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
|
||||
# pollute the captured stdout. The previous inline-substitution
|
||||
# shape produced "000000" on connection reset (curl wrote
|
||||
# "000" via -w, then the inline echo-fallback appended another
|
||||
# "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
|
||||
# set +e/-e keeps the non-zero curl exit from tripping the
|
||||
# outer pipeline. See lint-curl-status-capture.yml for the
|
||||
# CI gate that pins this fix shape.
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
set -e
|
||||
# Stderr from curl (e.g. dial errors with -sS) goes to the runner
|
||||
# log so operators can see WHY a connection failed. Stdout is
|
||||
# captured to $HTTP_CODE_FILE because that's where -w writes.
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
# Pretty-print per-tenant results in the job summary so
|
||||
# ops can see which tenants were redeployed without drilling
|
||||
# into the raw response.
|
||||
{
|
||||
echo "## Tenant redeploy fleet"
|
||||
echo ""
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**Canary:** \`$CANARY_SLUG\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "**Batch size:** $BATCH_SIZE"
|
||||
echo "**Dry run:** $DRY_RUN"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
exit 1
|
||||
fi
|
||||
OK=$(jq -r '.ok' "$HTTP_RESPONSE")
|
||||
if [ "$OK" != "true" ]; then
|
||||
echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
|
||||
|
||||
# Stash the response for the verify step. $RUNNER_TEMP outlasts
|
||||
# the step boundary; $HTTP_RESPONSE doesn't.
|
||||
cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
|
||||
|
||||
- name: Verify each tenant /buildinfo matches published SHA
|
||||
# ROOT FIX FOR #2395.
|
||||
#
|
||||
# `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
|
||||
# didn't error" — NOT "the new image is running on the tenant."
|
||||
# `:latest` lives in the local Docker daemon's image cache; if
|
||||
# the SSM document does `docker compose up -d` without an
|
||||
# explicit `docker pull`, the daemon serves the previously-
|
||||
# cached digest and the container restarts on stale code.
|
||||
# 2026-04-30 incident: hongmingwang's tenant reported
|
||||
# ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
|
||||
# chat_files for 30+ min — the lazy-heal fix never reached the
|
||||
# user despite green deploy + green redeploy.
|
||||
#
|
||||
# This step closes the gap by curling each tenant's /buildinfo
|
||||
# endpoint (added in workspace-server/internal/buildinfo +
|
||||
# /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
|
||||
# returned git_sha to the SHA the workflow expects. Mismatches
|
||||
# fail the workflow, which is what `ok=true` should have
|
||||
# guaranteed all along.
|
||||
#
|
||||
# When the redeploy was triggered by workflow_dispatch with a
|
||||
# specific tag (target_tag != "latest"), the expected SHA may
|
||||
# not equal ${{ github.sha }} — in that case we resolve via
|
||||
# GHCR's manifest. For workflow_run (default :latest) the
|
||||
# workflow_run.head_sha is the SHA that just published.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
# Tenant subdomain template — slugs from the response are
|
||||
# appended. Production CP issues `<slug>.moleculesai.app`;
|
||||
# staging CP issues `<slug>.staging.moleculesai.app`. This
|
||||
# workflow runs on main → prod CP → no `staging.` infix.
|
||||
TENANT_DOMAIN: 'moleculesai.app'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
|
||||
if [ "$TARGET_TAG" != "latest" ] \
|
||||
&& [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
|
||||
&& [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
|
||||
# workflow_dispatch with a pinned tag that isn't the head
|
||||
# SHA — operator is rolling back / pinning. Skip the
|
||||
# verification because we don't have the expected SHA in
|
||||
# this context (would need to crane-inspect the GHCR
|
||||
# manifest, which is a follow-up). Failing-open here is
|
||||
# safe: the operator chose the tag deliberately.
|
||||
#
|
||||
# `staging-<short_head_sha>` IS verified — it's the new
|
||||
# auto-trigger default (see Compute target tag step) and
|
||||
# the digest under that tag SHOULD match EXPECTED_SHA.
|
||||
echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RESP="$RUNNER_TEMP/redeploy-response.json"
|
||||
if [ ! -s "$RESP" ]; then
|
||||
echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pull only successfully-redeployed tenants. Any tenant that
|
||||
# halted the rollout already failed the previous step, so we
|
||||
# don't double-count them here.
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::warning::No tenants reported healthz_ok — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
|
||||
|
||||
# Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
|
||||
# vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
|
||||
# comment for the full rationale; same logic applies on prod even
|
||||
# though prod has fewer ephemeral tenants — the asymmetry would be a
|
||||
# gratuitous fork.
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
STALE_LINES=()
|
||||
UNREACHABLE_LINES=()
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
# 30s total: tenant just SSM-restarted, may still be coming
|
||||
# up. Retry-on-empty rather than retry-on-status — we want
|
||||
# to fail fast on "responded with wrong SHA", not "still
|
||||
# warming up".
|
||||
BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
|
||||
ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
|
||||
if [ -z "$ACTUAL_SHA" ]; then
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
|
||||
continue
|
||||
fi
|
||||
if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
|
||||
echo " $slug: ${ACTUAL_SHA:0:7} ✓"
|
||||
else
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Per-tenant /buildinfo verification"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
|
||||
echo ""
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${STALE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
|
||||
echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders sanity floor: same logic as the staging
|
||||
# variant — see that file's comment for the full rationale.
|
||||
# Floor only applies when fleet >= 4; below that, canary-verify
|
||||
# is the actual gate.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
|
||||
@@ -1,356 +0,0 @@
|
||||
name: redeploy-tenants-on-staging
|
||||
|
||||
# Ported from .github/workflows/redeploy-tenants-on-staging.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
# - **Gitea workflow_run trigger limitation**: Gitea 1.22.6's support
|
||||
# for the `workflow_run` event is partial. If this never fires on a
|
||||
# real publish-workspace-server-image completion, the follow-up
|
||||
# triage PR should replace the trigger with a push-with-paths-filter
|
||||
# on .gitea/workflows/publish-workspace-server-image.yml. Until
|
||||
# then continue-on-error+dead-workflow doesn't break anything.
|
||||
#
|
||||
|
||||
# Auto-refresh staging tenant EC2s after every staging-branch merge.
|
||||
#
|
||||
# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
|
||||
# the :staging-latest tag. Sister workflow exists for prod (rolls
|
||||
# :latest after canary-verify). Both share the same shape — just
|
||||
# different CP_URL + target_tag + admin token secret.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image now builds
|
||||
# on every staging-branch push (PR #2335), pushing
|
||||
# platform-tenant:staging-latest to GHCR. Existing tenants pulled
|
||||
# their image once at boot and never re-pull, so the new image just
|
||||
# sits unused until the tenant is reprovisioned.
|
||||
#
|
||||
# This workflow closes the gap by calling staging-CP's
|
||||
# /cp/admin/tenants/redeploy-fleet, which performs a canary-first,
|
||||
# batched, health-gated SSM redeploy across every live staging tenant.
|
||||
# Same endpoint shape as prod CP — only the host differs.
|
||||
#
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes on staging branch →
|
||||
# new :staging-latest in GHCR.
|
||||
# 2. This workflow fires via workflow_run, waits 30s for GHCR's CDN
|
||||
# to propagate the new tag.
|
||||
# 3. Calls redeploy-fleet with no canary (staging IS canary; we don't
|
||||
# need a sub-canary inside it). Soak still applies to the first
|
||||
# tenant in case of bad-deploy detection.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: re-run with workflow_dispatch + target_tag=staging-<sha>
|
||||
# of a known-good build.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize per-branch so two rapid staging pushes' redeploys don't
|
||||
# overlap and cause confusing per-tenant SSM state. cancel-in-progress
|
||||
# is false because aborting a half-rolled-out fleet leaves tenants
|
||||
# stuck on whatever image they happened to be on when cancelled.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
redeploy:
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
# NOTE (Gitea port): workflow_dispatch trigger dropped; only the
|
||||
# workflow_run path remains.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Wait for GHCR tag propagation
|
||||
# GHCR's edge cache takes ~15-30s to consistently serve the new
|
||||
# :staging-latest manifest after the registry accepts the push.
|
||||
# Same rationale as redeploy-tenants-on-main.yml.
|
||||
run: sleep 30
|
||||
|
||||
- name: Call staging-CP redeploy-fleet
|
||||
# CP_STAGING_ADMIN_API_TOKEN must be set as a repo/org secret
|
||||
# on molecule-ai/molecule-core, matching staging-CP's
|
||||
# CP_ADMIN_API_TOKEN env var (visible in Railway controlplane
|
||||
# / staging environment). Stored separately from the prod
|
||||
# CP_ADMIN_API_TOKEN so a leak of one doesn't auth the other.
|
||||
env:
|
||||
CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || '' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Schedule-vs-dispatch hardening (mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels): hard-fail on auto-trigger when the
|
||||
# secret is missing so a misconfigured-repo doesn't silently
|
||||
# serve stale staging tenants. Soft-skip on operator dispatch.
|
||||
if [ -z "${CP_STAGING_ADMIN_API_TOKEN:-}" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::CP_STAGING_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::warning::Set CP_STAGING_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
echo "::notice::Pull the value from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::staging redeploy cannot run — CP_STAGING_ADMIN_API_TOKEN secret missing"
|
||||
echo "::error::set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
--argjson soak "$SOAK_SECONDS" \
|
||||
--argjson batch "$BATCH_SIZE" \
|
||||
--argjson dry "$DRY_RUN" \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
# Route -w into its own tempfile so curl's exit code (e.g. 56
|
||||
# on connection-reset) can't pollute the captured stdout. The
|
||||
# previous inline-substitution shape produced "000000" on
|
||||
# connection reset — caught on main variant 2026-05-04
|
||||
# redeploying sha 2b862f6. Same fix shape as the synth-E2E
|
||||
# §9c gate (PR #2797). See lint-curl-status-capture.yml for
|
||||
# the CI gate that pins this fix shape.
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to the
|
||||
# runner log so operators can see WHY a connection failed.
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
{
|
||||
echo "## Staging tenant redeploy fleet"
|
||||
echo ""
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**Canary:** \`${CANARY_SLUG:-(none — staging is itself the canary)}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "**Batch size:** $BATCH_SIZE"
|
||||
echo "**Dry run:** $DRY_RUN"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Distinguish "real fleet failure" from "E2E teardown race".
|
||||
#
|
||||
# CP returns HTTP 500 + ok=false whenever ANY tenant in the
|
||||
# fleet failed SSM or healthz. In practice the recurring source
|
||||
# of these is ephemeral test tenants being torn down by their
|
||||
# parent E2E run mid-redeploy: the EC2 dies → SSM exit=2 or
|
||||
# healthz timeout → CP marks the fleet failed → this workflow
|
||||
# goes red even though every operator-facing tenant rolled fine.
|
||||
#
|
||||
# Ephemeral slug prefixes (kept in sync with sweep-stale-e2e-orgs.yml
|
||||
# — see that file for the source-of-truth list and rationale):
|
||||
# - e2e-* — canvas/saas/ext E2E suites
|
||||
# - rt-e2e-* — runtime-test harness fixtures (RFC #2251)
|
||||
# Long-lived prefixes that are NOT ephemeral and MUST hard-fail:
|
||||
# demo-prep, dryrun-*, dryrun2-*, plus all human tenant slugs.
|
||||
#
|
||||
# Filter: if HTTP=500/ok=false AND every failed slug matches an
|
||||
# ephemeral prefix, treat as soft-warn and let the verify step
|
||||
# downstream handle unreachable-vs-stale (#2402). Any non-ephemeral
|
||||
# failure or a non-500 HTTP response remains a hard failure.
|
||||
OK=$(jq -r '.ok // "false"' "$HTTP_RESPONSE")
|
||||
FAILED_SLUGS=$(jq -r '
|
||||
.results[]?
|
||||
| select((.healthz_ok != true) or (.ssm_status != "Success"))
|
||||
| .slug' "$HTTP_RESPONSE" 2>/dev/null || true)
|
||||
EPHEMERAL_PREFIX_RE='^(e2e-|rt-e2e-)'
|
||||
NON_EPHEMERAL_FAILED=$(printf '%s\n' "$FAILED_SLUGS" | grep -v '^$' | grep -Ev "$EPHEMERAL_PREFIX_RE" || true)
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ] && [ "$OK" = "true" ]; then
|
||||
: # happy path — fall through to verification
|
||||
elif [ "$HTTP_CODE" = "500" ] && [ -z "$NON_EPHEMERAL_FAILED" ] && [ -n "$FAILED_SLUGS" ]; then
|
||||
COUNT=$(printf '%s\n' "$FAILED_SLUGS" | grep -Ec "$EPHEMERAL_PREFIX_RE" || true)
|
||||
echo "::warning::redeploy-fleet returned HTTP 500 but every failed tenant ($COUNT) is ephemeral (e2e-*/rt-e2e-*) — treating as teardown race, soft-warning."
|
||||
printf '%s\n' "$FAILED_SLUGS" | sed 's/^/::warning:: failed: /'
|
||||
elif [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
if [ -n "$NON_EPHEMERAL_FAILED" ]; then
|
||||
echo "::error::non-ephemeral tenant(s) failed:"
|
||||
printf '%s\n' "$NON_EPHEMERAL_FAILED" | sed 's/^/::error:: /'
|
||||
fi
|
||||
exit 1
|
||||
else
|
||||
# HTTP=200 but ok=false (shouldn't happen with current CP
|
||||
# but keep the gate for completeness).
|
||||
echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Staging tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
|
||||
|
||||
cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
|
||||
|
||||
- name: Verify each staging tenant /buildinfo matches published SHA
|
||||
# Mirror of the verify step in redeploy-tenants-on-main.yml — see
|
||||
# there for the rationale (#2395 root fix). Staging has the same
|
||||
# ssm_status-success-but-stale-image hazard and benefits from the
|
||||
# same gate. Diff: TENANT_DOMAIN includes the `staging.` infix.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
|
||||
TENANT_DOMAIN: 'staging.moleculesai.app'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# staging-latest is the staging-side moving tag; treat it the
|
||||
# same way main treats `latest`. Operator-pinned SHAs skip
|
||||
# verification (see main variant for why).
|
||||
if [ "$TARGET_TAG" != "staging-latest" ] && [ "$TARGET_TAG" != "latest" ] && [ "$TARGET_TAG" != "$EXPECTED_SHA" ]; then
|
||||
echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RESP="$RUNNER_TEMP/redeploy-response.json"
|
||||
if [ ! -s "$RESP" ]; then
|
||||
echo "::error::redeploy-response.json missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::warning::No staging tenants reported healthz_ok — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Verifying ${#SLUGS[@]} staging tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
|
||||
|
||||
# Two distinct failure modes here:
|
||||
# STALE_COUNT — tenant returned a SHA that doesn't match. THIS is
|
||||
# the #2395 bug class: tenant up + serving old code.
|
||||
# Always hard-fail the workflow.
|
||||
# UNREACHABLE_COUNT — tenant didn't respond. Almost always a benign
|
||||
# teardown race: redeploy-fleet snapshot says
|
||||
# healthz_ok=true, then the E2E suite tears the
|
||||
# ephemeral tenant down before this step runs (the
|
||||
# e2e-* fixtures churn 5-10/hour on staging). Soft-
|
||||
# warn so we don't block staging→main on cleanup.
|
||||
# Real "tenant up but unreachable" is caught by CP's
|
||||
# own healthz monitor + the post-redeploy alert; we
|
||||
# don't need to double-count it here.
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
STALE_LINES=()
|
||||
UNREACHABLE_LINES=()
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
|
||||
ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
|
||||
if [ -z "$ACTUAL_SHA" ]; then
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
|
||||
continue
|
||||
fi
|
||||
if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
|
||||
echo " $slug: ${ACTUAL_SHA:0:7} ✓"
|
||||
else
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Per-tenant /buildinfo verification (staging)"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
|
||||
echo ""
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${STALE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely E2E teardown race (soft-warn, not failing):**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
|
||||
echo "All ${#SLUGS[@]} staging tenants returned matching SHA. ✓"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "::warning::$UNREACHABLE_COUNT staging tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders sanity floor: if MORE than half the fleet is
|
||||
# unreachable AND the fleet is large enough that "half down" is
|
||||
# statistically meaningful, this is a real outage (e.g. new image
|
||||
# crashes on startup), not a teardown race. Hard-fail.
|
||||
#
|
||||
# Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
|
||||
# canary-verify step is the actual gate for "all tenants down"
|
||||
# detection (it runs against the canary first and aborts the
|
||||
# rollout if the canary fails to come up). Without the >=4 gate,
|
||||
# a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
|
||||
# quiet staging push) would re-flake on the exact teardown-race
|
||||
# condition #2402 fixed: 1 of 1 unreachable = 100% > 50% → fail.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED staging tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "::error::$STALE_COUNT staging tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::Staging tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
|
||||
@@ -1,100 +0,0 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-pin-compat.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and
|
||||
# `workflow_dispatch:` (no inputs, but the trigger itself is
|
||||
# parser-rejected when inputs are absent in some Gitea 1.22.x
|
||||
# builds; safest to drop entirely — manual runs go via cron-trigger
|
||||
# bump or push-with-paths-filter).
|
||||
# - on.paths references .gitea/workflows/runtime-pin-compat.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.gitea/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking
|
||||
# the PR. Follow-up PR flips this off after surfaced defects are
|
||||
# triaged.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -1,139 +0,0 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Ported from .github/workflows/runtime-prbuild-compat.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` (no Gitea merge queue) and `workflow_dispatch:`
|
||||
# (Gitea 1.22.6 parser-rejects workflow_dispatch with inputs and is
|
||||
# finicky without them).
|
||||
# - `dorny/paths-filter@v4` replaced with inline `git diff` (per PR#372
|
||||
# pattern for ci.yml port).
|
||||
# - on.paths references .gitea/workflows/runtime-prbuild-compat.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on every job (RFC §1 contract).
|
||||
#
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin -> smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge -> publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy -> all crash on first boot with ImportError.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
# event_name + sha keeps PR sync and the subsequent staging push on the
|
||||
# same SHA from cancelling each other (per feedback_concurrency_group_per_sha).
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: decide
|
||||
run: |
|
||||
# Inline replacement for dorny/paths-filter — same pattern
|
||||
# PR#372's ci.yml port used. Diffs against the PR base or the
|
||||
# previous push SHA, then matches against the wheel-relevant
|
||||
# path set.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
fi
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
# New branch or no previous SHA: treat as wheel-relevant.
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$BASE" HEAD)
|
||||
if echo "$CHANGED" | grep -qE '^(workspace/|scripts/build_runtime_package\.py$|scripts/wheel_smoke\.py$|\.gitea/workflows/runtime-prbuild-compat\.yml$)'; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`.
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -1,70 +0,0 @@
|
||||
name: SECRET_PATTERNS drift lint
|
||||
|
||||
# Ported from .github/workflows/secret-pattern-drift.yml on 2026-05-11
|
||||
# per RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - on.paths references the new canonical .gitea/workflows/secret-scan.yml
|
||||
# (the .github/ copy is removed by Cat A of this sweep).
|
||||
# - CANONICAL_FILE inside scripts/lint_secret_pattern_drift.py was
|
||||
# updated in the same Cat C-1 PR to point at .gitea/workflows/secret-scan.yml.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Detects when the canonical SECRET_PATTERNS array in
|
||||
# .gitea/workflows/secret-scan.yml diverges from known consumer
|
||||
# mirrors (workspace-runtime's bundled pre-commit hook today; more
|
||||
# can be added as the consumer set grows).
|
||||
#
|
||||
# Why this exists: every side that scans for credentials has its own
|
||||
# copy of the pattern list. They drift — most recently the runtime
|
||||
# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
|
||||
# so a developer's local pre-commit would let a sk-cp- token through
|
||||
# while the org-wide CI scan would refuse it. The cost of that drift
|
||||
# is dev confusion + delayed feedback; the fix is automated detection.
|
||||
#
|
||||
# Triggers:
|
||||
# - schedule: daily 05:00 UTC. Catches drift introduced by edits
|
||||
# to a consumer copy that didn't update canonical here.
|
||||
# - push to main/staging where the canonical or this lint changed:
|
||||
# catches the inverse — canonical updated but consumers not yet
|
||||
# bumped. The lint will fail the push; that's intentional.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
|
||||
# email lands when humans are starting their day, not
|
||||
# interrupting it.
|
||||
- cron: "0 5 * * *"
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- ".gitea/workflows/secret-scan.yml"
|
||||
- ".gitea/workflows/secret-pattern-drift.yml"
|
||||
- ".github/scripts/lint_secret_pattern_drift.py"
|
||||
- ".githooks/pre-commit"
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
# Auto-injected GITHUB_TOKEN scoped to read-only. The lint only does git
|
||||
# checkout + HTTPS GETs to public consumer files; no writes to anything.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Detect SECRET_PATTERNS drift
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Run drift lint
|
||||
run: python3 .github/scripts/lint_secret_pattern_drift.py
|
||||
@@ -1,191 +0,0 @@
|
||||
name: Secret scan
|
||||
|
||||
# Hard CI gate. Refuses any PR / push whose diff additions contain a
|
||||
# recognisable credential. Defense-in-depth for the #2090-class incident
|
||||
# (2026-04-24): GitHub's hosted Copilot Coding Agent leaked a ghs_*
|
||||
# installation token into tenant-proxy/package.json via `npm init`
|
||||
# slurping the URL from a token-embedded origin remote. We can't fix
|
||||
# upstream's clone hygiene, so we gate here.
|
||||
#
|
||||
# Same regex set as the runtime's bundled pre-commit hook
|
||||
# (molecule-ai-workspace-runtime: molecule_runtime/scripts/pre-commit-checks.sh).
|
||||
# Keep the two sides aligned when adding patterns.
|
||||
#
|
||||
# Ported from .github/workflows/secret-scan.yml so the gate actually
|
||||
# fires on Gitea Actions. Differences from the GitHub version:
|
||||
# - drops `merge_group` event (Gitea has no merge queue)
|
||||
# - drops `workflow_call` (no cross-repo reusable invocation on Gitea)
|
||||
# - SELF path updated to .gitea/workflows/secret-scan.yml
|
||||
# The job name + step name are identical to the GitHub workflow so the
|
||||
# status-check context (`Secret scan / Scan diff for credential-shaped
|
||||
# strings (pull_request)`) matches branch protection on molecule-core/main.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan diff for credential-shaped strings
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base may be many commits behind
|
||||
# HEAD and absent from the shallow clone. Fetch it explicitly.
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
- name: Refuse if credential-shaped strings appear in diff additions
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# pull_request has pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Pattern set covers GitHub family (the actual #2090 vector),
|
||||
# Anthropic / OpenAI / Slack / AWS. Anchored on prefixes with low
|
||||
# false-positive rates against agent-generated content. Mirror of
|
||||
# molecule-ai-workspace-runtime/molecule_runtime/scripts/pre-commit-checks.sh
|
||||
# — keep aligned.
|
||||
SECRET_PATTERNS=(
|
||||
'ghp_[A-Za-z0-9]{36,}' # GitHub PAT (classic)
|
||||
'ghs_[A-Za-z0-9]{36,}' # GitHub App installation token
|
||||
'gho_[A-Za-z0-9]{36,}' # GitHub OAuth user-to-server
|
||||
'ghu_[A-Za-z0-9]{36,}' # GitHub OAuth user
|
||||
'ghr_[A-Za-z0-9]{36,}' # GitHub OAuth refresh
|
||||
'github_pat_[A-Za-z0-9_]{82,}' # GitHub fine-grained PAT
|
||||
'sk-ant-[A-Za-z0-9_-]{40,}' # Anthropic API key
|
||||
'sk-proj-[A-Za-z0-9_-]{40,}' # OpenAI project key
|
||||
'sk-svcacct-[A-Za-z0-9_-]{40,}' # OpenAI service-account key
|
||||
'sk-cp-[A-Za-z0-9_-]{60,}' # MiniMax API key (F1088 vector — caught only after the fact)
|
||||
'xox[baprs]-[A-Za-z0-9-]{20,}' # Slack tokens
|
||||
'AKIA[0-9A-Z]{16}' # AWS access key ID
|
||||
'ASIA[0-9A-Z]{16}' # AWS STS temp access key ID
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check the
|
||||
# entire tree as added content. Slower, but correct on first
|
||||
# push.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
DIFF_RANGE=""
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
DIFF_RANGE="$BASE $HEAD"
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Self-exclude: this workflow file legitimately contains the
|
||||
# pattern strings as regex literals. Without an exclude it would
|
||||
# block its own merge. Both the .github/ original and this
|
||||
# .gitea/ port are excluded so a sync between them stays clean.
|
||||
SELF_GITHUB=".github/workflows/secret-scan.yml"
|
||||
SELF_GITEA=".gitea/workflows/secret-scan.yml"
|
||||
|
||||
OFFENDING=""
|
||||
# `while IFS= read -r` (not `for f in $CHANGED`) so filenames
|
||||
# containing whitespace don't word-split silently — a path
|
||||
# with a space would otherwise produce two iterations on
|
||||
# tokens that aren't real filenames, breaking the
|
||||
# self-exclude + diff lookup.
|
||||
while IFS= read -r f; do
|
||||
[ -z "$f" ] && continue
|
||||
[ "$f" = "$SELF_GITHUB" ] && continue
|
||||
[ "$f" = "$SELF_GITEA" ] && continue
|
||||
if [ -n "$DIFF_RANGE" ]; then
|
||||
ADDED=$(git diff --no-color --unified=0 "$BASE" "$HEAD" -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
|
||||
else
|
||||
# No diff range (new branch first push) — scan the full file
|
||||
# contents as if every line were new.
|
||||
ADDED=$(cat "$f" 2>/dev/null || true)
|
||||
fi
|
||||
[ -z "$ADDED" ] && continue
|
||||
for pattern in "${SECRET_PATTERNS[@]}"; do
|
||||
if echo "$ADDED" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${f} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done <<< "$CHANGED"
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Credential-shaped strings detected in diff additions:"
|
||||
# `printf '%b' "$OFFENDING"` interprets backslash escapes
|
||||
# (the literal `\n` we appended above becomes a newline)
|
||||
# WITHOUT treating OFFENDING as a format string. Plain
|
||||
# `printf "$OFFENDING"` is a format-string sink: a filename
|
||||
# containing `%` would be interpreted as a conversion
|
||||
# specifier, corrupting the error message (or printing
|
||||
# `%(missing)` artifacts).
|
||||
printf '%b' "$OFFENDING"
|
||||
echo ""
|
||||
echo "The actual matched values are NOT echoed here, deliberately —"
|
||||
echo "round-tripping a leaked credential into CI logs widens the blast"
|
||||
echo "radius (logs are searchable + retained)."
|
||||
echo ""
|
||||
echo "Recovery:"
|
||||
echo " 1. Remove the secret from the file. Replace with an env var"
|
||||
echo " reference (e.g. \${{ secrets.GITHUB_TOKEN }} in workflows,"
|
||||
echo " process.env.X in code)."
|
||||
echo " 2. If the credential was already pushed (this PR's commit"
|
||||
echo " history reaches a public ref), treat it as compromised —"
|
||||
echo " ROTATE it immediately, do not just remove it. The token"
|
||||
echo " remains valid in git history forever and may be in any"
|
||||
echo " log/cache that consumed this branch."
|
||||
echo " 3. Force-push the cleaned commit (or stack a revert) and"
|
||||
echo " re-run CI."
|
||||
echo ""
|
||||
echo "If the match is a false positive (test fixture, docs example,"
|
||||
echo "or this workflow's own regex literals): use a clearly-fake"
|
||||
echo "placeholder like ghs_EXAMPLE_DO_NOT_USE that doesn't satisfy"
|
||||
echo "the length suffix, OR add the file path to the SELF exclude"
|
||||
echo "list in this workflow with a short reason."
|
||||
echo ""
|
||||
echo "Mirror of the regex set lives in the runtime's bundled"
|
||||
echo "pre-commit hook (molecule-ai-workspace-runtime:"
|
||||
echo "molecule_runtime/scripts/pre-commit-checks.sh) — keep aligned."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No credential-shaped strings in this change."
|
||||
@@ -1,126 +0,0 @@
|
||||
# sop-tier-check — canonical Gitea Actions workflow for §SOP-6 enforcement.
|
||||
#
|
||||
# Logic lives in `.gitea/scripts/sop-tier-check.sh` (extracted 2026-05-09
|
||||
# from the previous inline-bash version). The script is the single source
|
||||
# of truth; this workflow file just sets env + invokes it.
|
||||
#
|
||||
# Copy BOTH files (`.gitea/workflows/sop-tier-check.yml` +
|
||||
# `.gitea/scripts/sop-tier-check.sh`) into any repo that wants the
|
||||
# §SOP-6 PR gate enforced. Pair with branch protection on the protected
|
||||
# branch:
|
||||
# required_status_checks: ["sop-tier-check / tier-check (pull_request)"]
|
||||
# required_approving_reviews: 1
|
||||
# approving_review_teams: ["ceo", "managers", "engineers"]
|
||||
#
|
||||
# Tier → required-team expression (internal#189 AND-composition):
|
||||
# tier:low → engineers,managers,ceo (OR: any one suffices)
|
||||
# tier:medium → managers AND engineers AND qa???,security??? (AND: all required)
|
||||
# tier:high → ceo (OR: single team, wired for AND)
|
||||
#
|
||||
# "???" = teams not yet created in Gitea. When qa + security teams are
|
||||
# added, update TIER_EXPR["tier:medium"] in the script to remove the
|
||||
# markers. PRs already in-flight when qa/security are created continue
|
||||
# to work because their authors explicitly requested those reviews.
|
||||
#
|
||||
# Force-merge: Owners-team override remains available out-of-band via
|
||||
# the Gitea merge API; force-merge writes `incident.force_merge` to
|
||||
# `structure_events` per §Persistent structured logging gate (Phase 3).
|
||||
#
|
||||
# Environment variables:
|
||||
# SOP_DEBUG=1 — per-API-call diagnostic lines. Default: off.
|
||||
# SOP_LEGACY_CHECK=1 — revert to OR-gate for this run. Grace window
|
||||
# for PRs in-flight when AND-composition deployed.
|
||||
# Burn-in: remove after 2026-05-17 (7-day window).
|
||||
#
|
||||
# BURN-IN NOTE (internal#189 Phase 1): continue-on-error: true is set on
|
||||
# the tier-check job below. This prevents AND-composition from blocking
|
||||
# PRs during the 7-day burn-in. After 2026-05-17:
|
||||
# 1. Remove `continue-on-error: true` from this job block.
|
||||
# 2. Update this BURN-IN NOTE comment to mark the window closed.
|
||||
|
||||
name: sop-tier-check
|
||||
|
||||
# SECURITY: triggers MUST use `pull_request_target`, not `pull_request`.
|
||||
# `pull_request_target` loads the workflow definition from the BASE
|
||||
# branch (i.e. `main`), not the PR's HEAD. With `pull_request`, anyone
|
||||
# with write access to a feature branch could rewrite this file in
|
||||
# their PR to dump SOP_TIER_CHECK_TOKEN (org-read scope) to logs and
|
||||
# exfiltrate it. Verified 2026-05-09 against Gitea 1.22.6 —
|
||||
# `pull_request_target` (added in Gitea 1.21 via go-gitea/gitea#25229)
|
||||
# is the documented mitigation.
|
||||
#
|
||||
# This workflow does NOT call `actions/checkout` of PR HEAD code, so no
|
||||
# untrusted code is ever executed in the runner — we only HTTP-call the
|
||||
# Gitea API. If a future change adds a checkout step, it MUST pin to
|
||||
# `${{ github.event.pull_request.base.sha }}` (NOT `head.sha`) to keep
|
||||
# the trust boundary.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, synchronize, reopened, labeled, unlabeled]
|
||||
pull_request_review:
|
||||
types: [submitted, dismissed, edited]
|
||||
|
||||
jobs:
|
||||
tier-check:
|
||||
runs-on: ubuntu-latest
|
||||
# BURN-IN: continue-on-error prevents AND-composition from blocking
|
||||
# PRs during the 7-day window. Remove after 2026-05-17 (internal#189).
|
||||
continue-on-error: true
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
steps:
|
||||
- name: Check out base branch (for the script)
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Pin to base.sha — pull_request_target's protection only
|
||||
# works if we never check out PR HEAD. Same SHA the workflow
|
||||
# itself was loaded from.
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
- name: Install jq
|
||||
# Gitea Actions runners (ubuntu-latest label) do not bundle jq.
|
||||
# The sop-tier-check script uses jq for all JSON API parsing.
|
||||
# Install jq before the script runs so sop-tier-check can pass.
|
||||
#
|
||||
# Method: apt-get first (reliable for Ubuntu runners with internet
|
||||
# access to package mirrors). Falls back to GitHub binary download.
|
||||
# GitHub releases may be unreachable from some runner networks
|
||||
# (infra#241 follow-up: GitHub timeout after 3s on 5.78.80.188
|
||||
# runners). The sop-tier-check script has its own fallback as a
|
||||
# third line of defense. continue-on-error: true ensures this step
|
||||
# failing does not block the job.
|
||||
continue-on-error: true
|
||||
run: |
|
||||
# apt-get is the primary method — Ubuntu package mirrors are reliably
|
||||
# reachable from runner containers. GitHub releases may be blocked
|
||||
# or slow on some networks (infra#241 follow-up).
|
||||
if apt-get update -qq && apt-get install -y -qq jq; then
|
||||
echo "::notice::jq installed via apt-get: $(jq --version)"
|
||||
elif timeout 120 curl -sSL \
|
||||
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64" \
|
||||
-o /usr/local/bin/jq && chmod +x /usr/local/bin/jq; then
|
||||
echo "::notice::jq binary downloaded: $(/usr/local/bin/jq --version)"
|
||||
else
|
||||
echo "::warning::jq install failed — apt-get and GitHub download both failed."
|
||||
fi
|
||||
jq --version 2>/dev/null || echo "::notice::jq not yet available — script fallback will retry"
|
||||
|
||||
- name: Verify tier label + reviewer team membership
|
||||
# continue-on-error: true at step level — job-level is ignored by Gitea
|
||||
# Actions (quirk #10, internal runbooks). Belt-and-suspenders with
|
||||
# SOP_FAIL_OPEN=1 + || true below.
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.SOP_TIER_CHECK_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
GITEA_HOST: git.moleculesai.app
|
||||
REPO: ${{ github.repository }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
SOP_DEBUG: '0'
|
||||
SOP_LEGACY_CHECK: '0'
|
||||
# SOP_FAIL_OPEN=1 makes the script always exit 0. The UI enforces
|
||||
# the actual merge gate. Combined with continue-on-error: true
|
||||
# above, this step never fails the job regardless of script exit.
|
||||
SOP_FAIL_OPEN: '1'
|
||||
run: |
|
||||
bash .gitea/scripts/sop-tier-check.sh || true
|
||||
@@ -1,129 +0,0 @@
|
||||
name: Sweep stale AWS Secrets Manager secrets
|
||||
|
||||
# Ported from .github/workflows/sweep-aws-secrets.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for per-tenant AWS Secrets Manager secrets
|
||||
# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
|
||||
# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
|
||||
# sweep-cf-orphans.yml — different cloud, same justification.
|
||||
#
|
||||
# Why this exists separately from a long-term reconciler integration:
|
||||
# - molecule-controlplane's tenant_resources audit table (mig 024)
|
||||
# currently tracks four resource kinds: CloudflareTunnel,
|
||||
# CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
|
||||
# not in the list, so the existing reconciler doesn't catch
|
||||
# orphan secrets.
|
||||
# - At ~$0.40/secret/month the cost grew to ~$19/month before this
|
||||
# sweeper was written, indicating ~45+ orphan secrets from
|
||||
# crashed provisions and incomplete deprovision flows.
|
||||
# - The proper fix (KindSecretsManagerSecret + recorder hook +
|
||||
# reconciler enumerator) is filed as a separate controlplane
|
||||
# issue. This sweeper is the immediate cost-relief stopgap.
|
||||
#
|
||||
# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
|
||||
# This is a DEDICATED principal — the production `molecule-cp` IAM
|
||||
# user lacks `secretsmanager:ListSecrets` (it only has
|
||||
# Get/Create/Update/Delete on specific resources, scoped to its
|
||||
# operational needs). The janitor needs ListSecrets across the
|
||||
# `molecule/tenant/*` prefix, which warrants a separate principal so
|
||||
# we don't broaden the prod-CP policy.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
|
||||
# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
|
||||
# the mostly-orphan tunnels) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30 — offsets from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45) so the three janitors don't burst the
|
||||
# CP admin endpoints at the same minute.
|
||||
- cron: '30 * * * *'
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
group: sweep-aws-secrets
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
@@ -1,151 +0,0 @@
|
||||
name: Sweep stale Cloudflare DNS records
|
||||
|
||||
# Ported from .github/workflows/sweep-cf-orphans.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for Cloudflare DNS records whose backing tenant/workspace no
|
||||
# longer exists. Without this loop, every short-lived E2E or canary
|
||||
# leaves a CF record on the moleculesai.app zone — the zone has a
|
||||
# 200-record quota (controlplane#239 hit it 2026-04-23+) and provisions
|
||||
# start failing with code 81045 once exhausted.
|
||||
#
|
||||
# Why a separate workflow vs sweep-stale-e2e-orgs.yml:
|
||||
# - That workflow operates at the CP layer (DELETE /cp/admin/tenants/:slug
|
||||
# drives the cascade). It assumes CP has the org row to drive the
|
||||
# deprovision from. It doesn't catch records left behind when CP
|
||||
# itself never knew about the tenant (canary scratch, manual ops
|
||||
# experiments) or when the cascade's CF-delete branch failed.
|
||||
# - sweep-cf-orphans.sh enumerates the CF zone directly and matches
|
||||
# each record against live CP slugs + AWS EC2 names. It catches
|
||||
# leaks the CP-driven sweep can't.
|
||||
#
|
||||
# Safety: the script's own MAX_DELETE_PCT gate refuses to nuke more
|
||||
# than 50% of records in a single run. If something has gone weird
|
||||
# (CP admin endpoint returns no orgs → every tenant looks orphan) the
|
||||
# gate halts before damage. Decision-function unit tests in
|
||||
# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
|
||||
# classifier.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly. Mirrors sweep-stale-e2e-orgs cadence so the two janitors
|
||||
# converge on the same tick. CF API rate budget is generous (1200
|
||||
# req/5min); a single sweep makes ~1 list + N deletes (N<=quota/2).
|
||||
- cron: '15 * * * *' # offset from sweep-stale-e2e-orgs (top of hour)
|
||||
# No `merge_group:` trigger on purpose. This is a janitor — it doesn't
|
||||
# need to gate merges, and including it as written before #2088 fired
|
||||
# the full sweep job (or its secret-check) on every PR going through
|
||||
# the merge queue, generating one red CI run per merge-queue eval. If
|
||||
# this workflow is ever wired up as a required check, re-add
|
||||
# merge_group: { types: [checks_requested] }
|
||||
# AND gate the sweep step with `if: github.event_name != 'merge_group'`
|
||||
# so merge-queue evals report success without actually running.
|
||||
|
||||
# Don't let two sweeps race the same zone. workflow_dispatch during a
|
||||
# scheduled run would otherwise issue duplicate DELETE calls.
|
||||
concurrency:
|
||||
group: sweep-cf-orphans
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF orphans
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
|
||||
# within one cron interval instead of burning a full tick. Realistic
|
||||
# worst case is ~2 min: 4 sequential curls + 1 aws + N×CF-DELETE
|
||||
# each individually capped at 10s by the script's curl -m flag.
|
||||
timeout-minutes: 3
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split (hardened 2026-04-28
|
||||
# after the silent-no-op incident below):
|
||||
#
|
||||
# The earlier soft-skip-on-schedule policy hid a real leak. All
|
||||
# six secrets were unset on this repo for an unknown duration;
|
||||
# every hourly run printed a yellow ::warning:: and exited 0,
|
||||
# so the workflow registered as "passing" while doing nothing.
|
||||
# CF orphans accumulated to 152/200 (~76% of the zone quota
|
||||
# gone) before a manual `dig`-driven audit caught it. Anything
|
||||
# that runs as a janitor and reports green while idle is
|
||||
# indistinguishable from "the janitor is healthy" — so we now
|
||||
# treat schedule (and any future workflow_run/push triggers)
|
||||
# as a hard-fail when secrets are missing.
|
||||
#
|
||||
# - schedule / workflow_run / push → exit 1 (red CI run
|
||||
# surfaces the misconfiguration the next tick)
|
||||
# - workflow_dispatch → exit 0 with a warning
|
||||
# (an operator ran this ad-hoc; they already accepted the
|
||||
# state of the repo and want the workflow to short-circuit
|
||||
# so they can rerun after fixing the secret)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ZONE_ID CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry (intentional):
|
||||
# - Scheduled runs: github.event.inputs.dry_run is empty →
|
||||
# defaults to "false" below → script runs with --execute
|
||||
# (the whole point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default is true (line 38)
|
||||
# so an ad-hoc operator-triggered run is dry-run by default;
|
||||
# they have to flip the toggle to actually delete.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) is the second
|
||||
# line of defense regardless of mode.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-orphans.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-orphans.sh --execute
|
||||
fi
|
||||
@@ -1,128 +0,0 @@
|
||||
name: Sweep stale Cloudflare Tunnels
|
||||
|
||||
# Ported from .github/workflows/sweep-cf-tunnels.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for Cloudflare Tunnels whose backing tenant no longer
|
||||
# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
|
||||
# records); same justification, different CF resource.
|
||||
#
|
||||
# Why this exists separately from sweep-cf-orphans:
|
||||
# - DNS records live on the zone (`/zones/<id>/dns_records`).
|
||||
# - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
|
||||
# - Different CF API surface, different scopes; the existing CF
|
||||
# token might not have `account:cloudflare_tunnel:edit`. Splitting
|
||||
# the workflows keeps each one's secret-presence gate independent
|
||||
# so neither silent-skips when the other's secret is missing.
|
||||
# - Cleaner blast radius — operators can disable one without the
|
||||
# other if a regression surfaces.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
|
||||
# the DNS sweep's 50% because tenant-shaped tunnels are mostly
|
||||
# orphans by design) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
|
||||
# janitors don't issue parallel CF API bursts at the same minute.
|
||||
- cron: '45 * * * *'
|
||||
# Don't let two sweeps race the same account.
|
||||
concurrency:
|
||||
group: sweep-cf-tunnels
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF tunnels
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
# 30 min cap. Was 5 min on the theory that the only thing that
|
||||
# could take >5min is a CF-API hang — but on 2026-05-02 a backlog
|
||||
# of 672 stale tunnels accumulated (large staging E2E run + delayed
|
||||
# sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
|
||||
# ~7-8min to drain. The 5-min cap killed the run mid-sweep
|
||||
# (cancelled at 424/672, see run 25248788312); a manual rerun
|
||||
# finished the remainder fine.
|
||||
#
|
||||
# The fix is two-part: parallelize the delete loop (8-way xargs in
|
||||
# the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
|
||||
# cap so a one-off backlog doesn't trip a hangs-detector that
|
||||
# turned out to be a real-job-too-slow detector. With 8-way
|
||||
# parallelism, 600+ tunnels drains in ~60s; 30 min is generous
|
||||
# headroom for actual hangs to still surface (and is in line with
|
||||
# the sweep-cf-orphans companion job).
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# (hardened 2026-04-28 after the silent-no-op incident: the
|
||||
# janitor reported green while doing nothing because secrets
|
||||
# were unset, masking a 152/200 zone-record leak). Same
|
||||
# principle applies here:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ACCOUNT_ID CP_ADMIN_API_TOKEN CP_STAGING_ADMIN_API_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh --execute
|
||||
fi
|
||||
@@ -1,243 +0,0 @@
|
||||
name: Sweep stale e2e-* orgs (staging)
|
||||
|
||||
# Ported from .github/workflows/sweep-stale-e2e-orgs.yml on 2026-05-11 per RFC
|
||||
# internal#219 §1 sweep. Differences from the GitHub version:
|
||||
# - Dropped `workflow_dispatch.inputs` (Gitea 1.22.6 parser rejects them
|
||||
# per feedback_gitea_workflow_dispatch_inputs_unsupported).
|
||||
# - Dropped `merge_group:` (no Gitea merge queue).
|
||||
# - Dropped `environment:` blocks (Gitea has no environments).
|
||||
# - Workflow-level env.GITHUB_SERVER_URL pinned per
|
||||
# feedback_act_runner_github_server_url.
|
||||
# - `continue-on-error: true` on each job (RFC §1 contract).
|
||||
#
|
||||
|
||||
# Janitor for staging tenants left behind when E2E cleanup didn't run:
|
||||
# CI cancellations, runner crashes, transient AWS errors mid-cascade,
|
||||
# bash trap missed (signal 9), etc. Without this loop, every failed
|
||||
# teardown leaks an EC2 + DNS + DB row until manual ops cleanup —
|
||||
# 2026-04-23 staging hit the 64 vCPU AWS quota from ~27 such orphans.
|
||||
#
|
||||
# Why not rely on per-test-run teardown:
|
||||
# - Per-run teardown is best-effort by definition. Any process death
|
||||
# after the test starts but before the trap fires leaves debris.
|
||||
# - GH Actions cancellation kills the runner without grace period.
|
||||
# The workflow's `if: always()` step usually catches this, but it
|
||||
# too can fail (CP transient 5xx, runner network issue at the
|
||||
# wrong moment).
|
||||
# - Even when teardown runs, the CP cascade is best-effort in places
|
||||
# (cascadeTerminateWorkspaces logs+continues; DNS deletion same).
|
||||
# - This sweep is the catch-all that converges staging back to clean
|
||||
# regardless of which specific path leaked.
|
||||
#
|
||||
# The PROPER fix is making CP cleanup transactional + verify-after-
|
||||
# terminate (filed separately as cleanup-correctness work). This
|
||||
# workflow is the safety net that catches everything else AND any
|
||||
# future leak source we haven't yet identified.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
|
||||
# create to teardown — canary is ~8 min, full SaaS ~25 min). The
|
||||
# previous hourly + 120-min stale threshold meant a leaked tenant
|
||||
# could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
|
||||
# leak. Tightening the cadence + threshold reduces the worst-case
|
||||
# leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
|
||||
# threshold) without risk of catching in-progress runs (the longest
|
||||
# e2e run is the 25-min canary, well under the 30-min threshold).
|
||||
# See molecule-controlplane#420 for the leak-class accounting that
|
||||
# motivated this tightening.
|
||||
- cron: '*/15 * * * *'
|
||||
# Don't let two sweeps fight. Cron + workflow_dispatch could overlap
|
||||
# on a manual trigger; queue rather than parallel-delete.
|
||||
concurrency:
|
||||
group: sweep-stale-e2e-orgs
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep e2e orgs
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
# Refuse to delete more than this many orgs in one tick. If the
|
||||
# CP DB is briefly empty (or the admin endpoint goes weird and
|
||||
# returns no created_at), every e2e- org would look stale.
|
||||
# Bailing protects against runaway nukes.
|
||||
SAFETY_CAP: 50
|
||||
|
||||
steps:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Identify stale e2e orgs
|
||||
id: identify
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch into a file so the python step reads it via stdin —
|
||||
# cleaner than embedding $(curl ...) into a heredoc.
|
||||
curl -sS --fail-with-body --max-time 30 \
|
||||
"$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
> orgs.json
|
||||
|
||||
# Filter:
|
||||
# 1. slug starts with one of the ephemeral test prefixes:
|
||||
# - 'e2e-' — covers e2e-canary-, e2e-canvas-*, etc.
|
||||
# - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
|
||||
# missing this prefix left two such tenants
|
||||
# orphaned 8h on staging (2026-05-03), then
|
||||
# hard-failed redeploy-tenants-on-staging
|
||||
# and broke the staging→main auto-promote
|
||||
# chain. Kept in sync with the EPHEMERAL_PREFIX_RE
|
||||
# regex in redeploy-tenants-on-staging.yml.
|
||||
# 2. created_at is older than MAX_AGE_MINUTES ago
|
||||
# Output one slug per line to a file the next step reads.
|
||||
python3 > stale_slugs.txt <<'PY'
|
||||
import json, os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
# SSOT for this list lives in the controlplane Go code:
|
||||
# molecule-controlplane/internal/slugs/ephemeral.go
|
||||
# (var EphemeralPrefixes). The redeploy-fleet auto-rollout
|
||||
# also reads from there to SKIP these slugs — without that
|
||||
# filter, fleet redeploy SSM-failed in-flight E2E tenants
|
||||
# whose containers were still booting, breaking the test
|
||||
# that just spun them up (molecule-controlplane#493).
|
||||
# Update both files together.
|
||||
EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
|
||||
with open("orgs.json") as f:
|
||||
data = json.load(f)
|
||||
max_age = int(os.environ["MAX_AGE_MINUTES"])
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
|
||||
for o in data.get("orgs", []):
|
||||
slug = o.get("slug", "")
|
||||
if not slug.startswith(EPHEMERAL_PREFIXES):
|
||||
continue
|
||||
created = o.get("created_at")
|
||||
if not created:
|
||||
# Defensively skip rows without created_at — better
|
||||
# to leave one orphan than nuke a brand-new row
|
||||
# whose timestamp didn't render.
|
||||
continue
|
||||
# Python 3.11+ handles RFC3339 with Z directly via
|
||||
# fromisoformat; older runners need the trailing Z swap.
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
if created_dt < cutoff:
|
||||
print(slug)
|
||||
PY
|
||||
|
||||
count=$(wc -l < stale_slugs.txt | tr -d ' ')
|
||||
echo "Found $count stale e2e org(s) older than ${MAX_AGE_MINUTES}m"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo "First 20:"
|
||||
head -20 stale_slugs.txt | sed 's/^/ /'
|
||||
fi
|
||||
echo "count=$count" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Safety gate
|
||||
if: steps.identify.outputs.count != '0'
|
||||
run: |
|
||||
count="${{ steps.identify.outputs.count }}"
|
||||
if [ "$count" -gt "$SAFETY_CAP" ]; then
|
||||
echo "::error::Refusing to delete $count orgs in one sweep (cap=$SAFETY_CAP). Investigate manually — this usually means the CP admin API returned no created_at or returned a degraded result. Re-run with workflow_dispatch + max_age_minutes if intentional."
|
||||
exit 1
|
||||
fi
|
||||
echo "Within safety cap ($count ≤ $SAFETY_CAP) ✓"
|
||||
|
||||
- name: Delete stale orgs
|
||||
if: steps.identify.outputs.count != '0' && env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set -uo pipefail
|
||||
deleted=0
|
||||
failed=0
|
||||
while IFS= read -r slug; do
|
||||
[ -z "$slug" ] && continue
|
||||
# The DELETE handler requires {"confirm": "<slug>"} matching
|
||||
# the URL slug — fat-finger guard. Idempotent: re-issuing
|
||||
# picks up via org_purges.last_step.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/del_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/del_code
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to runner log.
|
||||
http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
||||
deleted=$((deleted+1))
|
||||
echo " deleted: $slug"
|
||||
else
|
||||
failed=$((failed+1))
|
||||
echo " FAILED ($http_code): $slug — $(cat /tmp/del_resp 2>/dev/null | head -c 200)"
|
||||
fi
|
||||
done < stale_slugs.txt
|
||||
echo ""
|
||||
echo "Sweep summary: deleted=$deleted failed=$failed"
|
||||
# Don't fail the workflow on per-org delete errors — the
|
||||
# sweeper is best-effort. Next hourly tick re-attempts. We
|
||||
# only fail loud at the safety-cap gate above.
|
||||
|
||||
- name: Sweep orphan tunnels
|
||||
# Stale-org cleanup deletes the org (which cascades to tunnel
|
||||
# delete inside the CP). But when that cascade fails partway —
|
||||
# CP transient 5xx after the org row is deleted but before the
|
||||
# CF tunnel delete completes — the tunnel persists with no
|
||||
# matching org row. The reconciler in internal/sweep flags this
|
||||
# as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
|
||||
#
|
||||
# `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
|
||||
# reaper. Calling it here at the end of every sweep tick
|
||||
# converges the staging CF account to clean even when CP
|
||||
# cascades half-fail.
|
||||
#
|
||||
# PR #492 made the underlying DeleteTunnel actually check
|
||||
# status — pre-fix it silent-succeeded on CF code 1022
|
||||
# ("active connections"), so this step would have been a no-op
|
||||
# against stuck connectors. Post-fix the cleanup invokes
|
||||
# CleanupTunnelConnections + retry, which actually clears the
|
||||
# 1022 case. (#2987)
|
||||
#
|
||||
# Best-effort. Failure here doesn't fail the workflow — next
|
||||
# tick re-attempts. Errors flow to step output for ops review.
|
||||
if: env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set +e
|
||||
curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
|
||||
set -e
|
||||
http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
|
||||
body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
|
||||
if [ "$http_code" = "200" ]; then
|
||||
count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
|
||||
failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
|
||||
echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
|
||||
else
|
||||
echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
|
||||
fi
|
||||
|
||||
- name: Dry-run summary
|
||||
if: env.DRY_RUN == 'true'
|
||||
run: |
|
||||
echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."
|
||||
@@ -1,65 +0,0 @@
|
||||
name: Ops Scripts Tests
|
||||
|
||||
# Ported from .github/workflows/test-ops-scripts.yml on 2026-05-11 per
|
||||
# RFC internal#219 §1 sweep.
|
||||
#
|
||||
# Differences from the GitHub version:
|
||||
# - Dropped `merge_group:` trigger (no Gitea merge queue).
|
||||
# - on.paths references .gitea/workflows/test-ops-scripts.yml (this
|
||||
# file) instead of the .github/ one.
|
||||
# - Workflow-level env.GITHUB_SERVER_URL set.
|
||||
# - `continue-on-error: true` on the job (RFC §1 contract).
|
||||
#
|
||||
# Runs the unittest suite for scripts/ on every PR + push that touches
|
||||
# anything under scripts/. Kept separate from the main CI so a script-only
|
||||
# change doesn't trigger the heavier Go/Canvas/Python pipelines.
|
||||
#
|
||||
# Discovery layout: tests sit alongside the code they test (see
|
||||
# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
|
||||
# test_build_runtime_package.py for the rewriter coverage). The job
|
||||
# below runs `unittest discover` TWICE — once from `scripts/`, once
|
||||
# from `scripts/ops/` — because neither dir has an `__init__.py`, so
|
||||
# a single discover from `scripts/` doesn't recurse into the ops
|
||||
# subdir. Two passes is simpler than retrofitting namespace packages.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.gitea/workflows/test-ops-scripts.yml'
|
||||
|
||||
env:
|
||||
GITHUB_SERVER_URL: https://git.moleculesai.app
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Ops scripts (unittest)
|
||||
runs-on: ubuntu-latest
|
||||
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Run scripts/ unittests (build_runtime_package, ...)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, ...)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
+8
-78
@@ -95,91 +95,21 @@ if [ -n "$STAGED_GO" ]; then
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 5. Go: build check — catches bot-generated structurally-invalid Go (#1770)
|
||||
# 5. Secrets: No tokens/keys in staged files
|
||||
# ──────────────────────────────────────────────────────────
|
||||
#
|
||||
# Background: bot agents have produced syntactically-broken Go that the
|
||||
# patch tool happily applied (e.g. PR #1769 commit 66ea0b64 — function
|
||||
# declaration nested inside another function's body). Compilation failed,
|
||||
# staging Platform(Go) was red for hours. CI catches this AT PR-time but
|
||||
# by then the malformed commit is already shared.
|
||||
#
|
||||
# Pre-commit guard: when ANY .go file in workspace-server/ is staged, run
|
||||
# `go build ./...` from workspace-server. If it fails, reject the commit.
|
||||
# Cost: ~5-10s on a warm cache; acceptable for the class of bug it
|
||||
# catches. Skip when go isn't available (CI runners that need to bypass).
|
||||
|
||||
if [ -n "$STAGED_GO" ]; then
|
||||
if command -v go >/dev/null 2>&1; then
|
||||
if ! (cd workspace-server && go build ./... >/tmp/precommit-go-build.log 2>&1); then
|
||||
echo "❌ GO BUILD FAILED — staged Go changes don't compile (workspace-server/)."
|
||||
echo " Output:"
|
||||
sed 's/^/ /' /tmp/precommit-go-build.log | head -20
|
||||
echo " Fix the build error before committing. See #1770 for context."
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
else
|
||||
# Bots and CI runners may bypass when go isn't installed — surface a
|
||||
# warning so the absence is visible, but don't block. Humans hit this
|
||||
# only if they didn't run setup.sh.
|
||||
echo "⚠️ go not installed — skipping go-build pre-commit check (#1770)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────
|
||||
# 6. Secrets: No tokens/keys in staged files
|
||||
# ──────────────────────────────────────────────────────────
|
||||
#
|
||||
# Pattern set MUST match .github/workflows/secret-scan.yml SECRET_PATTERNS
|
||||
# and molecule-ai-workspace-runtime/molecule_runtime/scripts/pre-commit-checks.sh —
|
||||
# .github/workflows/secret-pattern-drift.yml lints this invariant. Rebuilt
|
||||
# against canonical 2026-05-02 after #1569 Phase 1 discovery surfaced
|
||||
# real ghs_*/github_pat_* leaks that the prior pattern set
|
||||
# ('sk-ant-|sk-proj-|ghp_|gho_|AKIA|mol_pk_|cfut_') would have missed:
|
||||
# (a) it lacked ghs_ / ghu_ / ghr_ / github_pat_ / sk-svcacct- / sk-cp- /
|
||||
# xox[baprs]- / ASIA prefixes, (b) it skipped *.md and docs/* — but the
|
||||
# actual leaks lived in tick-reflections-temp.md, qa-audit-2026-04-21.md,
|
||||
# docs/incidents/INCIDENT_LOG.md.
|
||||
SECRET_PATTERNS=(
|
||||
'ghp_[A-Za-z0-9]{36,}' # GitHub PAT (classic)
|
||||
'ghs_[A-Za-z0-9]{36,}' # GitHub App installation token
|
||||
'gho_[A-Za-z0-9]{36,}' # GitHub OAuth user-to-server
|
||||
'ghu_[A-Za-z0-9]{36,}' # GitHub OAuth user
|
||||
'ghr_[A-Za-z0-9]{36,}' # GitHub OAuth refresh
|
||||
'github_pat_[A-Za-z0-9_]{82,}' # GitHub fine-grained PAT
|
||||
'sk-ant-[A-Za-z0-9_-]{40,}' # Anthropic API key
|
||||
'sk-proj-[A-Za-z0-9_-]{40,}' # OpenAI project key
|
||||
'sk-svcacct-[A-Za-z0-9_-]{40,}' # OpenAI service-account key
|
||||
'sk-cp-[A-Za-z0-9_-]{60,}' # MiniMax API key (F1088 vector — caught only after the fact)
|
||||
'xox[baprs]-[A-Za-z0-9-]{20,}' # Slack tokens (bot/app/user/refresh)
|
||||
'AKIA[0-9A-Z]{16}' # AWS access key ID
|
||||
'ASIA[0-9A-Z]{16}' # AWS STS temp access key ID
|
||||
)
|
||||
|
||||
ALL_STAGED=$(git diff --cached --name-only --diff-filter=ACM || true)
|
||||
if [ -n "$ALL_STAGED" ]; then
|
||||
for f in $ALL_STAGED; do
|
||||
# Skip ONLY binary + lockfiles + the hook itself. Markdown +
|
||||
# docs/* are NOT skipped — that was the bug (#1569 leaks were
|
||||
# all in *.md). If a doc legitimately needs a token-shaped
|
||||
# placeholder, use ghs_EXAMPLE_TOKEN_DO_NOT_USE — short enough
|
||||
# to dodge the {36,} length suffix.
|
||||
if echo "$f" | grep -qE '\.png$|\.jpg$|\.ico$|\.woff|node_modules|\.lock$|\.githooks/'; then
|
||||
# Skip binary, known safe files, hooks, docs, and markdown
|
||||
if echo "$f" | grep -qE '\.png$|\.jpg$|\.ico$|\.woff|node_modules|\.lock$|\.githooks/|\.md$|docs/'; then
|
||||
continue
|
||||
fi
|
||||
DIFF=$(git diff --cached --no-color --unified=0 -- "$f" 2>/dev/null | grep -E '^\+[^+]' || true)
|
||||
[ -z "$DIFF" ] && continue
|
||||
for pattern in "${SECRET_PATTERNS[@]}"; do
|
||||
if echo "$DIFF" | grep -qE "$pattern"; then
|
||||
echo "❌ POSSIBLE SECRET in $f (matched: ${pattern})"
|
||||
echo " The actual matched value is NOT echoed here — round-tripping a"
|
||||
echo " leaked credential into scrollback widens the blast radius."
|
||||
echo " If false positive (test/docs example), use a short placeholder"
|
||||
echo " like ghs_EXAMPLE_TOKEN_DO_NOT_USE that doesn't satisfy the length."
|
||||
ERRORS=$((ERRORS + 1))
|
||||
break
|
||||
fi
|
||||
done
|
||||
DIFF=$(git diff --cached "$f" 2>/dev/null | grep '^+' | grep -v '^+++' || true)
|
||||
if echo "$DIFF" | grep -qE 'sk-ant-|sk-proj-|ghp_|gho_|AKIA[A-Z0-9]|mol_pk_|cfut_' 2>/dev/null; then
|
||||
echo "❌ POSSIBLE SECRET in $f — do not commit API keys or tokens"
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
# Default reviewer routing for molecule-core.
|
||||
#
|
||||
# `*` matches every changed path, so every PR auto-requests review from
|
||||
# @hongmingwang-moleculeai. The agent-PR pattern is that the
|
||||
# HongmingWang-Rabbit (agent) account authors PRs; this file routes
|
||||
# them into the personal account's review queue automatically — no
|
||||
# manual `gh pr edit --add-reviewer` per PR.
|
||||
#
|
||||
# Why CODEOWNERS instead of branch-protection's review-from-anyone gate:
|
||||
# the gate just says "1 review needed"; CODEOWNERS specifies *which*
|
||||
# reviewer the request goes to. Without it, agent PRs sit unreviewed
|
||||
# until a human happens to look at the queue.
|
||||
#
|
||||
# Note: `require_code_owner_reviews` on the staging branch protection
|
||||
# is currently OFF, so the routing is informational rather than
|
||||
# enforced. Flip it on (in branch protection settings) if you want
|
||||
# CODEOWNERS approval to be the *required* review type. Until then,
|
||||
# any approving review still satisfies the 1-review gate — this just
|
||||
# makes sure the right person sees it.
|
||||
* @hongmingwang-moleculeai
|
||||
@@ -1,80 +0,0 @@
|
||||
# Dependabot — auto-bump pinned dependencies.
|
||||
#
|
||||
# Why this exists:
|
||||
#
|
||||
# All `uses:` references in .github/workflows/*.yml are pinned to commit
|
||||
# SHAs (with `# v<N>` comments for human readability) instead of mutable
|
||||
# tags like `@v4`. Tag pinning is a known supply-chain risk: a maintainer
|
||||
# (or compromised maintainer account) can repoint `@v4` to malicious code
|
||||
# and our pipelines silently pull it. SHA pinning closes that risk.
|
||||
#
|
||||
# But SHA pinning has a maintenance cost: each upstream legitimate fix
|
||||
# requires manually finding + bumping the SHA. Dependabot for Actions
|
||||
# closes that gap by opening PRs to bump pinned SHAs whenever upstream
|
||||
# tags a new version. Reviewer evaluates the bump like any other
|
||||
# dependency PR.
|
||||
#
|
||||
# Combined: SHA pinning gives us security, Dependabot keeps us current.
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
# GitHub Actions — every workflow file under .github/workflows/.
|
||||
# Weekly cadence is enough for a CI surface this size; the supply-
|
||||
# chain attack window is "minutes between repoint and pull," and
|
||||
# weekly auto-bumps don't help with zero-days regardless. The point
|
||||
# is to pull in non-zero-day fixes without operator effort, not to
|
||||
# be real-time.
|
||||
- package-ecosystem: github-actions
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- dependencies
|
||||
- github-actions
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
|
||||
# Go module — workspace-server. Bumps go.mod deps via PR weekly.
|
||||
- package-ecosystem: gomod
|
||||
directory: "/workspace-server"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- dependencies
|
||||
- go
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
|
||||
# npm — canvas (Next.js bundle). Largest dep tree in this repo;
|
||||
# weekly cadence keeps the security surface fresh without flooding
|
||||
# the queue. open-pull-requests-limit: 10 because npm churns more
|
||||
# than the others.
|
||||
- package-ecosystem: npm
|
||||
directory: "/canvas"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- dependencies
|
||||
- npm
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
|
||||
# Python — workspace runtime requirements. Pip/requirements.txt-
|
||||
# backed rather than pyproject.toml; Dependabot supports both.
|
||||
- package-ecosystem: pip
|
||||
directory: "/workspace"
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
labels:
|
||||
- dependencies
|
||||
- python
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
@@ -1,166 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lint SECRET_PATTERNS drift across known consumers of molecule-core's canonical.
|
||||
|
||||
The canonical SECRET_PATTERNS array in
|
||||
.github/workflows/secret-scan.yml is mirrored by every other side
|
||||
that scans for credentials: the workspace-runtime's bundled
|
||||
pre-commit hook, the molecule-controlplane inlined copy, etc. The
|
||||
mirror is enforced socially today — when someone adds a new pattern
|
||||
to canonical (e.g. the sk-cp- MiniMax token after F1088), the other
|
||||
sides are supposed to be updated in lockstep.
|
||||
|
||||
This script automates the check. Diffs the canonical's pattern set
|
||||
against each known public consumer and exits non-zero on any
|
||||
mismatch. Wired into a daily cron + on-push gate via
|
||||
.github/workflows/secret-pattern-drift.yml.
|
||||
|
||||
Private-repo consumers (currently molecule-controlplane's inlined
|
||||
copy) are out of scope here because the molecule-core workflow's
|
||||
GITHUB_TOKEN can't read other private repos in the org. They're
|
||||
expected to self-monitor via their own copy of this script — not a
|
||||
hard barrier, just a future expansion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
CANONICAL_FILE = Path(".gitea/workflows/secret-scan.yml")
|
||||
|
||||
# Public consumer mirrors. Each entry is (label, raw_url) — raw_url
|
||||
# points at the file's RAW content on the consumer's default branch
|
||||
# (or staging where applicable). Add an entry here when a new public
|
||||
# repo starts shipping its own SECRET_PATTERNS array.
|
||||
CONSUMERS: list[tuple[str, str]] = [
|
||||
(
|
||||
"molecule-ai-workspace-runtime/molecule_runtime/scripts/pre-commit-checks.sh",
|
||||
"https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime/raw/branch/main/molecule_runtime/scripts/pre-commit-checks.sh",
|
||||
),
|
||||
]
|
||||
|
||||
# In-repo consumers — paths read locally from the workflow checkout.
|
||||
# Read-from-disk avoids the staging→main lag that the URL fetcher
|
||||
# would hit (a freshly-edited canonical wouldn't yet be on the
|
||||
# consumer's default branch). Same drift semantics, no network.
|
||||
LOCAL_CONSUMERS: list[tuple[str, Path]] = [
|
||||
(
|
||||
".githooks/pre-commit (molecule-core local hook)",
|
||||
Path(".githooks/pre-commit"),
|
||||
),
|
||||
]
|
||||
|
||||
# Matches the SECRET_PATTERNS=( ... ) array in either yaml-indented
|
||||
# (the canonical workflow's `run:` block) or shell-flat (runtime
|
||||
# hook) format. Patterns inside are single-quoted Bash strings; we
|
||||
# pull each via _PATTERN_RE.
|
||||
#
|
||||
# Closing `)` is anchored to the start of a line (possibly indented)
|
||||
# because pattern comments like `# GitHub PAT (classic)` contain
|
||||
# their own `)` mid-line — a non-anchored regex would match through
|
||||
# the comment's paren and capture only the first pattern.
|
||||
_ARRAY_RE = re.compile(r"SECRET_PATTERNS=\((.*?)^\s*\)", re.DOTALL | re.MULTILINE)
|
||||
_PATTERN_RE = re.compile(r"'([^']+)'")
|
||||
|
||||
|
||||
def extract_patterns(content: str, source_label: str) -> list[str]:
|
||||
"""Pull the SECRET_PATTERNS list out of either format. Raises if missing."""
|
||||
m = _ARRAY_RE.search(content)
|
||||
if not m:
|
||||
raise SystemExit(f"::error::{source_label}: SECRET_PATTERNS=(...) array not found")
|
||||
return _PATTERN_RE.findall(m.group(1))
|
||||
|
||||
|
||||
def fetch(url: str) -> str:
|
||||
req = urllib.request.Request(
|
||||
url, headers={"User-Agent": "secret-pattern-drift-lint/1"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return resp.read().decode("utf-8")
|
||||
|
||||
|
||||
def diff_patterns(canonical: list[str], consumer: list[str]) -> tuple[list[str], list[str]]:
|
||||
"""Return (missing_from_consumer, extra_in_consumer) — both sorted."""
|
||||
canonical_set = set(canonical)
|
||||
consumer_set = set(consumer)
|
||||
return (
|
||||
sorted(canonical_set - consumer_set),
|
||||
sorted(consumer_set - canonical_set),
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not CANONICAL_FILE.exists():
|
||||
print(f"::error::canonical not found at {CANONICAL_FILE}")
|
||||
return 1
|
||||
|
||||
canonical = extract_patterns(CANONICAL_FILE.read_text(), str(CANONICAL_FILE))
|
||||
print(f"canonical ({CANONICAL_FILE}): {len(canonical)} patterns")
|
||||
|
||||
drift = False
|
||||
|
||||
# In-repo consumers first — these are read from the workflow's own
|
||||
# checkout, so they never lag behind the canonical and a missing
|
||||
# file IS a real error (not a fetch warning).
|
||||
for label, path in LOCAL_CONSUMERS:
|
||||
if not path.exists():
|
||||
print(f"::error::{label}: file not found at {path}")
|
||||
drift = True
|
||||
continue
|
||||
consumer = extract_patterns(path.read_text(), label)
|
||||
missing, extra = diff_patterns(canonical, consumer)
|
||||
if not missing and not extra:
|
||||
print(f" ✓ {label}: aligned ({len(consumer)} patterns)")
|
||||
continue
|
||||
drift = True
|
||||
print(f"::error::DRIFT in {label}:")
|
||||
for p in missing:
|
||||
print(f" - missing from consumer: {p!r}")
|
||||
for p in extra:
|
||||
print(f" - extra in consumer (not in canonical): {p!r}")
|
||||
|
||||
for label, url in CONSUMERS:
|
||||
try:
|
||||
content = fetch(url)
|
||||
except Exception as e:
|
||||
# Fetch failures are warnings, not errors. A consumer
|
||||
# whose default branch was just renamed (or whose file
|
||||
# moved) shouldn't fail the lint until someone updates
|
||||
# the URL above. Real drift is the failure mode this
|
||||
# gate exists to catch — fetch reliability isn't.
|
||||
print(f"::warning::{label}: fetch failed ({e}) — skipping")
|
||||
continue
|
||||
|
||||
consumer = extract_patterns(content, label)
|
||||
missing, extra = diff_patterns(canonical, consumer)
|
||||
if not missing and not extra:
|
||||
print(f" ✓ {label}: aligned ({len(consumer)} patterns)")
|
||||
continue
|
||||
|
||||
drift = True
|
||||
print(f"::error::DRIFT in {label}:")
|
||||
for p in missing:
|
||||
print(f" - missing from consumer: {p!r}")
|
||||
for p in extra:
|
||||
print(f" - extra in consumer (not in canonical): {p!r}")
|
||||
|
||||
if drift:
|
||||
print()
|
||||
print("::error::SECRET_PATTERNS drift detected. Bring consumer(s) into")
|
||||
print("alignment with the canonical SECRET_PATTERNS array in")
|
||||
print(f"{CANONICAL_FILE} by adding the missing patterns and removing")
|
||||
print("any extras. The two sides must stay byte-aligned on the pattern")
|
||||
print("list — the runtime hook is the developer's local pre-commit,")
|
||||
print("the canonical is the org-wide CI gate, divergence means a token")
|
||||
print("can pass one but get rejected by the other.")
|
||||
return 1
|
||||
|
||||
print()
|
||||
print("✓ All known consumers aligned with canonical SECRET_PATTERNS.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,154 +0,0 @@
|
||||
name: Block internal-flavored paths
|
||||
|
||||
# Hard CI gate. Internal content (positioning, competitive briefs, sales
|
||||
# playbooks, PMM/press drip, draft campaigns) lives in molecule-ai/internal —
|
||||
# this public monorepo must never re-acquire those paths. CEO directive
|
||||
# 2026-04-23 after a fleet-wide audit found 79 internal files leaked here.
|
||||
#
|
||||
# Failure mode without this gate: agents (PMM, Research, DevRel, Sales) drop
|
||||
# briefs into the easiest path their cwd resolves to (root /research,
|
||||
# /marketing, /docs/marketing) and gitignore alone won't catch a `git add -f`
|
||||
# or a stale gitignore line. This workflow is the mechanical backstop.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
push:
|
||||
branches: [main, staging]
|
||||
# Required for GitHub merge queue: the queue's pre-merge CI run on
|
||||
# `gh-readonly-queue/...` refs needs this check to fire so the queue
|
||||
# gets a real result instead of stalling forever AWAITING_CHECKS.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Block forbidden paths
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 2 # need previous commit to diff against on push events
|
||||
|
||||
# For pull_request events the diff base is github.event.pull_request.base.sha,
|
||||
# which may be many commits behind HEAD and therefore absent from the
|
||||
# shallow clone above. Fetch it explicitly (depth=1 keeps it fast).
|
||||
- name: Fetch PR base SHA (pull_request events only)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: git fetch --depth=1 origin ${{ github.event.pull_request.base.sha }}
|
||||
|
||||
# For merge_group events the queue's pre-merge ref is a commit on
|
||||
# `gh-readonly-queue/...` whose parent is the queue's base_sha.
|
||||
# That parent isn't part of the queue branch's shallow clone, so
|
||||
# we fetch it explicitly. Mirrors the equivalent step in
|
||||
# secret-scan.yml (#2120) — same shallow-clone bug class.
|
||||
- name: Fetch merge_group base SHA (merge_group events only)
|
||||
if: github.event_name == 'merge_group'
|
||||
run: git fetch --depth=1 origin ${{ github.event.merge_group.base_sha }}
|
||||
|
||||
- name: Refuse if forbidden paths appear
|
||||
env:
|
||||
# Plumb event-specific SHAs through env so the script doesn't
|
||||
# need conditional `${{ ... }}` interpolation per event type.
|
||||
# github.event.before/after only exist on push events;
|
||||
# merge_group has its own base_sha/head_sha; pull_request has
|
||||
# pull_request.base.sha / pull_request.head.sha.
|
||||
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
MG_BASE_SHA: ${{ github.event.merge_group.base_sha }}
|
||||
MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }}
|
||||
PUSH_BEFORE: ${{ github.event.before }}
|
||||
PUSH_AFTER: ${{ github.event.after }}
|
||||
run: |
|
||||
# Paths that must NEVER live in the public monorepo. Add to this
|
||||
# list narrowly — broader patterns belong in .gitignore so day-to-day
|
||||
# docs work isn't accidentally blocked.
|
||||
FORBIDDEN_PATTERNS=(
|
||||
"^research/"
|
||||
"^marketing/"
|
||||
"^docs/marketing/"
|
||||
"^comment-[0-9]+\.json$"
|
||||
"^test-pmm.*\.(txt|md)$"
|
||||
"^tick-reflections.*\.(txt|md)$"
|
||||
".*-temp\.(md|txt)$"
|
||||
)
|
||||
|
||||
# Determine the diff base. Each event type stores its SHAs in
|
||||
# a different place — see the env block above.
|
||||
case "${{ github.event_name }}" in
|
||||
pull_request)
|
||||
BASE="$PR_BASE_SHA"
|
||||
HEAD="$PR_HEAD_SHA"
|
||||
;;
|
||||
merge_group)
|
||||
BASE="$MG_BASE_SHA"
|
||||
HEAD="$MG_HEAD_SHA"
|
||||
;;
|
||||
*)
|
||||
BASE="$PUSH_BEFORE"
|
||||
HEAD="$PUSH_AFTER"
|
||||
;;
|
||||
esac
|
||||
|
||||
# On push events with shallow clones, BASE may be present in
|
||||
# the event payload but absent from the local object DB
|
||||
# (fetch-depth=2 doesn't always reach the previous commit
|
||||
# across true merges). Try fetching it on demand. If the
|
||||
# fetch fails — e.g. the SHA was force-overwritten — we fall
|
||||
# through to the empty-BASE branch below, which scans the
|
||||
# entire tree as if every file were new. Correct, just slow.
|
||||
# Same recovery shape as secret-scan.yml (#2120 — incident
|
||||
# 2026-04-27 06:50Z block-internal-paths exit 128 with
|
||||
# "fatal: bad object <sha>" on staging push).
|
||||
if [ -n "$BASE" ] && ! echo "$BASE" | grep -qE '^0+$'; then
|
||||
if ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
git fetch --depth=1 origin "$BASE" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Files added or modified in this change.
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$' || ! git cat-file -e "$BASE" 2>/dev/null; then
|
||||
# New branch / no previous SHA / BASE unreachable — check
|
||||
# the entire tree as if every file were new. Slower but
|
||||
# correct on first push or post-fetch-failure recovery.
|
||||
CHANGED=$(git ls-tree -r --name-only HEAD)
|
||||
else
|
||||
CHANGED=$(git diff --name-only --diff-filter=AM "$BASE" "$HEAD")
|
||||
fi
|
||||
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "No changed files to inspect."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
OFFENDING=""
|
||||
for path in $CHANGED; do
|
||||
for pattern in "${FORBIDDEN_PATTERNS[@]}"; do
|
||||
if echo "$path" | grep -qE "$pattern"; then
|
||||
OFFENDING="${OFFENDING}${path} (matched: ${pattern})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ -n "$OFFENDING" ]; then
|
||||
echo "::error::Forbidden internal-flavored paths detected:"
|
||||
printf "$OFFENDING"
|
||||
echo ""
|
||||
echo "These paths belong in molecule-ai/internal, not this public repo."
|
||||
echo "See docs/internal-content-policy.md for canonical locations."
|
||||
echo ""
|
||||
echo "If your file is genuinely public-facing (e.g. a blog post"
|
||||
echo "ready to ship), use one of these alternatives instead:"
|
||||
echo " • Public-bound blog posts: docs/blog/<slug>.md"
|
||||
echo " • Public-bound tutorials: docs/tutorials/<slug>.md"
|
||||
echo " • Public devrel content: docs/devrel/<slug>.md"
|
||||
echo ""
|
||||
echo "If you legitimately need to add a new top-level path that"
|
||||
echo "happens to match a forbidden pattern, edit"
|
||||
echo ".github/workflows/block-internal-paths.yml and update the"
|
||||
echo "FORBIDDEN_PATTERNS list with reviewer signoff."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ No forbidden paths in this change."
|
||||
@@ -1,320 +0,0 @@
|
||||
name: Canary — staging SaaS smoke (every 30 min)
|
||||
|
||||
# Minimum viable health check: provisions one Hermes workspace on a fresh
|
||||
# staging org, sends one A2A message, verifies PONG, tears down. ~8 min
|
||||
# wall clock. Pages on failure by opening a GitHub issue; auto-closes the
|
||||
# issue on the next green run.
|
||||
#
|
||||
# The full-SaaS workflow (e2e-staging-saas.yml) covers the broader surface
|
||||
# but runs only on provisioning-critical pushes + nightly — this one
|
||||
# catches drift in the 30-min window between those runs (AMI health, CF
|
||||
# cert rotation, WorkOS session stability, etc.).
|
||||
#
|
||||
# Lean mode: E2E_MODE=canary skips the child workspace + HMA memory +
|
||||
# peers/activity checks. One parent workspace + one A2A turn is enough
|
||||
# to signal "SaaS stack end-to-end is alive."
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 30 min. Cron on GitHub-hosted runners has a known drift of
|
||||
# a few minutes under load — that's fine for a canary.
|
||||
- cron: '*/30 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_on_failure:
|
||||
description: >-
|
||||
Skip teardown when the canary fails (debugging only). The
|
||||
tenant org + EC2 + CF tunnel + DNS stay alive so an operator
|
||||
can SSM into the workspace EC2 and capture docker logs of the
|
||||
failing claude-code container. REMEMBER to manually delete
|
||||
via DELETE /cp/admin/tenants/<slug> when done so the org
|
||||
doesn't accumulate cost. Only honored on workflow_dispatch;
|
||||
cron runs always tear down (we don't want unattended cron
|
||||
to leak resources).
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Serialise with the full-SaaS workflow so they don't contend for the
|
||||
# same org-create quota on staging. Different group key from
|
||||
# e2e-staging-saas since we don't mind queueing canaries behind one
|
||||
# full run, but two canaries SHOULD queue against each other.
|
||||
concurrency:
|
||||
group: canary-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
# Needed to open / close the alerting issue.
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
canary:
|
||||
name: Canary smoke
|
||||
runs-on: ubuntu-latest
|
||||
# 25 min headroom over the 15-min TLS-readiness deadline in
|
||||
# tests/e2e/test_staging_full_saas.sh (#2107). Without the buffer
|
||||
# the job is killed at the wall-clock 15:00 mark BEFORE the bash
|
||||
# `fail` + diagnostic burst can fire, leaving every cancellation
|
||||
# silent. Sibling staging E2E jobs run at 20-45 min — keeping
|
||||
# canary tighter than them so a true wedge still surfaces here
|
||||
# first.
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the canary's PRIMARY LLM auth path post-2026-05-04.
|
||||
# Switched from hermes+OpenAI after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the canary red the entire time). claude-code template's
|
||||
# `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot —
|
||||
# ~5-10x cheaper per token than gpt-4.1-mini AND on a separate
|
||||
# billing account, so OpenAI quota collapse no longer wedges the
|
||||
# canary. Mirrors the migration continuous-synth-e2e.yml made on
|
||||
# 2026-05-03 (#265) for the same reason. tests/e2e/test_staging_
|
||||
# full_saas.sh branches SECRETS_JSON on which key is present —
|
||||
# MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes overridden via workflow_dispatch can still
|
||||
# exercise the OpenAI path without re-editing the workflow.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
E2E_MODE: canary
|
||||
E2E_RUNTIME: claude-code
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default (which could resolve to "sonnet" →
|
||||
# direct Anthropic and defeat the cost saving). M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: MiniMax-M2.7-highspeed
|
||||
E2E_RUN_ID: "canary-${{ github.run_id }}"
|
||||
# Debug-only: when an operator dispatches with keep_on_failure=true,
|
||||
# the canary script's E2E_KEEP_ORG=1 path skips teardown so the
|
||||
# tenant org + EC2 stay alive for SSM-based log capture. Cron runs
|
||||
# never set this (the input only exists on workflow_dispatch) so
|
||||
# unattended cron always tears down. See molecule-core#129
|
||||
# failure mode #1 — capturing the actual exception requires
|
||||
# docker logs from the live container.
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_on_failure == 'true' && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per the lesson from synth E2E #2578:
|
||||
# an empty key silently falls through to the wrong
|
||||
# SECRETS_JSON branch and the canary fails 5 min later with
|
||||
# a confusing auth error instead of the clean "secret
|
||||
# missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain. Operators only need to set ONE of these
|
||||
# secrets; we don't force a choice between them.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — A2A will fail at request time with 'No LLM provider configured'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: Canary run
|
||||
id: canary
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Alerting: open a sticky issue on the FIRST failure; comment on
|
||||
# subsequent failures; auto-close on next green. Comment-on-existing
|
||||
# de-duplicates so a single open issue accumulates the streak —
|
||||
# ops sees one issue with N comments rather than N issues.
|
||||
#
|
||||
# Why no consecutive-failures threshold (e.g., wait 3 runs before
|
||||
# filing): the prior threshold check used
|
||||
# `github.rest.actions.listWorkflowRuns()` which Gitea 1.22.6 does
|
||||
# not expose (returns 404). On Gitea Actions the threshold call
|
||||
# ALWAYS failed, breaking the entire alerting step and going days
|
||||
# silent on real regressions (38h+ chronic red on 2026-05-07/08
|
||||
# before this fix; tracked in molecule-core#129). Filing on first
|
||||
# failure is also better UX — we want to know about the first red,
|
||||
# not wait 90 min for it to "count." Real flakes get one issue +
|
||||
# a quick close-on-green; persistent reds accumulate comments.
|
||||
- name: Open issue on failure
|
||||
if: failure()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const runURL = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
// Find an existing open canary issue (stable title match).
|
||||
// If one exists, this isn't a "first failure" — comment and exit.
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary still failing. ${runURL}`,
|
||||
});
|
||||
core.info(`Commented on existing issue #${match.number}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// No open issue yet — file one on this first failure. The
|
||||
// comment-on-existing branch above means subsequent failures
|
||||
// accumulate as comments on this same issue, so we don't
|
||||
// spam new issues per run.
|
||||
const body =
|
||||
`Canary run failed at ${new Date().toISOString()}.\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`This issue auto-closes on the next green canary run. ` +
|
||||
`Consecutive failures add a comment here rather than a new issue.`;
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['canary-staging', 'bug'],
|
||||
});
|
||||
core.info('Opened canary failure issue (first red)');
|
||||
|
||||
- name: Auto-close canary issue on success
|
||||
if: success()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = '🔴 Canary failing: staging SaaS smoke';
|
||||
const { data: open } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'canary-staging',
|
||||
per_page: 10,
|
||||
});
|
||||
const match = open.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Canary recovered at ${new Date().toISOString()}. Closing.`,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
state: 'closed',
|
||||
});
|
||||
core.info(`Closed recovered canary issue #${match.number}`);
|
||||
}
|
||||
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
# Slug prefix matches what test_staging_full_saas.sh emits
|
||||
# in canary mode:
|
||||
# SLUG="e2e-canary-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
|
||||
# Earlier this was `e2e-{today}-canary-` — that was the
|
||||
# full-mode pattern (date FIRST, mode SECOND); canary slugs
|
||||
# have mode FIRST, date SECOND. The mismatch silently
|
||||
# never matched, leaving every cancelled-canary EC2 alive
|
||||
# until the once-an-hour sweep eventually caught it
|
||||
# (incident 2026-04-26 21:03Z: 1h25m EC2 leak before manual
|
||||
# cleanup; same gap on three earlier cancellations today).
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope to slugs from THIS canary run when GITHUB_RUN_ID is
|
||||
# available; the canary workflow sets E2E_RUN_ID='canary-\${run_id}'
|
||||
# so the slug suffix is '-canary-\${run_id}-...'. Mirrors the
|
||||
# full-mode safety net's per-run scoping (e2e-staging-saas.yml)
|
||||
# added after the 2026-04-21 cross-run cleanup incident.
|
||||
# Sweep both today AND yesterday's UTC dates so a run that
|
||||
# crosses midnight still cleans up its own slug — see the
|
||||
# 2026-04-26→27 canvas-safety-net incident.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-canary-{d}-canary-{run_id}' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-canary-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug DELETE with HTTP-code verification. The previous
|
||||
# `... >/dev/null || true` swallowed every failure, so a 5xx
|
||||
# or timeout from CP looked identical to "successfully cleaned
|
||||
# up" and the tenant kept eating ~2 vCPU until the hourly
|
||||
# stale sweep caught it (up to 2h later). Now we capture the
|
||||
# response code and surface non-2xx as a workflow warning, so
|
||||
# the run page shows which slug leaked. We still don't `exit 1`
|
||||
# on cleanup failure — a single-canary cleanup miss shouldn't
|
||||
# fail-flag the canary itself when the actual smoke check
|
||||
# passed. The sweep-stale-e2e-orgs cron (now every 15 min,
|
||||
# 30-min threshold) is the safety net for whatever slips past.
|
||||
# See molecule-controlplane#420.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canary-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canary-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canary-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canary teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canary-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::canary teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,34 +1,19 @@
|
||||
name: canary-verify
|
||||
|
||||
# Runs the canary smoke suite against the staging canary tenant fleet
|
||||
# after a new :staging-<sha> image lands in ECR. On green, calls the
|
||||
# CP redeploy-fleet endpoint to promote :staging-<sha> → :latest so
|
||||
# the prod tenant fleet's 5-minute auto-updater picks up the verified
|
||||
# digest. On red, :latest stays on the prior known-good digest and
|
||||
# prod is untouched.
|
||||
#
|
||||
# Registry note (2026-05-10): This workflow previously used GHCR
|
||||
# (ghcr.io/molecule-ai/platform-tenant) — that registry was retired
|
||||
# during the 2026-05-06 Gitea suspension migration when publish-
|
||||
# workspace-server-image.yml switched to the operator's ECR org
|
||||
# (153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/
|
||||
# platform-tenant). The GHCR → ECR migration was never applied to
|
||||
# this file, so canary-verify was silently smoke-testing the stale
|
||||
# GHCR image while the actual staging/prod tenants ran the ECR image.
|
||||
# Result: smoke tests could not catch a broken ECR build. Fix:
|
||||
# - Wait step: reads SHA from running canary /health (tenant-
|
||||
# agnostic, works regardless of registry).
|
||||
# - Promote step: calls CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha>, same mechanism as redeploy-tenants-on-main.yml.
|
||||
# No longer attempts GHCR crane ops.
|
||||
# after a new :staging-<sha> image lands in GHCR. On green, promotes
|
||||
# :staging-<sha> → :latest so the prod tenant fleet's 5-minute
|
||||
# auto-updater picks up the verified digest. On red, :latest stays
|
||||
# on the prior known-good digest and prod is untouched.
|
||||
#
|
||||
# Dependencies:
|
||||
# - publish-workspace-server-image.yml publishes :staging-<sha>
|
||||
# to ECR on staging and main merges.
|
||||
# - Canary tenants are configured to pull :staging-<sha> from ECR
|
||||
# (TENANT_IMAGE env set to the ECR :staging-<sha> tag).
|
||||
# (NOT :latest) on main merge
|
||||
# - canary tenants are configured to pull :staging-<sha> as their
|
||||
# tenant image (set TENANT_IMAGE=ghcr.io/…:staging-<sha> on the
|
||||
# canary provisioner code path OR rotate via an admin endpoint)
|
||||
# - Repo secrets CANARY_TENANT_URLS / CANARY_ADMIN_TOKENS /
|
||||
# CANARY_CP_SHARED_SECRET are populated.
|
||||
# CANARY_CP_SHARED_SECRET are populated
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
@@ -42,24 +27,21 @@ permissions:
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# ECR registry (post-2026-05-06 SSOT for tenant images).
|
||||
# publish-workspace-server-image.yml pushes here.
|
||||
IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/platform-tenant
|
||||
# CP endpoint for redeploy-fleet (used in promote step below).
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
canary-smoke:
|
||||
# Skip when the upstream workflow failed — no image to test against.
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
|
||||
runs-on: ubuntu-latest
|
||||
# Self-hosted mac mini — GitHub-hosted minutes are quota-blocked on
|
||||
# this org (same reason publish/promote-latest moved earlier).
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
outputs:
|
||||
sha: ${{ steps.compute.outputs.sha }}
|
||||
smoke_ran: ${{ steps.smoke.outputs.ran }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Compute sha
|
||||
id: compute
|
||||
@@ -67,16 +49,11 @@ jobs:
|
||||
|
||||
- name: Wait for canary tenants to pick up :staging-<sha>
|
||||
# Poll canary health endpoints every 30s for up to 7 min instead
|
||||
# of a fixed 6-min sleep. Exits as soon as ALL canaries report
|
||||
# the new SHA (~2-3 min typical vs 6 min fixed). Falls back to
|
||||
# proceeding after 7 min even if not all canaries responded —
|
||||
# the smoke suite will catch any that didn't update.
|
||||
#
|
||||
# NOTE: The SHA is read from the running tenant's /health response,
|
||||
# NOT from a registry lookup. This is registry-agnostic and works
|
||||
# regardless of whether the tenant pulls from ECR, GHCR, or any
|
||||
# other registry — the canary is telling us what it's actually
|
||||
# running, which is the ground truth for smoke testing.
|
||||
# of a fixed 6-min sleep. Exits as soon as ALL canaries report the
|
||||
# new SHA, freeing the self-hosted runner slot sooner (~2-3 min
|
||||
# typical vs 6 min fixed). Falls back to proceeding after 7 min
|
||||
# even if not all canaries responded — the smoke suite will catch
|
||||
# any that didn't update.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
EXPECTED_SHA: ${{ steps.compute.outputs.sha }}
|
||||
@@ -111,38 +88,12 @@ jobs:
|
||||
echo "Timeout after ${MAX_WAIT}s — proceeding anyway (smoke suite will validate)"
|
||||
|
||||
- name: Run canary smoke suite
|
||||
id: smoke
|
||||
# Graceful-skip when no canary fleet is configured (Phase 2 not yet
|
||||
# stood up — see molecule-controlplane/docs/canary-tenants.md).
|
||||
# Sets `ran=false` on skip so promote-to-latest stays off (we don't
|
||||
# want every main merge auto-promoting without gating). Manual
|
||||
# promote-latest.yml is the release gate while canary is absent.
|
||||
# Once the fleet is real: delete the early-exit branch.
|
||||
env:
|
||||
CANARY_TENANT_URLS: ${{ secrets.CANARY_TENANT_URLS }}
|
||||
CANARY_ADMIN_TOKENS: ${{ secrets.CANARY_ADMIN_TOKENS }}
|
||||
CANARY_CP_BASE_URL: https://staging-api.moleculesai.app
|
||||
CANARY_CP_SHARED_SECRET: ${{ secrets.CANARY_CP_SHARED_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${CANARY_TENANT_URLS:-}" ] \
|
||||
|| [ -z "${CANARY_ADMIN_TOKENS:-}" ] \
|
||||
|| [ -z "${CANARY_CP_SHARED_SECRET:-}" ]; then
|
||||
{
|
||||
echo "## ⚠️ canary-verify skipped"
|
||||
echo
|
||||
echo "One or more canary secrets are unset (\`CANARY_TENANT_URLS\`, \`CANARY_ADMIN_TOKENS\`, \`CANARY_CP_SHARED_SECRET\`)."
|
||||
echo "Phase 2 canary fleet has not been stood up yet —"
|
||||
echo "see [canary-tenants.md](https://git.moleculesai.app/molecule-ai/molecule-controlplane/blob/main/docs/canary-tenants.md)."
|
||||
echo
|
||||
echo "**Skipped — promote-to-latest will NOT auto-fire.** Dispatch \`promote-latest.yml\` manually when ready."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "ran=false" >> "$GITHUB_OUTPUT"
|
||||
echo "::notice::canary-verify: skipped — no canary fleet configured"
|
||||
exit 0
|
||||
fi
|
||||
bash scripts/canary-smoke.sh
|
||||
echo "ran=true" >> "$GITHUB_OUTPUT"
|
||||
run: bash scripts/canary-smoke.sh
|
||||
|
||||
- name: Summary on failure
|
||||
if: ${{ failure() }}
|
||||
@@ -158,98 +109,51 @@ jobs:
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
promote-to-latest:
|
||||
# On green, calls the CP redeploy-fleet endpoint with target_tag=
|
||||
# staging-<sha> to promote the verified ECR image. This is the same
|
||||
# mechanism as redeploy-tenants-on-main.yml — no GHCR crane ops.
|
||||
#
|
||||
# Pre-fix history: the old GHCR promote step used `crane tag` against
|
||||
# ghcr.io/molecule-ai/platform-tenant, but publish-workspace-server-
|
||||
# image.yml had already migrated to ECR on 2026-05-07 (commit
|
||||
# 10e510f5). The GHCR tags were never updated, so this step was
|
||||
# silently promoting a stale GHCR image while actual prod tenants
|
||||
# pulled from ECR. Canary smoke tests were GHCR-targeted and could
|
||||
# not catch a broken ECR build.
|
||||
# On green, retag :staging-<sha> → :latest for BOTH images.
|
||||
# crane is a lightweight registry client (no Docker daemon needed on
|
||||
# the runner) that can retag remotely with a single API call each.
|
||||
needs: canary-smoke
|
||||
if: ${{ needs.canary-smoke.result == 'success' && needs.canary-smoke.outputs.smoke_ran == 'true' }}
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
SHA: ${{ needs.canary-smoke.outputs.sha }}
|
||||
CP_URL: ${{ vars.CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
# CP_ADMIN_API_TOKEN gates write access to the redeploy endpoint.
|
||||
# Stored at the repo level so all workflows pick it up automatically.
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
# canary_slug pin: deploy the verified :staging-<sha> to the canary
|
||||
# first (soak 120s), then fan out to the rest of the fleet.
|
||||
CANARY_SLUG: ${{ vars.CANARY_PROMOTE_SLUG || '' }}
|
||||
SOAK_SECONDS: ${{ vars.CANARY_PROMOTE_SOAK || '120' }}
|
||||
BATCH_SIZE: ${{ vars.CANARY_PROMOTE_BATCH || '3' }}
|
||||
if: ${{ needs.canary-smoke.result == 'success' }}
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
steps:
|
||||
- name: Check CP credentials
|
||||
- name: Ensure crane installed
|
||||
# Matches the install pattern in promote-latest.yml — brew
|
||||
# cleanup exits non-zero on the shared runner's /opt/homebrew
|
||||
# symlinks, so skip it.
|
||||
env:
|
||||
HOMEBREW_NO_INSTALL_CLEANUP: "1"
|
||||
HOMEBREW_NO_AUTO_UPDATE: "1"
|
||||
HOMEBREW_NO_ENV_HINTS: "1"
|
||||
run: |
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret is not set — promote step cannot call redeploy-fleet."
|
||||
echo "::error::Set it at: repo Settings → Actions → Variables and Secrets → New Secret."
|
||||
exit 1
|
||||
if ! command -v crane >/dev/null 2>&1; then
|
||||
brew install crane
|
||||
fi
|
||||
crane version
|
||||
|
||||
- name: Promote verified ECR image to :latest
|
||||
- name: GHCR login
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | \
|
||||
crane auth login ghcr.io -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
TARGET_TAG="staging-${SHA}"
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--argjson soak "${SOAK_SECONDS:-120}" \
|
||||
--argjson batch "${BATCH_SIZE:-3}" \
|
||||
--argjson dry false \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
- name: Retag platform :staging-<sha> → :latest
|
||||
run: |
|
||||
crane tag \
|
||||
"${IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}" \
|
||||
latest
|
||||
|
||||
if [ -n "${CANARY_SLUG:-}" ]; then
|
||||
BODY=$(jq '. * {canary_slug: $slug}' --arg slug "$CANARY_SLUG" <<<"$BODY")
|
||||
fi
|
||||
|
||||
echo "Calling: POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " target_tag: $TARGET_TAG"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
CURL_EXIT=$?
|
||||
set -e
|
||||
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE (curl exit $CURL_EXIT)"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
if [ "$HTTP_CODE" -ge 400 ]; then
|
||||
echo "::error::CP redeploy-fleet returned HTTP $HTTP_CODE — refusing to proceed."
|
||||
exit 1
|
||||
fi
|
||||
- name: Retag tenant :staging-<sha> → :latest
|
||||
run: |
|
||||
crane tag \
|
||||
"${TENANT_IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}" \
|
||||
latest
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Canary verified — :latest promoted via CP redeploy-fleet"
|
||||
echo ""
|
||||
echo "- **Target tag:** \`staging-${{ needs.canary-smoke.outputs.sha }}\`"
|
||||
echo "- **Registry:** ECR (\`${TENANT_IMAGE_NAME}\`)"
|
||||
echo "- **Canary slug:** \`${CANARY_SLUG:-<none>}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "- **Batch size:** ${BATCH_SIZE:-3}"
|
||||
echo ""
|
||||
echo "CP redeploy-fleet is rolling out the verified image across the prod fleet."
|
||||
echo "The fleet's 5-minute health-check loop will pick up the update automatically."
|
||||
echo "## Canary verified — :latest promoted"
|
||||
echo
|
||||
echo "- \`${IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}\` → \`${IMAGE_NAME}:latest\`"
|
||||
echo "- \`${TENANT_IMAGE_NAME}:staging-${{ needs.canary-smoke.outputs.sha }}\` → \`${TENANT_IMAGE_NAME}:latest\`"
|
||||
echo
|
||||
echo "Prod tenant fleet will pick up the new digest on its next 5-min auto-update cycle."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
name: cascade-list-drift-gate
|
||||
|
||||
# Structural gate: TEMPLATES list in publish-runtime.yml must match
|
||||
# manifest.json's workspace_templates exactly. Closes the recurrence
|
||||
# path of PR #2556 (the data fix) and is the first concrete deliverable
|
||||
# of RFC #388 PR-3.
|
||||
#
|
||||
# Why a gate, not just discipline: PR #2536 pruned the manifest, but the
|
||||
# cascade list wasn't updated for ~weeks before someone (PR #2556)
|
||||
# noticed during an unrelated audit. During that window, codex never
|
||||
# rebuilt on a runtime publish. A structural gate catches the drift
|
||||
# the same day either file changes.
|
||||
#
|
||||
# Triggers narrowly to keep CI quiet: only on PRs that actually change
|
||||
# one of the two files. The path-filtered split + always-emit-result
|
||||
# pattern (memory: "Required check names need a job that always runs")
|
||||
# is unnecessary here because the workflow IS the check name and PR
|
||||
# branch protection should require it directly. Future-proof: if this
|
||||
# becomes a required check, add a no-op aggregator with always() so the
|
||||
# name still emits when paths don't match.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staging, main]
|
||||
paths:
|
||||
- manifest.json
|
||||
- .github/workflows/publish-runtime.yml
|
||||
- scripts/check-cascade-list-vs-manifest.sh
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- name: Check cascade list matches manifest
|
||||
run: bash scripts/check-cascade-list-vs-manifest.sh
|
||||
@@ -1,58 +0,0 @@
|
||||
name: Check migration collisions
|
||||
|
||||
# Hard gate (#2341): fails a PR that adds a migration prefix already
|
||||
# claimed by the base branch or another open PR. Caught manually 2026-04-30
|
||||
# during PR #2276 rebase: 044_runtime_image_pins collided with
|
||||
# 044_platform_inbound_secret from RFC #2312. This workflow makes that
|
||||
# check automatic.
|
||||
#
|
||||
# Trigger model: pull_request only — there's no value running this on
|
||||
# pushes to staging or main (those are post-merge; the gate must fire
|
||||
# pre-merge to be useful). Path filter scopes to PRs that actually touch
|
||||
# migrations.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'scripts/ops/check_migration_collisions.py'
|
||||
- '.github/workflows/check-migration-collisions.yml'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# gh pr list/diff need read access to other PRs
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Migration version collision check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Need history to diff against base ref
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Detect collisions
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
BASE_REF: origin/${{ github.event.pull_request.base.ref }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.sha }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
# gh CLI uses GH_TOKEN from env
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Ensure the named base ref exists locally. checkout@v4 with
|
||||
# fetch-depth=0 pulls full history, but the explicit fetch is
|
||||
# cheap insurance against form-of-ref differences across runs.
|
||||
#
|
||||
# IMPORTANT: do NOT pass --depth=1 here. The script below uses
|
||||
# `git diff origin/<base>...<head>` (three-dot, merge-base form),
|
||||
# which fails with "fatal: no merge base" if the base ref is
|
||||
# shallow. The auto-promote staging→main PR (#2361) was blocked
|
||||
# by exactly this for ~5h on 2026-04-30 — the depth=1 fetch
|
||||
# overwrote checkout@v4's full-history clone with a shallow tip.
|
||||
git fetch origin "${{ github.event.pull_request.base.ref }}" || true
|
||||
python3 scripts/ops/check_migration_collisions.py
|
||||
+72
-312
@@ -5,24 +5,20 @@ on:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
# GitHub merge queue fires `merge_group` for the queue's pre-merge CI run.
|
||||
# Required so the queue gets a real check result instead of a false-green
|
||||
# from the absence of a triggered workflow. Safe to add unconditionally —
|
||||
# the event simply doesn't fire until the queue is enabled on the branch.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
# Cancel in-progress CI runs when a new commit arrives on the same ref.
|
||||
# This prevents stale runs from queuing behind each other. The merge_group
|
||||
# refs (refs/heads/gh-readonly-queue/...) get their own concurrency group
|
||||
# automatically because github.ref differs from the PR ref.
|
||||
# This prevents multiple stale runs from queuing and keeps the self-hosted
|
||||
# macOS arm64 runner (publish-canvas-image, publish-workspace-server-image)
|
||||
# available for the jobs that genuinely require it.
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Detect which paths changed so downstream jobs can skip when only
|
||||
# docs/markdown files were modified.
|
||||
# docs/markdown files were modified. Uses plain `git diff` — no macOS
|
||||
# dependency, so this runs on ubuntu-latest to free the self-hosted
|
||||
# macOS arm64 runner for jobs that genuinely need it.
|
||||
changes:
|
||||
name: Detect changes
|
||||
runs-on: ubuntu-latest
|
||||
@@ -32,22 +28,17 @@ jobs:
|
||||
python: ${{ steps.check.outputs.python }}
|
||||
scripts: ${{ steps.check.outputs.scripts }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- id: check
|
||||
run: |
|
||||
# For PR events: diff against the base branch (not HEAD~1 of the branch,
|
||||
# which may be unrelated after force-pushes). When a push updates a PR,
|
||||
# both pull_request and push events fire — prefer the PR base so that
|
||||
# the diff is always computed against the actual merge base, not the
|
||||
# previous SHA on the branch which may be on a different history line.
|
||||
BASE="${GITHUB_BASE_REF:-${{ github.event.before }}}"
|
||||
# GITHUB_BASE_REF is set by GitHub for PR events (the base branch name).
|
||||
# For pull_request events we use the stored base.sha; for push events
|
||||
# (or when base.sha is unavailable) fall back to github.event.before.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
# For push events: diff against previous commit (handles merge commits)
|
||||
# For PR events: diff against the base branch
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
else
|
||||
BASE="${{ github.event.before }}"
|
||||
fi
|
||||
# Fallback: if BASE is empty or all zeros (new branch), run everything
|
||||
if [ -z "$BASE" ] || echo "$BASE" | grep -qE '^0+$'; then
|
||||
@@ -61,252 +52,104 @@ jobs:
|
||||
echo "platform=$(echo "$DIFF" | grep -qE '^workspace-server/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "canvas=$(echo "$DIFF" | grep -qE '^canvas/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "python=$(echo "$DIFF" | grep -qE '^workspace/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^infra/scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
echo "scripts=$(echo "$DIFF" | grep -qE '^tests/e2e/|^scripts/|^\.github/workflows/ci\.yml$' && echo true || echo false)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Platform (Go) is a required check on staging. Always-run + per-step
|
||||
# gating (see Canvas (Next.js) for the rationale and the failure mode
|
||||
# this avoids).
|
||||
platform-build:
|
||||
name: Platform (Go)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.platform == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.changes.outputs.platform != 'true'
|
||||
working-directory: .
|
||||
run: echo "No platform/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go mod download
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: github.com/molecule-ai/molecule-cli
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
run: go vet ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run golangci-lint
|
||||
run: golangci-lint run --timeout 3m ./... || true
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Run tests with race detection and coverage
|
||||
- run: go mod download
|
||||
- run: go build ./cmd/server
|
||||
# CLI (molecli) moved to standalone repo: github.com/Molecule-AI/molecule-cli
|
||||
- run: go vet ./...
|
||||
# golangci-lint-action uses a Linux Docker image (ubuntu is the only arch+OS
|
||||
# combo the official image publishes for). Previously this step was pinned to
|
||||
# [self-hosted, macos, arm64] because the Docker image can't run on macOS ARM.
|
||||
# Now that the job itself runs on ubuntu-latest, the Docker image works natively.
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: latest
|
||||
working-directory: workspace-server
|
||||
args: --timeout 3m
|
||||
continue-on-error: true # Warn but don't block until codebase is clean
|
||||
- name: Run tests with race detection and coverage
|
||||
run: go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Per-file coverage report
|
||||
# Advisory — lists every source file with its coverage so reviewers
|
||||
# can see at-a-glance where gaps are. Sorted ascending so the worst
|
||||
# offenders float to the top. Does NOT fail the build; the hard
|
||||
# gate is the threshold check below. (#1823)
|
||||
- name: Check coverage baseline
|
||||
run: |
|
||||
echo "=== Per-file coverage (worst first) ==="
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%6.1f%% %s\n", s[f]/c[f], f}' \
|
||||
| sort -n
|
||||
|
||||
- if: needs.changes.outputs.platform == 'true'
|
||||
name: Check coverage thresholds
|
||||
# Enforces two gates from #1823 Layer 1:
|
||||
# 1. Total floor (25% — ratchet plan in COVERAGE_FLOOR.md).
|
||||
# 2. Per-file floor — non-test .go files in security-critical
|
||||
# paths with coverage <10% fail the build, UNLESS the file
|
||||
# path is listed in .coverage-allowlist.txt (acknowledged
|
||||
# historical debt with a tracking issue + expiry).
|
||||
run: |
|
||||
set -e
|
||||
TOTAL_FLOOR=25
|
||||
# Security-critical paths where a 0%-coverage file is a real risk.
|
||||
CRITICAL_PATHS=(
|
||||
"internal/handlers/tokens"
|
||||
"internal/handlers/workspace_provision"
|
||||
"internal/handlers/a2a_proxy"
|
||||
"internal/handlers/registry"
|
||||
"internal/handlers/secrets"
|
||||
"internal/middleware/wsauth"
|
||||
"internal/crypto"
|
||||
)
|
||||
|
||||
TOTAL=$(go tool cover -func=coverage.out | grep '^total:' | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${TOTAL}%"
|
||||
if awk "BEGIN{exit !($TOTAL < $TOTAL_FLOOR)}"; then
|
||||
echo "::error::Total coverage ${TOTAL}% is below the ${TOTAL_FLOOR}% floor. See COVERAGE_FLOOR.md for ratchet plan."
|
||||
COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//')
|
||||
echo "Total coverage: ${COVERAGE}%"
|
||||
THRESHOLD=25
|
||||
awk "BEGIN{if ($COVERAGE < $THRESHOLD) exit 1}" || {
|
||||
echo "::error::Coverage ${COVERAGE}% is below the ${THRESHOLD}% threshold"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Aggregate per-file coverage → /tmp/perfile.txt: "<fullpath> <pct>"
|
||||
go tool cover -func=coverage.out \
|
||||
| grep -v '^total:' \
|
||||
| awk '{file=$1; sub(/:[0-9][0-9.]*:.*/, "", file); pct=$NF; gsub(/%/,"",pct); s[file]+=pct; c[file]++}
|
||||
END {for (f in s) printf "%s %.1f\n", f, s[f]/c[f]}' \
|
||||
> /tmp/perfile.txt
|
||||
|
||||
# Build allowlist — paths relative to workspace-server, one per line.
|
||||
# Lines starting with # are comments.
|
||||
ALLOWLIST=""
|
||||
if [ -f ../.coverage-allowlist.txt ]; then
|
||||
ALLOWLIST=$(grep -vE '^(#|[[:space:]]*$)' ../.coverage-allowlist.txt || true)
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
WARNED=0
|
||||
for path in "${CRITICAL_PATHS[@]}"; do
|
||||
while read -r file pct; do
|
||||
[[ "$file" == *_test.go ]] && continue
|
||||
[[ "$file" == *"$path"* ]] || continue
|
||||
awk "BEGIN{exit !($pct < 10)}" || continue
|
||||
|
||||
# Strip the package-import prefix so we can match .coverage-allowlist.txt
|
||||
# entries written as paths relative to workspace-server/.
|
||||
# Handle both module paths: platform/workspace-server/... and platform/...
|
||||
rel=$(echo "$file" | sed 's|^github.com/molecule-ai/molecule-monorepo/platform/workspace-server/||; s|^github.com/molecule-ai/molecule-monorepo/platform/||')
|
||||
|
||||
if echo "$ALLOWLIST" | grep -qxF "$rel"; then
|
||||
echo "::warning file=workspace-server/$rel::Critical file at ${pct}% coverage (allowlisted, #1823) — fix before expiry."
|
||||
WARNED=$((WARNED+1))
|
||||
else
|
||||
echo "::error file=workspace-server/$rel::Critical file at ${pct}% coverage — must be >=10% (target 80%). See #1823. To acknowledge as known debt, add this path to .coverage-allowlist.txt."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done < /tmp/perfile.txt
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Critical-path check: $FAILED new failures, $WARNED allowlisted warnings."
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED security-critical file(s) have <10% test coverage and are"
|
||||
echo "NOT in the allowlist. These paths handle auth, tokens, secrets, or"
|
||||
echo "workspace provisioning — a 0% file here is the exact gap that let"
|
||||
echo "CWE-22, CWE-78, KI-005 slip through in past incidents. Either:"
|
||||
echo " (a) add tests to raise coverage above 10%, or"
|
||||
echo " (b) add the path to .coverage-allowlist.txt with an expiry date"
|
||||
echo " and a tracking issue reference."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Canvas (Next.js) — required check, always runs. See platform-build
|
||||
# comment above for the rationale.
|
||||
#
|
||||
# Supersedes the canvas-build-noop pattern attempted in PR #2321: two
|
||||
# jobs sharing `name:` doesn't actually satisfy branch protection
|
||||
# because the SKIPPED check run sibling is treated as not-passed
|
||||
# regardless of how many SUCCESS siblings it has. Verified empirically
|
||||
# on PR #2314 — mergeStateStatus stayed BLOCKED until I collapsed to
|
||||
# a single-job-with-conditional-steps shape.
|
||||
canvas-build:
|
||||
name: Canvas (Next.js)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.canvas == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
steps:
|
||||
- if: needs.changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: echo "No canvas/** changes — skipping real build steps; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: rm -f package-lock.json && npm install
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
run: npm run build
|
||||
- if: needs.changes.outputs.canvas == 'true'
|
||||
name: Run tests with coverage
|
||||
# Coverage instrumentation is configured in canvas/vitest.config.ts
|
||||
# (provider: v8, reporters: text + html + json-summary). Step 2 of
|
||||
# #1815 — wires coverage into CI so we get a baseline visible on
|
||||
# every PR. No threshold gate yet; thresholds dial in (Step 3, also
|
||||
# tracked in #1815) after the team sees what current coverage is.
|
||||
# Per the inline comment in vitest.config.ts: "first land
|
||||
# observability so we can see the baseline, then dial in
|
||||
# thresholds + a hard gate" — this PR ships the observability half.
|
||||
run: npx vitest run --coverage
|
||||
- name: Upload coverage summary as artifact
|
||||
if: needs.changes.outputs.canvas == 'true' && always()
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement, surfacing as `GHESNotSupportedError: @actions/artifact
|
||||
# v2.0.0+, upload-artifact@v4+ and download-artifact@v4+ are not
|
||||
# currently supported on GHES`. Drop this pin when Gitea ships
|
||||
# the v4 protocol (tracked: post-Gitea-1.23 followup).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: canvas-coverage-${{ github.run_id }}
|
||||
path: canvas/coverage/
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
- run: rm -f package-lock.json && npm install
|
||||
- run: npm run build
|
||||
- name: Run tests
|
||||
run: npx vitest run
|
||||
|
||||
# MCP Server + SDK removed from CI — now in standalone repos:
|
||||
# - github.com/molecule-ai/molecule-mcp-server (npm CI)
|
||||
# - github.com/molecule-ai/molecule-sdk-python (PyPI CI)
|
||||
# - github.com/Molecule-AI/molecule-mcp-server (npm CI)
|
||||
# - github.com/Molecule-AI/molecule-sdk-python (PyPI CI)
|
||||
|
||||
# e2e-api job moved to .github/workflows/e2e-api.yml (issue #458).
|
||||
# It now has workflow-level concurrency (cancel-in-progress: false) so
|
||||
# new pushes queue the E2E run rather than cancelling it at the run level.
|
||||
|
||||
# Shellcheck (E2E scripts) — required check, always runs. See
|
||||
# platform-build for the rationale.
|
||||
shellcheck:
|
||||
name: Shellcheck (E2E scripts)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.scripts == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- if: needs.changes.outputs.scripts != 'true'
|
||||
run: echo "No tests/e2e/ or infra/scripts/ changes — skipping real shellcheck; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run shellcheck on tests/e2e/*.sh and infra/scripts/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest runners (via apt).
|
||||
# infra/scripts/ is included because setup.sh + nuke.sh gate the
|
||||
# README quickstart — a shellcheck regression there silently breaks
|
||||
# new-user onboarding. scripts/ is intentionally excluded until its
|
||||
# pre-existing SC3040/SC3043 warnings are cleaned up.
|
||||
- uses: actions/checkout@v4
|
||||
- name: Run shellcheck on tests/e2e/*.sh
|
||||
# shellcheck is pre-installed on ubuntu-latest GitHub-hosted runners.
|
||||
run: |
|
||||
find tests/e2e infra/scripts -type f -name '*.sh' -print0 \
|
||||
if ! command -v shellcheck >/dev/null 2>&1; then
|
||||
echo "::error::shellcheck is not installed on the runner"
|
||||
exit 1
|
||||
fi
|
||||
find tests/e2e -type f -name '*.sh' -print0 \
|
||||
| xargs -0 shellcheck --severity=warning
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Lint cleanup-trap hygiene (RFC #2873)
|
||||
# Asserts every shell E2E test that calls `mktemp` also installs
|
||||
# an EXIT trap. Catches the /tmp-leak class — a missing trap
|
||||
# silently leaks scratch into CI runners (~10-100KB per run).
|
||||
# See tests/e2e/lint_cleanup_traps.sh for the rule + fix pattern.
|
||||
run: bash tests/e2e/lint_cleanup_traps.sh
|
||||
|
||||
- if: needs.changes.outputs.scripts == 'true'
|
||||
name: Run E2E bash unit tests (no live infra)
|
||||
# Pure-bash unit tests for E2E helper libs (lib/*.sh). These pin
|
||||
# behavior of dispatch logic that — when broken — silently masks as
|
||||
# "Could not resolve authentication method" only after a successful
|
||||
# tenant + workspace provision (PR #2571 incident, 2026-05-03). Add
|
||||
# new self-contained unit tests here as the lib/ directory grows;
|
||||
# tests requiring live CP/tenant credentials belong in the dedicated
|
||||
# e2e-staging-* workflows, not this job.
|
||||
run: |
|
||||
bash tests/e2e/test_model_slug.sh
|
||||
|
||||
canvas-deploy-reminder:
|
||||
name: Canvas Deploy Reminder
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, canvas-build]
|
||||
# Only fires on direct pushes to main (i.e. after staging→main promotion).
|
||||
if: needs.changes.outputs.canvas == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
permissions:
|
||||
# Required to post commit comments via the GitHub API.
|
||||
contents: write
|
||||
steps:
|
||||
- name: Write deploy reminder to step summary
|
||||
- name: Post deploy reminder as commit comment
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
COMMIT_SHA: ${{ github.sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
@@ -333,111 +176,28 @@ jobs:
|
||||
printf '\n> Posted automatically by CI · commit `%s` · [build log](%s)\n' \
|
||||
"$COMMIT_SHA" "$RUN_URL" >> /tmp/deploy-reminder.md
|
||||
|
||||
# Gitea has no commit-comments API (no equivalent of
|
||||
# POST /repos/{owner}/{repo}/commits/{commit_sha}/comments).
|
||||
# Write to GITHUB_STEP_SUMMARY instead — both GitHub Actions and
|
||||
# Gitea Actions render this as the workflow run's summary page,
|
||||
# which is where operators look for post-deploy action items.
|
||||
# (#75 / PR-D)
|
||||
cat /tmp/deploy-reminder.md >> "$GITHUB_STEP_SUMMARY"
|
||||
gh api \
|
||||
--method POST \
|
||||
"repos/${{ github.repository }}/commits/${{ github.sha }}/comments" \
|
||||
--field "body=@/tmp/deploy-reminder.md"
|
||||
|
||||
# Python Lint & Test — required check, always runs. See platform-build
|
||||
# for the rationale.
|
||||
python-lint:
|
||||
name: Python Lint & Test
|
||||
needs: changes
|
||||
if: needs.changes.outputs.python == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
WORKSPACE_ID: test
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace
|
||||
steps:
|
||||
- if: needs.changes.outputs.python != 'true'
|
||||
working-directory: .
|
||||
run: echo "No workspace/** changes — skipping real lint+test; this job always runs to satisfy the required-check name on branch protection."
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov sqlalchemy>=2.0.0
|
||||
# Coverage flags + fail-under floor moved into workspace/pytest.ini
|
||||
# (issue #1817) so local `pytest` and CI use identical config.
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
run: python -m pytest --tb=short
|
||||
|
||||
- if: needs.changes.outputs.python == 'true'
|
||||
name: Per-file critical-path coverage (MCP / inbox / auth)
|
||||
# MCP-critical Python files have a per-file floor on top of the
|
||||
# 86% total floor in pytest.ini. Rationale (issue #2790, after
|
||||
# the PR #2766 → PR #2771 cycle): the total floor averages ~6000
|
||||
# lines, so a single MCP file could regress to ~50% with no
|
||||
# complaint as long as other modules compensate. These five
|
||||
# files handle multi-tenant routing + auth + inbox dispatch —
|
||||
# a coverage drop here is the same risk shape as a Go-side
|
||||
# workspace-server token/secrets file dropping below 10%.
|
||||
#
|
||||
# Floor 75% sits below current actuals (80-96%) so this gate is
|
||||
# strictly additive — no existing PR fails. Ratchet plan in
|
||||
# COVERAGE_FLOOR.md.
|
||||
run: |
|
||||
set -e
|
||||
PER_FILE_FLOOR=75
|
||||
CRITICAL_FILES=(
|
||||
"a2a_mcp_server.py"
|
||||
"mcp_cli.py"
|
||||
"a2a_tools.py"
|
||||
"a2a_tools_inbox.py"
|
||||
"inbox.py"
|
||||
"platform_auth.py"
|
||||
)
|
||||
|
||||
# pytest already wrote .coverage; emit a JSON view scoped to
|
||||
# the critical files so jq/python can read the per-file pct
|
||||
# without parsing tabular text. --include uses fnmatch, and
|
||||
# the leading "*" allows the file to live anywhere under the
|
||||
# workspace root (today they sit at workspace/<name>.py).
|
||||
INCLUDES=$(printf '*%s,' "${CRITICAL_FILES[@]}")
|
||||
INCLUDES="${INCLUDES%,}"
|
||||
python -m coverage json -o /tmp/critical-cov.json --include="$INCLUDES"
|
||||
|
||||
FAILED=0
|
||||
for f in "${CRITICAL_FILES[@]}"; do
|
||||
# Match by top-level path key (e.g. "a2a_tools.py", not
|
||||
# "builtin_tools/a2a_tools.py" — different file at 100%).
|
||||
# The keys in coverage.json are paths relative to the run
|
||||
# cwd (workspace/), so the critical-path entry sits at the
|
||||
# bare basename.
|
||||
pct=$(jq -r --arg f "$f" '.files | to_entries | map(select(.key == $f)) | .[0].value.summary.percent_covered // "MISSING"' /tmp/critical-cov.json)
|
||||
if [ "$pct" = "MISSING" ]; then
|
||||
echo "::error file=workspace/$f::No coverage data — file may have moved or test exclusion mis-set."
|
||||
FAILED=$((FAILED+1))
|
||||
continue
|
||||
fi
|
||||
echo "$f: ${pct}%"
|
||||
if awk "BEGIN{exit !($pct < $PER_FILE_FLOOR)}"; then
|
||||
echo "::error file=workspace/$f::${pct}% < ${PER_FILE_FLOOR}% per-file floor (MCP critical path). See COVERAGE_FLOOR.md."
|
||||
FAILED=$((FAILED+1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$FAILED" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$FAILED MCP critical-path file(s) below the ${PER_FILE_FLOOR}% per-file floor."
|
||||
echo "These paths handle multi-tenant routing, auth tokens, and inbox dispatch."
|
||||
echo "A coverage drop here is the same risk shape as Go-side tokens/secrets files"
|
||||
echo "dropping below 10% (see COVERAGE_FLOOR.md). Either:"
|
||||
echo " (a) add tests to raise coverage back above ${PER_FILE_FLOOR}%, or"
|
||||
echo " (b) if this is unavoidable historical debt, file an issue and propose"
|
||||
echo " adjusting the floor with rationale in COVERAGE_FLOOR.md."
|
||||
exit 1
|
||||
fi
|
||||
- run: pip install -r requirements.txt pytest pytest-asyncio pytest-cov
|
||||
- run: WORKSPACE_ID=ci-placeholder python -m pytest --tb=short -q --cov=. --cov-report=term-missing
|
||||
|
||||
# SDK + plugin validation moved to standalone repo:
|
||||
# github.com/molecule-ai/molecule-sdk-python
|
||||
|
||||
# github.com/Molecule-AI/molecule-sdk-python
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
name: CodeQL
|
||||
|
||||
# Controls CodeQL scan triggers for this repo.
|
||||
#
|
||||
# GitHub's "Code quality" default setup (the UI-configured one) is
|
||||
# hardcoded to only scan the default branch — on this repo that's
|
||||
# `staging`, so PRs promoting staging→main would otherwise never be
|
||||
# scanned. This workflow fills that gap by explicitly scanning both
|
||||
# branches on push and PR.
|
||||
#
|
||||
# Runs on the self-hosted mac mini (matches the org-wide Code Quality
|
||||
# runner-label config). GHAS is NOT enabled on this repo, so results
|
||||
# are not uploaded to the Security tab — the scan fails the PR check
|
||||
# on findings, and the SARIF is kept as a workflow artifact for
|
||||
# triage.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
schedule:
|
||||
# Weekly run picks up findings in code that hasn't been touched.
|
||||
- cron: '30 1 * * 0'
|
||||
|
||||
# Workflow-level concurrency: only one CodeQL run per branch/PR at a time.
|
||||
# `cancel-in-progress: false` queues new runs — the 45-min analysis is the
|
||||
# longest CI occupant and fights the single mac mini runner the hardest.
|
||||
concurrency:
|
||||
group: codeql-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
# No security-events: write — we don't call the upload API.
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze (${{ matrix.language }})
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
timeout-minutes: 45
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: [go, javascript-typescript, python]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Checkout sibling plugin repo
|
||||
# Same reasoning as publish-workspace-server-image.yml — the Go
|
||||
# module's replace directive needs the plugin source so
|
||||
# CodeQL's "go build" phase can resolve.
|
||||
if: matrix.language == 'go'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: Molecule-AI/molecule-ai-plugin-github-app-auth
|
||||
path: molecule-ai-plugin-github-app-auth
|
||||
token: ${{ secrets.PLUGIN_REPO_PAT || secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Ensure jq installed
|
||||
# Follows the crane-install pattern in promote-latest.yml.
|
||||
# HOMEBREW_NO_* flags skip the cleanup that fails on the shared
|
||||
# runner's /opt/homebrew symlinks.
|
||||
env:
|
||||
HOMEBREW_NO_INSTALL_CLEANUP: "1"
|
||||
HOMEBREW_NO_AUTO_UPDATE: "1"
|
||||
HOMEBREW_NO_ENV_HINTS: "1"
|
||||
run: command -v jq >/dev/null || brew install jq
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# security-extended widens past the default to include the
|
||||
# full security-query set for a public SaaS surface.
|
||||
queries: security-extended
|
||||
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v3
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
id: analyze
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{ matrix.language }}"
|
||||
# upload: never — GHAS isn't enabled on this repo, so the
|
||||
# upload API 403s. Write SARIF locally instead.
|
||||
upload: never
|
||||
output: sarif-results/${{ matrix.language }}
|
||||
|
||||
- name: Parse SARIF + fail on findings
|
||||
# The analyze step writes <database>.sarif into the output
|
||||
# directory — database name is the short CodeQL lang id, not
|
||||
# the matrix value (e.g. "javascript-typescript" →
|
||||
# javascript.sarif), so glob rather than hardcode.
|
||||
# Filter to error/warning severity: security-extended emits
|
||||
# "note" rows for informational findings we don't want to fail
|
||||
# the build over.
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
dir="sarif-results/${{ matrix.language }}"
|
||||
sarif=$(ls "$dir"/*.sarif 2>/dev/null | head -1 || true)
|
||||
if [ -z "$sarif" ] || [ ! -f "$sarif" ]; then
|
||||
echo "::error::No SARIF file found under $dir"
|
||||
ls -la "$dir" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Parsing $sarif"
|
||||
count=$(jq '[.runs[].results[] | select(.level == "error" or .level == "warning")] | length' "$sarif")
|
||||
echo "CodeQL findings (error+warning) for ${{ matrix.language }}: $count"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo "::error::CodeQL found $count issues. Details below; full SARIF in the artifact."
|
||||
jq -r '.runs[].results[] | select(.level == "error" or .level == "warning") | " - [\(.level)] \(.ruleId // "?"): \(.message.text // "(no message)") @ \(.locations[0].physicalLocation.artifactLocation.uri // "?"):\(.locations[0].physicalLocation.region.startLine // "?")"' "$sarif"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload SARIF artifact
|
||||
# Keep SARIF around on success + failure so triagers can diff.
|
||||
# 14-day retention — longer than default 3, short enough not
|
||||
# to bloat quota.
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: codeql-sarif-${{ matrix.language }}
|
||||
path: sarif-results/${{ matrix.language }}/
|
||||
retention-days: 14
|
||||
@@ -1,257 +0,0 @@
|
||||
name: Continuous synthetic E2E (staging)
|
||||
|
||||
# Hard gate (#2342): cron-driven full-lifecycle E2E that catches
|
||||
# regressions visible only at runtime — schema drift, deployment-pipeline
|
||||
# gaps, vendor outages, env-var rotations, DNS / CF / Railway side-effects.
|
||||
#
|
||||
# Why this gate exists:
|
||||
# PR-time CI catches code-level regressions but not deployment-time or
|
||||
# integration-time ones. Today's empirical data:
|
||||
# • #2345 (A2A v0.2 silent drop) — passed all unit tests, broke at
|
||||
# JSON-RPC parse layer between sender and receiver. Visible only
|
||||
# to a sender exercising the full path.
|
||||
# • RFC #2312 chat upload — landed on staging-branch but never
|
||||
# reached staging tenants because publish-workspace-server-image
|
||||
# was main-only. Caught by manual dogfooding hours after deploy.
|
||||
# Both would have surfaced within 15-20 min of regression if a
|
||||
# continuous synth-E2E was running.
|
||||
#
|
||||
# Cadence: every 20 min (3x/hour). The script is conservatively
|
||||
# bounded at 10 min wall-clock; even on degraded staging it should
|
||||
# finish before the next firing. cron-overlap is guarded by the
|
||||
# concurrency group below.
|
||||
#
|
||||
# Cost: ~3 runs/hour × 5-10 min × $0.008/min GHA = ~$0.50-$1/day.
|
||||
# Plus a fresh tenant provisioned + torn down each run (Railway +
|
||||
# AWS pennies). Negligible.
|
||||
#
|
||||
# Failure handling: when the run fails, the workflow exits non-zero
|
||||
# and GitHub's standard email/notification path fires. Operators
|
||||
# can subscribe to this workflow's failure channel for paging-grade
|
||||
# alerting.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 10 minutes, on :02 :12 :22 :32 :42 :52. Three constraints:
|
||||
# 1. Stay off the top-of-hour. GitHub Actions scheduler drops
|
||||
# :00 firings under high load (own docs:
|
||||
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule).
|
||||
# Prior history: cron was '0,20,40' (2026-05-02) — only :00
|
||||
# ever survived. Bumped to '10,30,50' (2026-05-03) on the
|
||||
# theory that further-from-:00 wins. Empirically 2026-05-04
|
||||
# that ALSO dropped to ~60 min effective cadence (only ~1
|
||||
# schedule fire per hour — see molecule-core#2726). Detection
|
||||
# latency was claimed 20 min, actual 60 min.
|
||||
# 2. Avoid colliding with the existing :15 sweep-cf-orphans
|
||||
# and :45 sweep-cf-tunnels — both hit the CF API and we
|
||||
# don't want to fight for rate-limit tokens.
|
||||
# 3. Avoid the :30 heavy slot (canary-staging /30, sweep-aws-
|
||||
# secrets, sweep-stale-e2e-orgs every :15) — multiple
|
||||
# overlapping cron registrations on the same minute is part
|
||||
# of what GH drops under load.
|
||||
# Solution: bump fires-per-hour 3 → 6 AND keep all slots in clean
|
||||
# lanes (1-3 min away from any other cron). Even with empirically-
|
||||
# observed ~67% GH drop ratio, 6 attempts/hour yields ~2 effective
|
||||
# fires = ~30 min cadence; closer to the 20-min target than the
|
||||
# current shape and provides a real degradation alarm if drops
|
||||
# get worse.
|
||||
- cron: '2,12,22,32,42,52 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runtime:
|
||||
description: "Runtime to provision (claude-code = default + cheapest via MiniMax; langgraph = OpenAI-only; hermes = SDK-native path, slower)"
|
||||
required: false
|
||||
default: "claude-code"
|
||||
type: string
|
||||
model_slug:
|
||||
description: "Model id to provision the workspace with (default MiniMax-M2.7-highspeed; e.g. 'sonnet' to test direct Anthropic, 'openai/gpt-4o' for hermes)"
|
||||
required: false
|
||||
default: "MiniMax-M2.7-highspeed"
|
||||
type: string
|
||||
keep_org:
|
||||
description: "Skip teardown for post-mortem debugging (only manual dispatch — never set this for cron runs)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# No issue-write here — failures surface as red runs in the workflow
|
||||
# history. If you want auto-issue-on-fail, add a follow-up step that
|
||||
# uses gh issue create gated on `if: failure()`. Keeping the surface
|
||||
# minimal until that's actually wanted.
|
||||
|
||||
# Serialize so two firings can never overlap. Cron firing every 20 min
|
||||
# but scripts conservatively bounded at 10 min — overlap shouldn't
|
||||
# happen in steady state, but if a run hangs we don't want N more
|
||||
# stacking up.
|
||||
concurrency:
|
||||
group: continuous-synth-e2e
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
synth:
|
||||
name: Synthetic E2E against staging
|
||||
runs-on: ubuntu-latest
|
||||
# Bumped from 12 → 20 (2026-05-04). Tenant user-data install phase
|
||||
# (apt-get update + install docker.io/jq/awscli/caddy + snap install
|
||||
# ssm-agent) runs from raw Ubuntu on every boot — none of it is
|
||||
# pre-baked into the tenant AMI. Empirical fetch_secrets/ok timing
|
||||
# across today's canaries: 51s → 82s → 143s → 625s. apt-mirror tail
|
||||
# latency drives the boot-to-fetch_secrets phase from ~1min to >10min.
|
||||
# A 12min budget leaves only ~2min for the workspace (which needs
|
||||
# ~3.5min for claude-code cold boot) on slow-apt days, blowing the
|
||||
# budget. 20min absorbs the worst tenant tail so the workspace probe
|
||||
# gets the full ~7min it needs even on a slow apt day. Real fix:
|
||||
# pre-bake caddy + ssm-agent into the tenant AMI (controlplane#TBD).
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
# claude-code default: cold-start ~5 min (comparable to langgraph),
|
||||
# but uses MiniMax-M2.7-highspeed via the template's third-party-
|
||||
# Anthropic-compat path (workspace-configs-templates/claude-code-
|
||||
# default/config.yaml:64-69). MiniMax is ~5-10x cheaper than
|
||||
# gpt-4.1-mini per token AND avoids the recurring OpenAI quota-
|
||||
# exhaustion class that took the canary down 2026-05-03 (#265).
|
||||
# Operators can pick langgraph / hermes via workflow_dispatch
|
||||
# when they specifically need to exercise the OpenAI or SDK-
|
||||
# native paths.
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the canary to a specific MiniMax model rather than relying
|
||||
# on the per-runtime default ("sonnet" → routes to direct
|
||||
# Anthropic, defeats the cost saving). Operators can override
|
||||
# via workflow_dispatch by setting a different E2E_MODEL_SLUG
|
||||
# input if they need to exercise a specific model. M2.7-highspeed
|
||||
# is "Token Plan only" but cheap-per-token and fast.
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.model_slug || 'MiniMax-M2.7-highspeed' }}
|
||||
# Bound to 10 min so a stuck provision fails the run instead of
|
||||
# holding up the next cron firing. 15-min default in the script
|
||||
# is for the on-PR full lifecycle where we have more headroom.
|
||||
E2E_PROVISION_TIMEOUT_SECS: '600'
|
||||
# Slug suffix — namespaced "synth-" so these runs are
|
||||
# distinguishable from PR-driven runs in CP admin.
|
||||
E2E_RUN_ID: synth-${{ github.run_id }}
|
||||
# Forced false for cron; respected for manual dispatch
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org == 'true' && '1' || '' }}
|
||||
MOLECULE_CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
# MiniMax key is the canary's PRIMARY auth path. claude-code
|
||||
# template's `minimax` provider routes ANTHROPIC_BASE_URL to
|
||||
# api.minimax.io/anthropic and reads MINIMAX_API_KEY at boot.
|
||||
# tests/e2e/test_staging_full_saas.sh branches SECRETS_JSON on
|
||||
# which key is present — MiniMax wins when set.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so operators can dispatch with
|
||||
# E2E_RUNTIME=langgraph or =hermes and still have a working
|
||||
# canary path. The script picks the right blob shape based on
|
||||
# which key is non-empty.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
run: |
|
||||
# Hard-fail on missing secret REGARDLESS of trigger. Previously
|
||||
# this step soft-skipped on workflow_dispatch via `exit 0`, but
|
||||
# `exit 0` only ends the STEP — subsequent steps still ran with
|
||||
# the empty secret, the synth script fell through to the wrong
|
||||
# SECRETS_JSON branch, and the canary failed 5 min later with a
|
||||
# confusing "Agent error (Exception)" instead of the clean
|
||||
# "secret missing" message at the top. Caught 2026-05-04 by
|
||||
# dispatched run 25296530706: claude-code + missing MINIMAX
|
||||
# silently used OpenAI keys but kept model=MiniMax-M2.7, then
|
||||
# the workspace 401'd against MiniMax once it tried to call.
|
||||
# Fix: exit 1 in both cron and dispatch paths. Operators who
|
||||
# want to verify a YAML change without setting up the secret
|
||||
# can read the verify-secrets step's stderr — the failure is
|
||||
# itself the verification signal.
|
||||
if [ -z "${MOLECULE_ADMIN_TOKEN:-}" ]; then
|
||||
echo "::error::CP_STAGING_ADMIN_API_TOKEN secret missing — synth E2E cannot run"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# LLM-key requirement is per-runtime: claude-code accepts
|
||||
# EITHER MiniMax OR direct-Anthropic (whichever is set first),
|
||||
# langgraph + hermes use OpenAI (MOLECULE_STAGING_OPENAI_KEY).
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret missing — runtime=${E2E_RUNTIME} cannot authenticate against its LLM provider"
|
||||
echo "::error::Set it at Settings → Secrets and Variables → Actions, OR dispatch with a different runtime"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install required tools
|
||||
run: |
|
||||
# The script depends on jq + curl (already on ubuntu-latest)
|
||||
# and python3 (likewise). Verify they're all present so we
|
||||
# fail fast on a runner image regression rather than mid-script.
|
||||
for cmd in jq curl python3; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || {
|
||||
echo "::error::required tool '$cmd' not on PATH — runner image regression?"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
- name: Run synthetic E2E
|
||||
# The script handles its own teardown via EXIT trap; even on
|
||||
# failure (timeout, assertion), the org is deprovisioned and
|
||||
# leaks are reported. Exit code propagates from the script.
|
||||
run: |
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
- name: Failure summary
|
||||
# Runs only on failure. Adds a job summary so the workflow run
|
||||
# page shows a quick "what happened" instead of forcing readers
|
||||
# to scroll through script output.
|
||||
if: failure()
|
||||
run: |
|
||||
{
|
||||
echo "## Continuous synth E2E failed"
|
||||
echo ""
|
||||
echo "**Run ID:** ${{ github.run_id }}"
|
||||
echo "**Trigger:** ${{ github.event_name }}"
|
||||
echo "**Runtime:** ${E2E_RUNTIME}"
|
||||
echo "**Slug:** synth-${{ github.run_id }}"
|
||||
echo ""
|
||||
echo "### What this means"
|
||||
echo ""
|
||||
echo "Staging just regressed on a path that previously worked. Likely classes:"
|
||||
echo "- Schema mismatch between sender and receiver (#2345 class)"
|
||||
echo "- Deployment-pipeline gap (RFC #2312 / staging-tenant-image-stale class)"
|
||||
echo "- Vendor outage (Cloudflare, Railway, AWS, GHCR)"
|
||||
echo "- Staging-CP env var rotation"
|
||||
echo ""
|
||||
echo "### Next steps"
|
||||
echo ""
|
||||
echo "1. Check the script output above for the assertion that failed"
|
||||
echo "2. If it's a vendor outage, no action needed — next firing in ~20 min"
|
||||
echo "3. If it's a code regression, find the causing PR via \`git log\` against last green run and revert/fix"
|
||||
echo "4. Keep an eye on the next 1-2 firings — flake vs persistent fail differs in priority"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
+47
-218
@@ -2,204 +2,68 @@ name: E2E API Smoke Test
|
||||
# Extracted from ci.yml so workflow-level concurrency can protect this job
|
||||
# from run-level cancellation (issue #458).
|
||||
#
|
||||
# Trigger model (revised 2026-04-29):
|
||||
# Problem: the job-level `concurrency.cancel-in-progress: false` in ci.yml
|
||||
# prevented *sibling* E2E jobs from killing each other, but GitHub still
|
||||
# cancelled the parent *workflow run* when a new push arrived. Since the job
|
||||
# lived inside that run, it got cancelled too.
|
||||
#
|
||||
# Always FIRES on push/pull_request to staging+main. Real work is gated
|
||||
# per-step on `needs.detect-changes.outputs.api` — when paths under
|
||||
# `workspace-server/`, `tests/e2e/`, or this workflow file haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `E2E API Smoke Test` check, satisfying branch protection without
|
||||
# spending CI cycles. See the in-job comment on the `e2e-api` job for
|
||||
# why this is one job (not two-jobs-sharing-name) and the 2026-04-29
|
||||
# PR #2264 incident that drove the consolidation.
|
||||
#
|
||||
# Parallel-safety (Class B Hongming-owned CICD red sweep, 2026-05-08)
|
||||
# -------------------------------------------------------------------
|
||||
# Same substrate hazard as PR #98 (handlers-postgres-integration). Our
|
||||
# Gitea act_runner runs with `container.network: host` (operator host
|
||||
# `/opt/molecule/runners/config.yaml`), which means:
|
||||
#
|
||||
# * Two concurrent runs both try to bind their `-p 15432:5432` /
|
||||
# `-p 16379:6379` host ports — the second postgres/redis FATALs
|
||||
# with `Address in use` and `docker run` returns exit 125 with
|
||||
# `Conflict. The container name "/molecule-ci-postgres" is already
|
||||
# in use by container ...`. Verified in run a7/2727 on 2026-05-07.
|
||||
# * The fixed container names `molecule-ci-postgres` / `-redis` (the
|
||||
# pre-fix shape) collide on name AS WELL AS port. The cleanup-with-
|
||||
# `docker rm -f` at the start of the second job KILLS the first
|
||||
# job's still-running postgres/redis.
|
||||
#
|
||||
# Fix shape (mirrors PR #98's bridge-net pattern, adapted because
|
||||
# platform-server is a Go binary on the host, not a containerised
|
||||
# step):
|
||||
#
|
||||
# 1. Unique container names per run:
|
||||
# pg-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# redis-e2e-api-${RUN_ID}-${RUN_ATTEMPT}
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` is unique even across reruns of the
|
||||
# same run_id.
|
||||
# 2. Ephemeral host port per run (`-p 0:5432`), then read the actual
|
||||
# bound port via `docker port` and export DATABASE_URL/REDIS_URL
|
||||
# pointing at it. No fixed host-port → no port collision.
|
||||
# 3. `127.0.0.1` (NOT `localhost`) in URLs — IPv6 first-resolve was
|
||||
# the original flake fixed in #92 and the script's still IPv6-
|
||||
# enabled.
|
||||
# 4. `if: always()` cleanup so containers don't leak when test steps
|
||||
# fail.
|
||||
#
|
||||
# Issue #94 items #2 + #3 (also fixed here):
|
||||
# * Pre-pull `alpine:latest` so the platform-server's provisioner
|
||||
# (`internal/handlers/container_files.go`) can stand up its
|
||||
# ephemeral token-write helper without a daemon.io round-trip.
|
||||
# * Create `molecule-core-net` bridge network if missing so the
|
||||
# provisioner's container.HostConfig {NetworkMode: ...} attach
|
||||
# succeeds.
|
||||
# Item #1 (timeouts) — evidence on recent runs (77/3191, ae/4270, 0e/
|
||||
# 2318) shows Postgres ready in 3s, Redis in 1s, Platform in 1s when
|
||||
# they DO come up. Timeouts are not the bottleneck; not bumped.
|
||||
#
|
||||
# Item explicitly NOT fixed here: failing test `Status back online`
|
||||
# fails because the platform's langgraph workspace template image
|
||||
# (ghcr.io/molecule-ai/workspace-template-langgraph:latest) returns
|
||||
# 403 Forbidden post-2026-05-06 GitHub org suspension. That is a
|
||||
# template-registry resolution issue (ADR-002 / local-build mode) and
|
||||
# belongs in a separate change that touches workspace-server, not
|
||||
# this workflow file.
|
||||
# Fix: a dedicated workflow gets its own concurrency group at the workflow
|
||||
# level. New pushes to the same branch queue here instead of cancelling.
|
||||
# Fast jobs (platform-build, canvas-build, etc.) stay in ci.yml and continue
|
||||
# to benefit from run-level cancellation for quick feedback.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/e2e-api.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
workflow_dispatch:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/e2e-api.yml'
|
||||
|
||||
# Workflow-level concurrency: new runs queue rather than cancel.
|
||||
# `cancel-in-progress: false` is load-bearing — without it GitHub would still
|
||||
# cancel this run when the next push arrives, defeating the whole fix.
|
||||
# The group key includes github.ref so PRs don't compete with main.
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from per-ref). Per-ref had the
|
||||
# same auto-promote-staging brittleness as e2e-staging-canvas — back-
|
||||
# to-back staging pushes share refs/heads/staging, so the older push's
|
||||
# queued run gets cancelled when a newer push lands. Auto-promote-
|
||||
# staging then sees `completed/cancelled` for the older SHA and stays
|
||||
# put; the newer SHA's gates may eventually save the day, but if the
|
||||
# newer push gets cancelled too, we deadlock.
|
||||
#
|
||||
# See e2e-staging-canvas.yml's identical concurrency block for the full
|
||||
# rationale and the 2026-04-28 incident reference.
|
||||
group: e2e-api-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
group: e2e-api-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
api: ${{ steps.decide.outputs.api }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
api:
|
||||
- 'workspace-server/**'
|
||||
- 'tests/e2e/**'
|
||||
- '.github/workflows/e2e-api.yml'
|
||||
- id: decide
|
||||
# Always run real work for manual dispatch — no diff context to
|
||||
# filter against and ops dispatching this expects the suite to
|
||||
# actually exercise the platform.
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "api=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "api=${{ steps.filter.outputs.api }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `E2E API Smoke Test`. Real work is gated per-step
|
||||
# on `needs.detect-changes.outputs.api`. Reason: GitHub registers a
|
||||
# check run for every job that matches `name:`, and a job-level
|
||||
# `if: false` produces a SKIPPED check run. Branch protection treats
|
||||
# all check runs with a matching context name on the latest commit as a
|
||||
# SET — any SKIPPED in the set fails the required-check eval, even with
|
||||
# SUCCESS siblings. Verified 2026-04-29 on PR #2264 (staging→main):
|
||||
# 4 check runs (2 SKIPPED + 2 SUCCESS) at the head SHA blocked
|
||||
# promotion despite all real work succeeding. Collapsing to a single
|
||||
# always-running job with conditional steps emits exactly one SUCCESS
|
||||
# check run regardless of paths filter — branch-protection-clean.
|
||||
e2e-api:
|
||||
needs: detect-changes
|
||||
name: E2E API Smoke Test
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
timeout-minutes: 15
|
||||
# `services:` is Linux-only on self-hosted runners — we start postgres
|
||||
# and redis via `docker run` instead. Ports 15432/16379 avoid collision
|
||||
# with anything the host may already have on the standard ports.
|
||||
env:
|
||||
# Unique per-run container names so concurrent runs on the host-
|
||||
# network act_runner don't collide on name OR port.
|
||||
# `${RUN_ID}-${RUN_ATTEMPT}` stays unique across reruns of the
|
||||
# same run_id. PORT is set later (after docker port lookup) since
|
||||
# we let Docker assign an ephemeral host port.
|
||||
PG_CONTAINER: pg-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
REDIS_CONTAINER: redis-e2e-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
DATABASE_URL: postgres://dev:dev@localhost:15432/molecule?sslmode=disable
|
||||
REDIS_URL: redis://localhost:16379
|
||||
PORT: "8080"
|
||||
PG_CONTAINER: molecule-ci-postgres
|
||||
REDIS_CONTAINER: molecule-ci-redis
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.api != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / tests/e2e / workflow changes — E2E API gate satisfied without running tests."
|
||||
echo "::notice::E2E API Smoke Test no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.api == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
cache: true
|
||||
cache-dependency-path: workspace-server/go.sum
|
||||
- name: Pre-pull alpine + ensure provisioner network (Issue #94 items #2 + #3)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Provisioner uses alpine:latest for ephemeral token-write
|
||||
# containers (workspace-server/internal/handlers/container_files.go).
|
||||
# Pre-pull so the first provision in test_api.sh doesn't race
|
||||
# the daemon's pull cache. Idempotent — `docker pull` is a no-op
|
||||
# when the image is already present.
|
||||
docker pull alpine:latest >/dev/null
|
||||
# Provisioner attaches workspace containers to
|
||||
# molecule-core-net (workspace-server/internal/provisioner/
|
||||
# provisioner.go::DefaultNetwork). The bridge already exists on
|
||||
# the operator host's docker daemon — `network create` is
|
||||
# idempotent via `|| true`.
|
||||
docker network create molecule-core-net >/dev/null 2>&1 || true
|
||||
echo "alpine:latest pre-pulled; molecule-core-net ensured."
|
||||
- name: Start Postgres (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
# Defensive cleanup — only matches THIS run's container name,
|
||||
# so it cannot kill a sibling run's postgres. (Pre-fix the
|
||||
# name was static and this rm hit other runs' containers.)
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
# `-p 0:5432` requests an ephemeral host port; we read it back
|
||||
# below and export DATABASE_URL.
|
||||
docker run -d --name "$PG_CONTAINER" \
|
||||
-e POSTGRES_USER=dev -e POSTGRES_PASSWORD=dev -e POSTGRES_DB=molecule \
|
||||
-p 0:5432 postgres:16 >/dev/null
|
||||
# Resolve the host-side port assignment. `docker port` prints
|
||||
# `0.0.0.0:NNNN` (and on host-net runners may also print an
|
||||
# IPv6 line — take the first IPv4 line).
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
# Fallback: any first line. Some Docker versions print only
|
||||
# one line.
|
||||
PG_PORT=$(docker port "$PG_CONTAINER" 5432/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$PG_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $PG_CONTAINER"
|
||||
docker port "$PG_CONTAINER" 5432/tcp || true
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
# 127.0.0.1 (NOT localhost) — IPv6 first-resolve flake (#92).
|
||||
echo "PG_PORT=${PG_PORT}" >> "$GITHUB_ENV"
|
||||
echo "DATABASE_URL=postgres://dev:dev@127.0.0.1:${PG_PORT}/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Postgres host port: ${PG_PORT}"
|
||||
-e POSTGRES_USER=dev \
|
||||
-e POSTGRES_PASSWORD=dev \
|
||||
-e POSTGRES_DB=molecule \
|
||||
-p 15432:5432 \
|
||||
postgres:16
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$PG_CONTAINER" pg_isready -U dev >/dev/null 2>&1; then
|
||||
echo "Postgres ready after ${i}s"
|
||||
@@ -211,23 +75,9 @@ jobs:
|
||||
docker logs "$PG_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Start Redis (docker)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 0:6379 redis:7 >/dev/null
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | awk -F: '/^0\.0\.0\.0:/ {print $2; exit}')
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
REDIS_PORT=$(docker port "$REDIS_CONTAINER" 6379/tcp | head -1 | awk -F: '{print $NF}')
|
||||
fi
|
||||
if [ -z "$REDIS_PORT" ]; then
|
||||
echo "::error::Could not resolve host port for $REDIS_CONTAINER"
|
||||
docker port "$REDIS_CONTAINER" 6379/tcp || true
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "REDIS_PORT=${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "REDIS_URL=redis://127.0.0.1:${REDIS_PORT}" >> "$GITHUB_ENV"
|
||||
echo "Redis host port: ${REDIS_PORT}"
|
||||
docker run -d --name "$REDIS_CONTAINER" -p 16379:6379 redis:7
|
||||
for i in $(seq 1 15); do
|
||||
if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -q PONG; then
|
||||
echo "Redis ready after ${i}s"
|
||||
@@ -236,25 +86,19 @@ jobs:
|
||||
sleep 1
|
||||
done
|
||||
echo "::error::Redis did not become ready in 15s"
|
||||
docker logs "$REDIS_CONTAINER" || true
|
||||
exit 1
|
||||
- name: Build platform
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: go build -o platform-server ./cmd/server
|
||||
- name: Start platform (background)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
working-directory: workspace-server
|
||||
run: |
|
||||
# DATABASE_URL + REDIS_URL exported by the start-postgres /
|
||||
# start-redis steps point at this run's per-run host ports.
|
||||
./platform-server > platform.log 2>&1 &
|
||||
echo $! > platform.pid
|
||||
- name: Wait for /health
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: |
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://127.0.0.1:8080/health > /dev/null; then
|
||||
if curl -sf http://localhost:8080/health > /dev/null; then
|
||||
echo "Platform up after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
@@ -264,44 +108,29 @@ jobs:
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
- name: Assert migrations applied
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
# Migrations auto-run at platform boot. Fail fast if they silently
|
||||
# didn't — catches future migration-author mistakes before the E2E run.
|
||||
run: |
|
||||
tables=$(docker exec "$PG_CONTAINER" psql -U dev -d molecule -tAc "SELECT count(*) FROM information_schema.tables WHERE table_schema='public' AND table_name='workspaces'")
|
||||
if [ "$tables" != "1" ]; then
|
||||
echo "::error::Migrations did not apply"
|
||||
echo "::error::Migrations did not apply — 'workspaces' table missing"
|
||||
cat workspace-server/platform.log || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Migrations OK"
|
||||
echo "Migrations OK (workspaces table present)"
|
||||
- name: Run E2E API tests
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_api.sh
|
||||
- name: Run notify-with-attachments E2E
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_notify_attachments_e2e.sh
|
||||
- name: Run priority-runtimes E2E (claude-code + hermes — skips when keys absent)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_priority_runtimes_e2e.sh
|
||||
- name: Run poll-mode + since_id cursor E2E (#2339)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_e2e.sh
|
||||
- name: Run poll-mode chat upload E2E (RFC #2891)
|
||||
if: needs.detect-changes.outputs.api == 'true'
|
||||
run: bash tests/e2e/test_poll_mode_chat_upload_e2e.sh
|
||||
- name: Dump platform log on failure
|
||||
if: failure() && needs.detect-changes.outputs.api == 'true'
|
||||
if: failure()
|
||||
run: cat workspace-server/platform.log || true
|
||||
- name: Stop platform
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f workspace-server/platform.pid ]; then
|
||||
kill "$(cat workspace-server/platform.pid)" 2>/dev/null || true
|
||||
fi
|
||||
- name: Stop service containers
|
||||
# always() so containers don't leak when test steps fail. The
|
||||
# cleanup is best-effort: if the container is already gone
|
||||
# (e.g. concurrent rerun race), don't fail the job.
|
||||
if: always() && needs.detect-changes.outputs.api == 'true'
|
||||
if: always()
|
||||
run: |
|
||||
docker rm -f "$PG_CONTAINER" 2>/dev/null || true
|
||||
docker rm -f "$REDIS_CONTAINER" 2>/dev/null || true
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
name: E2E Staging Canvas (Playwright)
|
||||
|
||||
# Playwright test suite that provisions a fresh staging org per run and
|
||||
# verifies every workspace-panel tab renders without crashing. Complements
|
||||
# e2e-staging-saas.yml (which tests the API shape) by exercising the
|
||||
# actual browser + canvas bundle against live staging.
|
||||
#
|
||||
# Triggers: push to main/staging or PR touching canvas sources + this workflow,
|
||||
# manual dispatch, and weekly cron to catch browser/runtime drift even
|
||||
# when canvas is quiet.
|
||||
# Added staging to push/pull_request branches so the auto-promote gate
|
||||
# check (--event push --branch staging) can see a completed run for this
|
||||
# workflow — mirrors what PR #1891 does for e2e-api.yml.
|
||||
|
||||
on:
|
||||
# Trigger model (revised 2026-04-29):
|
||||
#
|
||||
# Always fires on push/pull_request; real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. When canvas/ paths haven't
|
||||
# changed, the no-op step alone runs and emits SUCCESS for the
|
||||
# `Canvas tabs E2E` check, satisfying branch protection without
|
||||
# spending CI cycles. See e2e-api.yml for the rationale on why this
|
||||
# is a single job rather than two-jobs-sharing-name.
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Weekly on Sunday 08:00 UTC — catches Chrome / Playwright / Next.js
|
||||
# release-note-shaped regressions that don't ride in with a PR.
|
||||
- cron: '0 8 * * 0'
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping (changed 2026-04-28 from a single global group). The
|
||||
# global group made auto-promote-staging brittle: when a staging push
|
||||
# queued behind an in-flight run and a third entrant (a PR run, a
|
||||
# follow-on push) entered the group, the staging push got cancelled —
|
||||
# leaving auto-promote-staging looking at `completed/cancelled` for a
|
||||
# required gate and refusing to advance main. Observed 2026-04-28
|
||||
# 23:51-23:53 on staging tip 3f99fede.
|
||||
#
|
||||
# The original intent of the global group was to throttle parallel
|
||||
# E2E provisions (each spins a fresh EC2). At our scale that throttle
|
||||
# isn't worth the correctness cost — fresh-org-per-run isolates the
|
||||
# state, and the cost of two parallel runs (~$0.001/min × 10min × 2)
|
||||
# is rounding error vs. the cost of a stuck pipeline.
|
||||
#
|
||||
# Per-SHA still dedupes accidental double-triggers for the SAME SHA.
|
||||
# It does NOT cancel obsolete-PR-version runs on force-push; that
|
||||
# wasted CI is acceptable given the alternative is losing staging-tip
|
||||
# data that auto-promote-staging needs.
|
||||
group: e2e-staging-canvas-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
canvas: ${{ steps.decide.outputs.canvas }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
canvas:
|
||||
- 'canvas/**'
|
||||
- '.github/workflows/e2e-staging-canvas.yml'
|
||||
- id: decide
|
||||
# Always run real tests for manual dispatch and the weekly cron —
|
||||
# both exist precisely to exercise the suite, regardless of diff.
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "schedule" ]; then
|
||||
echo "canvas=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "canvas=${{ steps.filter.outputs.canvas }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `Canvas tabs E2E`. Real work is gated per-step on
|
||||
# `needs.detect-changes.outputs.canvas`. See e2e-api.yml for the full
|
||||
# rationale — same path-filter check-name parity issue blocked PR #2264
|
||||
# (staging→main) on 2026-04-29 because branch protection treats matching-
|
||||
# name check runs as a SET, and any SKIPPED member fails the eval.
|
||||
playwright:
|
||||
needs: detect-changes
|
||||
name: Canvas tabs E2E
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
|
||||
env:
|
||||
CANVAS_E2E_STAGING: '1'
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: canvas
|
||||
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.canvas != 'true'
|
||||
working-directory: .
|
||||
run: |
|
||||
echo "No canvas / workflow changes — E2E Staging Canvas gate satisfied without running tests."
|
||||
echo "::notice::E2E Staging Canvas no-op pass (paths filter excluded this commit)."
|
||||
|
||||
- if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::Missing MOLECULE_STAGING_ADMIN_TOKEN"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Set up Node
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: '20'
|
||||
cache: 'npm'
|
||||
cache-dependency-path: canvas/package-lock.json
|
||||
|
||||
- name: Install canvas deps
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Install Playwright browsers
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Run staging canvas E2E
|
||||
if: needs.detect-changes.outputs.canvas == 'true'
|
||||
run: npx playwright test --config=playwright.staging.config.ts
|
||||
|
||||
- name: Upload Playwright report on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility — v4+ uses
|
||||
# the GHES 3.10+ artifact protocol that Gitea 1.22.x does NOT
|
||||
# implement (see ci.yml upload step for the canonical error
|
||||
# cite). Drop this pin when Gitea ships the v4 protocol.
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-report-staging
|
||||
path: canvas/playwright-report-staging/
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload screenshots on failure
|
||||
if: failure() && needs.detect-changes.outputs.canvas == 'true'
|
||||
# Pinned to v3 for Gitea act_runner v0.6 compatibility (see above).
|
||||
uses: actions/upload-artifact@c6a366c94c3e0affe28c06c8df20a878f24da3cf # v3.2.2
|
||||
with:
|
||||
name: playwright-screenshots
|
||||
path: canvas/test-results/
|
||||
retention-days: 14
|
||||
|
||||
# Safety-net teardown — fires only when Playwright's globalTeardown
|
||||
# didn't (worker crash, runner cancel). Reads the slug from
|
||||
# canvas/.playwright-staging-state.json (written by staging-setup
|
||||
# as its first action, before any CP call) and deletes only that
|
||||
# slug.
|
||||
#
|
||||
# Earlier versions of this step pattern-swept `e2e-canvas-<today>-*`
|
||||
# orgs to compensate for setup-crash-before-state-file-write. That
|
||||
# over-aggressive cleanup raced concurrent canvas-E2E runs and
|
||||
# poisoned each other's tenants — observed 2026-04-30 when three
|
||||
# real-test runs killed each other mid-test, surfacing as
|
||||
# `getaddrinfo ENOTFOUND` once CP had cleaned up the just-deleted
|
||||
# DNS record. Pattern-sweep removed; setup now writes the state
|
||||
# file before any CP work, so the slug is always recoverable.
|
||||
- name: Teardown safety net
|
||||
if: always() && needs.detect-changes.outputs.canvas == 'true'
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
STATE_FILE=".playwright-staging-state.json"
|
||||
if [ ! -f "$STATE_FILE" ]; then
|
||||
echo "::notice::No state file at canvas/$STATE_FILE — Playwright globalTeardown handled it (or setup never ran)."
|
||||
exit 0
|
||||
fi
|
||||
slug=$(python3 -c "import json; print(json.load(open('$STATE_FILE')).get('slug',''))")
|
||||
if [ -z "$slug" ]; then
|
||||
echo "::warning::State file present but slug missing; nothing to clean up."
|
||||
exit 0
|
||||
fi
|
||||
echo "Deleting orphan tenant: $slug"
|
||||
# Verify HTTP 2xx instead of `>/dev/null || true` swallowing
|
||||
# failures. A 5xx or timeout previously looked identical to
|
||||
# success, leaving the tenant alive for up to ~45 min until
|
||||
# sweep-stale-e2e-orgs caught it. Surface failures as
|
||||
# workflow warnings naming the slug. Don't `exit 1` — a single
|
||||
# cleanup miss shouldn't fail-flag the canvas test when the
|
||||
# actual smoke check passed; the sweeper is the safety net.
|
||||
# See molecule-controlplane#420.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/canvas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/canvas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/canvas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::canvas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/canvas-cleanup.out 2>/dev/null)"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,184 +0,0 @@
|
||||
name: E2E Staging External Runtime
|
||||
|
||||
# Regression for the four/five workspaces.status=awaiting_agent transitions
|
||||
# that silently failed in production for five days before migration 046
|
||||
# extended the workspace_status enum (see
|
||||
# workspace-server/migrations/046_workspace_status_awaiting_agent.up.sql).
|
||||
#
|
||||
# Why this is its own workflow (not folded into e2e-staging-saas.yml):
|
||||
# - The full-saas harness defaults to runtime=hermes, never exercises
|
||||
# external-runtime. Adding an `external` parameter to that script
|
||||
# would force every push to staging through both lifecycles in
|
||||
# series, doubling the EC2 cold-start budget.
|
||||
# - The external lifecycle has unique timing (REMOTE_LIVENESS_STALE_AFTER
|
||||
# window, 90s default + sweep interval), which we wait through
|
||||
# deliberately. Folding it into hermes would make the long path
|
||||
# even longer.
|
||||
# - It can run in parallel with the hermes E2E since both create
|
||||
# fresh tenant orgs with distinct slug prefixes (`e2e-ext-...` vs
|
||||
# `e2e-...`).
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to staging when any source affecting external runtime,
|
||||
# hibernation, or the migration set changes.
|
||||
# - PR review for the same set.
|
||||
# - Manual workflow_dispatch.
|
||||
# - Daily cron at 07:30 UTC (catches drift on quiet days; staggered
|
||||
# 30 min after e2e-staging-saas.yml's 07:00 UTC cron).
|
||||
#
|
||||
# Concurrency: serialized so two staging pushes don't fight for the
|
||||
# same EC2 quota window. cancel-in-progress=false so a half-rolled
|
||||
# tenant always finishes its teardown.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.github/workflows/e2e-staging-external.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/workspace.go'
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_restart.go'
|
||||
- 'workspace-server/internal/registry/healthsweep.go'
|
||||
- 'workspace-server/internal/registry/liveness.go'
|
||||
- 'workspace-server/migrations/**'
|
||||
- 'workspace-server/internal/db/workspace_status_enum_drift_test.go'
|
||||
- 'tests/e2e/test_staging_external_runtime.sh'
|
||||
- '.github/workflows/e2e-staging-external.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
keep_org:
|
||||
description: "Skip teardown for debugging (only via manual dispatch)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
stale_wait_secs:
|
||||
description: "Seconds to wait for the heartbeat-staleness sweep (default 180 = 90s window + 90s buffer)"
|
||||
required: false
|
||||
default: "180"
|
||||
schedule:
|
||||
- cron: '30 7 * * *'
|
||||
|
||||
concurrency:
|
||||
group: e2e-staging-external
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
e2e-staging-external:
|
||||
name: E2E Staging External Runtime
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
E2E_STALE_WAIT_SECS: ${{ github.event.inputs.stale_wait_secs || '180' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
# Schedule + push triggers must hard-fail when the token is
|
||||
# missing — silent skip would mask infra rot. Manual dispatch
|
||||
# gets the same hard-fail; an operator running this on a fork
|
||||
# without secrets configured needs to know up-front.
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run external-runtime E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_external_runtime.sh
|
||||
|
||||
# Mirror the e2e-staging-saas.yml safety net: if the runner is
|
||||
# cancelled (e.g. concurrent staging push), the test script's
|
||||
# EXIT trap may not fire, so we sweep e2e-ext-* slugs scoped to
|
||||
# *this* run id.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# Scope STRICTLY to this run id (e2e-ext-YYYYMMDD-<runid>-...)
|
||||
# so concurrent runs and unrelated dev probes are not touched.
|
||||
# Sweep today AND yesterday so a midnight-crossing run still
|
||||
# cleans up its own slug.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if not run_id:
|
||||
# Without a run id we cannot scope safely; bail rather
|
||||
# than risk deleting unrelated tenants.
|
||||
sys.exit(0)
|
||||
prefixes = tuple(f'e2e-ext-{d}-{run_id}-' for d in dates)
|
||||
for o in d.get('orgs', []):
|
||||
s = o.get('slug', '')
|
||||
if s.startswith(prefixes) and o.get('status') != 'purged':
|
||||
print(s)
|
||||
" 2>/dev/null)
|
||||
if [ -n "$orgs" ]; then
|
||||
echo "Safety-net sweep: deleting leftover orgs:"
|
||||
echo "$orgs"
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# `>/dev/null 2>&1` previously hid every failure; surface
|
||||
# non-2xx as workflow warnings so the run page names what
|
||||
# leaked. Sweeper catches the rest within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/external-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/external-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/external-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::external teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/external-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::external teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
else
|
||||
echo "Safety-net sweep: no leftover orgs to clean."
|
||||
fi
|
||||
@@ -1,246 +0,0 @@
|
||||
name: E2E Staging SaaS (full lifecycle)
|
||||
|
||||
# Dedicated workflow that provisions a fresh staging org per run, exercises
|
||||
# the full workspace lifecycle (register → heartbeat → A2A → delegation →
|
||||
# HMA memory → activity → peers), then tears down and asserts leak-free.
|
||||
#
|
||||
# Why a separate workflow (not folded into ci.yml):
|
||||
# - The run takes ~25-35 min (EC2 boot + cloudflared DNS + provision sweeps +
|
||||
# agent bootstrap), way too slow for every PR.
|
||||
# - Needs its own concurrency group so two pushes don't fight over the
|
||||
# same staging org slug prefix.
|
||||
# - Has its own required secrets (session cookie, admin token) that most
|
||||
# PRs don't need to read.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push to main (regression guard)
|
||||
# - workflow_dispatch (manual re-run from UI)
|
||||
# - Nightly cron (catches drift even when no pushes land)
|
||||
# - Changes to any provisioning-critical file under PR review (opt-in
|
||||
# via the same paths watcher that e2e-api.yml uses)
|
||||
|
||||
on:
|
||||
# Trunk-based (Phase 3 of internal#81): main is the only branch.
|
||||
# Previously this fired on staging push too because staging was a
|
||||
# superset of main and ran the gate ahead of auto-promote; with no
|
||||
# staging branch, main is where E2E gates the deploy.
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.github/workflows/e2e-staging-saas.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/internal/handlers/registry.go'
|
||||
- 'workspace-server/internal/handlers/workspace_provision.go'
|
||||
- 'workspace-server/internal/handlers/a2a_proxy.go'
|
||||
- 'workspace-server/internal/middleware/**'
|
||||
- 'workspace-server/internal/provisioner/**'
|
||||
- 'tests/e2e/test_staging_full_saas.sh'
|
||||
- '.github/workflows/e2e-staging-saas.yml'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runtime:
|
||||
description: "Runtime to test (claude-code [default, MiniMax] | hermes [OpenAI] | langgraph [OpenAI])"
|
||||
required: false
|
||||
default: "claude-code"
|
||||
keep_org:
|
||||
description: "Skip teardown for debugging (only use via manual dispatch!)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# 07:00 UTC every day — catches AMI drift, WorkOS cert rotation,
|
||||
# Cloudflare API regressions, etc. even on quiet days.
|
||||
- cron: '0 7 * * *'
|
||||
|
||||
# Serialize: staging has a finite per-hour org creation quota. Two pushes
|
||||
# landing in quick succession should queue, not race. `cancel-in-progress:
|
||||
# false` mirrors e2e-api.yml — GitHub would otherwise cancel the running
|
||||
# teardown step and leave orphan EC2s.
|
||||
concurrency:
|
||||
group: e2e-staging-saas
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
e2e-staging-saas:
|
||||
name: E2E Staging SaaS
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
# Single admin-bearer secret drives provision + tenant-token
|
||||
# retrieval + teardown. Configure in
|
||||
# Settings → Secrets and variables → Actions → Repository secrets.
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
# MiniMax is the PRIMARY LLM auth path post-2026-05-04. Switched
|
||||
# from hermes+OpenAI default after #2578 (the staging OpenAI key
|
||||
# account went over quota and stayed dead for 36+ hours, taking
|
||||
# the full-lifecycle E2E red on every provisioning-critical push).
|
||||
# claude-code template's `minimax` provider routes
|
||||
# ANTHROPIC_BASE_URL to api.minimax.io/anthropic and reads
|
||||
# MINIMAX_API_KEY at boot — separate billing account so an
|
||||
# OpenAI quota collapse no longer wedges the gate. Mirrors the
|
||||
# canary-staging.yml + continuous-synth-e2e.yml migrations.
|
||||
E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
|
||||
# Direct-Anthropic alternative for operators who don't want to
|
||||
# set up a MiniMax account (priority below MiniMax — first
|
||||
# non-empty wins in test_staging_full_saas.sh's secrets-injection
|
||||
# block). See #2578 PR comment for the rationale.
|
||||
E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
|
||||
# OpenAI fallback — kept wired so an operator-dispatched run with
|
||||
# E2E_RUNTIME=hermes or =langgraph via workflow_dispatch can still
|
||||
# exercise the OpenAI path.
|
||||
E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_KEY }}
|
||||
E2E_RUNTIME: ${{ github.event.inputs.runtime || 'claude-code' }}
|
||||
# Pin the model when running on the default claude-code path —
|
||||
# the per-runtime default ("sonnet") routes to direct Anthropic
|
||||
# and defeats the cost saving. Operators can override via the
|
||||
# workflow_dispatch flow (no input wired here yet — runtime
|
||||
# override is enough for ad-hoc).
|
||||
E2E_MODEL_SLUG: ${{ github.event.inputs.runtime == 'hermes' && 'openai/gpt-4o' || github.event.inputs.runtime == 'langgraph' && 'openai:gpt-4o' || 'MiniMax-M2.7-highspeed' }}
|
||||
E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
|
||||
E2E_KEEP_ORG: ${{ github.event.inputs.keep_org && '1' || '0' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Verify LLM key present
|
||||
run: |
|
||||
# Per-runtime key check — claude-code uses MiniMax; hermes /
|
||||
# langgraph (operator-dispatched only) use OpenAI. Hard-fail
|
||||
# rather than soft-skip per #2578's lesson — empty key
|
||||
# silently falls through to the wrong SECRETS_JSON branch and
|
||||
# produces a confusing auth error 5 min later instead of the
|
||||
# clean "secret missing" message at the top.
|
||||
case "${E2E_RUNTIME}" in
|
||||
claude-code)
|
||||
# Either MiniMax OR direct-Anthropic works — first
|
||||
# non-empty wins in the test script's secrets-injection
|
||||
# priority chain.
|
||||
if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY"
|
||||
required_secret_value="${E2E_MINIMAX_API_KEY}"
|
||||
elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
|
||||
required_secret_name="MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value="${E2E_ANTHROPIC_API_KEY}"
|
||||
else
|
||||
required_secret_name="MOLECULE_STAGING_MINIMAX_API_KEY or MOLECULE_STAGING_ANTHROPIC_API_KEY"
|
||||
required_secret_value=""
|
||||
fi
|
||||
;;
|
||||
langgraph|hermes)
|
||||
required_secret_name="MOLECULE_STAGING_OPENAI_KEY"
|
||||
required_secret_value="${E2E_OPENAI_API_KEY:-}"
|
||||
;;
|
||||
*)
|
||||
echo "::warning::Unknown E2E_RUNTIME='${E2E_RUNTIME}' — skipping LLM-key check"
|
||||
required_secret_name=""
|
||||
required_secret_value="present"
|
||||
;;
|
||||
esac
|
||||
if [ -n "$required_secret_name" ] && [ -z "$required_secret_value" ]; then
|
||||
echo "::error::${required_secret_name} secret not set for runtime=${E2E_RUNTIME} — workspaces will fail at boot with 'No provider API key found'"
|
||||
exit 2
|
||||
fi
|
||||
echo "LLM key present ✓ (runtime=${E2E_RUNTIME}, key=${required_secret_name}, len=${#required_secret_value})"
|
||||
|
||||
- name: CP staging health preflight
|
||||
run: |
|
||||
code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::Staging CP unhealthy (got HTTP $code). Skipping — not a workspace bug."
|
||||
exit 1
|
||||
fi
|
||||
echo "Staging CP healthy ✓"
|
||||
|
||||
- name: Run full-lifecycle E2E
|
||||
id: e2e
|
||||
run: bash tests/e2e/test_staging_full_saas.sh
|
||||
|
||||
# Belt-and-braces teardown: the test script itself installs a trap
|
||||
# for EXIT/INT/TERM, but if the GH runner itself is cancelled (e.g.
|
||||
# someone pushes a new commit and workflow concurrency is set to
|
||||
# cancel), the trap may not fire. This `always()` step runs even on
|
||||
# cancellation and attempts the delete a second time. The admin
|
||||
# DELETE endpoint is idempotent so double-invoking is safe.
|
||||
- name: Teardown safety net (runs on cancel/failure)
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
# Best-effort: find any e2e-YYYYMMDD-* orgs matching this run and
|
||||
# nuke them. Catches the case where the script died before
|
||||
# exporting its slug.
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys, os, datetime
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
d = json.load(sys.stdin)
|
||||
# ONLY sweep slugs from *this* CI run. Previously the filter was
|
||||
# f'e2e-{today}-' which stomped on parallel CI runs AND any manual
|
||||
# E2E probes a dev was running against staging (incident 2026-04-21
|
||||
# 15:02Z: this workflow's safety net deleted an unrelated manual
|
||||
# run's tenant 1s after it hit 'running').
|
||||
# Sweep both today AND yesterday's UTC dates so a run that crosses
|
||||
# midnight still matches its own slug — see the 2026-04-26→27
|
||||
# canvas-safety-net incident for the same bug class.
|
||||
today = datetime.date.today()
|
||||
yesterday = today - datetime.timedelta(days=1)
|
||||
dates = (today.strftime('%Y%m%d'), yesterday.strftime('%Y%m%d'))
|
||||
if run_id:
|
||||
prefixes = tuple(f'e2e-{d}-{run_id}-' for d in dates)
|
||||
else:
|
||||
prefixes = tuple(f'e2e-{d}-' for d in dates)
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if any(o.get('slug','').startswith(p) for p in prefixes)
|
||||
and o.get('instance_status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug verified DELETE (was `>/dev/null || true` — see
|
||||
# molecule-controlplane#420). Surface non-2xx as a workflow
|
||||
# warning naming the leaked slug; don't exit 1 (sweeper is
|
||||
# the safety net within ~45 min).
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
echo "Safety-net teardown: $slug"
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/saas-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/saas-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/saas-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::saas teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/saas-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::saas teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,171 +0,0 @@
|
||||
name: E2E Staging Sanity (leak-detection self-check)
|
||||
|
||||
# Periodic assertion that the teardown safety nets in e2e-staging-saas
|
||||
# and canary-staging actually work. Runs the E2E harness with
|
||||
# E2E_INTENTIONAL_FAILURE=1, which poisons the tenant admin token after
|
||||
# the org is provisioned. The workspace-provision step then fails, the
|
||||
# script exits non-zero, and the EXIT trap + workflow always()-step
|
||||
# must still tear down cleanly.
|
||||
#
|
||||
# A green run means:
|
||||
# - The script exited non-zero (intentional failure caught)
|
||||
# - The trap fired teardown
|
||||
# - The leak-detection poll found zero orphan orgs
|
||||
#
|
||||
# A red run means the teardown path itself is broken — act on this the
|
||||
# same way you'd act on a canary failure (the whole E2E safety net is
|
||||
# compromised until it's fixed).
|
||||
#
|
||||
# Cadence: once a week, Monday 06:00 UTC. Drift-slow, not per-PR — the
|
||||
# teardown path rarely changes, and a weekly heartbeat is enough to
|
||||
# catch silent regressions in cleanup code paths.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 6 * * 1'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
# Shares the group with canary + full so they don't collide on
|
||||
# staging org-create quota.
|
||||
group: e2e-staging-sanity
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sanity:
|
||||
name: Intentional-failure teardown sanity
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
MOLECULE_ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
E2E_MODE: canary # lean lifecycle; we only need the org to exist
|
||||
E2E_RUNTIME: hermes
|
||||
E2E_RUN_ID: "sanity-${{ github.run_id }}"
|
||||
E2E_INTENTIONAL_FAILURE: "1"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Inverted assertion: the run MUST fail. If it passes, the
|
||||
# E2E_INTENTIONAL_FAILURE path is broken (token not being
|
||||
# poisoned correctly, or the harness silently recovered).
|
||||
- name: Run harness — expecting exit !=0
|
||||
id: harness
|
||||
run: |
|
||||
set +e
|
||||
bash tests/e2e/test_staging_full_saas.sh
|
||||
rc=$?
|
||||
echo "harness_rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# The only acceptable outcomes:
|
||||
# 1 — harness failed mid-run, teardown ran, leak-check passed
|
||||
# (exit 4 means teardown left a leak — that's the real bug
|
||||
# this sanity check exists to catch)
|
||||
if [ "$rc" = "1" ]; then
|
||||
echo "✓ Harness failed as expected (rc=1); teardown trap ran, leak-check passed"
|
||||
exit 0
|
||||
elif [ "$rc" = "0" ]; then
|
||||
echo "::error::Harness succeeded under E2E_INTENTIONAL_FAILURE=1 — the poisoning path is broken"
|
||||
exit 1
|
||||
elif [ "$rc" = "4" ]; then
|
||||
echo "::error::LEAK DETECTED (rc=4) — teardown failed to clean up the org. Safety net broken."
|
||||
exit 4
|
||||
else
|
||||
echo "::error::Unexpected rc=$rc — neither clean-failure nor leak. Investigate harness."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Open issue if safety net is broken
|
||||
if: failure()
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = "🚨 E2E teardown safety net broken";
|
||||
const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const body =
|
||||
`The weekly sanity run (E2E_INTENTIONAL_FAILURE=1) did not exit ` +
|
||||
`as expected. This means one of:\n` +
|
||||
` - poisoning didn't actually cause failure (test harness regression), OR\n` +
|
||||
` - teardown left an orphan org (leak detection caught a real bug)\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`This is higher priority than a canary failure — the whole ` +
|
||||
`E2E safety net can't be trusted until this is resolved.`;
|
||||
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'e2e-safety-net',
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Still broken. ${runURL}`,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['e2e-safety-net', 'bug', 'priority-high'],
|
||||
});
|
||||
}
|
||||
|
||||
# Belt-and-braces: if teardown left anything behind, nuke it here
|
||||
# so we don't bleed staging quota. Different label from the
|
||||
# always()-steps in the other workflows so sanity-only orgs get
|
||||
# cleaned up by sanity runs.
|
||||
- name: Teardown safety net
|
||||
if: always()
|
||||
env:
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
today = __import__('datetime').date.today().strftime('%Y%m%d')
|
||||
candidates = [o['slug'] for o in d.get('orgs', [])
|
||||
if o.get('slug','').startswith(f'e2e-canary-{today}-sanity-')
|
||||
and o.get('status') not in ('purged',)]
|
||||
print('\n'.join(candidates))
|
||||
" 2>/dev/null)
|
||||
# Per-slug verified DELETE — see molecule-controlplane#420.
|
||||
# Failures surface as workflow warnings; the sweeper is the
|
||||
# safety net within ~45 min.
|
||||
leaks=()
|
||||
for slug in $orgs; do
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/sanity-cleanup.out -w "%{http_code}" \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/sanity-cleanup.code
|
||||
set -e
|
||||
code=$(cat /tmp/sanity-cleanup.code 2>/dev/null || echo "000")
|
||||
if [ "$code" = "200" ] || [ "$code" = "204" ]; then
|
||||
echo "[teardown] deleted $slug (HTTP $code)"
|
||||
else
|
||||
echo "::warning::sanity teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within ~45 min. Body: $(head -c 300 /tmp/sanity-cleanup.out 2>/dev/null)"
|
||||
leaks+=("$slug")
|
||||
fi
|
||||
done
|
||||
if [ ${#leaks[@]} -gt 0 ]; then
|
||||
echo "::warning::sanity teardown left ${#leaks[@]} leak(s): ${leaks[*]}"
|
||||
fi
|
||||
exit 0
|
||||
@@ -1,252 +0,0 @@
|
||||
name: Handlers Postgres Integration
|
||||
|
||||
# Real-Postgres integration tests for workspace-server/internal/handlers/.
|
||||
# Triggered on every PR/push that touches the handlers package.
|
||||
#
|
||||
# Why this workflow exists
|
||||
# ------------------------
|
||||
# Strict-sqlmock unit tests pin which SQL statements fire — they're fast
|
||||
# and let us iterate without a DB. But sqlmock CANNOT detect bugs that
|
||||
# depend on the row state AFTER the SQL runs. The result_preview-lost
|
||||
# bug shipped to staging in PR #2854 because every unit test was
|
||||
# satisfied with "an UPDATE statement fired" — none verified the row's
|
||||
# preview field actually landed. The local-postgres E2E that retrofit
|
||||
# self-review caught it took 2 minutes to set up and would have caught
|
||||
# the bug at PR-time.
|
||||
#
|
||||
# Why this workflow does NOT use `services: postgres:` (Class B fix)
|
||||
# ------------------------------------------------------------------
|
||||
# Our act_runner config has `container.network: host` (operator host
|
||||
# /opt/molecule/runners/config.yaml), which act_runner applies to BOTH
|
||||
# the job container AND every service container. With host-net, two
|
||||
# concurrent runs of this workflow both try to bind 0.0.0.0:5432 — the
|
||||
# second postgres FATALs with `could not create any TCP/IP sockets:
|
||||
# Address in use`, and Docker auto-removes it (act_runner sets
|
||||
# AutoRemove:true on service containers). By the time the migrations
|
||||
# step runs `psql`, the postgres container is gone, hence
|
||||
# `Connection refused` then `failed to remove container: No such
|
||||
# container` at cleanup time.
|
||||
#
|
||||
# Per-job `container.network` override is silently ignored by
|
||||
# act_runner — `--network and --net in the options will be ignored.`
|
||||
# appears in the runner log. Documented constraint.
|
||||
#
|
||||
# So we sidestep `services:` entirely. The job container still uses
|
||||
# host-net (inherited from runner config; required for cache server
|
||||
# discovery on the bridge IP 172.18.0.17:42631). We launch a sibling
|
||||
# postgres on the existing `molecule-core-net` bridge with a
|
||||
# UNIQUE name per run — `pg-handlers-${RUN_ID}-${RUN_ATTEMPT}` — and
|
||||
# read its bridge IP via `docker inspect`. A host-net job container
|
||||
# can reach a bridge-net container directly via the bridge IP (verified
|
||||
# manually on operator host 2026-05-08).
|
||||
#
|
||||
# Trade-offs vs. the original `services:` shape:
|
||||
# + No host-port collision; N parallel runs share the bridge cleanly
|
||||
# + `if: always()` cleanup runs even on test-step failure
|
||||
# - One more step in the workflow (+~3 lines)
|
||||
# - Requires `molecule-core-net` to exist on the operator host
|
||||
# (it does; declared in docker-compose.yml + docker-compose.infra.yml)
|
||||
#
|
||||
# Class B Hongming-owned CICD red sweep, 2026-05-08.
|
||||
#
|
||||
# Cost: ~30s job (postgres pull from cache + go build + 4 tests).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: handlers-pg-integ-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
name: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
handlers: ${{ steps.filter.outputs.handlers }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
handlers:
|
||||
- 'workspace-server/internal/handlers/**'
|
||||
- 'workspace-server/internal/wsauth/**'
|
||||
- 'workspace-server/migrations/**'
|
||||
- '.github/workflows/handlers-postgres-integration.yml'
|
||||
|
||||
# Single-job-with-per-step-if pattern: always runs to satisfy the
|
||||
# required-check name on branch protection; real work gates on the
|
||||
# paths filter. See ci.yml's Platform (Go) for the same shape.
|
||||
integration:
|
||||
name: Handlers Postgres Integration
|
||||
needs: detect-changes
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Unique name per run so concurrent jobs don't collide on the
|
||||
# bridge network. ${RUN_ID}-${RUN_ATTEMPT} is unique even across
|
||||
# workflow_dispatch reruns of the same run_id.
|
||||
PG_NAME: pg-handlers-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
# Bridge network already exists on the operator host (declared
|
||||
# in docker-compose.yml + docker-compose.infra.yml).
|
||||
PG_NETWORK: molecule-core-net
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workspace-server
|
||||
steps:
|
||||
- if: needs.detect-changes.outputs.handlers != 'true'
|
||||
working-directory: .
|
||||
run: echo "No handlers/migrations changes — skipping; this job always runs to satisfy the required-check name."
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
|
||||
with:
|
||||
go-version: 'stable'
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Start sibling Postgres on bridge network
|
||||
working-directory: .
|
||||
run: |
|
||||
# Sanity: the bridge network must exist on the operator host.
|
||||
# Hard-fail loud if it doesn't — easier to spot than a silent
|
||||
# auto-create that diverges from the rest of the stack.
|
||||
if ! docker network inspect "${PG_NETWORK}" >/dev/null 2>&1; then
|
||||
echo "::error::Bridge network '${PG_NETWORK}' missing on operator host. Re-run docker-compose.infra.yml or check ops handbook."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If a stale container with the same name exists (rerun on
|
||||
# the same run_id), wipe it first.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
|
||||
docker run -d \
|
||||
--name "${PG_NAME}" \
|
||||
--network "${PG_NETWORK}" \
|
||||
--health-cmd "pg_isready -U postgres" \
|
||||
--health-interval 5s \
|
||||
--health-timeout 5s \
|
||||
--health-retries 10 \
|
||||
-e POSTGRES_PASSWORD=test \
|
||||
-e POSTGRES_DB=molecule \
|
||||
postgres:15-alpine >/dev/null
|
||||
|
||||
# Read back the bridge IP. Always present immediately after
|
||||
# `docker run -d` for bridge networks.
|
||||
PG_HOST=$(docker inspect "${PG_NAME}" \
|
||||
--format "{{(index .NetworkSettings.Networks \"${PG_NETWORK}\").IPAddress}}")
|
||||
if [ -z "${PG_HOST}" ]; then
|
||||
echo "::error::Could not resolve PG_HOST for ${PG_NAME} on ${PG_NETWORK}"
|
||||
docker logs "${PG_NAME}" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_HOST=${PG_HOST}" >> "$GITHUB_ENV"
|
||||
echo "INTEGRATION_DB_URL=postgres://postgres:test@${PG_HOST}:5432/molecule?sslmode=disable" >> "$GITHUB_ENV"
|
||||
echo "Started ${PG_NAME} at ${PG_HOST}:5432"
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Apply migrations to Postgres service
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
# Wait for postgres to actually accept connections. Docker's
|
||||
# health-cmd handles container-side readiness, but the wire
|
||||
# to the bridge IP is best-tested with pg_isready directly.
|
||||
for i in {1..15}; do
|
||||
if pg_isready -h "${PG_HOST}" -p 5432 -U postgres -q; then break; fi
|
||||
echo "waiting for postgres at ${PG_HOST}:5432..."; sleep 2
|
||||
done
|
||||
|
||||
# Apply every .up.sql in lexicographic order with
|
||||
# ON_ERROR_STOP=0 — failing migrations are SKIPPED rather than
|
||||
# blocking the suite. This handles the current schema state
|
||||
# where a few historical migrations (e.g. 017_memories_fts_*)
|
||||
# depend on tables that were later renamed/dropped and so
|
||||
# cannot replay from scratch. The migrations that DO succeed
|
||||
# land their tables, which is sufficient for the integration
|
||||
# tests in handlers/.
|
||||
#
|
||||
# Why not maintain a curated allowlist: every new migration
|
||||
# touching a handlers/-tested table would have to update this
|
||||
# workflow. With apply-all-or-skip, a future migration that
|
||||
# adds a column to delegations runs automatically (its base
|
||||
# table 049_delegations.up.sql already succeeded above it in
|
||||
# the order). Operators only need to revisit this if the
|
||||
# migration chain becomes legitimately replayable end-to-end.
|
||||
#
|
||||
# Per-migration result is logged so a failed migration that
|
||||
# SHOULD have been replayable surfaces in the CI log instead
|
||||
# of silently failing.
|
||||
# Apply both *.sql (legacy, lives next to its module) and
|
||||
# *.up.sql (newer up/down convention) in a single
|
||||
# lexicographically-sorted pass. Excluding *.down.sql so the
|
||||
# newest-naming-convention pairs don't undo themselves mid-run.
|
||||
# Pre-#149-followup this loop only globbed *.up.sql, which
|
||||
# silently skipped 001_workspaces.sql + 009_activity_logs.sql
|
||||
# — fine while no integration test depended on those tables,
|
||||
# not fine once a cross-table atomicity test came in.
|
||||
set +e
|
||||
for migration in $(ls migrations/*.sql 2>/dev/null | grep -v '\.down\.sql$' | sort); do
|
||||
if psql -h "${PG_HOST}" -U postgres -d molecule -v ON_ERROR_STOP=1 \
|
||||
-f "$migration" >/dev/null 2>&1; then
|
||||
echo "✓ $(basename "$migration")"
|
||||
else
|
||||
echo "⊘ $(basename "$migration") (skipped — see comment in workflow)"
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
|
||||
# Sanity: the delegations + workspaces + activity_logs tables
|
||||
# MUST exist for the integration tests to be meaningful. Hard-
|
||||
# fail if any didn't land — that would be a real regression we
|
||||
# want loud.
|
||||
for tbl in delegations workspaces activity_logs pending_uploads; do
|
||||
if ! psql -h "${PG_HOST}" -U postgres -d molecule -tA \
|
||||
-c "SELECT 1 FROM information_schema.tables WHERE table_name = '$tbl'" \
|
||||
| grep -q 1; then
|
||||
echo "::error::$tbl table missing after migration replay — handler integration tests would be meaningless"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ $tbl table present"
|
||||
done
|
||||
|
||||
- if: needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Run integration tests
|
||||
run: |
|
||||
# INTEGRATION_DB_URL is exported by the start-postgres step;
|
||||
# points at the per-run bridge IP, not 127.0.0.1, so concurrent
|
||||
# workflow runs don't fight over a host-net 5432 port.
|
||||
go test -tags=integration -timeout 5m -v ./internal/handlers/ -run "^TestIntegration_"
|
||||
|
||||
- if: failure() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Diagnostic dump on failure
|
||||
env:
|
||||
PGPASSWORD: test
|
||||
run: |
|
||||
echo "::group::postgres container status"
|
||||
docker ps -a --filter "name=${PG_NAME}" --format '{{.Status}} {{.Names}}' || true
|
||||
docker logs "${PG_NAME}" 2>&1 | tail -50 || true
|
||||
echo "::endgroup::"
|
||||
echo "::group::delegations table state"
|
||||
psql -h "${PG_HOST}" -U postgres -d molecule -c "SELECT * FROM delegations LIMIT 50;" || true
|
||||
echo "::endgroup::"
|
||||
|
||||
- if: always() && needs.detect-changes.outputs.handlers == 'true'
|
||||
name: Stop sibling Postgres
|
||||
working-directory: .
|
||||
run: |
|
||||
# always() so containers don't leak when migrations or tests
|
||||
# fail. The cleanup is best-effort: if the container is
|
||||
# already gone (e.g. concurrent rerun race), don't fail the job.
|
||||
docker rm -f "${PG_NAME}" >/dev/null 2>&1 || true
|
||||
echo "Cleaned up ${PG_NAME}"
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
name: Harness Replays
|
||||
|
||||
# Boots tests/harness (production-shape compose topology with TenantGuard,
|
||||
# /cp/* proxy, canvas proxy, real production Dockerfile.tenant) and runs
|
||||
# every replay under tests/harness/replays/. Fails the PR if any replay
|
||||
# fails.
|
||||
#
|
||||
# Why this exists: 2026-04-30 we shipped #2398 which added /buildinfo as
|
||||
# a public route in router.go but forgot to add it to TenantGuard's
|
||||
# allowlist. The handler-level test in buildinfo_test.go constructed a
|
||||
# minimal gin engine without TenantGuard — green. The harness's
|
||||
# buildinfo-stale-image.sh replay would have caught it (cf-proxy doesn't
|
||||
# inject X-Molecule-Org-Id, so the curl path is identical to production's
|
||||
# redeploy verifier), but no one ran the harness pre-merge. The bug
|
||||
# shipped; the redeploy verifier silently soft-warned every tenant as
|
||||
# "unreachable" for ~1 day before being noticed.
|
||||
#
|
||||
# This gate makes "did you actually run the harness?" a CI invariant
|
||||
# instead of a memory-discipline thing.
|
||||
#
|
||||
# Trigger model — match e2e-api.yml: always FIRES on push/pull_request
|
||||
# to staging+main, real work is gated per-step on detect-changes output.
|
||||
# One job → one check run → branch-protection-clean (the SKIPPED-in-set
|
||||
# trap from PR #2264 is documented in e2e-api.yml's e2e-api job comment).
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.github/workflows/harness-replays.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'tests/harness/**'
|
||||
- '.github/workflows/harness-replays.yml'
|
||||
workflow_dispatch:
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
# Per-SHA grouping. Per-ref kept hitting the auto-promote-staging
|
||||
# cancellation deadlock — see e2e-api.yml's concurrency block for
|
||||
# the 2026-04-28 incident that codified this pattern.
|
||||
group: harness-replays-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
run: ${{ steps.decide.outputs.run }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- id: decide
|
||||
run: |
|
||||
# workflow_dispatch: always run (manual trigger)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=manual-trigger" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Determine the base commit to diff against.
|
||||
# For pull_request: use base.sha (the merge-base with main/staging).
|
||||
# For push: use github.event.before (the previous tip of the branch).
|
||||
# Fallback for new branches (all-zeros SHA): run everything.
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && \
|
||||
[ -n "${{ github.event.pull_request.base.sha }}" ]; then
|
||||
BASE="${{ github.event.pull_request.base.sha }}"
|
||||
elif [ -n "${{ github.event.before }}" ] && \
|
||||
! echo "${{ github.event.before }}" | grep -qE '^0+$'; then
|
||||
BASE="${{ github.event.before }}"
|
||||
else
|
||||
# New branch or github.event.before unavailable — run everything.
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
echo "debug=new-branch-fallback" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# GitHub Actions and Gitea Actions both expose github.sha for HEAD.
|
||||
DIFF=$(git diff --name-only "$BASE" "${{ github.sha }}" 2>/dev/null)
|
||||
echo "debug=diff-base=$BASE diff-files=$DIFF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if echo "$DIFF" | grep -qE '^workspace-server/|^canvas/|^tests/harness/|^.github/workflows/harness-replays\.yml$'; then
|
||||
echo "run=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "run=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job that always runs. Real work is gated per-step on
|
||||
# detect-changes.outputs.run so an unrelated PR (e.g. doc-only
|
||||
# change to molecule-controlplane wired here later) emits the
|
||||
# required check without spending CI cycles. Single-job pattern
|
||||
# matches e2e-api.yml — see that workflow's comment for why a
|
||||
# job-level `if: false` would block branch protection via the
|
||||
# SKIPPED-in-set bug.
|
||||
harness-replays:
|
||||
needs: detect-changes
|
||||
name: Harness Replays
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.run != 'true'
|
||||
run: |
|
||||
echo "No workspace-server / canvas / tests/harness / workflow changes — Harness Replays gate satisfied without running."
|
||||
echo "::notice::Harness Replays no-op pass (paths filter excluded this commit)."
|
||||
echo "::notice::Debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
- if: needs.detect-changes.outputs.run == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Log what files were detected so future failures include the diff.
|
||||
- name: Log detected changes
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: |
|
||||
echo "::notice::detect-changes debug: ${{ needs.detect-changes.outputs.debug }}"
|
||||
|
||||
# github-app-auth sibling-checkout removed 2026-05-07 (#157):
|
||||
# the plugin was dropped + Dockerfile.tenant no longer COPYs it.
|
||||
|
||||
# Pre-clone manifest deps before docker compose builds the tenant
|
||||
# image (Task #173 followup — same pattern as
|
||||
# publish-workspace-server-image.yml's "Pre-clone manifest deps"
|
||||
# step).
|
||||
#
|
||||
# Why pre-clone here too: tests/harness/compose.yml builds tenant-alpha
|
||||
# and tenant-beta from workspace-server/Dockerfile.tenant with
|
||||
# context=../.. (repo root). That Dockerfile expects
|
||||
# .tenant-bundle-deps/{workspace-configs-templates,org-templates,plugins}
|
||||
# to be present at build context root (post-#173 it COPYs from there
|
||||
# instead of running an in-image clone — the in-image clone failed
|
||||
# with "could not read Username for https://git.moleculesai.app"
|
||||
# because there's no auth path inside the build sandbox).
|
||||
#
|
||||
# Without this step harness-replays fails before any replay runs,
|
||||
# with `failed to calculate checksum of ref ...
|
||||
# "/.tenant-bundle-deps/plugins": not found`. Caught by run #892
|
||||
# (main, 2026-05-07T20:28:53Z) and run #964 (staging — same
|
||||
# symptom, different root cause: staging still has the in-image
|
||||
# clone path, hits the auth error directly).
|
||||
#
|
||||
# 2026-05-08 sub-finding (#192): the clone step ALSO fails when
|
||||
# any referenced workspace-template repo is private and the
|
||||
# AUTO_SYNC_TOKEN bearer (devops-engineer persona) lacks read
|
||||
# access. Root cause: 5 of 9 workspace-template repos
|
||||
# (openclaw, codex, crewai, deepagents, gemini-cli) had been
|
||||
# marked private with no team grant. Resolution: flipped them
|
||||
# to public per `feedback_oss_first_repo_visibility_default`
|
||||
# (the OSS surface should be public). Layer-3 (customer-private +
|
||||
# marketplace third-party repos) tracked separately in
|
||||
# internal#102.
|
||||
#
|
||||
# Token shape matches publish-workspace-server-image.yml: AUTO_SYNC_TOKEN
|
||||
# is the devops-engineer persona PAT, NOT the founder PAT (per
|
||||
# `feedback_per_agent_gitea_identity_default`). clone-manifest.sh
|
||||
# embeds it as basic-auth for the duration of the clones and strips
|
||||
# .git directories — the token never enters the resulting image.
|
||||
- name: Pre-clone manifest deps
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
env:
|
||||
MOLECULE_GITEA_TOKEN: ${{ secrets.AUTO_SYNC_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${MOLECULE_GITEA_TOKEN}" ]; then
|
||||
echo "::error::AUTO_SYNC_TOKEN secret is empty — register the devops-engineer persona PAT in repo Actions secrets"
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p .tenant-bundle-deps
|
||||
bash scripts/clone-manifest.sh \
|
||||
manifest.json \
|
||||
.tenant-bundle-deps/workspace-configs-templates \
|
||||
.tenant-bundle-deps/org-templates \
|
||||
.tenant-bundle-deps/plugins
|
||||
# Sanity-check counts so a silent partial clone fails fast
|
||||
# instead of producing a half-empty image.
|
||||
ws_count=$(find .tenant-bundle-deps/workspace-configs-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
org_count=$(find .tenant-bundle-deps/org-templates -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
plugins_count=$(find .tenant-bundle-deps/plugins -mindepth 1 -maxdepth 1 -type d | wc -l)
|
||||
echo "Cloned: ws=$ws_count org=$org_count plugins=$plugins_count"
|
||||
|
||||
- name: Install Python deps for replays
|
||||
# peer-discovery-404 (and future replays) eval Python against the
|
||||
# running tenant — importing workspace/a2a_client.py pulls in
|
||||
# httpx. tests/harness/requirements.txt holds just the HTTP-client
|
||||
# surface to keep CI install fast (~3s) vs the full
|
||||
# workspace/requirements.txt (~30s).
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
run: pip install -r tests/harness/requirements.txt
|
||||
|
||||
- name: Run all replays against the harness
|
||||
# run-all-replays.sh: boot via up.sh → seed via seed.sh → run
|
||||
# every replays/*.sh → tear down via down.sh on EXIT (trap).
|
||||
# Non-zero exit on any replay failure.
|
||||
#
|
||||
# KEEP_UP=1: without this, the script's trap-on-EXIT tears
|
||||
# down containers immediately on failure, leaving the dump
|
||||
# step below with nothing to dump (verified on PR #2410's
|
||||
# first run — tenant became unhealthy, trap fired, dump
|
||||
# step saw empty containers). Keeping them up lets the
|
||||
# failure path collect tenant/cp-stub/cf-proxy logs. The
|
||||
# always-run "Force teardown" step does the actual cleanup.
|
||||
if: needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
KEEP_UP: "1"
|
||||
run: ./run-all-replays.sh
|
||||
|
||||
- name: Dump compose logs on failure
|
||||
# SECRETS_ENCRYPTION_KEY: docker compose validates the entire compose
|
||||
# file even for read-only `logs` calls. up.sh generates a per-run key
|
||||
# and exports it to its OWN shell — this step runs in a fresh shell
|
||||
# that wouldn't see it, so without a placeholder the validate step
|
||||
# errors before logs print (verified against PR #2492's first run:
|
||||
# "required variable SECRETS_ENCRYPTION_KEY is missing a value").
|
||||
# A placeholder is fine — we're only reading log streams, not booting.
|
||||
if: failure() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
env:
|
||||
SECRETS_ENCRYPTION_KEY: dump-logs-placeholder
|
||||
run: |
|
||||
echo "=== docker compose ps ==="
|
||||
docker compose -f compose.yml ps || true
|
||||
echo "=== tenant-alpha logs ==="
|
||||
docker compose -f compose.yml logs tenant-alpha || true
|
||||
echo "=== tenant-beta logs ==="
|
||||
docker compose -f compose.yml logs tenant-beta || true
|
||||
echo "=== cp-stub logs ==="
|
||||
docker compose -f compose.yml logs cp-stub || true
|
||||
echo "=== cf-proxy logs ==="
|
||||
docker compose -f compose.yml logs cf-proxy || true
|
||||
echo "=== postgres-alpha logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-alpha || true
|
||||
echo "=== postgres-beta logs (last 100) ==="
|
||||
docker compose -f compose.yml logs --tail 100 postgres-beta || true
|
||||
|
||||
- name: Force teardown
|
||||
# We pass KEEP_UP=1 to run-all-replays.sh so the dump step
|
||||
# above sees real containers — that means we own teardown
|
||||
# explicitly here. Always run.
|
||||
if: always() && needs.detect-changes.outputs.run == 'true'
|
||||
working-directory: tests/harness
|
||||
run: ./down.sh || true
|
||||
@@ -1,94 +0,0 @@
|
||||
name: Lint curl status-code capture
|
||||
|
||||
# Pins the workflow-bash anti-pattern that produced "HTTP 000000" on the
|
||||
# 2026-05-04 redeploy-tenants-on-main run for sha 2b862f6:
|
||||
#
|
||||
# HTTP_CODE=$(curl ... -w '%{http_code}' ... || echo "000")
|
||||
#
|
||||
# When curl exits non-zero (connection reset → 56, --fail-with-body 4xx/5xx
|
||||
# → 22), the `-w '%{http_code}'` already wrote a status to stdout — usually
|
||||
# "000" for connection failures or the actual code for HTTP errors. The
|
||||
# `|| echo "000"` then fires AND appends ANOTHER "000" to the captured
|
||||
# stdout, producing values like "000000" or "409000" that fail string
|
||||
# comparisons against "200" while looking superficially right.
|
||||
#
|
||||
# Same class of bug the synth-E2E §7c gate hit twice (PRs #2779/#2783 +
|
||||
# #2797). Memory: feedback_curl_status_capture_pollution.md.
|
||||
#
|
||||
# Fix shape (route -w into a tempfile so curl's exit code can't pollute):
|
||||
#
|
||||
# set +e
|
||||
# curl ... -w '%{http_code}' >code.txt 2>/dev/null
|
||||
# set -e
|
||||
# HTTP_CODE=$(cat code.txt 2>/dev/null)
|
||||
# [ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths: ['.github/workflows/**']
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths: ['.github/workflows/**']
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
name: Scan workflows for curl status-capture pollution
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Find curl ... -w '%{http_code}' ... || echo "000" subshells
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Multi-line aware: look for `$(curl ... -w '%{http_code}' ... || echo "000")`
|
||||
# subshell where the entire command-substitution wraps a curl that
|
||||
# ends with `|| echo "000"`. Must distinguish from the SAFE shape
|
||||
# `$(cat tempfile 2>/dev/null || echo "000")` — `cat` with a missing
|
||||
# tempfile produces empty stdout, no pollution.
|
||||
python3 <<'PY'
|
||||
import os, re, sys, glob
|
||||
|
||||
BAD_FILES = []
|
||||
|
||||
# Match the buggy substitution across newlines: $(curl ... -w '%{http_code}' ... || echo "000")
|
||||
# The `\\n` is the bash line-continuation that lets curl flags span lines.
|
||||
# We collapse continuation lines first, then look for the single-line bad pattern.
|
||||
PATTERN = re.compile(
|
||||
r'\$\(\s*curl\b[^)]*-w\s*[\'"]%\{http_code\}[\'"][^)]*\|\|\s*echo\s+"000"\s*\)',
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Self-skip: this lint workflow contains the literal anti-pattern in
|
||||
# its own docstring — that's intentional, not a bug.
|
||||
SELF = ".github/workflows/lint-curl-status-capture.yml"
|
||||
|
||||
for f in sorted(glob.glob(".github/workflows/*.yml")):
|
||||
if f == SELF:
|
||||
continue
|
||||
with open(f) as fh:
|
||||
content = fh.read()
|
||||
# Collapse bash line-continuations (\\\n + leading whitespace)
|
||||
# into a single logical line so the regex can see the full
|
||||
# curl invocation as one chunk.
|
||||
flat = re.sub(r'\\\s*\n\s*', ' ', content)
|
||||
for m in PATTERN.finditer(flat):
|
||||
BAD_FILES.append((f, m.group(0)[:120]))
|
||||
|
||||
if not BAD_FILES:
|
||||
print("✓ No curl-status-capture pollution patterns detected")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"::error::Found {len(BAD_FILES)} curl-status-capture pollution site(s):")
|
||||
for f, snippet in BAD_FILES:
|
||||
print(f"::error file={f}::Curl status-capture pollution: '|| echo \"000\"' inside a $(curl ... -w '%{{http_code}}' ...) subshell. On non-2xx or connection failure, curl's -w writes a status, then exits non-zero, then the || echo appends another '000' — producing 'HTTP 000000' or '409000' that fails comparisons silently. Fix: route -w into a tempfile so the exit code can't pollute stdout. See memory feedback_curl_status_capture_pollution.md.")
|
||||
print(f" matched: {snippet}…")
|
||||
print()
|
||||
print("Fix template:")
|
||||
print(' set +e')
|
||||
print(' curl ... -w \'%{http_code}\' >code.txt 2>/dev/null')
|
||||
print(' set -e')
|
||||
print(' HTTP_CODE=$(cat code.txt 2>/dev/null)')
|
||||
print(' [ -z "$HTTP_CODE" ] && HTTP_CODE="000"')
|
||||
sys.exit(1)
|
||||
PY
|
||||
@@ -0,0 +1,100 @@
|
||||
name: promote-latest
|
||||
|
||||
# Manually retag ghcr.io/molecule-ai/platform:staging-<sha> → :latest
|
||||
# (and the same for the tenant image). Use this to:
|
||||
#
|
||||
# 1. Promote a :staging-<sha> to prod before the canary fleet is live
|
||||
# (one-off during the initial rollout).
|
||||
# 2. Roll back :latest to a prior known-good digest after a bad
|
||||
# promotion slipped past canary (use scripts/rollback-latest.sh
|
||||
# for a local / emergency path; this workflow is for scheduled
|
||||
# or from-browser promotions).
|
||||
#
|
||||
# Running this workflow needs no extra secrets — GitHub's default
|
||||
# GITHUB_TOKEN has write:packages for repo-owned GHCR images, which
|
||||
# is all we need for a remote retag via `crane tag`.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sha:
|
||||
description: 'Short sha to promote (e.g. 4c1d56e). Must match an existing :staging-<sha> tag.'
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
promote:
|
||||
# Self-hosted mac mini — GitHub-hosted minutes are currently quota-
|
||||
# blocked. mac mini already has crane available via homebrew.
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
steps:
|
||||
- name: Ensure crane installed
|
||||
# HOMEBREW_NO_INSTALL_CLEANUP + HOMEBREW_NO_AUTO_UPDATE stop
|
||||
# brew from touching unrelated symlinks in /opt/homebrew owned
|
||||
# by other users on this shared runner — cleanup was exiting
|
||||
# non-zero even though crane itself installed successfully.
|
||||
env:
|
||||
HOMEBREW_NO_INSTALL_CLEANUP: "1"
|
||||
HOMEBREW_NO_AUTO_UPDATE: "1"
|
||||
HOMEBREW_NO_ENV_HINTS: "1"
|
||||
run: |
|
||||
if ! command -v crane >/dev/null 2>&1; then
|
||||
brew install crane
|
||||
fi
|
||||
crane version
|
||||
|
||||
- name: GHCR login
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" \
|
||||
| crane auth login ghcr.io -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Retag platform image
|
||||
run: |
|
||||
set -eu
|
||||
SRC="${IMAGE_NAME}:staging-${{ inputs.sha }}"
|
||||
if ! crane digest "$SRC" >/dev/null 2>&1; then
|
||||
echo "::error::$SRC not found in registry — double-check the sha."
|
||||
exit 1
|
||||
fi
|
||||
EXPECTED=$(crane digest "$SRC")
|
||||
crane tag "$SRC" latest
|
||||
ACTUAL=$(crane digest "${IMAGE_NAME}:latest")
|
||||
if [ "$ACTUAL" != "$EXPECTED" ]; then
|
||||
echo "::error::retag digest mismatch (expected $EXPECTED, got $ACTUAL)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK ${IMAGE_NAME}:latest → $ACTUAL"
|
||||
|
||||
- name: Retag tenant image
|
||||
run: |
|
||||
set -eu
|
||||
SRC="${TENANT_IMAGE_NAME}:staging-${{ inputs.sha }}"
|
||||
if ! crane digest "$SRC" >/dev/null 2>&1; then
|
||||
echo "::error::$SRC not found — tenant image may not have built for this sha."
|
||||
exit 1
|
||||
fi
|
||||
EXPECTED=$(crane digest "$SRC")
|
||||
crane tag "$SRC" latest
|
||||
ACTUAL=$(crane digest "${TENANT_IMAGE_NAME}:latest")
|
||||
if [ "$ACTUAL" != "$EXPECTED" ]; then
|
||||
echo "::error::tenant retag digest mismatch"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK ${TENANT_IMAGE_NAME}:latest → $ACTUAL"
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## :latest promoted to staging-${{ inputs.sha }}"
|
||||
echo
|
||||
echo "Both platform + tenant images retagged. Prod tenants"
|
||||
echo "will auto-pull within their 5-min update cycle."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -39,36 +39,56 @@ env:
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build & push canvas image
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
- name: Configure GHCR auth (write auths map; do NOT call docker login)
|
||||
# `docker login` on macOS unconditionally writes credentials to the
|
||||
# osxkeychain credential helper, even when DOCKER_CONFIG/config.json
|
||||
# declares `credsStore: ""` and even when invoked with `--config`.
|
||||
# Verified locally 2026-04-16 — after a successful login, Docker
|
||||
# rewrites the same config file to:
|
||||
# { "auths": { "ghcr.io": {} }, "credsStore": "osxkeychain" }
|
||||
# i.e. the auth lives in the Keychain, not the config file. The
|
||||
# Mac mini runner is a launchd user agent with a locked Keychain,
|
||||
# so storage fails with `User interaction is not allowed (-25308)`.
|
||||
#
|
||||
# Six prior PRs (#273, #319, #322, #341, #484, #486) all kept calling
|
||||
# `docker login` and tried to coerce credsStore — none worked.
|
||||
# The only reliable fix is to skip `docker login` entirely and write
|
||||
# the auth string directly. `docker/build-push-action@v6` and the
|
||||
# daemon honor the `auths` map for push without needing login.
|
||||
shell: bash
|
||||
env:
|
||||
GHCR_USER: ${{ github.actor }}
|
||||
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p "${RUNNER_TEMP}/docker-config"
|
||||
AUTH=$(printf '%s:%s' "${GHCR_USER}" "${GHCR_TOKEN}" | base64)
|
||||
umask 077
|
||||
cat > "${RUNNER_TEMP}/docker-config/config.json" <<JSON
|
||||
{ "auths": { "ghcr.io": { "auth": "${AUTH}" } } }
|
||||
JSON
|
||||
echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config" >> "${GITHUB_ENV}"
|
||||
# Diagnostics that don't leak the token.
|
||||
echo "=== docker ==="
|
||||
command -v docker || echo "(docker not in PATH)"
|
||||
docker --version 2>&1 || true
|
||||
ls -la /usr/local/bin/docker /opt/homebrew/bin/docker 2>&1 || true
|
||||
echo "=== auths registries (no values) ==="
|
||||
grep -o '"[a-zA-Z0-9.-]*\.io"' "${RUNNER_TEMP}/docker-config/config.json" || true
|
||||
|
||||
- name: Set up QEMU
|
||||
# Apple-silicon runner building linux/amd64 images for x86 hosts.
|
||||
uses: docker/setup-qemu-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
# Health check: verify Docker daemon is accessible before attempting any
|
||||
# build steps. This fails loudly at step 1 when the runner's docker.sock
|
||||
# is inaccessible rather than silently continuing to the build step
|
||||
# where docker build fails deep in ECR auth with a cryptic error.
|
||||
- name: Verify Docker daemon access
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "::group::Docker daemon health check"
|
||||
docker info 2>&1 | head -5 || {
|
||||
echo "::error::Docker daemon is not accessible at /var/run/docker.sock"
|
||||
echo "::error::Check: (1) daemon running, (2) runner user in docker group, (3) sock perms 660+"
|
||||
exit 1
|
||||
}
|
||||
echo "Docker daemon OK"
|
||||
echo "::endgroup::"
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Compute tags
|
||||
id: tags
|
||||
@@ -101,7 +121,7 @@ jobs:
|
||||
echo "ws_url=${WS_URL}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build & push canvas image to GHCR
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./canvas
|
||||
file: ./canvas/Dockerfile
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
name: publish-workspace-server-image
|
||||
|
||||
# Builds and pushes Docker images to GHCR when staging is promoted to main.
|
||||
# PRs target staging (default branch). Only main push triggers production builds.
|
||||
# EC2 tenant instances pull the tenant image from GHCR.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'workspace-server/**'
|
||||
- 'canvas/**'
|
||||
- 'manifest.json'
|
||||
- '.github/workflows/publish-platform-image.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ghcr.io/molecule-ai/platform
|
||||
TENANT_IMAGE_NAME: ghcr.io/molecule-ai/platform-tenant
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: [self-hosted, macos, arm64]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Checkout sibling plugin repo
|
||||
# workspace-server/Dockerfile expects
|
||||
# ./molecule-ai-plugin-github-app-auth at build-context root because
|
||||
# the Go module has a `replace` directive pointing at /plugin inside
|
||||
# the image. Pre-repo-split the plugin lived in the monorepo; the
|
||||
# 2026-04-18 restructure moved it out but didn't add this clone step
|
||||
# — which is why publish has been failing since then.
|
||||
#
|
||||
# Uses a fine-grained PAT (PLUGIN_REPO_PAT) because the plugin repo
|
||||
# is private and the default GITHUB_TOKEN is scoped to THIS repo.
|
||||
# The PAT needs Contents:Read on Molecule-AI/molecule-ai-plugin-
|
||||
# github-app-auth. Falls back to the default token for the (rare)
|
||||
# case where an operator made the plugin repo public.
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: Molecule-AI/molecule-ai-plugin-github-app-auth
|
||||
path: molecule-ai-plugin-github-app-auth
|
||||
token: ${{ secrets.PLUGIN_REPO_PAT || secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Configure GHCR auth
|
||||
shell: bash
|
||||
env:
|
||||
GHCR_USER: ${{ github.actor }}
|
||||
GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p "${RUNNER_TEMP}/docker-config"
|
||||
GHCR_AUTH=$(printf '%s:%s' "${GHCR_USER}" "${GHCR_TOKEN}" | base64)
|
||||
umask 077
|
||||
printf '{"auths":{"ghcr.io":{"auth":"%s"}}}' "${GHCR_AUTH}" > "${RUNNER_TEMP}/docker-config/config.json"
|
||||
echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
with:
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Compute tags
|
||||
id: tags
|
||||
run: |
|
||||
echo "sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Canary-gated release: we publish :staging-<sha> ONLY here. The
|
||||
# :latest tag (which existing prod tenants auto-pull every 5 min)
|
||||
# is promoted by .github/workflows/canary-verify.yml after the
|
||||
# staging canary fleet green-lights this digest.
|
||||
# That means:
|
||||
# - Every main merge produces a :staging-<sha> image
|
||||
# - Canary tenants (configured to pull :staging-<sha>) pick it up
|
||||
# - canary-verify.yml runs smoke tests against them
|
||||
# - On green → canary-verify retags :staging-<sha> → :latest
|
||||
# - On red → :latest stays on the prior good digest, prod is safe
|
||||
- name: Build & push platform image to GHCR (staging-<sha> only)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./workspace-server/Dockerfile
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.IMAGE_NAME }}:staging-${{ steps.tags.outputs.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI platform (Go API server) — pending canary verify
|
||||
|
||||
- name: Build & push tenant image to GHCR (staging-<sha> only)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./workspace-server/Dockerfile.tenant
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: |
|
||||
${{ env.TENANT_IMAGE_NAME }}:staging-${{ steps.tags.outputs.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# Canvas uses same-origin fetches. The tenant Go platform
|
||||
# reverse-proxies /cp/* to the SaaS CP via its CP_UPSTREAM_URL
|
||||
# env; the tenant's /canvas/viewport, /approvals/pending,
|
||||
# /org/templates etc. live on the tenant platform itself.
|
||||
# Both legs share one origin (the tenant subdomain) so
|
||||
# PLATFORM_URL="" forces canvas to fetch paths as relative,
|
||||
# which land same-origin.
|
||||
#
|
||||
# Self-hosted / private-label deployments override this at
|
||||
# build time with a specific backend (e.g. local dev:
|
||||
# NEXT_PUBLIC_PLATFORM_URL=http://localhost:8080).
|
||||
build-args: |
|
||||
NEXT_PUBLIC_PLATFORM_URL=
|
||||
labels: |
|
||||
org.opencontainers.image.source=https://github.com/${{ github.repository }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
org.opencontainers.image.description=Molecule AI tenant platform + canvas — pending canary verify
|
||||
@@ -1,207 +0,0 @@
|
||||
name: Railway pin audit (drift detection)
|
||||
|
||||
# Daily audit of Railway env vars for drift-prone image-tag pins —
|
||||
# automation-cadence layer over the detection script + regression test
|
||||
# shipped in PR #2168 (#2001 closure).
|
||||
#
|
||||
# Background: on 2026-04-24 a stale `:staging-a14cf86` SHA pin in CP's
|
||||
# TENANT_IMAGE caused 3+ hours of E2E failure with the appearance that
|
||||
# "every fix didn't propagate" — really the tenant image was so old it
|
||||
# didn't read the env vars those fixes produced. The audit script
|
||||
# (scripts/ops/audit-railway-sha-pins.sh) flags drift; this workflow
|
||||
# runs the same check unattended on a daily cron.
|
||||
#
|
||||
# Cadence: once a day, 13:00 UTC (06:00 PT). Daily is the right
|
||||
# cadence for variables-tier config — Railway env var changes are
|
||||
# deliberate operator actions, low-frequency. Hourly would risk
|
||||
# Railway API rate-limit surprises and is overkill for the change rate.
|
||||
#
|
||||
# Issue-on-failure: drift triggers a priority-high issue, mirroring
|
||||
# .github/workflows/e2e-staging-sanity.yml's pattern. Drift is
|
||||
# medium-priority "config slipped, fix at next ops window," not
|
||||
# active-outage paging.
|
||||
#
|
||||
# Secret hardening: per feedback_schedule_vs_dispatch_secrets_hardening,
|
||||
# the schedule trigger HARD-FAILS on missing RAILWAY_AUDIT_TOKEN
|
||||
# (silent-success on schedule was the failure-mode class that bit the
|
||||
# team before; cron firing without checking anything is worse than no
|
||||
# cron). The workflow_dispatch trigger SOFT-SKIPS on missing secret so
|
||||
# an operator can dry-run the workflow shape during initial provisioning
|
||||
# without tripping a fake red.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 13 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: railway-pin-audit
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
name: Audit Railway env vars for drift-prone pins
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify RAILWAY_AUDIT_TOKEN present
|
||||
# Schedule trigger: hard-fail when the secret is missing —
|
||||
# otherwise the cron silently runs against the wrong scope (or
|
||||
# exits 2 from the script and we issue-spam) without anyone
|
||||
# noticing the token rot.
|
||||
# Dispatch trigger: soft-skip — operator may be dry-running the
|
||||
# workflow shape before provisioning the secret. Logged as a
|
||||
# workflow notice, not a failure.
|
||||
env:
|
||||
RAILWAY_AUDIT_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
id: secret_check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${RAILWAY_AUDIT_TOKEN:-}" ]; then
|
||||
echo "have_secret=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "have_secret=false" >> "$GITHUB_OUTPUT"
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
echo "::notice::RAILWAY_AUDIT_TOKEN not configured — soft-skipping (manual dispatch)"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::RAILWAY_AUDIT_TOKEN secret missing — schedule trigger requires it. Provision the token (read-only \`variables\` scope on the molecule-platform Railway project) and store as repo secret RAILWAY_AUDIT_TOKEN."
|
||||
exit 1
|
||||
|
||||
- name: Install Railway CLI
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
# Pinned hash matching the public install instructions; bump in
|
||||
# tandem with the audit-script's documented Railway CLI version.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -fsSL https://railway.com/install.sh | sh
|
||||
# The installer drops the binary in ~/.railway/bin
|
||||
echo "$HOME/.railway/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Verify Railway CLI authenticated
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# `railway whoami` exits non-zero when the token is
|
||||
# unauthenticated or doesn't have any project access.
|
||||
if ! railway whoami >/dev/null 2>&1; then
|
||||
echo "::error::Railway CLI failed to authenticate with RAILWAY_AUDIT_TOKEN — token may be revoked or scoped incorrectly"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
- name: Link molecule-platform project
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
# Project ID from reference_production_stack: molecule-platform
|
||||
# / 7ccc8c68-61f4-42ab-9be5-586eeee11768. Linking is per-process,
|
||||
# so we re-link in this CI shell (the audit script comment says
|
||||
# it deliberately doesn't chdir for you because the linked
|
||||
# project's identity matters).
|
||||
run: |
|
||||
set -euo pipefail
|
||||
railway link --project 7ccc8c68-61f4-42ab-9be5-586eeee11768
|
||||
|
||||
- name: Run drift audit
|
||||
if: steps.secret_check.outputs.have_secret == 'true'
|
||||
id: audit
|
||||
env:
|
||||
RAILWAY_TOKEN: ${{ secrets.RAILWAY_AUDIT_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
bash scripts/ops/audit-railway-sha-pins.sh 2>&1 | tee /tmp/audit.log
|
||||
rc=${PIPESTATUS[0]}
|
||||
echo "rc=$rc" >> "$GITHUB_OUTPUT"
|
||||
# Capture the audit log for the issue body.
|
||||
{
|
||||
echo 'log<<AUDIT_EOF'
|
||||
cat /tmp/audit.log
|
||||
echo 'AUDIT_EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
# Exit codes from the script:
|
||||
# 0 — no drift; workflow goes green
|
||||
# 1 — drift detected; we'll file an issue and fail the run
|
||||
# 2 — railway CLI unauthenticated / project unlinked; fail
|
||||
# Anything else: also fail.
|
||||
case "$rc" in
|
||||
0) exit 0 ;;
|
||||
1) echo "::warning::Drift-prone pin(s) detected — issue will be filed"; exit 1 ;;
|
||||
2) echo "::error::Railway CLI auth/link failed mid-script — token or project ID drift"; exit 2 ;;
|
||||
*) echo "::error::Unexpected audit rc=$rc"; exit 1 ;;
|
||||
esac
|
||||
|
||||
- name: Open / update drift issue
|
||||
if: failure() && steps.audit.outputs.rc == '1'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
AUDIT_LOG: ${{ steps.audit.outputs.log }}
|
||||
with:
|
||||
script: |
|
||||
const title = "🚨 Railway env-var drift detected";
|
||||
const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const body =
|
||||
`Daily Railway pin audit found drift-prone image-tag pins in the molecule-platform Railway project.\n\n` +
|
||||
`**What this means:** an env var (likely on \`controlplane\`) is pinned to a SHA-shaped or semver tag instead of a floating tag. ` +
|
||||
`Same pattern that caused the 2026-04-24 TENANT_IMAGE incident — fix-PRs land but the running service doesn't pick them up.\n\n` +
|
||||
`**Recovery:** open the Railway dashboard, replace the flagged value with a floating tag (\`:staging-latest\`, \`:main\`) unless the pin is intentional and documented in the ops runbook.\n\n` +
|
||||
`**Audit output:**\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\`\n\n` +
|
||||
`Run: ${runURL}\n\n` +
|
||||
`Closes automatically when a subsequent daily run reports clean.`;
|
||||
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'railway-drift',
|
||||
});
|
||||
const match = existing.find(i => i.title === title);
|
||||
if (match) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: match.number,
|
||||
body: `Still drifting. ${runURL}\n\n\`\`\`\n${process.env.AUDIT_LOG || '(log unavailable)'}\n\`\`\``,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
title, body,
|
||||
labels: ['railway-drift', 'bug', 'priority-high'],
|
||||
});
|
||||
}
|
||||
|
||||
- name: Close stale drift issue on clean run
|
||||
# When a previously-flagged drift gets fixed by an operator,
|
||||
# the next daily run goes green. Close any open `railway-drift`
|
||||
# issue with a confirmation comment so the queue doesn't carry
|
||||
# stale ones.
|
||||
if: success() && steps.audit.outputs.rc == '0'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const runURL = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const { data: existing } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
state: 'open', labels: 'railway-drift',
|
||||
});
|
||||
for (const issue of existing) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
body: `Daily audit clean — drift resolved. ${runURL}`,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner, repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
state: 'closed',
|
||||
state_reason: 'completed',
|
||||
});
|
||||
}
|
||||
@@ -1,400 +0,0 @@
|
||||
name: redeploy-tenants-on-main
|
||||
|
||||
# Auto-refresh prod tenant EC2s after every main merge.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image builds and
|
||||
# pushes a new platform-tenant :<sha> to ECR on every merge to main,
|
||||
# but running tenants pulled their image once at boot and never re-pull.
|
||||
# Users see stale code indefinitely.
|
||||
#
|
||||
# This workflow closes the gap by calling the control-plane admin
|
||||
# endpoint that performs a canary-first, batched, health-gated rolling
|
||||
# redeploy across every live tenant. Implemented in molecule-ai/
|
||||
# molecule-controlplane as POST /cp/admin/tenants/redeploy-fleet
|
||||
# (feat/tenant-auto-redeploy, landing alongside this workflow).
|
||||
#
|
||||
# Registry: ECR (153263036946.dkr.ecr.us-east-2.amazonaws.com/
|
||||
# molecule-ai/platform-tenant). GHCR was retired 2026-05-07 during the
|
||||
# Gitea suspension migration. The canary-verify.yml promote step now
|
||||
# uses the same redeploy-fleet endpoint (fixes the silent-GHCR gap).
|
||||
#
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes → new :staging-<sha> in ECR.
|
||||
# 2. This workflow fires via workflow_run, calls redeploy-fleet with
|
||||
# target_tag=staging-<sha>. No CDN propagation wait needed —
|
||||
# ECR image manifest is consistent immediately after push.
|
||||
# 3. Calls redeploy-fleet with canary_slug (if set) and a soak
|
||||
# period. Canary proves the image boots; batches follow.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: re-run this workflow with a specific SHA pinned via
|
||||
# the workflow_dispatch input. That calls redeploy-fleet with
|
||||
# target_tag=<sha>, re-pulling the older image on every tenant.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target_tag:
|
||||
# Empty default → auto-trigger and dispatch-without-input both
|
||||
# resolve to `staging-<short_head_sha>` (the digest publish-image
|
||||
# just pushed). Pre-fix this defaulted to 'latest', which only
|
||||
# gets retagged by canary-verify's promote-to-latest job — and
|
||||
# that job soft-skips when CANARY_TENANT_URLS is unset (the
|
||||
# current state, until Phase 2 canary fleet is live). Result:
|
||||
# `:latest` had been pinned to a 4-day-old digest (2026-04-28)
|
||||
# while every main push pushed fresh `staging-<sha>` images;
|
||||
# every prod redeploy pulled the stale `:latest` and the verify
|
||||
# step correctly flagged 3/3 tenants STALE. Pulling the
|
||||
# just-published `staging-<sha>` directly skips the dead retag
|
||||
# path. When canary fleet is real, this workflow should chain
|
||||
# on canary-verify completion (workflow_run from canary-verify),
|
||||
# not publish-image — separate, smaller PR.
|
||||
description: 'Tenant image tag to deploy (e.g. "latest", "staging-a59f1a6c"). Empty = auto staging-<head_sha>.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
canary_slug:
|
||||
description: 'Tenant slug to deploy first + soak (empty = skip canary, fan out immediately).'
|
||||
required: false
|
||||
type: string
|
||||
# Must be an actual prod tenant slug (current: hongming,
|
||||
# chloe-dong, reno-stars). The previous default 'hongmingwang'
|
||||
# didn't match any tenant — CP soft-skipped the missing canary
|
||||
# and the fleet rolled out without the soak gate, defeating the
|
||||
# whole point of canary-first.
|
||||
default: 'hongming'
|
||||
soak_seconds:
|
||||
description: 'Seconds to wait after canary before fanning out.'
|
||||
required: false
|
||||
type: string
|
||||
default: '60'
|
||||
batch_size:
|
||||
description: 'How many tenants SSM redeploys in parallel per batch.'
|
||||
required: false
|
||||
type: string
|
||||
default: '3'
|
||||
dry_run:
|
||||
description: 'Plan only — do not actually redeploy.'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize redeploys so two rapid main pushes' redeploys don't overlap
|
||||
# and cause confusing per-tenant SSM state. Without this, GitHub's
|
||||
# implicit workflow_run queueing would *probably* serialize them, but
|
||||
# the explicit block makes the invariant defensible. Mirrors the
|
||||
# concurrency block on redeploy-tenants-on-staging.yml for shape parity.
|
||||
#
|
||||
# cancel-in-progress: false → aborting a half-rolled-out fleet would
|
||||
# leave tenants stuck on whatever image they happened to be on when
|
||||
# cancelled. Better to finish the in-flight rollout before starting
|
||||
# the next one.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-main
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
redeploy:
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
if: |
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Note on ECR propagation
|
||||
# ECR image manifests are consistent immediately after push — no
|
||||
# CDN cache to wait for. The old GHCR-based workflow had a 30s
|
||||
# sleep to avoid race conditions; ECR makes that unnecessary.
|
||||
run: echo "ECR image available immediately after push — proceeding."
|
||||
|
||||
- name: Compute target tag
|
||||
id: tag
|
||||
# Resolution order:
|
||||
# 1. Operator-supplied input (workflow_dispatch with explicit
|
||||
# tag) → used verbatim. Lets ops pin `latest` for emergency
|
||||
# rollback to last canary-verified digest, or pin a specific
|
||||
# `staging-<sha>` to roll back to a known-good build.
|
||||
# 2. Default → `staging-<short_head_sha>`. The just-published
|
||||
# digest. Bypasses the `:latest` retag path that's currently
|
||||
# dead (canary-verify soft-skips without canary fleet, so
|
||||
# the only thing retagging `:latest` today is the manual
|
||||
# promote-latest.yml — last run 2026-04-28). Auto-trigger
|
||||
# from workflow_run uses workflow_run.head_sha; manual
|
||||
# dispatch with no input falls through to github.sha.
|
||||
env:
|
||||
INPUT_TAG: ${{ inputs.target_tag }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -n "${INPUT_TAG:-}" ]; then
|
||||
echo "target_tag=$INPUT_TAG" >> "$GITHUB_OUTPUT"
|
||||
echo "Using operator-pinned tag: $INPUT_TAG"
|
||||
else
|
||||
SHORT="${HEAD_SHA:0:7}"
|
||||
echo "target_tag=staging-$SHORT" >> "$GITHUB_OUTPUT"
|
||||
echo "Using auto tag: staging-$SHORT (head_sha=$HEAD_SHA)"
|
||||
fi
|
||||
|
||||
- name: Call CP redeploy-fleet
|
||||
# CP_ADMIN_API_TOKEN must be set as a repo/org secret on
|
||||
# molecule-ai/molecule-core, matching the staging/prod CP's
|
||||
# CP_ADMIN_API_TOKEN env. Stored in Railway, mirrored to this
|
||||
# repo's secrets for CI.
|
||||
env:
|
||||
CP_URL: ${{ vars.CP_URL || 'https://api.moleculesai.app' }}
|
||||
CP_ADMIN_API_TOKEN: ${{ secrets.CP_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || 'hongming' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ -z "${CP_ADMIN_API_TOKEN:-}" ]; then
|
||||
echo "::error::CP_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::notice::Set CP_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
--argjson soak "$SOAK_SECONDS" \
|
||||
--argjson batch "$BATCH_SIZE" \
|
||||
--argjson dry "$DRY_RUN" \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
# Route -w into its own tempfile so curl's exit code (e.g. 56
|
||||
# on connection-reset, 22 on --fail-with-body 4xx/5xx) can't
|
||||
# pollute the captured stdout. The previous inline-substitution
|
||||
# shape produced "000000" on connection reset (curl wrote
|
||||
# "000" via -w, then the inline echo-fallback appended another
|
||||
# "000") — caught on the 2026-05-04 redeploy of sha 2b862f6.
|
||||
# set +e/-e keeps the non-zero curl exit from tripping the
|
||||
# outer pipeline. See lint-curl-status-capture.yml for the
|
||||
# CI gate that pins this fix shape.
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
set -e
|
||||
# Stderr from curl (e.g. dial errors with -sS) goes to the runner
|
||||
# log so operators can see WHY a connection failed. Stdout is
|
||||
# captured to $HTTP_CODE_FILE because that's where -w writes.
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
# Pretty-print per-tenant results in the job summary so
|
||||
# ops can see which tenants were redeployed without drilling
|
||||
# into the raw response.
|
||||
{
|
||||
echo "## Tenant redeploy fleet"
|
||||
echo ""
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**Canary:** \`$CANARY_SLUG\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "**Batch size:** $BATCH_SIZE"
|
||||
echo "**Dry run:** $DRY_RUN"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
exit 1
|
||||
fi
|
||||
OK=$(jq -r '.ok' "$HTTP_RESPONSE")
|
||||
if [ "$OK" != "true" ]; then
|
||||
echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
|
||||
|
||||
# Stash the response for the verify step. $RUNNER_TEMP outlasts
|
||||
# the step boundary; $HTTP_RESPONSE doesn't.
|
||||
cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
|
||||
|
||||
- name: Verify each tenant /buildinfo matches published SHA
|
||||
# ROOT FIX FOR #2395.
|
||||
#
|
||||
# `redeploy-fleet`'s `ssm_status=Success` means "the SSM RPC
|
||||
# didn't error" — NOT "the new image is running on the tenant."
|
||||
# `:latest` lives in the local Docker daemon's image cache; if
|
||||
# the SSM document does `docker compose up -d` without an
|
||||
# explicit `docker pull`, the daemon serves the previously-
|
||||
# cached digest and the container restarts on stale code.
|
||||
# 2026-04-30 incident: hongmingwang's tenant reported
|
||||
# ssm_status=Success at 17:00:53Z but kept serving pre-501a42d7
|
||||
# chat_files for 30+ min — the lazy-heal fix never reached the
|
||||
# user despite green deploy + green redeploy.
|
||||
#
|
||||
# This step closes the gap by curling each tenant's /buildinfo
|
||||
# endpoint (added in workspace-server/internal/buildinfo +
|
||||
# /Dockerfile* GIT_SHA build-arg, this PR) and comparing the
|
||||
# returned git_sha to the SHA the workflow expects. Mismatches
|
||||
# fail the workflow, which is what `ok=true` should have
|
||||
# guaranteed all along.
|
||||
#
|
||||
# When the redeploy was triggered by workflow_dispatch with a
|
||||
# specific tag (target_tag != "latest"), the expected SHA may
|
||||
# not equal ${{ github.sha }} — in that case we resolve via
|
||||
# GHCR's manifest. For workflow_run (default :latest) the
|
||||
# workflow_run.head_sha is the SHA that just published.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ steps.tag.outputs.target_tag }}
|
||||
# Tenant subdomain template — slugs from the response are
|
||||
# appended. Production CP issues `<slug>.moleculesai.app`;
|
||||
# staging CP issues `<slug>.staging.moleculesai.app`. This
|
||||
# workflow runs on main → prod CP → no `staging.` infix.
|
||||
TENANT_DOMAIN: 'moleculesai.app'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
EXPECTED_SHORT="${EXPECTED_SHA:0:7}"
|
||||
if [ "$TARGET_TAG" != "latest" ] \
|
||||
&& [ "$TARGET_TAG" != "$EXPECTED_SHA" ] \
|
||||
&& [ "$TARGET_TAG" != "staging-$EXPECTED_SHORT" ]; then
|
||||
# workflow_dispatch with a pinned tag that isn't the head
|
||||
# SHA — operator is rolling back / pinning. Skip the
|
||||
# verification because we don't have the expected SHA in
|
||||
# this context (would need to crane-inspect the GHCR
|
||||
# manifest, which is a follow-up). Failing-open here is
|
||||
# safe: the operator chose the tag deliberately.
|
||||
#
|
||||
# `staging-<short_head_sha>` IS verified — it's the new
|
||||
# auto-trigger default (see Compute target tag step) and
|
||||
# the digest under that tag SHOULD match EXPECTED_SHA.
|
||||
echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RESP="$RUNNER_TEMP/redeploy-response.json"
|
||||
if [ ! -s "$RESP" ]; then
|
||||
echo "::error::redeploy-response.json missing or empty — verify step ran without a response to read"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pull only successfully-redeployed tenants. Any tenant that
|
||||
# halted the rollout already failed the previous step, so we
|
||||
# don't double-count them here.
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::warning::No tenants reported healthz_ok — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Verifying ${#SLUGS[@]} tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
|
||||
|
||||
# Two distinct failure modes — STALE (the #2395 bug class, hard-fail)
|
||||
# vs UNREACHABLE (teardown race, soft-warn). See the staging variant's
|
||||
# comment for the full rationale; same logic applies on prod even
|
||||
# though prod has fewer ephemeral tenants — the asymmetry would be a
|
||||
# gratuitous fork.
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
STALE_LINES=()
|
||||
UNREACHABLE_LINES=()
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
# 30s total: tenant just SSM-restarted, may still be coming
|
||||
# up. Retry-on-empty rather than retry-on-status — we want
|
||||
# to fail fast on "responded with wrong SHA", not "still
|
||||
# warming up".
|
||||
BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
|
||||
ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
|
||||
if [ -z "$ACTUAL_SHA" ]; then
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
|
||||
continue
|
||||
fi
|
||||
if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
|
||||
echo " $slug: ${ACTUAL_SHA:0:7} ✓"
|
||||
else
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Per-tenant /buildinfo verification"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
|
||||
echo ""
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${STALE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely teardown race (soft-warn, not failing):**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
|
||||
echo "All ${#SLUGS[@]} tenants returned matching SHA. ✓"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "::warning::$UNREACHABLE_COUNT tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders sanity floor: same logic as the staging
|
||||
# variant — see that file's comment for the full rationale.
|
||||
# Floor only applies when fleet >= 4; below that, canary-verify
|
||||
# is the actual gate.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "::error::$STALE_COUNT tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::Tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
|
||||
@@ -1,362 +0,0 @@
|
||||
name: redeploy-tenants-on-staging
|
||||
|
||||
# Auto-refresh staging tenant EC2s after every staging-branch merge.
|
||||
#
|
||||
# Mirror of redeploy-tenants-on-main.yml, with the staging-CP host and
|
||||
# the :staging-latest tag. Sister workflow exists for prod (rolls
|
||||
# :latest after canary-verify). Both share the same shape — just
|
||||
# different CP_URL + target_tag + admin token secret.
|
||||
#
|
||||
# Why this workflow exists: publish-workspace-server-image now builds
|
||||
# on every staging-branch push (PR #2335), pushing
|
||||
# platform-tenant:staging-latest to GHCR. Existing tenants pulled
|
||||
# their image once at boot and never re-pull, so the new image just
|
||||
# sits unused until the tenant is reprovisioned.
|
||||
#
|
||||
# This workflow closes the gap by calling staging-CP's
|
||||
# /cp/admin/tenants/redeploy-fleet, which performs a canary-first,
|
||||
# batched, health-gated SSM redeploy across every live staging tenant.
|
||||
# Same endpoint shape as prod CP — only the host differs.
|
||||
#
|
||||
# Runtime ordering:
|
||||
# 1. publish-workspace-server-image completes on staging branch →
|
||||
# new :staging-latest in GHCR.
|
||||
# 2. This workflow fires via workflow_run, waits 30s for GHCR's CDN
|
||||
# to propagate the new tag.
|
||||
# 3. Calls redeploy-fleet with no canary (staging IS canary; we don't
|
||||
# need a sub-canary inside it). Soak still applies to the first
|
||||
# tenant in case of bad-deploy detection.
|
||||
# 4. Any failure aborts the rollout and leaves older tenants on the
|
||||
# prior image — safer default than half-and-half state.
|
||||
#
|
||||
# Rollback path: re-run with workflow_dispatch + target_tag=staging-<sha>
|
||||
# of a known-good build.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['publish-workspace-server-image']
|
||||
types: [completed]
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target_tag:
|
||||
description: 'Tenant image tag to deploy (e.g. "staging-latest" or "staging-a59f1a6c"). Defaults to staging-latest when empty.'
|
||||
required: false
|
||||
type: string
|
||||
default: 'staging-latest'
|
||||
canary_slug:
|
||||
description: 'Tenant slug to deploy first + soak (empty = skip canary, fan out immediately). Default empty for staging since staging itself is the canary.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
soak_seconds:
|
||||
description: 'Seconds to wait after canary before fanning out. Only meaningful if canary_slug is set.'
|
||||
required: false
|
||||
type: string
|
||||
default: '60'
|
||||
batch_size:
|
||||
description: 'How many tenants SSM redeploys in parallel per batch.'
|
||||
required: false
|
||||
type: string
|
||||
default: '3'
|
||||
dry_run:
|
||||
description: 'Plan only — do not actually redeploy.'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# No write scopes needed — the workflow hits an external CP endpoint,
|
||||
# not the GitHub API.
|
||||
|
||||
# Serialize per-branch so two rapid staging pushes' redeploys don't
|
||||
# overlap and cause confusing per-tenant SSM state. cancel-in-progress
|
||||
# is false because aborting a half-rolled-out fleet leaves tenants
|
||||
# stuck on whatever image they happened to be on when cancelled.
|
||||
concurrency:
|
||||
group: redeploy-tenants-on-staging
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
redeploy:
|
||||
# Skip the auto-trigger if publish-workspace-server-image didn't
|
||||
# actually succeed. workflow_run fires on any completion state; we
|
||||
# don't want to redeploy against a half-built image.
|
||||
if: |
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- name: Wait for GHCR tag propagation
|
||||
# GHCR's edge cache takes ~15-30s to consistently serve the new
|
||||
# :staging-latest manifest after the registry accepts the push.
|
||||
# Same rationale as redeploy-tenants-on-main.yml.
|
||||
run: sleep 30
|
||||
|
||||
- name: Call staging-CP redeploy-fleet
|
||||
# CP_STAGING_ADMIN_API_TOKEN must be set as a repo/org secret
|
||||
# on molecule-ai/molecule-core, matching staging-CP's
|
||||
# CP_ADMIN_API_TOKEN env var (visible in Railway controlplane
|
||||
# / staging environment). Stored separately from the prod
|
||||
# CP_ADMIN_API_TOKEN so a leak of one doesn't auth the other.
|
||||
env:
|
||||
CP_URL: ${{ vars.STAGING_CP_URL || 'https://staging-api.moleculesai.app' }}
|
||||
CP_STAGING_ADMIN_API_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
|
||||
TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
|
||||
CANARY_SLUG: ${{ inputs.canary_slug || '' }}
|
||||
SOAK_SECONDS: ${{ inputs.soak_seconds || '60' }}
|
||||
BATCH_SIZE: ${{ inputs.batch_size || '3' }}
|
||||
DRY_RUN: ${{ inputs.dry_run || false }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Schedule-vs-dispatch hardening (mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels): hard-fail on auto-trigger when the
|
||||
# secret is missing so a misconfigured-repo doesn't silently
|
||||
# serve stale staging tenants. Soft-skip on operator dispatch.
|
||||
if [ -z "${CP_STAGING_ADMIN_API_TOKEN:-}" ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::CP_STAGING_ADMIN_API_TOKEN secret not set — skipping redeploy"
|
||||
echo "::warning::Set CP_STAGING_ADMIN_API_TOKEN in repo secrets to enable auto-redeploy."
|
||||
echo "::notice::Pull the value from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::staging redeploy cannot run — CP_STAGING_ADMIN_API_TOKEN secret missing"
|
||||
echo "::error::set it at Settings → Secrets and Variables → Actions; pull from staging-CP's CP_ADMIN_API_TOKEN env in Railway."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BODY=$(jq -nc \
|
||||
--arg tag "$TARGET_TAG" \
|
||||
--arg canary "$CANARY_SLUG" \
|
||||
--argjson soak "$SOAK_SECONDS" \
|
||||
--argjson batch "$BATCH_SIZE" \
|
||||
--argjson dry "$DRY_RUN" \
|
||||
'{
|
||||
target_tag: $tag,
|
||||
canary_slug: $canary,
|
||||
soak_seconds: $soak,
|
||||
batch_size: $batch,
|
||||
dry_run: $dry
|
||||
}')
|
||||
|
||||
echo "POST $CP_URL/cp/admin/tenants/redeploy-fleet"
|
||||
echo " body: $BODY"
|
||||
|
||||
HTTP_RESPONSE=$(mktemp)
|
||||
HTTP_CODE_FILE=$(mktemp)
|
||||
# Route -w into its own tempfile so curl's exit code (e.g. 56
|
||||
# on connection-reset) can't pollute the captured stdout. The
|
||||
# previous inline-substitution shape produced "000000" on
|
||||
# connection reset — caught on main variant 2026-05-04
|
||||
# redeploying sha 2b862f6. Same fix shape as the synth-E2E
|
||||
# §9c gate (PR #2797). See lint-curl-status-capture.yml for
|
||||
# the CI gate that pins this fix shape.
|
||||
set +e
|
||||
curl -sS -o "$HTTP_RESPONSE" -w '%{http_code}' \
|
||||
-m 1200 \
|
||||
-H "Authorization: Bearer $CP_STAGING_ADMIN_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST "$CP_URL/cp/admin/tenants/redeploy-fleet" \
|
||||
-d "$BODY" >"$HTTP_CODE_FILE"
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to the
|
||||
# runner log so operators can see WHY a connection failed.
|
||||
HTTP_CODE=$(cat "$HTTP_CODE_FILE" 2>/dev/null || echo "000")
|
||||
[ -z "$HTTP_CODE" ] && HTTP_CODE="000"
|
||||
|
||||
echo "HTTP $HTTP_CODE"
|
||||
cat "$HTTP_RESPONSE" | jq . || cat "$HTTP_RESPONSE"
|
||||
|
||||
{
|
||||
echo "## Staging tenant redeploy fleet"
|
||||
echo ""
|
||||
echo "**Target tag:** \`$TARGET_TAG\`"
|
||||
echo "**Canary:** \`${CANARY_SLUG:-(none — staging is itself the canary)}\` (soak ${SOAK_SECONDS}s)"
|
||||
echo "**Batch size:** $BATCH_SIZE"
|
||||
echo "**Dry run:** $DRY_RUN"
|
||||
echo "**HTTP:** $HTTP_CODE"
|
||||
echo ""
|
||||
echo "### Per-tenant result"
|
||||
echo ""
|
||||
echo '| Slug | Phase | SSM Status | Exit | Healthz | Error |'
|
||||
echo '|------|-------|------------|------|---------|-------|'
|
||||
jq -r '.results[]? | "| \(.slug) | \(.phase) | \(.ssm_status // "-") | \(.ssm_exit_code) | \(.healthz_ok) | \(.error // "-") |"' "$HTTP_RESPONSE" || true
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Distinguish "real fleet failure" from "E2E teardown race".
|
||||
#
|
||||
# CP returns HTTP 500 + ok=false whenever ANY tenant in the
|
||||
# fleet failed SSM or healthz. In practice the recurring source
|
||||
# of these is ephemeral test tenants being torn down by their
|
||||
# parent E2E run mid-redeploy: the EC2 dies → SSM exit=2 or
|
||||
# healthz timeout → CP marks the fleet failed → this workflow
|
||||
# goes red even though every operator-facing tenant rolled fine.
|
||||
#
|
||||
# Ephemeral slug prefixes (kept in sync with sweep-stale-e2e-orgs.yml
|
||||
# — see that file for the source-of-truth list and rationale):
|
||||
# - e2e-* — canvas/saas/ext E2E suites
|
||||
# - rt-e2e-* — runtime-test harness fixtures (RFC #2251)
|
||||
# Long-lived prefixes that are NOT ephemeral and MUST hard-fail:
|
||||
# demo-prep, dryrun-*, dryrun2-*, plus all human tenant slugs.
|
||||
#
|
||||
# Filter: if HTTP=500/ok=false AND every failed slug matches an
|
||||
# ephemeral prefix, treat as soft-warn and let the verify step
|
||||
# downstream handle unreachable-vs-stale (#2402). Any non-ephemeral
|
||||
# failure or a non-500 HTTP response remains a hard failure.
|
||||
OK=$(jq -r '.ok // "false"' "$HTTP_RESPONSE")
|
||||
FAILED_SLUGS=$(jq -r '
|
||||
.results[]?
|
||||
| select((.healthz_ok != true) or (.ssm_status != "Success"))
|
||||
| .slug' "$HTTP_RESPONSE" 2>/dev/null || true)
|
||||
EPHEMERAL_PREFIX_RE='^(e2e-|rt-e2e-)'
|
||||
NON_EPHEMERAL_FAILED=$(printf '%s\n' "$FAILED_SLUGS" | grep -v '^$' | grep -Ev "$EPHEMERAL_PREFIX_RE" || true)
|
||||
|
||||
if [ "$HTTP_CODE" = "200" ] && [ "$OK" = "true" ]; then
|
||||
: # happy path — fall through to verification
|
||||
elif [ "$HTTP_CODE" = "500" ] && [ -z "$NON_EPHEMERAL_FAILED" ] && [ -n "$FAILED_SLUGS" ]; then
|
||||
COUNT=$(printf '%s\n' "$FAILED_SLUGS" | grep -Ec "$EPHEMERAL_PREFIX_RE" || true)
|
||||
echo "::warning::redeploy-fleet returned HTTP 500 but every failed tenant ($COUNT) is ephemeral (e2e-*/rt-e2e-*) — treating as teardown race, soft-warning."
|
||||
printf '%s\n' "$FAILED_SLUGS" | sed 's/^/::warning:: failed: /'
|
||||
elif [ "$HTTP_CODE" != "200" ]; then
|
||||
echo "::error::redeploy-fleet returned HTTP $HTTP_CODE"
|
||||
if [ -n "$NON_EPHEMERAL_FAILED" ]; then
|
||||
echo "::error::non-ephemeral tenant(s) failed:"
|
||||
printf '%s\n' "$NON_EPHEMERAL_FAILED" | sed 's/^/::error:: /'
|
||||
fi
|
||||
exit 1
|
||||
else
|
||||
# HTTP=200 but ok=false (shouldn't happen with current CP
|
||||
# but keep the gate for completeness).
|
||||
echo "::error::redeploy-fleet reported ok=false (see summary for which tenant halted the rollout)"
|
||||
exit 1
|
||||
fi
|
||||
echo "::notice::Staging tenant fleet redeploy reported ssm_status=Success — verifying actual image roll on each tenant..."
|
||||
|
||||
cp "$HTTP_RESPONSE" "$RUNNER_TEMP/redeploy-response.json"
|
||||
|
||||
- name: Verify each staging tenant /buildinfo matches published SHA
|
||||
# Mirror of the verify step in redeploy-tenants-on-main.yml — see
|
||||
# there for the rationale (#2395 root fix). Staging has the same
|
||||
# ssm_status-success-but-stale-image hazard and benefits from the
|
||||
# same gate. Diff: TENANT_DOMAIN includes the `staging.` infix.
|
||||
env:
|
||||
EXPECTED_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
TARGET_TAG: ${{ inputs.target_tag || 'staging-latest' }}
|
||||
TENANT_DOMAIN: 'staging.moleculesai.app'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# staging-latest is the staging-side moving tag; treat it the
|
||||
# same way main treats `latest`. Operator-pinned SHAs skip
|
||||
# verification (see main variant for why).
|
||||
if [ "$TARGET_TAG" != "staging-latest" ] && [ "$TARGET_TAG" != "latest" ] && [ "$TARGET_TAG" != "$EXPECTED_SHA" ]; then
|
||||
echo "::notice::target_tag=$TARGET_TAG (operator-pinned) — skipping per-tenant SHA verification."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
RESP="$RUNNER_TEMP/redeploy-response.json"
|
||||
if [ ! -s "$RESP" ]; then
|
||||
echo "::error::redeploy-response.json missing or empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mapfile -t SLUGS < <(jq -r '.results[]? | select(.healthz_ok == true) | .slug' "$RESP")
|
||||
if [ ${#SLUGS[@]} -eq 0 ]; then
|
||||
echo "::warning::No staging tenants reported healthz_ok — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Verifying ${#SLUGS[@]} staging tenant(s) against EXPECTED_SHA=${EXPECTED_SHA:0:7}..."
|
||||
|
||||
# Two distinct failure modes here:
|
||||
# STALE_COUNT — tenant returned a SHA that doesn't match. THIS is
|
||||
# the #2395 bug class: tenant up + serving old code.
|
||||
# Always hard-fail the workflow.
|
||||
# UNREACHABLE_COUNT — tenant didn't respond. Almost always a benign
|
||||
# teardown race: redeploy-fleet snapshot says
|
||||
# healthz_ok=true, then the E2E suite tears the
|
||||
# ephemeral tenant down before this step runs (the
|
||||
# e2e-* fixtures churn 5-10/hour on staging). Soft-
|
||||
# warn so we don't block staging→main on cleanup.
|
||||
# Real "tenant up but unreachable" is caught by CP's
|
||||
# own healthz monitor + the post-redeploy alert; we
|
||||
# don't need to double-count it here.
|
||||
STALE_COUNT=0
|
||||
UNREACHABLE_COUNT=0
|
||||
STALE_LINES=()
|
||||
UNREACHABLE_LINES=()
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
URL="https://${slug}.${TENANT_DOMAIN}/buildinfo"
|
||||
BODY=$(curl -sS --max-time 30 --retry 3 --retry-delay 5 --retry-connrefused "$URL" || true)
|
||||
ACTUAL_SHA=$(echo "$BODY" | jq -r '.git_sha // ""' 2>/dev/null || echo "")
|
||||
if [ -z "$ACTUAL_SHA" ]; then
|
||||
UNREACHABLE_COUNT=$((UNREACHABLE_COUNT + 1))
|
||||
UNREACHABLE_LINES+=("| $slug | (no /buildinfo response) | ${EXPECTED_SHA:0:7} | ⚠ unreachable (likely teardown race) |")
|
||||
continue
|
||||
fi
|
||||
if [ "$ACTUAL_SHA" = "$EXPECTED_SHA" ]; then
|
||||
echo " $slug: ${ACTUAL_SHA:0:7} ✓"
|
||||
else
|
||||
STALE_COUNT=$((STALE_COUNT + 1))
|
||||
STALE_LINES+=("| $slug | ${ACTUAL_SHA:0:7} | ${EXPECTED_SHA:0:7} | ❌ stale |")
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "### Per-tenant /buildinfo verification (staging)"
|
||||
echo ""
|
||||
echo "Expected SHA: \`${EXPECTED_SHA:0:7}\`"
|
||||
echo ""
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "**${STALE_COUNT} STALE tenant(s) — these did NOT pick up the new image despite ssm_status=Success:**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${STALE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "**${UNREACHABLE_COUNT} unreachable tenant(s) — likely E2E teardown race (soft-warn, not failing):**"
|
||||
echo ""
|
||||
echo "| Slug | Actual /buildinfo SHA | Expected | Status |"
|
||||
echo "|------|----------------------|----------|--------|"
|
||||
for line in "${UNREACHABLE_LINES[@]}"; do echo "$line"; done
|
||||
echo ""
|
||||
fi
|
||||
if [ $STALE_COUNT -eq 0 ] && [ $UNREACHABLE_COUNT -eq 0 ]; then
|
||||
echo "All ${#SLUGS[@]} staging tenants returned matching SHA. ✓"
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
if [ $UNREACHABLE_COUNT -gt 0 ]; then
|
||||
echo "::warning::$UNREACHABLE_COUNT staging tenant(s) unreachable post-redeploy. Likely benign teardown race — CP healthz monitor catches real outages."
|
||||
fi
|
||||
|
||||
# Belt-and-suspenders sanity floor: if MORE than half the fleet is
|
||||
# unreachable AND the fleet is large enough that "half down" is
|
||||
# statistically meaningful, this is a real outage (e.g. new image
|
||||
# crashes on startup), not a teardown race. Hard-fail.
|
||||
#
|
||||
# Floor only applies when TOTAL_VERIFIED >= 4 — below that, the
|
||||
# canary-verify step is the actual gate for "all tenants down"
|
||||
# detection (it runs against the canary first and aborts the
|
||||
# rollout if the canary fails to come up). Without the >=4 gate,
|
||||
# a 1-tenant fleet (e.g. a single ephemeral e2e-* tenant on a
|
||||
# quiet staging push) would re-flake on the exact teardown-race
|
||||
# condition #2402 fixed: 1 of 1 unreachable = 100% > 50% → fail.
|
||||
TOTAL_VERIFIED=${#SLUGS[@]}
|
||||
if [ $TOTAL_VERIFIED -ge 4 ] && [ $UNREACHABLE_COUNT -gt $((TOTAL_VERIFIED / 2)) ]; then
|
||||
echo "::error::$UNREACHABLE_COUNT of $TOTAL_VERIFIED staging tenant(s) unreachable — exceeds 50% threshold on a fleet large enough that this signals a real outage, not teardown race."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $STALE_COUNT -gt 0 ]; then
|
||||
echo "::error::$STALE_COUNT staging tenant(s) returned a stale SHA. ssm_status=Success was misleading — see job summary."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "::notice::Staging tenant fleet redeploy complete — all reachable tenants on ${EXPECTED_SHA:0:7} (${UNREACHABLE_COUNT} unreachable, soft-warned)."
|
||||
@@ -1,91 +0,0 @@
|
||||
name: Runtime Pin Compatibility
|
||||
|
||||
# CI gate that prevents the 5-hour staging outage from 2026-04-24 from
|
||||
# recurring (controlplane#253). The original failure mode:
|
||||
# 1. molecule-ai-workspace-runtime 0.1.13 declared `a2a-sdk<1.0` in its
|
||||
# requires_dist metadata (incorrect — it actually imports
|
||||
# a2a.server.routes which only exists in a2a-sdk 1.0+)
|
||||
# 2. `pip install molecule-ai-workspace-runtime` resolved cleanly
|
||||
# 3. `from molecule_runtime.main import main_sync` raised ImportError
|
||||
# 4. Every tenant workspace crashed; the canary tenant caught it but
|
||||
# only after 5 hours of degraded staging
|
||||
#
|
||||
# This workflow installs the CURRENTLY PUBLISHED runtime from PyPI on
|
||||
# top of `workspace/requirements.txt` and smoke-imports. Catches:
|
||||
# - Upstream PyPI yanks
|
||||
# - Bad re-releases of molecule-ai-workspace-runtime
|
||||
# - Already-shipped wheels that stop importing because a transitive
|
||||
# dep moved underneath
|
||||
#
|
||||
# This is the "PyPI artifact health" half of pin compatibility. The
|
||||
# companion workflow `runtime-prbuild-compat.yml` covers the
|
||||
# "PR-introduced breakage" half by building the wheel from THIS PR's
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter — the pypi-latest job no longer fires on doc-only
|
||||
# workspace/ edits whose content can't change what's currently on PyPI.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
# Narrow filter: pypi-latest is sensitive only to changes that
|
||||
# affect what we're INSTALLING (requirements.txt) or WHAT THE
|
||||
# CHECK ITSELF DOES (this workflow file). Edits to workspace/
|
||||
# source code don't change what's on PyPI right now, so they
|
||||
# don't change this gate's verdict.
|
||||
- 'workspace/requirements.txt'
|
||||
- '.github/workflows/runtime-pin-compat.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'workspace/requirements.txt'
|
||||
- '.github/workflows/runtime-pin-compat.yml'
|
||||
# Daily catch for upstream PyPI publishes that break the pin combo
|
||||
# without any change in our repo (e.g. someone re-yanks an a2a-sdk
|
||||
# release or molecule-ai-workspace-runtime publishes a bad bump).
|
||||
schedule:
|
||||
- cron: '0 13 * * *' # 06:00 PT
|
||||
workflow_dispatch:
|
||||
# Required-check support: when this becomes a branch-protection gate,
|
||||
# merge_group runs let the queue green-check this in addition to PRs.
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pypi-latest-install:
|
||||
name: PyPI-latest install + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install runtime + workspace requirements
|
||||
# Install order is load-bearing: install the runtime FIRST so pip
|
||||
# honors whatever a2a-sdk constraint the runtime metadata declares
|
||||
# (this is the surface that broke in 2026-04-24 — runtime declared
|
||||
# `a2a-sdk<1.0` but actually needed >=1.0). The follow-up install
|
||||
# of workspace/requirements.txt then upgrades a2a-sdk to the
|
||||
# constraint our runtime image actually pins. The import smoke
|
||||
# below verifies the upgraded combination is consistent.
|
||||
run: |
|
||||
python -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install --upgrade pip
|
||||
/tmp/venv/bin/pip install molecule-ai-workspace-runtime
|
||||
/tmp/venv/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import — fail if metadata declares deps that don't satisfy real imports
|
||||
# WORKSPACE_ID is validated at import time by platform_auth.py — EC2
|
||||
# user-data sets it from the cloud-init template; set a placeholder
|
||||
# here so the import smoke doesn't trip on the env-var guard.
|
||||
env:
|
||||
WORKSPACE_ID: 00000000-0000-0000-0000-000000000001
|
||||
run: |
|
||||
/tmp/venv/bin/python -c "from molecule_runtime.main import main_sync; print('runtime imports OK')"
|
||||
@@ -1,152 +0,0 @@
|
||||
name: Runtime PR-Built Compatibility
|
||||
|
||||
# Companion to `runtime-pin-compat.yml`. That workflow tests what's
|
||||
# CURRENTLY PUBLISHED on PyPI; this workflow tests what WOULD BE
|
||||
# PUBLISHED if THIS PR merges.
|
||||
#
|
||||
# Why two workflows: the chicken-and-egg #128 fix added a "PR-built
|
||||
# wheel" job to the original runtime-pin-compat.yml, but both jobs
|
||||
# shared a `paths:` filter that was the union of their needs
|
||||
# (`workspace/**`). That meant the PyPI-latest job ran on every doc
|
||||
# edit even though the upstream PyPI artifact can't change with our
|
||||
# workspace/ source. Splitting the two means each gets a narrow
|
||||
# `paths:` filter that matches the inputs it actually depends on.
|
||||
#
|
||||
# Catches the failure mode where a PR adds an import requiring a newer
|
||||
# SDK than `workspace/requirements.txt` pins:
|
||||
# 1. Pip resolves the existing PyPI wheel + the old SDK pin → smoke
|
||||
# passes (it imports the OLD main.py from the wheel, not the PR's
|
||||
# new main.py).
|
||||
# 2. Merge → publish-runtime.yml ships a wheel WITH the new import.
|
||||
# 3. Tenant images redeploy → all crash on first boot with
|
||||
# ImportError.
|
||||
#
|
||||
# By building from the PR's source and smoke-importing THAT wheel, we
|
||||
# fail at PR-time instead of after publish.
|
||||
#
|
||||
# Required-check shape (2026-05-01): the workflow runs on EVERY push +
|
||||
# PR + merge_group event with no top-level `paths:` filter, then uses a
|
||||
# detect-changes job + per-step `if:` gates inside ONE always-running
|
||||
# job named `PR-built wheel + import smoke`. PRs that don't touch
|
||||
# wheel-relevant paths get a no-op SUCCESS check run, satisfying branch
|
||||
# protection without re-running the heavy build. Same pattern as
|
||||
# e2e-api.yml — see its comment for the full rationale + the 2026-04-29
|
||||
# PR #2264 incident that motivated the always-run-with-if-gates shape.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
workflow_dispatch:
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
# Include event_name so a PR sync (event=pull_request) and the
|
||||
# subsequent staging push (event=push) on the SAME merge SHA don't
|
||||
# collide in one group. Without event_name, both runs hashed to
|
||||
# the same key and cancel-in-progress=true cancelled whichever
|
||||
# arrived second — usually the push run, which staging branch-
|
||||
# protection then sees as a CANCELLED required check and refuses
|
||||
# to mark merged. Caught 2026-05-05 across PR #2869's runs (run
|
||||
# ids 25371863455 / 25371811486 / 25371078157 / 25370403142 — every
|
||||
# staging push run cancelled, every matching PR run green).
|
||||
#
|
||||
# Per memory `feedback_concurrency_group_per_sha.md` — same drift
|
||||
# class that broke auto-promote-staging on 2026-04-28. Pin invariant:
|
||||
# event_name + sha is the minimum unique key for these workflows.
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.head.sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
detect-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
wheel: ${{ steps.decide.outputs.wheel }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
wheel:
|
||||
- 'workspace/**'
|
||||
- 'scripts/build_runtime_package.py'
|
||||
- 'scripts/wheel_smoke.py'
|
||||
- '.github/workflows/runtime-prbuild-compat.yml'
|
||||
- id: decide
|
||||
# Always run real work for manual dispatch + merge_group — no
|
||||
# diff-against-base in those contexts, and the gate exists to
|
||||
# validate the to-be-merged state regardless of which paths it
|
||||
# touched (paths-filter would default to "no changes" which is
|
||||
# the wrong answer when the queue is composing many PRs).
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ] || [ "${{ github.event_name }}" = "merge_group" ]; then
|
||||
echo "wheel=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "wheel=${{ steps.filter.outputs.wheel }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ONE job (no job-level `if:`) that always runs and reports under the
|
||||
# required-check name `PR-built wheel + import smoke`. Real work is
|
||||
# gated per-step on `needs.detect-changes.outputs.wheel`. Same shape
|
||||
# as e2e-api.yml's e2e-api job — see its comment block for the full
|
||||
# rationale (SKIPPED check runs block branch protection even with
|
||||
# SUCCESS siblings; collapsing to one always-run job emits exactly
|
||||
# one SUCCESS check run).
|
||||
local-build-install:
|
||||
needs: detect-changes
|
||||
name: PR-built wheel + import smoke
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: No-op pass (paths filter excluded this commit)
|
||||
if: needs.detect-changes.outputs.wheel != 'true'
|
||||
run: |
|
||||
echo "No workspace/ / scripts/{build_runtime_package,wheel_smoke}.py / workflow changes — wheel gate satisfied without rebuilding."
|
||||
echo "::notice::PR-built wheel + import smoke no-op pass (paths filter excluded this commit)."
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- if: needs.detect-changes.outputs.wheel == 'true'
|
||||
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
cache: pip
|
||||
cache-dependency-path: workspace/requirements.txt
|
||||
- name: Install build tooling
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: pip install build
|
||||
- name: Build wheel from PR source (mirrors publish-runtime.yml)
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Use a fixed test version so the wheel filename is predictable.
|
||||
# Doesn't reach PyPI — this build is local-only for the smoke.
|
||||
# Use the SAME build script with the SAME args as
|
||||
# publish-runtime.yml's build step. The temp dir path differs
|
||||
# (`/tmp/runtime-build` here vs `${{ runner.temp }}/runtime-build`
|
||||
# in publish-runtime.yml — they coincide on ubuntu-latest but
|
||||
# the call sites are not byte-identical). The smoke import is
|
||||
# also intentionally narrower than publish's: this gate exists
|
||||
# to catch SDK-version-import drift specifically; full invariant
|
||||
# coverage lives in publish-runtime.yml's own pre-PyPI smoke.
|
||||
run: |
|
||||
python scripts/build_runtime_package.py \
|
||||
--version "0.0.0.dev0+pin-compat" \
|
||||
--out /tmp/runtime-build
|
||||
cd /tmp/runtime-build && python -m build
|
||||
- name: Install built wheel + workspace requirements
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
run: |
|
||||
python -m venv /tmp/venv-built
|
||||
/tmp/venv-built/bin/pip install --upgrade pip
|
||||
/tmp/venv-built/bin/pip install /tmp/runtime-build/dist/*.whl
|
||||
/tmp/venv-built/bin/pip install -r workspace/requirements.txt
|
||||
/tmp/venv-built/bin/pip show molecule-ai-workspace-runtime a2a-sdk \
|
||||
| grep -E '^(Name|Version):'
|
||||
- name: Smoke import the PR-built wheel
|
||||
if: needs.detect-changes.outputs.wheel == 'true'
|
||||
# Same script publish-runtime.yml runs against the to-be-PyPI wheel.
|
||||
# Closes the PR-time vs publish-time gap: a PR adding a new SDK
|
||||
# call-shape no longer passes here (narrow `import main_sync`) only
|
||||
# to fail post-merge in publish-runtime's broader smoke.
|
||||
run: |
|
||||
/tmp/venv-built/bin/python "$GITHUB_WORKSPACE/scripts/wheel_smoke.py"
|
||||
@@ -1,58 +0,0 @@
|
||||
name: SECRET_PATTERNS drift lint
|
||||
|
||||
# Detects when the canonical SECRET_PATTERNS array in
|
||||
# .github/workflows/secret-scan.yml diverges from known consumer
|
||||
# mirrors (workspace-runtime's bundled pre-commit hook today; more
|
||||
# can be added as the consumer set grows).
|
||||
#
|
||||
# Why this exists: every side that scans for credentials has its own
|
||||
# copy of the pattern list. They drift — most recently the runtime
|
||||
# hook lagged the canonical by one pattern (sk-cp- / MiniMax F1088),
|
||||
# so a developer's local pre-commit would let a sk-cp- token through
|
||||
# while the org-wide CI scan would refuse it. The cost of that drift
|
||||
# is dev confusion + delayed feedback; the fix is automated detection.
|
||||
#
|
||||
# Triggers:
|
||||
# - schedule: daily 05:00 UTC. Catches drift introduced by edits
|
||||
# to a consumer copy that didn't update canonical here.
|
||||
# - push to main/staging where the canonical or this lint changed:
|
||||
# catches the inverse — canonical updated but consumers not yet
|
||||
# bumped. The lint will fail the push; that's intentional, the
|
||||
# person editing canonical is the right person to also update
|
||||
# the consumer.
|
||||
# - workflow_dispatch: ad-hoc operator runs.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 05:00 UTC = 22:00 PT / 01:00 ET. Quiet hours so a failure
|
||||
# email lands when humans are starting their day, not
|
||||
# interrupting it.
|
||||
- cron: "0 5 * * *"
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- ".github/workflows/secret-scan.yml"
|
||||
- ".github/workflows/secret-pattern-drift.yml"
|
||||
- ".github/scripts/lint_secret_pattern_drift.py"
|
||||
- ".githooks/pre-commit"
|
||||
workflow_dispatch:
|
||||
|
||||
# GITHUB_TOKEN scoped to read-only. The lint only does git checkout
|
||||
# + HTTPS GETs to public consumer files; no writes to anything.
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
name: Detect SECRET_PATTERNS drift
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Run drift lint
|
||||
run: python3 .github/scripts/lint_secret_pattern_drift.py
|
||||
@@ -1,129 +0,0 @@
|
||||
name: Sweep stale AWS Secrets Manager secrets
|
||||
|
||||
# Janitor for per-tenant AWS Secrets Manager secrets
|
||||
# (`molecule/tenant/<org_id>/bootstrap`) whose backing tenant no
|
||||
# longer exists. Parallel-shape to sweep-cf-tunnels.yml and
|
||||
# sweep-cf-orphans.yml — different cloud, same justification.
|
||||
#
|
||||
# Why this exists separately from a long-term reconciler integration:
|
||||
# - molecule-controlplane's tenant_resources audit table (mig 024)
|
||||
# currently tracks four resource kinds: CloudflareTunnel,
|
||||
# CloudflareDNS, EC2Instance, SecurityGroup. SecretsManager is
|
||||
# not in the list, so the existing reconciler doesn't catch
|
||||
# orphan secrets.
|
||||
# - At ~$0.40/secret/month the cost grew to ~$19/month before this
|
||||
# sweeper was written, indicating ~45+ orphan secrets from
|
||||
# crashed provisions and incomplete deprovision flows.
|
||||
# - The proper fix (KindSecretsManagerSecret + recorder hook +
|
||||
# reconciler enumerator) is filed as a separate controlplane
|
||||
# issue. This sweeper is the immediate cost-relief stopgap.
|
||||
#
|
||||
# IAM principal: AWS_JANITOR_ACCESS_KEY_ID / AWS_JANITOR_SECRET_ACCESS_KEY.
|
||||
# This is a DEDICATED principal — the production `molecule-cp` IAM
|
||||
# user lacks `secretsmanager:ListSecrets` (it only has
|
||||
# Get/Create/Update/Delete on specific resources, scoped to its
|
||||
# operational needs). The janitor needs ListSecrets across the
|
||||
# `molecule/tenant/*` prefix, which warrants a separate principal so
|
||||
# we don't broaden the prod-CP policy.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 50%, mirroring
|
||||
# sweep-cf-orphans.yml — tenant secrets are durable by design, unlike
|
||||
# the mostly-orphan tunnels) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :30 — offsets from sweep-cf-orphans (:15) and
|
||||
# sweep-cf-tunnels (:45) so the three janitors don't burst the
|
||||
# CP admin endpoints at the same minute.
|
||||
- cron: '30 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted, no deletion"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
max_delete_pct:
|
||||
description: "Override safety gate (default 50, set higher only for major cleanup)"
|
||||
required: false
|
||||
default: "50"
|
||||
grace_hours:
|
||||
description: "Skip secrets created within this many hours (default 24)"
|
||||
required: false
|
||||
default: "24"
|
||||
|
||||
# Don't let two sweeps race the same AWS account.
|
||||
concurrency:
|
||||
group: sweep-aws-secrets
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep AWS Secrets Manager
|
||||
runs-on: ubuntu-latest
|
||||
# 30 min cap, mirroring the other janitors. AWS DeleteSecret is
|
||||
# fast (~0.3s/call) so even a 100+ backlog drains in seconds
|
||||
# under the 8-way xargs parallelism, but the cap is set generously
|
||||
# to leave headroom for any actual API hang.
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
AWS_REGION: ${{ secrets.AWS_REGION || 'us-east-1' }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_JANITOR_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_JANITOR_SECRET_ACCESS_KEY }}
|
||||
CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
|
||||
CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
GRACE_HOURS: ${{ github.event.inputs.grace_hours || '24' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# and sweep-cf-tunnels (hardened 2026-04-28). Same principle:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/* (the prod molecule-cp principal lacks ListSecrets)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::AWS_JANITOR_* must belong to a principal with secretsmanager:ListSecrets and secretsmanager:DeleteSecret on molecule/tenant/*."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-tunnels:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-aws-secrets.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-aws-secrets.sh --execute
|
||||
fi
|
||||
@@ -1,146 +0,0 @@
|
||||
name: Sweep stale Cloudflare DNS records
|
||||
|
||||
# Janitor for Cloudflare DNS records whose backing tenant/workspace no
|
||||
# longer exists. Without this loop, every short-lived E2E or canary
|
||||
# leaves a CF record on the moleculesai.app zone — the zone has a
|
||||
# 200-record quota (controlplane#239 hit it 2026-04-23+) and provisions
|
||||
# start failing with code 81045 once exhausted.
|
||||
#
|
||||
# Why a separate workflow vs sweep-stale-e2e-orgs.yml:
|
||||
# - That workflow operates at the CP layer (DELETE /cp/admin/tenants/:slug
|
||||
# drives the cascade). It assumes CP has the org row to drive the
|
||||
# deprovision from. It doesn't catch records left behind when CP
|
||||
# itself never knew about the tenant (canary scratch, manual ops
|
||||
# experiments) or when the cascade's CF-delete branch failed.
|
||||
# - sweep-cf-orphans.sh enumerates the CF zone directly and matches
|
||||
# each record against live CP slugs + AWS EC2 names. It catches
|
||||
# leaks the CP-driven sweep can't.
|
||||
#
|
||||
# Safety: the script's own MAX_DELETE_PCT gate refuses to nuke more
|
||||
# than 50% of records in a single run. If something has gone weird
|
||||
# (CP admin endpoint returns no orgs → every tenant looks orphan) the
|
||||
# gate halts before damage. Decision-function unit tests in
|
||||
# scripts/ops/test_sweep_cf_decide.py (#2027) cover the rule
|
||||
# classifier.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly. Mirrors sweep-stale-e2e-orgs cadence so the two janitors
|
||||
# converge on the same tick. CF API rate budget is generous (1200
|
||||
# req/5min); a single sweep makes ~1 list + N deletes (N<=quota/2).
|
||||
- cron: '15 * * * *' # offset from sweep-stale-e2e-orgs (top of hour)
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted, no deletion"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
max_delete_pct:
|
||||
description: "Override safety gate (default 50, set higher only for major cleanup)"
|
||||
required: false
|
||||
default: "50"
|
||||
# No `merge_group:` trigger on purpose. This is a janitor — it doesn't
|
||||
# need to gate merges, and including it as written before #2088 fired
|
||||
# the full sweep job (or its secret-check) on every PR going through
|
||||
# the merge queue, generating one red CI run per merge-queue eval. If
|
||||
# this workflow is ever wired up as a required check, re-add
|
||||
# merge_group: { types: [checks_requested] }
|
||||
# AND gate the sweep step with `if: github.event_name != 'merge_group'`
|
||||
# so merge-queue evals report success without actually running.
|
||||
|
||||
# Don't let two sweeps race the same zone. workflow_dispatch during a
|
||||
# scheduled run would otherwise issue duplicate DELETE calls.
|
||||
concurrency:
|
||||
group: sweep-cf-orphans
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF orphans
|
||||
runs-on: ubuntu-latest
|
||||
# 3 min surfaces hangs (CF API stall, AWS describe-instances stuck)
|
||||
# within one cron interval instead of burning a full tick. Realistic
|
||||
# worst case is ~2 min: 4 sequential curls + 1 aws + N×CF-DELETE
|
||||
# each individually capped at 10s by the script's curl -m flag.
|
||||
timeout-minutes: 3
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ZONE_ID: ${{ secrets.CF_ZONE_ID }}
|
||||
CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
|
||||
CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: us-east-2
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '50' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split (hardened 2026-04-28
|
||||
# after the silent-no-op incident below):
|
||||
#
|
||||
# The earlier soft-skip-on-schedule policy hid a real leak. All
|
||||
# six secrets were unset on this repo for an unknown duration;
|
||||
# every hourly run printed a yellow ::warning:: and exited 0,
|
||||
# so the workflow registered as "passing" while doing nothing.
|
||||
# CF orphans accumulated to 152/200 (~76% of the zone quota
|
||||
# gone) before a manual `dig`-driven audit caught it. Anything
|
||||
# that runs as a janitor and reports green while idle is
|
||||
# indistinguishable from "the janitor is healthy" — so we now
|
||||
# treat schedule (and any future workflow_run/push triggers)
|
||||
# as a hard-fail when secrets are missing.
|
||||
#
|
||||
# - schedule / workflow_run / push → exit 1 (red CI run
|
||||
# surfaces the misconfiguration the next tick)
|
||||
# - workflow_dispatch → exit 0 with a warning
|
||||
# (an operator ran this ad-hoc; they already accepted the
|
||||
# state of the repo and want the workflow to short-circuit
|
||||
# so they can rerun after fixing the secret)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ZONE_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::a silent skip masked an active CF DNS leak (152/200 zone records) caught only by a manual audit on 2026-04-28; this gate exists to make the gap visible."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry (intentional):
|
||||
# - Scheduled runs: github.event.inputs.dry_run is empty →
|
||||
# defaults to "false" below → script runs with --execute
|
||||
# (the whole point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default is true (line 38)
|
||||
# so an ad-hoc operator-triggered run is dry-run by default;
|
||||
# they have to flip the toggle to actually delete.
|
||||
# The script's MAX_DELETE_PCT gate (default 50%) is the second
|
||||
# line of defense regardless of mode.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-orphans.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-orphans.sh --execute
|
||||
fi
|
||||
@@ -1,124 +0,0 @@
|
||||
name: Sweep stale Cloudflare Tunnels
|
||||
|
||||
# Janitor for Cloudflare Tunnels whose backing tenant no longer
|
||||
# exists. Parallel-shape to sweep-cf-orphans.yml (which sweeps DNS
|
||||
# records); same justification, different CF resource.
|
||||
#
|
||||
# Why this exists separately from sweep-cf-orphans:
|
||||
# - DNS records live on the zone (`/zones/<id>/dns_records`).
|
||||
# - Tunnels live on the account (`/accounts/<id>/cfd_tunnel`).
|
||||
# - Different CF API surface, different scopes; the existing CF
|
||||
# token might not have `account:cloudflare_tunnel:edit`. Splitting
|
||||
# the workflows keeps each one's secret-presence gate independent
|
||||
# so neither silent-skips when the other's secret is missing.
|
||||
# - Cleaner blast radius — operators can disable one without the
|
||||
# other if a regression surfaces.
|
||||
#
|
||||
# Safety: the script's MAX_DELETE_PCT gate (default 90% — higher than
|
||||
# the DNS sweep's 50% because tenant-shaped tunnels are mostly
|
||||
# orphans by design) refuses to nuke past the threshold.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Hourly at :45 — offset from sweep-cf-orphans (:15) so the two
|
||||
# janitors don't issue parallel CF API bursts at the same minute.
|
||||
- cron: '45 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted, no deletion"
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
max_delete_pct:
|
||||
description: "Override safety gate (default 90, set higher only for major cleanup)"
|
||||
required: false
|
||||
default: "90"
|
||||
|
||||
# Don't let two sweeps race the same account.
|
||||
concurrency:
|
||||
group: sweep-cf-tunnels
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep CF tunnels
|
||||
runs-on: ubuntu-latest
|
||||
# 30 min cap. Was 5 min on the theory that the only thing that
|
||||
# could take >5min is a CF-API hang — but on 2026-05-02 a backlog
|
||||
# of 672 stale tunnels accumulated (large staging E2E run + delayed
|
||||
# sweep) and the serial `curl -X DELETE` loop (~0.7s/tunnel) needed
|
||||
# ~7-8min to drain. The 5-min cap killed the run mid-sweep
|
||||
# (cancelled at 424/672, see run 25248788312); a manual rerun
|
||||
# finished the remainder fine.
|
||||
#
|
||||
# The fix is two-part: parallelize the delete loop (8-way xargs in
|
||||
# the script — see scripts/ops/sweep-cf-tunnels.sh), AND raise the
|
||||
# cap so a one-off backlog doesn't trip a hangs-detector that
|
||||
# turned out to be a real-job-too-slow detector. With 8-way
|
||||
# parallelism, 600+ tunnels drains in ~60s; 30 min is generous
|
||||
# headroom for actual hangs to still surface (and is in line with
|
||||
# the sweep-cf-orphans companion job).
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CF_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
|
||||
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
|
||||
CP_PROD_ADMIN_TOKEN: ${{ secrets.CP_PROD_ADMIN_TOKEN }}
|
||||
CP_STAGING_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_TOKEN }}
|
||||
MAX_DELETE_PCT: ${{ github.event.inputs.max_delete_pct || '90' }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Verify required secrets present
|
||||
id: verify
|
||||
# Schedule-vs-dispatch behaviour split mirrors sweep-cf-orphans
|
||||
# (hardened 2026-04-28 after the silent-no-op incident: the
|
||||
# janitor reported green while doing nothing because secrets
|
||||
# were unset, masking a 152/200 zone-record leak). Same
|
||||
# principle applies here:
|
||||
# - schedule → exit 1 on missing secrets (red CI surfaces it)
|
||||
# - workflow_dispatch → exit 0 with warning (operator-driven,
|
||||
# they already accepted the repo state)
|
||||
run: |
|
||||
missing=()
|
||||
for var in CF_API_TOKEN CF_ACCOUNT_ID CP_PROD_ADMIN_TOKEN CP_STAGING_ADMIN_TOKEN; do
|
||||
if [ -z "${!var:-}" ]; then
|
||||
missing+=("$var")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
echo "::warning::skipping sweep — secrets not configured: ${missing[*]}"
|
||||
echo "::warning::set them at Settings → Secrets and Variables → Actions, then rerun."
|
||||
echo "::warning::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope (separate from the zone:dns:edit scope used by sweep-cf-orphans)."
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "::error::sweep cannot run — required secrets missing: ${missing[*]}"
|
||||
echo "::error::set them at Settings → Secrets and Variables → Actions, or disable this workflow."
|
||||
echo "::error::CF_API_TOKEN must include account:cloudflare_tunnel:edit scope."
|
||||
exit 1
|
||||
fi
|
||||
echo "All required secrets present ✓"
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Run sweep
|
||||
if: steps.verify.outputs.skip != 'true'
|
||||
# Schedule-vs-dispatch dry-run asymmetry mirrors sweep-cf-orphans:
|
||||
# - Scheduled: input empty → "false" → --execute (the whole
|
||||
# point of an hourly janitor).
|
||||
# - Manual workflow_dispatch: input default true → dry-run;
|
||||
# operator must flip it to actually delete.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event.inputs.dry_run || 'false' }}" = "true" ]; then
|
||||
echo "Running in dry-run mode — no deletions"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh
|
||||
else
|
||||
echo "Running with --execute — will delete identified orphans"
|
||||
bash scripts/ops/sweep-cf-tunnels.sh --execute
|
||||
fi
|
||||
@@ -1,239 +0,0 @@
|
||||
name: Sweep stale e2e-* orgs (staging)
|
||||
|
||||
# Janitor for staging tenants left behind when E2E cleanup didn't run:
|
||||
# CI cancellations, runner crashes, transient AWS errors mid-cascade,
|
||||
# bash trap missed (signal 9), etc. Without this loop, every failed
|
||||
# teardown leaks an EC2 + DNS + DB row until manual ops cleanup —
|
||||
# 2026-04-23 staging hit the 64 vCPU AWS quota from ~27 such orphans.
|
||||
#
|
||||
# Why not rely on per-test-run teardown:
|
||||
# - Per-run teardown is best-effort by definition. Any process death
|
||||
# after the test starts but before the trap fires leaves debris.
|
||||
# - GH Actions cancellation kills the runner without grace period.
|
||||
# The workflow's `if: always()` step usually catches this, but it
|
||||
# too can fail (CP transient 5xx, runner network issue at the
|
||||
# wrong moment).
|
||||
# - Even when teardown runs, the CP cascade is best-effort in places
|
||||
# (cascadeTerminateWorkspaces logs+continues; DNS deletion same).
|
||||
# - This sweep is the catch-all that converges staging back to clean
|
||||
# regardless of which specific path leaked.
|
||||
#
|
||||
# The PROPER fix is making CP cleanup transactional + verify-after-
|
||||
# terminate (filed separately as cleanup-correctness work). This
|
||||
# workflow is the safety net that catches everything else AND any
|
||||
# future leak source we haven't yet identified.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Every 15 min. E2E orgs are short-lived (~8-25 min wall clock from
|
||||
# create to teardown — canary is ~8 min, full SaaS ~25 min). The
|
||||
# previous hourly + 120-min stale threshold meant a leaked tenant
|
||||
# could keep an EC2 alive for up to 2 hours, eating ~2 vCPU per
|
||||
# leak. Tightening the cadence + threshold reduces the worst-case
|
||||
# leak window from 120 min to ~45 min (15-min sweep cadence + 30-min
|
||||
# threshold) without risk of catching in-progress runs (the longest
|
||||
# e2e run is the 25-min canary, well under the 30-min threshold).
|
||||
# See molecule-controlplane#420 for the leak-class accounting that
|
||||
# motivated this tightening.
|
||||
- cron: '*/15 * * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
max_age_minutes:
|
||||
description: "Delete e2e-* orgs older than N minutes (default 30)"
|
||||
required: false
|
||||
default: "30"
|
||||
dry_run:
|
||||
description: "Dry run only — list what would be deleted"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# Don't let two sweeps fight. Cron + workflow_dispatch could overlap
|
||||
# on a manual trigger; queue rather than parallel-delete.
|
||||
concurrency:
|
||||
group: sweep-stale-e2e-orgs
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
name: Sweep e2e orgs
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
MOLECULE_CP_URL: https://staging-api.moleculesai.app
|
||||
ADMIN_TOKEN: ${{ secrets.MOLECULE_STAGING_ADMIN_TOKEN }}
|
||||
MAX_AGE_MINUTES: ${{ github.event.inputs.max_age_minutes || '30' }}
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }}
|
||||
# Refuse to delete more than this many orgs in one tick. If the
|
||||
# CP DB is briefly empty (or the admin endpoint goes weird and
|
||||
# returns no created_at), every e2e- org would look stale.
|
||||
# Bailing protects against runaway nukes.
|
||||
SAFETY_CAP: 50
|
||||
|
||||
steps:
|
||||
- name: Verify admin token present
|
||||
run: |
|
||||
if [ -z "$ADMIN_TOKEN" ]; then
|
||||
echo "::error::MOLECULE_STAGING_ADMIN_TOKEN not set"
|
||||
exit 2
|
||||
fi
|
||||
echo "Admin token present ✓"
|
||||
|
||||
- name: Identify stale e2e orgs
|
||||
id: identify
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Fetch into a file so the python step reads it via stdin —
|
||||
# cleaner than embedding $(curl ...) into a heredoc.
|
||||
curl -sS --fail-with-body --max-time 30 \
|
||||
"$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
> orgs.json
|
||||
|
||||
# Filter:
|
||||
# 1. slug starts with one of the ephemeral test prefixes:
|
||||
# - 'e2e-' — covers e2e-canary-, e2e-canvas-*, etc.
|
||||
# - 'rt-e2e-' — runtime-test harness fixtures (RFC #2251);
|
||||
# missing this prefix left two such tenants
|
||||
# orphaned 8h on staging (2026-05-03), then
|
||||
# hard-failed redeploy-tenants-on-staging
|
||||
# and broke the staging→main auto-promote
|
||||
# chain. Kept in sync with the EPHEMERAL_PREFIX_RE
|
||||
# regex in redeploy-tenants-on-staging.yml.
|
||||
# 2. created_at is older than MAX_AGE_MINUTES ago
|
||||
# Output one slug per line to a file the next step reads.
|
||||
python3 > stale_slugs.txt <<'PY'
|
||||
import json, os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
# SSOT for this list lives in the controlplane Go code:
|
||||
# molecule-controlplane/internal/slugs/ephemeral.go
|
||||
# (var EphemeralPrefixes). The redeploy-fleet auto-rollout
|
||||
# also reads from there to SKIP these slugs — without that
|
||||
# filter, fleet redeploy SSM-failed in-flight E2E tenants
|
||||
# whose containers were still booting, breaking the test
|
||||
# that just spun them up (molecule-controlplane#493).
|
||||
# Update both files together.
|
||||
EPHEMERAL_PREFIXES = ("e2e-", "rt-e2e-")
|
||||
with open("orgs.json") as f:
|
||||
data = json.load(f)
|
||||
max_age = int(os.environ["MAX_AGE_MINUTES"])
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(minutes=max_age)
|
||||
for o in data.get("orgs", []):
|
||||
slug = o.get("slug", "")
|
||||
if not slug.startswith(EPHEMERAL_PREFIXES):
|
||||
continue
|
||||
created = o.get("created_at")
|
||||
if not created:
|
||||
# Defensively skip rows without created_at — better
|
||||
# to leave one orphan than nuke a brand-new row
|
||||
# whose timestamp didn't render.
|
||||
continue
|
||||
# Python 3.11+ handles RFC3339 with Z directly via
|
||||
# fromisoformat; older runners need the trailing Z swap.
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
if created_dt < cutoff:
|
||||
print(slug)
|
||||
PY
|
||||
|
||||
count=$(wc -l < stale_slugs.txt | tr -d ' ')
|
||||
echo "Found $count stale e2e org(s) older than ${MAX_AGE_MINUTES}m"
|
||||
if [ "$count" -gt 0 ]; then
|
||||
echo "First 20:"
|
||||
head -20 stale_slugs.txt | sed 's/^/ /'
|
||||
fi
|
||||
echo "count=$count" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Safety gate
|
||||
if: steps.identify.outputs.count != '0'
|
||||
run: |
|
||||
count="${{ steps.identify.outputs.count }}"
|
||||
if [ "$count" -gt "$SAFETY_CAP" ]; then
|
||||
echo "::error::Refusing to delete $count orgs in one sweep (cap=$SAFETY_CAP). Investigate manually — this usually means the CP admin API returned no created_at or returned a degraded result. Re-run with workflow_dispatch + max_age_minutes if intentional."
|
||||
exit 1
|
||||
fi
|
||||
echo "Within safety cap ($count ≤ $SAFETY_CAP) ✓"
|
||||
|
||||
- name: Delete stale orgs
|
||||
if: steps.identify.outputs.count != '0' && env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set -uo pipefail
|
||||
deleted=0
|
||||
failed=0
|
||||
while IFS= read -r slug; do
|
||||
[ -z "$slug" ] && continue
|
||||
# The DELETE handler requires {"confirm": "<slug>"} matching
|
||||
# the URL slug — fat-finger guard. Idempotent: re-issuing
|
||||
# picks up via org_purges.last_step.
|
||||
# Tempfile-routed -w + set +e/-e prevents curl-exit-code
|
||||
# pollution of the captured status (lint-curl-status-capture.yml).
|
||||
set +e
|
||||
curl -sS -o /tmp/del_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"confirm\":\"$slug\"}" >/tmp/del_code
|
||||
set -e
|
||||
# Stderr from curl (-sS shows dial errors etc.) goes to runner log.
|
||||
http_code=$(cat /tmp/del_code 2>/dev/null || echo "000")
|
||||
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
||||
deleted=$((deleted+1))
|
||||
echo " deleted: $slug"
|
||||
else
|
||||
failed=$((failed+1))
|
||||
echo " FAILED ($http_code): $slug — $(cat /tmp/del_resp 2>/dev/null | head -c 200)"
|
||||
fi
|
||||
done < stale_slugs.txt
|
||||
echo ""
|
||||
echo "Sweep summary: deleted=$deleted failed=$failed"
|
||||
# Don't fail the workflow on per-org delete errors — the
|
||||
# sweeper is best-effort. Next hourly tick re-attempts. We
|
||||
# only fail loud at the safety-cap gate above.
|
||||
|
||||
- name: Sweep orphan tunnels
|
||||
# Stale-org cleanup deletes the org (which cascades to tunnel
|
||||
# delete inside the CP). But when that cascade fails partway —
|
||||
# CP transient 5xx after the org row is deleted but before the
|
||||
# CF tunnel delete completes — the tunnel persists with no
|
||||
# matching org row. The reconciler in internal/sweep flags this
|
||||
# as `cf_tunnel kind=orphan`, but nothing automatically reaps it.
|
||||
#
|
||||
# `/cp/admin/orphan-tunnels/cleanup` is the operator-triggered
|
||||
# reaper. Calling it here at the end of every sweep tick
|
||||
# converges the staging CF account to clean even when CP
|
||||
# cascades half-fail.
|
||||
#
|
||||
# PR #492 made the underlying DeleteTunnel actually check
|
||||
# status — pre-fix it silent-succeeded on CF code 1022
|
||||
# ("active connections"), so this step would have been a no-op
|
||||
# against stuck connectors. Post-fix the cleanup invokes
|
||||
# CleanupTunnelConnections + retry, which actually clears the
|
||||
# 1022 case. (#2987)
|
||||
#
|
||||
# Best-effort. Failure here doesn't fail the workflow — next
|
||||
# tick re-attempts. Errors flow to step output for ops review.
|
||||
if: env.DRY_RUN != 'true'
|
||||
run: |
|
||||
set +e
|
||||
curl -sS -o /tmp/cleanup_resp -w "%{http_code}" \
|
||||
--max-time 60 \
|
||||
-X POST "$MOLECULE_CP_URL/cp/admin/orphan-tunnels/cleanup" \
|
||||
-H "Authorization: Bearer $ADMIN_TOKEN" >/tmp/cleanup_code
|
||||
set -e
|
||||
http_code=$(cat /tmp/cleanup_code 2>/dev/null || echo "000")
|
||||
body=$(cat /tmp/cleanup_resp 2>/dev/null | head -c 500)
|
||||
if [ "$http_code" = "200" ]; then
|
||||
count=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(d.get('deleted_count', 0))" 2>/dev/null || echo "0")
|
||||
failed_n=$(echo "$body" | python3 -c "import sys,json; d=json.loads(sys.stdin.read() or '{}'); print(len(d.get('failed') or {}))" 2>/dev/null || echo "0")
|
||||
echo "Orphan-tunnel sweep: deleted=$count failed=$failed_n"
|
||||
else
|
||||
echo "::warning::orphan-tunnels cleanup returned HTTP $http_code — body: $body"
|
||||
fi
|
||||
|
||||
- name: Dry-run summary
|
||||
if: env.DRY_RUN == 'true'
|
||||
run: |
|
||||
echo "DRY RUN — would have deleted ${{ steps.identify.outputs.count }} org(s) AND triggered orphan-tunnels cleanup. Re-run with dry_run=false to actually delete."
|
||||
@@ -1,52 +0,0 @@
|
||||
name: Ops Scripts Tests
|
||||
|
||||
# Runs the unittest suite for scripts/ on every PR + push that touches
|
||||
# anything under scripts/. Kept separate from the main CI so a script-only
|
||||
# change doesn't trigger the heavier Go/Canvas/Python pipelines.
|
||||
#
|
||||
# Discovery layout: tests sit alongside the code they test (see
|
||||
# scripts/ops/test_sweep_cf_decide.py for the pattern; scripts/
|
||||
# test_build_runtime_package.py for the rewriter coverage). The job
|
||||
# below runs `unittest discover` TWICE — once from `scripts/`, once
|
||||
# from `scripts/ops/` — because neither dir has an `__init__.py`, so
|
||||
# a single discover from `scripts/` doesn't recurse into the ops
|
||||
# subdir. Two passes is simpler than retrofitting namespace packages.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.github/workflows/test-ops-scripts.yml'
|
||||
pull_request:
|
||||
branches: [main, staging]
|
||||
paths:
|
||||
- 'scripts/**'
|
||||
- '.github/workflows/test-ops-scripts.yml'
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Ops scripts (unittest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Run scripts/ unittests (build_runtime_package, …)
|
||||
# Top-level scripts/ tests live alongside their target file
|
||||
# (e.g. scripts/test_build_runtime_package.py exercises
|
||||
# scripts/build_runtime_package.py). discover from scripts/
|
||||
# picks up only top-level test_*.py because scripts/ops/ has
|
||||
# no __init__.py — that's intentional, so we run two passes.
|
||||
working-directory: scripts
|
||||
run: python -m unittest discover -t . -p 'test_*.py' -v
|
||||
- name: Run scripts/ops/ unittests (sweep_cf_decide, …)
|
||||
working-directory: scripts/ops
|
||||
run: python -m unittest discover -p 'test_*.py' -v
|
||||
+5
-31
@@ -117,40 +117,14 @@ backups/
|
||||
|
||||
# Cloned-via-manifest dirs — populated locally by scripts/clone-manifest.sh,
|
||||
# tracked in their own standalone repos. Never commit to core.
|
||||
# org-templates live in Molecule-AI/molecule-ai-org-template-* repos
|
||||
# (including molecule-dev — no checkin exception).
|
||||
# org-templates live in Molecule-AI/molecule-ai-org-template-* repos.
|
||||
# plugins live in Molecule-AI/molecule-ai-plugin-* repos.
|
||||
# All three directories are populated by scripts/clone-manifest.sh
|
||||
# (now auto-run by infra/scripts/setup.sh). The in-tree exception for
|
||||
# molecule-dev was removed because the checked-in copy drifted from
|
||||
# the standalone repo and shipped with broken !include references to
|
||||
# role files that never existed in the snapshot.
|
||||
/org-templates/
|
||||
# Exception: molecule-dev is checked in so it doubles as the internal-team
|
||||
# seed template (not fetched via clone-manifest).
|
||||
/org-templates/*
|
||||
!/org-templates/molecule-dev/
|
||||
/plugins/
|
||||
/workspace-configs-templates/
|
||||
# Cloned by publish-workspace-server-image.yml so the Dockerfile's
|
||||
# replace-directive path resolves. Lives in its own repo.
|
||||
/molecule-ai-plugin-github-app-auth/
|
||||
# Tenant-image build context — populated by the workflow's
|
||||
# "Pre-clone manifest deps" step. Mirrors the public manifest, holds the
|
||||
# same content as the three /<>/ dirs above but namespaced under one
|
||||
# parent so the Docker build context is a single COPY-friendly tree.
|
||||
# Each entry is a transient working-dir, never source-of-truth, never
|
||||
# committed.
|
||||
/.tenant-bundle-deps/
|
||||
|
||||
# Internal-flavored content lives in Molecule-AI/internal — NEVER in this
|
||||
# public monorepo. Migrated 2026-04-23 (CEO directive). The CI workflow
|
||||
# .github/workflows/block-internal-paths.yml enforces this; this gitignore
|
||||
# is the second line of defence so accidental local writes don't reach a
|
||||
# commit. See docs/internal-content-policy.md for the full rationale.
|
||||
/research/
|
||||
/marketing/
|
||||
/docs/marketing/
|
||||
# Common temp/scratch patterns agents have produced
|
||||
/comment-*.json
|
||||
*-temp.md
|
||||
*-temp.txt
|
||||
/test-pmm-*.txt
|
||||
/tick-reflections-*.md
|
||||
tests/harness/cp-stub/cp-stub
|
||||
|
||||
+3
-58
@@ -12,29 +12,21 @@ development workflow, conventions, and how to get your changes merged.
|
||||
- **Python 3.11+** — workspace runtime
|
||||
- **Docker** — infrastructure services (Postgres, Redis)
|
||||
- **Git** — with hooks path set to `.githooks`
|
||||
- **jq** — parses `manifest.json` during `setup.sh` to clone the
|
||||
template/plugin registry. Install via `brew install jq` (macOS) or
|
||||
`apt install jq` (Debian). Without it, setup.sh prints a note and
|
||||
leaves the registry dirs empty (recoverable by installing jq and
|
||||
re-running).
|
||||
|
||||
### Setup
|
||||
|
||||
```bash
|
||||
# Clone the repo
|
||||
git clone https://git.moleculesai.app/molecule-ai/molecule-core.git
|
||||
cd molecule-core
|
||||
git clone https://github.com/Molecule-AI/molecule-monorepo.git
|
||||
cd molecule-monorepo
|
||||
|
||||
# Install git hooks
|
||||
git config core.hooksPath .githooks
|
||||
|
||||
# Copy and edit .env (generate ADMIN_TOKEN + SECRETS_ENCRYPTION_KEY)
|
||||
cp .env.example .env
|
||||
|
||||
# Start infrastructure (Postgres, Redis, Langfuse, Temporal)
|
||||
./infra/scripts/setup.sh
|
||||
|
||||
# Build and run the platform — applies pending migrations on first boot
|
||||
# Build and run the platform
|
||||
cd workspace-server
|
||||
go run ./cmd/server
|
||||
|
||||
@@ -53,29 +45,6 @@ cp .env.example .env
|
||||
|
||||
See `CLAUDE.md` for a full list of environment variables and their purposes.
|
||||
|
||||
## What goes where (content vs code)
|
||||
|
||||
This repo is scoped to **code** (canvas, workspace, workspace-server, related
|
||||
infra). Public content (blog posts, marketing copy, OG images, SEO briefs,
|
||||
DevRel demos) lives in [`Molecule-AI/docs`](https://git.moleculesai.app/molecule-ai/docs).
|
||||
The `Block forbidden paths` CI gate fails any PR that writes to `marketing/`
|
||||
or other removed paths — open against `Molecule-AI/docs` instead.
|
||||
|
||||
| Content type | Target |
|
||||
|---|---|
|
||||
| Blog posts | `Molecule-AI/docs` → `content/blog/<YYYY-MM-DD-slug>/` |
|
||||
| Doc pages | `Molecule-AI/docs` → `content/docs/` |
|
||||
| Marketing copy / PMM positioning | `Molecule-AI/docs` → `marketing/` |
|
||||
| OG images, visual assets | `Molecule-AI/docs` → `app/` or `marketing/` |
|
||||
| SEO briefs | `Molecule-AI/docs` → `marketing/` |
|
||||
| DevRel demos (runnable code) | Standalone repo under `Molecule-AI/`, OR embedded in `Molecule-AI/docs` |
|
||||
| Launch checklists, internal tracking | GitHub Issues — **not** committed files |
|
||||
| Engineering docs (`docs/adr/`, `docs/architecture/`, `docs/incidents/`) | This repo (internal, not published) |
|
||||
| Live product pages (e.g. `canvas/src/app/pricing/page.tsx`) | This repo (these are app code, not marketing copy) |
|
||||
|
||||
If a PR fails the `Block forbidden paths` check, the contents belong in
|
||||
`Molecule-AI/docs`. No CI drag, no Canvas E2E, content lands in minutes.
|
||||
|
||||
## Development Workflow
|
||||
|
||||
### Branch Naming
|
||||
@@ -104,19 +73,6 @@ causing a render loop when any node position changed.
|
||||
- Include a test plan in the PR description
|
||||
- PRs are merged with **merge commits** (not squash or rebase)
|
||||
|
||||
#### Auto-merge & the "extra commit" trap
|
||||
|
||||
**Two system guards protect against pushing commits after auto-merge has been enabled.** Don't try to work around them — they exist because we shipped a half-merged PR on 2026-04-27 (`#2174` merged with only its first commit; the second was orphaned on a branch GitHub had already deleted).
|
||||
|
||||
1. **Repo-wide:** "Automatically delete head branches" is on. Once a PR merges, the branch is deleted server-side. Any subsequent `git push` to that branch fails with `remote rejected — no such branch`.
|
||||
|
||||
2. **CI:** the `pr-guards` workflow (calling [molecule-ci `disable-auto-merge-on-push`](https://git.moleculesai.app/molecule-ai/molecule-ci/src/branch/main/.github/workflows/disable-auto-merge-on-push.yml)) fires on every push to an open PR. If auto-merge was already enabled, it's disabled and a comment is posted. You must explicitly re-enable after verifying the new commit.
|
||||
|
||||
**Workflow rules that follow from the guards:**
|
||||
- Push **all** commits before running `gh pr merge --auto`.
|
||||
- If you realize you need another commit after enabling auto-merge: push it, then **re-run** `gh pr merge --auto` — the guard will already have disabled it. The disable + re-enable is the verification step.
|
||||
- For changes that depend on each other across PRs (e.g. a build-script change + a workflow that consumes it), prefer a **stack** of PRs (PR-B branched off PR-A's branch, opened only after PR-A is in queue) over amending one in-flight PR.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
@@ -175,17 +131,6 @@ and run CI manually.
|
||||
- Type hints on public functions
|
||||
- pytest for all tests
|
||||
|
||||
## External integrations
|
||||
|
||||
Code in this repo lands in molecule-core. Some related runtime artifacts
|
||||
live in their own repos:
|
||||
|
||||
- [`Molecule-AI/molecule-ai-workspace-runtime`](https://git.moleculesai.app/molecule-ai/molecule-ai-workspace-runtime) — Python adapter SDK (`molecule_runtime`) that runs inside containerized Molecule workspaces. Bridges Claude Code SDK / hermes / langgraph / etc. → A2A queue.
|
||||
- [`Molecule-AI/molecule-sdk-python`](https://git.moleculesai.app/molecule-ai/molecule-sdk-python) — `A2AServer` + `RemoteAgentClient` for external agents that register over the public `/registry/register` flow.
|
||||
- [`Molecule-AI/molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel) — Claude Code channel plugin. Bridges A2A traffic into a running Claude Code session via MCP `notifications/claude/channel`. Polling-based (no tunnel required); install with `claude --channels plugin:molecule@Molecule-AI/molecule-mcp-claude-channel`.
|
||||
|
||||
When extending the **A2A surface** in molecule-core (`workspace-server/internal/handlers/a2a_proxy.go` etc.), consider whether the change has a downstream impact on the runtime SDK or the channel plugin — they're versioned independently but share the wire shape.
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
See `CLAUDE.md` for detailed architecture documentation, including:
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
# Coverage Floor
|
||||
|
||||
CI enforces coverage gates on two surfaces — `workspace-server` (Go) and
|
||||
`workspace/` (Python). All defined in `.github/workflows/ci.yml`.
|
||||
|
||||
## Current floors (2026-04-23)
|
||||
|
||||
| Gate | Threshold | What fails |
|
||||
|---|---|---|
|
||||
| **Total floor** | `25%` | `go tool cover -func` reports total below floor |
|
||||
| **Critical-path per-file floor** | `10%` | Any non-test source file in a security-critical path with coverage ≤10% |
|
||||
| **Per-file report** | advisory | Printed in CI log, sorted worst-first, does not fail |
|
||||
|
||||
Total floor starts at 25% (unchanged from pre-#1823 to keep this PR strictly
|
||||
additive). The new protection is the critical-path per-file floor, which
|
||||
directly closes the gap that prompted the issue. Ratchet plan below begins
|
||||
the month after to let the team first observe the gate in action.
|
||||
|
||||
## Security-critical paths (Gate 2)
|
||||
|
||||
Changes to these paths have historically introduced security issues (CWE-22,
|
||||
CWE-78, KI-005, SSRF) or billing/auth risk. Coverage must not drop to zero.
|
||||
|
||||
- `internal/handlers/tokens*`
|
||||
- `internal/handlers/workspace_provision*`
|
||||
- `internal/handlers/a2a_proxy*`
|
||||
- `internal/handlers/registry*`
|
||||
- `internal/handlers/secrets*`
|
||||
- `internal/middleware/wsauth*`
|
||||
- `internal/crypto*`
|
||||
|
||||
## Ratchet plan
|
||||
|
||||
Floor ratchets upward on a fixed cadence. Any ratchet is a PR — reviewable,
|
||||
reversible, and creates history. The table below is the intended schedule.
|
||||
|
||||
| Date | Total floor | Critical-path floor | Notes |
|
||||
|---|---|---|---|
|
||||
| 2026-04-23 | 25% | 10% | Initial gate (this file). |
|
||||
| 2026-05-23 | 30% | 20% | First ratchet |
|
||||
| 2026-06-23 | 40% | 30% | |
|
||||
| 2026-07-23 | 50% | 40% | |
|
||||
| 2026-08-23 | 55% | 50% | |
|
||||
| 2026-09-23 | 60% | 60% | |
|
||||
| 2026-10-23 | 70% | 70% | Target steady-state |
|
||||
|
||||
The target end-state matches the per-role QA prompts which specify
|
||||
"coverage >80% on changed files". CI enforces the floor; reviewers still
|
||||
enforce the per-PR bar.
|
||||
|
||||
## Exceptions
|
||||
|
||||
If a critical-path file genuinely cannot have coverage above the floor (e.g.
|
||||
thin wrapper around a third-party SDK with no branches to test), add an entry
|
||||
here with:
|
||||
|
||||
1. **File**: `internal/handlers/example.go`
|
||||
2. **Reason**: Why coverage can't hit the floor
|
||||
3. **Tracking issue**: GitHub issue for the real fix
|
||||
4. **Expiry**: 14 days from entry date; after expiry either coverage is fixed
|
||||
or the issue is closed as "accepted technical debt"
|
||||
|
||||
### Active exceptions
|
||||
|
||||
*(none — add here if you need to land code that legitimately can't clear the floor)*
|
||||
|
||||
## Why this gate exists
|
||||
|
||||
Issue #1823: an external audit found critical files at 0% coverage despite
|
||||
test files existing with hundreds of lines. The existing CI step measured
|
||||
coverage but didn't enforce a meaningful threshold. Any file could go from
|
||||
80% → 0% and CI stayed green, because the single gate (total ≥25%) ignored
|
||||
per-file distribution.
|
||||
|
||||
This gate makes "no untested critical paths merged" a mechanical property of
|
||||
the CI, not a behavioural property of QA agents or individual reviewers —
|
||||
which is the only way to make it survive fleet outages, agent rotations, or
|
||||
QA process changes.
|
||||
|
||||
## Python (workspace/) — added 2026-05-04 from #2790
|
||||
|
||||
The Python side has its own gates in the `python-lint` job:
|
||||
|
||||
| Gate | Threshold | Where |
|
||||
|---|---|---|
|
||||
| **Total floor** | `86%` | `workspace/pytest.ini` `--cov-fail-under=86` (issue #1817) |
|
||||
| **Critical-path per-file floor** | `75%` | Inline shell step after the pytest run |
|
||||
|
||||
### Critical-path Python files
|
||||
|
||||
These handle multi-tenant routing, auth tokens, and inbox dispatch. A
|
||||
coverage drop here is the same risk shape as a Go-side `tokens*` /
|
||||
`secrets*` file regressing below 10%.
|
||||
|
||||
- `workspace/a2a_mcp_server.py` — MCP dispatcher (PR #2766 / #2771)
|
||||
- `workspace/mcp_cli.py` — molecule-mcp standalone CLI entry
|
||||
- `workspace/a2a_tools.py` — workspace-scoped tool implementations
|
||||
- `workspace/inbox.py` — multi-workspace inbox + per-workspace cursors
|
||||
- `workspace/platform_auth.py` — per-workspace token resolver
|
||||
|
||||
### Why 75% (vs 86% total)
|
||||
|
||||
The total floor averages ~6000 lines across `workspace/`. A single MCP
|
||||
file could drop to ~50% with no CI complaint as long as other modules
|
||||
compensate. The per-file floor closes that distribution gap. 75% sits
|
||||
below current actuals (80–96% as of 2026-05-04) — strictly additive,
|
||||
no existing PR fails.
|
||||
|
||||
### Python ratchet plan
|
||||
|
||||
| Date | Total | Per-file critical | Notes |
|
||||
|---|---|---|---|
|
||||
| 2026-05-04 | 86% | 75% | Initial gate (this file). |
|
||||
| 2026-06-04 | 86% | 80% | First ratchet — at-floor files must catch up. |
|
||||
| 2026-07-04 | 88% | 85% | |
|
||||
| 2026-08-04 | 90% | 90% | Target steady-state. |
|
||||
|
||||
### Why this Python gate exists
|
||||
|
||||
Issue #2790, after the PR #2766 → PR #2771 cycle. PR #2766 added
|
||||
multi-workspace routing through `a2a_tools.py` + `a2a_mcp_server.py`,
|
||||
shipped to main with green CI, but the dispatcher silently dropped a
|
||||
load-bearing kwarg for 4 of 9 tools — caught only by post-merge code
|
||||
review. The structural drift gate (`test_dispatcher_schema_drift.py`,
|
||||
PR #2791) catches the schema↔dispatcher mismatch class; this floor
|
||||
catches the broader "MCP-critical file regressed" class.
|
||||
@@ -1,28 +0,0 @@
|
||||
# Top-level Makefile — convenience wrappers around docker compose.
|
||||
#
|
||||
# Most molecule-core dev work happens via these shortcuts. CI doesn't
|
||||
# use this Makefile; CI calls docker compose / go test directly so the
|
||||
# Makefile can evolve without breaking the build.
|
||||
|
||||
.PHONY: help dev up down logs build test
|
||||
|
||||
help: ## Show this help.
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-12s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
dev: ## Start the full stack with air hot-reload for the platform service.
|
||||
docker compose -f docker-compose.yml -f docker-compose.dev.yml up
|
||||
|
||||
up: ## Start the full stack in production-shape mode (no air, normal Dockerfile).
|
||||
docker compose up
|
||||
|
||||
down: ## Stop the stack and remove containers (volumes preserved).
|
||||
docker compose down
|
||||
|
||||
logs: ## Tail logs from all services (Ctrl-C to detach).
|
||||
docker compose logs -f
|
||||
|
||||
build: ## Force a fresh build of the platform image (no cache).
|
||||
docker compose build --no-cache platform
|
||||
|
||||
test: ## Run Go unit tests in workspace-server/.
|
||||
cd workspace-server && go test -race ./...
|
||||
@@ -1,7 +1,7 @@
|
||||
<div align="center">
|
||||
|
||||
<p>
|
||||
<img src="./docs/assets/branding/molecule-icon.svg" alt="Molecule AI" width="160" />
|
||||
<img src="./docs/assets/branding/molecule-icon.png" alt="Molecule AI Icon Logo" width="160" />
|
||||
</p>
|
||||
|
||||
<p>
|
||||
@@ -39,8 +39,8 @@
|
||||
<a href="./docs/agent-runtime/workspace-runtime.md"><strong>Workspace Runtime</strong></a>
|
||||
</p>
|
||||
|
||||
[](https://railway.app/new/template?template=https://git.moleculesai.app/molecule-ai/molecule-core)
|
||||
[](https://render.com/deploy?repo=https://git.moleculesai.app/molecule-ai/molecule-core)
|
||||
[](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
[](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
|
||||
</div>
|
||||
|
||||
@@ -53,8 +53,8 @@ Molecule AI is the most powerful way to govern an AI agent organization in produ
|
||||
It combines the parts that are usually scattered across demos, internal glue code, and framework-specific tooling into one product:
|
||||
|
||||
- one org-native control plane for teams, roles, hierarchy, and lifecycle
|
||||
- one runtime layer that lets **eight** agent runtimes — LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, **Hermes**, **Gemini CLI**, and OpenClaw — run side by side behind one workspace contract
|
||||
- one memory model that keeps recall, sharing, and skill evolution aligned with organizational boundaries (Memory v2 backed by pgvector for semantic recall)
|
||||
- one runtime layer that lets LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, and OpenClaw run side by side
|
||||
- one memory model that keeps recall, sharing, and skill evolution aligned with organizational boundaries
|
||||
- one operational surface for observing, pausing, restarting, inspecting, and improving live workspaces
|
||||
|
||||
Most teams can build a workflow, a strong single agent, a coding agent, or a custom multi-agent graph.
|
||||
@@ -75,7 +75,7 @@ You do not wire collaboration paths by hand. Hierarchy defines the default commu
|
||||
|
||||
### 3. Runtime choice stops being a dead-end decision
|
||||
|
||||
LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, Hermes, Gemini CLI, and OpenClaw can all plug into the same workspace abstraction. Teams can standardize governance without forcing every group onto one runtime.
|
||||
LangGraph, DeepAgents, Claude Code, CrewAI, AutoGen, and OpenClaw can all plug into the same workspace abstraction. Teams can standardize governance without forcing every group onto one runtime.
|
||||
|
||||
### 4. Memory is treated like infrastructure
|
||||
|
||||
@@ -117,8 +117,6 @@ Molecule AI is not trying to replace the frameworks below. It is the system that
|
||||
| **Claude Code** | Shipping on `main` | Real coding workflows, CLI-native continuity | Secure workspace abstraction, A2A delegation, org boundaries, shared control plane |
|
||||
| **CrewAI** | Shipping on `main` | Role-based crews | Persistent workspace identity, policy consistency, shared canvas and registry |
|
||||
| **AutoGen** | Shipping on `main` | Assistant/tool orchestration | Standardized deployment, hierarchy-aware collaboration, shared ops plane |
|
||||
| **Hermes 4** | Shipping on `main` | Hybrid reasoning, native tools, json_schema (NousResearch/hermes-agent) | Option B upstream hook, A2A bridge to OpenAI-compat API, multi-provider provider derivation |
|
||||
| **Gemini CLI** | Shipping on `main` | Google Gemini CLI continuity | Workspace lifecycle, A2A, hierarchy-aware collaboration, shared ops plane |
|
||||
| **OpenClaw** | Shipping on `main` | CLI-native runtime with its own session model | Workspace lifecycle, templates, activity logs, topology-aware collaboration |
|
||||
| **NemoClaw** | WIP on `feat/nemoclaw-t4-docker` | NVIDIA-oriented runtime path | Planned to join the same abstraction once merged; not yet part of `main` |
|
||||
|
||||
@@ -184,10 +182,9 @@ The result is not just “an agent that learns.” It is **an organization that
|
||||
|
||||
## What Ships In `main`
|
||||
|
||||
### Canvas (v4)
|
||||
### Canvas
|
||||
|
||||
- Next.js 15 + React Flow + Zustand
|
||||
- **warm-paper theme system** — light / dark / follow-system, SSR cookie + nonce'd boot script + ThemeProvider; terminal + code surfaces stay dark unconditionally
|
||||
- drag-to-nest team building
|
||||
- empty-state deployment + onboarding wizard
|
||||
- template palette
|
||||
@@ -196,9 +193,8 @@ The result is not just “an agent that learns.” It is **an organization that
|
||||
|
||||
### Platform
|
||||
|
||||
- Go 1.25 / Gin control plane (80+ HTTP endpoints + Gorilla WebSocket fanout)
|
||||
- workspace CRUD and provisioning (pluggable Provisioner — Docker locally, EC2 + SSM in production)
|
||||
- **A2A response path is a typed discriminated union (RFC #2967)** — frozen dataclasses + total parser; 100% unit + adversarial fuzz coverage
|
||||
- Go/Gin control plane
|
||||
- workspace CRUD and provisioning
|
||||
- registry and heartbeats
|
||||
- browser-safe A2A proxy
|
||||
- team expansion/collapse
|
||||
@@ -208,10 +204,10 @@ The result is not just “an agent that learns.” It is **an organization that
|
||||
|
||||
### Runtime
|
||||
|
||||
- unified `workspace/` image; thin AMI in production (us-east-2)
|
||||
- adapter-driven execution across **8 runtimes** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw)
|
||||
- unified `workspace/` image
|
||||
- adapter-driven execution
|
||||
- Agent Card registration
|
||||
- awareness-backed memory integration; **Memory v2 backed by pgvector** for semantic recall
|
||||
- awareness-backed memory integration
|
||||
- plugin-mounted shared rules/skills
|
||||
- hot-reloadable local skills
|
||||
- coordinator-only delegation path
|
||||
@@ -225,21 +221,6 @@ The result is not just “an agent that learns.” It is **an organization that
|
||||
- runtime tiers
|
||||
- direct workspace inspection through terminal and files
|
||||
|
||||
### SaaS (via [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane))
|
||||
|
||||
- multi-tenant on AWS EC2 + Neon (per-tenant Postgres branch) + Cloudflare Tunnels (per-tenant, no public ports)
|
||||
- WorkOS AuthKit + Stripe Checkout + Customer Portal
|
||||
- AWS KMS envelope encryption (DB / Redis connection strings); AWS Secrets Manager for tenant bootstrap
|
||||
- `tenant_resources` audit table + 30-min boot-event-aware reconciler — every CF / AWS lifecycle event recorded, claim vs live state diffed
|
||||
|
||||
### Bring your own Claude Code session (via [`molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel))
|
||||
|
||||
- Claude Code plugin that bridges Molecule A2A traffic into a local Claude Code session via MCP
|
||||
- subscribe to one or more workspaces; peer messages surface as conversation turns; replies route back through Molecule's A2A
|
||||
- no tunnel, no public endpoint — the plugin self-registers each watched workspace as `delivery_mode=poll` and long-polls `/activity?since_id=…`
|
||||
- multi-tenant friendly: one plugin install can watch workspaces across multiple Molecule tenants (`MOLECULE_PLATFORM_URLS` per-workspace)
|
||||
- install via the standard marketplace flow: `/plugin marketplace add Molecule-AI/molecule-mcp-claude-channel` → `/plugin install molecule-channel@molecule-mcp-claude-channel`
|
||||
|
||||
## Built For Teams That Need More Than A Demo
|
||||
|
||||
Molecule AI is especially strong when you need to run:
|
||||
@@ -252,49 +233,33 @@ Molecule AI is especially strong when you need to run:
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
Canvas (Next.js 15, warm-paper :3000) <--HTTP / WS--> Platform (Go 1.25 :8080) <---> Postgres + Redis
|
||||
| |
|
||||
| +--> Provisioner: Docker (local) / EC2 + SSM (prod)
|
||||
| +--> bundles · templates · secrets · KMS
|
||||
Canvas (Next.js :3000) <--HTTP / WS--> Platform (Go :8080) <---> Postgres + Redis
|
||||
| |
|
||||
| +--> Docker provisioner / bundles / templates / secrets
|
||||
|
|
||||
+------------------------- shows ------------------------> workspaces, teams, tasks, traces, events
|
||||
+-------------------- shows --------------------> workspaces, teams, tasks, traces, events
|
||||
|
||||
Workspace Runtime (Python ≥3.11, image with adapters)
|
||||
- 8 adapters: LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / Hermes / Gemini CLI / OpenClaw
|
||||
- Agent Card + A2A server (typed-SSOT response path, RFC #2967)
|
||||
- heartbeat + activity + awareness-backed memory (Memory v2 — pgvector semantic recall)
|
||||
Workspace Runtime (Python image with adapters)
|
||||
- LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / OpenClaw
|
||||
- Agent Card + A2A server
|
||||
- heartbeat + activity + awareness-backed memory
|
||||
- skills + plugins + hot reload
|
||||
|
||||
SaaS Control Plane (molecule-controlplane, private)
|
||||
- per-tenant EC2 + Neon (Postgres branch) + Cloudflare Tunnel
|
||||
- WorkOS · Stripe · KMS · AWS Secrets Manager
|
||||
- tenant_resources audit + 30-min reconciler
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
git clone https://git.moleculesai.app/molecule-ai/molecule-core.git
|
||||
cd molecule-core
|
||||
|
||||
cp .env.example .env
|
||||
# Defaults boot the stack locally out of the box. See .env.example for
|
||||
# production hardening knobs (ADMIN_TOKEN, SECRETS_ENCRYPTION_KEY, etc.).
|
||||
git clone https://github.com/Molecule-AI/molecule-monorepo.git
|
||||
cd molecule-monorepo
|
||||
|
||||
./infra/scripts/setup.sh
|
||||
# Boots Postgres (:5432), Redis (:6379), Langfuse (:3001),
|
||||
# and Temporal (:7233 gRPC, :8233 UI) on the shared
|
||||
# `molecule-core-net` Docker network. Temporal runs with
|
||||
# `molecule-monorepo-net` Docker network. Temporal runs with
|
||||
# no auth on localhost — dev-only; production must gate it.
|
||||
#
|
||||
# Also populates the template/plugin registry by cloning every repo
|
||||
# listed in manifest.json into workspace-configs-templates/,
|
||||
# org-templates/, and plugins/. Requires jq — install via
|
||||
# `brew install jq` (macOS) or `apt install jq` (Debian). Idempotent:
|
||||
# re-runs skip any target dir that's already populated.
|
||||
|
||||
cd workspace-server
|
||||
go run ./cmd/server # applies pending migrations on first boot
|
||||
go run ./cmd/server
|
||||
|
||||
cd ../canvas
|
||||
npm install
|
||||
@@ -319,20 +284,12 @@ Then open `http://localhost:3000`:
|
||||
- [Workspace Runtime](./docs/agent-runtime/workspace-runtime.md)
|
||||
- [Canvas UI](./docs/frontend/canvas.md)
|
||||
- [Local Development](./docs/development/local-development.md)
|
||||
- [Backend Parity Matrix](./docs/architecture/backends.md) — Docker vs EC2 feature parity tracker
|
||||
- [Testing Strategy](./docs/engineering/testing-strategy.md) — tiered coverage floors, not blanket 100%
|
||||
- [PR Hygiene](./docs/engineering/pr-hygiene.md) — small PRs, clean branches, cherry-pick on drift
|
||||
- [Engineering Postmortems](./docs/engineering/) — architecture + testing lessons from real incidents
|
||||
- [Ecosystem Watch](./docs/ecosystem-watch.md) — adjacent projects we track (Holaboss, Hermes, gstack, …)
|
||||
- [Glossary](./docs/glossary.md) — how we use "harness", "workspace", "plugin", "flow" vs. ecosystem neighbors
|
||||
|
||||
## Current Scope
|
||||
|
||||
The current `main` branch ships the core platform, Canvas v4 (warm-paper themed), Memory v2 (pgvector semantic recall), the typed-SSOT A2A response path (RFC #2967), **eight production adapters** (Claude Code, Hermes, Gemini CLI, LangGraph, DeepAgents, CrewAI, AutoGen, OpenClaw), skill lifecycle, and operational surfaces.
|
||||
|
||||
The companion private repo [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) provides the SaaS surface — multi-tenant orchestration on EC2 + Neon + Cloudflare Tunnels, KMS envelope encryption, WorkOS auth, Stripe billing, and a `tenant_resources` audit table with a 30-min reconciler.
|
||||
|
||||
Adjacent runtime work such as **NemoClaw** remains branch-level until merged, and this README keeps that distinction explicit on purpose.
|
||||
The current `main` branch already includes the core platform, canvas, memory model, six production adapters, skill lifecycle, and operational surfaces. Adjacent runtime work such as **NemoClaw** remains branch-level until merged, and this README keeps that distinction explicit on purpose.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+25
-63
@@ -1,7 +1,7 @@
|
||||
<div align="center">
|
||||
|
||||
<p>
|
||||
<img src="./docs/assets/branding/molecule-icon.svg" alt="Molecule AI" width="160" />
|
||||
<img src="./docs/assets/branding/molecule-icon.png" alt="Molecule AI 图案 Logo" width="160" />
|
||||
</p>
|
||||
|
||||
<p>
|
||||
@@ -38,8 +38,8 @@
|
||||
<a href="./docs/agent-runtime/workspace-runtime.md"><strong>Workspace Runtime</strong></a>
|
||||
</p>
|
||||
|
||||
[](https://railway.app/new/template?template=https://git.moleculesai.app/molecule-ai/molecule-core)
|
||||
[](https://render.com/deploy?repo=https://git.moleculesai.app/molecule-ai/molecule-core)
|
||||
[](https://railway.app/new/template?template=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
[](https://render.com/deploy?repo=https://github.com/Molecule-AI/molecule-monorepo)
|
||||
|
||||
</div>
|
||||
|
||||
@@ -52,8 +52,8 @@ Molecule AI 是目前最强的 AI Agent 组织治理方案之一,用来把 age
|
||||
它把过去分散在 demo、内部胶水代码和各类 framework 私有工具里的关键能力,收敛成一个产品:
|
||||
|
||||
- 一套组织原生 control plane,管理团队、角色、层级和生命周期
|
||||
- 一套 runtime abstraction,让 **8 个** agent runtime —— LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、**Hermes**、**Gemini CLI**、OpenClaw —— 共用一套 workspace 契约
|
||||
- 一套与组织边界对齐的 memory 模型,把 recall、sharing 和 skill evolution 放进同一体系(Memory v2 由 pgvector 支撑语义召回)
|
||||
- 一套 runtime abstraction,让 LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、OpenClaw 并存运行
|
||||
- 一套与组织边界对齐的 memory 模型,把 recall、sharing 和 skill evolution 放进同一体系
|
||||
- 一套面向线上 workspace 的运维面,统一完成观测、暂停、重启、检查和持续改进
|
||||
|
||||
今天很多团队能做好 workflow、单 agent、coding agent,或者自定义 multi-agent graph 中的一种。
|
||||
@@ -74,7 +74,7 @@ Molecule AI 填的就是这个空白。
|
||||
|
||||
### 3. Runtime 选择不再是死路
|
||||
|
||||
LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、Hermes、Gemini CLI、OpenClaw 都可以挂到同一个 workspace abstraction 下。团队可以统一治理方式,而不必统一到底层 runtime。
|
||||
LangGraph、DeepAgents、Claude Code、CrewAI、AutoGen、OpenClaw 都可以挂到同一个 workspace abstraction 下。团队可以统一治理方式,而不必统一到底层 runtime。
|
||||
|
||||
### 4. Memory 被当成基础设施来做
|
||||
|
||||
@@ -116,8 +116,6 @@ Molecule AI 并不是要替代下面这些 framework,而是把它们纳入更
|
||||
| **Claude Code** | `main` 已支持 | 真实编码工作流、CLI-native continuity | 安全 workspace 抽象、A2A delegation、组织边界、共享 control plane |
|
||||
| **CrewAI** | `main` 已支持 | 角色型 crew 模式清晰 | 持久 workspace 身份、统一策略、共享 Canvas 和 registry |
|
||||
| **AutoGen** | `main` 已支持 | assistant/tool orchestration | 统一部署、层级协作、共享运维平面 |
|
||||
| **Hermes 4** | `main` 已支持 | 混合推理、原生工具调用、json_schema 输出(NousResearch/hermes-agent) | Option B 上游 hook、A2A 桥接 OpenAI 兼容 API、多 provider 自动派生 |
|
||||
| **Gemini CLI** | `main` 已支持 | Google Gemini CLI 持续会话 | workspace 生命周期、A2A、层级感知协作、共享运维平面 |
|
||||
| **OpenClaw** | `main` 已支持 | CLI-native runtime,自有 session 模型 | workspace 生命周期、templates、activity logs、拓扑感知协作 |
|
||||
| **NemoClaw** | `feat/nemoclaw-t4-docker` 分支 WIP | NVIDIA 方向 runtime 路线 | 计划并入同一抽象层,但当前还不是 `main` 已合并能力 |
|
||||
|
||||
@@ -183,10 +181,9 @@ Molecule AI 并不是要替代下面这些 framework,而是把它们纳入更
|
||||
|
||||
## `main` 分支已经具备什么
|
||||
|
||||
### Canvas(v4)
|
||||
### Canvas
|
||||
|
||||
- Next.js 15 + React Flow + Zustand
|
||||
- **warm-paper 主题系统** —— light / dark / 跟随系统;SSR cookie + nonce'd boot 脚本 + ThemeProvider;终端与代码面板始终保持深色
|
||||
- drag-to-nest 团队构建
|
||||
- empty state + onboarding wizard
|
||||
- template palette
|
||||
@@ -195,9 +192,8 @@ Molecule AI 并不是要替代下面这些 framework,而是把它们纳入更
|
||||
|
||||
### Platform
|
||||
|
||||
- Go 1.25 / Gin control plane(80+ HTTP 端点 + Gorilla WebSocket fanout)
|
||||
- workspace CRUD 和 provisioning(可插拔 Provisioner —— 本地 Docker、生产 EC2 + SSM)
|
||||
- **A2A 响应路径已收敛为类型化的判别联合(RFC #2967)** —— 冻结 dataclass + 全量 parser;100% 单元测试 + 对抗性 fuzz 覆盖
|
||||
- Go/Gin control plane
|
||||
- workspace CRUD 和 provisioning
|
||||
- registry 与 heartbeat
|
||||
- 浏览器安全的 A2A proxy
|
||||
- team expansion/collapse
|
||||
@@ -207,10 +203,10 @@ Molecule AI 并不是要替代下面这些 framework,而是把它们纳入更
|
||||
|
||||
### Runtime
|
||||
|
||||
- 统一 `workspace/` 镜像;生产环境采用 thin AMI(us-east-2)
|
||||
- adapter 驱动执行,覆盖 **8 个 runtime**(Claude Code、Hermes、Gemini CLI、LangGraph、DeepAgents、CrewAI、AutoGen、OpenClaw)
|
||||
- 统一 `workspace/` 镜像
|
||||
- adapter 驱动执行
|
||||
- Agent Card 注册
|
||||
- awareness-backed memory;**Memory v2 由 pgvector 支撑**语义召回
|
||||
- awareness-backed memory
|
||||
- plugin 挂载共享 rules/skills
|
||||
- 本地 skills 热加载
|
||||
- coordinator-only delegation 路径
|
||||
@@ -224,21 +220,6 @@ Molecule AI 并不是要替代下面这些 framework,而是把它们纳入更
|
||||
- runtime tiers
|
||||
- 终端与文件层面的 workspace 直接排障
|
||||
|
||||
### SaaS(由 [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) 提供)
|
||||
|
||||
- 多租户运行在 AWS EC2 + Neon(每租户一个 Postgres branch)+ Cloudflare Tunnels(每租户一条隧道,对外不开任何端口)
|
||||
- WorkOS AuthKit + Stripe Checkout + Customer Portal
|
||||
- AWS KMS 信封加密(DB / Redis 连接串);AWS Secrets Manager 负责租户 bootstrap
|
||||
- `tenant_resources` 审计表 + 30 分钟 boot-event-aware reconciler —— 每个 CF / AWS lifecycle 事件都有记录,每 30 分钟比对 claim 与实际状态
|
||||
|
||||
### 在 Claude Code 里直接接入(由 [`molecule-mcp-claude-channel`](https://git.moleculesai.app/molecule-ai/molecule-mcp-claude-channel) 提供)
|
||||
|
||||
- 把 Molecule A2A 流量桥接到本地 Claude Code 会话的 MCP 插件
|
||||
- 订阅一个或多个 workspace;peer 的消息会以 user-turn 出现,回复会经 Molecule A2A 路由出去
|
||||
- 无需公网隧道、无需公开端点 —— 插件启动时自动把每个 watched workspace 注册成 `delivery_mode=poll`,长轮询 `/activity?since_id=…`
|
||||
- 多租户友好:单次安装即可同时 watch 跨多个 Molecule 租户的 workspace(`MOLECULE_PLATFORM_URLS` 按 workspace 配置)
|
||||
- 通过标准 marketplace 流程安装:`/plugin marketplace add Molecule-AI/molecule-mcp-claude-channel` → `/plugin install molecule-channel@molecule-mcp-claude-channel`
|
||||
|
||||
## 适合什么团队
|
||||
|
||||
Molecule AI 特别适合下面这些场景:
|
||||
@@ -251,48 +232,33 @@ Molecule AI 特别适合下面这些场景:
|
||||
## 架构总览
|
||||
|
||||
```text
|
||||
Canvas (Next.js 15, warm-paper :3000) <--HTTP / WS--> Platform (Go 1.25 :8080) <---> Postgres + Redis
|
||||
| |
|
||||
| +--> Provisioner: Docker (本地) / EC2 + SSM (生产)
|
||||
| +--> bundles · templates · secrets · KMS
|
||||
Canvas (Next.js :3000) <--HTTP / WS--> Platform (Go :8080) <---> Postgres + Redis
|
||||
| |
|
||||
| +--> Docker provisioner / bundles / templates / secrets
|
||||
|
|
||||
+------------------------- 展示 ------------------------> workspaces, teams, tasks, traces, events
|
||||
+-------------------- 展示 --------------------> workspaces, teams, tasks, traces, events
|
||||
|
||||
Workspace Runtime (Python ≥3.11,含 adapter 集合的镜像)
|
||||
- 8 个 adapter: LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / Hermes / Gemini CLI / OpenClaw
|
||||
- Agent Card + A2A server(typed-SSOT 响应路径,RFC #2967)
|
||||
- heartbeat + activity + awareness-backed memory(Memory v2 —— pgvector 语义召回)
|
||||
Workspace Runtime (Python image with adapters)
|
||||
- LangGraph / DeepAgents / Claude Code / CrewAI / AutoGen / OpenClaw
|
||||
- Agent Card + A2A server
|
||||
- heartbeat + activity + awareness-backed memory
|
||||
- skills + plugins + hot reload
|
||||
|
||||
SaaS Control Plane (molecule-controlplane,私有)
|
||||
- 每租户 EC2 + Neon (Postgres branch) + Cloudflare Tunnel
|
||||
- WorkOS · Stripe · KMS · AWS Secrets Manager
|
||||
- tenant_resources 审计 + 30 分钟 reconciler
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
git clone https://git.moleculesai.app/molecule-ai/molecule-core.git
|
||||
cd molecule-core
|
||||
|
||||
cp .env.example .env
|
||||
# 默认值即可在本地启动整套服务。.env.example 里有针对生产部署的
|
||||
# 安全配置说明(ADMIN_TOKEN、SECRETS_ENCRYPTION_KEY 等)。
|
||||
git clone https://github.com/Molecule-AI/molecule-monorepo.git
|
||||
cd molecule-monorepo
|
||||
|
||||
./infra/scripts/setup.sh
|
||||
# 启动 Postgres (:5432)、Redis (:6379)、Langfuse (:3001)
|
||||
# 以及 Temporal (:7233 gRPC, :8233 UI),全部挂在共享的
|
||||
# `molecule-core-net` Docker 网络上。Temporal 默认无鉴权,
|
||||
# `molecule-monorepo-net` Docker 网络上。Temporal 默认无鉴权,
|
||||
# 仅用于本地开发;生产环境必须加 mTLS / API Key。
|
||||
#
|
||||
# 同时会根据 manifest.json 拉取所有模板/插件仓库到
|
||||
# workspace-configs-templates/、org-templates/、plugins/ 三个目录。
|
||||
# 需要安装 jq:`brew install jq`(macOS)或 `apt install jq`(Debian)。
|
||||
# 脚本幂等:已经存在内容的目录会被跳过,可以安全重跑。
|
||||
|
||||
cd workspace-server
|
||||
go run ./cmd/server # 首次启动会自动跑 schema_migrations 里未应用的迁移
|
||||
go run ./cmd/server
|
||||
|
||||
cd ../canvas
|
||||
npm install
|
||||
@@ -321,11 +287,7 @@ npm run dev
|
||||
|
||||
## 当前范围说明
|
||||
|
||||
当前 `main` 已经包含核心平台、Canvas v4(warm-paper 主题)、Memory v2(pgvector 语义召回)、typed-SSOT A2A 响应路径(RFC #2967)、**8 个正式 adapter**(Claude Code、Hermes、Gemini CLI、LangGraph、DeepAgents、CrewAI、AutoGen、OpenClaw)、skill lifecycle,以及主要运维面。
|
||||
|
||||
配套的私有仓库 [`molecule-controlplane`](https://git.moleculesai.app/molecule-ai/molecule-controlplane) 提供 SaaS 层 —— 多租户编排(EC2 + Neon + Cloudflare Tunnels)、KMS 信封加密、WorkOS 鉴权、Stripe 计费,以及 `tenant_resources` 审计表加 30 分钟 reconciler。
|
||||
|
||||
像 **NemoClaw** 这样的相邻 runtime 路线仍然属于分支级工作,只有合并后才会进入正式支持列表,这里会明确区分。
|
||||
当前 `main` 已经包含核心平台、Canvas、memory model、6 个正式 adapter、skill lifecycle 和主要运维面。像 **NemoClaw** 这样的相邻 runtime 路线仍然属于分支级工作,只有合并后才会进入正式支持列表,这里会明确区分。
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
# Excluded from `docker build` context. Without this, the COPY . . step in
|
||||
# canvas/Dockerfile clobbers the freshly-installed node_modules with the
|
||||
# host's (potentially broken / wrong-arch) copy — the @tailwindcss/oxide
|
||||
# native binary disagreed and broke `next build`.
|
||||
node_modules
|
||||
.next
|
||||
.git
|
||||
*.log
|
||||
.env*
|
||||
!.env.example
|
||||
+5
-9
@@ -1,11 +1,7 @@
|
||||
FROM node:22-alpine@sha256:cb15fca92530d7ac113467696cf1001208dac49c3c64355fd1348c11a88ddf8f AS builder
|
||||
FROM node:20-alpine AS builder
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json* ./
|
||||
# `npm ci` (not `install`) for lockfile-exact reproducibility.
|
||||
# `--include=optional` ensures the platform-specific @tailwindcss/oxide
|
||||
# native binary lands — without it, postcss fails with "Cannot read
|
||||
# properties of undefined (reading 'All')" at build time.
|
||||
RUN npm ci --include=optional
|
||||
RUN npm install
|
||||
COPY . .
|
||||
ARG NEXT_PUBLIC_PLATFORM_URL=http://localhost:8080
|
||||
ARG NEXT_PUBLIC_WS_URL=ws://localhost:8080/ws
|
||||
@@ -15,7 +11,7 @@ ENV NEXT_PUBLIC_WS_URL=$NEXT_PUBLIC_WS_URL
|
||||
ENV NEXT_PUBLIC_ADMIN_TOKEN=$NEXT_PUBLIC_ADMIN_TOKEN
|
||||
RUN npm run build
|
||||
|
||||
FROM node:22-alpine@sha256:cb15fca92530d7ac113467696cf1001208dac49c3c64355fd1348c11a88ddf8f
|
||||
FROM node:20-alpine
|
||||
WORKDIR /app
|
||||
COPY --from=builder /app/.next/standalone ./
|
||||
COPY --from=builder /app/.next/static ./.next/static
|
||||
@@ -24,7 +20,7 @@ COPY --from=builder /app/public ./public
|
||||
EXPOSE 3000
|
||||
ENV PORT=3000
|
||||
ENV HOSTNAME="0.0.0.0"
|
||||
# Non-root runtime — use addgroup/adduser without fixed GID/UID to avoid conflicts with base image
|
||||
RUN addgroup canvas 2>/dev/null || true && adduser -G canvas -s /bin/sh -D canvas 2>/dev/null || true
|
||||
# Non-root runtime — node image defaults to root, explicitly drop.
|
||||
RUN addgroup -g 1000 canvas && adduser -u 1000 -G canvas -s /bin/sh -D canvas
|
||||
USER canvas
|
||||
CMD ["node", "server.js"]
|
||||
|
||||
@@ -4,9 +4,10 @@
|
||||
"rsc": true,
|
||||
"tsx": true,
|
||||
"tailwind": {
|
||||
"config": "tailwind.config.ts",
|
||||
"css": "src/app/globals.css",
|
||||
"baseColor": "neutral",
|
||||
"cssVariables": true
|
||||
"baseColor": "zinc",
|
||||
"cssVariables": false
|
||||
},
|
||||
"aliases": {
|
||||
"components": "@/components",
|
||||
|
||||
@@ -1,293 +0,0 @@
|
||||
/**
|
||||
* Playwright global setup for the staging canvas E2E.
|
||||
*
|
||||
* Provisions a fresh staging org per run (POST /cp/admin/orgs), fetches
|
||||
* the per-tenant admin token, provisions one hermes workspace, waits
|
||||
* for online, then exports:
|
||||
*
|
||||
* STAGING_TENANT_URL https://<slug>.staging.moleculesai.app
|
||||
* STAGING_WORKSPACE_ID UUID of the hermes workspace
|
||||
* STAGING_TENANT_TOKEN per-tenant admin bearer (for spec requests)
|
||||
* STAGING_SLUG org slug (used by teardown)
|
||||
*
|
||||
* Required env:
|
||||
* MOLECULE_CP_URL default: https://staging-api.moleculesai.app
|
||||
* MOLECULE_ADMIN_TOKEN CP admin bearer (Railway staging
|
||||
* CP_ADMIN_API_TOKEN). Drives provision +
|
||||
* tenant-token retrieval + teardown via a
|
||||
* single credential.
|
||||
* STAGING_TENANT_DOMAIN default: staging.moleculesai.app — the
|
||||
* DNS suffix the CP provisioner writes for
|
||||
* staging tenants. Override only when
|
||||
* running this harness against a non-default
|
||||
* zone.
|
||||
*/
|
||||
|
||||
import type { FullConfig } from "@playwright/test";
|
||||
import { writeFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
|
||||
const CP_URL = process.env.MOLECULE_CP_URL || "https://staging-api.moleculesai.app";
|
||||
const ADMIN_TOKEN = process.env.MOLECULE_ADMIN_TOKEN;
|
||||
const STAGING = process.env.CANVAS_E2E_STAGING === "1";
|
||||
// Tenant DNS zone for staging. CP provisioner registers DNS as
|
||||
// `<slug>.staging.moleculesai.app` (see internal/provisioner/ec2.go's
|
||||
// EC2 provisioner: DNS log line). The previous default of plain
|
||||
// `moleculesai.app` matched prod tenant naming and silently broke
|
||||
// every staging E2E at the TLS readiness step — DNS literally didn't
|
||||
// resolve, fetch threw NXDOMAIN, waitFor saw null on every poll, and
|
||||
// the harness wedged at TLS_TIMEOUT_MS instead of failing loud.
|
||||
const TENANT_DOMAIN = process.env.STAGING_TENANT_DOMAIN || "staging.moleculesai.app";
|
||||
|
||||
// Tenant cold boot on staging regularly takes 12-15 min when the
|
||||
// workspace-server Docker image isn't already cached on the AMI. Raised
|
||||
// to 20 min to match tests/e2e/test_staging_full_saas.sh (PR #1930)
|
||||
// after repeated "tenant provision: timed out after 900s" flakes
|
||||
// were blocking staging→main syncs on 2026-04-24.
|
||||
const PROVISION_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
const WORKSPACE_ONLINE_TIMEOUT_MS = 20 * 60 * 1000;
|
||||
|
||||
// TLS readiness depends on (1) Cloudflare DNS propagation through the
|
||||
// edge, (2) the tenant's CF Tunnel registering the new hostname, (3)
|
||||
// CF's edge ACME cert provisioning + cache. Each of these layers can
|
||||
// add 1-3 min on its own under heavy staging load. Bumped 10→15 min
|
||||
// after a burst of canary failures correlated with CP changes (#2090).
|
||||
// Stays below the 20-min PROVISION_TIMEOUT envelope so a genuinely-
|
||||
// stuck tenant fails-loud at the provision step rather than
|
||||
// masquerading as a TLS issue. Kept aligned with
|
||||
// tests/e2e/test_staging_full_saas.sh.
|
||||
const TLS_TIMEOUT_MS = 15 * 60 * 1000;
|
||||
|
||||
async function jsonFetch(
|
||||
url: string,
|
||||
init: RequestInit = {},
|
||||
): Promise<{ status: number; body: any }> {
|
||||
const res = await fetch(url, {
|
||||
...init,
|
||||
headers: { "Content-Type": "application/json", ...(init.headers || {}) },
|
||||
});
|
||||
let body: any = null;
|
||||
try {
|
||||
body = await res.json();
|
||||
} catch {
|
||||
/* non-JSON */
|
||||
}
|
||||
return { status: res.status, body };
|
||||
}
|
||||
|
||||
async function waitFor<T>(
|
||||
op: () => Promise<T | null>,
|
||||
deadlineMs: number,
|
||||
intervalMs: number,
|
||||
desc: string,
|
||||
): Promise<T> {
|
||||
const deadline = Date.now() + deadlineMs;
|
||||
while (Date.now() < deadline) {
|
||||
const v = await op();
|
||||
if (v !== null) return v;
|
||||
await new Promise((r) => setTimeout(r, intervalMs));
|
||||
}
|
||||
throw new Error(`${desc}: timed out after ${Math.round(deadlineMs / 1000)}s`);
|
||||
}
|
||||
|
||||
function makeSlug(): string {
|
||||
const y = new Date().toISOString().slice(0, 10).replace(/-/g, "");
|
||||
const rand = Math.random().toString(36).slice(2, 8);
|
||||
return `e2e-canvas-${y}-${rand}`.slice(0, 32);
|
||||
}
|
||||
|
||||
export default async function globalSetup(_config: FullConfig): Promise<void> {
|
||||
if (!STAGING) {
|
||||
console.log("[staging-setup] CANVAS_E2E_STAGING not set, skipping");
|
||||
return;
|
||||
}
|
||||
if (!ADMIN_TOKEN) {
|
||||
throw new Error(
|
||||
"MOLECULE_ADMIN_TOKEN required (Railway staging CP_ADMIN_API_TOKEN)",
|
||||
);
|
||||
}
|
||||
|
||||
const slug = makeSlug();
|
||||
const adminAuth = { Authorization: `Bearer ${ADMIN_TOKEN}` };
|
||||
console.log(`[staging-setup] Using slug=${slug}`);
|
||||
|
||||
// Write the state file FIRST, before any CP call. Teardown (both
|
||||
// Playwright globalTeardown and the workflow safety-net) reads this
|
||||
// file to identify the slug it must clean up. If we wait until the
|
||||
// end of setup to write it (the previous behavior), a crash during
|
||||
// any of steps 1-6 leaves the org orphaned in CP with no record on
|
||||
// disk — forcing the workflow safety-net into a pattern-sweep over
|
||||
// every `e2e-canvas-<date>-*` org, which races with concurrent
|
||||
// canvas-E2E runs and deletes their live tenants. Race observed
|
||||
// 2026-04-30 on PR #2264 staging→main: three real-test runs killed
|
||||
// each other's tenants mid-test, surfacing as `getaddrinfo ENOTFOUND`
|
||||
// when CP cleaned up the just-deleted DNS record.
|
||||
const stateFile = join(process.cwd(), ".playwright-staging-state.json");
|
||||
writeFileSync(stateFile, JSON.stringify({ slug }, null, 2));
|
||||
|
||||
// 1. Create org via admin endpoint — no WorkOS session needed
|
||||
const create = await jsonFetch(`${CP_URL}/cp/admin/orgs`, {
|
||||
method: "POST",
|
||||
headers: adminAuth,
|
||||
body: JSON.stringify({
|
||||
slug,
|
||||
name: `E2E Canvas ${slug}`,
|
||||
owner_user_id: `e2e-runner:${slug}`,
|
||||
}),
|
||||
});
|
||||
if (create.status >= 400) {
|
||||
throw new Error(
|
||||
`POST /cp/admin/orgs ${create.status}: ${JSON.stringify(create.body)}`,
|
||||
);
|
||||
}
|
||||
console.log(`[staging-setup] Org created: ${slug}`);
|
||||
|
||||
// 2. Wait for tenant running (admin-orgs list is the status source).
|
||||
//
|
||||
// The CP /cp/admin/orgs endpoint returns each org with an
|
||||
// `instance_status` field (handlers/admin.go:adminOrgSummary,
|
||||
// sourced from `org_instances.status`). NOT `status` — there's no
|
||||
// top-level `status` on the row at all. A previous version of this
|
||||
// test polled `row.status`, which was always undefined, so this
|
||||
// waitFor never resolved truthy and the harness invariably timed
|
||||
// out at 1200s — masking real CP bugs (see #242 chain) AND
|
||||
// surviving real CP fixes alike.
|
||||
// Capture the org UUID alongside the running check — every request
|
||||
// we send to the tenant URL after this point needs an
|
||||
// X-Molecule-Org-Id header (see workspace-server middleware/tenant_guard.go).
|
||||
// Without it, TenantGuard returns 404 ("must not be inferable by
|
||||
// probing other orgs' machines"). The CP returns the id on the
|
||||
// admin-orgs row; capture it here while we're already polling.
|
||||
let orgID = "";
|
||||
await waitFor<boolean>(
|
||||
async () => {
|
||||
const r = await jsonFetch(`${CP_URL}/cp/admin/orgs`, { headers: adminAuth });
|
||||
if (r.status !== 200) return null;
|
||||
const row = (r.body?.orgs || []).find((o: any) => o.slug === slug);
|
||||
if (!row) return null;
|
||||
if (row.instance_status === "running") {
|
||||
orgID = row.id;
|
||||
return true;
|
||||
}
|
||||
if (row.instance_status === "failed") {
|
||||
// Dump every diagnostic field the admin row carries — boot stage,
|
||||
// last error, terraform/SSM state, etc. The bare slug message used
|
||||
// to surface ZERO context, so triaging a failed provision meant
|
||||
// re-running locally to repro. Now the failure log carries enough
|
||||
// to point at the right subsystem (CP/AWS/SSM/runtime) without a
|
||||
// second round-trip.
|
||||
throw new Error(
|
||||
`provision failed: ${slug} — admin-orgs row: ${JSON.stringify(row)}`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
PROVISION_TIMEOUT_MS,
|
||||
15_000,
|
||||
"tenant provision",
|
||||
);
|
||||
if (!orgID) {
|
||||
throw new Error(`expected admin-orgs row to carry id, got empty for slug=${slug}`);
|
||||
}
|
||||
console.log(`[staging-setup] Tenant running (org_id=${orgID})`);
|
||||
|
||||
// 3. Fetch per-tenant admin token
|
||||
const tokRes = await jsonFetch(
|
||||
`${CP_URL}/cp/admin/orgs/${slug}/admin-token`,
|
||||
{ headers: adminAuth },
|
||||
);
|
||||
if (tokRes.status !== 200 || !tokRes.body?.admin_token) {
|
||||
throw new Error(
|
||||
`tenant-token fetch ${tokRes.status}: ${JSON.stringify(tokRes.body)}`,
|
||||
);
|
||||
}
|
||||
const tenantToken: string = tokRes.body.admin_token;
|
||||
const tenantURL = `https://${slug}.${TENANT_DOMAIN}`;
|
||||
console.log(`[staging-setup] Tenant URL: ${tenantURL}`);
|
||||
|
||||
// 4. TLS readiness
|
||||
await waitFor<boolean>(
|
||||
async () => {
|
||||
try {
|
||||
const res = await fetch(`${tenantURL}/health`, {
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
return res.ok ? true : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
TLS_TIMEOUT_MS,
|
||||
5_000,
|
||||
"tenant TLS",
|
||||
);
|
||||
|
||||
// 5. Provision workspace
|
||||
//
|
||||
// tenantAuth carries TWO headers, both required:
|
||||
// - Authorization: Bearer <admin-token> — wsAdmin middleware gate
|
||||
// - X-Molecule-Org-Id: <uuid> — TenantGuard cross-org gate
|
||||
// Missing the org-id header silently 404s every non-allowlisted
|
||||
// route, with no body and no security headers. The 404 is intentional
|
||||
// (existence-non-inference) which makes it look like a missing route.
|
||||
const tenantAuth = {
|
||||
"Authorization": `Bearer ${tenantToken}`,
|
||||
"X-Molecule-Org-Id": orgID,
|
||||
};
|
||||
const ws = await jsonFetch(`${tenantURL}/workspaces`, {
|
||||
method: "POST",
|
||||
headers: tenantAuth,
|
||||
body: JSON.stringify({
|
||||
name: "E2E Canvas Test",
|
||||
runtime: "hermes",
|
||||
tier: 2,
|
||||
model: "gpt-4o",
|
||||
}),
|
||||
});
|
||||
if (ws.status >= 400 || !ws.body?.id) {
|
||||
throw new Error(`Workspace create ${ws.status}: ${JSON.stringify(ws.body)}`);
|
||||
}
|
||||
const workspaceId = ws.body.id as string;
|
||||
console.log(`[staging-setup] Workspace created: ${workspaceId}`);
|
||||
|
||||
// 6. Wait for workspace online
|
||||
await waitFor<boolean>(
|
||||
async () => {
|
||||
const r = await jsonFetch(`${tenantURL}/workspaces/${workspaceId}`, {
|
||||
headers: tenantAuth,
|
||||
});
|
||||
if (r.status !== 200) return null;
|
||||
if (r.body?.status === "online") return true;
|
||||
if (r.body?.status === "failed") {
|
||||
// last_sample_error is often empty when the failure happens before
|
||||
// the agent emits a sample (e.g. boot crash, image pull error,
|
||||
// missing PYTHONPATH, OpenAI quota at startup). Dumping the full
|
||||
// body gives triage the boot_stage / last_error / image fields it
|
||||
// needs without a second probe. Otherwise this propagates as a
|
||||
// bare "Workspace failed: " — the exact useless message that
|
||||
// sent #2632 to the issue tracker.
|
||||
const detail = r.body.last_sample_error
|
||||
? r.body.last_sample_error
|
||||
: `(no last_sample_error) full body: ${JSON.stringify(r.body)}`;
|
||||
throw new Error(`Workspace failed: ${detail}`);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
WORKSPACE_ONLINE_TIMEOUT_MS,
|
||||
10_000,
|
||||
"workspace online",
|
||||
);
|
||||
console.log(`[staging-setup] Workspace online`);
|
||||
|
||||
// 7. Hand state off to tests + teardown — overwrite the slug-only
|
||||
// bootstrap state with the full state spec tests need.
|
||||
writeFileSync(
|
||||
stateFile,
|
||||
JSON.stringify({ slug, tenantURL, workspaceId, tenantToken }, null, 2),
|
||||
);
|
||||
process.env.STAGING_SLUG = slug;
|
||||
process.env.STAGING_TENANT_URL = tenantURL;
|
||||
process.env.STAGING_WORKSPACE_ID = workspaceId;
|
||||
process.env.STAGING_TENANT_TOKEN = tenantToken;
|
||||
console.log(`[staging-setup] Ready — ${stateFile}`);
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
/**
|
||||
* Staging canvas E2E — opens each of the 13 workspace-panel tabs against a
|
||||
* fresh staging org provisioned in the global setup. Asserts each tab
|
||||
* renders without throwing and captures a screenshot for visual review.
|
||||
*
|
||||
* Auth model: the tenant platform's AdminAuth middleware accepts a bearer
|
||||
* token OR a WorkOS session cookie. Playwright can't mint a WorkOS
|
||||
* session, so we feed the per-tenant admin token (fetched in global
|
||||
* setup via GET /cp/admin/orgs/:slug/admin-token) as an Authorization:
|
||||
* Bearer header via context.setExtraHTTPHeaders(). Every browser
|
||||
* request inherits the header.
|
||||
*
|
||||
* Known SaaS gaps — documented in #1369 and allowed to render errored
|
||||
* content without failing the test (the gate is "no hard crash, no
|
||||
* 'Failed to load' toast"):
|
||||
* - Files tab: empty (platform can't docker exec into a remote EC2)
|
||||
* - Terminal tab: WS connect fails
|
||||
* - Peers tab: 401 without workspace-scoped token
|
||||
*/
|
||||
|
||||
import { test, expect } from "@playwright/test";
|
||||
|
||||
// Tab ids as declared in canvas/src/components/SidePanel.tsx TABS.
|
||||
const TAB_IDS = [
|
||||
"chat",
|
||||
"activity",
|
||||
"details",
|
||||
"skills",
|
||||
"terminal",
|
||||
"config",
|
||||
"schedule",
|
||||
"channels",
|
||||
"files",
|
||||
"memory",
|
||||
"traces",
|
||||
"events",
|
||||
"audit",
|
||||
] as const;
|
||||
|
||||
const STAGING = process.env.CANVAS_E2E_STAGING === "1";
|
||||
|
||||
test.skip(!STAGING, "CANVAS_E2E_STAGING not set — skipping staging-only tests");
|
||||
|
||||
test.describe("staging canvas tabs", () => {
|
||||
test("each workspace-panel tab renders without error", async ({
|
||||
page,
|
||||
context,
|
||||
}) => {
|
||||
const tenantURL = process.env.STAGING_TENANT_URL;
|
||||
const tenantToken = process.env.STAGING_TENANT_TOKEN;
|
||||
const workspaceId = process.env.STAGING_WORKSPACE_ID;
|
||||
|
||||
if (!tenantURL || !tenantToken || !workspaceId) {
|
||||
throw new Error(
|
||||
"staging-setup.ts did not export STAGING_TENANT_URL / STAGING_TENANT_TOKEN / STAGING_WORKSPACE_ID — did global setup run?",
|
||||
);
|
||||
}
|
||||
|
||||
// Attach the per-tenant admin bearer to every outbound request.
|
||||
// The tenant platform's AdminAuth middleware accepts this; no
|
||||
// WorkOS session needed.
|
||||
await context.setExtraHTTPHeaders({
|
||||
Authorization: `Bearer ${tenantToken}`,
|
||||
});
|
||||
|
||||
// canvas/src/components/AuthGate.tsx fetches /cp/auth/me on mount
|
||||
// and redirects to the login page on 401. The bearer header above
|
||||
// is for platform API calls — it does NOT satisfy /cp/auth/me,
|
||||
// which is cookie-based (WorkOS session). Without this mock, the
|
||||
// canvas page mounts AuthGate, sees 401 from /cp/auth/me, and
|
||||
// redirects away from the tenant URL before the React Flow root
|
||||
// ever renders. The [aria-label] selector wait then times out.
|
||||
//
|
||||
// Intercept /cp/auth/me + return a fake Session shape so AuthGate
|
||||
// resolves to "authenticated" and renders {children}. The session
|
||||
// contents are cosmetic — the canvas only inspects org_id/user_id
|
||||
// in a few places that don't fail when these are dummy values.
|
||||
await context.route("**/cp/auth/me", (route) =>
|
||||
route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: JSON.stringify({
|
||||
user_id: `e2e-test-user-${workspaceId}`,
|
||||
org_id: "e2e-test-org",
|
||||
email: "e2e@test.local",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
// Universal 401 → empty-200 fallback (defense-in-depth).
|
||||
//
|
||||
// The original product bug was canvas/src/lib/api.ts:62-74 calling
|
||||
// `redirectToLogin` on EVERY 401 — a single workspace-scoped 401
|
||||
// (e.g. /workspaces/:id/peers, /plugins) yanked the user (and the
|
||||
// test) to AuthKit. That's now fixed at the source: api.ts probes
|
||||
// /cp/auth/me before redirecting, so a 401 from a non-auth path
|
||||
// with a live session throws a regular error instead.
|
||||
//
|
||||
// This route handler stays as a SAFETY NET, not the primary
|
||||
// defense:
|
||||
// 1. It silences resource-load console noise from the browser
|
||||
// (those messages don't include the URL — useless in
|
||||
// diagnostics, captured by the filter in the assertion
|
||||
// block but having no 401s reach the network is cleaner).
|
||||
// 2. It guards against panels that DON'T have try/catch around
|
||||
// their api calls — an unhandled rejection would surface
|
||||
// as console.error → fail the assertion. Panels SHOULD
|
||||
// handle errors, but until they're all audited, this is
|
||||
// the test's belt to api.ts's braces.
|
||||
//
|
||||
// Pass-through real responses; swap 401s for 200 + empty body.
|
||||
// Skip /cp/auth/me (mocked above) and non-fetch resources
|
||||
// (HTML/JS/CSS bundles that should NOT be intercepted).
|
||||
await context.route("**", async (route, request) => {
|
||||
if (request.resourceType() !== "fetch") {
|
||||
return route.fallback();
|
||||
}
|
||||
// /cp/auth/me is mocked above with a fixed Session shape — let
|
||||
// that handler win without us round-tripping the network.
|
||||
if (request.url().includes("/cp/auth/me")) {
|
||||
return route.fallback();
|
||||
}
|
||||
let resp;
|
||||
try {
|
||||
resp = await route.fetch();
|
||||
} catch {
|
||||
return route.fallback();
|
||||
}
|
||||
if (resp.status() !== 401) {
|
||||
return route.fulfill({ response: resp });
|
||||
}
|
||||
const lastSeg =
|
||||
new URL(request.url()).pathname.split("/").filter(Boolean).pop() || "";
|
||||
const looksLikeList = !/^[0-9a-f-]{8,}$/.test(lastSeg);
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: "application/json",
|
||||
body: looksLikeList ? "[]" : "{}",
|
||||
});
|
||||
});
|
||||
|
||||
const consoleErrors: string[] = [];
|
||||
page.on("console", (msg) => {
|
||||
if (msg.type() === "error") {
|
||||
consoleErrors.push(msg.text());
|
||||
}
|
||||
});
|
||||
|
||||
// Capture the URL of any failed network request so a "Failed to load
|
||||
// resource: 404" console message we filter out below leaves a
|
||||
// breadcrumb. Browser console messages for resource-load failures
|
||||
// omit the URL, so we'd otherwise be flying blind. Logged to the
|
||||
// test's stdout (visible in the workflow log under the failed step).
|
||||
page.on("requestfailed", (req) => {
|
||||
console.log(`[e2e/requestfailed] ${req.method()} ${req.url()}: ${req.failure()?.errorText ?? "?"}`);
|
||||
});
|
||||
page.on("response", (res) => {
|
||||
if (res.status() >= 400) {
|
||||
console.log(`[e2e/response-${res.status()}] ${res.request().method()} ${res.url()}`);
|
||||
}
|
||||
});
|
||||
|
||||
// waitUntil="networkidle" is wrong here — the canvas keeps a
|
||||
// WebSocket open + polls /events and /workspaces every few
|
||||
// seconds, so the network is *never* idle for 500ms. page.goto
|
||||
// would hang until its 45s default timeout. "domcontentloaded"
|
||||
// returns as soon as the HTML is parsed; React hydration + the
|
||||
// selector wait below is what actually gates ready-for-interaction.
|
||||
await page.goto(tenantURL, { waitUntil: "domcontentloaded" });
|
||||
|
||||
// Canvas hydration races WebSocket connect + /workspaces fetch.
|
||||
// Wait for the React Flow canvas wrapper (always present once
|
||||
// hydrated, even with zero workspaces) or the hydration-error
|
||||
// banner — whichever wins first. Previous version of this wait
|
||||
// used `[role="tablist"]`, but that selector only appears AFTER
|
||||
// a workspace node is clicked (which happens below at L100), so
|
||||
// the wait would always time out at 45s before any meaningful
|
||||
// failure surfaced.
|
||||
await page.waitForSelector(
|
||||
'[aria-label="Molecule AI workspace canvas"], [data-testid="hydration-error"]',
|
||||
{ timeout: 45_000 },
|
||||
);
|
||||
|
||||
const hydrationErr = await page
|
||||
.locator('[data-testid="hydration-error"]')
|
||||
.count();
|
||||
expect(
|
||||
hydrationErr,
|
||||
"canvas hydration failed — check staging CP + tenant reachability",
|
||||
).toBe(0);
|
||||
|
||||
// Click the workspace node to open the side panel. Try a data
|
||||
// attribute first, fall back to a generic role-based selector so
|
||||
// the test doesn't break when the node-card markup changes.
|
||||
const byDataAttr = page.locator(`[data-workspace-id="${workspaceId}"]`).first();
|
||||
if ((await byDataAttr.count()) > 0) {
|
||||
await byDataAttr.click({ timeout: 10_000 });
|
||||
} else {
|
||||
const firstNode = page
|
||||
.locator('[role="button"][aria-label*="Workspace" i]')
|
||||
.first();
|
||||
await firstNode.click({ timeout: 10_000 });
|
||||
}
|
||||
|
||||
await page.waitForSelector('[role="tablist"]', { timeout: 15_000 });
|
||||
|
||||
for (const tabId of TAB_IDS) {
|
||||
await test.step(`tab: ${tabId}`, async () => {
|
||||
const tabButton = page.locator(`#tab-${tabId}`);
|
||||
// The TABS bar is `overflow-x-auto` (SidePanel.tsx:~tabs
|
||||
// wrapper) — tabs after position ~3 are clipped behind the
|
||||
// right-edge fade gradient on smaller viewports. Playwright's
|
||||
// `toBeVisible()` returns false for clipped elements, so a
|
||||
// bare visibility check fails on `skills` and later tabs in
|
||||
// CI. scrollIntoViewIfNeeded brings the button into view
|
||||
// before the visibility check, mirroring what SidePanel's own
|
||||
// keyboard handler does on arrow-key navigation.
|
||||
await tabButton.scrollIntoViewIfNeeded({ timeout: 5_000 });
|
||||
await expect(
|
||||
tabButton,
|
||||
`tab-${tabId} button missing — TABS list may have drifted`,
|
||||
).toBeVisible({ timeout: 5_000 });
|
||||
await tabButton.click();
|
||||
|
||||
const panel = page.locator(`#panel-${tabId}`);
|
||||
await expect(panel, `panel for ${tabId} never rendered`).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
|
||||
// "Failed to load" toast = hard crash. Known SaaS-mode gaps
|
||||
// (Files empty, Terminal disconnected, Peers 401) surface as
|
||||
// in-panel content, not toasts.
|
||||
const errorToasts = await page
|
||||
.locator('[role="alert"]:has-text("Failed to load")')
|
||||
.count();
|
||||
expect(errorToasts, `tab ${tabId}: "Failed to load" toast`).toBe(0);
|
||||
|
||||
await page.screenshot({
|
||||
path: `test-results/staging-tab-${tabId}.png`,
|
||||
fullPage: false,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Aggregate console-error budget. Known-noisy sources whitelisted:
|
||||
// Sentry, Vercel analytics, WS reconnects (expected on SaaS
|
||||
// terminal), favicon 404 (cosmetic), and the browser's generic
|
||||
// "Failed to load resource: ... 404" message which never includes
|
||||
// the URL — uninformative on its own and impossible to filter
|
||||
// meaningfully without a URL. The page.on('requestfailed') +
|
||||
// page.on('response>=400') logging above captures the actual URLs
|
||||
// so a real bug still leaves a breadcrumb in the workflow log;
|
||||
// a real exception (panel crash, JS error) surfaces as a typed
|
||||
// error with file path which the filter still catches.
|
||||
const appErrors = consoleErrors.filter(
|
||||
(msg) =>
|
||||
!msg.includes("sentry") &&
|
||||
!msg.includes("vercel") &&
|
||||
!msg.includes("WebSocket") &&
|
||||
!msg.includes("favicon") &&
|
||||
!msg.includes("molecule-icon.png") && // cosmetic 404
|
||||
!msg.includes("Failed to load resource"),
|
||||
);
|
||||
expect(
|
||||
appErrors,
|
||||
`unexpected console errors:\n${appErrors.join("\n")}`,
|
||||
).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,70 +0,0 @@
|
||||
/**
|
||||
* Playwright global teardown — deletes the staging org provisioned by
|
||||
* staging-setup.ts via DELETE /cp/admin/tenants/:slug. Runs on success AND
|
||||
* failure (Playwright calls globalTeardown regardless).
|
||||
*
|
||||
* The workflow's always()-step safety net also catches orphan orgs
|
||||
* tagged with the run ID, so this is the primary cleanup and the
|
||||
* workflow step is the belt-and-braces backup.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, unlinkSync } from "fs";
|
||||
import { join } from "path";
|
||||
|
||||
const CP_URL = process.env.MOLECULE_CP_URL || "https://staging-api.moleculesai.app";
|
||||
const ADMIN_TOKEN = process.env.MOLECULE_ADMIN_TOKEN;
|
||||
const STAGING = process.env.CANVAS_E2E_STAGING === "1";
|
||||
|
||||
export default async function globalTeardown(): Promise<void> {
|
||||
if (!STAGING) return;
|
||||
if (!ADMIN_TOKEN) {
|
||||
console.warn("[staging-teardown] no MOLECULE_ADMIN_TOKEN, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
const stateFile = join(process.cwd(), ".playwright-staging-state.json");
|
||||
if (!existsSync(stateFile)) {
|
||||
// staging-setup writes this file as its first action, before any
|
||||
// CP call. Missing here means setup never ran (CANVAS_E2E_STAGING
|
||||
// unset, or ran in a different cwd) — there's no slug we created
|
||||
// that needs cleaning up.
|
||||
console.warn("[staging-teardown] no state file — nothing to tear down");
|
||||
return;
|
||||
}
|
||||
|
||||
let slug: string;
|
||||
try {
|
||||
const state = JSON.parse(readFileSync(stateFile, "utf-8"));
|
||||
slug = state.slug;
|
||||
} catch (e) {
|
||||
console.warn(`[staging-teardown] state file unreadable: ${e}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[staging-teardown] Deleting org ${slug}...`);
|
||||
try {
|
||||
const res = await fetch(`${CP_URL}/cp/admin/tenants/${slug}`, {
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
Authorization: `Bearer ${ADMIN_TOKEN}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ confirm: slug }),
|
||||
});
|
||||
if (res.ok) {
|
||||
console.log(`[staging-teardown] ${slug} deleted`);
|
||||
} else {
|
||||
console.warn(
|
||||
`[staging-teardown] DELETE returned ${res.status} (may already be gone)`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(`[staging-teardown] DELETE failed: ${e}`);
|
||||
}
|
||||
|
||||
try {
|
||||
unlinkSync(stateFile);
|
||||
} catch {
|
||||
/* non-fatal */
|
||||
}
|
||||
}
|
||||
@@ -1,155 +1,7 @@
|
||||
import type { NextConfig } from "next";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
// Load NEXT_PUBLIC_* vars from the monorepo root .env so a fresh
|
||||
// `pnpm dev` works without a per-developer canvas/.env.local. Next.js
|
||||
// only auto-loads .env from the project root by default — but our
|
||||
// canonical config (NEXT_PUBLIC_PLATFORM_URL, NEXT_PUBLIC_WS_URL,
|
||||
// MOLECULE_ENV, etc.) lives at the monorepo root, gitignored, shared
|
||||
// by the Go platform binary. Without this, the canvas falls back to
|
||||
// `window.location` (`ws://localhost:3000/ws`) and the WS pill stays
|
||||
// "Reconnecting" forever because Next.js dev doesn't serve /ws.
|
||||
//
|
||||
// Mirrors workspace-server/cmd/server/dotenv.go's monorepo-rooted .env
|
||||
// loader. Both processes look for the SAME marker (`workspace-server/
|
||||
// go.mod`) so a developer renaming or relocating the repo only has to
|
||||
// update one heuristic. Production is unaffected: `output: "standalone"`
|
||||
// bakes resolved env into the build, and the marker file isn't shipped.
|
||||
loadMonorepoEnv();
|
||||
// Boot-time matched-pair guard for ADMIN_TOKEN / NEXT_PUBLIC_ADMIN_TOKEN.
|
||||
// When ADMIN_TOKEN is set on the workspace-server (server-side bearer
|
||||
// gate, wsauth_middleware.go ~L245), the canvas MUST send the matching
|
||||
// NEXT_PUBLIC_ADMIN_TOKEN as `Authorization: Bearer ...` on every API
|
||||
// call. If only one is set, every workspace API call 401s silently —
|
||||
// the canvas hydrates with empty data and the user sees a broken page
|
||||
// with no console hint about the auth-config mismatch.
|
||||
//
|
||||
// Pre-fix the matched-pair contract was descriptive only (a comment in
|
||||
// .env): future devs/agents could re-misconfigure with one of the two
|
||||
// unset and silently 401. Closes the post-PR-#174 self-review gap.
|
||||
//
|
||||
// Warn-only (not exit) — production canvas Docker images bake these
|
||||
// vars into the build at image-build time, and a missed pair there
|
||||
// would still emit the warning at runtime via the standalone server's
|
||||
// startup. Killing the process on misconfiguration would turn a
|
||||
// recoverable auth issue into a hard crashloop.
|
||||
checkAdminTokenPair();
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
output: "standalone",
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
|
||||
function loadMonorepoEnv() {
|
||||
const root = findMonorepoRoot(__dirname);
|
||||
if (!root) return;
|
||||
const envPath = join(root, ".env");
|
||||
if (!existsSync(envPath)) return;
|
||||
const body = readFileSync(envPath, "utf8");
|
||||
let loaded = 0;
|
||||
let skipped = 0;
|
||||
for (const line of body.split(/\r?\n/)) {
|
||||
const kv = parseLine(line);
|
||||
if (!kv) continue;
|
||||
const [k, v] = kv;
|
||||
// Existing env wins. NOTE: an explicitly-set empty string
|
||||
// (`KEY=` exported from a parent shell, where Node represents it
|
||||
// as `""` not `undefined`) counts as "set" — we keep the empty
|
||||
// value rather than backfilling from the file. Matches Go's
|
||||
// os.LookupEnv check in workspace-server/cmd/server/dotenv.go so
|
||||
// both processes treat the same input identically. Operators who
|
||||
// want the file value to win must `unset KEY` in the launching
|
||||
// shell.
|
||||
if (process.env[k] !== undefined) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
process.env[k] = v;
|
||||
loaded++;
|
||||
}
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[next.config] loaded ${loaded} vars from ${envPath} (${skipped} already set in env)`,
|
||||
);
|
||||
}
|
||||
|
||||
// Boot-time matched-pair guard. Runs after .env has been loaded so the
|
||||
// check sees the post-load state. The two env vars must be set or
|
||||
// unset together; one-without-the-other is the silent-401 footgun.
|
||||
//
|
||||
// Treats empty string ("") as unset. An explicitly-empty `KEY=` in
|
||||
// .env counts as set-to-empty in `process.env`, but for auth purposes
|
||||
// an empty bearer token is equivalent to no token — so both
|
||||
// `ADMIN_TOKEN=` and an unset ADMIN_TOKEN are equivalent relative to
|
||||
// the matched-pair invariant.
|
||||
//
|
||||
// Returns void; side effect is the console.error warning. Kept as a
|
||||
// separate function (exported) so a future test can reset env, call
|
||||
// this, and assert on captured stderr.
|
||||
export function checkAdminTokenPair(): void {
|
||||
const serverSet = !!process.env.ADMIN_TOKEN;
|
||||
const clientSet = !!process.env.NEXT_PUBLIC_ADMIN_TOKEN;
|
||||
if (serverSet === clientSet) return;
|
||||
// Distinct messages so the operator can tell which half is missing
|
||||
// — the fix is symmetric (set the other one) but the diagnostic
|
||||
// mentions which side is currently set so they don't have to grep.
|
||||
if (serverSet && !clientSet) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
"[next.config] ADMIN_TOKEN is set but NEXT_PUBLIC_ADMIN_TOKEN is not — " +
|
||||
"canvas will 401 against workspace-server because the bearer header " +
|
||||
"is never attached. Set both to the same value, or unset both.",
|
||||
);
|
||||
} else {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
"[next.config] NEXT_PUBLIC_ADMIN_TOKEN is set but ADMIN_TOKEN is not — " +
|
||||
"workspace-server will reject the bearer because no AdminAuth gate " +
|
||||
"is configured. Set both to the same value, or unset both.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function findMonorepoRoot(start: string): string | null {
|
||||
let dir = start;
|
||||
for (let i = 0; i < 6; i++) {
|
||||
if (existsSync(join(dir, "workspace-server", "go.mod"))) return dir;
|
||||
const parent = dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Mirror of workspace-server/cmd/server/dotenv.go's parseDotEnvLine
|
||||
// — same rules so the two loaders agree on every line in the shared
|
||||
// .env. If you change one parser, change the other.
|
||||
function parseLine(raw: string): [string, string] | null {
|
||||
let line = raw.replace(/^/, "").trim();
|
||||
if (line === "" || line.startsWith("#")) return null;
|
||||
// `export ` prefix uses a literal space — `export\tFOO=bar` with a
|
||||
// tab is intentionally rejected, matching the Go mirror in
|
||||
// workspace-server/cmd/server/dotenv.go. Shells emit the prefix
|
||||
// with a space; tabs would only appear in hand-mangled files.
|
||||
if (line.startsWith("export ")) line = line.slice("export ".length).trimStart();
|
||||
const eq = line.indexOf("=");
|
||||
if (eq <= 0) return null;
|
||||
const k = line.slice(0, eq).trim();
|
||||
let v = line.slice(eq + 1).replace(/^[ \t]+/, "");
|
||||
if (v.length >= 2 && (v[0] === '"' || v[0] === "'")) {
|
||||
const quote = v[0];
|
||||
const end = v.indexOf(quote, 1);
|
||||
if (end >= 0) return [k, v.slice(1, end)];
|
||||
// unterminated — fall through to bare-value handling
|
||||
}
|
||||
for (let i = 0; i < v.length; i++) {
|
||||
if (v[i] !== "#") continue;
|
||||
if (i === 0 || v[i - 1] === " " || v[i - 1] === "\t") {
|
||||
v = v.slice(0, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return [k, v.trim()];
|
||||
}
|
||||
|
||||
Generated
+1580
-944
File diff suppressed because it is too large
Load Diff
+7
-9
@@ -3,12 +3,11 @@
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev --turbopack -p 3000",
|
||||
"dev": "next dev --turbopack",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage"
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@radix-ui/react-alert-dialog": "^1.1.15",
|
||||
@@ -32,15 +31,14 @@
|
||||
"@playwright/test": "^1.59.1",
|
||||
"@testing-library/jest-dom": "^6.6.0",
|
||||
"@testing-library/react": "^16.1.0",
|
||||
"@types/node": "^25.6.0",
|
||||
"@types/node": "^22.0.0",
|
||||
"@types/react": "^19.0.0",
|
||||
"@types/react-dom": "^19.0.0",
|
||||
"@vitejs/plugin-react": "^6.0.1",
|
||||
"@vitest/coverage-v8": "^4.1.5",
|
||||
"@tailwindcss/postcss": "^4.0.0",
|
||||
"jsdom": "^29.1.1",
|
||||
"postcss": "^8.5.13",
|
||||
"tailwindcss": "^4.0.0",
|
||||
"autoprefixer": "^10.4.0",
|
||||
"jsdom": "^25.0.0",
|
||||
"postcss": "^8.4.0",
|
||||
"tailwindcss": "^3.4.0",
|
||||
"typescript": "^5.7.0",
|
||||
"vitest": "^4.1.2"
|
||||
}
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
/**
|
||||
* Playwright config for staging canvas E2E.
|
||||
*
|
||||
* Separate from playwright.config.ts (local dev) so:
|
||||
* - globalSetup / globalTeardown don't run for every local `pnpm test`
|
||||
* - Retries + timeouts can be longer (staging is remote + shared)
|
||||
* - baseURL is dynamic (set by globalSetup → STAGING_TENANT_URL)
|
||||
*
|
||||
* Invoked by the e2e-staging-canvas GH Actions workflow:
|
||||
* npx playwright test --config=playwright.staging.config.ts
|
||||
*/
|
||||
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./e2e",
|
||||
// Only the staging-*.spec.ts files run under this config. The smoke +
|
||||
// unit specs (chat-separation, filestab-smoke, etc.) stay on the local
|
||||
// config so they don't hit staging.
|
||||
testMatch: /staging-.*\.spec\.ts/,
|
||||
// Global setup provisions the org; budget generously because EC2 boot
|
||||
// is ~5 min and can drift to 10+ on cold AMI days.
|
||||
timeout: 120_000,
|
||||
expect: { timeout: 15_000 },
|
||||
fullyParallel: false,
|
||||
// A transient network blip shouldn't cost us the whole run. Two retries
|
||||
// mean up to 3 attempts — staging flakes fall within that budget.
|
||||
retries: 2,
|
||||
// One worker: the setup provisions exactly one org/workspace, and
|
||||
// parallel specs would fight over the shared workspace selector state.
|
||||
workers: 1,
|
||||
globalSetup: "./e2e/staging-setup.ts",
|
||||
globalTeardown: "./e2e/staging-teardown.ts",
|
||||
use: {
|
||||
// STAGING_TENANT_URL gets written to process.env in global setup, but
|
||||
// Playwright resolves baseURL before setup runs. We read it inside
|
||||
// each spec instead — don't hard-code here.
|
||||
headless: true,
|
||||
screenshot: "only-on-failure",
|
||||
video: "retain-on-failure",
|
||||
trace: "retain-on-failure",
|
||||
navigationTimeout: 45_000,
|
||||
actionTimeout: 15_000,
|
||||
},
|
||||
reporter: [
|
||||
["list"],
|
||||
["html", { outputFolder: "playwright-report-staging", open: "never" }],
|
||||
],
|
||||
projects: [{ name: "chromium", use: { browserName: "chromium" } }],
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
module.exports = {
|
||||
plugins: {
|
||||
"@tailwindcss/postcss": {},
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
* - Polling: provisioning orgs schedule a 5s refresh (fake timers)
|
||||
*/
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
||||
import { act } from "react";
|
||||
import { render, screen, cleanup } from "@testing-library/react";
|
||||
import { render, screen, waitFor, cleanup } from "@testing-library/react";
|
||||
|
||||
// ── Hoisted mocks ────────────────────────────────────────────────────────────
|
||||
// vi.mock factories are hoisted above imports; any captured references must
|
||||
@@ -128,10 +127,14 @@ describe("/orgs — auth guard", () => {
|
||||
describe("/orgs — error state", () => {
|
||||
it("shows error + Retry button when /cp/orgs fails", async () => {
|
||||
mockFetchSession.mockResolvedValue({ userId: "u-1" });
|
||||
mockFetch.mockResolvedValueOnce(notOk(500, "db down"));
|
||||
mockFetch.mockImplementationOnce(() =>
|
||||
Promise.reject(new Error("GET /cp/orgs: 500"))
|
||||
);
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
expect(screen.getByText(/Error:/)).toBeTruthy();
|
||||
// PR #1243 replaced waitFor polling with vi.advanceTimersByTimeAsync(50),
|
||||
// which fires the timer but does not guarantee React render flush completes
|
||||
// before the assertion runs. Restores waitFor for the error-state test.
|
||||
await waitFor(() => expect(screen.getByText(/Error:/)).toBeTruthy());
|
||||
expect(screen.getByRole("button", { name: /retry/i })).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -141,7 +144,7 @@ describe("/orgs — empty list", () => {
|
||||
mockFetchSession.mockResolvedValue({ userId: "u-1" });
|
||||
mockFetch.mockResolvedValueOnce(okJson({ orgs: [] }));
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
expect(screen.getByText(/don't have any organizations/i)).toBeTruthy();
|
||||
expect(screen.getByRole("button", { name: /create organization/i })).toBeTruthy();
|
||||
});
|
||||
@@ -168,7 +171,7 @@ describe("/orgs — CTAs by status", () => {
|
||||
})
|
||||
);
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
const link = screen.getByRole("link", { name: /open/i }) as HTMLAnchorElement;
|
||||
expect(link.href).toBe("https://acme.moleculesai.app/");
|
||||
});
|
||||
@@ -191,7 +194,7 @@ describe("/orgs — CTAs by status", () => {
|
||||
})
|
||||
);
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
const link = screen.getByRole("link", {
|
||||
name: /complete payment/i,
|
||||
}) as HTMLAnchorElement;
|
||||
@@ -216,7 +219,7 @@ describe("/orgs — CTAs by status", () => {
|
||||
})
|
||||
);
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
const link = screen.getByRole("link", {
|
||||
name: /contact support/i,
|
||||
}) as HTMLAnchorElement;
|
||||
@@ -245,7 +248,7 @@ describe("/orgs — post-checkout banner", () => {
|
||||
})
|
||||
);
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
expect(screen.getByText(/Payment confirmed/i)).toBeTruthy();
|
||||
// URL must be rewritten to drop the ?checkout flag so reload doesn't re-show the banner
|
||||
expect(replaceState).toHaveBeenCalled();
|
||||
@@ -257,7 +260,7 @@ describe("/orgs — post-checkout banner", () => {
|
||||
mockFetchSession.mockResolvedValue({ userId: "u-1" });
|
||||
mockFetch.mockResolvedValueOnce(okJson({ orgs: [] }));
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
expect(screen.getByText(/don't have any organizations/i)).toBeTruthy();
|
||||
expect(screen.queryByText(/Payment confirmed/i)).toBeNull();
|
||||
});
|
||||
@@ -268,7 +271,7 @@ describe("/orgs — fetch includes credentials + timeout signal", () => {
|
||||
mockFetchSession.mockResolvedValue({ userId: "u-1" });
|
||||
mockFetch.mockResolvedValueOnce(okJson({ orgs: [] }));
|
||||
render(<OrgsPage />);
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(50); });
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
const callArgs = mockFetch.mock.calls.find((c) =>
|
||||
String(c[0]).includes("/cp/orgs")
|
||||
);
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
/**
|
||||
* Canvas /api/buildinfo — version-display endpoint mirroring
|
||||
* workspace-server's /buildinfo. Lets `curl <url>/api/buildinfo`
|
||||
* confirm which git SHA is live on a canvas deployment.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
||||
import { GET } from "../route";
|
||||
|
||||
const ENV_KEYS = ["VERCEL_GIT_COMMIT_SHA", "VERCEL_GIT_COMMIT_REF", "VERCEL_ENV"];
|
||||
|
||||
describe("GET /api/buildinfo", () => {
|
||||
let saved: Record<string, string | undefined>;
|
||||
|
||||
beforeEach(() => {
|
||||
saved = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
|
||||
for (const k of ENV_KEYS) delete process.env[k];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
for (const k of ENV_KEYS) {
|
||||
if (saved[k] === undefined) delete process.env[k];
|
||||
else process.env[k] = saved[k];
|
||||
}
|
||||
});
|
||||
|
||||
it("returns dev sentinel when Vercel env vars are unset", async () => {
|
||||
const res = await GET();
|
||||
const body = await res.json();
|
||||
expect(body).toEqual({ git_sha: "dev", git_ref: "", vercel_env: "local" });
|
||||
});
|
||||
|
||||
it("reports the SHA Vercel injected at build time", async () => {
|
||||
process.env.VERCEL_GIT_COMMIT_SHA = "abc1234567890";
|
||||
process.env.VERCEL_GIT_COMMIT_REF = "main";
|
||||
process.env.VERCEL_ENV = "production";
|
||||
const res = await GET();
|
||||
const body = await res.json();
|
||||
expect(body.git_sha).toBe("abc1234567890");
|
||||
expect(body.git_ref).toBe("main");
|
||||
expect(body.vercel_env).toBe("production");
|
||||
});
|
||||
|
||||
it("returns 200 status and JSON content type", async () => {
|
||||
const res = await GET();
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.headers.get("content-type")).toContain("application/json");
|
||||
});
|
||||
});
|
||||
@@ -1,18 +0,0 @@
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
// Mirror of workspace-server's GET /buildinfo (PR #2398). Lets a developer
|
||||
// confirm which git SHA is live on a canvas deployment with the same
|
||||
// `curl <url>/buildinfo` flow they use against tenant workspaces.
|
||||
//
|
||||
// Vercel injects VERCEL_GIT_COMMIT_SHA / _REF / VERCEL_ENV at build time
|
||||
// from the deploying commit; outside Vercel (local `next dev`, harness)
|
||||
// these are unset and the endpoint reports `git_sha: "dev"`. Same sentinel
|
||||
// the workspace-server uses pre-ldflags-injection so both surfaces speak
|
||||
// the same vocabulary.
|
||||
export async function GET() {
|
||||
return NextResponse.json({
|
||||
git_sha: process.env.VERCEL_GIT_COMMIT_SHA ?? "dev",
|
||||
git_ref: process.env.VERCEL_GIT_COMMIT_REF ?? "",
|
||||
vercel_env: process.env.VERCEL_ENV ?? "local",
|
||||
});
|
||||
}
|
||||
@@ -1,240 +0,0 @@
|
||||
---
|
||||
title: "Give Your AI Agent Browser Superpowers: Chrome DevTools MCP Integration"
|
||||
date: "2026-04-20"
|
||||
canonical: "https://docs.molecule.ai/blog/chrome-devtools-mcp"
|
||||
og_title: "Give Your AI Agent Browser Superpowers with Chrome DevTools MCP"
|
||||
og_description: "Chrome DevTools MCP brings AI agent browser control to Molecule AI. Every browser action is audit-attributed via org API keys. MCP browser automation with governance built in."
|
||||
og_image: "/blog/chrome-devtools-mcp/chrome-devtools-mcp-social-card.png"
|
||||
twitter_card: "summary_large_image"
|
||||
author: "Molecule AI"
|
||||
keywords:
|
||||
- "AI agent browser control"
|
||||
- "MCP browser automation"
|
||||
- "browser automation AI agents"
|
||||
- "browser automation governance"
|
||||
- "Chrome DevTools MCP"
|
||||
- "MCP governance layer"
|
||||
- "AI agent web UI automation"
|
||||
---
|
||||
|
||||
import { Callout } from '@/components/blog/Callout'
|
||||
import { CodeBlock } from '@/components/blog/CodeBlock'
|
||||
|
||||
# Give Your AI Agent Browser Superpowers: Chrome DevTools MCP Integration
|
||||
|
||||
Every AI agent platform eventually gets asked the same question: "Can it interact with a web interface?" The answer is usually some variant of "sort of — give it your credentials and hope for the best." That's not a real answer. It's a trust fall.
|
||||
|
||||
Chrome DevTools MCP changes this. It gives your AI agent a structured, governed interface to a real Chrome browser session — with full **MCP browser automation** capability and an audit trail that actually answers the question: "which agent touched what, and what did it do?"
|
||||
|
||||
This post covers what Chrome DevTools MCP is, how Molecule AI's governance layer makes it enterprise-safe, and how to put it to work in your agent fleet.
|
||||
|
||||
---
|
||||
|
||||
## What is Chrome DevTools MCP?
|
||||
|
||||
Chrome DevTools MCP is an integration between the [MCP (Model Context Protocol)](https://modelcontextprotocol.io) and Google Chrome's DevTools Protocol. MCP is a standardized interface layer that lets AI agents connect to external tools with consistent tooling, authentication, and telemetry. The DevTools Protocol is Chrome's native debugging interface — the same interface your browser's developer tools use to inspect pages, capture network traffic, and control the browser.
|
||||
|
||||
When you connect an AI agent to Chrome DevTools via MCP, you get:
|
||||
|
||||
- **Full CDP access** — navigate, click, type, screenshot, evaluate JavaScript, read network logs, intercept requests, read cookies and local storage
|
||||
- **MCP protocol layer** — structured JSON-RPC instead of raw CDP, consistent tool naming, type-safe parameters
|
||||
- **Molecule AI governance layer** — org API key attribution, audit logging, session scoping, instant revocation
|
||||
|
||||
The third item is what separates this from "use Puppeteer with an API key." It's the difference between browser automation AI agents and browser automation AI agents with a compliance story.
|
||||
|
||||
---
|
||||
|
||||
## The Browser Problem: Trust Falls and Black Boxes
|
||||
|
||||
When most teams give an AI agent browser access, the workflow looks like this:
|
||||
|
||||
1. Agent receives a task ("find our competitors' pricing pages")
|
||||
2. Agent uses browser credentials to log into Chrome
|
||||
3. Agent navigates, reads, screenshots, and reports
|
||||
4. Nobody knows exactly what the agent did, which session it used, or whether credentials were exposed
|
||||
|
||||
This is a trust fall, not a governance model. The agent *can* do the task. But you have no audit trail if something goes wrong. No way to revoke access if the agent's behavior becomes unexpected. No attribution if you need to trace a call back to a specific integration.
|
||||
|
||||
The **MCP governance layer** in Molecule AI addresses all three:
|
||||
|
||||
- Every browser action is logged with the org API key prefix that initiated it
|
||||
- Chrome sessions are token-scoped — Agent A's session is never Agent B's
|
||||
- Revocation is one API call — the key stops working, the session closes, no redeploy required
|
||||
|
||||
---
|
||||
|
||||
## How MCP Browser Automation Works in Molecule AI
|
||||
|
||||
The integration uses Chrome's CDP over a WebSocket connection managed by the MCP server. Molecule AI's MCP server exposes a structured set of tools that map to CDP commands. Your agent calls these tools like any other MCP tool — the same interface whether you're automating Chrome, reading memory, or querying the platform API.
|
||||
|
||||
Here's the sequence:
|
||||
|
||||
1. **Workspace starts with a Chrome session attached** — the session is scoped to a specific Chrome profile or fresh browser context, isolated from other agents
|
||||
2. **Agent calls MCP tools** — `cdp_navigate`, `cdp_click`, `cdp_evaluate`, `cdp_screenshot`, and others are available as structured tools with type-safe parameters
|
||||
3. **Every call is audit-attributed** — the org API key prefix (e.g., `mole_a1b2`) is logged with the tool name, parameters, and result for every CDP call
|
||||
4. **Session is revocable at any time** — revoke the org API key and the agent loses Chrome access immediately
|
||||
|
||||
### AI Agent Browser Control: What You Can Do
|
||||
|
||||
**Navigation and interaction:**
|
||||
- `cdp_navigate` — navigate to any URL (supports `data:` and `about:` URLs via browser UI)
|
||||
- `cdp_click` — click a DOM element by selector
|
||||
- `cdp_type` — type text into a focused element
|
||||
- `cdp_hover` — hover over a DOM element
|
||||
- `cdp_scroll` — scroll an element or the page
|
||||
|
||||
**Inspection and debugging:**
|
||||
- `cdp_screenshot` — capture a full-page or viewport screenshot
|
||||
- `cdp_evaluate` — execute JavaScript in the page context
|
||||
- `cdp_get_cookies` / `cdp_set_cookies` — read and write cookies for authenticated sessions
|
||||
- `cdp_get_local_storage` / `cdp_set_local_storage` — read and write localStorage
|
||||
|
||||
**Network and performance:**
|
||||
- `cdp_get_requests` — capture and filter network requests (XHR, fetch, WS)
|
||||
- `cdp_block_urls` — block specific URL patterns to simulate adblocked environments
|
||||
- `cdp_set_throttle` — throttle network conditions (3G, LTE, offline)
|
||||
|
||||
---
|
||||
|
||||
## Browser Automation AI Agents: Use Cases That Actually Ship
|
||||
|
||||
The Chrome DevTools MCP integration is most useful in workflows where browser state is the source of truth — and where audit attribution matters.
|
||||
|
||||
### Automated Lighthouse audits on every PR
|
||||
|
||||
A research agent runs a Lighthouse audit against every pull request in your repo. It navigates to the preview URL, captures the performance score, flags regressions below your threshold, and reports to the PM agent. Every audit run is logged with the org API key — your observability team can trace which agent ran which audit and when.
|
||||
|
||||
```bash
|
||||
# Agent calls cdp_navigate to the PR preview URL
|
||||
# Agent calls cdp_evaluate to run Lighthouse inline
|
||||
# Agent calls cdp_screenshot to capture the score
|
||||
# Agent delegates results to PM workspace
|
||||
```
|
||||
|
||||
### Visual regression detection
|
||||
|
||||
An agent maintains a baseline set of screenshots for your key user flows. On every code change, it navigates to each flow, captures screenshots, and diffs against the baseline. Drift beyond your threshold opens a ticket automatically. The governance layer means your QA team can review the full history of which screenshots were captured, when, and by which agent.
|
||||
|
||||
### Auth scraping
|
||||
|
||||
An agent reads authenticated browser state from an existing Chrome session — cookies, localStorage, session tokens — and uses that state to authenticate API calls that would otherwise require separate credential management. The session is scoped; the credentials never leave the browser context.
|
||||
|
||||
---
|
||||
|
||||
## MCP Governance Layer: Why It Matters
|
||||
|
||||
The MCP protocol gives you tool connectivity. The governance layer is what makes it enterprise-ready.
|
||||
|
||||
### Per-action audit logging
|
||||
|
||||
Every CDP call your agent makes generates an audit log entry. The log includes:
|
||||
|
||||
- **Org API key prefix** — which integration made the call (e.g., `mole_a1b2`)
|
||||
- **Tool name and parameters** — `cdp_navigate(url=https://...)`
|
||||
- **Result or error** — success, timeout, or CDP error code
|
||||
- **Timestamp and workspace ID** — for timeline reconstruction
|
||||
|
||||
This is the audit trail your security team will ask for in the next compliance review. It exists because Molecule AI's MCP server generates it — not because you built a custom logging pipeline.
|
||||
|
||||
### Token-scoped Chrome sessions
|
||||
|
||||
Chrome sessions are isolated per org API key. When you create an org API key for a specific integration (`lighthouse-reporter`), that key's Chrome session is separate from every other key's session. No credential cross-contamination — Agent A cannot read Agent B's authenticated state because their sessions are isolated at the MCP tool layer.
|
||||
|
||||
### Instant revocation without redeployment
|
||||
|
||||
If you need to revoke access — the integration is compromised, the agent behavior is unexpected, the contractor relationship ended — you revoke the org API key:
|
||||
|
||||
```bash
|
||||
curl -X DELETE https://platform.moleculesai.app/org/tokens/<token-id> \
|
||||
-H "Authorization: Bearer <admin-session-token>"
|
||||
```
|
||||
|
||||
The key stops working immediately. The Chrome session is closed. The agent loses browser access before the next heartbeat. No redeploy, no container restart, no waiting for DNS cache expiration.
|
||||
|
||||
---
|
||||
|
||||
## Setting Up Chrome DevTools MCP
|
||||
|
||||
Chrome DevTools MCP requires a Chrome instance running with the remote debugging port enabled, and a `chromedp` or equivalent CDP client connected through Molecule AI's MCP server.
|
||||
|
||||
### Step 1: Enable Chrome remote debugging
|
||||
|
||||
Start Chrome with the `--remote-debugging-port=9222` flag:
|
||||
|
||||
```bash
|
||||
# macOS
|
||||
/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome \
|
||||
--remote-debugging-port=9222 \
|
||||
--user-data-dir=/tmp/chrome-debug
|
||||
|
||||
# Linux
|
||||
google-chrome --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-debug
|
||||
```
|
||||
|
||||
### Step 2: Configure Molecule AI
|
||||
|
||||
In your workspace config, add the Chrome DevTools MCP server URL:
|
||||
|
||||
```yaml
|
||||
# config.yaml
|
||||
mcpServers:
|
||||
- name: chrome-devtools
|
||||
url: "http://localhost:9222" # CDP WebSocket endpoint
|
||||
transport: cdp
|
||||
```
|
||||
|
||||
### Step 3: Verify the connection
|
||||
|
||||
Your agent can now call CDP tools. Test with a simple navigation:
|
||||
|
||||
```
|
||||
Agent: navigate to https://example.com and screenshot the page
|
||||
```
|
||||
|
||||
The audit log should show `cdp_navigate` and `cdp_screenshot` entries attributed to the workspace's org API key prefix.
|
||||
|
||||
---
|
||||
|
||||
## What the Security Review Looks Like
|
||||
|
||||
When your security team asks "what does this integration actually do?", here's the answer:
|
||||
|
||||
**What it can do:**
|
||||
- Navigate to any URL (with org API key attribution on every navigation)
|
||||
- Read and write browser state (cookies, localStorage, session tokens)
|
||||
- Screenshot pages and DOM elements
|
||||
- Execute JavaScript in the page context
|
||||
|
||||
**What it can't do (by default):**
|
||||
- Access the host machine beyond the Chrome sandbox
|
||||
- Read files outside the browser context
|
||||
- Exfiltrate session tokens across session boundaries
|
||||
|
||||
**What revocation looks like:**
|
||||
- Revoke org API key → immediate session close
|
||||
- No redeploy, no agent restart
|
||||
- Audit trail shows every action taken before revocation
|
||||
|
||||
---
|
||||
|
||||
## Browser Automation Governance: The Bigger Picture
|
||||
|
||||
Chrome DevTools MCP is one piece of Molecule AI's broader MCP governance story. MCP is a general-purpose protocol — it connects agents to any tool that speaks CDP, stdio, or HTTP. The governance layer applies uniformly: every MCP call gets the same treatment — org API key attribution, audit logging, instant revocation.
|
||||
|
||||
This means you can add new MCP integrations — databases, APIs, code execution environments — with the same governance posture. The MCP protocol is the connectivity layer. Molecule AI's MCP governance layer is the control plane.
|
||||
|
||||
If you're evaluating AI agent platforms for browser automation governance, the question to ask is not "can it control a browser?" It's "can I audit every action, attribute every call, and revoke access in one step?" Chrome DevTools MCP with Molecule AI's MCP governance layer is the answer to that question.
|
||||
|
||||
---
|
||||
|
||||
## Get Started
|
||||
|
||||
Chrome DevTools MCP is available on all Molecule AI deployments running Phase 30 or later.
|
||||
|
||||
- [MCP Server Setup Guide](/docs/guides/mcp-server-setup) — configure MCP tools in your workspace
|
||||
- [Org API Keys: Audit Attribution Setup](/blog/org-scoped-api-keys) — set up org API keys with attribution
|
||||
- [A2A Protocol Reference](/docs/api-protocol/a2a-protocol) — how agents delegate browser tasks to each other
|
||||
|
||||
<Callout variant="info">
|
||||
Chrome DevTools MCP requires Chrome running with the remote debugging port enabled. CDP access is scoped per org API key — multiple agents can share Chrome sessions only if intentionally scoped that way via key design.
|
||||
</Callout>
|
||||
+12
-154
@@ -1,139 +1,24 @@
|
||||
@import "tailwindcss";
|
||||
@plugin "@tailwindcss/typography";
|
||||
|
||||
/*
|
||||
* Tailwind v4 defaults the `dark:` variant to `prefers-color-scheme: dark`.
|
||||
* Our theme switcher writes `data-theme="dark"` on <html> instead (so user
|
||||
* choice via the toggle wins over OS preference). Re-bind `dark:` to that
|
||||
* attribute so component classes like `dark:bg-zinc-800` track the same
|
||||
* source of truth as the `[data-theme="dark"]` token overrides below.
|
||||
*/
|
||||
@custom-variant dark (&:where([data-theme="dark"], [data-theme="dark"] *));
|
||||
|
||||
/*
|
||||
* Load order:
|
||||
* 1. Tailwind core (v4) — provides preflight + utility generation.
|
||||
* 2. xterm — overrides preflight on its own .xterm-* class names; must
|
||||
* load AFTER tailwind so its specificity wins.
|
||||
* 3. theme-tokens.css — canvas-only motion + deploy animation vars
|
||||
* (--mol-duration-*, --mol-easing-*, --mol-deploy-*). NOT colour
|
||||
* tokens; the warm-paper @theme block below owns those.
|
||||
* 4. settings-panel.css / org-deploy.css — feature stylesheets that
|
||||
* reference the variables above.
|
||||
*/
|
||||
@import "xterm/css/xterm.css";
|
||||
@import "../styles/theme-tokens.css";
|
||||
@import "../styles/settings-panel.css";
|
||||
@import "../styles/org-deploy.css";
|
||||
|
||||
/*
|
||||
* Warm-paper semantic tokens — light defaults via @theme, dark
|
||||
* overrides via [data-theme="dark"]. Names are role-based
|
||||
* (`bg-surface`, `text-ink`, `border-line`) not colour-based, so the
|
||||
* same component classes work in either mode.
|
||||
*
|
||||
* Source of truth: molecule-app/app/globals.css. Keep aligned across
|
||||
* surfaces (landing, market, app, canvas) so a token tweak ripples
|
||||
* everywhere via a single PR per repo.
|
||||
*
|
||||
* Theme preference is persisted in the `mol_theme` cookie scoped to
|
||||
* Domain=.moleculesai.app so the choice follows the user across
|
||||
* subdomains. The inline boot script in app/layout.tsx applies it
|
||||
* before paint to eliminate flash.
|
||||
*/
|
||||
@theme {
|
||||
/* Surface — page / elevated card / sunken input / deep card */
|
||||
--color-surface: #fafaf7;
|
||||
--color-surface-elevated: #ffffff;
|
||||
--color-surface-sunken: #f3f1ec;
|
||||
--color-surface-card: #efece4;
|
||||
|
||||
/* Borders */
|
||||
--color-line: #e6e2d8;
|
||||
--color-line-soft: #efece4;
|
||||
|
||||
/* Text */
|
||||
--color-ink: #15181c;
|
||||
--color-ink-mid: #5a5e66;
|
||||
--color-ink-soft: #8b8e95;
|
||||
|
||||
/* Brand + state */
|
||||
--color-accent: #3b5bdb;
|
||||
--color-accent-strong: #1a2f99;
|
||||
--color-warm: #c0532b;
|
||||
--color-good: #2f7a4d;
|
||||
--color-bad: #b94e4a;
|
||||
}
|
||||
|
||||
[data-theme="dark"] {
|
||||
--color-surface: #0e1014;
|
||||
--color-surface-elevated: #15181c;
|
||||
--color-surface-sunken: #0a0b0e;
|
||||
--color-surface-card: #1a1d23;
|
||||
|
||||
--color-line: #2a2f3a;
|
||||
--color-line-soft: #1f2329;
|
||||
|
||||
--color-ink: #f4f1e9;
|
||||
--color-ink-mid: #c8c2b4;
|
||||
--color-ink-soft: #8d92a0;
|
||||
|
||||
/* Accents brighten slightly for AA contrast on dark backgrounds. */
|
||||
--color-accent: #6883e8;
|
||||
--color-accent-strong: #8aa1ee;
|
||||
--color-warm: #d96f48;
|
||||
--color-good: #4ca06e;
|
||||
--color-bad: #d27773;
|
||||
}
|
||||
|
||||
:root {
|
||||
color-scheme: light;
|
||||
}
|
||||
[data-theme="dark"] {
|
||||
color-scheme: dark;
|
||||
}
|
||||
|
||||
/*
|
||||
* Always-dark surface tokens. Terminals (xterm), the console modal,
|
||||
* and log streams stay dark in both modes — readable green-on-black
|
||||
* code surfaces don't translate cleanly to a light theme. Components
|
||||
* that should not light-flip use `bg-bg`, `bg-bg-elev`, `bg-bg-card`,
|
||||
* `text-ink-mute`, `text-ink-dim`, `border-line-strong` instead of
|
||||
* the warm-paper utilities above.
|
||||
*
|
||||
* Distinct names (bg-* / ink-mute / ink-dim / line-strong) so they
|
||||
* don't collide with the warm-paper namespace (surface / ink /
|
||||
* line). Both palettes coexist; the choice between them is per
|
||||
* component, not per theme.
|
||||
*/
|
||||
@theme {
|
||||
--color-bg: rgb(9 9 11); /* zinc-950 */
|
||||
--color-bg-elev: rgb(24 24 27); /* zinc-900 */
|
||||
--color-bg-card: rgb(39 39 42); /* zinc-800 */
|
||||
--color-line-strong: rgb(63 63 70); /* zinc-700 */
|
||||
--color-ink-mute: rgb(161 161 170); /* zinc-400 */
|
||||
--color-ink-dim: rgb(113 113 122); /* zinc-500 */
|
||||
--color-accent-dim: rgb(96 165 250);/* blue-400 */
|
||||
--color-plasma: rgb(59 130 246); /* blue-500 */
|
||||
--color-warn: rgb(251 191 36); /* amber-400 */
|
||||
}
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
background-color: var(--color-surface);
|
||||
color: var(--color-ink);
|
||||
background: #09090b;
|
||||
color: #e4e4e7;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
/* React Flow overrides for both themes. Edge stroke pulls from the
|
||||
semantic line token so dark mode keeps its existing zinc-700 look
|
||||
and light mode picks up the warm-paper line colour. */
|
||||
/* React Flow overrides for dark theme */
|
||||
.react-flow__edge-path {
|
||||
stroke: var(--color-line) !important;
|
||||
stroke: #3f3f46 !important;
|
||||
stroke-width: 1.5 !important;
|
||||
}
|
||||
|
||||
@@ -153,24 +38,10 @@ body {
|
||||
}
|
||||
|
||||
.react-flow__node {
|
||||
/* Transform transition drives the "spawn from parent" motion —
|
||||
org-deploy sets the node's initial position to the parent's
|
||||
absolute coords, then repositions to the real slot, and this
|
||||
transition interpolates the translate() in between.
|
||||
Non-deploy workspace moves (drag, nest) get the same smoothing
|
||||
for free. */
|
||||
transition:
|
||||
box-shadow var(--mol-duration-fast) ease,
|
||||
transform var(--mol-duration-spawn) var(--mol-easing-bounce-out);
|
||||
}
|
||||
/* Drag events must feel instant — React Flow adds this class
|
||||
for the lifetime of the gesture. */
|
||||
.react-flow__node.dragging {
|
||||
transition: box-shadow var(--mol-duration-fast) ease;
|
||||
transition: box-shadow 0.2s ease;
|
||||
}
|
||||
|
||||
/* Scrollbar styling. Track + thumb pull from the surface tokens so
|
||||
they feel native to either theme. */
|
||||
/* Scrollbar styling */
|
||||
::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
@@ -181,17 +52,17 @@ body {
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: var(--color-line);
|
||||
background: #3f3f46;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: var(--color-line-strong, var(--color-ink-soft));
|
||||
background: #52525b;
|
||||
}
|
||||
|
||||
/* Selection */
|
||||
::selection {
|
||||
background: color-mix(in srgb, var(--color-accent) 30%, transparent);
|
||||
background: rgba(59, 130, 246, 0.3);
|
||||
}
|
||||
|
||||
/* Panel slide animation */
|
||||
@@ -274,17 +145,4 @@ body {
|
||||
.react-flow__node {
|
||||
animation: none !important;
|
||||
}
|
||||
|
||||
/* React Flow Controls toolbar buttons — WCAG 2.4.7 focus-visible */
|
||||
.react-flow__controls button:focus-visible {
|
||||
outline: 2px solid var(--accent, #3b5bdb);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
/* React Flow Minimap nodes — WCAG 2.4.7 focus-visible */
|
||||
.react-flow__minimap:focus-visible,
|
||||
.react-flow__minimap svg:focus-visible {
|
||||
outline: 2px solid var(--accent, #3b5bdb);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user