fix(queue): re-fetch PR head before merge to detect stale SHA

If CI updates the PR head between the initial status check and the merge call, the queue might try to merge an outdated head. Add a pre-merge PR re-fetch that bails out if the head changed, letting the next tick re-evaluate with the current head. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
fix(queue): handle Gitea empty-body 200 on merge endpoint
2026-05-17 04:06:22 +00:00 · 2026-05-17 04:05:37 +00:00 · 2026-05-17 04:03:11 +00:00 · 2026-05-17 02:47:52 +00:00 · 2026-05-16 22:48:49 +00:00 · 2026-05-16 21:09:20 +00:00
33 changed files with 1631 additions and 121 deletions
@@ -23,6 +23,7 @@ import dataclasses
 import json
 import os
 import sys
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -326,6 +327,43 @@ def update_pull(pr_number: int, *, dry_run: bool) -> None:
    )


+def wait_for_ci(
+    head_sha: str,
+    contexts: list[str],
+    *,
+    max_wait_seconds: int = 300,
+    poll_interval: int = 15,
+) -> bool:
+    """Poll CI statuses for head_sha until all required contexts are terminal.
+
+    Returns True if all contexts reached 'success', False if timeout expired
+    (some still pending or failed).
+
+    Background: after a queue-triggered PR update, CI re-runs on the new head.
+    The queue must not update again until CI completes — otherwise the
+    update-then-wait loop keeps the PR in a perpetually-updating state where
+    CI never finishes on any single head.
+    """
+    deadline = time.time() + max_wait_seconds
+    while time.time() < deadline:
+        time.sleep(poll_interval)
+        try:
+            pr_status = get_combined_status(head_sha)
+        except Exception as exc:
+            sys.stderr.write(f"::warning::wait_for_ci: status fetch failed: {exc}\n")
+            continue
+        latest = latest_statuses_by_context(pr_status.get("statuses") or [])
+        ok, bad = required_contexts_green(latest, contexts)
+        if ok:
+            sys.stderr.write(f"::notice::wait_for_ci: all contexts green after {int(time.time() - (deadline - max_wait_seconds))}s\n")
+            return True
+        # Log progress
+        pending = [f"{c}={latest.get(c, {}).get('status', 'missing')}" for c in contexts if latest.get(c, {}).get('status') != 'success']
+        sys.stderr.write(f"::notice::wait_for_ci: still waiting ({int(deadline - time.time())}s left): {', '.join(pending[:3])}\n")
+    sys.stderr.write(f"::warning::wait_for_ci: timeout after {max_wait_seconds}s; proceeding with merge check\n")
+    return False
+
+
 def merge_pull(pr_number: int, *, dry_run: bool) -> None:
    payload = {
        "Do": "merge",
@@ -338,7 +376,24 @@ def merge_pull(pr_number: int, *, dry_run: bool) -> None:
    print(f"::notice::merging PR #{pr_number}")
    if dry_run:
        return
-    api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
+    # Gitea's merge endpoint returns HTTP 200 with an empty body on success.
+    # The generic api() wrapper raises ApiError on non-2xx, so a 200 with an
+    # empty body reaches the json.loads() path and raises JSONDecodeError,
+    # which api() re-raises as ApiError — making the queue think the merge
+    # failed when it actually succeeded.  Work around this by catching the
+    # expected JSONDecodeError here and treating it as success.
+    try:
+        api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
+    except ApiError as exc:
+        # Surface non-merge errors (5xx server errors, 403 forbidden, etc.)
+        if "merge" in str(exc).lower() or "405" in str(exc) or "409" in str(exc):
+            # 405 = PR not mergeable (already merged or CI still running by
+            #    the time we got here — the PR will be re-checked next tick)
+            # 409 = merge conflict detected at merge time
+            # In both cases the PR stays open and the next tick re-evaluates.
+            sys.stderr.write(f"::warning::merge call returned: {exc}\n")
+        else:
+            raise


 def process_once(*, dry_run: bool = False) -> int:
@@ -390,6 +445,32 @@ def process_once(*, dry_run: bool = False) -> int:
    print(f"::notice::PR #{pr_number} decision={decision.action}: {decision.reason}")
    if decision.action == "update":
        update_pull(pr_number, dry_run=dry_run)
+        # After an update, CI re-runs on the new head. If we check statuses
+        # immediately we see pending (CI not started yet on the new head), so
+        # the next tick updates again — CI never completes on any single head.
+        # Fix: re-fetch the PR to get the new head SHA, then poll CI for up
+        # to 5 min until all required contexts reach terminal state.  If CI
+        # finishes in time, proceed to merge on the same tick.
+        if not dry_run:
+            updated_pr = get_pull(pr_number)
+            new_head = updated_pr.get("head", {}).get("sha", "")
+            if new_head and new_head != head_sha:
+                sys.stderr.write(f"::notice::PR #{pr_number}: update created new head {new_head[:8]}; waiting for CI...\n")
+                waited = wait_for_ci(new_head, contexts, max_wait_seconds=300, poll_interval=15)
+                if waited:
+                    # CI completed — re-fetch main to confirm it hasn't moved,
+                    # then merge immediately without another update cycle.
+                    current_main_sha = get_branch_head(WATCH_BRANCH)
+                    if current_main_sha != main_sha:
+                        sys.stderr.write(f"::notice::PR #{pr_number}: main moved {main_sha[:8]} -> {current_main_sha[:8]}; deferring\n")
+                        return 0
+                    sys.stderr.write(f"::notice::PR #{pr_number}: CI complete; merging now\n")
+                    merge_pull(pr_number, dry_run=dry_run)
+                    return 0
+                else:
+                    sys.stderr.write(f"::warning::PR #{pr_number}: CI did not finish within 5 min; will retry next tick\n")
+            else:
+                sys.stderr.write(f"::notice::PR #{pr_number}: update did not change head SHA; will retry\n")
        post_comment(
            pr_number,
            (
@@ -400,6 +481,13 @@ def process_once(*, dry_run: bool = False) -> int:
        )
        return 0
    if decision.ready:
+        # Re-fetch PR to confirm head hasn't changed since we last checked
+        # (CI may have updated the head while we were evaluating).
+        current_pr = get_pull(pr_number)
+        current_head = current_pr.get("head", {}).get("sha", "")
+        if current_head != head_sha:
+            print(f"::notice::PR #{pr_number} head changed {head_sha[:8]} -> {current_head[:8]}; re-evaluating")
+            return 0
        latest_main_sha = get_branch_head(WATCH_BRANCH)
        if latest_main_sha != main_sha:
            print(
@@ -68,7 +68,7 @@ import sys
 import urllib.error
 import urllib.parse
 import urllib.request
-from typing import Any
+from typing import Any, Callable


 # ---------------------------------------------------------------------------
@@ -110,7 +110,7 @@ def normalize_slug(raw: str, numeric_aliases: dict[int, str] | None = None) -> s
 # for /sop-revoke (RFC#351 open question 4 — reason is captured but not
 # yet validated; future iteration may require a min-length).
 _DIRECTIVE_RE = re.compile(
-    r"^[ \t]*/(sop-ack|sop-revoke)[ \t]+([A-Za-z0-9_\- ]+?)(?:[ \t]+(.*))?[ \t]*$",
+    r"^[ \t]*/(sop-ack|sop-revoke|sop-n/a)[ \t]+([A-Za-z0-9_\- ]+?)(?:[ \t]+(.*))?[ \t]*$",
    re.MULTILINE,
 )

@@ -118,19 +118,21 @@ _DIRECTIVE_RE = re.compile(
 def parse_directives(
    comment_body: str,
    numeric_aliases: dict[int, str],
-) -> tuple[list[tuple[str, str, str]], list]:
-    """Extract /sop-ack and /sop-revoke directives from a comment body.
+) -> tuple[list[tuple[str, str, str]], list[tuple[str, str, str]]]:
+    """Extract /sop-ack, /sop-revoke, and /sop-n/a directives from a comment body.

-    Returns (directives, na_directives) where:
-      directives is a list of (kind, canonical_slug, note) tuples
-        kind is "sop-ack" or "sop-revoke"
-        canonical_slug is the normalized form (or "" if unparseable)
-        note is the trailing free-text (may be "")
-      na_directives is reserved for future N/A handling (always [] for now)
+    Returns (directives, na_directives) where each is a list of
+    (kind, canonical_slug, note) tuples:
+      kind is "sop-ack", "sop-revoke", or "sop-n/a"
+      canonical_slug is the normalized form (or "" if unparseable)
+      note is the trailing free-text (may be "")
+    The two lists are kept separate so call sites can unpack them
+    directly (e.g. directives, na_directives = parse_directives(...)).
    """
-    out: list[tuple[str, str, str]] = []
+    directives: list[tuple[str, str, str]] = []
+    na_directives: list[tuple[str, str, str]] = []
    if not comment_body:
-        return out, []
+        return directives, na_directives
    for m in _DIRECTIVE_RE.finditer(comment_body):
        kind = m.group(1)
        raw_slug = (m.group(2) or "").strip()
@@ -160,8 +162,12 @@ def parse_directives(
        note_from_group = (m.group(3) or "").strip()
        # If we collapsed multi-word slug into kebab and there's a
        # trailing-text group too, append it.
-        out.append((kind, canonical, note_from_group))
-    return out, []
+        entry = (kind, canonical, note_from_group)
+        if kind == "sop-n/a":
+            na_directives.append(entry)
+        else:
+            directives.append(entry)
+    return directives, na_directives


 # ---------------------------------------------------------------------------
@@ -174,8 +180,8 @@ def section_marker_present(body: str, marker: str) -> bool:
    on a non-empty line (i.e. the author actually filled it in).

    We require the marker substring AND non-whitespace content on the
-    same line OR within the next line — this prevents trivially-empty
-    checklists like:
+    same line OR within the next non-blank line — this prevents
+    trivially-empty checklists like:

        ## SOP-Checklist
        - [ ] **Comprehensive testing performed**:
@@ -184,9 +190,18 @@ def section_marker_present(body: str, marker: str) -> bool:
    from auto-passing the section-present check. The peer-ack is still
    required, but answering with empty content is captured as a soft
    finding via the section-present test alone.
+
+    NOTE: we scan forward through blank lines (the markdown-header pattern
+    is ## Header\\n\\ncontent) so that a header + blank-line + content
+    structure still satisfies the check. The backward checkbox fallback
+    catches inline markers without a preceding checkbox (mc#1099).
    """
    if not body or not marker:
        return False
+    # Strip trailing whitespace so the blank-line scan below can find
+    # content that appears on the very last line of the body (without
+    # being misled by a trailing \n or spaces).
+    body = body.rstrip()
    body_lower = body.lower()
    marker_lower = marker.lower()
    idx = body_lower.find(marker_lower)
@@ -202,13 +217,44 @@ def section_marker_present(body: str, marker: str) -> bool:
    stripped = re.sub(r"[\s\*:\-\[\]]+", "", line)
    if stripped:
        return True
-    # Fall through: check the NEXT line (multi-line answers).
-    next_line_end = body.find("\n", line_end + 1)
-    if next_line_end < 0:
-        next_line_end = len(body)
-    next_line = body[line_end + 1:next_line_end]
-    stripped_next = re.sub(r"[\s\*:\-\[\]]+", "", next_line)
-    return bool(stripped_next)
+    # Fall through: scan forward, skipping blank-only lines, until we find
+    # non-empty content or run out of body.  Handles:
+    #   ## Header          ← marker line (empty after marker)
+    #                      ← blank line (skipped)
+    #   - actual content   ← found
+    pos = line_end
+    while True:
+        # Skip the current newline and any additional newlines (blank lines).
+        while pos < len(body) and body[pos] == "\n":
+            pos += 1
+        if pos >= len(body):
+            break
+        line_end = body.find("\n", pos)
+        if line_end < 0:
+            line_end = len(body)
+        line = body[pos:line_end]
+        stripped = re.sub(r"[\s\*:\-\[\]]+", "", line)
+        if stripped:
+            return True
+        pos = line_end
+    # Last resort: the marker may appear mid-sentence (e.g.
+    # **Memory/saved-feedback consulted**: No applicable...).
+    # Search backward within the CURRENT LINE only (not preceding lines)
+    # to find a checkbox on the same line before the marker text.
+    # mc#1099 follow-up: memory-consulted detection was failing because
+    # the checkbox was on the same line before the inline marker.
+    _CHECKBOX_RE = re.compile(r"- \[[ x\]]|<input", re.IGNORECASE)
+    line_start = body.rfind("\n", 0, idx) + 1  # 0 if no newline before idx
+    before = body[line_start:idx]
+    m = _CHECKBOX_RE.search(before)
+    if not m:
+        return False
+    # Require meaningful content between the checkbox and the marker text
+    # (markdown formatting like ** or * must also be stripped).
+    # If only whitespace/markdown chars remain, the checkbox line is empty.
+    between = before[m.end() :]
+    stripped_between = re.sub(r"[\s\*:#\[\]_\-]+", "", between)
+    return bool(stripped_between)


 # ---------------------------------------------------------------------------
@@ -251,8 +297,7 @@ def compute_ack_state(
        user = (c.get("user") or {}).get("login", "")
        if not user:
            continue
-        directives, _na = parse_directives(body, numeric_aliases)
-        for kind, slug, _note in directives:
+        for kind, slug, _note in parse_directives(body, numeric_aliases)[0]:
            if not slug:
                unparseable_per_user[user] = unparseable_per_user.get(user, 0) + 1
                continue
@@ -304,6 +349,63 @@ def compute_ack_state(
    }


+# ---------------------------------------------------------------------------
+# N/A-gate evaluation
+# ---------------------------------------------------------------------------
+
+
+def compute_na_state(
+    comments: list[dict[str, Any]],
+    author: str,
+    na_gates: dict[str, Any],
+    probe: Callable[[str, list[str]], list[str]],
+) -> dict[str, dict[str, Any]]:
+    """Evaluate which N/A gates have a valid declaration from a team member.
+
+    Returns dict[gate_name, dict] where each dict has:
+      declared: bool — at least one valid non-author team-member declared N/A
+      decl_ackers: list[str] — usernames who declared this gate N/A
+      rejected: dict with keys:
+        not_in_team: list[str] — users who tried but aren't in required teams
+    """
+    # Build per-user latest N/A directive (most-recent wins per RFC#324).
+    latest_na: dict[str, tuple[str, str]] = {}  # user → (gate, note)
+    for c in comments:
+        body = c.get("body", "") or ""
+        user = (c.get("user") or {}).get("login", "")
+        if not user:
+            continue
+        for kind, gate, note in parse_directives(body, {})[1]:
+            # [1] = na_directives only
+            if gate in na_gates:
+                latest_na[user] = (gate, note)
+
+    result: dict[str, dict[str, Any]] = {}
+    for gate, gate_cfg in na_gates.items():
+        result[gate] = {
+            "declared": False,
+            "decl_ackers": [],
+            "rejected": {"not_in_team": []},
+        }
+        decl_ackers: list[str] = []
+        not_in_team: list[str] = []
+        for user, (g, _note) in latest_na.items():
+            if g != gate:
+                continue
+            if user == author:
+                continue  # authors cannot self-declare N/A
+            approved = probe(gate, [user])
+            if approved:
+                decl_ackers.append(user)
+            else:
+                not_in_team.append(user)
+        result[gate]["declared"] = bool(decl_ackers)
+        result[gate]["decl_ackers"] = decl_ackers
+        result[gate]["rejected"]["not_in_team"] = not_in_team
+
+    return result
+
+
 # ---------------------------------------------------------------------------
 # Gitea API client
 # ---------------------------------------------------------------------------
@@ -698,6 +800,7 @@ def main(argv: list[str] | None = None) -> int:
    cfg = load_config(args.config)
    items: list[dict[str, Any]] = cfg["items"]
    items_by_slug = {it["slug"]: it for it in items}
+    na_gates: dict[str, Any] = cfg.get("n/a_gates", {})
    numeric_aliases = {
        int(it["numeric_alias"]): it["slug"] for it in items if it.get("numeric_alias")
    }
@@ -818,6 +921,46 @@ def main(argv: list[str] | None = None) -> int:
        description=description, target_url=target_url,
    )
    print(f"::notice::status posted: {args.status_context} → {state}")
+
+    # --- N/A gate status (RFC#324 §N/A follow-up) ---
+    # Post a separate status so review-check.sh can discover N/A declarations
+    # and waive the Gitea-approve requirement for that gate.
+    na_state: dict[str, dict[str, Any]] = {}
+    if na_gates:
+        na_state = compute_na_state(comments, author, na_gates, probe)
+
+        na_descs: list[str] = []
+        for gate, s in na_state.items():
+            if s["declared"]:
+                na_descs.append(gate)
+            decl = s["decl_ackers"]
+            rej = s["rejected"]["not_in_team"]
+            if decl:
+                print(f"::notice::  [N/A OK] {gate} — declared by {','.join(decl)}")
+            if rej:
+                print(
+                    f"::notice::  [N/A REJ] {gate} — not-in-team: {','.join(rej)}",
+                    file=sys.stderr,
+                )
+
+        na_desc = ", ".join(sorted(na_descs)) if na_descs else "(none)"
+        na_status_state = "success" if na_descs else "pending"
+        # review-check.sh reads the description to discover which gates are N/A.
+        # Include the gate names so it can grep for them.
+        na_description = f"N/A: {na_desc}" if na_descs else "N/A: (none)"
+
+        if not args.dry_run:
+            client.post_status(
+                args.owner, args.repo, head_sha,
+                state=na_status_state,
+                context="sop-checklist / na-declarations (pull_request)",
+                description=na_description,
+                target_url=target_url,
+            )
+            print(
+                f"::notice::na-declarations status → {na_status_state}: {na_description}"
+            )
+
    # By default exit 0 — the POSTed status IS the gate, NOT the job
    # conclusion. If the job exits 1 BP will see TWO failure signals
    # (one from the job's auto-status, one from our POST), making the
@@ -551,3 +551,55 @@ class TestEndToEndAckFlow(unittest.TestCase):

 if __name__ == "__main__":
    unittest.main(verbosity=2)
+
+
+# ---------------------------------------------------------------------------
+# compute_na_state
+# ---------------------------------------------------------------------------
+
+
+class TestComputeNaState(unittest.TestCase):
+    """Tests for /sop-n/a directive evaluation."""
+
+    def test_no_na_declarations(self):
+        cfg = sop.load_config(CONFIG_PATH)
+        na_gates = cfg.get("n/a_gates", {})
+        comments = []
+        na_state = sop.compute_na_state(comments, "alice", na_gates, lambda *_: [])
+        self.assertFalse(na_state["qa-review"]["declared"])
+        self.assertFalse(na_state["security-review"]["declared"])
+
+    def test_na_declared_by_authorized_user(self):
+        cfg = sop.load_config(CONFIG_PATH)
+        na_gates = cfg.get("n/a_gates", {})
+        comments = [_comment("bob", "/sop-n/a qa-review N/A: pure tooling change")]
+        na_state = sop.compute_na_state(comments, "alice", na_gates, lambda g, u: u)
+        self.assertTrue(na_state["qa-review"]["declared"])
+        self.assertEqual(na_state["qa-review"]["decl_ackers"], ["bob"])
+
+    def test_na_declared_by_unauthorized_user_rejected(self):
+        cfg = sop.load_config(CONFIG_PATH)
+        na_gates = cfg.get("n/a_gates", {})
+        comments = [_comment("mallory", "/sop-n/a qa-review N/A: not real team")]
+        na_state = sop.compute_na_state(comments, "alice", na_gates, lambda g, u: [])
+        self.assertFalse(na_state["qa-review"]["declared"])
+        self.assertEqual(na_state["qa-review"]["rejected"]["not_in_team"], ["mallory"])
+
+    def test_author_cannot_self_declare_na(self):
+        cfg = sop.load_config(CONFIG_PATH)
+        na_gates = cfg.get("n/a_gates", {})
+        comments = [_comment("alice", "/sop-n/a qa-review N/A: I am the author")]
+        na_state = sop.compute_na_state(comments, "alice", na_gates, lambda g, u: u)
+        self.assertFalse(na_state["qa-review"]["declared"])
+
+    def test_parse_directives_separates_na_from_ack(self):
+        directives, na_directives = sop.parse_directives(
+            "/sop-ack comprehensive-testing\n/sop-n/a qa-review N/A: no surface",
+            {},
+        )
+        self.assertEqual(len(directives), 1)
+        self.assertEqual(directives[0][0], "sop-ack")
+        self.assertEqual(len(na_directives), 1)
+        self.assertEqual(na_directives[0][0], "sop-n/a")
+        self.assertEqual(na_directives[0][1], "qa-review")
+        self.assertIn("no surface", na_directives[0][2])
@@ -564,14 +564,7 @@ jobs:
    #
    continue-on-error: false
    runs-on: ubuntu-latest
-    timeout-minutes: 1
-    needs:
-      - changes
-      - platform-build
-      - canvas-build
-      - shellcheck
-      - python-lint
-    if: ${{ always() }}
+    timeout-minutes: 45
    steps:
      - name: Wait for required CI contexts
        env:
@@ -175,6 +175,19 @@ jobs:
          echo "E2E_PLATFORM_URL=http://127.0.0.1:${PLATFORM_PORT}" >> "$GITHUB_ENV"
          echo "Platform host port: ${PLATFORM_PORT}"

+      - name: Pick canvas port
+        if: needs.detect-changes.outputs.chat == 'true'
+        run: |
+          CANVAS_PORT=$(python3 - <<'PY'
+          import socket
+          with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+              s.bind(("127.0.0.1", 0))
+              print(s.getsockname()[1])
+          PY
+          )
+          echo "CANVAS_PORT=${CANVAS_PORT}" >> "$GITHUB_ENV"
+          echo "Canvas host port: ${CANVAS_PORT}"
+
      - name: Start platform (background)
        if: needs.detect-changes.outputs.chat == 'true'
        working-directory: workspace-server
@@ -183,6 +196,7 @@ jobs:
          export DATABASE_URL="${DATABASE_URL}"
          export REDIS_URL="${REDIS_URL}"
          export PORT="${PLATFORM_PORT}"
+          export CORS_ORIGINS="http://localhost:3000,http://localhost:3001,http://localhost:${CANVAS_PORT},http://127.0.0.1:${CANVAS_PORT}"
          ./platform-server > platform.log 2>&1 &
          echo $! > platform.pid

@@ -216,10 +230,10 @@ jobs:
        run: |
          export NEXT_PUBLIC_PLATFORM_URL="http://127.0.0.1:${PLATFORM_PORT}"
          export NEXT_PUBLIC_WS_URL="ws://127.0.0.1:${PLATFORM_PORT}/ws"
-          npm run dev > canvas.log 2>&1 &
+          npx next dev --turbopack -p "${CANVAS_PORT}" > canvas.log 2>&1 &
          echo $! > canvas.pid
          for i in $(seq 1 30); do
-            if curl -sf http://localhost:3000 > /dev/null 2>&1; then
+            if curl -sf "http://localhost:${CANVAS_PORT}" > /dev/null 2>&1; then
              echo "Canvas up after ${i}s"
              exit 0
            fi
@@ -235,6 +249,7 @@ jobs:
        run: |
          export E2E_PLATFORM_URL="http://127.0.0.1:${PLATFORM_PORT}"
          export E2E_DATABASE_URL="${DATABASE_URL}"
+          export PLAYWRIGHT_BASE_URL="http://localhost:${CANVAS_PORT}"
          npx playwright test e2e/chat-desktop.spec.ts e2e/chat-mobile.spec.ts

      - name: Dump platform log on failure
@@ -0,0 +1,225 @@
+name: E2E Peer Visibility (literal MCP list_peers)
+
+# WHY A DEDICATED WORKFLOW (not folded into e2e-staging-saas.yml)
+# --------------------------------------------------------------
+# This is the systemic fix for a real trust failure. Hermes and OpenClaw
+# were reported "fleet-verified / cascade-complete" because the *proxy*
+# signals were green (registry registration + heartbeat for Hermes; model
+# round-trip 200 for OpenClaw). A freshly-provisioned workspace asked on
+# canvas "can you see your peers" actually FAILS:
+#   - Hermes: 401 on the molecule MCP `list_peers` call
+#   - OpenClaw: native `sessions_list` fallback, sees no platform peers
+# Tasks #142/#159 were even marked "completed" under this proxy flaw.
+#
+# A dedicated workflow (vs extending e2e-staging-saas.yml) because:
+#   - It must provision MULTIPLE distinct runtimes (hermes, openclaw,
+#     claude-code) in ONE org and assert each sees the others. The
+#     full-saas script is single-runtime-per-run (E2E_RUNTIME) and folding
+#     a multi-runtime matrix into it would conflate concerns and bloat its
+#     already-45-min run.
+#   - It needs its own concurrency group so it doesn't fight full-saas /
+#     canvas for the staging org-creation quota.
+#   - It needs an independent, non-required status-context name so it can
+#     be RED today (the in-flight Hermes-401 / OpenClaw-MCP-wiring fixes
+#     have not landed) WITHOUT wedging unrelated merges — and flipped to
+#     REQUIRED in one branch-protection edit once it goes green
+#     (flip-to-required checklist: molecule-core#1296).
+#
+# THE ASSERTION IS NOT A PROXY. The driving script
+# tests/e2e/test_peer_visibility_mcp_staging.sh issues the byte-for-byte
+# JSON-RPC `tools/call name=list_peers` envelope to `POST
+# /workspaces/:id/mcp` using each workspace's OWN bearer token, through
+# the real WorkspaceAuth + MCPRateLimiter middleware chain — the exact
+# call mcp_molecule_list_peers makes from a canvas agent. It does NOT
+# read a registry row, /health, the heartbeat table, or
+# GET /registry/:id/peers.
+#
+# HONEST GATE — NO continue-on-error. Per feedback_fix_root_not_symptom a
+# fake-green mask would defeat the entire purpose. This workflow goes red
+# on today's broken behavior and green only when the root-cause fixes
+# actually land. It is intentionally NOT in branch_protections — see PR
+# body for the required-vs-not decision + flip tracking issue.
+#
+# Gitea 1.22.6 / act_runner notes honored:
+#   - No cross-repo `uses:` (feedback_gitea_cross_repo_uses_blocked). The
+#     actions/checkout SHA is the one e2e-staging-canvas.yml already uses
+#     successfully (a mirrored SHA — see #1277/PR#1292 root-cause).
+#   - Per-SHA concurrency, not global (feedback_concurrency_group_per_sha).
+#   - Workflow-level GITHUB_SERVER_URL pinned
+#     (feedback_act_runner_github_server_url).
+#   - pr-validate posts a status under the same check name so a
+#     workflow-only PR is not silently statusless and the context is
+#     flip-to-required-ready (mirrors e2e-staging-saas.yml's proven shape;
+#     real EC2-provisioning E2E is push/dispatch/cron only — it is 30+ min
+#     and cannot run per-PR-update).
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/mcp.go'
+      - 'workspace-server/internal/handlers/mcp_tools.go'
+      - 'workspace-server/internal/middleware/**'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace/a2a_mcp_server.py'
+      - 'workspace/platform_tools/registry.py'
+      - 'tests/e2e/test_peer_visibility_mcp_staging.sh'
+      - '.gitea/workflows/e2e-peer-visibility.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'workspace-server/internal/handlers/mcp.go'
+      - 'workspace-server/internal/handlers/mcp_tools.go'
+      - 'workspace-server/internal/middleware/**'
+      - 'workspace-server/internal/handlers/registry.go'
+      - 'workspace-server/internal/handlers/workspace.go'
+      - 'workspace/a2a_mcp_server.py'
+      - 'workspace/platform_tools/registry.py'
+      - 'tests/e2e/test_peer_visibility_mcp_staging.sh'
+      - '.gitea/workflows/e2e-peer-visibility.yml'
+  workflow_dispatch:
+  schedule:
+    # 07:30 UTC daily — catches AMI / template-hermes / template-openclaw
+    # drift even on quiet days. Offset 30m from e2e-staging-saas (07:00)
+    # so the two don't collide on the staging org-creation quota.
+    - cron: '30 7 * * *'
+
+concurrency:
+  # Per-SHA (feedback_concurrency_group_per_sha). A single global group
+  # would let a queued staging/main push behind a PR run get cancelled,
+  # leaving any gate that reads "completed run at SHA" stuck.
+  group: e2e-peer-visibility-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: false
+
+env:
+  GITHUB_SERVER_URL: https://git.moleculesai.app
+
+jobs:
+  # PR path: post a real status under the required-ready check name so a
+  # workflow-only PR is never silently statusless. The actual EC2 E2E is
+  # push/dispatch/cron only (30+ min). This is NOT a fake-green mask of
+  # the real assertion — it validates the driving script's bash syntax
+  # and inline-python so a broken test script fails at PR time.
+  pr-validate:
+    name: E2E Peer Visibility
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Validate driving script
+        run: |
+          bash -n tests/e2e/test_peer_visibility_mcp_staging.sh
+          echo "test_peer_visibility_mcp_staging.sh — bash syntax OK"
+          echo "Real fresh-provision MCP list_peers E2E runs on push to"
+          echo "main / workflow_dispatch / daily cron (30+ min EC2 boot)."
+
+  # Real gate: provisions a throwaway org + sibling-per-runtime, drives
+  # the LITERAL list_peers MCP call per runtime, asserts 200 + expected
+  # peer set, then scoped teardown. push(main)/dispatch/cron only.
+  peer-visibility:
+    name: E2E Peer Visibility
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    timeout-minutes: 60
+
+    env:
+      MOLECULE_CP_URL: https://staging-api.moleculesai.app
+      MOLECULE_ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+      # LLM provider key so each runtime can authenticate at boot.
+      # Priority MiniMax → direct-Anthropic → OpenAI matches
+      # test_staging_full_saas.sh's secrets-injection chain.
+      E2E_MINIMAX_API_KEY: ${{ secrets.MOLECULE_STAGING_MINIMAX_API_KEY }}
+      E2E_ANTHROPIC_API_KEY: ${{ secrets.MOLECULE_STAGING_ANTHROPIC_API_KEY }}
+      E2E_OPENAI_API_KEY: ${{ secrets.MOLECULE_STAGING_OPENAI_API_KEY }}
+      E2E_RUN_ID: "${{ github.run_id }}-${{ github.run_attempt }}"
+      PV_RUNTIMES: "hermes openclaw claude-code"
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Verify admin token present
+        run: |
+          if [ -z "$MOLECULE_ADMIN_TOKEN" ]; then
+            echo "::error::CP_STAGING_ADMIN_API_TOKEN secret not set (Railway staging CP_ADMIN_API_TOKEN)"
+            exit 2
+          fi
+          echo "Admin token present"
+
+      - name: Verify an LLM key present
+        run: |
+          if [ -z "${E2E_MINIMAX_API_KEY:-}" ] && [ -z "${E2E_ANTHROPIC_API_KEY:-}" ] && [ -z "${E2E_OPENAI_API_KEY:-}" ]; then
+            echo "::error::No LLM provider key set — workspaces fail at boot with 'No provider API key found'. Set MOLECULE_STAGING_MINIMAX_API_KEY (or ANTHROPIC / OPENAI)."
+            exit 2
+          fi
+          echo "LLM key present"
+
+      - name: CP staging health preflight
+        run: |
+          code=$(curl -sS -o /dev/null -w "%{http_code}" --max-time 10 "$MOLECULE_CP_URL/health")
+          if [ "$code" != "200" ]; then
+            echo "::error::Staging CP unhealthy (HTTP $code) — infra, not a workspace bug. Failing loud per feedback_fix_root_not_symptom."
+            exit 1
+          fi
+          echo "Staging CP healthy"
+
+      - name: Run fresh-provision peer-visibility E2E (literal MCP list_peers)
+        run: bash tests/e2e/test_peer_visibility_mcp_staging.sh
+
+      # Belt-and-braces scoped teardown: the script installs an EXIT/INT/
+      # TERM trap, but if the runner itself is cancelled the trap may not
+      # fire. This always() step deletes ONLY the e2e-pv-<run_id> org this
+      # run created — never a cluster-wide sweep
+      # (feedback_never_run_cluster_cleanup_tests_on_live_platform). The
+      # admin DELETE is idempotent so double-invoking is safe;
+      # sweep-stale-e2e-orgs is the final net (slug starts with 'e2e-').
+      - name: Teardown safety net (runs on cancel/failure)
+        if: always()
+        env:
+          ADMIN_TOKEN: ${{ secrets.CP_STAGING_ADMIN_API_TOKEN }}
+        run: |
+          set +e
+          orgs=$(curl -sS "$MOLECULE_CP_URL/cp/admin/orgs?limit=500" \
+            -H "Authorization: Bearer $ADMIN_TOKEN" 2>/dev/null \
+            | python3 -c "
+          import json, sys, os, datetime
+          run_id = os.environ.get('GITHUB_RUN_ID', '')
+          try:
+              d = json.load(sys.stdin)
+          except Exception:
+              print(''); sys.exit(0)
+          # ONLY sweep slugs from THIS run. e2e-pv-<YYYYMMDD>-<run_id>-...
+          # Sweep today AND yesterday's UTC date so a midnight-crossing run
+          # still matches its own slug (same bug class as the saas/canvas
+          # safety nets).
+          today = datetime.date.today()
+          yest = today - datetime.timedelta(days=1)
+          dates = (today.strftime('%Y%m%d'), yest.strftime('%Y%m%d'))
+          if run_id:
+              prefixes = tuple(f'e2e-pv-{dt}-{run_id}-' for dt in dates)
+          else:
+              prefixes = tuple(f'e2e-pv-{dt}-' for dt in dates)
+          orgs = d if isinstance(d, list) else d.get('orgs', [])
+          cands = [o['slug'] for o in orgs
+                   if any(o.get('slug','').startswith(p) for p in prefixes)
+                   and o.get('instance_status') not in ('purged',)]
+          print('\n'.join(cands))
+          " 2>/dev/null)
+          for slug in $orgs; do
+            echo "Safety-net teardown: $slug"
+            set +e
+            curl -sS -o /tmp/pv-cleanup.out -w "%{http_code}" \
+              -X DELETE "$MOLECULE_CP_URL/cp/admin/tenants/$slug" \
+              -H "Authorization: Bearer $ADMIN_TOKEN" \
+              -H "Content-Type: application/json" \
+              -d "{\"confirm\":\"$slug\"}" >/tmp/pv-cleanup.code
+            set -e
+            code=$(cat /tmp/pv-cleanup.code 2>/dev/null || echo "000")
+            if [ "$code" = "200" ] || [ "$code" = "204" ]; then
+              echo "[teardown] deleted $slug (HTTP $code)"
+            else
+              echo "::warning::pv teardown for $slug returned HTTP $code — sweep-stale-e2e-orgs will catch it within MAX_AGE_MINUTES. Body: $(head -c 300 /tmp/pv-cleanup.out 2>/dev/null)"
+            fi
+          done
+          exit 0
@@ -32,6 +32,12 @@ on:
  # iterating all open PRs when PR_NUMBER is empty.
  workflow_dispatch:

+# Cancel stale runs so the 8-runner pool stays available for PR jobs.
+# Per-SHA group ensures push and cron runs at different SHAs don't cancel each other.
+concurrency:
+  group: gate-check-v3-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
 permissions:
  # read: contents — for checkout (base ref, not PR head for security)
  # read: pull-requests — for reading PR info via API
@@ -49,13 +49,17 @@ jobs:
  # bp-exempt: post-merge image publication side effect; CI / all-required gates source changes.
  build-and-push:
    name: Build & push canvas image
-    # REVERTED (infra/revert-docker-runner-label): `runs-on: ubuntu-latest` restored.
-    # The `docker` label is not registered on any act_runner. `runs-on: [ubuntu-latest, docker]`
-    # causes jobs to queue indefinitely with zero eligible runners — strictly worse than the
-    # pre-#599 coin-flip (50% success rate). Once the `docker` label is registered on
-    # ≥2 runners, re-apply the fix from #599 (infra/docker-runner-label).
-    # See issue #576 + infra-lead pulse ~00:30Z.
-    runs-on: ubuntu-latest
+    # Dedicated publish/release lane (internal#462 / #394 / #399). Ship
+    # path (on: push:main, canvas/**) — reserved capacity so a merged
+    # canvas fix's image build never FIFO-queues behind PR required-CI.
+    # The `publish` label resolves ONLY to the molecule-runner-publish-*
+    # sub-pool (config.publish.yaml). HARD DEPENDENCY: this MUST land
+    # AFTER the publish-lane runners are registered/advertising `publish`
+    # — the earlier #599 `docker` label attempt queued indefinitely with
+    # zero eligible runners precisely because the label was targeted
+    # before any runner advertised it (see #576). The lane is registered
+    # in this rollout (internal#462) so the precondition holds.
+    runs-on: publish
    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
    continue-on-error: true
@@ -66,7 +66,10 @@ concurrency:

 jobs:
  publish:
-    runs-on: ubuntu-latest
+    # Dedicated publish/release lane (internal#462 / #394 / #399). Ship
+    # path (on: push tag runtime-v*) — reserved capacity, never FIFO
+    # behind PR-CI. `publish` resolves only to molecule-runner-publish-*.
+    runs-on: publish
    outputs:
      version: ${{ steps.version.outputs.version }}
      wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
@@ -166,7 +169,9 @@ jobs:

  cascade:
    needs: publish
-    runs-on: ubuntu-latest
+    # Publish/release lane (internal#462) — downstream of the runtime
+    # publish ship job; keep it on the reserved lane too.
+    runs-on: publish
    steps:
      - name: Wait for PyPI to propagate the new version
        env:
@@ -54,7 +54,14 @@ env:

 jobs:
  build-and-push:
-    runs-on: ubuntu-latest
+    # Dedicated publish/release lane (internal#462 / #394 / #399). This
+    # is a post-merge ship job (on: push:main) — it must NOT FIFO-compete
+    # with PR required-CI on the shared pool (PR#1350's prod image build
+    # was delayed ~25min this way). The `publish` label resolves ONLY to
+    # the reserved molecule-runner-publish-* sub-pool (config.publish.yaml,
+    # OUTSIDE the managed 1..20 range) so a merged fix's image build
+    # starts immediately while PR-CI keeps the general pool.
+    runs-on: publish
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -181,7 +188,9 @@ jobs:
    name: Production auto-deploy
    needs: build-and-push
    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-    runs-on: ubuntu-latest
+    # Publish/release lane (internal#462) — production deploy of a merged
+    # fix; reserved capacity, never queued behind PR-CI.
+    runs-on: publish
    timeout-minutes: 75
    env:
      CP_URL: ${{ vars.PROD_CP_URL || 'https://api.moleculesai.app' }}
@@ -68,7 +68,10 @@ jobs:
  # bp-exempt: production redeploy is a side-effect workflow, not a merge gate.
  redeploy:
    if: ${{ github.event_name == 'workflow_dispatch' }}
-    runs-on: ubuntu-latest
+    # Dedicated publish/release lane (internal#462 / #394 / #399).
+    # Production tenant redeploy — a deploy action, reserved capacity so
+    # it never queues behind PR-CI. `publish` -> molecule-runner-publish-*.
+    runs-on: publish
    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
    continue-on-error: true
@@ -75,7 +75,10 @@ env:
 jobs:
  # bp-exempt: post-merge staging redeploy side effect; CI / all-required gates source changes.
  redeploy:
-    runs-on: ubuntu-latest
+    # Dedicated publish/release lane (internal#462 / #394 / #399).
+    # Post-merge staging redeploy — a deploy action, reserved capacity.
+    # `publish` -> molecule-runner-publish-* sub-pool.
+    runs-on: publish
    # Phase 3 (RFC #219 §1): surface broken workflows without blocking.
    # mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
    continue-on-error: true
@@ -44,6 +44,12 @@ on:
      - ".github/scripts/lint_secret_pattern_drift.py"
      - ".githooks/pre-commit"

+# Cancel stale runs to keep the 8-runner pool available for PR jobs.
+# Per-SHA group ensures push and scheduled runs at different SHAs don't cancel each other.
+concurrency:
+  group: secret-pattern-drift-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
 env:
  GITHUB_SERVER_URL: https://git.moleculesai.app

@@ -22,6 +22,11 @@ on:
    - cron: '17 4 * * 1'  # Mondays at 04:17 UTC
  workflow_dispatch:

+# Cancel stale runs to keep the 8-runner pool available for PR jobs.
+concurrency:
+  group: weekly-platform-go-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
+
 permissions:
  contents: read
  statuses: write
@@ -8,7 +8,7 @@ export default defineConfig({
  workers: 1,
  retries: 0,
  use: {
-    baseURL: "http://localhost:3000",
+    baseURL: process.env.PLAYWRIGHT_BASE_URL || "http://localhost:3000",
    headless: true,
    screenshot: "only-on-failure",
  },
@@ -176,7 +176,7 @@ export function deriveProvidersFromModels(models: ModelSpec[]): string[] {
 // exactly the point of the platform adaptor. The deep `~/.hermes/
 // config.yaml` on the container is a separate runtime-internal file,
 // not this one.
-const RUNTIMES_WITH_OWN_CONFIG = new Set<string>(["external", "kimi", "kimi-cli"]);
+const RUNTIMES_WITH_OWN_CONFIG = new Set<string>(["external", "kimi", "kimi-cli", "openclaw"]);

 const FALLBACK_RUNTIME_OPTIONS: RuntimeOption[] = [
  { value: "", label: "LangGraph (default)", models: [], providers: [] },
@@ -8,14 +8,18 @@ import { getTenantSlug } from "./tenant";
 export const PLATFORM_URL =
  process.env.NEXT_PUBLIC_PLATFORM_URL ?? "http://localhost:8080";

-// 15s is long enough for slow CP queries but short enough that a
-// hung backend doesn't leave the UI spinning forever. The abort
-// propagates through AbortController so React components can observe
-// the error and render a retry affordance. Callers that know the
-// endpoint is intentionally slow (org import walks a tree of
-// workspaces with server-side pacing) can pass `timeoutMs` to
-// override.
-const DEFAULT_TIMEOUT_MS = 15_000;
+// 35s is long enough for the slowest server-side path (EIC SSH
+// tunnel for tenant EC2 file operations, bounded server-side by
+// `eicFileOpTimeout = 30 * time.Second` in
+// workspace-server/internal/handlers/template_files_eic.go) so the
+// canvas surfaces the server's real error instead of aborting first
+// with a generic timeout. Shorter values caused "Save & Restart" to
+// time out at the client before the backend returned its 5xx. The
+// abort still propagates through AbortController so React components
+// can render a retry affordance. Callers that know an endpoint is
+// intentionally slow (org import walks a tree of workspaces with
+// server-side pacing) can pass `timeoutMs` to override.
+const DEFAULT_TIMEOUT_MS = 35_000;

 export interface RequestOptions {
  timeoutMs?: number;
@@ -30,10 +30,7 @@
    {"name": "openclaw", "repo": "molecule-ai/molecule-ai-workspace-template-openclaw", "ref": "main"},
    {"name": "codex", "repo": "molecule-ai/molecule-ai-workspace-template-codex", "ref": "main"},
    {"name": "langgraph", "repo": "molecule-ai/molecule-ai-workspace-template-langgraph", "ref": "main"},
-    {"name": "crewai", "repo": "molecule-ai/molecule-ai-workspace-template-crewai", "ref": "main"},
-    {"name": "autogen", "repo": "molecule-ai/molecule-ai-workspace-template-autogen", "ref": "main"},
-    {"name": "deepagents", "repo": "molecule-ai/molecule-ai-workspace-template-deepagents", "ref": "main"},
-    {"name": "gemini-cli", "repo": "molecule-ai/molecule-ai-workspace-template-gemini-cli", "ref": "main"}
+    {"name": "autogen", "repo": "molecule-ai/molecule-ai-workspace-template-autogen", "ref": "main"}
  ],
  "org_templates": [
    {"name": "molecule-dev", "repo": "molecule-ai/molecule-ai-org-template-molecule-dev", "ref": "main"},
@@ -0,0 +1,376 @@
+#!/usr/bin/env bash
+# Staging E2E — fresh-provision peer-visibility gate via the LITERAL MCP path.
+#
+# WHY THIS EXISTS
+# ---------------
+# Hermes and OpenClaw were repeatedly reported "fleet-verified / cascade-
+# complete" because the *proxy* signals were green:
+#   - registry-registration + heartbeat (Hermes), and
+#   - model round-trip 200 (OpenClaw).
+# But a freshly-provisioned workspace, asked on canvas "can you see your
+# peers", actually FAILS:
+#   - Hermes: 401 on the molecule MCP `list_peers` call,
+#   - OpenClaw: falls back to native `sessions_list`, sees no platform peers.
+# Tasks #142/#159 were even marked "completed" under this same proxy flaw.
+#
+# This script codifies the LITERAL user-facing path so it can never silently
+# regress: it provisions a brand-new throwaway org + sibling workspaces via
+# the real control-plane provisioning path, then for each runtime that should
+# have platform peer-visibility it drives the EXACT MCP call the canvas agent
+# makes — `POST /workspaces/:id/mcp` JSON-RPC tools/call name=list_peers,
+# authenticated by that workspace's own bearer token through the real
+# WorkspaceAuth + MCPRateLimiter middleware chain. It then asserts:
+#   (1) HTTP 200,
+#   (2) JSON-RPC `result` present (NOT an `error` object — a -32000
+#       "tool call failed" or a 401 from WorkspaceAuth fails here),
+#   (3) the returned peer set CONTAINS the other provisioned sibling
+#       workspace IDs — not an empty list, not a native-sessions fallback.
+#
+# This is NOT a proxy. It does not look at a registry row, /health, the
+# heartbeat table, or `GET /registry/:id/peers`. It drives the byte-for-byte
+# JSON-RPC envelope that mcp_molecule_list_peers issues from a real agent.
+#
+# It is written to FAIL on today's broken Hermes/OpenClaw behavior and go
+# green only when the in-flight root-cause fixes (Hermes-401, OpenClaw MCP
+# wiring) actually land. That is the point: it is the objective proof gate.
+#
+# AUTH MODEL (mirrors tests/e2e/test_staging_full_saas.sh)
+# --------------------------------------------------------
+#   Single MOLECULE_ADMIN_TOKEN (= CP_ADMIN_API_TOKEN on Railway staging)
+#   drives: POST /cp/admin/orgs (provision), GET
+#   /cp/admin/orgs/:slug/admin-token (per-tenant token), DELETE
+#   /cp/admin/tenants/:slug (teardown). The per-tenant admin token drives
+#   tenant workspace creation; each workspace's OWN auth_token (returned by
+#   POST /workspaces) drives its MCP call.
+#
+# Required env:
+#   MOLECULE_ADMIN_TOKEN   CP admin bearer — Railway staging CP_ADMIN_API_TOKEN
+# Optional env:
+#   MOLECULE_CP_URL        default https://staging-api.moleculesai.app
+#   E2E_RUN_ID             slug suffix; CI passes ${GITHUB_RUN_ID}
+#   PV_RUNTIMES            space list; default "hermes openclaw claude-code"
+#   E2E_PROVISION_TIMEOUT_SECS  default 1800 (hermes/openclaw cold EC2 budget)
+#   E2E_MINIMAX_API_KEY / E2E_ANTHROPIC_API_KEY / E2E_OPENAI_API_KEY
+#                          LLM provider key injected so the runtime can boot
+#   E2E_KEEP_ORG           1 → skip teardown (local debugging only)
+#
+# Exit codes:
+#   0  every runtime saw its peers via the literal MCP call
+#   1  generic failure
+#   2  missing required env
+#   3  provisioning timed out
+#   4  teardown left orphan resources
+#   10 peer-visibility regression reproduced (the gate firing as designed)
+
+set -uo pipefail
+
+CP_URL="${MOLECULE_CP_URL:-https://staging-api.moleculesai.app}"
+ADMIN_TOKEN="${MOLECULE_ADMIN_TOKEN:?MOLECULE_ADMIN_TOKEN required — Railway staging CP_ADMIN_API_TOKEN}"
+RUN_ID_SUFFIX="${E2E_RUN_ID:-$(date +%H%M%S)-$$}"
+PV_RUNTIMES="${PV_RUNTIMES:-hermes openclaw claude-code}"
+PROVISION_TIMEOUT_SECS="${E2E_PROVISION_TIMEOUT_SECS:-1800}"
+
+# Slug MUST start with 'e2e-' so the sweep-stale-e2e-orgs safety net
+# (EPHEMERAL_PREFIXES) catches any leak this run fails to tear down.
+SLUG="e2e-pv-$(date +%Y%m%d)-${RUN_ID_SUFFIX}"
+SLUG=$(echo "$SLUG" | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z0-9-' | head -c 32)
+
+ORG_ID=""
+TENANT_URL=""
+TENANT_TOKEN=""
+
+log()  { echo "[$(date +%H:%M:%S)] $*"; }
+fail() { echo "[$(date +%H:%M:%S)] ❌ $*" >&2; exit 1; }
+ok()   { echo "[$(date +%H:%M:%S)] ✅ $*"; }
+
+admin_call() {
+  local method="$1" path="$2"; shift 2
+  curl -sS -X "$method" "$CP_URL$path" \
+    -H "Authorization: Bearer $ADMIN_TOKEN" \
+    -H "Content-Type: application/json" "$@"
+}
+tenant_call() {
+  local method="$1" path="$2"; shift 2
+  curl -sS -X "$method" "$TENANT_URL$path" \
+    -H "Authorization: Bearer $TENANT_TOKEN" \
+    -H "X-Molecule-Org-Id: $ORG_ID" \
+    -H "Content-Type: application/json" "$@"
+}
+
+# ─── Scoped teardown ───────────────────────────────────────────────────
+# Deletes ONLY the org this run created (DELETE /cp/admin/tenants/$SLUG
+# with the {"confirm":$SLUG} fat-finger guard). Never a cluster-wide
+# sweep — honors feedback_cleanup_after_each_test and
+# feedback_never_run_cluster_cleanup_tests_on_live_platform. The
+# workflow's always() step + sweep-stale-e2e-orgs are the outer nets.
+teardown() {
+  local rc=$?
+  set +e
+  if [ "${E2E_KEEP_ORG:-0}" = "1" ]; then
+    echo ""
+    log "[teardown] E2E_KEEP_ORG=1 — leaving $SLUG for debugging (REMEMBER TO DELETE)"
+    exit $rc
+  fi
+  echo ""
+  log "[teardown] DELETE /cp/admin/tenants/$SLUG (scoped to this run only)"
+  admin_call DELETE "/cp/admin/tenants/$SLUG" --max-time 120 \
+    -d "{\"confirm\":\"$SLUG\"}" >/dev/null 2>&1
+  for j in $(seq 1 24); do
+    LIST=$(admin_call GET "/cp/admin/orgs?limit=500" 2>/dev/null)
+    LEAK=$(echo "$LIST" | python3 -c "
+import sys, json
+try: d = json.load(sys.stdin)
+except Exception: print(1); sys.exit(0)
+orgs = d if isinstance(d, list) else d.get('orgs', [])
+print(sum(1 for o in orgs if o.get('slug') == '$SLUG' and o.get('instance_status') not in ('purged',) and o.get('status') != 'purged'))
+" 2>/dev/null || echo 1)
+    if [ "$LEAK" = "0" ]; then
+      log "[teardown] ✓ $SLUG purged (after ${j}x5s)"
+      exit $rc
+    fi
+    sleep 5
+  done
+  echo "::warning::[teardown] $SLUG still present after 120s — sweep-stale-e2e-orgs will catch it within MAX_AGE_MINUTES" >&2
+  [ $rc -eq 0 ] && rc=4
+  exit $rc
+}
+trap teardown EXIT INT TERM
+
+# ─── 1. Provision the throwaway org ────────────────────────────────────
+log "1/6 POST /cp/admin/orgs — slug=$SLUG"
+CREATE=$(admin_call POST /cp/admin/orgs \
+  -d "{\"slug\":\"$SLUG\",\"name\":\"E2E peer-visibility $SLUG\",\"owner_user_id\":\"e2e-runner:$SLUG\"}")
+ORG_ID=$(echo "$CREATE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null)
+[ -n "$ORG_ID" ] || fail "org creation failed: $(echo "$CREATE" | head -c 300)"
+log "    ORG_ID=$ORG_ID"
+
+# ─── 2. Wait for tenant EC2 + DNS ──────────────────────────────────────
+log "2/6 waiting for tenant instance_status=running (cold EC2 + cloudflared)..."
+DEADLINE=$(( $(date +%s) + PROVISION_TIMEOUT_SECS ))
+while true; do
+  [ "$(date +%s)" -gt "$DEADLINE" ] && fail "tenant never came up within ${PROVISION_TIMEOUT_SECS}s"
+  STATUS=$(admin_call GET "/cp/admin/orgs?limit=500" 2>/dev/null | python3 -c "
+import sys, json
+try: d = json.load(sys.stdin)
+except Exception: sys.exit(0)
+orgs = d if isinstance(d, list) else d.get('orgs', [])
+for o in orgs:
+    if o.get('slug') == '$SLUG':
+        print(o.get('instance_status') or o.get('status') or 'unknown'); break
+" 2>/dev/null)
+  case "$STATUS" in running|online|ready) break ;; esac
+  sleep 10
+done
+log "    tenant status=$STATUS"
+
+# ─── 3. Per-tenant admin token + tenant URL ────────────────────────────
+log "3/6 fetching per-tenant admin token..."
+TT_RESP=$(admin_call GET "/cp/admin/orgs/$SLUG/admin-token")
+TENANT_TOKEN=$(echo "$TT_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('admin_token',''))" 2>/dev/null)
+[ -n "$TENANT_TOKEN" ] || fail "tenant token fetch failed: $(echo "$TT_RESP" | head -c 200)"
+
+CP_HOST=$(echo "$CP_URL" | sed -E 's#^https?://##; s#/.*$##')
+case "$CP_HOST" in
+  api.*)         DERIVED_DOMAIN="${CP_HOST#api.}" ;;
+  staging-api.*) DERIVED_DOMAIN="staging.${CP_HOST#staging-api.}" ;;
+  *)             DERIVED_DOMAIN="$CP_HOST" ;;
+esac
+TENANT_URL="https://${SLUG}.${DERIVED_DOMAIN}"
+log "    tenant url: $TENANT_URL"
+
+log "3b. waiting for tenant /health (TLS/DNS, up to 10min)..."
+for i in $(seq 1 120); do
+  curl -fsS "$TENANT_URL/health" -m 5 -k >/dev/null 2>&1 && { log "    /health ok (attempt $i)"; break; }
+  sleep 5
+done
+
+# ─── 4. Provision the parent + one sibling per runtime under test ──────
+# Inject the LLM provider key so each runtime can authenticate at boot.
+# Priority: MiniMax → direct-Anthropic → OpenAI (mirrors
+# test_staging_full_saas.sh's secrets-injection chain).
+SECRETS_JSON='{}'
+if [ -n "${E2E_MINIMAX_API_KEY:-}" ]; then
+  SECRETS_JSON=$(python3 -c "import json,os;k=os.environ['E2E_MINIMAX_API_KEY'];print(json.dumps({'ANTHROPIC_BASE_URL':'https://api.minimax.io/anthropic','ANTHROPIC_AUTH_TOKEN':k,'MINIMAX_API_KEY':k}))")
+elif [ -n "${E2E_ANTHROPIC_API_KEY:-}" ]; then
+  SECRETS_JSON=$(python3 -c "import json,os;k=os.environ['E2E_ANTHROPIC_API_KEY'];print(json.dumps({'ANTHROPIC_API_KEY':k}))")
+elif [ -n "${E2E_OPENAI_API_KEY:-}" ]; then
+  SECRETS_JSON=$(python3 -c "import json,os;k=os.environ['E2E_OPENAI_API_KEY'];print(json.dumps({'OPENAI_API_KEY':k,'OPENAI_BASE_URL':'https://api.openai.com/v1','MODEL_PROVIDER':'openai:gpt-4o','HERMES_INFERENCE_PROVIDER':'custom','HERMES_CUSTOM_BASE_URL':'https://api.openai.com/v1','HERMES_CUSTOM_API_KEY':k,'HERMES_CUSTOM_API_MODE':'chat_completions'}))")
+fi
+
+log "4/6 provisioning parent (claude-code) + one sibling per runtime under test..."
+P_RESP=$(tenant_call POST /workspaces \
+  -d "{\"name\":\"pv-parent\",\"runtime\":\"claude-code\",\"tier\":3,\"secrets\":$SECRETS_JSON}")
+PARENT_ID=$(echo "$P_RESP" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null)
+[ -n "$PARENT_ID" ] || fail "parent create failed: $(echo "$P_RESP" | head -c 300)"
+log "    PARENT_ID=$PARENT_ID"
+
+# WS_IDS[runtime]=id ; WS_TOKENS[runtime]=auth_token (the MCP bearer)
+declare -A WS_IDS WS_TOKENS
+ALL_WS_IDS="$PARENT_ID"
+for rt in $PV_RUNTIMES; do
+  R=$(tenant_call POST /workspaces \
+    -d "{\"name\":\"pv-$rt\",\"runtime\":\"$rt\",\"tier\":2,\"parent_id\":\"$PARENT_ID\",\"secrets\":$SECRETS_JSON}")
+  WID=$(echo "$R" | python3 -c "import sys,json; print(json.load(sys.stdin).get('id',''))" 2>/dev/null)
+  # auth_token is top-level for container runtimes; external-like nest it
+  # under connection.auth_token (verified vs staging response shape).
+  WTOK=$(echo "$R" | python3 -c "
+import sys, json
+try: d = json.load(sys.stdin)
+except Exception: print(''); sys.exit(0)
+print(d.get('auth_token') or d.get('connection', {}).get('auth_token') or '')
+" 2>/dev/null)
+  [ -n "$WID" ] || fail "$rt workspace create failed: $(echo "$R" | head -c 300)"
+  [ -n "$WTOK" ] || fail "$rt workspace did not return an auth_token — cannot drive its MCP call (resp: $(echo "$R" | head -c 300))"
+  WS_IDS[$rt]="$WID"
+  WS_TOKENS[$rt]="$WTOK"
+  ALL_WS_IDS="$ALL_WS_IDS $WID"
+  log "    $rt → $WID"
+done
+
+# ─── 5. Wait for every sibling online ──────────────────────────────────
+log "5/6 waiting for all workspaces status=online (up to ${PROVISION_TIMEOUT_SECS}s — cold boot)..."
+WS_DEADLINE=$(( $(date +%s) + PROVISION_TIMEOUT_SECS ))
+for rt in $PV_RUNTIMES; do
+  wid="${WS_IDS[$rt]}"
+  LAST=""
+  while true; do
+    [ "$(date +%s)" -gt "$WS_DEADLINE" ] && fail "$rt ($wid) never reached online (last=$LAST)"
+    S=$(tenant_call GET "/workspaces/$wid" 2>/dev/null | python3 -c "
+import sys, json
+try: d = json.load(sys.stdin)
+except Exception: sys.exit(0)
+w = d.get('workspace') if isinstance(d.get('workspace'), dict) else d
+print(w.get('status') or '')
+" 2>/dev/null)
+    [ "$S" != "$LAST" ] && { log "    $rt → $S"; LAST="$S"; }
+    case "$S" in
+      online) break ;;
+      failed) sleep 10 ;;   # transient: bootstrap-watcher 5-min deadline, heartbeat recovers
+      *)      sleep 10 ;;
+    esac
+  done
+  ok "    $rt online"
+done
+
+# ─── 6. THE GATE — literal mcp_molecule_list_peers via POST /:id/mcp ────
+# This is the byte-for-byte user-facing call. NOT GET /registry/:id/peers,
+# NOT /health, NOT the heartbeat table. JSON-RPC 2.0 tools/call,
+# name=list_peers, authenticated by the workspace's OWN bearer token
+# through WorkspaceAuth + MCPRateLimiter.
+log "6/6 driving the LITERAL list_peers MCP call per runtime..."
+echo ""
+RPC_BODY='{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"list_peers","arguments":{}}}'
+REGRESSED=0
+declare -A VERDICT
+
+for rt in $PV_RUNTIMES; do
+  wid="${WS_IDS[$rt]}"
+  wtok="${WS_TOKENS[$rt]}"
+  # The expected peer set = every OTHER provisioned workspace (parent +
+  # the sibling runtimes), excluding the caller itself.
+  EXPECT_IDS=$(echo "$ALL_WS_IDS" | tr ' ' '\n' | grep -v "^${wid}$" | grep -v '^$')
+
+  set +e
+  RESP=$(curl -sS -X POST "$TENANT_URL/workspaces/$wid/mcp" \
+    -H "Authorization: Bearer $wtok" \
+    -H "X-Molecule-Org-Id: $ORG_ID" \
+    -H "Content-Type: application/json" \
+    -d "$RPC_BODY" \
+    -o /tmp/pv_mcp_body.json -w "%{http_code}" 2>/dev/null)
+  set -e
+  HTTP_CODE="$RESP"
+  BODY=$(cat /tmp/pv_mcp_body.json 2>/dev/null || echo '')
+
+  echo "--- $rt (ws=$wid) ---"
+  echo "    HTTP $HTTP_CODE"
+  echo "    body: $(echo "$BODY" | head -c 600)"
+
+  # (1) HTTP 200 — a 401 (WorkspaceAuth reject, the Hermes symptom) fails here.
+  if [ "$HTTP_CODE" != "200" ]; then
+    echo "  ✗ $rt: list_peers MCP call returned HTTP $HTTP_CODE (expected 200)"
+    VERDICT[$rt]="FAIL(http=$HTTP_CODE)"
+    REGRESSED=1
+    continue
+  fi
+
+  # (2) JSON-RPC result present, not an error object.
+  PARSE=$(echo "$BODY" | python3 -c "
+import sys, json
+expect = set(filter(None, '''$EXPECT_IDS'''.split()))
+try:
+    d = json.load(sys.stdin)
+except Exception as e:
+    print('PARSE_ERROR:' + str(e)); sys.exit(0)
+if isinstance(d, dict) and d.get('error') is not None:
+    print('RPC_ERROR:' + json.dumps(d['error'])[:200]); sys.exit(0)
+res = d.get('result') if isinstance(d, dict) else None
+if res is None:
+    print('NO_RESULT'); sys.exit(0)
+# MCP tools/call result shape: {content:[{type:text,text:'<json or prose>'}]}
+text = ''
+if isinstance(res, dict):
+    for c in res.get('content', []):
+        if c.get('type') == 'text':
+            text += c.get('text', '')
+text_l = text.lower()
+# Native-sessions fallback signature (the OpenClaw symptom): the agent
+# answered from its own runtime session list, not the platform peer set.
+if 'sessions_list' in text_l or 'no platform peers' in text_l or 'native session' in text_l:
+    print('NATIVE_FALLBACK:' + text[:200]); sys.exit(0)
+# The expected sibling IDs must literally appear in the returned peer text.
+found = sorted(i for i in expect if i in text)
+missing = sorted(expect - set(found))
+if not expect:
+    print('NO_EXPECTED_PEERS_CONFIGURED'); sys.exit(0)
+if missing:
+    print('MISSING_PEERS:found=%d/%d missing=%s' % (len(found), len(expect), ','.join(m[:8] for m in missing)))
+    sys.exit(0)
+print('OK:found=%d/%d' % (len(found), len(expect)))
+" 2>/dev/null)
+
+  case "$PARSE" in
+    OK:*)
+      echo "  ✓ $rt: list_peers returned 200 and contains all expected peers ($PARSE)"
+      VERDICT[$rt]="OK"
+      ;;
+    NATIVE_FALLBACK:*)
+      echo "  ✗ $rt: list_peers fell back to NATIVE sessions — sees no platform peers ($PARSE)"
+      VERDICT[$rt]="FAIL(native-fallback)"
+      REGRESSED=1
+      ;;
+    RPC_ERROR:*|NO_RESULT|PARSE_ERROR:*)
+      echo "  ✗ $rt: list_peers MCP call did not return a usable result ($PARSE)"
+      VERDICT[$rt]="FAIL(rpc=$PARSE)"
+      REGRESSED=1
+      ;;
+    MISSING_PEERS:*)
+      echo "  ✗ $rt: list_peers returned 200 but peer set is wrong/empty ($PARSE)"
+      VERDICT[$rt]="FAIL(peers=$PARSE)"
+      REGRESSED=1
+      ;;
+    *)
+      echo "  ✗ $rt: unexpected verdict '$PARSE'"
+      VERDICT[$rt]="FAIL(unknown)"
+      REGRESSED=1
+      ;;
+  esac
+  echo ""
+done
+
+echo "=== SUMMARY — fresh-provision peer-visibility (literal MCP list_peers) ==="
+for rt in $PV_RUNTIMES; do
+  printf '  %-14s %s\n' "$rt" "${VERDICT[$rt]:-NO_RUN}"
+done
+echo ""
+
+if [ "$REGRESSED" -ne 0 ]; then
+  echo "✗ GATE FAILED — at least one runtime cannot see its peers via the"
+  echo "  literal mcp_molecule_list_peers call. This is the real user-facing"
+  echo "  failure the proxy signals (registry row / heartbeat / model 200)"
+  echo "  were hiding. Expected RED until the Hermes-401 + OpenClaw-MCP-wiring"
+  echo "  root-cause fixes land; goes green only when they actually do."
+  exit 10
+fi
+
+ok "GATE PASSED — every runtime under test sees its platform peers via the literal MCP call."
+exit 0
@@ -0,0 +1,160 @@
+package handlers
+
+// Regression coverage for the POLL-mode arm of the canvas user-message
+// data-loss bug (internal#470 sibling — tracked on internal#471).
+//
+// Bug (reported 2026-05-16 by CTO Hongming): "in canvas i sometimes lose
+// my own message when i exit chat". The push-mode arm was fixed by
+// #1347 (persistUserMessageAtIngest — a SYNCHRONOUS, before-dispatch,
+// context.WithoutCancel INSERT). #1347's framing asserted "poll-mode
+// workspaces were never affected — logA2AReceiveQueued already persists
+// at ingest". That assertion is OVERSTATED.
+//
+// Hongming's tenant (slug `hongming`, org 2c940477-...) has 4 workspaces,
+// ALL runtime=external with empty URL → ALL delivery_mode=poll (proven
+// empirically: a benign A2A probe returns the synthetic
+// {"delivery_mode":"poll","status":"queued"} envelope for every one).
+// So his reported loss is the POLL path, NOT the push path #1347 fixes.
+//
+// Root cause (poll arm): the poll-mode short-circuit (a2a_proxy.go ~402)
+// calls logA2AReceiveQueued and then IMMEDIATELY returns the synthetic
+// 200 {status:"queued"} to the canvas. But logA2AReceiveQueued's durable
+// INSERT runs inside h.goAsync(...) — a DETACHED goroutine with NO
+// happens-before barrier against the HTTP response. The canvas sees 200
+// ("message accepted") while the activity_logs row may not yet be — and,
+// on a workspace-server restart / deploy / OOM / EC2 hibernation between
+// the 200 and the goroutine's commit, NEVER will be — durable. There is
+// also no fallback (unlike push-mode's legacy-INSERT fallback): a
+// swallowed LogActivity error loses the message with only a log line.
+// Chat-history reads activity_logs (postgres_store.go:165-187); a missing
+// row = message gone on reopen. That is exactly Hongming's symptom.
+//
+// Fix (parity with push-mode): the poll-mode ingest persist of the
+// canvas user message must be SYNCHRONOUS — committed before the queued
+// 200 is returned — on a context.WithoutCancel derived context, so a
+// client disconnect on chat-exit and a post-response restart cannot lose
+// it. Behavior is never worse than today (best-effort; a persist error
+// still returns queued).
+//
+// TEST DESIGN NOTE: sqlmock.ExpectationsWereMet() hangs indefinitely if
+// the expected query never fires. We use a select+default+time.After
+// pattern so the test FAILS fast (not hangs) when the production code
+// regresses to async (the INSERT never fires before handler returns),
+// while still returning promptly when all expectations are met. The
+// insertDelay is kept small (50ms) to minimise suite-level timing
+// impact under -race detection, where mock delays are amplified by
+// the instrumenter's goroutine overhead.
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/gin-gonic/gin"
+)
+
+// TestProxyA2A_PollMode_PersistsUserMessageSynchronouslyBeforeQueuedResponse
+// is the defining contract: for a poll-mode workspace, the canvas user
+// message MUST be durably INSERTed into activity_logs BEFORE the synthetic
+// queued 200 is returned to the client — with NO reliance on a detached
+// async goroutine completing later.
+//
+// The test proves the ordering by making the INSERT block briefly and
+// asserting the handler does NOT return until the INSERT has completed.
+// Pre-fix (INSERT in h.goAsync, response returned immediately) the
+// handler returns ~instantly while the INSERT is still pending in the
+// goroutine → the elapsed time is far below the injected INSERT delay and
+// ExpectationsWereMet() is racy/unmet at return. Post-fix (synchronous
+// persist before the queued response) the handler return is gated on the
+// INSERT, so elapsed >= the injected delay and the expectation is met
+// deterministically at return WITHOUT any waitAsyncForTest()/sleep.
+func TestProxyA2A_PollMode_PersistsUserMessageSynchronouslyBeforeQueuedResponse(t *testing.T) {
+	mock := setupTestDB(t)
+	setupTestRedis(t)
+	broadcaster := newTestBroadcaster()
+	handler := NewWorkspaceHandler(broadcaster, nil, "http://localhost:8080", t.TempDir())
+
+	const wsID = "ws-poll-sync-persist"
+	// Keep delay small: -race detection amplifies mock delays significantly.
+	// A 50ms delay is sufficient to prove synchronous blocking (~50× the
+	// normal INSERT latency) without bloating the full ./... suite runtime.
+	const insertDelay = 50 * time.Millisecond
+
+	expectBudgetCheck(mock, wsID)
+
+	// lookupDeliveryMode → poll, triggering the short-circuit.
+	mock.ExpectQuery("SELECT delivery_mode FROM workspaces WHERE id").
+		WithArgs(wsID).
+		WillReturnRows(sqlmock.NewRows([]string{"delivery_mode"}).AddRow("poll"))
+
+	// workspace-name lookup inside logA2AReceiveQueued.
+	mock.ExpectQuery(`SELECT name FROM workspaces WHERE id`).
+		WithArgs(wsID).
+		WillReturnRows(sqlmock.NewRows([]string{"name"}).AddRow("Poll WS"))
+
+	// The durable user-message write. We delay it so a synchronous
+	// persist visibly gates the handler return; a detached-goroutine
+	// persist (pre-fix) does not. The fix must keep using
+	// context.WithoutCancel so this write survives a chat-exit cancel.
+	mock.ExpectExec("INSERT INTO activity_logs").
+		WillDelayFor(insertDelay).
+		WillReturnResult(sqlmock.NewResult(0, 1))
+
+	w := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(w)
+	c.Params = gin.Params{{Key: "id", Value: wsID}}
+
+	// callerID == "" (no X-Workspace-ID) → this is a canvas_user message,
+	// exactly Hongming's case.
+	body := `{"jsonrpc":"2.0","id":"poll-canvas-1","method":"message/send","params":{"message":{"role":"user","parts":[{"text":"my own message"}]}}}`
+	c.Request = httptest.NewRequest("POST", "/workspaces/"+wsID+"/a2a", bytes.NewBufferString(body))
+	c.Request.Header.Set("Content-Type", "application/json")
+
+	start := time.Now()
+	handler.ProxyA2A(c)
+	elapsed := time.Since(start)
+
+	// Defining assertion #1: the handler must not have returned the
+	// queued response before the durable INSERT committed. Pre-fix this
+	// fails (elapsed ≈ 0, INSERT still racing in goAsync).
+	if elapsed < insertDelay {
+		t.Fatalf("poll-mode queued response returned in %v, before the %v user-message INSERT — "+
+			"the message is not durable when the client/process goes away (DATA LOSS). "+
+			"Persist must be synchronous before the queued 200.", elapsed, insertDelay)
+	}
+
+	// Defining assertion #2: the durable write actually happened by the
+	// time the handler returned. ExpectionsWereMet() hangs indefinitely if
+	// the mock never fires (e.g. production code regressed to async),
+	// so we check it in a goroutine with a hard 2s timeout — fails fast
+	// (no CI hang) on regression while returning promptly on success.
+	expectDone := make(chan error, 1)
+	go func() { expectDone <- mock.ExpectationsWereMet() }()
+	select {
+	case err := <-expectDone:
+		if err != nil {
+			t.Fatalf("user-message INSERT was not durable at handler return (unmet sqlmock expectations): %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("ExpectationsWereMet() hung for >2s — INSERT mock never fired. " +
+			"Likely cause: production code regressed logA2AReceiveQueued to goAsync " +
+			"(INSERT fires after handler returns, not before).")
+	}
+
+	// Sanity: still the correct poll-mode envelope + status.
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200 (queued), got %d: %s", w.Code, w.Body.String())
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("response is not valid JSON: %v", err)
+	}
+	if resp["status"] != "queued" || resp["delivery_mode"] != "poll" {
+		t.Errorf("poll envelope changed: got status=%v delivery_mode=%v, want queued/poll",
+			resp["status"], resp["delivery_mode"])
+	}
+}
@@ -504,25 +504,49 @@ func lookupDeliveryMode(ctx context.Context, workspaceID string) string {
 // reads in PR 3 — that's how a poll-mode workspace receives inbound A2A
 // without a public URL.
 func (h *WorkspaceHandler) logA2AReceiveQueued(ctx context.Context, workspaceID, callerID string, body []byte, a2aMethod string) {
+	// DATA-LOSS FIX (internal#471 — poll-mode sibling of #1347/internal#470):
+	// this is the ONLY durable write of a poll-mode inbound message,
+	// including a canvas_user message (callerID == "") typed in the canvas
+	// chat. It MUST be SYNCHRONOUS and complete BEFORE the caller returns
+	// the synthetic {status:"queued"} 200 — otherwise the canvas sees the
+	// send acknowledged while the activity_logs row is still racing in a
+	// detached goroutine, and a workspace-server restart / deploy / OOM /
+	// EC2 hibernation between the 200 and the goroutine's commit loses the
+	// user's message permanently (chat-history reads activity_logs, so a
+	// missing row = message gone on reopen). Hongming's tenant is entirely
+	// poll-mode (4 external workspaces, no URL — verified empirically), so
+	// his reported loss is THIS path; #1347 (push-mode, persists AFTER the
+	// poll short-circuit) structurally cannot cover it.
+	//
+	// Mirrors persistUserMessageAtIngest's discipline:
+	//   - context.WithoutCancel: a client disconnect on chat-exit (which
+	//     cancels the inbound request ctx) MUST NOT abort this write.
+	//   - SYNCHRONOUS (no goAsync): the row must be durable before the
+	//     queued 200 is returned to the caller.
+	//   - Best-effort: LogActivity already logs+swallows INSERT errors, so
+	//     a hiccup never blocks or fails the user's send (behavior for
+	//     that one request is never worse than the pre-fix async path).
+	// The post-commit broadcast still fires inside LogActivity; a missed
+	// WebSocket event is not data loss (the durable row is the truth the
+	// canvas re-reads on reopen).
+	insCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+	defer cancel()
+
 	var wsName string
-	db.DB.QueryRowContext(ctx, `SELECT name FROM workspaces WHERE id = $1`, workspaceID).Scan(&wsName)
+	db.DB.QueryRowContext(insCtx, `SELECT name FROM workspaces WHERE id = $1`, workspaceID).Scan(&wsName)
 	if wsName == "" {
 		wsName = workspaceID
 	}
 	summary := a2aMethod + " → " + wsName + " (queued for poll)"
-	h.goAsync(func() {
-		logCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
-		defer cancel()
-		LogActivity(logCtx, h.broadcaster, ActivityParams{
-			WorkspaceID:  workspaceID,
-			ActivityType: "a2a_receive",
-			SourceID:     nilIfEmpty(callerID),
-			TargetID:     &workspaceID,
-			Method:       &a2aMethod,
-			Summary:      &summary,
-			RequestBody:  json.RawMessage(body),
-			Status:       "ok",
-		})
+	LogActivity(insCtx, h.broadcaster, ActivityParams{
+		WorkspaceID:  workspaceID,
+		ActivityType: "a2a_receive",
+		SourceID:     nilIfEmpty(callerID),
+		TargetID:     &workspaceID,
+		Method:       &a2aMethod,
+		Summary:      &summary,
+		RequestBody:  json.RawMessage(body),
+		Status:       "ok",
 	})
 }

@@ -44,8 +44,8 @@ func NewWorkspaceImageService(docker *dockerclient.Client) *WorkspaceImageServic
 // AllRuntimes is the canonical list mirroring docs/workspace-runtime-package.md.
 // Update both when a new template is added.
 var AllRuntimes = []string{
-	"claude-code", "langgraph", "crewai", "autogen",
-	"deepagents", "hermes", "gemini-cli", "openclaw",
+	"claude-code", "langgraph", "autogen",
+	"hermes", "openclaw",
 }

 // RefreshResult is the per-call outcome surfaced to HTTP callers AND logged
@@ -177,7 +177,7 @@ func isEnvIdentPart(c byte) bool {
 	return isEnvIdentStart(c) || (c >= '0' && c <= '9')
 }

-// loadWorkspaceEnv reads the org root .env and the workspace-specific .env .env and the workspace-specific .env
+// loadWorkspaceEnv reads the org root .env and the workspace-specific .env
 // (workspace overrides org root). Used by both secret injection and channel
 // config expansion.
 //
@@ -23,8 +23,8 @@ package models
 //   - claude-code: "sonnet" — Anthropic's CLI accepts the short
 //     name and resolves it via the operator's anthropic-oauth or
 //     ANTHROPIC_API_KEY chain.
-//   - everything else (hermes, langgraph, crewai, autogen, deepagents,
-//     codex, openclaw, gemini-cli, external, ""): a fully-qualified
+//   - everything else (hermes, langgraph, autogen, codex, openclaw,
+//     external, ""): a fully-qualified
 //     vendor:model slug that the universal MODEL_PROVIDER chain in
 //     molecule-core PR #247 can route via per-vendor required_env.
 //
@@ -21,12 +21,9 @@ func TestDefaultModel(t *testing.T) {
 		// as a generic "unknown" failure.
 		{"hermes", "anthropic:claude-opus-4-7"},
 		{"langgraph", "anthropic:claude-opus-4-7"},
-		{"crewai", "anthropic:claude-opus-4-7"},
 		{"autogen", "anthropic:claude-opus-4-7"},
-		{"deepagents", "anthropic:claude-opus-4-7"},
 		{"codex", "anthropic:claude-opus-4-7"},
 		{"openclaw", "anthropic:claude-opus-4-7"},
-		{"gemini-cli", "anthropic:claude-opus-4-7"},
 		{"external", "anthropic:claude-opus-4-7"},

 		// Unknown / empty — fall through to universal default rather
@@ -190,7 +190,7 @@ func TestEnsureLocalImage_RepoNotFound(t *testing.T) {
 	opts.HTTPClient = srv.Client()
 	opts.remoteHeadSha = nil // exercise real HTTP path

-	_, err := ensureLocalImageWithOpts(context.Background(), "crewai", opts)
+	_, err := ensureLocalImageWithOpts(context.Background(), "hermes", opts)
 	if err == nil {
 		t.Fatalf("expected error, got nil")
 	}
@@ -35,6 +35,19 @@ import (
 // drift-risk #6.
 var ErrNoBackend = errors.New("provisioner: no backend configured (zero-valued receiver)")

+// ErrUnresolvableRuntime is returned by selectImage when a workspace
+// names a runtime that has no resolvable image (not in RuntimeImages and
+// no operator-pinned cfg.Image). RFC internal#483 + security review 4269:
+// previously such a request silently fell through to DefaultImage
+// (langgraph) — a user asking for crewai would get a langgraph container
+// with no signal. The CTO standing directive
+// (feedback_platform_must_hardgate_base_contract) is fail-closed: a
+// named-but-unresolvable runtime must reject with a structured,
+// runtime-naming error so the existing provision-failed notify/log path
+// surfaces it, NOT silently degrade. The genuinely-unspecified (empty)
+// runtime is still a distinct, legitimate path that keeps DefaultImage.
+var ErrUnresolvableRuntime = errors.New("provisioner: requested runtime has no resolvable image")
+
 // RuntimeImages maps runtime names to their Docker image refs.
 // Each standalone template repo publishes its image via the reusable
 // publish-template-image workflow in molecule-ci on every main merge.
@@ -104,20 +117,33 @@ type WorkspaceConfig struct {
 // selectImage resolves the final Docker image ref for a workspace. The handler
 // layer is the source of truth — if it set cfg.Image (the digest-pinned form
 // from runtime_image_pins, #2272), honor that. Otherwise fall back to the
-// runtime→tag lookup in RuntimeImages (legacy `:latest` behavior). When the
-// runtime isn't recognized either, fall back to DefaultImage so Start() still
-// has something to hand Docker — surfacing a "No such image" later is more
-// actionable than a silent "" panic in ContainerCreate.
-func selectImage(cfg WorkspaceConfig) string {
+// runtime→tag lookup in RuntimeImages (legacy `:latest` behavior).
+//
+// Fail-closed contract (RFC internal#483 / security review 4269 /
+// feedback_platform_must_hardgate_base_contract): if the workspace NAMES a
+// runtime that resolves to no image (not in RuntimeImages, no pinned
+// cfg.Image), reject with ErrUnresolvableRuntime instead of silently
+// substituting DefaultImage. Pre-fix, removing crewai/deepagents/gemini-cli
+// from the catalog left those create requests silently provisioning a
+// langgraph container — the user asked for crewai and got langgraph with no
+// signal. The error propagates through Start → markProvisionFailed, which
+// already broadcasts WorkspaceProvisionFailed and records the message.
+//
+// The genuinely-unspecified runtime (empty cfg.Runtime, e.g. an org template
+// that doesn't pin one) is an intended distinct path and still resolves to
+// DefaultImage — only a NAMED-but-unresolvable runtime is rejected.
+func selectImage(cfg WorkspaceConfig) (string, error) {
 	if cfg.Image != "" {
-		return cfg.Image
+		return cfg.Image, nil
 	}
 	if cfg.Runtime != "" {
 		if img, ok := RuntimeImages[cfg.Runtime]; ok {
-			return img
+			return img, nil
 		}
+		return "", fmt.Errorf("%w: runtime %q (known runtimes: %v)",
+			ErrUnresolvableRuntime, cfg.Runtime, knownRuntimes)
 	}
-	return DefaultImage
+	return DefaultImage, nil
 }

 // Workspace-access constants for #65. Matches the CHECK constraint on
@@ -189,6 +215,24 @@ const containerNamePrefix = "ws-"
 // (the wiped-DB case after `docker compose down -v`).
 const LabelManaged = "molecule.platform.managed"

+// AgentUID / AgentGID are the uid/gid of the unprivileged `agent` user that
+// every workspace template creates and drops to via `gosu agent` before
+// exec'ing the runtime (the a2a_mcp_server runs under this uid). The value is
+// fixed at 1000:1000 across all templates — see:
+//   - workspace-configs-templates/claude-code-default/Dockerfile (`useradd -u 1000 ... agent`)
+//   - workspace-configs-templates/hermes/Dockerfile               (`useradd -u 1000 ... agent`)
+//   - workspace/entrypoint.sh                                     (`exec gosu agent` — "uid 1000")
+//
+// Files the platform injects into /configs AFTER the entrypoint's
+// `chown -R agent:agent /configs` (the post-start #418 re-injection and the
+// pre-start #1877 volume write) must be owned by this uid/gid, otherwise the
+// agent-uid MCP server hits EACCES reading /configs/.auth_token, sends an
+// empty bearer, and the platform 401s on /registry/{id}/peers (list_peers).
+const (
+	AgentUID = 1000
+	AgentGID = 1000
+)
+
 // managedLabels is the canonical label map applied to every workspace
 // container + volume. Pulled out so a future addition (e.g. instance
 // UUID for multi-platform-shared-daemon disambiguation) is one edit.
@@ -318,7 +362,15 @@ func (p *Provisioner) Start(ctx context.Context, cfg WorkspaceConfig) (string, e

 	env := buildContainerEnv(cfg)

-	image := selectImage(cfg)
+	image, imgErr := selectImage(cfg)
+	if imgErr != nil {
+		// Fail-closed: a named-but-unresolvable runtime must not silently
+		// become DefaultImage (RFC internal#483 / review 4269). The caller's
+		// error path (markProvisionFailed) broadcasts the failure + records
+		// the message so the canvas surfaces it.
+		log.Printf("Provisioner: refusing to start %s: %v", cfg.WorkspaceID, imgErr)
+		return "", imgErr
+	}

 	// Local-build mode (issue #63 / Task #194): when MOLECULE_IMAGE_REGISTRY
 	// is unset, the OSS contributor path skips the registry pull entirely
@@ -862,8 +914,18 @@ func buildTemplateTar(templatePath string) (*bytes.Buffer, error) {
 	return &buf, nil
 }

-// WriteFilesToContainer writes in-memory files into /configs in the container.
-func (p *Provisioner) WriteFilesToContainer(ctx context.Context, containerID string, files map[string][]byte) error {
+// buildConfigFilesTar builds the tar stream that WriteFilesToContainer streams
+// into /configs via CopyToContainer. Every entry is stamped Uid/Gid = agent
+// (AgentUID/AgentGID) so the files land agent-owned after extraction. This is
+// the issue #418 post-start re-injection path: it runs AFTER the template
+// entrypoint's `chown -R agent:agent /configs`, so without explicit ownership
+// in the tar header the files extract as root:root (tar Uid/Gid default 0) and
+// the agent-uid MCP server can no longer read /configs/.auth_token (and
+// /configs/.platform_inbound_secret) → empty bearer → list_peers 401.
+//
+// Pulled out as a pure function so the ownership contract is unit-testable
+// without a live Docker daemon (mirrors buildTemplateTar).
+func buildConfigFilesTar(files map[string][]byte) (*bytes.Buffer, error) {
 	var buf bytes.Buffer
 	tw := tar.NewWriter(&buf)

@@ -876,8 +938,10 @@ func (p *Provisioner) WriteFilesToContainer(ctx context.Context, containerID str
 				Typeflag: tar.TypeDir,
 				Name:     dir + "/",
 				Mode:     0755,
+				Uid:      AgentUID,
+				Gid:      AgentGID,
 			}); err != nil {
-				return fmt.Errorf("failed to write tar dir header for %s: %w", dir, err)
+				return nil, fmt.Errorf("failed to write tar dir header for %s: %w", dir, err)
 			}
 			createdDirs[dir] = true
 		}
@@ -886,19 +950,30 @@ func (p *Provisioner) WriteFilesToContainer(ctx context.Context, containerID str
 			Name: name,
 			Mode: 0644,
 			Size: int64(len(data)),
+			Uid:  AgentUID,
+			Gid:  AgentGID,
 		}
 		if err := tw.WriteHeader(header); err != nil {
-			return fmt.Errorf("failed to write tar header for %s: %w", name, err)
+			return nil, fmt.Errorf("failed to write tar header for %s: %w", name, err)
 		}
 		if _, err := tw.Write(data); err != nil {
-			return fmt.Errorf("failed to write tar data for %s: %w", name, err)
+			return nil, fmt.Errorf("failed to write tar data for %s: %w", name, err)
 		}
 	}
 	if err := tw.Close(); err != nil {
-		return fmt.Errorf("failed to close tar writer: %w", err)
+		return nil, fmt.Errorf("failed to close tar writer: %w", err)
 	}
+	return &buf, nil
+}

-	return p.cli.CopyToContainer(ctx, containerID, "/configs", &buf, container.CopyToContainerOptions{})
+// WriteFilesToContainer writes in-memory files into /configs in the container,
+// agent-owned (see buildConfigFilesTar).
+func (p *Provisioner) WriteFilesToContainer(ctx context.Context, containerID string, files map[string][]byte) error {
+	buf, err := buildConfigFilesTar(files)
+	if err != nil {
+		return err
+	}
+	return p.cli.CopyToContainer(ctx, containerID, "/configs", buf, container.CopyToContainerOptions{})
 }

 // CopyToContainer exposes CopyToContainer from the Docker client for use by other packages.
@@ -988,13 +1063,28 @@ func (p *Provisioner) ReadFromVolume(ctx context.Context, volumeName, filePath s
 	return clean, nil
 }

+// writeAuthTokenVolumeCmd is the shell command the throwaway alpine container
+// runs to seed /vol/.auth_token. alpine runs it as root, so without the
+// explicit `chown 1000:1000` the file stays root:root after the template
+// entrypoint's `chown -R agent:agent /configs` has already run — the agent-uid
+// (AgentUID) MCP server then gets EACCES reading it → empty bearer →
+// list_peers 401. Pulled out as a pure function so the ownership contract is
+// unit-testable without a live Docker daemon. Issue #1877.
+func writeAuthTokenVolumeCmd() string {
+	return fmt.Sprintf(
+		"mkdir -p /vol && printf '%%s' $TOKEN > /vol/.auth_token && chmod 0600 /vol/.auth_token && chown %d:%d /vol/.auth_token",
+		AgentUID, AgentGID,
+	)
+}
+
 // WriteAuthTokenToVolume writes the workspace auth token into the config volume
 // BEFORE the container starts, eliminating the token-injection race window where
 // a restarted container could read a stale token from /configs/.auth_token before
 // WriteFilesToContainer writes the new one. Issue #1877.
 //
 // Uses a throwaway alpine container to write directly to the named volume,
-// bypassing the container lifecycle entirely.
+// bypassing the container lifecycle entirely. The written file is chowned to
+// the agent uid/gid (see writeAuthTokenVolumeCmd).
 func (p *Provisioner) WriteAuthTokenToVolume(ctx context.Context, workspaceID, token string) error {
 	if p == nil || p.cli == nil {
 		return ErrNoBackend
@@ -1002,7 +1092,7 @@ func (p *Provisioner) WriteAuthTokenToVolume(ctx context.Context, workspaceID, t
 	volName := ConfigVolumeName(workspaceID)
 	resp, err := p.cli.ContainerCreate(ctx, &container.Config{
 		Image: "alpine",
-		Cmd:   []string{"sh", "-c", "mkdir -p /vol && printf '%s' $TOKEN > /vol/.auth_token && chmod 0600 /vol/.auth_token"},
+		Cmd:   []string{"sh", "-c", writeAuthTokenVolumeCmd()},
 		Env:   []string{"TOKEN=" + token},
 	}, &container.HostConfig{
 		Binds: []string{volName + ":/vol"},
@@ -513,7 +513,10 @@ func TestWorkspaceConfig_ResetClaudeSessionFieldPresent(t *testing.T) {
 // we lose the "one bad publish doesn't break every workspace" guarantee.
 func TestSelectImage_PrefersExplicitImage(t *testing.T) {
 	pinned := "ghcr.io/molecule-ai/workspace-template-claude-code@sha256:3d6761a97ed07d7d33cfc19a8fbab81175d9d9179618d493dbc00c5f7ef076a3"
-	got := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: pinned})
+	got, err := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: pinned})
+	if err != nil {
+		t.Fatalf("selectImage with cfg.Image=pinned: unexpected error %v", err)
+	}
 	if got != pinned {
 		t.Errorf("selectImage with cfg.Image=pinned: got %q, want %q", got, pinned)
 	}
@@ -523,28 +526,46 @@ func TestSelectImage_PrefersExplicitImage(t *testing.T) {
 // pin lookup deliberately bypassed via WORKSPACE_IMAGE_LOCAL_OVERRIDE).
 // selectImage must use the legacy runtime→:latest map.
 func TestSelectImage_FallsBackToRuntimeMap(t *testing.T) {
-	got := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: ""})
+	got, err := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: ""})
+	if err != nil {
+		t.Fatalf("selectImage with empty Image: unexpected error %v", err)
+	}
 	want := RuntimeImages["claude-code"]
 	if got != want {
 		t.Errorf("selectImage with empty Image: got %q, want %q", got, want)
 	}
 }

-// TestSelectImage_UnknownRuntimeFallsBackToDefault preserves today's
-// behavior — an unrecognized runtime resolves to DefaultImage rather than
-// "" so ContainerCreate gets a usable arg and surfaces a meaningful
-// "No such image" error if the default itself is missing.
-func TestSelectImage_UnknownRuntimeFallsBackToDefault(t *testing.T) {
-	got := selectImage(WorkspaceConfig{Runtime: "no-such-runtime"})
-	if got != DefaultImage {
-		t.Errorf("selectImage with unknown runtime: got %q, want DefaultImage %q", got, DefaultImage)
+// TestSelectImage_NamedUnresolvableRuntimeRejects pins the fail-closed
+// contract (RFC internal#483 / security review 4269 /
+// feedback_platform_must_hardgate_base_contract): a NAMED runtime with no
+// resolvable image must reject with ErrUnresolvableRuntime, NOT silently
+// substitute DefaultImage. Pre-fix this returned langgraph — a user asking
+// for a removed runtime (crewai/deepagents/gemini-cli) silently got a
+// langgraph container. "crewai" is the concrete regression from the
+// security finding.
+func TestSelectImage_NamedUnresolvableRuntimeRejects(t *testing.T) {
+	for _, rt := range []string{"no-such-runtime", "crewai", "deepagents", "gemini-cli"} {
+		got, err := selectImage(WorkspaceConfig{Runtime: rt})
+		if !errors.Is(err, ErrUnresolvableRuntime) {
+			t.Errorf("selectImage(%q): got err %v, want ErrUnresolvableRuntime", rt, err)
+		}
+		if got != "" {
+			t.Errorf("selectImage(%q): got image %q, want \"\" on reject", rt, got)
+		}
+		if err != nil && !strings.Contains(err.Error(), rt) {
+			t.Errorf("selectImage(%q): error must name the offending runtime, got %v", rt, err)
+		}
 	}
 }

 // TestSelectImage_EmptyRuntimeFallsBackToDefault: same invariant for the
 // no-runtime-supplied path (legacy callers / older handler code).
 func TestSelectImage_EmptyRuntimeFallsBackToDefault(t *testing.T) {
-	got := selectImage(WorkspaceConfig{})
+	got, err := selectImage(WorkspaceConfig{})
+	if err != nil {
+		t.Fatalf("selectImage with zero cfg: unexpected error %v (empty runtime is a legitimate DefaultImage path)", err)
+	}
 	if got != DefaultImage {
 		t.Errorf("selectImage with zero cfg: got %q, want DefaultImage %q", got, DefaultImage)
 	}
@@ -808,7 +829,7 @@ func TestIsImageNotFoundErr(t *testing.T) {
 		{"nil", nil, false},
 		{"moby no such image", fmtErr(`Error response from daemon: No such image: workspace-template:openclaw`), true},
 		{"no such image lowercase", fmtErr(`error: no such image: foo:bar`), true},
-		{"image not found", fmtErr(`Error: image "workspace-template:crewai" not found`), true},
+		{"image not found", fmtErr(`Error: image "workspace-template:hermes" not found`), true},
 		{"generic not found without image", fmtErr(`container not found`), false},
 		{"unrelated error", fmtErr(`connection refused`), false},
 		{"permission denied", fmtErr(`permission denied`), false},
@@ -21,9 +21,6 @@ var knownRuntimes = []string{
 	"autogen",
 	"claude-code",
 	"codex",
-	"crewai",
-	"deepagents",
-	"gemini-cli",
 	"hermes",
 	"langgraph",
 	"openclaw",
@@ -53,8 +53,8 @@ func TestRuntimeImage_AllKnownRuntimes(t *testing.T) {
 		}
 	}
 	// Pin the count so adding a runtime requires explicit test acknowledgement.
-	if len(knownRuntimes) != 9 {
-		t.Errorf("knownRuntimes length = %d, want 9 (autogen, claude-code, codex, crewai, deepagents, gemini-cli, hermes, langgraph, openclaw)", len(knownRuntimes))
+	if len(knownRuntimes) != 6 {
+		t.Errorf("knownRuntimes length = %d, want 6 (autogen, claude-code, codex, hermes, langgraph, openclaw)", len(knownRuntimes))
 	}
 }

@@ -0,0 +1,95 @@
+package provisioner
+
+import (
+	"archive/tar"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+)
+
+// These tests pin the P0 fix for the fleet-wide list_peers 401 (Hermes and
+// every other template): the workspace-server token-injection paths wrote
+// /configs/.auth_token (and /configs/.platform_inbound_secret) as root:root
+// AFTER the template entrypoint's `chown -R agent:agent /configs` ran, so the
+// agent-uid (1000) MCP server (a2a_mcp_server, running via `gosu agent`) hit
+// `[Errno 13] Permission denied` reading the bearer → empty bearer → platform
+// 401 on /registry/{id}/peers (the literal tool_list_peers path).
+//
+// The agent uid is 1000:1000, verified from the templates:
+//   - workspace-configs-templates/claude-code-default/Dockerfile: `useradd -u 1000 ... agent`
+//   - workspace-configs-templates/hermes/Dockerfile:               `useradd -u 1000 ... agent`
+//   - workspace/entrypoint.sh / claude-code-default/entrypoint.sh:  `exec gosu agent` ("uid 1000")
+//
+// Both tests assert the real artifact (the tar headers Docker's CopyToContainer
+// honours for ownership, and the literal shell command the throwaway alpine
+// container runs), not a mock that bypasses ownership. They FAIL on pre-fix
+// code (no Uid/Gid in tar headers; no chown in the alpine command → root:root)
+// and PASS post-fix (agent-owned).
+
+// TestWriteFilesToContainerTar_FilesAreAgentOwned covers the issue #418
+// post-start re-injection path (WriteFilesToContainer): the tar it streams
+// into /configs via CopyToContainer must carry Uid/Gid = agent (1000) so the
+// extracted files land agent-readable, not root:root. This is the path that
+// (re)writes BOTH .auth_token and .platform_inbound_secret on a cadence.
+func TestWriteFilesToContainerTar_FilesAreAgentOwned(t *testing.T) {
+	files := map[string][]byte{
+		".auth_token":              []byte("tok-abc123"),
+		".platform_inbound_secret": []byte("inbound-secret-xyz"),
+		"nested/dir/file.txt":      []byte("data"),
+	}
+
+	buf, err := buildConfigFilesTar(files)
+	if err != nil {
+		t.Fatalf("buildConfigFilesTar: %v", err)
+	}
+
+	tr := tar.NewReader(buf)
+	seen := map[string]bool{}
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			t.Fatalf("read tar: %v", err)
+		}
+		if _, err := io.Copy(io.Discard, tr); err != nil {
+			t.Fatalf("drain %s: %v", hdr.Name, err)
+		}
+		seen[hdr.Name] = true
+		if hdr.Uid != AgentUID {
+			t.Fatalf("tar entry %q Uid = %d, want %d (agent) — root-owned injection causes the list_peers 401",
+				hdr.Name, hdr.Uid, AgentUID)
+		}
+		if hdr.Gid != AgentGID {
+			t.Fatalf("tar entry %q Gid = %d, want %d (agent)", hdr.Name, hdr.Gid, AgentGID)
+		}
+	}
+
+	for _, want := range []string{".auth_token", ".platform_inbound_secret"} {
+		if !seen[want] {
+			t.Fatalf("tar missing %q (seen: %v)", want, seen)
+		}
+	}
+}
+
+// TestWriteAuthTokenVolumeCmd_ChownsToAgent covers the issue #1877 pre-start
+// volume-write path (WriteAuthTokenToVolume): the throwaway alpine container
+// writes /vol/.auth_token then chmod 0600 but, pre-fix, never chowns it, so it
+// stays root:root (alpine runs the command as root). The literal command must
+// chown the file to the agent uid:gid so the agent-uid MCP server can read it.
+func TestWriteAuthTokenVolumeCmd_ChownsToAgent(t *testing.T) {
+	cmd := writeAuthTokenVolumeCmd()
+
+	if !strings.Contains(cmd, "chmod 0600 /vol/.auth_token") {
+		t.Fatalf("alpine cmd lost the 0600 chmod (regression): %q", cmd)
+	}
+
+	wantChown := "chown 1000:1000 /vol/.auth_token"
+	if !strings.Contains(cmd, wantChown) {
+		t.Fatalf("alpine cmd = %q, missing %q — without it .auth_token stays root:root "+
+			"and the agent-uid MCP server gets EACCES → empty bearer → list_peers 401",
+			cmd, wantChown)
+	}
+}
@@ -431,6 +431,43 @@ def _is_self_notify_row(row: dict[str, Any]) -> bool:
    return source_id is None or source_id == ""


+def _is_self_echo_row(row: dict[str, Any], workspace_id: str) -> bool:
+    """Return True if ``row`` is a self-originated a2a_receive row.
+
+    Internal #469: when a workspace delegates to a target that never picks
+    up the task, ``tool_delegate_task`` calls ``report_activity`` which
+    POSTs to the platform with source_id set to the *sender's* workspace
+    UUID (mandated by spoof-defense in workspace-server's a2a_proxy). The
+    activity API exposes that row under type=a2a_receive, so the inbox
+    poller re-fetches it. Without this guard the row is surfaced as
+    kind='peer_agent' with the workspace's own identity as peer_id —
+    the workspace sees its own delegation-failure echoed back as if a
+    peer had delegated to it.
+
+    The guard mirrors the existing _is_self_notify_row pattern: both
+    skip rows that would otherwise create spurious inbound signal. The
+    long-term fix (making the platform write a distinct activity_type
+    for agent-outbound rows) is tracked separately; this guard stays
+    because it only excludes rows the agent never wants.
+
+    ``workspace_id`` must be non-empty — an empty-string workspace_id
+    (single-workspace legacy path) can never match a UUID source_id, so
+    the predicate is always False there, which is safe.
+
+    RFC #2829 PR-2 note: rows with method="delegate_result" are excluded
+    from the self-echo guard even when source_id matches our workspace_id.
+    The platform may write a delegation-result row with source_id set to
+    our workspace_id (e.g. a self-delegation or edge case in the platform's
+    result-writing path). Such rows must reach the inbox so that
+    message_from_activity can surface them as peer_agent inbound and the
+    runtime receives the delegation result. Silently filtering them as
+    self-echo would break delegation result delivery.
+    """
+    if not workspace_id:
+        return False
+    return row.get("source_id") == workspace_id and row.get("method") != "delegate_result"
+
+
 def message_from_activity(row: dict[str, Any]) -> InboxMessage:
    """Convert one /activity row into an InboxMessage.

@@ -623,6 +660,16 @@ def _poll_once(
            # the same self-notify on every iteration.
            last_id = str(row.get("id", "")) or last_id
            continue
+        if _is_self_echo_row(row, workspace_id):
+            # Internal #469: tool_delegate_task writes its own a2a_receive
+            # row with source_id = this workspace's UUID (spoof-defense).
+            # The poll fetches it back as kind='peer_agent', making the
+            # workspace echo its own delegation-failure as an inbound from
+            # a phantom peer. Skip it — the real delegation-result path
+            # (delegate_result push) is separate and unaffected. Cursor
+            # still advances so the next poll doesn't re-seen this row.
+            last_id = str(row.get("id", "")) or last_id
+            continue
        message = message_from_activity(row)
        if not message.activity_id:
            continue
@@ -495,6 +495,151 @@ def test_poll_once_skips_self_notify_rows(state: inbox.InboxState):
    assert [m.activity_id for m in queue] == ["act-real"]


+# ---------------------------------------------------------------------------
+# _is_self_echo_row — internal #469 fix
+# ---------------------------------------------------------------------------
+#
+# When a workspace delegates to a target that never picks up the task,
+# tool_delegate_task calls report_activity("a2a_receive", ...) which POSTs
+# to the platform with source_id set to the *sender's* workspace UUID
+# (spoof-defense). The activity API returns that row under type=a2a_receive
+# on the next poll, so message_from_activity sets peer_id = workspace's own
+# UUID — the workspace sees its own delegation-failure as an inbound from
+# a phantom peer. _is_self_echo_row guards against this.
+#
+# Internal #469 was live-reproduced on hongming.moleculesai.app 2026-05-16.
+
+
+def test_is_self_echo_row_true_when_source_id_matches_workspace():
+    row = {"source_id": "ws-abc123", "method": "a2a_receive"}
+    assert inbox._is_self_echo_row(row, "ws-abc123") is True
+
+
+def test_is_self_echo_row_false_when_source_id_differs():
+    """A real peer agent (different workspace_id) must NOT be filtered."""
+    row = {"source_id": "ws-peer", "method": "a2a_receive"}
+    assert inbox._is_self_echo_row(row, "ws-1") is False
+
+
+def test_is_self_echo_row_false_when_source_id_is_none():
+    """Canvas-user inbound has no source_id — never an echo."""
+    row = {"source_id": None, "method": "a2a_receive"}
+    assert inbox._is_self_echo_row(row, "ws-1") is False
+
+
+def test_is_self_echo_row_false_when_workspace_id_is_empty():
+    """Single-workspace legacy path with empty workspace_id cannot
+    match a UUID source_id — predicate is always False, which is safe."""
+    row = {"source_id": "ws-abc123", "method": "a2a_receive"}
+    assert inbox._is_self_echo_row(row, "") is False
+
+
+def test_is_self_echo_row_false_when_source_id_key_absent():
+    row = {"method": "a2a_receive"}
+    assert inbox._is_self_echo_row(row, "ws-1") is False
+
+
+def test_is_self_echo_row_false_for_delegate_result():
+    """RFC #2829 PR-2 regression pin: a row with source_id matching our
+    workspace_id but method=delegate_result must NOT be filtered as a
+    self-echo. The platform may write a delegation-result row with our
+    workspace_id as source_id; such rows must reach the inbox so the
+    runtime receives the delegation result. Silently filtering them would
+    break delegate_result delivery."""
+    row = {"source_id": "ws-1", "method": "delegate_result"}
+    assert inbox._is_self_echo_row(row, "ws-1") is False
+
+
+def test_poll_once_skips_self_echo_rows(state: inbox.InboxState):
+    """Internal #469 regression pin: a row with source_id matching our
+    workspace_id must NOT land in the inbox queue — it is our own
+    delegation-report echoing back, not a real peer inbound."""
+    rows = [
+        {
+            "id": "act-real-peer",
+            "source_id": "ws-peer",
+            "method": "a2a_receive",
+            "summary": None,
+            "request_body": {"parts": [{"type": "text", "text": "real peer inbound"}]},
+            "created_at": "2026-04-30T22:00:00Z",
+        },
+        {
+            "id": "act-self-echo",
+            "source_id": "ws-1",
+            "method": "a2a_receive",
+            "summary": "task result: target timed out",
+            "request_body": None,
+            "created_at": "2026-04-30T22:00:01Z",
+        },
+    ]
+    resp = _make_response(200, rows)
+    p, _ = _patch_httpx(resp)
+    with p:
+        n = inbox._poll_once(state, "http://platform", "ws-1", {})
+
+    # Only the real peer inbound counted; self-echo silently dropped.
+    assert n == 1
+    queue = state.peek(10)
+    assert [m.activity_id for m in queue] == ["act-real-peer"]
+    assert queue[0].peer_id == "ws-peer"
+
+
+def test_poll_once_advances_cursor_past_self_echo(state: inbox.InboxState):
+    """Cursor must advance past self-echo rows even though we don't
+    enqueue them. Otherwise the next poll re-fetches the same self-echo
+    on every iteration, wasting requests and blocking real inbound."""
+    state.save_cursor("act-old")
+    rows = [
+        {
+            "id": "act-self-echo",
+            "source_id": "ws-1",
+            "method": "a2a_receive",
+            "summary": "task result: timeout",
+            "request_body": None,
+            "created_at": "2026-04-30T22:00:00Z",
+        },
+    ]
+    resp = _make_response(200, rows)
+    p, _ = _patch_httpx(resp)
+    with p:
+        n = inbox._poll_once(state, "http://platform", "ws-1", {})
+
+    assert n == 0
+    assert state.peek(10) == []
+    # Cursor must move past the skipped row so we don't re-poll it.
+    assert state.load_cursor() == "act-self-echo"
+
+
+def test_poll_once_self_echo_does_not_fire_notification(state: inbox.InboxState):
+    """The notification callback (channel push to Claude Code etc.)
+    must not fire for self-echo rows. Same rationale as self-notify:
+    push-capable hosts would see the echo loop on the push channel."""
+    rows = [
+        {
+            "id": "act-self-echo",
+            "source_id": "ws-1",
+            "method": "a2a_receive",
+            "summary": "task result: timeout",
+            "request_body": None,
+            "created_at": "2026-04-30T22:00:00Z",
+        },
+    ]
+    received: list[dict] = []
+    inbox.set_notification_callback(received.append)
+    try:
+        resp = _make_response(200, rows)
+        p, _ = _patch_httpx(resp)
+        with p:
+            inbox._poll_once(state, "http://platform", "ws-1", {})
+    finally:
+        inbox.set_notification_callback(None)
+
+    assert received == [], (
+        "self-echo rows must not surface as MCP notifications — "
+        "doing so re-creates the echo loop on push-capable hosts"
+    )
+
+
 def test_poll_once_advances_cursor_past_self_notify(state: inbox.InboxState):
    """Cursor must advance past self-notify rows even though we don't
    enqueue them. Otherwise the next poll re-fetches the same self-