Compare commits

..

1 Commits

Author SHA1 Message Date
infra-runtime-be 9aba046dff fix(a2a-mcp): use readline() not read(65536) for pipe-safe stdio
a2a_mcp_server.py main()'s stdio read loop used
`await loop.run_in_executor(None, stdin.read, 65536)`. On a PIPE,
read(n) blocks until n bytes accumulate OR EOF. A live MCP client
(openclaw bundle-mcp, Claude Code, Cursor) sends one ~150-byte
newline-delimited request and keeps stdin OPEN waiting for the reply,
so neither condition is met: the server never parses `initialize` and
the client times out (~30s; openclaw: "MCP error -32000: Connection
closed"). This silently broke peer visibility for every pipe-spawned
MCP host while passing all existing stdio tests, which only fed stdin
from a regular file or a heredoc-pipe that CLOSES (EOF returns
immediately). readline() returns as soon as one newline-delimited
line is available — exactly the JSON-RPC framing — and is
backward-compatible with the EOF/file cases.

Root cause of the 2026-05-15 openclaw peer-visibility outage
(workspace 95744c11): the molecule MCP server could not complete the
handshake over openclaw's stdio pipe, so the agent fell back to
native sessions_list. The openclaw template adapter fix
(template-openclaw#16) works around this via HTTP transport; this
patch fixes the stdio root cause so stdio works for all CLI MCP hosts.

Regression coverage:
- tests/test_a2a_mcp_server.py::TestStdioKeepOpenPipe — spawns the
  real a2a_mcp_server.py, writes one request over a pipe, and
  DELIBERATELY keeps stdin open. FAILS (15s timeout, empty response)
  on read(65536); PASSES on readline(). Verified both directions.
- ci-mcp-stdio-transport.yml: new "pipe held OPEN, no EOF" step that
  reproduces the literal openclaw failure (the prior steps only
  exercised EOF-closing stdin, which is why the outage shipped green).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-16 15:56:04 +00:00
28 changed files with 302 additions and 750 deletions
+3 -86
View File
@@ -44,15 +44,9 @@ REQUIRED_CONTEXTS_RAW = _env(
"REQUIRED_CONTEXTS",
default=(
"CI / all-required (pull_request),"
"sop-checklist / all-items-acked (pull_request),"
"E2E Chat / E2E Chat (pull_request)"
"sop-checklist / all-items-acked (pull_request)"
),
)
# E2E Chat is not in branch protection's status_check_contexts, but Gitea's
# merge gate evaluates the full combined status including it. Adding it here
# prevents the queue from attempting a merge that will be 405'd by Gitea when
# E2E Chat is failing (e.g. runner-stall Quirk #9 on a flaky test).
# See: mc#420 / molecule-core runbooks/gitea-operational-quirks.md Quirk #9.
# Required contexts for push (main/staging) runs. The push CI uses the same
# aggregator names with " (push)" suffix. Checking these explicitly instead of
# the combined state avoids false-pause when non-blocking jobs (e.g. Platform
@@ -71,11 +65,6 @@ class ApiError(RuntimeError):
pass
class MergePermissionError(ApiError):
"""Merge failed with a permanent permission error (403/404/405).
The queue should skip this PR and move to the next one."""
@dataclasses.dataclass(frozen=True)
class MergeDecision:
ready: bool
@@ -325,31 +314,6 @@ def post_comment(pr_number: int, body: str, *, dry_run: bool) -> None:
api("POST", f"/repos/{OWNER}/{NAME}/issues/{pr_number}/comments", body={"body": body})
def add_hold_label(pr_number: int, *, dry_run: bool) -> None:
"""Add HOLD_LABEL to a PR if not already present."""
if not HOLD_LABEL:
return
# Check current labels first to avoid a no-op API call in dry-run.
_, current = api("GET", f"/repos/{OWNER}/{NAME}/issues/{pr_number}/labels")
current_names = {
l["name"] for l in (current if isinstance(current, list) else [])
}
if HOLD_LABEL in current_names:
print(f"::notice::PR #{pr_number} already has hold label; skipping add")
return
print(f"::notice::PR #{pr_number} adding hold label `{HOLD_LABEL}`")
if dry_run:
return
# Gitea accepts {"labels": ["label1", "label2"]} to append labels.
new_labels = list(current_names) + [HOLD_LABEL]
api(
"PATCH",
f"/repos/{OWNER}/{NAME}/issues/{pr_number}",
body={"labels": new_labels},
expect_json=False,
)
def update_pull(pr_number: int, *, dry_run: bool) -> None:
print(f"::notice::updating PR #{pr_number} with base branch via style={UPDATE_STYLE}")
if dry_run:
@@ -374,16 +338,7 @@ def merge_pull(pr_number: int, *, dry_run: bool) -> None:
print(f"::notice::merging PR #{pr_number}")
if dry_run:
return
try:
api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
except ApiError as exc:
# Re-raise permission-like errors so process_once can skip this PR.
# 403 = no push access, 404 = repo/pr not found, 405 = not allowed.
msg = str(exc)
for code in ("403", "404", "405"):
if code in msg:
raise MergePermissionError(msg) from exc
raise # re-raise other ApiErrors unchanged
api("POST", f"/repos/{OWNER}/{NAME}/pulls/{pr_number}/merge", body=payload, expect_json=False)
def process_once(*, dry_run: bool = False) -> int:
@@ -452,45 +407,7 @@ def process_once(*, dry_run: bool = False) -> int:
"deferring to next tick"
)
return 0
try:
merge_pull(pr_number, dry_run=dry_run)
except MergePermissionError as exc:
msg = str(exc)
is_status_check_failure = "not all required status checks successful" in msg
if is_status_check_failure:
# Gitea's merge gate failed due to a status check that passed our
# pre-flight but is failing at Gitea's side (e.g. runner-stall Quirk
# #9, or a context not in REQUIRED_CONTEXTS). Auto-add hold so the
# queue skips this PR and processes the next one. The hold can be
# removed once CI is green again.
add_hold_label(pr_number, dry_run=dry_run)
post_comment(
pr_number,
(
"merge-queue: merge blocked by Gitea's status-check gate "
"(E2E Chat or other non-required context failing). "
"Auto-held via `merge-queue-hold`. "
"Remove the hold label to requeue once CI is green. "
"If E2E Chat is stuck (runner stall / Quirk #9), CI will "
"self-recover after ~90 min and the hold can then be removed."
),
dry_run=dry_run,
)
return 0
else:
# Genuine permission error — token lacks Can-merge.
sys.stderr.write(f"::error::merge permission error for PR #{pr_number}: {exc}\n")
post_comment(
pr_number,
(
"merge-queue: merge failed with HTTP 405 'User not allowed to merge PR'. "
"No available token has Can-merge permission on this repo. "
"Fix: grant Can-merge to a token, or add a maintain/admin collaborator. "
"Skipping to next queued PR on next tick."
),
dry_run=dry_run,
)
return 0
merge_pull(pr_number, dry_run=dry_run)
return 0
return 0
+25 -168
View File
@@ -68,7 +68,7 @@ import sys
import urllib.error
import urllib.parse
import urllib.request
from typing import Any, Callable
from typing import Any
# ---------------------------------------------------------------------------
@@ -110,7 +110,7 @@ def normalize_slug(raw: str, numeric_aliases: dict[int, str] | None = None) -> s
# for /sop-revoke (RFC#351 open question 4 — reason is captured but not
# yet validated; future iteration may require a min-length).
_DIRECTIVE_RE = re.compile(
r"^[ \t]*/(sop-ack|sop-revoke|sop-n/a)[ \t]+([A-Za-z0-9_\- ]+?)(?:[ \t]+(.*))?[ \t]*$",
r"^[ \t]*/(sop-ack|sop-revoke)[ \t]+([A-Za-z0-9_\- ]+?)(?:[ \t]+(.*))?[ \t]*$",
re.MULTILINE,
)
@@ -118,21 +118,19 @@ _DIRECTIVE_RE = re.compile(
def parse_directives(
comment_body: str,
numeric_aliases: dict[int, str],
) -> tuple[list[tuple[str, str, str]], list[tuple[str, str, str]]]:
"""Extract /sop-ack, /sop-revoke, and /sop-n/a directives from a comment body.
) -> tuple[list[tuple[str, str, str]], list]:
"""Extract /sop-ack and /sop-revoke directives from a comment body.
Returns (directives, na_directives) where each is a list of
(kind, canonical_slug, note) tuples:
kind is "sop-ack", "sop-revoke", or "sop-n/a"
canonical_slug is the normalized form (or "" if unparseable)
note is the trailing free-text (may be "")
The two lists are kept separate so call sites can unpack them
directly (e.g. directives, na_directives = parse_directives(...)).
Returns (directives, na_directives) where:
directives is a list of (kind, canonical_slug, note) tuples
kind is "sop-ack" or "sop-revoke"
canonical_slug is the normalized form (or "" if unparseable)
note is the trailing free-text (may be "")
na_directives is reserved for future N/A handling (always [] for now)
"""
directives: list[tuple[str, str, str]] = []
na_directives: list[tuple[str, str, str]] = []
out: list[tuple[str, str, str]] = []
if not comment_body:
return directives, na_directives
return out, []
for m in _DIRECTIVE_RE.finditer(comment_body):
kind = m.group(1)
raw_slug = (m.group(2) or "").strip()
@@ -162,12 +160,8 @@ def parse_directives(
note_from_group = (m.group(3) or "").strip()
# If we collapsed multi-word slug into kebab and there's a
# trailing-text group too, append it.
entry = (kind, canonical, note_from_group)
if kind == "sop-n/a":
na_directives.append(entry)
else:
directives.append(entry)
return directives, na_directives
out.append((kind, canonical, note_from_group))
return out, []
# ---------------------------------------------------------------------------
@@ -180,8 +174,8 @@ def section_marker_present(body: str, marker: str) -> bool:
on a non-empty line (i.e. the author actually filled it in).
We require the marker substring AND non-whitespace content on the
same line OR within the next non-blank line — this prevents
trivially-empty checklists like:
same line OR within the next line — this prevents trivially-empty
checklists like:
## SOP-Checklist
- [ ] **Comprehensive testing performed**:
@@ -190,18 +184,9 @@ def section_marker_present(body: str, marker: str) -> bool:
from auto-passing the section-present check. The peer-ack is still
required, but answering with empty content is captured as a soft
finding via the section-present test alone.
NOTE: we scan forward through blank lines (the markdown-header pattern
is ## Header\\n\\ncontent) so that a header + blank-line + content
structure still satisfies the check. The backward checkbox fallback
catches inline markers without a preceding checkbox (mc#1099).
"""
if not body or not marker:
return False
# Strip trailing whitespace so the blank-line scan below can find
# content that appears on the very last line of the body (without
# being misled by a trailing \n or spaces).
body = body.rstrip()
body_lower = body.lower()
marker_lower = marker.lower()
idx = body_lower.find(marker_lower)
@@ -217,44 +202,13 @@ def section_marker_present(body: str, marker: str) -> bool:
stripped = re.sub(r"[\s\*:\-\[\]]+", "", line)
if stripped:
return True
# Fall through: scan forward, skipping blank-only lines, until we find
# non-empty content or run out of body. Handles:
# ## Header ← marker line (empty after marker)
# ← blank line (skipped)
# - actual content ← found
pos = line_end
while True:
# Skip the current newline and any additional newlines (blank lines).
while pos < len(body) and body[pos] == "\n":
pos += 1
if pos >= len(body):
break
line_end = body.find("\n", pos)
if line_end < 0:
line_end = len(body)
line = body[pos:line_end]
stripped = re.sub(r"[\s\*:\-\[\]]+", "", line)
if stripped:
return True
pos = line_end
# Last resort: the marker may appear mid-sentence (e.g.
# **Memory/saved-feedback consulted**: No applicable...).
# Search backward within the CURRENT LINE only (not preceding lines)
# to find a checkbox on the same line before the marker text.
# mc#1099 follow-up: memory-consulted detection was failing because
# the checkbox was on the same line before the inline marker.
_CHECKBOX_RE = re.compile(r"- \[[ x\]]|<input", re.IGNORECASE)
line_start = body.rfind("\n", 0, idx) + 1 # 0 if no newline before idx
before = body[line_start:idx]
m = _CHECKBOX_RE.search(before)
if not m:
return False
# Require meaningful content between the checkbox and the marker text
# (markdown formatting like ** or * must also be stripped).
# If only whitespace/markdown chars remain, the checkbox line is empty.
between = before[m.end() :]
stripped_between = re.sub(r"[\s\*:#\[\]_\-]+", "", between)
return bool(stripped_between)
# Fall through: check the NEXT line (multi-line answers).
next_line_end = body.find("\n", line_end + 1)
if next_line_end < 0:
next_line_end = len(body)
next_line = body[line_end + 1:next_line_end]
stripped_next = re.sub(r"[\s\*:\-\[\]]+", "", next_line)
return bool(stripped_next)
# ---------------------------------------------------------------------------
@@ -297,7 +251,8 @@ def compute_ack_state(
user = (c.get("user") or {}).get("login", "")
if not user:
continue
for kind, slug, _note in parse_directives(body, numeric_aliases)[0]:
directives, _na = parse_directives(body, numeric_aliases)
for kind, slug, _note in directives:
if not slug:
unparseable_per_user[user] = unparseable_per_user.get(user, 0) + 1
continue
@@ -349,63 +304,6 @@ def compute_ack_state(
}
# ---------------------------------------------------------------------------
# N/A-gate evaluation
# ---------------------------------------------------------------------------
def compute_na_state(
comments: list[dict[str, Any]],
author: str,
na_gates: dict[str, Any],
probe: Callable[[str, list[str]], list[str]],
) -> dict[str, dict[str, Any]]:
"""Evaluate which N/A gates have a valid declaration from a team member.
Returns dict[gate_name, dict] where each dict has:
declared: bool — at least one valid non-author team-member declared N/A
decl_ackers: list[str] — usernames who declared this gate N/A
rejected: dict with keys:
not_in_team: list[str] — users who tried but aren't in required teams
"""
# Build per-user latest N/A directive (most-recent wins per RFC#324).
latest_na: dict[str, tuple[str, str]] = {} # user → (gate, note)
for c in comments:
body = c.get("body", "") or ""
user = (c.get("user") or {}).get("login", "")
if not user:
continue
for kind, gate, note in parse_directives(body, {})[1]:
# [1] = na_directives only
if gate in na_gates:
latest_na[user] = (gate, note)
result: dict[str, dict[str, Any]] = {}
for gate, gate_cfg in na_gates.items():
result[gate] = {
"declared": False,
"decl_ackers": [],
"rejected": {"not_in_team": []},
}
decl_ackers: list[str] = []
not_in_team: list[str] = []
for user, (g, _note) in latest_na.items():
if g != gate:
continue
if user == author:
continue # authors cannot self-declare N/A
approved = probe(gate, [user])
if approved:
decl_ackers.append(user)
else:
not_in_team.append(user)
result[gate]["declared"] = bool(decl_ackers)
result[gate]["decl_ackers"] = decl_ackers
result[gate]["rejected"]["not_in_team"] = not_in_team
return result
# ---------------------------------------------------------------------------
# Gitea API client
# ---------------------------------------------------------------------------
@@ -800,7 +698,6 @@ def main(argv: list[str] | None = None) -> int:
cfg = load_config(args.config)
items: list[dict[str, Any]] = cfg["items"]
items_by_slug = {it["slug"]: it for it in items}
na_gates: dict[str, Any] = cfg.get("n/a_gates", {})
numeric_aliases = {
int(it["numeric_alias"]): it["slug"] for it in items if it.get("numeric_alias")
}
@@ -921,46 +818,6 @@ def main(argv: list[str] | None = None) -> int:
description=description, target_url=target_url,
)
print(f"::notice::status posted: {args.status_context}{state}")
# --- N/A gate status (RFC#324 §N/A follow-up) ---
# Post a separate status so review-check.sh can discover N/A declarations
# and waive the Gitea-approve requirement for that gate.
na_state: dict[str, dict[str, Any]] = {}
if na_gates:
na_state = compute_na_state(comments, author, na_gates, probe)
na_descs: list[str] = []
for gate, s in na_state.items():
if s["declared"]:
na_descs.append(gate)
decl = s["decl_ackers"]
rej = s["rejected"]["not_in_team"]
if decl:
print(f"::notice:: [N/A OK] {gate} — declared by {','.join(decl)}")
if rej:
print(
f"::notice:: [N/A REJ] {gate} — not-in-team: {','.join(rej)}",
file=sys.stderr,
)
na_desc = ", ".join(sorted(na_descs)) if na_descs else "(none)"
na_status_state = "success" if na_descs else "pending"
# review-check.sh reads the description to discover which gates are N/A.
# Include the gate names so it can grep for them.
na_description = f"N/A: {na_desc}" if na_descs else "N/A: (none)"
if not args.dry_run:
client.post_status(
args.owner, args.repo, head_sha,
state=na_status_state,
context="sop-checklist / na-declarations (pull_request)",
description=na_description,
target_url=target_url,
)
print(
f"::notice::na-declarations status → {na_status_state}: {na_description}"
)
# By default exit 0 — the POSTed status IS the gate, NOT the job
# conclusion. If the job exits 1 BP will see TWO failure signals
# (one from the job's auto-status, one from our POST), making the
@@ -118,13 +118,3 @@ def test_merge_decision_updates_stale_pr_before_merge():
assert decision.ready is False
assert decision.action == "update"
def test_MergePermissionError_inherits_from_ApiError():
assert issubclass(mq.MergePermissionError, mq.ApiError)
def test_MergePermissionError_message_preserved():
exc = mq.MergePermissionError("POST /merge -> HTTP 405: User not allowed")
assert "405" in str(exc)
assert "User not allowed" in str(exc)
@@ -551,55 +551,3 @@ class TestEndToEndAckFlow(unittest.TestCase):
if __name__ == "__main__":
unittest.main(verbosity=2)
# ---------------------------------------------------------------------------
# compute_na_state
# ---------------------------------------------------------------------------
class TestComputeNaState(unittest.TestCase):
"""Tests for /sop-n/a directive evaluation."""
def test_no_na_declarations(self):
cfg = sop.load_config(CONFIG_PATH)
na_gates = cfg.get("n/a_gates", {})
comments = []
na_state = sop.compute_na_state(comments, "alice", na_gates, lambda *_: [])
self.assertFalse(na_state["qa-review"]["declared"])
self.assertFalse(na_state["security-review"]["declared"])
def test_na_declared_by_authorized_user(self):
cfg = sop.load_config(CONFIG_PATH)
na_gates = cfg.get("n/a_gates", {})
comments = [_comment("bob", "/sop-n/a qa-review N/A: pure tooling change")]
na_state = sop.compute_na_state(comments, "alice", na_gates, lambda g, u: u)
self.assertTrue(na_state["qa-review"]["declared"])
self.assertEqual(na_state["qa-review"]["decl_ackers"], ["bob"])
def test_na_declared_by_unauthorized_user_rejected(self):
cfg = sop.load_config(CONFIG_PATH)
na_gates = cfg.get("n/a_gates", {})
comments = [_comment("mallory", "/sop-n/a qa-review N/A: not real team")]
na_state = sop.compute_na_state(comments, "alice", na_gates, lambda g, u: [])
self.assertFalse(na_state["qa-review"]["declared"])
self.assertEqual(na_state["qa-review"]["rejected"]["not_in_team"], ["mallory"])
def test_author_cannot_self_declare_na(self):
cfg = sop.load_config(CONFIG_PATH)
na_gates = cfg.get("n/a_gates", {})
comments = [_comment("alice", "/sop-n/a qa-review N/A: I am the author")]
na_state = sop.compute_na_state(comments, "alice", na_gates, lambda g, u: u)
self.assertFalse(na_state["qa-review"]["declared"])
def test_parse_directives_separates_na_from_ack(self):
directives, na_directives = sop.parse_directives(
"/sop-ack comprehensive-testing\n/sop-n/a qa-review N/A: no surface",
{},
)
self.assertEqual(len(directives), 1)
self.assertEqual(directives[0][0], "sop-ack")
self.assertEqual(len(na_directives), 1)
self.assertEqual(na_directives[0][0], "sop-n/a")
self.assertEqual(na_directives[0][1], "qa-review")
self.assertIn("no surface", na_directives[0][2])
+61 -1
View File
@@ -158,8 +158,68 @@ jobs:
echo "NOTE: No warning in output (may be suppressed by log level)"
fi
- name: Reproduce openclaw failure — pipe held OPEN, no EOF
run: |
set -euo pipefail
echo "=== keep-stdin-open pipe (the real openclaw / Claude Code case) ==="
echo ""
echo "Before the readline() fix this HANGS: main() did"
echo " stdin.read(65536) -> on a pipe, blocks until 64KB OR EOF."
echo "An MCP client sends one ~150B initialize and keeps stdin"
echo "open waiting for the response, so the server never parsed"
echo "the request and the client timed out (openclaw: 'MCP error"
echo "-32000: Connection closed'). The earlier regular-file /"
echo "heredoc-pipe steps PASSED through this bug because a file"
echo "(or a closing heredoc) yields EOF immediately."
echo ""
# Drive the server through a real pipe that stays OPEN: write
# one initialize, do NOT close stdin, and require a response
# within a hard timeout. read(65536) -> no output -> timeout
# kills it -> FAIL. readline() -> immediate response -> PASS.
python - <<'PYEOF'
import json, subprocess, sys, time, select
proc = subprocess.Popen(
[sys.executable, "a2a_mcp_server.py"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env={**__import__("os").environ},
)
req = json.dumps({
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2024-11-05",
"capabilities": {},
"clientInfo": {"name": "keepopen", "version": "1"}},
}) + "\n"
proc.stdin.write(req.encode())
proc.stdin.flush()
# Deliberately DO NOT close proc.stdin — mirror a live MCP client.
deadline = time.time() + 15
line = b""
while time.time() < deadline:
r, _, _ = select.select([proc.stdout], [], [], 1)
if r:
line = proc.stdout.readline()
if line:
break
proc.kill()
if not line:
print("FAIL: no response within 15s on an open pipe — "
"stdin.read(65536) regression is back")
sys.exit(1)
resp = json.loads(line.decode())
assert resp.get("id") == 1 and "result" in resp, \
f"unexpected response: {line[:200]!r}"
assert resp["result"]["serverInfo"]["name"] == "molecule", \
f"wrong serverInfo: {line[:200]!r}"
print("PASS: server answered initialize on a still-open pipe")
PYEOF
- name: Run unit tests for stdio transport
run: |
set -euo pipefail
echo "=== Running stdio transport unit tests ==="
python -m pytest tests/test_a2a_mcp_server.py::TestStdioPipeAssertion -v --no-cov
python -m pytest tests/test_a2a_mcp_server.py::TestStdioPipeAssertion tests/test_a2a_mcp_server.py::TestStdioKeepOpenPipe -v --no-cov
+1 -1
View File
@@ -57,7 +57,7 @@ permissions:
# can produce duplicate comments before the title-search dedup wins.
concurrency:
group: ci-required-drift
cancel-in-progress: true
cancel-in-progress: false
jobs:
drift:
+1 -1
View File
@@ -22,7 +22,7 @@ permissions:
concurrency:
group: gitea-merge-queue-${{ github.repository }}
cancel-in-progress: true
cancel-in-progress: false
jobs:
queue:
+1 -5
View File
@@ -56,13 +56,9 @@ permissions:
# Workflow-scoped serialisation — two simultaneous runs would race on the
# `[main-red] {SHA}` open/PATCH path. Idempotent by title, but parallel
# POSTs can produce duplicates before the title search dedup wins.
# NOTE: cancel-in-progress: true is safe here — the idempotent design means
# a cancelled run produces identical output to a completed one. This also
# prevents the Gitea scheduler freeze that occurs when a cron tick fires
# while a previous run is still executing (Quirk #8).
concurrency:
group: main-red-watchdog
cancel-in-progress: true
cancel-in-progress: false
jobs:
watchdog:
+7 -11
View File
@@ -49,17 +49,13 @@ jobs:
# bp-exempt: post-merge image publication side effect; CI / all-required gates source changes.
build-and-push:
name: Build & push canvas image
# Dedicated publish/release lane (internal#462 / #394 / #399). Ship
# path (on: push:main, canvas/**) — reserved capacity so a merged
# canvas fix's image build never FIFO-queues behind PR required-CI.
# The `publish` label resolves ONLY to the molecule-runner-publish-*
# sub-pool (config.publish.yaml). HARD DEPENDENCY: this MUST land
# AFTER the publish-lane runners are registered/advertising `publish`
# — the earlier #599 `docker` label attempt queued indefinitely with
# zero eligible runners precisely because the label was targeted
# before any runner advertised it (see #576). The lane is registered
# in this rollout (internal#462) so the precondition holds.
runs-on: publish
# REVERTED (infra/revert-docker-runner-label): `runs-on: ubuntu-latest` restored.
# The `docker` label is not registered on any act_runner. `runs-on: [ubuntu-latest, docker]`
# causes jobs to queue indefinitely with zero eligible runners — strictly worse than the
# pre-#599 coin-flip (50% success rate). Once the `docker` label is registered on
# ≥2 runners, re-apply the fix from #599 (infra/docker-runner-label).
# See issue #576 + infra-lead pulse ~00:30Z.
runs-on: ubuntu-latest
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
continue-on-error: true
+2 -8
View File
@@ -66,10 +66,7 @@ concurrency:
jobs:
publish:
# Dedicated publish/release lane (internal#462 / #394 / #399). Ship
# path (on: push tag runtime-v*) — reserved capacity, never FIFO
# behind PR-CI. `publish` resolves only to molecule-runner-publish-*.
runs-on: publish
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
wheel_sha256: ${{ steps.wheel_hash.outputs.wheel_sha256 }}
@@ -162,7 +159,6 @@ jobs:
exit 1
fi
python -m twine upload \
--verbose \
--repository pypi \
--username __token__ \
--password "$PYPI_TOKEN" \
@@ -170,9 +166,7 @@ jobs:
cascade:
needs: publish
# Publish/release lane (internal#462) — downstream of the runtime
# publish ship job; keep it on the reserved lane too.
runs-on: publish
runs-on: ubuntu-latest
steps:
- name: Wait for PyPI to propagate the new version
env:
@@ -54,14 +54,7 @@ env:
jobs:
build-and-push:
# Dedicated publish/release lane (internal#462 / #394 / #399). This
# is a post-merge ship job (on: push:main) — it must NOT FIFO-compete
# with PR required-CI on the shared pool (PR#1350's prod image build
# was delayed ~25min this way). The `publish` label resolves ONLY to
# the reserved molecule-runner-publish-* sub-pool (config.publish.yaml,
# OUTSIDE the managed 1..20 range) so a merged fix's image build
# starts immediately while PR-CI keeps the general pool.
runs-on: publish
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -188,9 +181,7 @@ jobs:
name: Production auto-deploy
needs: build-and-push
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
# Publish/release lane (internal#462) — production deploy of a merged
# fix; reserved capacity, never queued behind PR-CI.
runs-on: publish
runs-on: ubuntu-latest
timeout-minutes: 75
env:
CP_URL: ${{ vars.PROD_CP_URL || 'https://api.moleculesai.app' }}
@@ -68,10 +68,7 @@ jobs:
# bp-exempt: production redeploy is a side-effect workflow, not a merge gate.
redeploy:
if: ${{ github.event_name == 'workflow_dispatch' }}
# Dedicated publish/release lane (internal#462 / #394 / #399).
# Production tenant redeploy — a deploy action, reserved capacity so
# it never queues behind PR-CI. `publish` -> molecule-runner-publish-*.
runs-on: publish
runs-on: ubuntu-latest
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
continue-on-error: true
@@ -75,10 +75,7 @@ env:
jobs:
# bp-exempt: post-merge staging redeploy side effect; CI / all-required gates source changes.
redeploy:
# Dedicated publish/release lane (internal#462 / #394 / #399).
# Post-merge staging redeploy — a deploy action, reserved capacity.
# `publish` -> molecule-runner-publish-* sub-pool.
runs-on: publish
runs-on: ubuntu-latest
# Phase 3 (RFC #219 §1): surface broken workflows without blocking.
# mc#774: pre-existing continue-on-error mask; root-fix and remove, do not renew silently.
continue-on-error: true
+4 -1
View File
@@ -30,7 +30,10 @@
{"name": "openclaw", "repo": "molecule-ai/molecule-ai-workspace-template-openclaw", "ref": "main"},
{"name": "codex", "repo": "molecule-ai/molecule-ai-workspace-template-codex", "ref": "main"},
{"name": "langgraph", "repo": "molecule-ai/molecule-ai-workspace-template-langgraph", "ref": "main"},
{"name": "autogen", "repo": "molecule-ai/molecule-ai-workspace-template-autogen", "ref": "main"}
{"name": "crewai", "repo": "molecule-ai/molecule-ai-workspace-template-crewai", "ref": "main"},
{"name": "autogen", "repo": "molecule-ai/molecule-ai-workspace-template-autogen", "ref": "main"},
{"name": "deepagents", "repo": "molecule-ai/molecule-ai-workspace-template-deepagents", "ref": "main"},
{"name": "gemini-cli", "repo": "molecule-ai/molecule-ai-workspace-template-gemini-cli", "ref": "main"}
],
"org_templates": [
{"name": "molecule-dev", "repo": "molecule-ai/molecule-ai-org-template-molecule-dev", "ref": "main"},
-25
View File
@@ -77,31 +77,6 @@ does not replace the queue. The queue still performs its own current-main
check immediately before merge because branch protection alone cannot
serialize two already-green PRs.
### Correct API field names (Gitea 1.22.6)
When setting branch protection via API, use these exact field names — several
intuitively-correct names are silently ignored (see `gitea-operational-quirks.md`
Quirk #7):
```json
{
"branch_name": "main",
"enable_merge_whitelist": true,
"merge_whitelist_usernames": ["devops-engineer", "hongming", "core-devops"],
"enable_status_check": true,
"status_check_contexts": ["CI / all-required"],
"required_approvals": 1,
"block_on_rejected_reviews": true
}
```
After any `POST /branch_protections`, immediately GET and verify the values
persisted — the API returns 201 even when fields are silently dropped.
If the queue returns HTTP 405 ("User not allowed to merge"), the first
diagnostic step is `GET /branch_protections/main` and checking whether
`merge_whitelist_usernames` still contains `devops-engineer`.
## Failure Handling
If `main` is not green, the queue pauses and does not merge anything.
+21 -86
View File
@@ -196,134 +196,69 @@ primary consumer of combined status and is affected.
---
## Quirk #7 — Gitea branch protection API silently ignores some field names
## Quirk #7 — TBD
*[Placeholder — document here when a new Gitea Actions quirk is discovered.]*
### Finding
The Gitea 1.22.6 `POST /repos/{org}/{repo}/branch_protections` API accepts a
non-obvious set of field names. Several intuitively-correct names are silently
ignored — the call returns 201 but the field is dropped:
| Intended field | Correct API name | Silently ignored aliases |
|---|---|---|
| Enable merge whitelist | `enable_merge_whitelist` | `user_can_merge`, `merge_whitelist_enabled` |
| Users who can merge | `merge_whitelist_usernames` | `merge_whitelist_users`, `whitelisted_users` |
| Enable status check | `enable_status_check` | `enable_status_checks`, `require_status_checks` |
| Required status contexts | `status_check_contexts` | `required_status_checks.contexts` |
| Block on rejected reviews | `block_on_rejected_reviews` | (this one works) |
| Required approvals | `required_approvals` | `required_reviewers` |
The GET response after a POST shows the actual stored values. A naive
GET → modify → POST cycle (without using the exact GET field names) will
silently reset the merge whitelist on every call.
*[What Gitea Actions does differently from GitHub Actions.]*
### Impact
- Branch protection merge whitelist resets to empty after any API mis-invocation
- Queue AUTO_SYNC_TOKEN (`devops-engineer`) loses Can-merge permission → HTTP 405
- All queued PRs blocked until whitelist is restored
- Confirmed reset on Gitea server restart/upgrade (Gitea uses default values)
*[Which workflows or operations are affected.]*
### Workaround
1. Always GET the current protection first and use **exact** field names from the
GET response when modifying
2. After any `POST /branch_protections`, immediately GET and verify
`enable_merge_whitelist: true` and `merge_whitelist_usernames` contains
`["devops-engineer", "hongming", "core-devops"]`
3. The queue bot should verify branch protection before each merge tick
4. For queue to work: `enable_merge_whitelist: true` +
`merge_whitelist_usernames: ["devops-engineer", "hongming", "core-devops"]` +
`enable_status_check: true` + `status_check_contexts: ["CI / all-required"]`
*[How to work around this quirk.]*
### References
- SEV-1 2026-05-17: 3x branch protection resets caused 405 on all queue merges
- `feedback_gitea_branch_protection_api_field_names`
- internal#[N]: first observation
---
## Quirk #8 — Scheduled workflow with `cancel-in-progress: false` causes scheduler freeze
## Quirk #8 — TBD
*[Placeholder — document here when a new Gitea Actions quirk is discovered.]*
### Finding
When a `schedule:` workflow has `concurrency.cancel-in-progress: false`, and a
new cron tick fires while the previous run is still executing, the Gitea Actions
scheduler stops dispatching the workflow entirely. Pending entries accumulate
indefinitely — the scheduler shows the workflow as "scheduled" but never dispatches.
This is dangerous for workflows with variable execution time (e.g., workflows that
wait for downstream CI, or workflows that run on slow/degraded runners).
*[What Gitea Actions does differently from GitHub Actions.]*
### Impact
- `gitea-merge-queue.yml` with `cancel-in-progress: false` froze on 2026-05-17
starting ~16:44Z — pending runs accumulated, no new runs dispatched
- Queue appeared stalled; all 22 queued PRs blocked
- The `gitea-merge-queue` workflow itself becomes invisible to operators
*[Which workflows or operations are affected.]*
### Workaround
**Always set `cancel-in-progress: true` on `schedule:` workflows:**
```yaml
concurrency:
group: workflow-name
cancel-in-progress: true # ← always true for schedule: workflows
```
If the freeze has already occurred: the scheduler recovers automatically after the
currently-running instance completes (Gitea dispatches the next queued tick).
*[How to work around this quirk.]*
### References
- SEV-1 2026-05-17: queue frozen since 16:44Z; fixed by setting `cancel-in-progress: true`
- PR #1358: `fix(scheduled-workflows): enable cancel-in-progress` (pending merge)
- internal#[N]: first observation
---
## Quirk #9 — Gitea Actions runner accepts runs but stalls (jobs never start)
## Quirk #9 — TBD
*[Placeholder — document here when a new Gitea Actions quirk is discovered.]*
### Finding
The Gitea Actions runner on host `5.78.80.188` can enter a degraded state where:
1. It accepts new workflow runs (shows "in_progress" in the UI)
2. It never starts any jobs — pending count grows indefinitely
3. The runner shows as "online" and accepting runs
4. After ~6090 minutes, the runner self-recovers and all pending jobs start
This is distinct from a true runner crash (which would show as offline).
*[What Gitea Actions does differently from GitHub Actions.]*
### Impact
- All CI jobs for all PRs stall — no status updates posted
- Queue waits indefinitely for CI (which never posts success)
- `sop-checklist` and other workflows time out on affected PRs
- Looks like the runner is working (green in UI) but nothing executes
### How to diagnose
Add a debug step to a known-failing workflow:
```bash
# In a stalled job:
curl -s http://localhost:8088/debug/pprof/trace?seconds=5 | head
# Check runner process CPU — if near 0% while jobs are pending, runner is stalled
```
Check runner logs on the host (`/var/log/actrunner.log` or similar).
*[Which workflows or operations are affected.]*
### Workaround
No operator workaround while stalled — the runner self-recovers. Options:
1. **Wait** — runner typically recovers within 90 minutes
2. **Restart the runner service**`systemctl restart act_runner` (requires host access)
3. **Move to a second runner** — if registered, re-route dispatch
*[How to work around this quirk.]*
### References
- SEV-1 2026-05-17: runner stalled; self-recovered ~21:33Z after ~90 min
- `feedback_gitea_runner_stall_accepted_jobs_no_execution`
- internal#[N]: first observation
---
@@ -44,8 +44,8 @@ func NewWorkspaceImageService(docker *dockerclient.Client) *WorkspaceImageServic
// AllRuntimes is the canonical list mirroring docs/workspace-runtime-package.md.
// Update both when a new template is added.
var AllRuntimes = []string{
"claude-code", "langgraph", "autogen",
"hermes", "openclaw",
"claude-code", "langgraph", "crewai", "autogen",
"deepagents", "hermes", "gemini-cli", "openclaw",
}
// RefreshResult is the per-call outcome surfaced to HTTP callers AND logged
@@ -23,8 +23,8 @@ package models
// - claude-code: "sonnet" — Anthropic's CLI accepts the short
// name and resolves it via the operator's anthropic-oauth or
// ANTHROPIC_API_KEY chain.
// - everything else (hermes, langgraph, autogen, codex, openclaw,
// external, ""): a fully-qualified
// - everything else (hermes, langgraph, crewai, autogen, deepagents,
// codex, openclaw, gemini-cli, external, ""): a fully-qualified
// vendor:model slug that the universal MODEL_PROVIDER chain in
// molecule-core PR #247 can route via per-vendor required_env.
//
@@ -21,9 +21,12 @@ func TestDefaultModel(t *testing.T) {
// as a generic "unknown" failure.
{"hermes", "anthropic:claude-opus-4-7"},
{"langgraph", "anthropic:claude-opus-4-7"},
{"crewai", "anthropic:claude-opus-4-7"},
{"autogen", "anthropic:claude-opus-4-7"},
{"deepagents", "anthropic:claude-opus-4-7"},
{"codex", "anthropic:claude-opus-4-7"},
{"openclaw", "anthropic:claude-opus-4-7"},
{"gemini-cli", "anthropic:claude-opus-4-7"},
{"external", "anthropic:claude-opus-4-7"},
// Unknown / empty — fall through to universal default rather
@@ -190,7 +190,7 @@ func TestEnsureLocalImage_RepoNotFound(t *testing.T) {
opts.HTTPClient = srv.Client()
opts.remoteHeadSha = nil // exercise real HTTP path
_, err := ensureLocalImageWithOpts(context.Background(), "hermes", opts)
_, err := ensureLocalImageWithOpts(context.Background(), "crewai", opts)
if err == nil {
t.Fatalf("expected error, got nil")
}
@@ -35,19 +35,6 @@ import (
// drift-risk #6.
var ErrNoBackend = errors.New("provisioner: no backend configured (zero-valued receiver)")
// ErrUnresolvableRuntime is returned by selectImage when a workspace
// names a runtime that has no resolvable image (not in RuntimeImages and
// no operator-pinned cfg.Image). RFC internal#483 + security review 4269:
// previously such a request silently fell through to DefaultImage
// (langgraph) — a user asking for crewai would get a langgraph container
// with no signal. The CTO standing directive
// (feedback_platform_must_hardgate_base_contract) is fail-closed: a
// named-but-unresolvable runtime must reject with a structured,
// runtime-naming error so the existing provision-failed notify/log path
// surfaces it, NOT silently degrade. The genuinely-unspecified (empty)
// runtime is still a distinct, legitimate path that keeps DefaultImage.
var ErrUnresolvableRuntime = errors.New("provisioner: requested runtime has no resolvable image")
// RuntimeImages maps runtime names to their Docker image refs.
// Each standalone template repo publishes its image via the reusable
// publish-template-image workflow in molecule-ci on every main merge.
@@ -117,33 +104,20 @@ type WorkspaceConfig struct {
// selectImage resolves the final Docker image ref for a workspace. The handler
// layer is the source of truth — if it set cfg.Image (the digest-pinned form
// from runtime_image_pins, #2272), honor that. Otherwise fall back to the
// runtime→tag lookup in RuntimeImages (legacy `:latest` behavior).
//
// Fail-closed contract (RFC internal#483 / security review 4269 /
// feedback_platform_must_hardgate_base_contract): if the workspace NAMES a
// runtime that resolves to no image (not in RuntimeImages, no pinned
// cfg.Image), reject with ErrUnresolvableRuntime instead of silently
// substituting DefaultImage. Pre-fix, removing crewai/deepagents/gemini-cli
// from the catalog left those create requests silently provisioning a
// langgraph container — the user asked for crewai and got langgraph with no
// signal. The error propagates through Start → markProvisionFailed, which
// already broadcasts WorkspaceProvisionFailed and records the message.
//
// The genuinely-unspecified runtime (empty cfg.Runtime, e.g. an org template
// that doesn't pin one) is an intended distinct path and still resolves to
// DefaultImage — only a NAMED-but-unresolvable runtime is rejected.
func selectImage(cfg WorkspaceConfig) (string, error) {
// runtime→tag lookup in RuntimeImages (legacy `:latest` behavior). When the
// runtime isn't recognized either, fall back to DefaultImage so Start() still
// has something to hand Docker — surfacing a "No such image" later is more
// actionable than a silent "" panic in ContainerCreate.
func selectImage(cfg WorkspaceConfig) string {
if cfg.Image != "" {
return cfg.Image, nil
return cfg.Image
}
if cfg.Runtime != "" {
if img, ok := RuntimeImages[cfg.Runtime]; ok {
return img, nil
return img
}
return "", fmt.Errorf("%w: runtime %q (known runtimes: %v)",
ErrUnresolvableRuntime, cfg.Runtime, knownRuntimes)
}
return DefaultImage, nil
return DefaultImage
}
// Workspace-access constants for #65. Matches the CHECK constraint on
@@ -362,15 +336,7 @@ func (p *Provisioner) Start(ctx context.Context, cfg WorkspaceConfig) (string, e
env := buildContainerEnv(cfg)
image, imgErr := selectImage(cfg)
if imgErr != nil {
// Fail-closed: a named-but-unresolvable runtime must not silently
// become DefaultImage (RFC internal#483 / review 4269). The caller's
// error path (markProvisionFailed) broadcasts the failure + records
// the message so the canvas surfaces it.
log.Printf("Provisioner: refusing to start %s: %v", cfg.WorkspaceID, imgErr)
return "", imgErr
}
image := selectImage(cfg)
// Local-build mode (issue #63 / Task #194): when MOLECULE_IMAGE_REGISTRY
// is unset, the OSS contributor path skips the registry pull entirely
@@ -513,10 +513,7 @@ func TestWorkspaceConfig_ResetClaudeSessionFieldPresent(t *testing.T) {
// we lose the "one bad publish doesn't break every workspace" guarantee.
func TestSelectImage_PrefersExplicitImage(t *testing.T) {
pinned := "ghcr.io/molecule-ai/workspace-template-claude-code@sha256:3d6761a97ed07d7d33cfc19a8fbab81175d9d9179618d493dbc00c5f7ef076a3"
got, err := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: pinned})
if err != nil {
t.Fatalf("selectImage with cfg.Image=pinned: unexpected error %v", err)
}
got := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: pinned})
if got != pinned {
t.Errorf("selectImage with cfg.Image=pinned: got %q, want %q", got, pinned)
}
@@ -526,46 +523,28 @@ func TestSelectImage_PrefersExplicitImage(t *testing.T) {
// pin lookup deliberately bypassed via WORKSPACE_IMAGE_LOCAL_OVERRIDE).
// selectImage must use the legacy runtime→:latest map.
func TestSelectImage_FallsBackToRuntimeMap(t *testing.T) {
got, err := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: ""})
if err != nil {
t.Fatalf("selectImage with empty Image: unexpected error %v", err)
}
got := selectImage(WorkspaceConfig{Runtime: "claude-code", Image: ""})
want := RuntimeImages["claude-code"]
if got != want {
t.Errorf("selectImage with empty Image: got %q, want %q", got, want)
}
}
// TestSelectImage_NamedUnresolvableRuntimeRejects pins the fail-closed
// contract (RFC internal#483 / security review 4269 /
// feedback_platform_must_hardgate_base_contract): a NAMED runtime with no
// resolvable image must reject with ErrUnresolvableRuntime, NOT silently
// substitute DefaultImage. Pre-fix this returned langgraph — a user asking
// for a removed runtime (crewai/deepagents/gemini-cli) silently got a
// langgraph container. "crewai" is the concrete regression from the
// security finding.
func TestSelectImage_NamedUnresolvableRuntimeRejects(t *testing.T) {
for _, rt := range []string{"no-such-runtime", "crewai", "deepagents", "gemini-cli"} {
got, err := selectImage(WorkspaceConfig{Runtime: rt})
if !errors.Is(err, ErrUnresolvableRuntime) {
t.Errorf("selectImage(%q): got err %v, want ErrUnresolvableRuntime", rt, err)
}
if got != "" {
t.Errorf("selectImage(%q): got image %q, want \"\" on reject", rt, got)
}
if err != nil && !strings.Contains(err.Error(), rt) {
t.Errorf("selectImage(%q): error must name the offending runtime, got %v", rt, err)
}
// TestSelectImage_UnknownRuntimeFallsBackToDefault preserves today's
// behavior — an unrecognized runtime resolves to DefaultImage rather than
// "" so ContainerCreate gets a usable arg and surfaces a meaningful
// "No such image" error if the default itself is missing.
func TestSelectImage_UnknownRuntimeFallsBackToDefault(t *testing.T) {
got := selectImage(WorkspaceConfig{Runtime: "no-such-runtime"})
if got != DefaultImage {
t.Errorf("selectImage with unknown runtime: got %q, want DefaultImage %q", got, DefaultImage)
}
}
// TestSelectImage_EmptyRuntimeFallsBackToDefault: same invariant for the
// no-runtime-supplied path (legacy callers / older handler code).
func TestSelectImage_EmptyRuntimeFallsBackToDefault(t *testing.T) {
got, err := selectImage(WorkspaceConfig{})
if err != nil {
t.Fatalf("selectImage with zero cfg: unexpected error %v (empty runtime is a legitimate DefaultImage path)", err)
}
got := selectImage(WorkspaceConfig{})
if got != DefaultImage {
t.Errorf("selectImage with zero cfg: got %q, want DefaultImage %q", got, DefaultImage)
}
@@ -829,7 +808,7 @@ func TestIsImageNotFoundErr(t *testing.T) {
{"nil", nil, false},
{"moby no such image", fmtErr(`Error response from daemon: No such image: workspace-template:openclaw`), true},
{"no such image lowercase", fmtErr(`error: no such image: foo:bar`), true},
{"image not found", fmtErr(`Error: image "workspace-template:hermes" not found`), true},
{"image not found", fmtErr(`Error: image "workspace-template:crewai" not found`), true},
{"generic not found without image", fmtErr(`container not found`), false},
{"unrelated error", fmtErr(`connection refused`), false},
{"permission denied", fmtErr(`permission denied`), false},
@@ -21,6 +21,9 @@ var knownRuntimes = []string{
"autogen",
"claude-code",
"codex",
"crewai",
"deepagents",
"gemini-cli",
"hermes",
"langgraph",
"openclaw",
@@ -53,8 +53,8 @@ func TestRuntimeImage_AllKnownRuntimes(t *testing.T) {
}
}
// Pin the count so adding a runtime requires explicit test acknowledgement.
if len(knownRuntimes) != 6 {
t.Errorf("knownRuntimes length = %d, want 6 (autogen, claude-code, codex, hermes, langgraph, openclaw)", len(knownRuntimes))
if len(knownRuntimes) != 9 {
t.Errorf("knownRuntimes length = %d, want 9 (autogen, claude-code, codex, crewai, deepagents, gemini-cli, hermes, langgraph, openclaw)", len(knownRuntimes))
}
}
+17 -1
View File
@@ -776,7 +776,23 @@ async def main(): # pragma: no cover
buffer = b""
while True:
try:
chunk = await loop.run_in_executor(None, stdin.read, 65536)
# MUST be readline(), NOT read(65536). MCP is a line-delimited
# JSON-RPC stream where the client (openclaw bundle-mcp,
# Claude Code, Cursor, ...) sends one small (~150B) request
# and keeps stdin OPEN waiting for the response. A fixed-size
# `stdin.read(65536)` on a PIPE blocks until either 64KB
# accumulate OR EOF — neither happens during a normal MCP
# handshake — so the server never parses `initialize` and the
# client times out (~30s; openclaw: "MCP error -32000:
# Connection closed"). This made the stdio transport unusable
# for every pipe-spawned MCP host while passing tests/manual
# checks that fed stdin from a regular FILE (where read()
# returns immediately at the short file's end). readline()
# returns as soon as one newline-terminated line is available,
# which is exactly the JSON-RPC framing. Diagnosed 2026-05-15
# against a live openclaw workspace; see
# molecule-ai-workspace-runtime#61 (same fd-compat lineage).
chunk = await loop.run_in_executor(None, stdin.readline)
if not chunk:
break
buffer += chunk
-47
View File
@@ -431,43 +431,6 @@ def _is_self_notify_row(row: dict[str, Any]) -> bool:
return source_id is None or source_id == ""
def _is_self_echo_row(row: dict[str, Any], workspace_id: str) -> bool:
"""Return True if ``row`` is a self-originated a2a_receive row.
Internal #469: when a workspace delegates to a target that never picks
up the task, ``tool_delegate_task`` calls ``report_activity`` which
POSTs to the platform with source_id set to the *sender's* workspace
UUID (mandated by spoof-defense in workspace-server's a2a_proxy). The
activity API exposes that row under type=a2a_receive, so the inbox
poller re-fetches it. Without this guard the row is surfaced as
kind='peer_agent' with the workspace's own identity as peer_id —
the workspace sees its own delegation-failure echoed back as if a
peer had delegated to it.
The guard mirrors the existing _is_self_notify_row pattern: both
skip rows that would otherwise create spurious inbound signal. The
long-term fix (making the platform write a distinct activity_type
for agent-outbound rows) is tracked separately; this guard stays
because it only excludes rows the agent never wants.
``workspace_id`` must be non-empty — an empty-string workspace_id
(single-workspace legacy path) can never match a UUID source_id, so
the predicate is always False there, which is safe.
RFC #2829 PR-2 note: rows with method="delegate_result" are excluded
from the self-echo guard even when source_id matches our workspace_id.
The platform may write a delegation-result row with source_id set to
our workspace_id (e.g. a self-delegation or edge case in the platform's
result-writing path). Such rows must reach the inbox so that
message_from_activity can surface them as peer_agent inbound and the
runtime receives the delegation result. Silently filtering them as
self-echo would break delegation result delivery.
"""
if not workspace_id:
return False
return row.get("source_id") == workspace_id and row.get("method") != "delegate_result"
def message_from_activity(row: dict[str, Any]) -> InboxMessage:
"""Convert one /activity row into an InboxMessage.
@@ -660,16 +623,6 @@ def _poll_once(
# the same self-notify on every iteration.
last_id = str(row.get("id", "")) or last_id
continue
if _is_self_echo_row(row, workspace_id):
# Internal #469: tool_delegate_task writes its own a2a_receive
# row with source_id = this workspace's UUID (spoof-defense).
# The poll fetches it back as kind='peer_agent', making the
# workspace echo its own delegation-failure as an inbound from
# a phantom peer. Skip it — the real delegation-result path
# (delegate_result push) is separate and unaffected. Cursor
# still advances so the next poll doesn't re-seen this row.
last_id = str(row.get("id", "")) or last_id
continue
message = message_from_activity(row)
if not message.activity_id:
continue
+121
View File
@@ -2097,3 +2097,124 @@ def test_peer_metadata_set_replaces_existing_entry_in_place(_reset_peer_metadata
)
cached = a2a_client._peer_metadata[peer]
assert cached[1]["name"] == "v2", "re-write must update the value in place"
class TestStdioKeepOpenPipe:
"""Regression for the openclaw peer-visibility outage (2026-05-15).
main()'s read loop used `await loop.run_in_executor(None,
stdin.read, 65536)`. On a PIPE, `read(n)` blocks until n bytes
accumulate OR EOF. A real MCP client (openclaw bundle-mcp, Claude
Code, Cursor) sends ONE ~150-byte newline-delimited request and
keeps stdin OPEN waiting for the reply — so neither condition is
met, the server never parses `initialize`, and the client times
out (~30s; openclaw surfaced "MCP error -32000: Connection
closed"). Every prior stdio test fed stdin from a regular file or
a heredoc-pipe that CLOSES (EOF), masking the bug.
These spawn the real a2a_mcp_server.py process, write one request
over a pipe, and DELIBERATELY keep stdin open. With the buggy
read(65536) the assertion times out and fails; with readline() it
passes promptly. This is the literal user-facing path, not a
mock — see feedback_smoke_test_vendor_truth_not_shape_match.
"""
def _spawn(self):
import subprocess
env = dict(os.environ)
env.setdefault("WORKSPACE_ID", "00000000-0000-0000-0000-000000000001")
server = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"a2a_mcp_server.py",
)
return subprocess.Popen(
["python3", server],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
)
def _read_line_with_deadline(self, proc, deadline_s=15):
import select
import time
end = time.time() + deadline_s
while time.time() < end:
r, _, _ = select.select([proc.stdout], [], [], 1)
if r:
line = proc.stdout.readline()
if line:
return line
return b""
def test_initialize_answered_on_still_open_pipe(self):
"""One initialize, stdin kept OPEN, response required <15s.
FAILS (times out -> empty line) on stdin.read(65536).
PASSES on stdin.readline().
"""
proc = self._spawn()
try:
req = json.dumps({
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {
"protocolVersion": "2024-11-05",
"capabilities": {},
"clientInfo": {"name": "keepopen", "version": "1"},
},
}) + "\n"
proc.stdin.write(req.encode())
proc.stdin.flush()
# NOTE: stdin is intentionally NOT closed — mirrors a live
# MCP client. Closing it here would yield EOF and let the
# buggy read(65536) return, hiding the regression.
line = self._read_line_with_deadline(proc, 15)
finally:
proc.kill()
proc.wait(timeout=5)
assert line, (
"no response within 15s on a still-open pipe — the "
"stdin.read(65536) pipe-blocking regression is back "
"(this is the exact openclaw peer-visibility outage)"
)
resp = json.loads(line.decode())
assert resp.get("id") == 1, f"unexpected id: {line[:200]!r}"
assert "result" in resp, f"no result envelope: {line[:200]!r}"
assert resp["result"]["serverInfo"]["name"] == "molecule", (
f"wrong serverInfo: {line[:200]!r}"
)
def test_two_sequential_requests_on_open_pipe(self):
"""initialize THEN tools/list on the same open pipe — proves
the loop keeps reading line-by-line, not just the first 64KB
chunk. tools/list must include list_peers (the peer-visibility
tool the outage was about)."""
proc = self._spawn()
try:
proc.stdin.write((json.dumps({
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2024-11-05",
"capabilities": {},
"clientInfo": {"name": "x", "version": "1"}},
}) + "\n").encode())
proc.stdin.flush()
init = self._read_line_with_deadline(proc, 15)
assert init, "initialize unanswered on open pipe"
proc.stdin.write((json.dumps({
"jsonrpc": "2.0", "id": 2, "method": "tools/list",
}) + "\n").encode())
proc.stdin.flush()
tl = self._read_line_with_deadline(proc, 15)
finally:
proc.kill()
proc.wait(timeout=5)
assert tl, "tools/list unanswered — loop stopped after one read"
resp = json.loads(tl.decode())
names = {t["name"] for t in resp["result"]["tools"]}
assert "list_peers" in names, (
f"list_peers missing from tools/list: {sorted(names)}"
)
-145
View File
@@ -495,151 +495,6 @@ def test_poll_once_skips_self_notify_rows(state: inbox.InboxState):
assert [m.activity_id for m in queue] == ["act-real"]
# ---------------------------------------------------------------------------
# _is_self_echo_row — internal #469 fix
# ---------------------------------------------------------------------------
#
# When a workspace delegates to a target that never picks up the task,
# tool_delegate_task calls report_activity("a2a_receive", ...) which POSTs
# to the platform with source_id set to the *sender's* workspace UUID
# (spoof-defense). The activity API returns that row under type=a2a_receive
# on the next poll, so message_from_activity sets peer_id = workspace's own
# UUID — the workspace sees its own delegation-failure as an inbound from
# a phantom peer. _is_self_echo_row guards against this.
#
# Internal #469 was live-reproduced on hongming.moleculesai.app 2026-05-16.
def test_is_self_echo_row_true_when_source_id_matches_workspace():
row = {"source_id": "ws-abc123", "method": "a2a_receive"}
assert inbox._is_self_echo_row(row, "ws-abc123") is True
def test_is_self_echo_row_false_when_source_id_differs():
"""A real peer agent (different workspace_id) must NOT be filtered."""
row = {"source_id": "ws-peer", "method": "a2a_receive"}
assert inbox._is_self_echo_row(row, "ws-1") is False
def test_is_self_echo_row_false_when_source_id_is_none():
"""Canvas-user inbound has no source_id — never an echo."""
row = {"source_id": None, "method": "a2a_receive"}
assert inbox._is_self_echo_row(row, "ws-1") is False
def test_is_self_echo_row_false_when_workspace_id_is_empty():
"""Single-workspace legacy path with empty workspace_id cannot
match a UUID source_id — predicate is always False, which is safe."""
row = {"source_id": "ws-abc123", "method": "a2a_receive"}
assert inbox._is_self_echo_row(row, "") is False
def test_is_self_echo_row_false_when_source_id_key_absent():
row = {"method": "a2a_receive"}
assert inbox._is_self_echo_row(row, "ws-1") is False
def test_is_self_echo_row_false_for_delegate_result():
"""RFC #2829 PR-2 regression pin: a row with source_id matching our
workspace_id but method=delegate_result must NOT be filtered as a
self-echo. The platform may write a delegation-result row with our
workspace_id as source_id; such rows must reach the inbox so the
runtime receives the delegation result. Silently filtering them would
break delegate_result delivery."""
row = {"source_id": "ws-1", "method": "delegate_result"}
assert inbox._is_self_echo_row(row, "ws-1") is False
def test_poll_once_skips_self_echo_rows(state: inbox.InboxState):
"""Internal #469 regression pin: a row with source_id matching our
workspace_id must NOT land in the inbox queue — it is our own
delegation-report echoing back, not a real peer inbound."""
rows = [
{
"id": "act-real-peer",
"source_id": "ws-peer",
"method": "a2a_receive",
"summary": None,
"request_body": {"parts": [{"type": "text", "text": "real peer inbound"}]},
"created_at": "2026-04-30T22:00:00Z",
},
{
"id": "act-self-echo",
"source_id": "ws-1",
"method": "a2a_receive",
"summary": "task result: target timed out",
"request_body": None,
"created_at": "2026-04-30T22:00:01Z",
},
]
resp = _make_response(200, rows)
p, _ = _patch_httpx(resp)
with p:
n = inbox._poll_once(state, "http://platform", "ws-1", {})
# Only the real peer inbound counted; self-echo silently dropped.
assert n == 1
queue = state.peek(10)
assert [m.activity_id for m in queue] == ["act-real-peer"]
assert queue[0].peer_id == "ws-peer"
def test_poll_once_advances_cursor_past_self_echo(state: inbox.InboxState):
"""Cursor must advance past self-echo rows even though we don't
enqueue them. Otherwise the next poll re-fetches the same self-echo
on every iteration, wasting requests and blocking real inbound."""
state.save_cursor("act-old")
rows = [
{
"id": "act-self-echo",
"source_id": "ws-1",
"method": "a2a_receive",
"summary": "task result: timeout",
"request_body": None,
"created_at": "2026-04-30T22:00:00Z",
},
]
resp = _make_response(200, rows)
p, _ = _patch_httpx(resp)
with p:
n = inbox._poll_once(state, "http://platform", "ws-1", {})
assert n == 0
assert state.peek(10) == []
# Cursor must move past the skipped row so we don't re-poll it.
assert state.load_cursor() == "act-self-echo"
def test_poll_once_self_echo_does_not_fire_notification(state: inbox.InboxState):
"""The notification callback (channel push to Claude Code etc.)
must not fire for self-echo rows. Same rationale as self-notify:
push-capable hosts would see the echo loop on the push channel."""
rows = [
{
"id": "act-self-echo",
"source_id": "ws-1",
"method": "a2a_receive",
"summary": "task result: timeout",
"request_body": None,
"created_at": "2026-04-30T22:00:00Z",
},
]
received: list[dict] = []
inbox.set_notification_callback(received.append)
try:
resp = _make_response(200, rows)
p, _ = _patch_httpx(resp)
with p:
inbox._poll_once(state, "http://platform", "ws-1", {})
finally:
inbox.set_notification_callback(None)
assert received == [], (
"self-echo rows must not surface as MCP notifications — "
"doing so re-creates the echo loop on push-capable hosts"
)
def test_poll_once_advances_cursor_past_self_notify(state: inbox.InboxState):
"""Cursor must advance past self-notify rows even though we don't
enqueue them. Otherwise the next poll re-fetches the same self-