fix(ci): split publish-runtime into tags-only + autobump (closes #351) (#352)

publish-runtime.yml has never fired since the .gitea port (0 rows in
action_run.workflow_id='publish-runtime.yml' ever), which is why PyPI
is still at 0.1.129 despite Gitea having a runtime-v1.0.0 tag.

Root cause hypothesis: Gitea Actions evaluates the on.push.paths filter
against tag-push events too (no path diff → workflow skipped). PR #349
made this visible by adding the paths trigger, but the same defect
existed for the originally-ported tags-only trigger on this Gitea version
— hence the runtime-v1.0.0 tag also never published.

Fix: split into two files, each with a single unambiguous trigger shape.

  - publish-runtime.yml          : on.push.tags only       (the publisher)
  - publish-runtime-autobump.yml : on.push.branches+paths  (NEW; the bumper)

The autobump file computes next version from PyPI latest, pushes
'runtime-v$VERSION' tag via DISPATCH_TOKEN (not GITHUB_TOKEN — needed
to trigger downstream workflows on Gitea), and exits. The tag push
then triggers publish-runtime.yml.

Test plan after merge:
  1. Push no-op commit to workspace/. Observe autobump fire, push tag.
  2. Observe publish-runtime.yml fire on the tag, publish 0.1.130 to
     PyPI, cascade to template repos.
  3. Verify 'action_run' shows >0 rows for both workflow_ids.

.gitea/workflows/publish-runtime-autobump.yml

@@ -0,0 +1,100 @@
+name: publish-runtime-autobump
+
+# Auto-bump-on-workspace-edit half of the publish pipeline.
+#
+# Why this file exists (issue #351):
+#   Gitea Actions does not correctly disambiguate `paths:` from `tags:`
+#   when both are bundled under a single `on.push` key. The result is
+#   that tag pushes get filtered out and `publish-runtime.yml` never
+#   fires — `action_run` rows: 0. This was unnoticed pre-2026-05-11
+#   because PYPI_TOKEN was absent (publishes would have failed anyway).
+#
+#   Split design:
+#     - publish-runtime.yml         : on.push.tags only        (the publisher)
+#     - publish-runtime-autobump.yml: on.push.branches+paths   (this file — the version-bumper)
+#
+#   This file computes the next version from PyPI's latest, pushes a
+#   `runtime-v$VERSION` tag, and exits. The tag push then triggers
+#   publish-runtime.yml via its tags-only trigger.
+#
+# Concurrency: shares the `publish-runtime` group with publish-runtime.yml
+# so concurrent workspace pushes serialize at the bump step. Without
+# this, two pushes minutes apart could both read PyPI latest=0.1.129
+# and try to tag 0.1.130 simultaneously, only one of which would land.
+
+on:
+  push:
+    branches:
+      - main
+      - staging
+    paths:
+      - "workspace/**"
+
+permissions:
+  contents: write  # required to push tags back
+
+concurrency:
+  group: publish-runtime
+  cancel-in-progress: false
+
+jobs:
+  autobump-and-tag:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # Fetch full tag list so the bump logic can sanity-check against
+          # what's already in this repo (catches collision with prior
+          # manual tag pushes).
+          fetch-depth: 0
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.11"
+
+      - name: Compute next version from PyPI latest
+        id: bump
+        run: |
+          set -eu
+          LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
+            | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
+          MAJOR=$(echo "$LATEST" | cut -d. -f1)
+          MINOR=$(echo "$LATEST" | cut -d. -f2)
+          PATCH=$(echo "$LATEST" | cut -d. -f3)
+          VERSION="${MAJOR}.${MINOR}.$((PATCH+1))"
+          echo "PyPI latest=$LATEST -> next=$VERSION"
+          if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
+            echo "::error::computed version $VERSION does not match PEP 440 X.Y.Z"
+            exit 1
+          fi
+          if git tag --list | grep -qx "runtime-v$VERSION"; then
+            echo "::error::tag runtime-v$VERSION already exists in this repo. Manual intervention required (PyPI and Gitea tag history are out of sync)."
+            exit 1
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+
+      - name: Push runtime-v$VERSION tag
+        env:
+          DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+          VERSION: ${{ steps.bump.outputs.version }}
+          GITEA_URL: https://git.moleculesai.app
+        run: |
+          set -eu
+          if [ -z "$DISPATCH_TOKEN" ]; then
+            echo "::error::DISPATCH_TOKEN secret is not set — needed to push the tag back to molecule-core."
+            exit 1
+          fi
+          git config user.name  "publish-runtime autobump"
+          git config user.email "publish-runtime@moleculesai.app"
+          git tag -a "runtime-v$VERSION" \
+            -m "Auto-bump on workspace/** edit on $GITHUB_REF" \
+            -m "Triggered by: $GITHUB_REF @ $GITHUB_SHA" \
+            -m "publish-runtime.yml will pick up this tag and upload to PyPI"
+          # Push via DISPATCH_TOKEN (a Gitea PAT). Using the bot identity
+          # ensures the resulting tag-push event is dispatched to
+          # publish-runtime.yml; act_runner's default GITHUB_TOKEN cannot
+          # trigger downstream workflows.
+          git remote set-url origin "${GITEA_URL#https://}"
+          git remote set-url origin "https://x-access-token:${DISPATCH_TOKEN}@${GITEA_URL#https://}/molecule-ai/molecule-core.git"
+          git push origin "runtime-v$VERSION"
+          echo "✓ pushed runtime-v$VERSION — publish-runtime.yml should fire next"

.gitea/workflows/publish-runtime.yml

@@ -12,7 +12,24 @@ name: publish-runtime
 #   - Replaced `github.ref_name` (GitHub-only) with `${GITHUB_REF#refs/tags/}`
 #     — Gitea Actions exposes github.ref (the full ref) but not ref_name
 #   - Dropped `merge_group` trigger (Gitea has no merge queue)
-#   - Dropped `staging` branch trigger (no staging branch exists in this repo)
+#
+# 2026-05-10 (issue #348): originally restored `staging`/`main` branch +
+# `workspace/**` path-filter trigger in PR #349.
+#
+# 2026-05-11 (issue #351): REVERTED the branches+paths trigger from THIS
+# file. Bundling `paths` with `tags` under a single `on.push` key caused
+# Gitea Actions to never dispatch the workflow for tag-push events (0
+# runs in `action_run` for workflow_id='publish-runtime.yml' since the
+# port, including the runtime-v1.0.0 tag — which is why PyPI is still at
+# 0.1.129 despite a v1.0.0 Gitea tag existing).
+#
+# The auto-bump-on-workspace-edit trigger now lives in
+# `.gitea/workflows/publish-runtime-autobump.yml`. That file computes the
+# next version from PyPI's latest and pushes a `runtime-v$VERSION` tag,
+# which THIS file then picks up via the tags-only trigger below.
+#
+# This decoupling means Gitea's path-vs-tag evaluator never has to
+# disambiguate — each file has a single unambiguous trigger shape.
 #
 # PyPI publishing: requires PYPI_TOKEN repository secret (or org-level secret).
 # Set via: repo Settings → Actions → Variables and Secrets → New Secret.
@@ -65,10 +82,9 @@ jobs:
             VERSION="${GITHUB_REF#refs/tags/runtime-v}"
           else
             # Fallback: derive from PyPI latest + patch bump.
-            # (The staging-push auto-bump trigger is dropped on Gitea —
-            # no staging branch exists. This fallback path is kept for
-            # robustness if a future automation uses workflow_dispatch without
-            # an explicit version input.)
+            # Used by the restored `push.branches: [main, staging]` +
+            # `paths: workspace/**` auto-bump trigger (issue #348). Also kept
+            # for workflow_dispatch invocations that omit the version input.
             LATEST=$(curl -fsS --retry 3 https://pypi.org/pypi/molecule-ai-workspace-runtime/json \
               | python -c "import sys,json; print(json.load(sys.stdin)['info']['version'])")
             MAJOR=$(echo "$LATEST" | cut -d. -f1)

docker-compose.infra.yml

@@ -1,6 +1,7 @@
 services:
+  # digest-pinned 2026-05-10 (sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579, linux/amd64)
   postgres:
-    image: postgres:16-alpine
+    image: postgres@sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579
     environment:
       POSTGRES_USER: ${POSTGRES_USER:-dev}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev}
@@ -17,7 +18,7 @@ services:
       retries: 10
 
   langfuse-db-init:
-    image: postgres:16-alpine
+    image: postgres@sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579
     depends_on:
       postgres:
         condition: service_healthy
@@ -36,8 +37,9 @@ services:
           psql -h postgres -U "$${POSTGRES_USER}" -d postgres -c "CREATE DATABASE langfuse"
         fi
 
+  # digest-pinned 2026-05-10 (sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7, linux/amd64)
   redis:
-    image: redis:7-alpine
+    image: redis@sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7
     command: ["redis-server", "--notify-keyspace-events", "KEA"]
     ports:
       - "6379:6379"
@@ -49,8 +51,9 @@ services:
       timeout: 5s
       retries: 10
 
+  # digest-pinned 2026-05-10 (sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe, linux/amd64)
   clickhouse:
-    image: clickhouse/clickhouse-server:24-alpine
+    image: clickhouse/clickhouse-server@sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe
     environment:
       CLICKHOUSE_DB: langfuse
       CLICKHOUSE_USER: langfuse
@@ -64,8 +67,9 @@ services:
       retries: 10
 
   # dev-only: no-auth on 0.0.0.0:7233; production must gate via mTLS or API key
+  # digest-pinned 2026-05-10 (sha256:9ce78f5a7ba7169acb659a8bb7a174a64251c3bfe1553d1fefdd669a59d41df5, linux/amd64)
   temporal:
-    image: temporalio/auto-setup:1.25
+    image: temporalio/auto-setup@sha256:9ce78f5a7ba7169acb659a8bb7a174a64251c3bfe1553d1fefdd669a59d41df5
     depends_on:
       postgres:
         condition: service_healthy
@@ -85,8 +89,9 @@ services:
       timeout: 5s
       retries: 10
 
+  # digest-pinned 2026-05-10 (sha256:7be8d6e41d4846ccb718c4f35956c9557512f8085e94a73954286a4e95113703, linux/amd64)
   temporal-ui:
-    image: temporalio/ui:2.31.2
+    image: temporalio/ui@sha256:7be8d6e41d4846ccb718c4f35956c9557512f8085e94a73954286a4e95113703
     depends_on:
       - temporal
     environment:
@@ -95,8 +100,9 @@ services:
     ports:
       - "8233:8080"
 
+  # digest-pinned 2026-05-10 (sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d, linux/amd64)
   langfuse-web:
-    image: langfuse/langfuse:2
+    image: langfuse/langfuse@sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d
     depends_on:
       clickhouse:
         condition: service_healthy

docker-compose.yml

@@ -4,8 +4,9 @@ include:
 
 services:
   # --- Infrastructure ---
+  # digest-pinned 2026-05-10 (sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579, linux/amd64)
   postgres:
-    image: postgres:16-alpine
+    image: postgres@sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579
     environment:
       POSTGRES_USER: ${POSTGRES_USER:-dev}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev}
@@ -25,7 +26,7 @@ services:
       retries: 10
 
   langfuse-db-init:
-    image: postgres:16-alpine
+    image: postgres@sha256:4941ef97aaa2633ce9808f7766f8b8d746dd039ce8c51ca6da185c3dc63ab579
     depends_on:
       postgres:
         condition: service_healthy
@@ -46,8 +47,9 @@ services:
     networks:
       - molecule-core-net
 
+  # digest-pinned 2026-05-10 (sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7, linux/amd64)
   redis:
-    image: redis:7-alpine
+    image: redis@sha256:b1addbe72465a718643cff9e60a58e6df1841e29d6d7d60c9a85d8d72f08d1a7
     command: ["redis-server", "--notify-keyspace-events", "KEA"]
     ports:
       - "6379:6379"
@@ -63,8 +65,9 @@ services:
       retries: 10
 
   # --- Observability ---
+  # digest-pinned 2026-05-10 (sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe, linux/amd64)
   langfuse-clickhouse:
-    image: clickhouse/clickhouse-server:24-alpine
+    image: clickhouse/clickhouse-server@sha256:5b296e0ba1da74efea3143c773ddd60245f249fb7c72eb1d866c2d6ebc759fbe
     environment:
       CLICKHOUSE_DB: langfuse
       CLICKHOUSE_USER: langfuse
@@ -79,8 +82,9 @@ services:
       timeout: 5s
       retries: 10
 
+  # digest-pinned 2026-05-10 (sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d, linux/amd64)
   langfuse:
-    image: langfuse/langfuse:2
+    image: langfuse/langfuse@sha256:e7aafd3ccf721821b40f8b2251220b4bb8af5e4877b5c5a8846af5b3318aaf1d
     depends_on:
       langfuse-clickhouse:
         condition: service_healthy
@@ -239,6 +243,8 @@ services:
     # First-time local setup or testing unreleased changes — build from source:
     #   docker compose build canvas && docker compose up -d canvas
     # Note: ECR images require AWS auth — `aws ecr get-login-password --region us-east-2 | docker login --username AWS --password-stdin 153263036946.dkr.ecr.us-east-2.amazonaws.com` before pull.
+    # Digest-pin requires: aws ecr describe-images --repository-name molecule-ai/canvas --image-tags latest --query 'imageDetails[0].imageDigest'
+    # TODO: pin canvas ECR image digest once AWS creds are available in CI.
     image: 153263036946.dkr.ecr.us-east-2.amazonaws.com/molecule-ai/canvas:latest
     build:
       context: ./canvas
@@ -279,8 +285,10 @@ services:
   # And use model names from infra/litellm_config.yml (e.g. "claude-opus-4-5",
   # "gpt-4o", "openrouter/deepseek-r1", "ollama/llama3.2").
   # Edit infra/litellm_config.yml to add/remove providers and models.
+  # digest-pinned 2026-05-10 (sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186)
+  # Refresh: curl -sI https://ghcr.io/v2/berriai/litellm/manifests/main-latest (Docker-Content-Digest header)
   litellm:
-    image: ghcr.io/berriai/litellm:main-latest
+    image: ghcr.io/berriai/litellm/main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
     profiles:
       - multi-provider
     ports:
@@ -311,8 +319,10 @@ services:
   #   docker compose exec ollama ollama pull qwen2.5-coder:7b
   # Then set MODEL_PROVIDER=ollama:llama3.2 in your workspace config.yaml
   # Workspace agents reach Ollama at http://ollama:11434 (internal Docker network).
+  # digest-pinned 2026-05-10 (sha256:90bd8ed1ad1853fbfb1ef5835f9d7a24fe890e05ace521e2d8d7a6f56bb667dd, linux/amd64)
+  # Refresh: curl -s https://hub.docker.com/v2/repositories/ollama/ollama/tags/latest | python3 -c "import json,sys; ..."
   ollama:
-    image: ollama/ollama:latest
+    image: ollama/ollama@sha256:90bd8ed1ad1853fbfb1ef5835f9d7a24fe890e05ace521e2d8d7a6f56bb667dd
     profiles:
       - local-models
     ports:

scripts/clone-manifest.sh

@@ -37,6 +37,50 @@ PLUGINS_DIR="${4:?Missing plugins dir}"
 EXPECTED=0
 CLONED=0
 
+# clone_one_with_retry — clone a single repo, retrying on transient failure.
+#
+# Why: the publish-workspace-server-image (and harness-replays) CI jobs
+# clone the full manifest (~36 repos) serially on a memory-constrained
+# Gitea Actions runner. Under host memory pressure the OOM killer
+# occasionally SIGKILLs git-remote-https mid-clone:
+#
+#   error: git-remote-https died of signal 9
+#   fatal: the remote end hung up unexpectedly
+#
+# (observed in publish-workspace-server-image run 4622 on 2026-05-10 — the
+# job died on the 14th of 36 clones, which wedged staging→main). One
+# transient SIGKILL / network blip would otherwise fail the whole tenant
+# image rebuild. Retrying after a short backoff lets the pressure subside.
+# The durable fix is more runner RAM/swap (tracked with Infra-SRE); this
+# just stops a single flake from being release-blocking.
+#
+# Args: <target_dir> <name> <clone_url> <display_url> <ref>
+clone_one_with_retry() {
+    local tdir="$1" name="$2" url="$3" display="$4" ref="$5"
+    local attempt=1 max_attempts=3 backoff
+
+    while : ; do
+        # A killed attempt can leave a partial directory behind; git clone
+        # refuses a non-empty target, so wipe it before each try.
+        rm -rf "$tdir/$name"
+
+        if [ "$ref" = "main" ]; then
+            if git clone --depth=1 -q "$url" "$tdir/$name"; then return 0; fi
+        else
+            if git clone --depth=1 -q --branch "$ref" "$url" "$tdir/$name"; then return 0; fi
+        fi
+
+        if [ "$attempt" -ge "$max_attempts" ]; then
+            echo "::error::clone failed after ${max_attempts} attempts: ${display}" >&2
+            return 1
+        fi
+        backoff=$((attempt * 3))   # 3s, then 6s
+        echo "  ⚠ clone attempt ${attempt}/${max_attempts} failed for ${display} — retrying in ${backoff}s" >&2
+        sleep "$backoff"
+        attempt=$((attempt + 1))
+    done
+}
+
 clone_category() {
     local category="$1"
     local target_dir="$2"
@@ -82,11 +126,7 @@ clone_category() {
         fi
 
         echo "  cloning $display_url -> $target_dir/$name (ref=$ref)"
-        if [ "$ref" = "main" ]; then
-            git clone --depth=1 -q "$clone_url" "$target_dir/$name"
-        else
-            git clone --depth=1 -q --branch "$ref" "$clone_url" "$target_dir/$name"
-        fi
+        clone_one_with_retry "$target_dir" "$name" "$clone_url" "$display_url" "$ref"
         CLONED=$((CLONED + 1))
         i=$((i + 1))
     done