From 1f3fd90adcadd9908fe44607de77f73d4bf0f98d Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Fri, 24 Apr 2026 17:02:45 +0300 Subject: [PATCH 001/120] ci: add automatic Claude Code review on PRs (#101) * ci: add Claude Code PR review + @claude mention workflows * ci: add id-token: write permission required by claude-code-action --- .github/workflows/claude-review.yml | 41 +++++++++++++++++++++++++++++ .github/workflows/claude.yml | 35 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 .github/workflows/claude-review.yml create mode 100644 .github/workflows/claude.yml diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml new file mode 100644 index 000000000..e9edaaf92 --- /dev/null +++ b/.github/workflows/claude-review.yml @@ -0,0 +1,41 @@ +name: Claude PR Review + +on: + pull_request: + types: [opened, synchronize, ready_for_review] + +permissions: + contents: read + pull-requests: write + issues: read + id-token: write + +concurrency: + group: claude-review-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + review: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + track_progress: true + prompt: | + Review this pull request against the main branch. Focus on: + - Correctness and likely bugs + - Security issues (auth, input validation, secrets, injection) + - Performance regressions, especially in the agent loop and streaming paths + - Breakages in LiteLLM / Bedrock routing (model ids, params, prompt caching) + - Test coverage for new behavior + - Backend/frontend contract drift (FastAPI routes ↔ React client) + + Be concise. Prefer inline comments over long summaries. Skip nitpicks on + style that ruff already catches. If the PR looks good, say so briefly + instead of inventing issues. diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 000000000..d3036a232 --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,35 @@ +name: Claude on Mention + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + pull_request_review: + types: [submitted] + issues: + types: [opened, assigned] + +permissions: + contents: write + pull-requests: write + issues: write + id-token: write + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + track_progress: true From b292d83aa78e6bb2801cd3ff99fd46789122807e Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:11:47 +0300 Subject: [PATCH 002/120] ci: add REVIEW.md for tunable Claude reviews (#104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: add REVIEW.md and inject it into the review prompt REVIEW.md is a repo-root freeform instructions file that gets prepended to the review prompt as highest-priority guidance. Lets maintainers tune severity calibration, nit caps, skip lists, and repo-specific must-checks by editing one file instead of the workflow YAML. Mirrors the pattern used by the managed Anthropic Code Review product so we keep the same levers on our self-hosted Actions setup. * review: add merge-bias, pushback norms, and What-I-checked summary Insights from the Latent Space 'harness engineering' interview: review agents should default to merge, not block; 🟔/🟣 are informational not required; author pushback without a fix is legitimate for non-Important findings; repeated disagreement is a signal REVIEW.md is missing a rule. Also adds a 'What I checked' bullet list to the summary shape so even clean LGTM reviews surface the coverage the reviewer actually applied. * review: rename severity markers to P0/P1/P2 Replace šŸ”“ Important / 🟔 Nit / 🟣 Pre-existing with plain P0/P1/P2 labels throughout REVIEW.md and the workflow prompt. Matches the priority scheme from the Latent Space harness-engineering interview and reads cleaner in terminal-rendered GitHub diffs. * review: swap merge-bias for rigor; require deep investigation + merge verdict Maintainer feedback: default-bias-merge was borrowed from a closed AI-loop context (Ryan's harness) where the PR author is also an agent and merge-and- iterate is cheap. For an open-source repo taking one-shot external PRs with a small maintainer team, the risk flips: false negatives ship bugs, false positives cost one contributor round trip. Rigor is the correct default. Three concrete changes: - 'Default bias: rigor' replaces 'default bias: merge'. Hold the line on P0 even under contributor pushback. P1/P2 still accept deferral silently. - New 'Investigate before posting' section requires reading callers and callees (not just the diff), tracing routing/auth chains end-to-end, and checking established patterns before flagging divergence. - Summary now carries an explicit 'Verdict: ready to merge / changes requested / needs discussion' so the maintainer sees the call at a glance. * review: add Dependency PRs rubric to catch supply-chain bait Empirical test against the current open-PR queue surfaced a false-negative: a bot PR (orbisai0security, #96) titled 'upgrade authlib to 1.6.9 for CVE-2026-27962' actually bumps 1.6.5 → 1.7.0 in the lockfile, the CVE isn't in NVD, and the bump silently introduces a new transitive dep (joserfc). Existing REVIEW.md rules are routing/auth/agent-loop centric and would LGTM it. New 'Dependency PRs' section requires: CVE verification against NVD or GH Advisory DB, title-version ↔ lockfile-diff match, justification for any new transitive dep, and P0 framing-flag when a dep-only PR claims a code-behavior fix. * review: trim REVIEW.md — drop enumerations, tighten P1 cap to 3 - Remove 'What counts as P0 in this repo' enumeration: P0 is implicitly for Claude to figure out from context, not a static checklist. - Remove 'Always check' repo-specific enumeration: same rationale. The rigor + investigate-before-posting framing carries the weight. - Remove 'Anything CI already enforces' block under 'Do not report': rigor framing plus the skip-paths list already covers it. - Drop 'If you cannot invest the depth to verify, do not post the finding' tail from Investigate-before-posting (implicit in rigor). - Drop routing/effort/caching citation expansion from Verification bar (implicit in generic citation rule). - Drop the concrete What-I-checked example from Summary shape. - Drop 'one paragraph of context at most' from Summary shape. - Tighten P1 cap from 5 to 3. * review: compress dep-PR section to one paragraph, drop test-nag example Dep-PR rubric was carrying four bulleted cases that amounted to one idea: claims in the PR body must match the diff, new deps need justification, lying framing is P0. Collapsed to a single paragraph. Also drops 'Consider adding a test' from the speculative examples — that heuristic tends to manufacture P1s rather than filter noise. --- .github/workflows/claude-review.yml | 45 +++++++--- REVIEW.md | 135 ++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 REVIEW.md diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index e9edaaf92..a38ce8c37 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -23,19 +23,40 @@ jobs: with: fetch-depth: 0 + - name: Compose review prompt + id: compose + run: | + { + printf 'prompt<> "$GITHUB_OUTPUT" + - uses: anthropics/claude-code-action@v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} track_progress: true - prompt: | - Review this pull request against the main branch. Focus on: - - Correctness and likely bugs - - Security issues (auth, input validation, secrets, injection) - - Performance regressions, especially in the agent loop and streaming paths - - Breakages in LiteLLM / Bedrock routing (model ids, params, prompt caching) - - Test coverage for new behavior - - Backend/frontend contract drift (FastAPI routes ↔ React client) - - Be concise. Prefer inline comments over long summaries. Skip nitpicks on - style that ruff already catches. If the PR looks good, say so briefly - instead of inventing issues. + prompt: ${{ steps.compose.outputs.prompt }} diff --git a/REVIEW.md b/REVIEW.md new file mode 100644 index 000000000..3f08c60a8 --- /dev/null +++ b/REVIEW.md @@ -0,0 +1,135 @@ +# Review instructions + +These rules override the default review guidance. Treat them as the highest-priority +instruction block for any review of this repo. If something here contradicts a more +generic review habit, follow these. + +## Severity levels + +Every finding carries one of three priority labels: + +- **P0** — blocks merge. +- **P1** — worth fixing, not blocking. +- **P2** — informational. + +Write labels as plain text (`P0`, `P1`, `P2`) in finding headers. Do not use +emoji or colored markers. Use judgment on what belongs at which level — this +repo does not enumerate P0 cases; read the code and decide. + +## Default bias: rigor + +Reviews gate merges. This is an open-source repo that takes PRs from anyone; the +maintainer team is small and relies on the review to catch what they don't have +time to verify themselves. **Default bias is rigor, not speed.** When in doubt +on a P0-class concern, investigate further before deciding whether to flag — a +false negative ships a bug to production, a false positive costs the contributor +one round trip. + +Rigor is not nitpicking. The P1 cap, "do not report" skip list, and verification +bar all still apply. Rigor means going deep on a small number of real concerns, +not surfacing a large number of shallow ones. Prefer one well-investigated P0 +over three speculative P1s. + +**Hold the line on P0.** If the author pushes back on a P0 finding without a fix +that actually addresses the root cause, re-state the concern with added +citations. Only accept the pushback if the author points to code or behavior you +missed. Do not soften a P0 because the contributor is polite or new to the repo. + +For P1 and P2: if the author defers or pushes back without fixing, accept it +silently — do not re-flag on subsequent commits. P1/P2 are informational; the +author may defer to a follow-up issue at their discretion. + +If Claude and the author repeatedly disagree on the same class of finding, the +signal is that REVIEW.md is missing a rule; note it once in the PR summary as +`suggest-rule: ` and stop. + +## Investigate before posting + +The depth of your analysis determines the strength of your finding. For any +P0-class concern, before writing it up: + +- Read the relevant callers and callees, not just the diff. Use Read and Grep + to open files the diff doesn't touch but the changed code interacts with. +- Trace the full chain end-to-end for routing, auth, and agent-loop findings. + Cite each hop by `file:line`, not just the suspicious line. +- Check whether the codebase already has an established pattern for this kind + of change (`grep` for similar call sites, similar tool definitions, similar + route guards). If the PR introduces a new approach where an established + pattern exists, flag that — divergence from the existing pattern is usually a + regression vector even when the new code "works." +- Confirm the specific behavior you're claiming. "This breaks X" must be + grounded in either the code handling X or a test exercising X, not in + inference from naming or structure. + +A finding you "spotted" by scanning the diff is more likely to be a false +positive than a finding you verified by reading the code around it. + +## P1 cap + +Report at most **3** P1 findings per review. If you found more, say "plus N +similar items" in the summary. If everything you found is P1 or below, open the +summary with "No blocking issues." + +## Re-review convergence + +If this PR has already received a Claude review (there is a prior review comment +by the `claude` bot), suppress new P1 findings and post only P0 ones. Do not +re-post P1s that were already flagged on earlier commits. If the author pushed a +fix for a previously flagged issue, acknowledge it in one line rather than +re-flagging. + +## Do not report + +Anything in these paths — skip entirely: + +- `frontend/node_modules/**`, `**/*.lock`, `uv.lock`, `package-lock.json` +- `hf_agent.egg-info/**`, `.ruff_cache/**`, `.pytest_cache/**`, `.venv/**` +- `session_logs/**`, `reports/**` +- Anything under a `gen/` or `generated/` path + +Anything speculative — do not post: + +- "This might be slow" without a concrete complexity claim tied to a specific + input size +- Hypothetical race conditions without a concrete interleaving + +## Dependency PRs + +For PRs whose diff is only a lockfile bump, a `pyproject.toml` change, or a +new dependency, the code rules above don't apply — risks shift to provenance +and framing. Every claim in the title or body (CVE IDs, version numbers, +behavior fixes) must match what the diff actually does, and any new +transitive dep needs justification. A PR that lies in its framing is P0 +regardless of whether the code change is safe in isolation. + +## Verification bar + +Every behavior claim in a finding must cite `file:line`. "This breaks X" is not +actionable without a line reference. If you cannot cite a line, do not post +the finding. + +## Summary shape + +Open the review body with a single-line tally and an explicit merge verdict, on +two lines: + +``` +2 P0, 3 P1 +Verdict: changes requested +``` + +Valid verdicts: + +- **Verdict: ready to merge** — no P0 findings, contributor can merge as-is + once any CI passes +- **Verdict: changes requested** — at least one P0 that must be addressed + before merging +- **Verdict: needs discussion** — a design-level concern the maintainer should + weigh in on before the contributor iterates (use sparingly) + +If it's a clean review, write `LGTM` followed by `Verdict: ready to merge`. + +Then a **What I checked** bullet list — one line per major area you examined, +regardless of whether you found anything. This gives the maintainer visible +coverage at a glance and lets them decide whether to spot-check areas you +didn't touch. From 2a2e1700bf0fa0cdeca2d9921cbe18b22b9db62f Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:55:59 +0300 Subject: [PATCH 003/120] feat(observability) --- agent/config.py | 9 +- agent/core/agent_loop.py | 28 +- agent/core/redact.py | 68 +++ agent/core/session.py | 50 +- agent/core/session_uploader.py | 39 +- agent/core/telemetry.py | 289 ++++++++++ agent/sft/__init__.py | 0 agent/sft/tagger.py | 324 +++++++++++ agent/tools/jobs_tool.py | 23 +- agent/tools/sandbox_tool.py | 9 + backend/kpis_scheduler.py | 146 +++++ backend/main.py | 29 + backend/routes/agent.py | 38 ++ backend/session_manager.py | 16 +- configs/main_agent_config.json | 2 +- .../src/components/Chat/AssistantMessage.tsx | 49 +- .../src/components/Chat/MessageBubble.tsx | 3 + frontend/src/components/Chat/MessageList.tsx | 4 +- frontend/src/components/SessionChat.tsx | 1 + pyproject.toml | 1 + scripts/build_kpis.py | 517 ++++++++++++++++++ scripts/build_sft.py | 204 +++++++ tests/unit/test_build_kpis.py | 164 ++++++ tests/unit/test_build_sft.py | 78 +++ tests/unit/test_heartbeat.py | 134 +++++ tests/unit/test_kpis_scheduler.py | 107 ++++ tests/unit/test_redact.py | 76 +++ tests/unit/test_sft_tagger.py | 197 +++++++ 28 files changed, 2584 insertions(+), 21 deletions(-) create mode 100644 agent/core/redact.py create mode 100644 agent/core/telemetry.py create mode 100644 agent/sft/__init__.py create mode 100644 agent/sft/tagger.py create mode 100644 backend/kpis_scheduler.py create mode 100644 scripts/build_kpis.py create mode 100644 scripts/build_sft.py create mode 100644 tests/unit/test_build_kpis.py create mode 100644 tests/unit/test_build_sft.py create mode 100644 tests/unit/test_heartbeat.py create mode 100644 tests/unit/test_kpis_scheduler.py create mode 100644 tests/unit/test_redact.py create mode 100644 tests/unit/test_sft_tagger.py diff --git a/agent/config.py b/agent/config.py index b7e698ad7..7e696dd78 100644 --- a/agent/config.py +++ b/agent/config.py @@ -24,8 +24,13 @@ class Config(BaseModel): model_name: str mcpServers: dict[str, MCPServerConfig] = {} save_sessions: bool = True - session_dataset_repo: str = "akseljoonas/hf-agent-sessions" - auto_save_interval: int = 3 # Save every N user turns (0 = disabled) + session_dataset_repo: str = "smolagents/ml-intern-sessions" + auto_save_interval: int = 1 # Save every N user turns (0 = disabled) + # Mid-turn heartbeat: save + upload every N seconds while events are being + # emitted. Guards against losing trace data on long-running turns that + # crash before turn_complete (e.g. a multi-hour hf_jobs wait that OOMs). + # 0 = disabled. Consumed by agent.core.telemetry.HeartbeatSaver. + heartbeat_interval_s: int = 60 yolo_mode: bool = False # Auto-approve all tool calls without confirmation max_iterations: int = 300 # Max LLM calls per agent turn (-1 = unlimited) diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index c3fd88bc8..3d68d54d8 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -6,12 +6,14 @@ import json import logging import os -from dataclasses import dataclass +import time +from dataclasses import dataclass, field from litellm import ChatCompletionMessageToolCall, Message, acompletion from litellm.exceptions import ContextWindowExceededError from agent.config import Config +from agent.core import telemetry from agent.core.doom_loop import check_for_doom_loop from agent.core.llm_params import _resolve_llm_params from agent.core.prompt_caching import with_prompt_caching @@ -291,6 +293,7 @@ class LLMResult: tool_calls_acc: dict[int, dict] token_count: int finish_reason: str | None + usage: dict = field(default_factory=dict) async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> LLMResult: @@ -298,6 +301,7 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> response = None _healed_effort = False # one-shot safety net per call messages, tools = with_prompt_caching(messages, tools, llm_params.get("model")) + t_start = time.monotonic() for _llm_attempt in range(_MAX_LLM_RETRIES): try: response = await acompletion( @@ -339,6 +343,7 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> tool_calls_acc: dict[int, dict] = {} token_count = 0 finish_reason = None + final_usage_chunk = None async for chunk in response: if session.is_cancelled: @@ -349,6 +354,7 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> if not choice: if hasattr(chunk, "usage") and chunk.usage: token_count = chunk.usage.total_tokens + final_usage_chunk = chunk continue delta = choice.delta @@ -379,12 +385,22 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> if hasattr(chunk, "usage") and chunk.usage: token_count = chunk.usage.total_tokens + final_usage_chunk = chunk + + usage = await telemetry.record_llm_call( + session, + model=llm_params.get("model", session.config.model_name), + response=final_usage_chunk, + latency_ms=int((time.monotonic() - t_start) * 1000), + finish_reason=finish_reason, + ) return LLMResult( content=full_content or None, tool_calls_acc=tool_calls_acc, token_count=token_count, finish_reason=finish_reason, + usage=usage, ) @@ -393,6 +409,7 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) response = None _healed_effort = False messages, tools = with_prompt_caching(messages, tools, llm_params.get("model")) + t_start = time.monotonic() for _llm_attempt in range(_MAX_LLM_RETRIES): try: response = await acompletion( @@ -454,11 +471,20 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) Event(event_type="assistant_message", data={"content": content}) ) + usage = await telemetry.record_llm_call( + session, + model=llm_params.get("model", session.config.model_name), + response=response, + latency_ms=int((time.monotonic() - t_start) * 1000), + finish_reason=finish_reason, + ) + return LLMResult( content=content, tool_calls_acc=tool_calls_acc, token_count=token_count, finish_reason=finish_reason, + usage=usage, ) diff --git a/agent/core/redact.py b/agent/core/redact.py new file mode 100644 index 000000000..8978942c8 --- /dev/null +++ b/agent/core/redact.py @@ -0,0 +1,68 @@ +"""Secret scrubbing for session trajectories before upload. + +Users frequently paste HF / API / GitHub tokens into the chat, or scripts echo +them via env dumps. This module applies regex-based redaction to any string +value found recursively in a trajectory payload. The goal is best-effort — +strict formats are matched; we won't catch free-form leaks like "my password +is hunter2". +""" + +from __future__ import annotations + +import re +from typing import Any + +# Each entry: (compiled regex, replacement placeholder). +# Patterns are conservative: they only match tokens with the canonical prefix +# and a minimum body length so we don't paint over normal text. +_PATTERNS: list[tuple[re.Pattern, str]] = [ + # Hugging Face tokens: hf_[A-Za-z0-9]{30,} + (re.compile(r"hf_[A-Za-z0-9]{30,}"), "[REDACTED_HF_TOKEN]"), + # Anthropic: sk-ant-[A-Za-z0-9_\-]{20,} + (re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,}"), "[REDACTED_ANTHROPIC_KEY]"), + # OpenAI: sk-[A-Za-z0-9]{40,} (legacy + proj keys) + (re.compile(r"sk-(?!ant-)[A-Za-z0-9_\-]{40,}"), "[REDACTED_OPENAI_KEY]"), + # GitHub classic PATs: ghp_, gho_, ghu_, ghs_, ghr_ followed by 36+ chars + (re.compile(r"gh[pousr]_[A-Za-z0-9]{36,}"), "[REDACTED_GITHUB_TOKEN]"), + # GitHub fine-grained PATs: github_pat_ + (re.compile(r"github_pat_[A-Za-z0-9_]{36,}"), "[REDACTED_GITHUB_TOKEN]"), + # AWS access key IDs: AKIA / ASIA + 16 uppercase alnum + (re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"), "[REDACTED_AWS_KEY_ID]"), + # Generic 'Bearer ' header values + (re.compile(r"(?i)bearer\s+[A-Za-z0-9_\-\.=]{20,}"), "Bearer [REDACTED]"), +] + +# Env-var-like exports: we scrub the value but keep the name so callers can +# still see which secret was referenced. Covers `KEY=value` and `KEY: value` +# when the key looks secret-y. +_SECRETY_NAMES = re.compile( + r"(?i)\b(HF_TOKEN|HUGGINGFACEHUB_API_TOKEN|ANTHROPIC_API_KEY|OPENAI_API_KEY|" + r"GITHUB_TOKEN|AWS_SECRET_ACCESS_KEY|AWS_ACCESS_KEY_ID|PASSWORD|SECRET|API_KEY)" + r"\s*[:=]\s*([^\s\"']+)" +) + + +def scrub_string(s: str) -> str: + """Apply all redaction patterns to a single string. Safe on non-strings.""" + if not isinstance(s, str) or not s: + return s + out = s + for pat, repl in _PATTERNS: + out = pat.sub(repl, out) + out = _SECRETY_NAMES.sub(lambda m: f"{m.group(1)}=[REDACTED]", out) + return out + + +def scrub(obj: Any) -> Any: + """Recursively scrub every string value in a nested dict/list structure. + + Returns a new object — inputs are not mutated.""" + if isinstance(obj, str): + return scrub_string(obj) + if isinstance(obj, dict): + return {k: scrub(v) for k, v in obj.items()} + if isinstance(obj, list): + return [scrub(v) for v in obj] + if isinstance(obj, tuple): + return tuple(scrub(v) for v in obj) + return obj diff --git a/agent/core/session.py b/agent/core/session.py index 4b6390d84..0cf9524a1 100644 --- a/agent/core/session.py +++ b/agent/core/session.py @@ -108,6 +108,11 @@ def __init__( self.session_start_time = datetime.now().isoformat() self.turn_count: int = 0 self.last_auto_save_turn: int = 0 + # Stable local save path so heartbeat saves overwrite one file instead + # of spamming session_logs/. ``_last_heartbeat_ts`` is owned by + # ``agent.core.telemetry.HeartbeatSaver`` and lazily initialised there. + self._local_save_path: Optional[str] = None + self._last_heartbeat_ts: Optional[float] = None # Per-model probed reasoning-effort cache. Populated by the probe # on /model switch, read by ``effective_effort_for`` below. Keys are @@ -132,6 +137,10 @@ async def send_event(self, event: Event) -> None: } ) + # Mid-turn heartbeat flush (owned by telemetry module). + from agent.core.telemetry import HeartbeatSaver + HeartbeatSaver.maybe_fire(self) + def cancel(self) -> None: """Signal cancellation to the running agent loop.""" self._cancelled.set() @@ -184,6 +193,12 @@ async def auto_save_if_needed(self) -> None: def get_trajectory(self) -> dict: """Serialize complete session trajectory for logging""" + tools: list = [] + if self.tool_router is not None: + try: + tools = self.tool_router.get_tool_specs_for_llm() or [] + except Exception: + tools = [] return { "session_id": self.session_id, "session_start_time": self.session_start_time, @@ -191,6 +206,7 @@ def get_trajectory(self) -> dict: "model_name": self.config.model_name, "messages": [msg.model_dump() for msg in self.context_manager.items], "events": self.logged_events, + "tools": tools, } def save_trajectory_local( @@ -216,16 +232,42 @@ def save_trajectory_local( trajectory = self.get_trajectory() + # Scrub secrets at save time so session_logs/ never holds raw + # tokens on disk — a log aggregator, crash dump, or filesystem + # snapshot between heartbeats would otherwise leak them. + try: + from agent.core.redact import scrub + for key in ("messages", "events", "tools"): + if key in trajectory: + trajectory[key] = scrub(trajectory[key]) + except Exception as _e: + logger.debug("Redact-on-save failed (non-fatal): %s", _e) + # Add upload metadata trajectory["upload_status"] = upload_status trajectory["upload_url"] = dataset_url trajectory["last_save_time"] = datetime.now().isoformat() - filename = f"session_{self.session_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" - filepath = log_dir / filename - - with open(filepath, "w") as f: + # Reuse one stable path per session so heartbeat saves overwrite + # the same file instead of creating a new timestamped file every + # minute. The timestamp in the filename is kept for first-save + # ordering; subsequent saves just rewrite that file. + if self._local_save_path and Path(self._local_save_path).parent == log_dir: + filepath = Path(self._local_save_path) + else: + filename = ( + f"session_{self.session_id}_" + f"{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + ) + filepath = log_dir / filename + self._local_save_path = str(filepath) + + # Atomic-ish write: stage to .tmp then rename so a crash mid-write + # doesn't leave a truncated JSON that breaks the retry scanner. + tmp_path = filepath.with_suffix(filepath.suffix + ".tmp") + with open(tmp_path, "w") as f: json.dump(trajectory, f, indent=2) + tmp_path.replace(filepath) return str(filepath) except Exception as e: diff --git a/agent/core/session_uploader.py b/agent/core/session_uploader.py index ef2f9496d..f22b52010 100644 --- a/agent/core/session_uploader.py +++ b/agent/core/session_uploader.py @@ -15,8 +15,15 @@ load_dotenv() -# Token for session uploads — loaded from env var (never hardcode tokens in source) -_SESSION_TOKEN = os.environ.get("HF_SESSION_UPLOAD_TOKEN", "") +# Token for session uploads. Fallback chain (least-privilege first) — matches +# backend/kpis_scheduler.py so one write-scoped token on the Space covers every +# telemetry dataset. Never hardcode tokens in source. +_SESSION_TOKEN = ( + os.environ.get("HF_SESSION_UPLOAD_TOKEN") + or os.environ.get("HF_TOKEN") + or os.environ.get("HF_ADMIN_TOKEN") + or "" +) def upload_session_as_file( @@ -58,15 +65,37 @@ def upload_session_as_file( json.dump(data, f, indent=2) return False + # Scrub secrets (HF tokens, API keys, etc.) from messages + events + # before they leave the local disk. Best-effort regex-based redaction — + # see agent/core/redact.py for the patterns covered. + try: + from agent.core.redact import scrub # type: ignore + except Exception: + # Fallback for environments where the agent package isn't importable + # (shouldn't happen in our subprocess, but be defensive). + import importlib.util + _spec = importlib.util.spec_from_file_location( + "_redact", + Path(__file__).parent / "redact.py", + ) + _mod = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(_mod) # type: ignore + scrub = _mod.scrub + scrubbed_messages = scrub(data["messages"]) + scrubbed_events = scrub(data["events"]) + scrubbed_tools = scrub(data.get("tools") or []) + # Prepare JSONL content (single line) - # Store messages and events as JSON strings to avoid schema conflicts + # Store messages/events/tools as JSON strings to avoid schema conflicts + # across sessions with different tool rosters. session_row = { "session_id": data["session_id"], "session_start_time": data["session_start_time"], "session_end_time": data["session_end_time"], "model_name": data["model_name"], - "messages": json.dumps(data["messages"]), - "events": json.dumps(data["events"]), + "messages": json.dumps(scrubbed_messages), + "events": json.dumps(scrubbed_events), + "tools": json.dumps(scrubbed_tools), } # Create temporary JSONL file diff --git a/agent/core/telemetry.py b/agent/core/telemetry.py new file mode 100644 index 000000000..11818585d --- /dev/null +++ b/agent/core/telemetry.py @@ -0,0 +1,289 @@ +"""All agent observability in one module. + +Every telemetry signal the agent emits — LLM-call usage / cost, hf_jobs +lifecycle, sandbox lifecycle, user feedback, mid-turn heartbeat saves — is +defined here so business-logic files stay free of instrumentation noise. + +Callsites are one-liners:: + + await telemetry.record_llm_call(session, model=..., response=r, ...) + await telemetry.record_hf_job_submit(session, job, args, image=..., job_type="Python") + HeartbeatSaver.maybe_fire(session) + +All ``record_*`` functions emit a single ``Event`` via ``session.send_event`` +and never raise — telemetry is best-effort and must not break the agent. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from typing import Any + +logger = logging.getLogger(__name__) + + +# ── usage extraction ──────────────────────────────────────────────────────── + +def extract_usage(response_or_chunk: Any) -> dict: + """Flat usage dict from a litellm response or final-chunk usage object. + + Normalizes across providers: Anthropic exposes cache tokens as + ``cache_read_input_tokens`` / ``cache_creation_input_tokens``; OpenAI uses + ``prompt_tokens_details.cached_tokens``. Exposed under the stable keys + ``cache_read_tokens`` / ``cache_creation_tokens``. + """ + u = getattr(response_or_chunk, "usage", None) + if u is None and isinstance(response_or_chunk, dict): + u = response_or_chunk.get("usage") + if u is None: + return {} + + def _g(name, default=0): + if isinstance(u, dict): + return u.get(name, default) or default + return getattr(u, name, default) or default + + prompt = _g("prompt_tokens") + completion = _g("completion_tokens") + total = _g("total_tokens") or (prompt + completion) + + cache_read = _g("cache_read_input_tokens") + cache_creation = _g("cache_creation_input_tokens") + + if not cache_read: + details = _g("prompt_tokens_details", None) + if details is not None: + if isinstance(details, dict): + cache_read = details.get("cached_tokens", 0) or 0 + else: + cache_read = getattr(details, "cached_tokens", 0) or 0 + + return { + "prompt_tokens": int(prompt), + "completion_tokens": int(completion), + "total_tokens": int(total), + "cache_read_tokens": int(cache_read), + "cache_creation_tokens": int(cache_creation), + } + + +# ── llm_call ──────────────────────────────────────────────────────────────── + +async def record_llm_call( + session: Any, + *, + model: str, + response: Any = None, + latency_ms: int, + finish_reason: str | None, +) -> dict: + """Emit an ``llm_call`` event and return the extracted usage dict so + callers can stash it on their result object if they want.""" + usage = extract_usage(response) if response is not None else {} + cost_usd = 0.0 + if response is not None: + try: + from litellm import completion_cost + cost_usd = float(completion_cost(completion_response=response) or 0.0) + except Exception: + cost_usd = 0.0 + from agent.core.session import Event # local import to avoid cycle + try: + await session.send_event(Event( + event_type="llm_call", + data={ + "model": model, + "latency_ms": latency_ms, + "finish_reason": finish_reason, + "cost_usd": cost_usd, + **usage, + }, + )) + except Exception as e: + logger.debug("record_llm_call failed (non-fatal): %s", e) + return usage + + +# ── hf_jobs ──────────────────────────────────────────────────────────────── + +def _infer_push_to_hub(script_or_cmd: Any) -> bool: + if not isinstance(script_or_cmd, str): + return False + return ( + "push_to_hub=True" in script_or_cmd + or "push_to_hub=true" in script_or_cmd + or "hub_model_id" in script_or_cmd + ) + + +async def record_hf_job_submit( + session: Any, + job: Any, + args: dict, + *, + image: str, + job_type: str, +) -> float: + """Emit ``hf_job_submit``. Returns the monotonic start timestamp so the + caller can pass it back into :func:`record_hf_job_complete`.""" + from agent.core.session import Event + t_start = time.monotonic() + try: + script_text = args.get("script") or args.get("command") or "" + await session.send_event(Event( + event_type="hf_job_submit", + data={ + "job_id": getattr(job, "id", None), + "job_url": getattr(job, "url", None), + "flavor": args.get("hardware_flavor", "cpu-basic"), + "timeout": args.get("timeout", "30m"), + "job_type": job_type, + "image": image, + "push_to_hub": _infer_push_to_hub(script_text), + }, + )) + except Exception as e: + logger.debug("record_hf_job_submit failed (non-fatal): %s", e) + return t_start + + +async def record_hf_job_complete( + session: Any, + job: Any, + *, + flavor: str, + final_status: str, + submit_ts: float, +) -> None: + from agent.core.session import Event + try: + wall_time_s = int(time.monotonic() - submit_ts) + await session.send_event(Event( + event_type="hf_job_complete", + data={ + "job_id": getattr(job, "id", None), + "flavor": flavor, + "final_status": final_status, + "wall_time_s": wall_time_s, + }, + )) + except Exception as e: + logger.debug("record_hf_job_complete failed (non-fatal): %s", e) + + +# ── sandbox ───────────────────────────────────────────────────────────────── + +async def record_sandbox_create( + session: Any, + sandbox: Any, + *, + hardware: str, + create_latency_s: int, +) -> None: + from agent.core.session import Event + try: + # Pin created-at on the session so record_sandbox_destroy can diff. + session._sandbox_created_at = time.monotonic() - create_latency_s + await session.send_event(Event( + event_type="sandbox_create", + data={ + "sandbox_id": getattr(sandbox, "space_id", None), + "hardware": hardware, + "create_latency_s": int(create_latency_s), + }, + )) + except Exception as e: + logger.debug("record_sandbox_create failed (non-fatal): %s", e) + + +async def record_sandbox_destroy(session: Any, sandbox: Any) -> None: + from agent.core.session import Event + try: + created = getattr(session, "_sandbox_created_at", None) + lifetime_s = int(time.monotonic() - created) if created else None + await session.send_event(Event( + event_type="sandbox_destroy", + data={ + "sandbox_id": getattr(sandbox, "space_id", None), + "lifetime_s": lifetime_s, + }, + )) + except Exception as e: + logger.debug("record_sandbox_destroy failed (non-fatal): %s", e) + + +# ── feedback ─────────────────────────────────────────────────────────────── + +async def record_feedback( + session: Any, + *, + rating: str, + turn_index: int | None = None, + message_id: str | None = None, + comment: str | None = None, +) -> None: + from agent.core.session import Event + try: + await session.send_event(Event( + event_type="feedback", + data={ + "rating": rating, + "turn_index": turn_index, + "message_id": message_id, + "comment": (comment or "")[:500], + }, + )) + except Exception as e: + logger.debug("record_feedback failed (non-fatal): %s", e) + + +# ── heartbeat ────────────────────────────────────────────────────────────── + +# Module-level reference set for fire-and-forget heartbeat tasks. asyncio only +# keeps *weak* references to tasks, so the returned Task would otherwise be +# eligible for GC before running — the task gets discarded and the upload +# silently never happens. Hold strong refs until the task completes. +_heartbeat_tasks: set[asyncio.Task] = set() + + +class HeartbeatSaver: + """Time-gated mid-turn flush. + + Called from ``Session.send_event`` after every event. Fires + ``save_and_upload_detached`` in a worker thread at most once per + ``heartbeat_interval_s`` (default 60s). Guards against losing trace data + on long-running turns that crash before ``turn_complete``. + """ + + @staticmethod + def maybe_fire(session: Any) -> None: + if not getattr(session.config, "save_sessions", False): + return + interval = getattr(session.config, "heartbeat_interval_s", 0) or 0 + if interval <= 0: + return + now = time.monotonic() + last = getattr(session, "_last_heartbeat_ts", None) + if last is None: + # Initialise on first event; no save yet. + session._last_heartbeat_ts = now + return + if now - last < interval: + return + session._last_heartbeat_ts = now + repo_id = session.config.session_dataset_repo + try: + task = asyncio.get_running_loop().create_task( + asyncio.to_thread(session.save_and_upload_detached, repo_id) + ) + # Hold a strong reference until the task finishes so asyncio can't + # GC it. ``set.discard`` is a no-op on missing keys → safe callback. + _heartbeat_tasks.add(task) + task.add_done_callback(_heartbeat_tasks.discard) + except RuntimeError: + try: + session.save_and_upload_detached(repo_id) + except Exception as e: + logger.debug("Heartbeat save failed (non-fatal): %s", e) diff --git a/agent/sft/__init__.py b/agent/sft/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/agent/sft/tagger.py b/agent/sft/tagger.py new file mode 100644 index 000000000..7c47434d9 --- /dev/null +++ b/agent/sft/tagger.py @@ -0,0 +1,324 @@ +"""Derive tags for a session trajectory. + +``tag_session(trajectory)`` → ``list[str]``. Pure function. No filtering, no +mutation — tags are purely metadata so downstream pipelines can slice the raw +SFT dataset (``where 'hf_job:succeeded' in tags``) without re-reading trajectories. + +Tag namespaces (all tags are ``":"`` strings): + +* ``tool:`` — every tool called at least once (``tool:hf_jobs``, …) +* ``outcome:`` — ``completed`` / ``errored`` / ``interrupted`` / + ``ongoing`` / ``doom_loop`` / ``context_exceeded`` +* ``hf_job:`` — ``submitted``, ``succeeded``, ``failed``, + ``multi`` (>1), ``oom``, ``push_to_hub`` +* ``gpu:`` — ``none``, ``t4``, ``a10g``, ``a100``, ``l40s``, + ``h100``, plus ``gpu:multi`` for x2/x4/x8 flavors +* ``sandbox:`` — ``created``, ``gpu``, ``cpu``, ``long_lived`` (>30 min) +* ``feedback:`` — ``up``, ``down``, ``mixed``, ``none`` +* ``model:`` — ``opus`` / ``sonnet`` / ``haiku`` / ``kimi`` / + ``gpt`` / ``deepseek`` / ``qwen`` / ``other`` +* ``turns:`` — ``short`` (<5) / ``medium`` (5–20) / ``long`` (>20) +* ``cost:`` — ``low`` (<$0.10) / ``med`` (<$1) / ``high`` +* ``task:`` — ``training`` / ``inference`` / ``data_prep`` / + ``research_only`` (heuristic on tools + scripts) + +Tags are deduplicated before returning. +""" + +from __future__ import annotations + +from typing import Any, Iterable + +# Flavor → GPU-family mapping. Keep conservative; unknown flavors → "none". +_GPU_FAMILY = { + "cpu-basic": "none", "cpu-upgrade": "none", + "t4-small": "t4", "t4-medium": "t4", + "l4x1": "l40s", "l4x4": "l40s", + "l40sx1": "l40s", "l40sx4": "l40s", "l40sx8": "l40s", + "a10g-small": "a10g", "a10g-large": "a10g", + "a10g-largex2": "a10g", "a10g-largex4": "a10g", + "a100-large": "a100", "a100x2": "a100", + "a100x4": "a100", "a100x8": "a100", + "h100": "h100", "h100x8": "h100", +} + +# Substrings that count a flavor as multi-GPU. +_MULTI_GPU_MARKERS = ("x2", "x4", "x8") + +# Tool names that don't touch training/inference or sandbox/jobs. If a session +# only used these, we tag it research_only. +_RESEARCH_ONLY_TOOLS = { + "research", "github_find_examples", "github_read_file", "github_list_repos", + "hf_papers", "explore_hf_docs", "fetch_hf_docs", "hub_repo_details", + "plan", "hf_inspect_dataset", "web_search", +} + +# Tool names that signal data manipulation workflows. +_DATA_PREP_TOOLS = {"hf_inspect_dataset", "dataset_tools", "hub_repo_details"} + + +def _model_family(model_name: str | None) -> str: + if not model_name: + return "other" + n = model_name.lower() + if "opus" in n: + return "opus" + if "sonnet" in n: + return "sonnet" + if "haiku" in n: + return "haiku" + if "kimi" in n: + return "kimi" + if "gpt" in n: + return "gpt" + if "deepseek" in n: + return "deepseek" + if "qwen" in n: + return "qwen" + if "llama" in n: + return "llama" + return "other" + + +def _turns_bucket(n: int) -> str: + if n < 5: + return "short" + if n <= 20: + return "medium" + return "long" + + +def _cost_bucket(cost_usd: float) -> str: + if cost_usd < 0.10: + return "low" + if cost_usd < 1.0: + return "med" + return "high" + + +def _flavor_to_gpu_tags(flavor: str) -> list[str]: + family = _GPU_FAMILY.get(flavor, "none") + tags = [f"gpu:{family}"] + if any(m in flavor for m in _MULTI_GPU_MARKERS): + tags.append("gpu:multi") + return tags + + +def _has_oom_signal(tool_outputs: Iterable[str]) -> bool: + for out in tool_outputs: + if not isinstance(out, str): + continue + low = out.lower() + if "outofmemoryerror" in low or "cuda out of memory" in low or "oom" in low: + return True + return False + + +def _infer_task_tag( + tool_names: set[str], + hf_job_submit_scripts: list[str], +) -> str | None: + """Return a ``task:*`` tag or None if we can't tell. + + Heuristic order: training > inference > data_prep > research_only. + """ + # training: any hf_jobs script with a Trainer/SFT/training keyword, OR uses + # hf_jobs at all and a script mentions training APIs. + for script in hf_job_submit_scripts: + low = script.lower() + if any(k in low for k in ( + "sftconfig", "sfttrainer", "trainer(", "trainingarguments", + "grpo", "dpo", ".train(", "transformers import", + "trainer import", "fine-tune", "finetune", + )): + return "training" + + # inference: sessions that use inference tools but never hf_jobs/sandbox + uses_compute = bool(tool_names & {"hf_jobs", "sandbox_create", "sandbox_exec"}) + if not uses_compute and tool_names & {"inference", "generate", "run_inference"}: + return "inference" + + # data_prep: primarily dataset tools and no training/inference + if tool_names & _DATA_PREP_TOOLS and not uses_compute: + return "data_prep" + + # research_only: every tool used is in the research allow-list + if tool_names and tool_names <= _RESEARCH_ONLY_TOOLS: + return "research_only" + + return None + + +def tag_session(trajectory: dict) -> list[str]: + """Derive tags from a session trajectory. Pure function.""" + tags: set[str] = set() + + events: list[dict] = trajectory.get("events") or [] + messages: list[dict] = trajectory.get("messages") or [] + model_name: str | None = trajectory.get("model_name") + + # model + tags.add(f"model:{_model_family(model_name)}") + + # turns + user_turns = sum(1 for m in messages if m.get("role") == "user") + tags.add(f"turns:{_turns_bucket(user_turns)}") + + # cost + tool-name enumeration + outcome detection + cost_usd = 0.0 + tool_names: set[str] = set() + tool_outputs: list[str] = [] + hf_job_submit_count = 0 + hf_job_submit_scripts: list[str] = [] + hf_job_success_count = 0 + hf_job_fail_count = 0 + hf_job_push_to_hub = False + gpu_tags_seen: set[str] = set() + + # Outcome is the *last* terminal signal. Seed with "ongoing" — overridden + # if we see a terminal event. + outcome = "ongoing" + had_error = False + had_doom_loop = False + had_compact = False + + feedback_up = 0 + feedback_down = 0 + + sandbox_created = False + sandbox_hardware: str | None = None + sandbox_lifetime_s: int | None = None + + for ev in events: + et = ev.get("event_type") + data = ev.get("data") or {} + + if et == "llm_call": + cost_usd += float(data.get("cost_usd") or 0.0) + + elif et == "tool_call": + name = data.get("tool") + if name: + tool_names.add(name) + + elif et == "tool_output": + out = data.get("output") + if isinstance(out, str): + tool_outputs.append(out) + + elif et == "hf_job_submit": + hf_job_submit_count += 1 + if data.get("push_to_hub"): + hf_job_push_to_hub = True + flavor = data.get("flavor") or "cpu-basic" + for t in _flavor_to_gpu_tags(flavor): + gpu_tags_seen.add(t) + + elif et == "hf_job_complete": + final = (data.get("final_status") or "").lower() + if final in ("completed", "succeeded", "success"): + hf_job_success_count += 1 + elif final in ("failed", "error", "timeout", "cancelled"): + hf_job_fail_count += 1 + + elif et == "sandbox_create": + sandbox_created = True + sandbox_hardware = data.get("hardware") + + elif et == "sandbox_destroy": + lt = data.get("lifetime_s") + if isinstance(lt, (int, float)): + sandbox_lifetime_s = int(lt) + + elif et == "feedback": + rating = data.get("rating") + if rating == "up": + feedback_up += 1 + elif rating == "down": + feedback_down += 1 + + elif et == "error": + had_error = True + elif et == "turn_complete": + if not had_error: + outcome = "completed" + elif et == "interrupted": + outcome = "interrupted" + elif et == "compacted": + had_compact = True + elif et == "tool_log": + log_text = (data.get("log") or "").lower() + if "doom loop" in log_text: + had_doom_loop = True + + if had_error and outcome not in ("completed", "interrupted"): + outcome = "errored" + + tags.add(f"outcome:{outcome}") + if had_doom_loop: + tags.add("outcome:doom_loop") + if had_compact: + tags.add("outcome:context_exceeded") + + # tools + for name in tool_names: + tags.add(f"tool:{name}") + + # hf_jobs facets + if hf_job_submit_count >= 1: + tags.add("hf_job:submitted") + if hf_job_submit_count > 1: + tags.add("hf_job:multi") + if hf_job_success_count > 0: + tags.add("hf_job:succeeded") + if hf_job_fail_count > 0: + tags.add("hf_job:failed") + if hf_job_push_to_hub: + tags.add("hf_job:push_to_hub") + if _has_oom_signal(tool_outputs): + tags.add("hf_job:oom") + + # gpu tags (from all submitted jobs) + tags.update(gpu_tags_seen) + if "gpu:none" in tags and len(gpu_tags_seen) > 1: + # If any GPU flavor was used, drop the "none" tag for clarity. + tags.discard("gpu:none") + + # sandbox facets + if sandbox_created: + tags.add("sandbox:created") + if sandbox_hardware: + fam = _GPU_FAMILY.get(sandbox_hardware, "none") + tags.add("sandbox:cpu" if fam == "none" else "sandbox:gpu") + if sandbox_lifetime_s is not None and sandbox_lifetime_s > 1800: + tags.add("sandbox:long_lived") + + # feedback + if feedback_up and feedback_down: + tags.add("feedback:mixed") + elif feedback_up: + tags.add("feedback:up") + elif feedback_down: + tags.add("feedback:down") + else: + tags.add("feedback:none") + + # cost bucket + tags.add(f"cost:{_cost_bucket(cost_usd)}") + + # task heuristic (needs scripts — pull from the hf_job_submit events' + # matching tool_call arguments in the event list). + for ev in events: + if ev.get("event_type") == "tool_call": + data = ev.get("data") or {} + if data.get("tool") == "hf_jobs": + args = data.get("arguments") or {} + script = args.get("script") or args.get("command") or "" + if isinstance(script, str): + hf_job_submit_scripts.append(script) + + task_tag = _infer_task_tag(tool_names, hf_job_submit_scripts) + if task_tag: + tags.add(f"task:{task_tag}") + + return sorted(tags) diff --git a/agent/tools/jobs_tool.py b/agent/tools/jobs_tool.py index 2c6ebf6c7..474ee4cc7 100644 --- a/agent/tools/jobs_tool.py +++ b/agent/tools/jobs_tool.py @@ -528,14 +528,16 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: job_type = "Docker" # Run the job + flavor = args.get("hardware_flavor", "cpu-basic") + timeout_str = args.get("timeout", "30m") job = await _async_call( self.api.run_job, image=image, command=command, env=_add_default_env(args.get("env")), secrets=_add_environment_variables(args.get("secrets"), self.hf_token), - flavor=args.get("hardware_flavor", "cpu-basic"), - timeout=args.get("timeout", "30m"), + flavor=flavor, + timeout=timeout_str, namespace=self.namespace, ) @@ -557,6 +559,16 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: ) ) + # Telemetry: job submission + completion (infra consumption signal). + submit_ts = None + if self.session: + from agent.core import telemetry + submit_ts = await telemetry.record_hf_job_submit( + self.session, job, + {**args, "hardware_flavor": flavor, "timeout": timeout_str}, + image=image, job_type=job_type, + ) + # Wait for completion and stream logs logger.info(f"{job_type} job started: {job.url}") logger.info("Streaming logs...") @@ -566,6 +578,13 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: namespace=self.namespace, ) + if self.session and submit_ts is not None: + from agent.core import telemetry + await telemetry.record_hf_job_complete( + self.session, job, + flavor=flavor, final_status=final_status, submit_ts=submit_ts, + ) + # Untrack job ID (completed or failed, no longer needs cancellation) if self.session: self.session._running_job_ids.discard(job.id) diff --git a/agent/tools/sandbox_tool.py b/agent/tools/sandbox_tool.py index 74c6a7885..be38c7340 100644 --- a/agent/tools/sandbox_tool.py +++ b/agent/tools/sandbox_tool.py @@ -131,6 +131,8 @@ async def _watch_cancel(): } if hardware != "cpu-basic": kwargs["sleep_time"] = 2700 + import time as _t + _t_start = _t.monotonic() try: sb = await asyncio.to_thread(Sandbox.create, **kwargs) except Sandbox.Cancelled: @@ -139,6 +141,13 @@ async def _watch_cancel(): watcher_task.cancel() session.sandbox = sb + # Telemetry: sandbox creation (infra consumption signal) + from agent.core import telemetry + await telemetry.record_sandbox_create( + session, sb, hardware=hardware, + create_latency_s=int(_t.monotonic() - _t_start), + ) + # Set a descriptive title (template title is inherited on duplicate) from huggingface_hub import metadata_update diff --git a/backend/kpis_scheduler.py b/backend/kpis_scheduler.py new file mode 100644 index 000000000..f044c8ee5 --- /dev/null +++ b/backend/kpis_scheduler.py @@ -0,0 +1,146 @@ +"""In-process hourly KPI rollup, owned by the backend Space lifespan. + +Replaces an external GitHub Actions cron so the rollup lives next to the data +and reuses the Space's existing HF token — no production secrets on the +public source repo. See ``scripts/build_kpis.py`` for the data-flow diagram +and metric definitions. + +Behaviour:: + + lifespan startup → start APScheduler with cron("5 * * * *", UTC) + → fire a best-effort 6-hour backfill (fire-and-forget) + each :05 → run ``build_kpis.run_for_hour`` for the just-completed hour + lifespan shutdown → scheduler.shutdown(wait=False) + +Environment:: + + HF_KPI_WRITE_TOKEN | HF_SESSION_UPLOAD_TOKEN | HF_TOKEN | HF_ADMIN_TOKEN + First one found is used. Least-privilege first. + KPI_SOURCE_REPO default smolagents/ml-intern-sessions + KPI_TARGET_REPO default smolagents/ml-intern-kpis + ML_INTERN_KPIS_DISABLED if truthy, the scheduler is not started +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import logging +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent + +# Hold strong refs to backfill tasks so asyncio doesn't GC them mid-run. +_background_tasks: set[asyncio.Task] = set() + +_scheduler = None # AsyncIOScheduler instance (lazy import) + + +def _resolve_token() -> Optional[str]: + """Pick the first available HF token. Least-privilege first.""" + for var in ( + "HF_KPI_WRITE_TOKEN", + "HF_SESSION_UPLOAD_TOKEN", + "HF_TOKEN", + "HF_ADMIN_TOKEN", + ): + val = os.environ.get(var) + if val: + return val + return None + + +def _load_build_kpis(): + """Import ``scripts/build_kpis.py`` without putting ``scripts/`` on sys.path.""" + spec = importlib.util.spec_from_file_location( + "build_kpis", _PROJECT_ROOT / "scripts" / "build_kpis.py", + ) + mod = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(mod) + return mod + + +async def _run_hour(hour_dt: datetime) -> None: + """Run one hourly rollup off the event loop. Best-effort, never raises.""" + token = _resolve_token() + if not token: + logger.warning("kpis_scheduler: no HF token available, skipping %s", hour_dt) + return + try: + mod = _load_build_kpis() + from huggingface_hub import HfApi + api = HfApi() + source = os.environ.get("KPI_SOURCE_REPO", "smolagents/ml-intern-sessions") + target = os.environ.get("KPI_TARGET_REPO", "smolagents/ml-intern-kpis") + await asyncio.to_thread(mod.run_for_hour, api, source, target, hour_dt, token) + except Exception as e: + logger.warning("kpis_scheduler: rollup for %s failed: %s", hour_dt, e) + + +async def run_last_completed_hour() -> None: + """The scheduled-at-:05 job. Rolls up the previous whole hour.""" + now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + await _run_hour(now - timedelta(hours=1)) + + +async def backfill(hours: int = 6) -> None: + """Catch-up pass for hours the Space was down. Idempotent (overwrites).""" + now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + for i in range(1, hours + 1): + await _run_hour(now - timedelta(hours=i)) + + +def start(backfill_hours: int = 6) -> None: + """Called from FastAPI lifespan startup.""" + global _scheduler + if os.environ.get("ML_INTERN_KPIS_DISABLED"): + logger.info("kpis_scheduler: disabled via ML_INTERN_KPIS_DISABLED") + return + if _scheduler is not None: + return + + try: + from apscheduler.schedulers.asyncio import AsyncIOScheduler + from apscheduler.triggers.cron import CronTrigger + except ImportError: + logger.warning("kpis_scheduler: apscheduler not installed, skipping") + return + + _scheduler = AsyncIOScheduler(timezone="UTC") + _scheduler.add_job( + run_last_completed_hour, + CronTrigger(minute=5), + id="kpis_hourly", + misfire_grace_time=600, # tolerate a 10-min misfire window + coalesce=True, # collapse multiple missed fires into one + max_instances=1, + replace_existing=True, + ) + _scheduler.start() + logger.info("kpis_scheduler: started (cron '5 * * * *' UTC)") + + # Non-blocking backfill. Hold a strong ref until done so asyncio doesn't + # GC the task before it finishes. + try: + task = asyncio.get_running_loop().create_task(backfill(backfill_hours)) + _background_tasks.add(task) + task.add_done_callback(_background_tasks.discard) + except RuntimeError: + # Not in an event loop (tests); skip backfill. + pass + + +async def shutdown() -> None: + """Called from FastAPI lifespan shutdown.""" + global _scheduler + if _scheduler is None: + return + _scheduler.shutdown(wait=False) + _scheduler = None + logger.info("kpis_scheduler: stopped") diff --git a/backend/main.py b/backend/main.py index 888740e53..9aa939a08 100644 --- a/backend/main.py +++ b/backend/main.py @@ -27,8 +27,37 @@ async def lifespan(app: FastAPI): """Application lifespan handler.""" logger.info("Starting HF Agent backend...") + # Start in-process hourly KPI rollup. Replaces an external cron so the + # rollup lives next to the data and reuses the Space's HF token. + try: + import kpis_scheduler + kpis_scheduler.start() + except Exception as e: + logger.warning("KPI scheduler failed to start: %s", e) + yield + logger.info("Shutting down HF Agent backend...") + try: + import kpis_scheduler + await kpis_scheduler.shutdown() + except Exception as e: + logger.warning("KPI scheduler shutdown failed: %s", e) + + # Final-flush: save every still-active session so we don't lose traces on + # server restart. Uploads are detached subprocesses — this is fast. + try: + from session_manager import session_manager + for sid, agent_session in list(session_manager.sessions.items()): + sess = agent_session.session + if sess.config.save_sessions: + try: + sess.save_and_upload_detached(sess.config.session_dataset_repo) + logger.info("Flushed session %s on shutdown", sid) + except Exception as e: + logger.warning("Failed to flush session %s: %s", sid, e) + except Exception as e: + logger.warning("Lifespan final-flush skipped: %s", e) app = FastAPI( diff --git a/backend/routes/agent.py b/backend/routes/agent.py index 7f5779952..0224f258a 100644 --- a/backend/routes/agent.py +++ b/backend/routes/agent.py @@ -693,3 +693,41 @@ async def shutdown_session( return {"status": "shutdown_requested", "session_id": session_id} +@router.post("/feedback/{session_id}") +async def submit_feedback( + session_id: str, + body: dict, + user: dict = Depends(get_current_user), +) -> dict: + """Attach a user feedback signal to a session's event log. + + Body: {rating: "up"|"down"|"outcome_success"|"outcome_fail", + turn_index?: int, comment?: str, message_id?: str} + Appended as a `feedback` event and saved with the session trajectory. + """ + _check_session_access(session_id, user) + agent_session = session_manager.sessions.get(session_id) + if not agent_session: + raise HTTPException(status_code=404, detail="Session not found") + + rating = body.get("rating") + if rating not in {"up", "down", "outcome_success", "outcome_fail"}: + raise HTTPException(status_code=400, detail="invalid rating") + + from agent.core import telemetry + await telemetry.record_feedback( + agent_session.session, + rating=rating, + turn_index=body.get("turn_index"), + message_id=body.get("message_id"), + comment=body.get("comment"), + ) + # Fire-and-forget save so feedback reaches the dataset even if the user + # closes the tab right after clicking. + if agent_session.session.config.save_sessions: + agent_session.session.save_and_upload_detached( + agent_session.session.config.session_dataset_repo + ) + return {"status": "ok"} + + diff --git a/backend/session_manager.py b/backend/session_manager.py index 7293f9cf3..d52cd2754 100644 --- a/backend/session_manager.py +++ b/backend/session_manager.py @@ -290,11 +290,14 @@ async def _cleanup_sandbox(session: Session) -> None: """Delete the sandbox Space if one was created for this session.""" sandbox = getattr(session, "sandbox", None) if sandbox and getattr(sandbox, "_owns_space", False): + space_id = getattr(sandbox, "space_id", None) try: - logger.info(f"Deleting sandbox {sandbox.space_id}...") + logger.info(f"Deleting sandbox {space_id}...") await asyncio.to_thread(sandbox.delete) + from agent.core import telemetry + await telemetry.record_sandbox_destroy(session, sandbox) except Exception as e: - logger.warning(f"Failed to delete sandbox {sandbox.space_id}: {e}") + logger.warning(f"Failed to delete sandbox {space_id}: {e}") async def _run_session( self, @@ -356,6 +359,15 @@ async def _run_session( await self._cleanup_sandbox(session) + # Final-flush: always save on session death so we capture ended + # sessions even if the client disconnects without /shutdown. + # Idempotent via session_id key; detached subprocess. + if session.config.save_sessions: + try: + session.save_and_upload_detached(session.config.session_dataset_repo) + except Exception as e: + logger.warning(f"Final-flush failed for {session_id}: {e}") + async with self._lock: if session_id in self.sessions: self.sessions[session_id].is_active = False diff --git a/configs/main_agent_config.json b/configs/main_agent_config.json index af76608f3..c73ea380f 100644 --- a/configs/main_agent_config.json +++ b/configs/main_agent_config.json @@ -1,7 +1,7 @@ { "model_name": "bedrock/us.anthropic.claude-opus-4-6-v1", "save_sessions": true, - "session_dataset_repo": "akseljoonas/hf-agent-sessions", + "session_dataset_repo": "smolagents/ml-intern-sessions", "yolo_mode": false, "confirm_cpu_jobs": true, "auto_file_upload": true, diff --git a/frontend/src/components/Chat/AssistantMessage.tsx b/frontend/src/components/Chat/AssistantMessage.tsx index 83bd8cae5..91c7b8c10 100644 --- a/frontend/src/components/Chat/AssistantMessage.tsx +++ b/frontend/src/components/Chat/AssistantMessage.tsx @@ -1,13 +1,19 @@ -import { useMemo } from 'react'; -import { Box, Stack, Typography } from '@mui/material'; +import { useMemo, useState } from 'react'; +import { Box, IconButton, Stack, Tooltip, Typography } from '@mui/material'; +import ThumbUpOutlined from '@mui/icons-material/ThumbUpOutlined'; +import ThumbUp from '@mui/icons-material/ThumbUp'; +import ThumbDownOutlined from '@mui/icons-material/ThumbDownOutlined'; +import ThumbDown from '@mui/icons-material/ThumbDown'; import MarkdownContent from './MarkdownContent'; import ToolCallGroup from './ToolCallGroup'; +import { apiFetch } from '@/utils/api'; import type { UIMessage } from 'ai'; import type { MessageMeta } from '@/types/agent'; interface AssistantMessageProps { message: UIMessage; isStreaming?: boolean; + sessionId?: string | null; approveTools: (approvals: Array<{ tool_call_id: string; approved: boolean; feedback?: string | null }>) => Promise; } @@ -43,8 +49,27 @@ function groupParts(parts: UIMessage['parts']) { return groups; } -export default function AssistantMessage({ message, isStreaming = false, approveTools }: AssistantMessageProps) { +export default function AssistantMessage({ message, isStreaming = false, sessionId, approveTools }: AssistantMessageProps) { const groups = useMemo(() => groupParts(message.parts), [message.parts]); + const [feedback, setFeedback] = useState<'up' | 'down' | null>(null); + const [feedbackBusy, setFeedbackBusy] = useState(false); + + const sendFeedback = async (rating: 'up' | 'down') => { + if (!sessionId || feedbackBusy) return; + setFeedbackBusy(true); + // Optimistic toggle — feedback is observability, not a hard requirement. + setFeedback(rating); + try { + await apiFetch(`/api/feedback/${sessionId}`, { + method: 'POST', + body: JSON.stringify({ rating, message_id: message.id }), + }); + } catch { + // Silently swallow — don't block chat UX on a telemetry write. + } finally { + setFeedbackBusy(false); + } + }; // Find the last text group index for streaming cursor let lastTextIdx = -1; @@ -114,6 +139,24 @@ export default function AssistantMessage({ message, isStreaming = false, approve return null; })} + {!isStreaming && sessionId && ( + + + sendFeedback('up')}> + {feedback === 'up' ? : } + + + + sendFeedback('down')}> + {feedback === 'down' ? : } + + + + )} ); } diff --git a/frontend/src/components/Chat/MessageBubble.tsx b/frontend/src/components/Chat/MessageBubble.tsx index ede0f8a7f..ab971205c 100644 --- a/frontend/src/components/Chat/MessageBubble.tsx +++ b/frontend/src/components/Chat/MessageBubble.tsx @@ -9,6 +9,7 @@ interface MessageBubbleProps { onEditAndRegenerate?: (messageId: string, newText: string) => void | Promise; isProcessing?: boolean; isStreaming?: boolean; + sessionId?: string | null; approveTools: (approvals: Array<{ tool_call_id: string; approved: boolean; feedback?: string | null }>) => Promise; } @@ -19,6 +20,7 @@ export default function MessageBubble({ onEditAndRegenerate, isProcessing = false, isStreaming = false, + sessionId, approveTools, }: MessageBubbleProps) { if (message.role === 'user') { @@ -38,6 +40,7 @@ export default function MessageBubble({ ); diff --git a/frontend/src/components/Chat/MessageList.tsx b/frontend/src/components/Chat/MessageList.tsx index b50a66626..5e3efcaea 100644 --- a/frontend/src/components/Chat/MessageList.tsx +++ b/frontend/src/components/Chat/MessageList.tsx @@ -8,6 +8,7 @@ import type { UIMessage } from 'ai'; interface MessageListProps { messages: UIMessage[]; isProcessing: boolean; + sessionId?: string | null; approveTools: (approvals: Array<{ tool_call_id: string; approved: boolean; feedback?: string | null }>) => Promise; onUndoLastTurn: () => void | Promise; onEditAndRegenerate?: (messageId: string, newText: string) => void | Promise; @@ -57,7 +58,7 @@ function WelcomeGreeting() { ); } -export default function MessageList({ messages, isProcessing, approveTools, onUndoLastTurn, onEditAndRegenerate }: MessageListProps) { +export default function MessageList({ messages, isProcessing, sessionId, approveTools, onUndoLastTurn, onEditAndRegenerate }: MessageListProps) { const scrollContainerRef = useRef(null); const stickToBottom = useRef(true); @@ -139,6 +140,7 @@ export default function MessageList({ messages, isProcessing, approveTools, onUn onEditAndRegenerate={onEditAndRegenerate} isProcessing={isProcessing} isStreaming={isProcessing && msg.id === lastAssistantId} + sessionId={sessionId} approveTools={approveTools} /> )) diff --git a/frontend/src/components/SessionChat.tsx b/frontend/src/components/SessionChat.tsx index adc98876c..8f1823806 100644 --- a/frontend/src/components/SessionChat.tsx +++ b/frontend/src/components/SessionChat.tsx @@ -102,6 +102,7 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess =0.32.0", "httpx>=0.27.0", "websockets>=13.0", + "apscheduler>=3.10,<4", ] [project.optional-dependencies] diff --git a/scripts/build_kpis.py b/scripts/build_kpis.py new file mode 100644 index 000000000..6fcda8753 --- /dev/null +++ b/scripts/build_kpis.py @@ -0,0 +1,517 @@ +#!/usr/bin/env python3 +"""Hourly KPI rollup for the session-trajectory dataset. + +================================================================================ + Data flow +================================================================================ + + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” heartbeat ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” + │ agent (CLI/web) │ ───────────────▶ │ hf-agent-sessions (dataset) │ + │ Session.send_event│ │ sessions/YYYY-MM-DD/.jsonl│ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + │ cron @:05 each hour + ā–¼ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” + │ scripts/build_kpis.py │ + │ (GitHub Actions) │ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”¬ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + │ upload CSV + ā–¼ + ā”Œā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā” + │ hf-agent-kpis (dataset) │ + │ hourly/YYYY-MM-DD/HH.csv │ + ā””ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”˜ + +Each hourly run reads today's + yesterday's session folders (to cover sessions +that crossed midnight), filters events into the target hour window +``[hour, hour+1h)``, computes aggregates, and writes one CSV at +``hourly//.csv`` in the target dataset. Uploads are idempotent — +re-running the same hour overwrites. + +================================================================================ + Metrics (one row per hour) +================================================================================ + + sessions — distinct session_ids with ≄1 event in window + users — distinct user ids (when present on session rows) + turns — sum of user-message counts across active sessions + llm_calls — count of llm_call events + tokens_prompt / _completion / _cache_read / _cache_creation + cost_usd — sum of llm_call.cost_usd + cache_hit_ratio — cache_read / (cache_read + prompt) + tool_success_rate — tool_output success=True / total tool_output + failure_rate — sessions that ended with an `error` event / sessions + regenerate_rate — sessions with any `undo_complete` event / sessions + time_to_first_action_s_p50 / _p95 — from session_start to first tool_call + thumbs_up / thumbs_down + hf_jobs_submitted / _succeeded + gpu_hours_by_flavor_json — JSON-serialised {flavor: gpu-hours} + +================================================================================ + Usage +================================================================================ + + # Run for the most recently completed hour (default — the cron path): + python scripts/build_kpis.py + + # Backfill last 24 hours: + python scripts/build_kpis.py --hours 24 + + # Explicit hour (UTC): + python scripts/build_kpis.py --datetime 2026-04-24T14 + +Env: + HF_TOKEN (or HF_KPI_WRITE_TOKEN) — write access to the target dataset. + +================================================================================ + Deploy +================================================================================ + +See ``.github/workflows/build-kpis.yml`` — runs every hour at :05. To provision: + + 1. Create the target dataset (once): + huggingface-cli repo create hf-agent-kpis --type dataset + 2. Put ``HF_KPI_WRITE_TOKEN`` (or ``HF_TOKEN``) into repo Actions secrets. + 3. Merge this file; the first scheduled run fires within the hour. +""" + +from __future__ import annotations + +import argparse +import io +import json +import logging +import os +import sys +import tempfile +from collections import defaultdict +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable + +logger = logging.getLogger("build_kpis") + +# Rough gpu-hour pricing for hf_jobs flavor strings. Keep conservative; used +# only to compute gpu-hours (not dollars) — wall_time_s * flavor_gpu_count. +_FLAVOR_GPU_COUNT = { + "cpu-basic": 0, "cpu-upgrade": 0, + "t4-small": 1, "t4-medium": 1, + "l4x1": 1, "l4x4": 4, + "l40sx1": 1, "l40sx4": 4, "l40sx8": 8, + "a10g-small": 1, "a10g-large": 1, "a10g-largex2": 2, "a10g-largex4": 4, + "a100-large": 1, "a100x2": 2, "a100x4": 4, "a100x8": 8, + "h100": 1, "h100x8": 8, +} + + +def _percentile(values: list[float], p: float) -> float: + if not values: + return 0.0 + values = sorted(values) + k = (len(values) - 1) * p + f = int(k) + c = min(f + 1, len(values) - 1) + if f == c: + return float(values[f]) + return float(values[f] + (values[c] - values[f]) * (k - f)) + + +def _parse_ts(s: Any) -> datetime | None: + if not s or not isinstance(s, str): + return None + try: + dt = datetime.fromisoformat(s) + except Exception: + return None + # Normalise to aware UTC so comparisons work against window bounds. + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + + +def _iter_session_files(api, repo_id: str, day: date, token: str) -> Iterable[str]: + """Yield repo-relative paths for all sessions under ``sessions/YYYY-MM-DD/``.""" + prefix = f"sessions/{day.isoformat()}/" + try: + files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", token=token) + except Exception as e: + logger.warning("list_repo_files(%s) failed: %s", repo_id, e) + return [] + return [f for f in files if f.startswith(prefix) and f.endswith(".jsonl")] + + +def _download_session(repo_id: str, path: str, token: str) -> dict | None: + """Fetch one session JSONL and decode its single row. + + ``hf_hub_download`` caches; second run within the same process / runner + directory is near-free. + """ + from huggingface_hub import hf_hub_download + try: + local = hf_hub_download( + repo_id=repo_id, filename=path, repo_type="dataset", token=token, + ) + except Exception as e: + logger.warning("hf_hub_download(%s) failed: %s", path, e) + return None + try: + with open(local, "r") as f: + line = f.readline().strip() + if not line: + return None + row = json.loads(line) + # Session uploader stores messages/events as JSON strings — unpack. + for key in ("messages", "events", "tools"): + v = row.get(key) + if isinstance(v, str): + try: + row[key] = json.loads(v) + except Exception: + row[key] = [] + return row + except Exception as e: + logger.warning("parse(%s) failed: %s", path, e) + return None + + +def _filter_session_to_window( + session: dict, start: datetime, end: datetime, +) -> dict | None: + """Return a copy of ``session`` whose events are only those in ``[start, end)``. + + ``None`` if no event falls in the window — the caller drops the session + from this hour's aggregate. + """ + events = session.get("events") or [] + in_window = [] + for ev in events: + ts = _parse_ts(ev.get("timestamp")) + if ts is None: + continue + if start <= ts < end: + in_window.append(ev) + if not in_window: + return None + return {**session, "events": in_window} + + +def _session_metrics(session: dict) -> dict: + """Reduce a single session trajectory to its KPI contributions. + + Assumes ``events`` are already filtered to the target window by the caller. + """ + # Pre-seed every numeric key so downstream aggregation can sum without + # having to special-case empty sessions. + out: dict = { + "sessions": 0, "turns": 0, "llm_calls": 0, + "tokens_prompt": 0, "tokens_completion": 0, + "tokens_cache_read": 0, "tokens_cache_creation": 0, + "cost_usd": 0.0, + "tool_calls_total": 0, "tool_calls_success": 0, + "failures": 0, "regenerate_sessions": 0, + "thumbs_up": 0, "thumbs_down": 0, + "hf_jobs_submitted": 0, "hf_jobs_succeeded": 0, + "first_tool_s": -1, + } + events = session.get("events") or [] + messages = session.get("messages") or [] + + turn_count = sum(1 for m in messages if m.get("role") == "user") + out["turns"] = turn_count + out["sessions"] = 1 + + tool_success = 0 + tool_total = 0 + had_error = False + had_undo = False + first_tool_ts = None + session_start = session.get("session_start_time") + gpu_hours_by_flavor: dict[str, float] = defaultdict(float) + jobs_submitted = 0 + jobs_succeeded = 0 + thumbs_up = 0 + thumbs_down = 0 + + start_dt = _parse_ts(session_start) + + for ev in events: + et = ev.get("event_type") + data = ev.get("data") or {} + ts = _parse_ts(ev.get("timestamp")) + + if et == "llm_call": + out["llm_calls"] += 1 + out["tokens_prompt"] += int(data.get("prompt_tokens") or 0) + out["tokens_completion"] += int(data.get("completion_tokens") or 0) + out["tokens_cache_read"] += int(data.get("cache_read_tokens") or 0) + out["tokens_cache_creation"] += int(data.get("cache_creation_tokens") or 0) + out["cost_usd"] += float(data.get("cost_usd") or 0.0) + + elif et == "tool_output": + tool_total += 1 + if data.get("success"): + tool_success += 1 + if first_tool_ts is None and ts is not None and start_dt is not None: + first_tool_ts = (ts - start_dt).total_seconds() + + elif et == "tool_call": + if first_tool_ts is None and ts is not None and start_dt is not None: + first_tool_ts = (ts - start_dt).total_seconds() + + elif et == "error": + had_error = True + + elif et == "undo_complete": + had_undo = True + + elif et == "feedback": + rating = data.get("rating") + if rating == "up": + thumbs_up += 1 + elif rating == "down": + thumbs_down += 1 + + elif et == "hf_job_submit": + jobs_submitted += 1 + + elif et == "hf_job_complete": + flavor = data.get("flavor") or "unknown" + status = (data.get("final_status") or "").lower() + wall = float(data.get("wall_time_s") or 0.0) + gpus = _FLAVOR_GPU_COUNT.get(flavor, 0) + gpu_hours_by_flavor[flavor] += wall * gpus / 3600.0 + if status in ("completed", "succeeded", "success"): + jobs_succeeded += 1 + + out["tool_calls_total"] = tool_total + out["tool_calls_success"] = tool_success + out["failures"] = 1 if had_error else 0 + out["regenerate_sessions"] = 1 if had_undo else 0 + out["thumbs_up"] = thumbs_up + out["thumbs_down"] = thumbs_down + out["hf_jobs_submitted"] = jobs_submitted + out["hf_jobs_succeeded"] = jobs_succeeded + out["first_tool_s"] = first_tool_ts if first_tool_ts is not None else -1 + out["_gpu_hours_by_flavor"] = dict(gpu_hours_by_flavor) + out["_user"] = session.get("user_id") or session.get("session_id") + return dict(out) + + +def _aggregate(per_session: list[dict]) -> dict: + """Collapse a bucket's worth of session rollups into the final KPI row.""" + ttfa_values = [s["first_tool_s"] for s in per_session if s.get("first_tool_s", -1) >= 0] + gpu_hours: dict[str, float] = defaultdict(float) + for s in per_session: + for f, h in (s.get("_gpu_hours_by_flavor") or {}).items(): + gpu_hours[f] += h + + total_sessions = sum(s["sessions"] for s in per_session) + total_turns = sum(s["turns"] for s in per_session) + tokens_prompt = sum(s["tokens_prompt"] for s in per_session) + tokens_cache_read = sum(s["tokens_cache_read"] for s in per_session) + tool_total = sum(s["tool_calls_total"] for s in per_session) + tool_success = sum(s["tool_calls_success"] for s in per_session) + + unique_users = {s.get("_user") for s in per_session if s.get("_user")} + + return { + "sessions": total_sessions, + "users": len(unique_users), + "turns": total_turns, + "llm_calls": int(sum(s["llm_calls"] for s in per_session)), + "tokens_prompt": int(tokens_prompt), + "tokens_completion": int(sum(s["tokens_completion"] for s in per_session)), + "tokens_cache_read": int(tokens_cache_read), + "tokens_cache_creation": int(sum(s["tokens_cache_creation"] for s in per_session)), + "cost_usd": round(sum(s["cost_usd"] for s in per_session), 4), + "cache_hit_ratio": round( + tokens_cache_read / (tokens_cache_read + tokens_prompt), 4 + ) if (tokens_cache_read + tokens_prompt) > 0 else 0.0, + "tool_success_rate": round(tool_success / tool_total, 4) if tool_total > 0 else 0.0, + "failure_rate": round( + sum(s["failures"] for s in per_session) / total_sessions, 4 + ) if total_sessions > 0 else 0.0, + "regenerate_rate": round( + sum(s["regenerate_sessions"] for s in per_session) / total_sessions, 4 + ) if total_sessions > 0 else 0.0, + "time_to_first_action_s_p50": round(_percentile(ttfa_values, 0.5), 2), + "time_to_first_action_s_p95": round(_percentile(ttfa_values, 0.95), 2), + "thumbs_up": int(sum(s["thumbs_up"] for s in per_session)), + "thumbs_down": int(sum(s["thumbs_down"] for s in per_session)), + "hf_jobs_submitted": int(sum(s["hf_jobs_submitted"] for s in per_session)), + "hf_jobs_succeeded": int(sum(s["hf_jobs_succeeded"] for s in per_session)), + "gpu_hours_by_flavor_json": json.dumps(dict(gpu_hours), sort_keys=True), + } + + +# Back-compat alias: older tests call _aggregate_day. +_aggregate_day = _aggregate + + +def _csv_cell(v: Any) -> str: + s = str(v) + if "," in s or '"' in s or "\n" in s: + return '"' + s.replace('"', '""') + '"' + return s + + +def _write_csv( + api, row: dict, bucket_key: str, path_in_repo: str, target_repo: str, token: str, +) -> None: + """Render ``row`` to CSV with a leading ``bucket`` column and upload. + + ``bucket_key`` is the hour string (ISO ``YYYY-MM-DDTHH``) or date string; + written as the ``bucket`` column so downstream consumers can union all + CSVs without date-parsing paths. ``api`` is the caller's ``HfApi`` + instance — reused so we don't spin up a fresh one per CSV. + """ + columns = list(row.keys()) + buf = io.StringIO() + buf.write(",".join(["bucket", *columns]) + "\n") + buf.write(",".join([bucket_key, *[_csv_cell(row[c]) for c in columns]]) + "\n") + + with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as tmp: + tmp.write(buf.getvalue()) + tmp_path = tmp.name + + try: + api.create_repo( + repo_id=target_repo, repo_type="dataset", exist_ok=True, token=token, + ) + api.upload_file( + path_or_fileobj=tmp_path, + path_in_repo=path_in_repo, + repo_id=target_repo, + repo_type="dataset", + token=token, + commit_message=f"KPIs for {bucket_key}", + ) + finally: + try: + os.unlink(tmp_path) + except Exception: + pass + + +def run_for_hour( + api, source_repo: str, target_repo: str, hour_dt: datetime, token: str, +) -> dict: + """Roll up one UTC hour [hour_dt, hour_dt+1h). + + Reads today's + yesterday's session folders so sessions that crossed + midnight land in the right hourly bucket. + """ + if hour_dt.tzinfo is None: + hour_dt = hour_dt.replace(tzinfo=timezone.utc) + window_start = hour_dt.replace(minute=0, second=0, microsecond=0) + window_end = window_start + timedelta(hours=1) + + # Sessions partition by session_start_time date. A session that started + # at 23:50 yesterday can still emit events in today's first hours, so we + # look at both folders. + candidate_dates = {window_start.date(), (window_start - timedelta(days=1)).date()} + + per_session: list[dict] = [] + for d in sorted(candidate_dates): + for path in _iter_session_files(api, source_repo, d, token): + sess = _download_session(source_repo, path, token) + if not sess: + continue + windowed = _filter_session_to_window(sess, window_start, window_end) + if windowed is None: + continue + per_session.append(_session_metrics(windowed)) + + if not per_session: + logger.info("No sessions in window %s — skipping", window_start.isoformat()) + return {} + + row = _aggregate(per_session) + bucket_key = window_start.strftime("%Y-%m-%dT%H") + path_in_repo = f"hourly/{window_start.strftime('%Y-%m-%d')}/{window_start.strftime('%H')}.csv" + _write_csv(api, row, bucket_key, path_in_repo, target_repo, token) + logger.info("Wrote KPIs for %s (%d sessions): %s", + bucket_key, per_session and len(per_session), row) + return row + + +# Back-compat for daily backfills — unchanged behaviour. +def run_for_day(api, source_repo: str, target_repo: str, day: date, token: str) -> dict: + paths = _iter_session_files(api, source_repo, day, token) + per_session: list[dict] = [] + for path in paths: + sess = _download_session(source_repo, path, token) + if not sess: + continue + per_session.append(_session_metrics(sess)) + if not per_session: + logger.info("No sessions found for %s — skipping", day) + return {} + row = _aggregate(per_session) + path_in_repo = f"daily/{day.isoformat()}.csv" + _write_csv(api, row, day.isoformat(), path_in_repo, target_repo, token) + return row + + +def _parse_hour_arg(s: str) -> datetime: + """Accept ``YYYY-MM-DDTHH`` or full ISO — always pinned to the start of the hour, UTC.""" + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.replace(minute=0, second=0, microsecond=0) + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--source", default="smolagents/ml-intern-sessions") + ap.add_argument("--target", default="smolagents/ml-intern-kpis") + ap.add_argument( + "--hours", type=int, default=1, + help="Number of trailing hours to roll up (default: 1 = last completed hour).", + ) + ap.add_argument( + "--datetime", type=str, default=None, + help="Single hour, ISO ``YYYY-MM-DDTHH`` (UTC); overrides --hours.", + ) + ap.add_argument( + "--daily-backfill", type=str, default=None, + help="Escape hatch: aggregate a whole day at once (YYYY-MM-DD). " + "Writes to daily/.csv. Use for historical backfill only.", + ) + args = ap.parse_args(argv) + + token = ( + os.environ.get("HF_KPI_WRITE_TOKEN") + or os.environ.get("HF_SESSION_UPLOAD_TOKEN") + or os.environ.get("HF_TOKEN") + or os.environ.get("HF_ADMIN_TOKEN") + ) + if not token: + logger.error( + "No HF token found. Set one of: HF_KPI_WRITE_TOKEN, " + "HF_SESSION_UPLOAD_TOKEN, HF_TOKEN, HF_ADMIN_TOKEN." + ) + return 1 + + from huggingface_hub import HfApi + api = HfApi() + + if args.daily_backfill: + run_for_day(api, args.source, args.target, date.fromisoformat(args.daily_backfill), token) + return 0 + + if args.datetime: + target_hours = [_parse_hour_arg(args.datetime)] + else: + now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + # Roll up *completed* hours: start from the hour before ``now``. + target_hours = [now - timedelta(hours=i) for i in range(1, args.hours + 1)] + + for hour in target_hours: + run_for_hour(api, args.source, args.target, hour, token) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/build_sft.py b/scripts/build_sft.py new file mode 100644 index 000000000..ac2344c9a --- /dev/null +++ b/scripts/build_sft.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Export session trajectories as raw multi-turn tool-calling SFT data. + +Reads the source sessions dataset (JSONL, one file per session at +``sessions/YYYY-MM-DD/.jsonl``) and writes a re-shaped row to a +target dataset at ``sft/YYYY-MM-DD/.jsonl``. + +**No filtering, no cleaning, no dedup.** Raw passthrough of messages + tools, +with session-level metadata and derived tags (see ``agent/sft/tagger.py``) +attached for downstream slicing. + +Output row schema:: + + { + "session_id": "...", + "model": "claude-opus-4-6", + "timestamp": "2026-04-24T...", + "tags": ["tool:hf_jobs", "gpu:a100", "hf_job:succeeded", ...], + "messages": [...], # OpenAI / TRL SFTTrainer format + "tools": [...] # OpenAI tool schemas the session had access to + } + +Usage:: + + python scripts/build_sft.py \\ + --source smolagents/ml-intern-sessions \\ + --target smolagents/ml-intern-sft \\ + --days 7 + +Env: + HF_TOKEN (or HF_SFT_WRITE_TOKEN) — write access to target dataset. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import tempfile +from datetime import date, datetime, timedelta, timezone +from typing import Iterable + +# Make ``agent`` importable when this script is run outside the project venv. +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from agent.sft.tagger import tag_session # noqa: E402 + +logger = logging.getLogger("build_sft") + + +def _iter_session_files(api, repo_id: str, day: date, token: str) -> Iterable[str]: + prefix = f"sessions/{day.isoformat()}/" + try: + files = api.list_repo_files(repo_id=repo_id, repo_type="dataset", token=token) + except Exception as e: + logger.warning("list_repo_files(%s) failed: %s", repo_id, e) + return [] + return [f for f in files if f.startswith(prefix) and f.endswith(".jsonl")] + + +def _download_and_parse(repo_id: str, path: str, token: str) -> dict | None: + from huggingface_hub import hf_hub_download + try: + local = hf_hub_download( + repo_id=repo_id, filename=path, repo_type="dataset", token=token, + ) + except Exception as e: + logger.warning("hf_hub_download(%s) failed: %s", path, e) + return None + try: + with open(local, "r") as f: + line = f.readline().strip() + if not line: + return None + row = json.loads(line) + # Session uploader stores messages/events/tools as JSON strings. + for key in ("messages", "events", "tools"): + v = row.get(key) + if isinstance(v, str): + try: + row[key] = json.loads(v) + except Exception: + row[key] = [] + return row + except Exception as e: + logger.warning("parse(%s) failed: %s", path, e) + return None + + +def _reshape_to_sft(row: dict) -> dict: + """Raw passthrough: reshape one session row into SFT format + tags. + + Trajectories predating the ``tools`` addition to ``get_trajectory`` will + have an empty tools list — still valid, just less useful downstream. + """ + trajectory = { + "events": row.get("events") or [], + "messages": row.get("messages") or [], + "model_name": row.get("model_name"), + } + return { + "session_id": row.get("session_id"), + "model": row.get("model_name"), + "timestamp": row.get("session_start_time"), + "tags": tag_session(trajectory), + "messages": row.get("messages") or [], + "tools": row.get("tools") or [], + } + + +def _upload_row(api, row: dict, day: date, target_repo: str, token: str) -> None: + session_id = row["session_id"] + path_in_repo = f"sft/{day.isoformat()}/{session_id}.jsonl" + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as tmp: + json.dump(row, tmp, ensure_ascii=False) + tmp_path = tmp.name + try: + api.create_repo( + repo_id=target_repo, repo_type="dataset", exist_ok=True, token=token, + ) + api.upload_file( + path_or_fileobj=tmp_path, + path_in_repo=path_in_repo, + repo_id=target_repo, + repo_type="dataset", + token=token, + commit_message=f"Add SFT row {session_id}", + ) + finally: + try: + os.unlink(tmp_path) + except Exception: + pass + + +def run_for_day( + api, source_repo: str, target_repo: str, day: date, token: str, +) -> int: + paths = _iter_session_files(api, source_repo, day, token) + n = 0 + for path in paths: + sess = _download_and_parse(source_repo, path, token) + if not sess: + continue + sft_row = _reshape_to_sft(sess) + if not sft_row.get("session_id"): + continue + try: + _upload_row(api, sft_row, day, target_repo, token) + n += 1 + except Exception as e: + logger.warning("upload failed for %s: %s", sft_row["session_id"], e) + logger.info("Exported %d sessions for %s", n, day) + return n + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + ap = argparse.ArgumentParser() + ap.add_argument("--source", default="smolagents/ml-intern-sessions") + ap.add_argument("--target", default="smolagents/ml-intern-sft") + ap.add_argument( + "--days", type=int, default=1, + help="Number of trailing days to export (default: 1 = yesterday).", + ) + ap.add_argument( + "--date", type=str, default=None, + help="Single YYYY-MM-DD to export; overrides --days.", + ) + args = ap.parse_args(argv) + + token = ( + os.environ.get("HF_SFT_WRITE_TOKEN") + or os.environ.get("HF_SESSION_UPLOAD_TOKEN") + or os.environ.get("HF_TOKEN") + or os.environ.get("HF_ADMIN_TOKEN") + ) + if not token: + logger.error( + "No HF token found. Set one of: HF_SFT_WRITE_TOKEN, " + "HF_SESSION_UPLOAD_TOKEN, HF_TOKEN, HF_ADMIN_TOKEN." + ) + return 1 + + from huggingface_hub import HfApi + api = HfApi() + + if args.date: + target_days = [date.fromisoformat(args.date)] + else: + today = datetime.now(timezone.utc).date() + target_days = [today - timedelta(days=i) for i in range(1, args.days + 1)] + + total = 0 + for day in target_days: + total += run_for_day(api, args.source, args.target, day, token) + logger.info("Total exported: %d sessions", total) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_build_kpis.py b/tests/unit/test_build_kpis.py new file mode 100644 index 000000000..b9f744b02 --- /dev/null +++ b/tests/unit/test_build_kpis.py @@ -0,0 +1,164 @@ +"""Unit tests for the KPI rollup math. + +We exercise the pure functions (``_session_metrics`` and ``_aggregate_day``) +on hand-crafted session trajectories — no network, no HF Hub. +""" + +import importlib.util +import sys +from pathlib import Path + + +def _load(): + """Load ``scripts/build_kpis.py`` without treating ``scripts`` as a package.""" + path = Path(__file__).parent.parent.parent / "scripts" / "build_kpis.py" + spec = importlib.util.spec_from_file_location("build_kpis", path) + mod = importlib.util.module_from_spec(spec) + sys.modules["build_kpis"] = mod + spec.loader.exec_module(mod) # type: ignore + return mod + + +def _ev(event_type, data=None, ts="2026-04-24T10:00:00"): + return {"timestamp": ts, "event_type": event_type, "data": data or {}} + + +def _session(events, user_id="u1", start="2026-04-24T09:59:00"): + return { + "session_id": "sess-" + user_id, + "session_start_time": start, + "session_end_time": "2026-04-24T10:05:00", + "model_name": "claude-opus-4-6", + "messages": [{"role": "user", "content": "hi"}], + "events": events, + "user_id": user_id, + } + + +def test_llm_call_accumulates_tokens_and_cost(): + mod = _load() + events = [ + _ev("llm_call", { + "prompt_tokens": 100, "completion_tokens": 50, + "cache_read_tokens": 40, "cache_creation_tokens": 10, + "cost_usd": 0.01, + }), + _ev("llm_call", { + "prompt_tokens": 200, "completion_tokens": 100, + "cache_read_tokens": 80, "cost_usd": 0.02, + }), + ] + m = mod._session_metrics(_session(events)) + assert m["llm_calls"] == 2 + assert m["tokens_prompt"] == 300 + assert m["tokens_completion"] == 150 + assert m["tokens_cache_read"] == 120 + assert m["tokens_cache_creation"] == 10 + assert abs(m["cost_usd"] - 0.03) < 1e-9 + + +def test_tool_success_rate_and_first_action(): + mod = _load() + events = [ + _ev("tool_call", {"tool": "bash"}, ts="2026-04-24T10:00:05"), + _ev("tool_output", {"success": True}), + _ev("tool_output", {"success": False}), + ] + m = mod._session_metrics(_session(events)) + assert m["tool_calls_total"] == 2 + assert m["tool_calls_success"] == 1 + # 65s from start to first action + assert m["first_tool_s"] == 65 + + +def test_hf_job_gpu_hours(): + mod = _load() + events = [ + _ev("hf_job_submit", {"flavor": "a100-large", "job_id": "j1"}), + _ev("hf_job_complete", { + "flavor": "a100-large", + "final_status": "COMPLETED", + "wall_time_s": 3600, + }), + ] + m = mod._session_metrics(_session(events)) + assert m["hf_jobs_submitted"] == 1 + assert m["hf_jobs_succeeded"] == 1 + # a100-large = 1 gpu * 1 hour = 1 gpu-hour + assert abs(m["_gpu_hours_by_flavor"]["a100-large"] - 1.0) < 1e-6 + + +def test_feedback_counts(): + mod = _load() + events = [ + _ev("feedback", {"rating": "up"}), + _ev("feedback", {"rating": "up"}), + _ev("feedback", {"rating": "down"}), + ] + m = mod._session_metrics(_session(events)) + assert m["thumbs_up"] == 2 + assert m["thumbs_down"] == 1 + + +def test_aggregate_day_cache_hit_and_users(): + mod = _load() + s1 = mod._session_metrics(_session( + [_ev("llm_call", {"prompt_tokens": 100, "cache_read_tokens": 400, "cost_usd": 0.5})], + user_id="u1", + )) + s2 = mod._session_metrics(_session( + [_ev("llm_call", {"prompt_tokens": 200, "cache_read_tokens": 100, "cost_usd": 1.0})], + user_id="u2", + )) + row = mod._aggregate_day([s1, s2]) + assert row["sessions"] == 2 + assert row["users"] == 2 + assert row["tokens_prompt"] == 300 + assert row["tokens_cache_read"] == 500 + # 500 / (500 + 300) = 0.625 + assert abs(row["cache_hit_ratio"] - 0.625) < 1e-9 + assert abs(row["cost_usd"] - 1.5) < 1e-9 + + +def test_failure_and_regenerate_rates(): + mod = _load() + s1 = mod._session_metrics(_session([_ev("error", {"error": "boom"})], user_id="a")) + s2 = mod._session_metrics(_session([_ev("undo_complete")], user_id="b")) + s3 = mod._session_metrics(_session([], user_id="c")) + row = mod._aggregate_day([s1, s2, s3]) + assert row["failure_rate"] == round(1 / 3, 4) + assert row["regenerate_rate"] == round(1 / 3, 4) + + +def test_window_filter_keeps_only_events_in_range(): + from datetime import datetime, timezone + mod = _load() + events = [ + _ev("llm_call", {"prompt_tokens": 100}, ts="2026-04-24T09:45:00"), + _ev("llm_call", {"prompt_tokens": 200}, ts="2026-04-24T10:05:00"), + _ev("tool_call", {"tool": "bash"}, ts="2026-04-24T10:30:00"), + _ev("llm_call", {"prompt_tokens": 400}, ts="2026-04-24T11:10:00"), + ] + session = _session(events, start="2026-04-24T09:44:00") + # Only events in [10:00, 11:00) should remain. + window_start = datetime(2026, 4, 24, 10, 0, 0, tzinfo=timezone.utc) + window_end = datetime(2026, 4, 24, 11, 0, 0, tzinfo=timezone.utc) + windowed = mod._filter_session_to_window(session, window_start, window_end) + assert windowed is not None + types = [e["event_type"] for e in windowed["events"]] + assert types == ["llm_call", "tool_call"] + # Metrics only reflect in-window events. + m = mod._session_metrics(windowed) + assert m["tokens_prompt"] == 200 + assert m["llm_calls"] == 1 + assert m["tool_calls_total"] == 0 # tool_call not tool_output + + +def test_window_filter_returns_none_when_nothing_in_range(): + from datetime import datetime, timezone + mod = _load() + events = [_ev("llm_call", {"prompt_tokens": 100}, ts="2026-04-24T09:45:00")] + session = _session(events) + window_start = datetime(2026, 4, 24, 10, 0, 0, tzinfo=timezone.utc) + window_end = datetime(2026, 4, 24, 11, 0, 0, tzinfo=timezone.utc) + assert mod._filter_session_to_window(session, window_start, window_end) is None diff --git a/tests/unit/test_build_sft.py b/tests/unit/test_build_sft.py new file mode 100644 index 000000000..538ede29d --- /dev/null +++ b/tests/unit/test_build_sft.py @@ -0,0 +1,78 @@ +"""Smoke test for the SFT reshape — raw passthrough with tags attached.""" + +import importlib.util +import sys +from pathlib import Path + + +def _load(): + path = Path(__file__).parent.parent.parent / "scripts" / "build_sft.py" + spec = importlib.util.spec_from_file_location("build_sft", path) + mod = importlib.util.module_from_spec(spec) + sys.modules["build_sft"] = mod + spec.loader.exec_module(mod) # type: ignore + return mod + + +def _session_row(): + return { + "session_id": "abc", + "session_start_time": "2026-04-24T10:00:00", + "session_end_time": "2026-04-24T10:05:00", + "model_name": "claude-opus-4-6", + "messages": [ + {"role": "system", "content": "You are an agent"}, + {"role": "user", "content": "fine-tune llama"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "c1", "type": "function", + "function": {"name": "hf_jobs", "arguments": '{"script":"from trl import SFTTrainer"}'}}, + ]}, + {"role": "tool", "tool_call_id": "c1", "content": "ok"}, + {"role": "assistant", "content": "done"}, + ], + "events": [ + {"timestamp": "2026-04-24T10:00:05", "event_type": "tool_call", + "data": {"tool": "hf_jobs", + "arguments": {"script": "from trl import SFTTrainer"}}}, + {"timestamp": "2026-04-24T10:00:06", "event_type": "hf_job_submit", + "data": {"flavor": "a100-large", "push_to_hub": True}}, + {"timestamp": "2026-04-24T10:45:00", "event_type": "hf_job_complete", + "data": {"flavor": "a100-large", "final_status": "COMPLETED", + "wall_time_s": 2700}}, + {"timestamp": "2026-04-24T10:45:05", "event_type": "turn_complete", + "data": {}}, + ], + "tools": [{"type": "function", "function": {"name": "hf_jobs"}}], + } + + +def test_reshape_preserves_messages_and_tools_and_adds_tags(): + mod = _load() + row = mod._reshape_to_sft(_session_row()) + assert row["session_id"] == "abc" + assert row["model"] == "claude-opus-4-6" + assert row["timestamp"] == "2026-04-24T10:00:00" + # Messages preserved verbatim, in order, with tool_calls + tool role rows. + assert len(row["messages"]) == 5 + assert row["messages"][2]["tool_calls"][0]["function"]["name"] == "hf_jobs" + assert row["messages"][3]["role"] == "tool" + # Tools preserved verbatim. + assert row["tools"] == [{"type": "function", "function": {"name": "hf_jobs"}}] + # Tags include the expected signals. + tags = set(row["tags"]) + assert "tool:hf_jobs" in tags + assert "hf_job:succeeded" in tags + assert "hf_job:push_to_hub" in tags + assert "gpu:a100" in tags + assert "outcome:completed" in tags + assert "task:training" in tags + assert "model:opus" in tags + + +def test_reshape_handles_missing_tools_field(): + mod = _load() + row = _session_row() + del row["tools"] + out = mod._reshape_to_sft(row) + assert out["tools"] == [] + assert isinstance(out["tags"], list) # still computes tags diff --git a/tests/unit/test_heartbeat.py b/tests/unit/test_heartbeat.py new file mode 100644 index 000000000..29d8079fd --- /dev/null +++ b/tests/unit/test_heartbeat.py @@ -0,0 +1,134 @@ +"""Heartbeat + stable-local-path tests for Session. + +We don't spin up the real agent loop — we build a minimal Session with a +stubbed config and an in-memory queue, then call send_event repeatedly while +monkeypatching time.monotonic to simulate seconds passing. +""" + +import asyncio +import json +from pathlib import Path +from unittest.mock import patch + +import pytest + +from agent.core.session import Event, Session + + +class _FakeConfig: + model_name = "claude-opus-4-6" + save_sessions = True + session_dataset_repo = "fake/repo" + auto_save_interval = 1 + heartbeat_interval_s = 60 + max_iterations = 10 + yolo_mode = False + confirm_cpu_jobs = False + auto_file_upload = False + reasoning_effort = None + mcpServers: dict = {} + + +def _mk_session(tmp_path: Path) -> Session: + import os + os.chdir(tmp_path) # so session_logs/ lands under tmp_path + # Stub out the context manager to avoid litellm lookups. + from agent.context_manager.manager import ContextManager + cm = ContextManager.__new__(ContextManager) + cm.items = [] + cm.tool_specs = [] + cm.model_max_tokens = 200_000 + cm.running_context_usage = 0 + cm.compact_size = 0.1 + cm.untouched_messages = 5 + cm.hf_token = None + cm.local_mode = True + s = Session( + event_queue=asyncio.Queue(), + config=_FakeConfig(), + tool_router=None, + context_manager=cm, + hf_token=None, + local_mode=True, + ) + return s + + +def test_heartbeat_fires_after_interval(tmp_path, monkeypatch): + # Use asyncio.run rather than pytest-asyncio so the test works without the + # plugin installed (same pattern elsewhere in this repo). + async def body(): + s = _mk_session(tmp_path) + calls = [] + + def fake_upload(repo_id): + calls.append(repo_id) + return "fake/path.json" + + monkeypatch.setattr(s, "save_and_upload_detached", fake_upload) + + # t=0: first event, should NOT trigger (initial _last_heartbeat_ts = now) + with patch("agent.core.telemetry.time.monotonic", return_value=100.0): + s._last_heartbeat_ts = 100.0 + await s.send_event(Event(event_type="x")) + assert calls == [] + + # t=+30s: still under interval → no save + with patch("agent.core.telemetry.time.monotonic", return_value=130.0): + await s.send_event(Event(event_type="y")) + assert calls == [] + + # t=+61s: over 60s → save fires once + with patch("agent.core.telemetry.time.monotonic", return_value=161.0): + await s.send_event(Event(event_type="z")) + # create_task runs on the event loop; wait for the to_thread to complete + await asyncio.sleep(0.05) + assert calls == ["fake/repo"] + + # Next event shortly after → no second save (interval resets to 161) + with patch("agent.core.telemetry.time.monotonic", return_value=170.0): + await s.send_event(Event(event_type="w")) + await asyncio.sleep(0.05) + assert len(calls) == 1 + + asyncio.run(body()) + + +def test_stable_local_path_overwrites(tmp_path): + import os + os.chdir(tmp_path) + from agent.context_manager.manager import ContextManager + cm = ContextManager.__new__(ContextManager) + cm.items = [] + cm.tool_specs = [] + cm.model_max_tokens = 200_000 + cm.running_context_usage = 0 + cm.compact_size = 0.1 + cm.untouched_messages = 5 + cm.hf_token = None + cm.local_mode = True + + s = Session( + event_queue=asyncio.Queue(), + config=_FakeConfig(), + tool_router=None, + context_manager=cm, + hf_token=None, + local_mode=True, + ) + + p1 = s.save_trajectory_local(directory="session_logs") + assert p1 is not None + p2 = s.save_trajectory_local(directory="session_logs") + p3 = s.save_trajectory_local(directory="session_logs") + # All three saves land on the same file — heartbeat should not spam files. + assert p1 == p2 == p3 + files = list(Path("session_logs").glob("session_*.json")) + # Exactly one final file; the .tmp should be renamed away. + assert len(files) == 1 + + # File is valid JSON (atomic write → no torn content). + with open(p1) as f: + data = json.load(f) + assert data["session_id"] == s.session_id + assert data["upload_status"] == "pending" diff --git a/tests/unit/test_kpis_scheduler.py b/tests/unit/test_kpis_scheduler.py new file mode 100644 index 000000000..8c52f0513 --- /dev/null +++ b/tests/unit/test_kpis_scheduler.py @@ -0,0 +1,107 @@ +"""Smoke tests for backend/kpis_scheduler.py. + +Exercise the pure / fast paths only: + * token resolution order + * build_kpis import path + * start()/shutdown() lifecycle without APScheduler actually running a job + * backfill() passes the right hour values through to _run_hour +""" + +from __future__ import annotations + +import asyncio +import importlib.util +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def _load(): + path = Path(__file__).parent.parent.parent / "backend" / "kpis_scheduler.py" + spec = importlib.util.spec_from_file_location("kpis_scheduler", path) + mod = importlib.util.module_from_spec(spec) + sys.modules["kpis_scheduler"] = mod + assert spec.loader is not None + spec.loader.exec_module(mod) + return mod + + +def test_token_resolution_order(monkeypatch): + mod = _load() + for var in ("HF_KPI_WRITE_TOKEN", "HF_SESSION_UPLOAD_TOKEN", "HF_TOKEN", "HF_ADMIN_TOKEN"): + monkeypatch.delenv(var, raising=False) + assert mod._resolve_token() is None + + monkeypatch.setenv("HF_ADMIN_TOKEN", "admin") + assert mod._resolve_token() == "admin" + + monkeypatch.setenv("HF_TOKEN", "generic") + assert mod._resolve_token() == "generic" + + monkeypatch.setenv("HF_SESSION_UPLOAD_TOKEN", "sessions") + assert mod._resolve_token() == "sessions" + + monkeypatch.setenv("HF_KPI_WRITE_TOKEN", "kpis") + assert mod._resolve_token() == "kpis" + + +def test_load_build_kpis_exposes_run_for_hour(): + mod = _load() + bk = mod._load_build_kpis() + assert hasattr(bk, "run_for_hour") + assert callable(bk.run_for_hour) + + +def test_backfill_calls_run_hour_for_each_hour(monkeypatch): + mod = _load() + monkeypatch.setenv("HF_KPI_WRITE_TOKEN", "x") + calls: list[datetime] = [] + + async def fake_run_hour(hour_dt): + calls.append(hour_dt) + + monkeypatch.setattr(mod, "_run_hour", fake_run_hour) + asyncio.run(mod.backfill(hours=3)) + assert len(calls) == 3 + # Hours are returned most-recent-first + assert calls[0] > calls[1] > calls[2] + # All aligned to the top of the hour + for c in calls: + assert c.minute == 0 and c.second == 0 and c.microsecond == 0 + assert c.tzinfo == timezone.utc + + +def test_start_is_no_op_when_disabled(monkeypatch): + mod = _load() + # Ensure clean state — _scheduler is module-global + mod._scheduler = None + monkeypatch.setenv("ML_INTERN_KPIS_DISABLED", "1") + mod.start() + assert mod._scheduler is None # never instantiated + + +def test_start_skips_cleanly_without_apscheduler(monkeypatch): + mod = _load() + mod._scheduler = None + monkeypatch.delenv("ML_INTERN_KPIS_DISABLED", raising=False) + + # Force the apscheduler import to fail — start() should log and return. + real_import = __builtins__["__import__"] if isinstance(__builtins__, dict) else __builtins__.__import__ + + def fake_import(name, *args, **kwargs): + if name.startswith("apscheduler"): + raise ImportError("apscheduler unavailable in test") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr( + "builtins.__import__", + fake_import, + ) + mod.start() # should not raise + assert mod._scheduler is None + + +def test_shutdown_is_no_op_when_not_started(): + mod = _load() + mod._scheduler = None + asyncio.run(mod.shutdown()) # must not raise diff --git a/tests/unit/test_redact.py b/tests/unit/test_redact.py new file mode 100644 index 000000000..24c142536 --- /dev/null +++ b/tests/unit/test_redact.py @@ -0,0 +1,76 @@ +"""Tests for the secret scrubber used before session upload.""" + +from agent.core.redact import scrub, scrub_string + + +def test_hf_token(): + s = "here is a token hf_" + "A" * 35 + " ok" + out = scrub_string(s) + assert "hf_" not in out + assert "[REDACTED_HF_TOKEN]" in out + + +def test_anthropic_key(): + s = "key=sk-ant-api03_" + "a" * 40 + out = scrub_string(s) + # The env-var name prefix matches too; just verify we don't leave the body. + assert "sk-ant-api03_" not in out + + +def test_github_token(): + s = "ghp_" + "a" * 40 + out = scrub_string(s) + assert out == "[REDACTED_GITHUB_TOKEN]" + + +def test_github_fine_grained_pat(): + # Fine-grained PATs: github_pat_, 36+ chars + s = "github_pat_" + "A1B2_" * 10 + out = scrub_string(s) + assert "github_pat_" not in out + assert "[REDACTED_GITHUB_TOKEN]" in out + + +def test_aws_key_id(): + s = "AWS_ACCESS_KEY_ID=AKIAABCDEFGHIJKLMNOP" + out = scrub_string(s) + assert "AKIAABCDEFGHIJKLMNOP" not in out + + +def test_bearer_header(): + s = "Authorization: Bearer abcdef0123456789abcdef0123456789" + out = scrub_string(s) + assert "abcdef0123456789abcdef0123456789" not in out + assert "Bearer [REDACTED]" in out + + +def test_env_var_style(): + s = "HF_TOKEN=hf_" + "x" * 40 + " run" + out = scrub_string(s) + # Either the value-scrubber or the HF-token regex should fire. + assert "hf_xxxx" not in out + + +def test_scrub_nested_dict_and_list(): + payload = { + "msg": "token hf_" + "Z" * 35, + "tools": [ + {"args": {"secret": "ghp_" + "Q" * 40}}, + "no secrets here", + ], + "n": 42, + } + out = scrub(payload) + # Original not mutated + assert "hf_" in payload["msg"] + # Redacted copy + assert "[REDACTED_HF_TOKEN]" in out["msg"] + assert out["tools"][0]["args"]["secret"] == "[REDACTED_GITHUB_TOKEN]" + assert out["tools"][1] == "no secrets here" + assert out["n"] == 42 + + +def test_scrub_preserves_non_strings(): + assert scrub(None) is None + assert scrub(123) == 123 + assert scrub(True) is True diff --git a/tests/unit/test_sft_tagger.py b/tests/unit/test_sft_tagger.py new file mode 100644 index 000000000..2ade0f64d --- /dev/null +++ b/tests/unit/test_sft_tagger.py @@ -0,0 +1,197 @@ +"""Tests for agent.sft.tagger — one test per tag namespace.""" + +from agent.sft.tagger import tag_session + + +def _ev(event_type, data=None, ts="2026-04-24T10:00:00"): + return {"timestamp": ts, "event_type": event_type, "data": data or {}} + + +def _traj(events=None, messages=None, model="claude-opus-4-6"): + return { + "session_id": "sess-1", + "model_name": model, + "session_start_time": "2026-04-24T09:59:00", + "session_end_time": "2026-04-24T10:05:00", + "messages": messages + or [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "ok"}], + "events": events or [], + } + + +def test_model_family(): + assert "model:opus" in tag_session(_traj(model="claude-opus-4-6")) + assert "model:sonnet" in tag_session(_traj(model="bedrock/claude-sonnet-4-5")) + assert "model:kimi" in tag_session(_traj(model="moonshotai/Kimi-K2.6")) + assert "model:other" in tag_session(_traj(model="unknown-model-xyz")) + + +def test_turns_buckets(): + short = _traj(messages=[{"role": "user", "content": "hi"}]) + medium = _traj(messages=[{"role": "user", "content": "q"} for _ in range(10)]) + long = _traj(messages=[{"role": "user", "content": "q"} for _ in range(25)]) + assert "turns:short" in tag_session(short) + assert "turns:medium" in tag_session(medium) + assert "turns:long" in tag_session(long) + + +def test_cost_buckets(): + cheap = _traj(events=[_ev("llm_call", {"cost_usd": 0.05})]) + med = _traj(events=[_ev("llm_call", {"cost_usd": 0.5})]) + expensive = _traj(events=[_ev("llm_call", {"cost_usd": 5.0})]) + assert "cost:low" in tag_session(cheap) + assert "cost:med" in tag_session(med) + assert "cost:high" in tag_session(expensive) + + +def test_tool_tags(): + events = [ + _ev("tool_call", {"tool": "hf_jobs", "arguments": {}}), + _ev("tool_call", {"tool": "research"}), + _ev("tool_call", {"tool": "bash"}), + ] + tags = tag_session(_traj(events)) + assert "tool:hf_jobs" in tags + assert "tool:research" in tags + assert "tool:bash" in tags + + +def test_outcome_completed(): + events = [_ev("turn_complete", {"history_size": 10})] + assert "outcome:completed" in tag_session(_traj(events)) + + +def test_outcome_errored(): + events = [_ev("error", {"error": "boom"})] + assert "outcome:errored" in tag_session(_traj(events)) + + +def test_outcome_interrupted(): + events = [_ev("interrupted")] + assert "outcome:interrupted" in tag_session(_traj(events)) + + +def test_outcome_ongoing(): + # No terminal events → session was still running at save time + events = [_ev("llm_call", {"cost_usd": 0.01})] + assert "outcome:ongoing" in tag_session(_traj(events)) + + +def test_outcome_doom_loop_and_context(): + events = [ + _ev("tool_log", {"tool": "system", "log": "Doom loop detected — injecting corrective prompt"}), + _ev("compacted", {"old_tokens": 100, "new_tokens": 50}), + _ev("turn_complete", {"history_size": 10}), + ] + tags = tag_session(_traj(events)) + assert "outcome:doom_loop" in tags + assert "outcome:context_exceeded" in tags + + +def test_hf_job_tags(): + events = [ + _ev("tool_call", {"tool": "hf_jobs", "arguments": {"script": "from trl import SFTTrainer"}}), + _ev("hf_job_submit", { + "flavor": "a100-large", "push_to_hub": True, "job_id": "j1", + }), + _ev("hf_job_complete", {"flavor": "a100-large", "final_status": "COMPLETED", "wall_time_s": 3600}), + _ev("hf_job_submit", {"flavor": "a100x4", "push_to_hub": False}), + _ev("hf_job_complete", {"flavor": "a100x4", "final_status": "FAILED"}), + ] + tags = tag_session(_traj(events)) + assert "hf_job:submitted" in tags + assert "hf_job:multi" in tags + assert "hf_job:succeeded" in tags + assert "hf_job:failed" in tags + assert "hf_job:push_to_hub" in tags + assert "gpu:a100" in tags + assert "gpu:multi" in tags + + +def test_hf_job_oom(): + events = [ + _ev("tool_call", {"tool": "hf_jobs", "arguments": {}}), + _ev("hf_job_submit", {"flavor": "a100-large"}), + _ev("tool_output", {"success": False, "output": "RuntimeError: CUDA out of memory. Tried to allocate..."}), + ] + tags = tag_session(_traj(events)) + assert "hf_job:oom" in tags + + +def test_sandbox_tags(): + events = [ + _ev("sandbox_create", {"hardware": "t4-small", "sandbox_id": "s1", "create_latency_s": 5}), + _ev("sandbox_destroy", {"sandbox_id": "s1", "lifetime_s": 3600}), + ] + tags = tag_session(_traj(events)) + assert "sandbox:created" in tags + assert "sandbox:gpu" in tags + assert "sandbox:long_lived" in tags + + +def test_sandbox_cpu_short(): + events = [ + _ev("sandbox_create", {"hardware": "cpu-basic"}), + _ev("sandbox_destroy", {"lifetime_s": 120}), + ] + tags = tag_session(_traj(events)) + assert "sandbox:cpu" in tags + assert "sandbox:long_lived" not in tags + + +def test_feedback_tags(): + up_only = _traj(events=[_ev("feedback", {"rating": "up"})]) + down_only = _traj(events=[_ev("feedback", {"rating": "down"})]) + mixed = _traj(events=[_ev("feedback", {"rating": "up"}), _ev("feedback", {"rating": "down"})]) + none = _traj() + assert "feedback:up" in tag_session(up_only) + assert "feedback:down" in tag_session(down_only) + assert "feedback:mixed" in tag_session(mixed) + assert "feedback:none" in tag_session(none) + + +def test_task_training(): + events = [ + _ev("tool_call", {"tool": "hf_jobs", "arguments": { + "script": "from trl import SFTTrainer\ntrainer = SFTTrainer(...)" + }}), + _ev("hf_job_submit", {"flavor": "a100-large"}), + ] + assert "task:training" in tag_session(_traj(events)) + + +def test_task_research_only(): + events = [ + _ev("tool_call", {"tool": "research"}), + _ev("tool_call", {"tool": "explore_hf_docs"}), + ] + assert "task:research_only" in tag_session(_traj(events)) + + +def test_task_data_prep(): + events = [ + _ev("tool_call", {"tool": "hf_inspect_dataset", "arguments": {}}), + _ev("tool_call", {"tool": "hub_repo_details"}), + ] + tags = tag_session(_traj(events)) + assert "task:data_prep" in tags + + +def test_no_duplicates_and_sorted(): + events = [ + _ev("tool_call", {"tool": "hf_jobs"}), + _ev("tool_call", {"tool": "hf_jobs"}), # duplicate + _ev("hf_job_submit", {"flavor": "a10g-small"}), + _ev("hf_job_submit", {"flavor": "a10g-small"}), + ] + tags = tag_session(_traj(events)) + assert tags == sorted(tags) + assert len(tags) == len(set(tags)) + + +def test_empty_trajectory_has_required_tags(): + tags = tag_session(_traj()) + namespaces = {t.split(":", 1)[0] for t in tags} + # Every session must have at least model/turns/cost/outcome/feedback. + for required in ("model", "turns", "cost", "outcome", "feedback"): + assert required in namespaces, f"missing {required} — got {tags}" From 2158d6b783f07344185aacced49f1699f0343567 Mon Sep 17 00:00:00 2001 From: Clem Date: Sat, 25 Apr 2026 12:11:28 -0400 Subject: [PATCH 004/120] fix typo in readme (#117) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29fe439b8..3e5e37b46 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # ML Intern -An ML intern that autonomously researches, writes, and ships good quality ML releated code using the Hugging Face ecosystem — with deep access to docs, papers, datasets, and cloud compute. +An ML intern that autonomously researches, writes, and ships good quality ML related code using the Hugging Face ecosystem — with deep access to docs, papers, datasets, and cloud compute. ## Quick Start From 0545e4074dbf019acf90309a4b0ca778d891932c Mon Sep 17 00:00:00 2001 From: lewtun Date: Sat, 25 Apr 2026 18:37:22 +0200 Subject: [PATCH 005/120] Switch from Bedrock to Anthropic endpoint as default. Include support for gpt-5.5 (#118) * Default ml-intern to Anthropic Co-authored-by: OpenAI Codex * Add direct OpenAI GPT-5 model support Co-authored-by: OpenAI Codex * Raise probe budget for GPT-5 models Co-authored-by: OpenAI Codex * Fix deps * Fix stale xhigh provider messaging Co-authored-by: OpenAI Codex --------- Co-authored-by: OpenAI Codex --- README.md | 2 + agent/core/effort_probe.py | 12 ++- agent/core/llm_params.py | 4 +- agent/core/model_switcher.py | 7 +- agent/main.py | 5 +- configs/main_agent_config.json | 2 +- pyproject.toml | 4 +- tests/unit/test_llm_params.py | 25 +++++ uv.lock | 170 +++++++++++++++++++-------------- 9 files changed, 146 insertions(+), 85 deletions(-) create mode 100644 tests/unit/test_llm_params.py diff --git a/README.md b/README.md index 3e5e37b46..20d95cd49 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Create a `.env` file in the project root (or export these in your shell): ```bash ANTHROPIC_API_KEY= # if using anthropic models +OPENAI_API_KEY= # if using openai models HF_TOKEN= GITHUB_TOKEN= ``` @@ -50,6 +51,7 @@ ml-intern "fine-tune llama on my dataset" ```bash ml-intern --model anthropic/claude-opus-4-6 "your prompt" +ml-intern --model openai/gpt-5.5 "your prompt" ml-intern --max-iterations 100 "your prompt" ml-intern --no-stream "your prompt" ``` diff --git a/agent/core/effort_probe.py b/agent/core/effort_probe.py index 142feaaa1..2c0c79ea3 100644 --- a/agent/core/effort_probe.py +++ b/agent/core/effort_probe.py @@ -32,9 +32,10 @@ # Cascade: for each user-stated preference, the ordered list of levels to -# try. First success wins. ``max`` / ``xhigh`` are Anthropic-only; providers -# that don't accept them raise ``UnsupportedEffortError`` synchronously (no -# wasted network round-trip) and we advance to the next level. +# try. First success wins. ``max`` is Anthropic-only; ``xhigh`` is also +# supported on current OpenAI GPT-5 models. Providers that don't accept a +# requested level raise ``UnsupportedEffortError`` synchronously (no wasted +# network round-trip) and we advance to the next level. _EFFORT_CASCADE: dict[str, list[str]] = { "max": ["max", "xhigh", "high", "medium", "low"], "xhigh": ["xhigh", "high", "medium", "low"], @@ -45,7 +46,10 @@ } _PROBE_TIMEOUT = 15.0 -_PROBE_MAX_TOKENS = 16 +# Keep the probe cheap, but high enough that frontier reasoning models can +# finish a trivial reply instead of tripping a false "output limit reached" +# error during capability detection. +_PROBE_MAX_TOKENS = 64 class ProbeInconclusive(Exception): diff --git a/agent/core/llm_params.py b/agent/core/llm_params.py index d6843df10..bac507354 100644 --- a/agent/core/llm_params.py +++ b/agent/core/llm_params.py @@ -66,13 +66,13 @@ def _widened(model: str) -> bool: # Effort levels accepted on the wire. # Anthropic (4.6+): low | medium | high | xhigh | max (output_config.effort) -# OpenAI direct: minimal | low | medium | high (reasoning_effort top-level) +# OpenAI direct: minimal | low | medium | high | xhigh (reasoning_effort top-level) # HF router: low | medium | high (extra_body.reasoning_effort) # # We validate *shape* here and let the probe cascade walk down on rejection; # we deliberately do NOT maintain a per-model capability table. _ANTHROPIC_EFFORTS = {"low", "medium", "high", "xhigh", "max"} -_OPENAI_EFFORTS = {"minimal", "low", "medium", "high"} +_OPENAI_EFFORTS = {"minimal", "low", "medium", "high", "xhigh"} _HF_EFFORTS = {"low", "medium", "high"} diff --git a/agent/core/model_switcher.py b/agent/core/model_switcher.py index afb8d52c6..63c0f40c6 100644 --- a/agent/core/model_switcher.py +++ b/agent/core/model_switcher.py @@ -24,8 +24,11 @@ # ":cheapest" / ":preferred" / ":" to override the default # routing policy (auto = fastest with failover). SUGGESTED_MODELS = [ - {"id": "bedrock/us.anthropic.claude-opus-4-7", "label": "Claude Opus 4.7"}, - {"id": "bedrock/us.anthropic.claude-opus-4-6-v1", "label": "Claude Opus 4.6"}, + {"id": "openai/gpt-5.5", "label": "GPT-5.5"}, + {"id": "openai/gpt-5.4", "label": "GPT-5.4"}, + {"id": "anthropic/claude-opus-4-7", "label": "Claude Opus 4.7"}, + {"id": "anthropic/claude-opus-4-6", "label": "Claude Opus 4.6"}, + {"id": "bedrock/us.anthropic.claude-opus-4-6-v1", "label": "Claude Opus 4.6 via Bedrock"}, {"id": "MiniMaxAI/MiniMax-M2.7", "label": "MiniMax M2.7"}, {"id": "moonshotai/Kimi-K2.6", "label": "Kimi K2.6"}, {"id": "zai-org/GLM-5.1", "label": "GLM 5.1"}, diff --git a/agent/main.py b/agent/main.py index 4ecbefc50..4b08c7266 100644 --- a/agent/main.py +++ b/agent/main.py @@ -771,8 +771,9 @@ async def _handle_slash_command( console.print(f" [dim]{m}: {eff or 'off'}[/dim]") console.print( "[dim]Set with '/effort minimal|low|medium|high|xhigh|max|off'. " - "'max' and 'xhigh' are Anthropic-only; the cascade falls back " - "to whatever the model actually accepts.[/dim]" + "'max' is Anthropic-only; 'xhigh' is also supported by current " + "OpenAI GPT-5 models. The cascade falls back to whatever the " + "model actually accepts.[/dim]" ) return None level = arg.lower() diff --git a/configs/main_agent_config.json b/configs/main_agent_config.json index c73ea380f..99335ca71 100644 --- a/configs/main_agent_config.json +++ b/configs/main_agent_config.json @@ -1,5 +1,5 @@ { - "model_name": "bedrock/us.anthropic.claude-opus-4-6-v1", + "model_name": "anthropic/claude-opus-4-6", "save_sessions": true, "session_dataset_repo": "smolagents/ml-intern-sessions", "yolo_mode": false, diff --git a/pyproject.toml b/pyproject.toml index 2544f36b7..4b263213e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "hf-agent" +name = "ml-intern" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -46,7 +46,7 @@ dev = [ # All dependencies (eval + dev) all = [ - "hf-agent[eval,dev]", + "ml-intern[eval,dev]", ] [project.scripts] diff --git a/tests/unit/test_llm_params.py b/tests/unit/test_llm_params.py new file mode 100644 index 000000000..ee6cf62c6 --- /dev/null +++ b/tests/unit/test_llm_params.py @@ -0,0 +1,25 @@ +from agent.core.llm_params import UnsupportedEffortError, _resolve_llm_params + + +def test_openai_xhigh_effort_is_forwarded(): + params = _resolve_llm_params( + "openai/gpt-5.5", + reasoning_effort="xhigh", + strict=True, + ) + + assert params["model"] == "openai/gpt-5.5" + assert params["reasoning_effort"] == "xhigh" + + +def test_openai_max_effort_is_still_rejected(): + try: + _resolve_llm_params( + "openai/gpt-5.4", + reasoning_effort="max", + strict=True, + ) + except UnsupportedEffortError as exc: + assert "OpenAI doesn't accept effort='max'" in str(exc) + else: + raise AssertionError("Expected UnsupportedEffortError for max effort") diff --git a/uv.lock b/uv.lock index 7546e793a..c214134b5 100644 --- a/uv.lock +++ b/uv.lock @@ -228,6 +228,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] +[[package]] +name = "apscheduler" +version = "3.11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/12/3e4389e5920b4c1763390c6d371162f3784f86f85cd6d6c1bfe68eef14e2/apscheduler-3.11.2.tar.gz", hash = "sha256:2a9966b052ec805f020c8c4c3ae6e6a06e24b1bf19f2e11d91d8cca0473eef41", size = 108683, upload-time = "2025-12-22T00:39:34.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/64/2e54428beba8d9992aa478bb8f6de9e4ecaa5f8f513bcfd567ed7fb0262d/apscheduler-3.11.2-py3-none-any.whl", hash = "sha256:ce005177f741409db4e4dd40a7431b76feb856b9dd69d57e0da49d6715bfd26d", size = 64439, upload-time = "2025-12-22T00:39:33.303Z" }, +] + [[package]] name = "attrs" version = "25.4.0" @@ -992,78 +1004,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] -[[package]] -name = "hf-agent" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "boto3" }, - { name = "datasets" }, - { name = "fastapi" }, - { name = "fastmcp" }, - { name = "httpx" }, - { name = "huggingface-hub" }, - { name = "litellm" }, - { name = "nbconvert" }, - { name = "nbformat" }, - { name = "prompt-toolkit" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "requests" }, - { name = "rich" }, - { name = "thefuzz" }, - { name = "uvicorn", extra = ["standard"] }, - { name = "websockets" }, - { name = "whoosh" }, -] - -[package.optional-dependencies] -all = [ - { name = "datasets" }, - { name = "inspect-ai" }, - { name = "pandas" }, - { name = "pytest" }, - { name = "tenacity" }, -] -dev = [ - { name = "pytest" }, -] -eval = [ - { name = "datasets" }, - { name = "inspect-ai" }, - { name = "pandas" }, - { name = "tenacity" }, -] - -[package.metadata] -requires-dist = [ - { name = "boto3", specifier = ">=1.35.0" }, - { name = "datasets", specifier = ">=4.4.1" }, - { name = "datasets", marker = "extra == 'eval'", specifier = ">=4.3.0" }, - { name = "fastapi", specifier = ">=0.115.0" }, - { name = "fastmcp", specifier = ">=3.2.0" }, - { name = "hf-agent", extras = ["eval", "dev"], marker = "extra == 'all'" }, - { name = "httpx", specifier = ">=0.27.0" }, - { name = "huggingface-hub", specifier = ">=1.0.1" }, - { name = "inspect-ai", marker = "extra == 'eval'", specifier = ">=0.3.149" }, - { name = "litellm", specifier = ">=1.83.0" }, - { name = "nbconvert", specifier = ">=7.16.6" }, - { name = "nbformat", specifier = ">=5.10.4" }, - { name = "pandas", marker = "extra == 'eval'", specifier = ">=2.3.3" }, - { name = "prompt-toolkit", specifier = ">=3.0.0" }, - { name = "pydantic", specifier = ">=2.12.3" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, - { name = "python-dotenv", specifier = ">=1.2.1" }, - { name = "requests", specifier = ">=2.33.0" }, - { name = "rich", specifier = ">=13.0.0" }, - { name = "tenacity", marker = "extra == 'eval'", specifier = ">=8.0.0" }, - { name = "thefuzz", specifier = ">=0.22.1" }, - { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, - { name = "websockets", specifier = ">=13.0" }, - { name = "whoosh", specifier = ">=2.7.4" }, -] -provides-extras = ["eval", "dev", "all"] - [[package]] name = "hf-xet" version = "1.2.0" @@ -1827,6 +1767,80 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/f7/4a5e785ec9fbd65146a27b6b70b6cdc161a66f2024e4b04ac06a67f5578b/mistune-3.2.0-py3-none-any.whl", hash = "sha256:febdc629a3c78616b94393c6580551e0e34cc289987ec6c35ed3f4be42d0eee1", size = 53598, upload-time = "2025-12-23T11:36:33.211Z" }, ] +[[package]] +name = "ml-intern" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "apscheduler" }, + { name = "boto3" }, + { name = "datasets" }, + { name = "fastapi" }, + { name = "fastmcp" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "litellm" }, + { name = "nbconvert" }, + { name = "nbformat" }, + { name = "prompt-toolkit" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "rich" }, + { name = "thefuzz" }, + { name = "uvicorn", extra = ["standard"] }, + { name = "websockets" }, + { name = "whoosh" }, +] + +[package.optional-dependencies] +all = [ + { name = "datasets" }, + { name = "inspect-ai" }, + { name = "pandas" }, + { name = "pytest" }, + { name = "tenacity" }, +] +dev = [ + { name = "pytest" }, +] +eval = [ + { name = "datasets" }, + { name = "inspect-ai" }, + { name = "pandas" }, + { name = "tenacity" }, +] + +[package.metadata] +requires-dist = [ + { name = "apscheduler", specifier = ">=3.10,<4" }, + { name = "boto3", specifier = ">=1.35.0" }, + { name = "datasets", specifier = ">=4.4.1" }, + { name = "datasets", marker = "extra == 'eval'", specifier = ">=4.3.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "fastmcp", specifier = ">=3.2.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "huggingface-hub", specifier = ">=1.0.1" }, + { name = "inspect-ai", marker = "extra == 'eval'", specifier = ">=0.3.149" }, + { name = "litellm", specifier = ">=1.83.0" }, + { name = "ml-intern", extras = ["eval", "dev"], marker = "extra == 'all'" }, + { name = "nbconvert", specifier = ">=7.16.6" }, + { name = "nbformat", specifier = ">=5.10.4" }, + { name = "pandas", marker = "extra == 'eval'", specifier = ">=2.3.3" }, + { name = "prompt-toolkit", specifier = ">=3.0.0" }, + { name = "pydantic", specifier = ">=2.12.3" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, + { name = "python-dotenv", specifier = ">=1.2.1" }, + { name = "requests", specifier = ">=2.33.0" }, + { name = "rich", specifier = ">=13.0.0" }, + { name = "tenacity", marker = "extra == 'eval'", specifier = ">=8.0.0" }, + { name = "thefuzz", specifier = ">=0.22.1" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.32.0" }, + { name = "websockets", specifier = ">=13.0" }, + { name = "whoosh", specifier = ">=2.7.4" }, +] +provides-extras = ["eval", "dev", "all"] + [[package]] name = "mmh3" version = "5.2.0" @@ -3619,6 +3633,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "uc-micro-py" version = "1.0.3" From 3eec386de3526519db92f40b886adf5c539e9148 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Sat, 25 Apr 2026 22:55:16 +0300 Subject: [PATCH 006/120] Fix CLI rendering corruption and split CLI/frontend model defaults (#121) * Stabilize CLI rendering and make surface defaults explicit The interactive CLI was interleaving live sub-agent redraws with streamed markdown output, which corrupted ANSI rendering and leaked raw control sequences into the terminal. The CLI and web app also shared one default model config even though they need different Anthropic routing defaults. Constraint: CLI default must use direct Anthropic credentials while web sessions must default to Bedrock Anthropic Constraint: Interactive terminal output must remain readable while sub-agent progress is live Rejected: Single shared config file with runtime overrides | keeps ownership of defaults implicit across surfaces Rejected: Keep background redraw ticker | concurrent terminal writers still corrupt streamed output Confidence: high Scope-risk: moderate Reversibility: clean Directive: Keep CLI and frontend default models in separate config files unless both surfaces intentionally converge again Tested: python -m compileall agent backend Tested: ./frontend/node_modules/.bin/tsc -p frontend/tsconfig.json --noEmit Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --with pytest python -m pytest -q tests/unit/test_cli_rendering.py Not-tested: Full pytest suite (blocked by pre-existing tests/unit/test_llm_error_classification.py import error during collection) * Restore regression coverage and make the full test suite green The earlier PR fixed the CLI rendering and model-default split, but the full local suite exposed additional regressions in tool-result patching, doom-loop polling detection, sandbox reuse messaging, and async test support. This follow-up commit restores the missing helpers and updates those production paths so the new regression tests pass for real. Constraint: Provider message histories must keep tool_use/tool_result pairing valid across interrupted turns Constraint: Legitimate polling with changing results must not trip doom-loop recovery Rejected: Only fix the original collection blocker | leaves the full suite red and the PR note stale Rejected: Silence the failing tests without restoring runtime helpers | would hide real production regressions Confidence: high Scope-risk: moderate Reversibility: clean Directive: Keep the local regression tests in sync with the production recovery paths they exercise Tested: python -m compileall agent/context_manager/manager.py agent/core/agent_loop.py agent/core/doom_loop.py agent/tools/sandbox_tool.py backend/user_quotas.py Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --with pytest --with pytest-asyncio python -m pytest -q tests/unit/test_dangling_tool_calls.py tests/unit/test_doom_loop_polling.py tests/unit/test_sandbox_already_active_message.py tests/unit/test_user_quotas.py Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --with pytest --with pytest-asyncio python -m pytest -q Not-tested: Remote CI environment parity * Tighten rate-limit retries and drop the orphaned shared config The review was right about two follow-up issues: the old shared config file was still present after the CLI/frontend split, and the Bedrock rate-limit retry schedule still had a dead third entry because the loop only ever consumed two retry delays. This commit removes the orphaned config and makes the rate-limit schedule line up with the actual retry budget. Constraint: Retry budget for Bedrock token throttling must exceed the provider's ~60s bucket recovery window in the retries that actually run Rejected: Keep a third delay entry in the schedule | the current retry loop never reaches it Confidence: high Scope-risk: narrow Reversibility: clean Directive: Keep retry schedules aligned with the retry loop's real number of sleeps, not the raw retry constant count Tested: python -m compileall agent/core/agent_loop.py tests/unit/test_llm_error_classification.py Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --with pytest --with pytest-asyncio python -m pytest -q tests/unit/test_llm_error_classification.py Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --with pytest --with pytest-asyncio python -m pytest -q Not-tested: Remote CI environment parity --- README.md | 3 +- agent/context_manager/manager.py | 68 +++++---- agent/core/agent_loop.py | 143 +++++++++++++++++- agent/core/doom_loop.py | 28 +++- agent/main.py | 18 ++- agent/tools/research_tool.py | 4 +- agent/tools/sandbox_tool.py | 12 +- agent/utils/terminal_display.py | 20 +-- backend/session_manager.py | 2 +- ...gent_config.json => cli_agent_config.json} | 0 configs/frontend_agent_config.json | 14 ++ frontend/src/components/Chat/ChatInput.tsx | 6 +- frontend/src/utils/model.ts | 7 +- pyproject.toml | 4 + tests/unit/test_cli_rendering.py | 44 ++++++ tests/unit/test_dangling_tool_calls.py | 121 +++++++++++++++ tests/unit/test_doom_loop_polling.py | 96 ++++++++++++ tests/unit/test_llm_error_classification.py | 100 ++++++++++++ tests/unit/test_malformed_args_recovery.py | 66 ++++++++ .../test_sandbox_already_active_message.py | 47 ++++++ uv.lock | 16 ++ 21 files changed, 743 insertions(+), 76 deletions(-) rename configs/{main_agent_config.json => cli_agent_config.json} (100%) create mode 100644 configs/frontend_agent_config.json create mode 100644 tests/unit/test_cli_rendering.py create mode 100644 tests/unit/test_dangling_tool_calls.py create mode 100644 tests/unit/test_doom_loop_polling.py create mode 100644 tests/unit/test_llm_error_classification.py create mode 100644 tests/unit/test_malformed_args_recovery.py create mode 100644 tests/unit/test_sandbox_already_active_message.py diff --git a/README.md b/README.md index 20d95cd49..8e46063a2 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,8 @@ def create_builtin_tools() -> list[ToolSpec]: ### Adding MCP Servers -Edit `configs/main_agent_config.json`: +Edit `configs/cli_agent_config.json` for CLI defaults, or +`configs/frontend_agent_config.json` for web-session defaults: ```json { diff --git a/agent/context_manager/manager.py b/agent/context_manager/manager.py index 373fb11ec..64584b6d5 100644 --- a/agent/context_manager/manager.py +++ b/agent/context_manager/manager.py @@ -253,45 +253,49 @@ def _normalize_tool_calls(msg: Message) -> None: def _patch_dangling_tool_calls(self) -> None: """Add stub tool results for any tool_calls that lack a matching result. - Scans backwards to find the last assistant message with tool_calls, - which may not be items[-1] if some tool results were already added. + Ensures each assistant message's tool_calls are followed immediately + by matching tool-result messages. This has to work across the whole + history, not just the most recent turn, because a cancelled tool use + in an earlier turn can still poison the next provider request. """ if not self.items: return - # Find the last assistant message with tool_calls - assistant_msg = None - for i in range(len(self.items) - 1, -1, -1): + i = 0 + while i < len(self.items): msg = self.items[i] - if getattr(msg, "role", None) == "assistant" and getattr( - msg, "tool_calls", None - ): - assistant_msg = msg - break - # Stop scanning once we hit a user message — anything before - # that belongs to a previous (complete) turn. - if getattr(msg, "role", None) == "user": - break + if getattr(msg, "role", None) != "assistant" or not getattr(msg, "tool_calls", None): + i += 1 + continue + + self._normalize_tool_calls(msg) + + # Consume the contiguous tool-result block that immediately follows + # this assistant message. Any missing tool ids must be inserted + # before the next non-tool message to satisfy provider ordering. + j = i + 1 + immediate_ids: set[str | None] = set() + while j < len(self.items) and getattr(self.items[j], "role", None) == "tool": + immediate_ids.add(getattr(self.items[j], "tool_call_id", None)) + j += 1 + + missing: list[Message] = [] + for tc in msg.tool_calls: + if tc.id not in immediate_ids: + missing.append( + Message( + role="tool", + content="Tool was not executed (interrupted or error).", + tool_call_id=tc.id, + name=tc.function.name, + ) + ) - if not assistant_msg: - return + if missing: + self.items[j:j] = missing + j += len(missing) - self._normalize_tool_calls(assistant_msg) - answered_ids = { - getattr(m, "tool_call_id", None) - for m in self.items - if getattr(m, "role", None) == "tool" - } - for tc in assistant_msg.tool_calls: - if tc.id not in answered_ids: - self.items.append( - Message( - role="tool", - content="Tool was not executed (interrupted or error).", - tool_call_id=tc.id, - name=tc.function.name, - ) - ) + i = j def undo_last_turn(self) -> bool: """Remove the last complete turn (user msg + all assistant/tool msgs that follow). diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index 3d68d54d8..fae5465b8 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -25,6 +25,61 @@ ToolCall = ChatCompletionMessageToolCall +_MALFORMED_TOOL_PREFIX = "ERROR: Tool call to '" +_MALFORMED_TOOL_SUFFIX = "' had malformed JSON arguments" + + +def _malformed_tool_name(message: Message) -> str | None: + """Return the tool name for malformed-json tool-result messages.""" + if getattr(message, "role", None) != "tool": + return None + content = getattr(message, "content", None) + if not isinstance(content, str): + return None + if not content.startswith(_MALFORMED_TOOL_PREFIX): + return None + end = content.find(_MALFORMED_TOOL_SUFFIX, len(_MALFORMED_TOOL_PREFIX)) + if end == -1: + return None + return content[len(_MALFORMED_TOOL_PREFIX):end] + + +def _detect_repeated_malformed( + items: list[Message], threshold: int = 2, +) -> str | None: + """Return the repeated malformed tool name if the tail contains a streak. + + Walk backward over the current conversation tail. A streak counts only + consecutive malformed tool-result messages for the same tool; any other + tool result breaks it. + """ + if threshold <= 0: + return None + + streak_tool: str | None = None + streak = 0 + + for item in reversed(items): + if getattr(item, "role", None) != "tool": + continue + + malformed_tool = _malformed_tool_name(item) + if malformed_tool is None: + break + + if streak_tool is None: + streak_tool = malformed_tool + streak = 1 + elif malformed_tool == streak_tool: + streak += 1 + else: + break + + if streak >= threshold: + return streak_tool + + return None + def _validate_tool_args(tool_args: dict) -> tuple[bool, str | None]: """ @@ -121,6 +176,54 @@ def _needs_approval( # -- LLM retry constants -------------------------------------------------- _MAX_LLM_RETRIES = 3 _LLM_RETRY_DELAYS = [5, 15, 30] # seconds between retries +_LLM_RATE_LIMIT_RETRY_DELAYS = [30, 60] # exceed Bedrock's ~60s TPM bucket window + + +def _is_rate_limit_error(error: Exception) -> bool: + """Return True for rate-limit / quota-bucket style provider errors.""" + err_str = str(error).lower() + rate_limit_patterns = [ + "429", + "rate limit", + "rate_limit", + "too many requests", + "too many tokens", + "request limit", + "throttl", + ] + return any(pattern in err_str for pattern in rate_limit_patterns) + + +def _is_context_overflow_error(error: Exception) -> bool: + """Return True when the prompt exceeded the model's context window.""" + if isinstance(error, ContextWindowExceededError): + return True + + err_str = str(error).lower() + overflow_patterns = [ + "context window exceeded", + "maximum context length", + "max context length", + "prompt is too long", + "context length exceeded", + "too many input tokens", + "input is too long", + ] + return any(pattern in err_str for pattern in overflow_patterns) + + +def _retry_delay_for(error: Exception, attempt_index: int) -> int | None: + """Return the delay for this retry attempt, or None if it should not retry.""" + if _is_rate_limit_error(error): + schedule = _LLM_RATE_LIMIT_RETRY_DELAYS + elif _is_transient_error(error): + schedule = _LLM_RETRY_DELAYS + else: + return None + + if attempt_index >= len(schedule): + return None + return schedule[attempt_index] def _is_transient_error(error: Exception) -> bool: @@ -128,7 +231,6 @@ def _is_transient_error(error: Exception) -> bool: err_str = str(error).lower() transient_patterns = [ "timeout", "timed out", - "429", "rate limit", "rate_limit", "503", "service unavailable", "502", "bad gateway", "500", "internal server error", @@ -136,7 +238,7 @@ def _is_transient_error(error: Exception) -> bool: "connection reset", "connection refused", "connection error", "eof", "broken pipe", ] - return any(pattern in err_str for pattern in transient_patterns) + return _is_rate_limit_error(error) or any(pattern in err_str for pattern in transient_patterns) def _is_effort_config_error(error: Exception) -> bool: @@ -317,6 +419,8 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> except ContextWindowExceededError: raise except Exception as e: + if _is_context_overflow_error(e): + raise ContextWindowExceededError(str(e)) from e if not _healed_effort and _is_effort_config_error(e): _healed_effort = True llm_params = await _heal_effort_and_rebuild_params(session, e, llm_params) @@ -325,8 +429,8 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> data={"tool": "system", "log": "Reasoning effort not supported for this model — adjusting and retrying."}, )) continue - if _llm_attempt < _MAX_LLM_RETRIES - 1 and _is_transient_error(e): - _delay = _LLM_RETRY_DELAYS[_llm_attempt] + _delay = _retry_delay_for(e, _llm_attempt) + if _llm_attempt < _MAX_LLM_RETRIES - 1 and _delay is not None: logger.warning( "Transient LLM error (attempt %d/%d): %s — retrying in %ds", _llm_attempt + 1, _MAX_LLM_RETRIES, e, _delay, @@ -424,6 +528,8 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) except ContextWindowExceededError: raise except Exception as e: + if _is_context_overflow_error(e): + raise ContextWindowExceededError(str(e)) from e if not _healed_effort and _is_effort_config_error(e): _healed_effort = True llm_params = await _heal_effort_and_rebuild_params(session, e, llm_params) @@ -432,8 +538,8 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) data={"tool": "system", "log": "Reasoning effort not supported for this model — adjusting and retrying."}, )) continue - if _llm_attempt < _MAX_LLM_RETRIES - 1 and _is_transient_error(e): - _delay = _LLM_RETRY_DELAYS[_llm_attempt] + _delay = _retry_delay_for(e, _llm_attempt) + if _llm_attempt < _MAX_LLM_RETRIES - 1 and _delay is not None: logger.warning( "Transient LLM error (attempt %d/%d): %s — retrying in %ds", _llm_attempt + 1, _MAX_LLM_RETRIES, e, _delay, @@ -585,6 +691,31 @@ async def run_agent( ) ) + malformed_tool = _detect_repeated_malformed(session.context_manager.items) + if malformed_tool: + recovery_prompt = ( + "[SYSTEM: Repeated malformed tool arguments detected for " + f"'{malformed_tool}'. Stop retrying the same tool call shape. " + "Use a different strategy that produces smaller, valid JSON. " + "For large file writes, prefer bash with a heredoc or split the " + "edit into multiple smaller tool calls.]" + ) + session.context_manager.add_message( + Message(role="user", content=recovery_prompt) + ) + await session.send_event( + Event( + event_type="tool_log", + data={ + "tool": "system", + "log": ( + "Repeated malformed tool arguments detected — " + f"forcing a different strategy for {malformed_tool}" + ), + }, + ) + ) + messages = session.context_manager.get_messages() tools = session.tool_router.get_tool_specs_for_llm() try: diff --git a/agent/core/doom_loop.py b/agent/core/doom_loop.py index 5050d7550..fbc3510a1 100644 --- a/agent/core/doom_loop.py +++ b/agent/core/doom_loop.py @@ -17,10 +17,11 @@ @dataclass(frozen=True) class ToolCallSignature: - """Hashable signature for a single tool call (name + args hash).""" + """Hashable signature for a single tool call plus its observed result.""" name: str args_hash: str + result_hash: str | None = None def _hash_args(args_str: str) -> str: @@ -31,11 +32,16 @@ def _hash_args(args_str: str) -> str: def extract_recent_tool_signatures( messages: list[Message], lookback: int = 30 ) -> list[ToolCallSignature]: - """Extract tool call signatures from recent assistant messages.""" + """Extract tool call signatures from recent assistant messages. + + Includes the immediate tool result hash when present. This prevents + legitimate polling from being classified as a doom loop when the poll + arguments stay constant but the observed result keeps changing. + """ signatures: list[ToolCallSignature] = [] recent = messages[-lookback:] if len(messages) > lookback else messages - for msg in recent: + for idx, msg in enumerate(recent): if getattr(msg, "role", None) != "assistant": continue tool_calls = getattr(msg, "tool_calls", None) @@ -47,7 +53,21 @@ def extract_recent_tool_signatures( continue name = getattr(fn, "name", "") or "" args_str = getattr(fn, "arguments", "") or "" - signatures.append(ToolCallSignature(name=name, args_hash=_hash_args(args_str))) + result_hash = None + for follow in recent[idx + 1:]: + role = getattr(follow, "role", None) + if role == "tool" and getattr(follow, "tool_call_id", None) == getattr(tc, "id", None): + result_hash = _hash_args(str(getattr(follow, "content", "") or "")) + break + if role in {"assistant", "user"}: + break + signatures.append( + ToolCallSignature( + name=name, + args_hash=_hash_args(args_str), + result_hash=result_hash, + ) + ) return signatures diff --git a/agent/main.py b/agent/main.py index 4b08c7266..fd13e8dc6 100644 --- a/agent/main.py +++ b/agent/main.py @@ -50,6 +50,16 @@ # on every error — users don't need it, and our friendly errors cover the case. litellm.suppress_debug_info = True +CLI_CONFIG_PATH = Path(__file__).parent.parent / "configs" / "cli_agent_config.json" + + +def _configure_runtime_logging() -> None: + """Keep third-party warning spam from punching through the interactive UI.""" + import logging + + logging.getLogger("LiteLLM").setLevel(logging.ERROR) + logging.getLogger("litellm").setLevel(logging.ERROR) + def _safe_get_args(arguments: dict) -> dict: """Safely extract args dict from arguments, handling cases where LLM passes string.""" args = arguments.get("args", {}) @@ -846,8 +856,7 @@ async def main(): ready_event = asyncio.Event() # Start agent loop in background - config_path = Path(__file__).parent.parent / "configs" / "main_agent_config.json" - config = load_config(config_path) + config = load_config(CLI_CONFIG_PATH) # Create tool router with local mode tool_router = ToolRouter(config.mcpServers, hf_token=hf_token, local_mode=True) @@ -1037,6 +1046,7 @@ async def headless_main( import logging logging.basicConfig(level=logging.WARNING) + _configure_runtime_logging() hf_token = _get_hf_token() if not hf_token: @@ -1045,8 +1055,7 @@ async def headless_main( print(f"HF token loaded", file=sys.stderr) - config_path = Path(__file__).parent.parent / "configs" / "main_agent_config.json" - config = load_config(config_path) + config = load_config(CLI_CONFIG_PATH) config.yolo_mode = True # Auto-approve everything in headless mode if model: @@ -1222,6 +1231,7 @@ def cli(): import warnings # Suppress aiohttp "Unclosed client session" noise during event loop teardown _logging.getLogger("asyncio").setLevel(_logging.CRITICAL) + _configure_runtime_logging() # Suppress litellm pydantic deprecation warnings warnings.filterwarnings("ignore", category=DeprecationWarning, module="litellm") # Suppress whoosh invalid escape sequence warnings (third-party, unfixed upstream) diff --git a/agent/tools/research_tool.py b/agent/tools/research_tool.py index 79692383a..18ae2ad65 100644 --- a/agent/tools/research_tool.py +++ b/agent/tools/research_tool.py @@ -216,7 +216,9 @@ def _get_research_model(main_model: str) -> str: """Pick a cheaper model for research based on the main model.""" - if "anthropic" in main_model: + if main_model.startswith("anthropic/"): + return "anthropic/claude-sonnet-4-6" + if main_model.startswith("bedrock/") and "anthropic" in main_model: return "bedrock/us.anthropic.claude-sonnet-4-6" # For non-Anthropic models (HF router etc.), use the same model return main_model diff --git a/agent/tools/sandbox_tool.py b/agent/tools/sandbox_tool.py index be38c7340..6dfd3db19 100644 --- a/agent/tools/sandbox_tool.py +++ b/agent/tools/sandbox_tool.py @@ -213,16 +213,26 @@ async def sandbox_create_handler( args: dict[str, Any], session: Any = None ) -> tuple[str, bool]: """Handle sandbox_create tool calls.""" + hardware = args.get("hardware", "cpu-basic") + # If sandbox already exists, return its info if session and getattr(session, "sandbox", None): sb = session.sandbox + requested_hardware = args.get("hardware") + lockout_note = "" + if requested_hardware: + lockout_note = ( + f"\nRequested hardware: {requested_hardware}\n" + "Hardware cannot be changed by calling sandbox_create again. " + "Delete the existing sandbox first if you need a different tier." + ) return ( f"Sandbox already active: {sb.space_id}\n" f"URL: {sb.url}\n" + f"{lockout_note}\n" f"Use bash/read/write/edit to interact with it." ), True - hardware = args.get("hardware", "cpu-basic") create_kwargs = {} if "private" in args: create_kwargs["private"] = args["private"] diff --git a/agent/utils/terminal_display.py b/agent/utils/terminal_display.py index 34d879108..3509a13ec 100644 --- a/agent/utils/terminal_display.py +++ b/agent/utils/terminal_display.py @@ -180,10 +180,8 @@ class SubAgentDisplayManager: def __init__(self): self._agents: dict[str, dict] = {} # agent_id -> state dict self._lines_on_screen = 0 - self._ticker_task = None def start(self, agent_id: str, label: str = "research") -> None: - import asyncio import time self._agents[agent_id] = { "label": label, @@ -192,8 +190,6 @@ def start(self, agent_id: str, label: str = "research") -> None: "token_count": 0, "start_time": time.monotonic(), } - if not self._ticker_task: - self._ticker_task = asyncio.ensure_future(self._tick()) self._redraw() def set_tokens(self, agent_id: str, tokens: int) -> None: @@ -222,11 +218,7 @@ def clear(self, agent_id: str) -> None: _console.file.write(line + "\n") _console.file.flush() self._lines_on_screen = 0 - if not self._agents: - if self._ticker_task: - self._ticker_task.cancel() - self._ticker_task = None - else: + if self._agents: self._redraw() @staticmethod @@ -239,16 +231,6 @@ def _render_completion_line(agent: dict) -> str: line += f" \033[2m({stats})\033[0m" return line - async def _tick(self) -> None: - import asyncio - try: - while True: - await asyncio.sleep(1.0) - if self._agents: - self._redraw() - except asyncio.CancelledError: - pass - @staticmethod def _format_stats(agent: dict) -> str: import time diff --git a/backend/session_manager.py b/backend/session_manager.py index d52cd2754..68177fc12 100644 --- a/backend/session_manager.py +++ b/backend/session_manager.py @@ -15,7 +15,7 @@ # Get project root (parent of backend directory) PROJECT_ROOT = Path(__file__).parent.parent -DEFAULT_CONFIG_PATH = str(PROJECT_ROOT / "configs" / "main_agent_config.json") +DEFAULT_CONFIG_PATH = str(PROJECT_ROOT / "configs" / "frontend_agent_config.json") # These dataclasses match agent/main.py structure diff --git a/configs/main_agent_config.json b/configs/cli_agent_config.json similarity index 100% rename from configs/main_agent_config.json rename to configs/cli_agent_config.json diff --git a/configs/frontend_agent_config.json b/configs/frontend_agent_config.json new file mode 100644 index 000000000..c73ea380f --- /dev/null +++ b/configs/frontend_agent_config.json @@ -0,0 +1,14 @@ +{ + "model_name": "bedrock/us.anthropic.claude-opus-4-6-v1", + "save_sessions": true, + "session_dataset_repo": "smolagents/ml-intern-sessions", + "yolo_mode": false, + "confirm_cpu_jobs": true, + "auto_file_upload": true, + "mcpServers": { + "hf-mcp-server": { + "transport": "http", + "url": "https://huggingface.co/mcp?login" + } + } +} diff --git a/frontend/src/components/Chat/ChatInput.tsx b/frontend/src/components/Chat/ChatInput.tsx index d9fe5c4dc..13a6f5443 100644 --- a/frontend/src/components/Chat/ChatInput.tsx +++ b/frontend/src/components/Chat/ChatInput.tsx @@ -7,7 +7,7 @@ import { apiFetch } from '@/utils/api'; import { useUserQuota } from '@/hooks/useUserQuota'; import ClaudeCapDialog from '@/components/ClaudeCapDialog'; import { useAgentStore } from '@/store/agentStore'; -import { FIRST_FREE_MODEL_PATH } from '@/utils/model'; +import { CLAUDE_MODEL_PATH, FIRST_FREE_MODEL_PATH, isClaudePath } from '@/utils/model'; // Model configuration interface ModelOption { @@ -37,7 +37,7 @@ const MODEL_OPTIONS: ModelOption[] = [ id: 'claude-opus', name: 'Claude Opus 4.6', description: 'Anthropic', - modelPath: 'anthropic/claude-opus-4-6', + modelPath: CLAUDE_MODEL_PATH, avatarUrl: 'https://huggingface.co/api/avatars/Anthropic', recommended: true, }, @@ -70,7 +70,7 @@ interface ChatInputProps { placeholder?: string; } -const isClaudeModel = (m: ModelOption) => m.modelPath.startsWith('anthropic/'); +const isClaudeModel = (m: ModelOption) => isClaudePath(m.modelPath); const firstFreeModel = () => MODEL_OPTIONS.find(m => !isClaudeModel(m)) ?? MODEL_OPTIONS[0]; export default function ChatInput({ sessionId, onSend, onStop, isProcessing = false, disabled = false, placeholder = 'Ask anything...' }: ChatInputProps) { diff --git a/frontend/src/utils/model.ts b/frontend/src/utils/model.ts index 89f23fe71..37cdece00 100644 --- a/frontend/src/utils/model.ts +++ b/frontend/src/utils/model.ts @@ -3,13 +3,12 @@ * ClaudeCapDialog "Use a free model" escape hatch. * * Keep in sync with MODEL_OPTIONS in components/Chat/ChatInput.tsx and - * AVAILABLE_MODELS in backend/routes/agent.py. Bare HF ids (no - * `huggingface/` prefix) — matches upstream's auto-router. + * AVAILABLE_MODELS in backend/routes/agent.py. */ -export const CLAUDE_MODEL_PATH = 'anthropic/claude-opus-4-6'; +export const CLAUDE_MODEL_PATH = 'bedrock/us.anthropic.claude-opus-4-6-v1'; export const FIRST_FREE_MODEL_PATH = 'moonshotai/Kimi-K2.6'; export function isClaudePath(modelPath: string | undefined): boolean { - return !!modelPath && modelPath.startsWith('anthropic/'); + return !!modelPath && modelPath.includes('anthropic'); } diff --git a/pyproject.toml b/pyproject.toml index 4b263213e..89cadf94b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ eval = [ # Development and testing dependencies dev = [ "pytest>=9.0.2", + "pytest-asyncio>=0.26.0", ] # All dependencies (eval + dev) @@ -61,3 +62,6 @@ include = ["agent*"] [tool.uv] package = true + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/tests/unit/test_cli_rendering.py b/tests/unit/test_cli_rendering.py new file mode 100644 index 000000000..7704afd58 --- /dev/null +++ b/tests/unit/test_cli_rendering.py @@ -0,0 +1,44 @@ +"""Regression tests for interactive CLI rendering and research model routing.""" + +from io import StringIO +from types import SimpleNamespace + +from agent.tools.research_tool import _get_research_model +from agent.utils import terminal_display + + +def test_direct_anthropic_research_model_stays_off_bedrock(): + assert _get_research_model("anthropic/claude-opus-4-6") == "anthropic/claude-sonnet-4-6" + + +def test_bedrock_anthropic_research_model_stays_on_bedrock(): + assert ( + _get_research_model("bedrock/us.anthropic.claude-opus-4-6-v1") + == "bedrock/us.anthropic.claude-sonnet-4-6" + ) + + +def test_non_anthropic_research_model_is_unchanged(): + assert _get_research_model("openai/gpt-5.4") == "openai/gpt-5.4" + + +def test_subagent_display_does_not_spawn_background_redraw(monkeypatch): + calls: list[object] = [] + + def _unexpected_future(*args, **kwargs): + calls.append((args, kwargs)) + raise AssertionError("background redraw task should not be created") + + monkeypatch.setattr("asyncio.ensure_future", _unexpected_future) + monkeypatch.setattr( + terminal_display, + "_console", + SimpleNamespace(file=StringIO(), width=100), + ) + + mgr = terminal_display.SubAgentDisplayManager() + mgr.start("agent-1", "research") + mgr.add_call("agent-1", "ā–ø hf_papers {\"operation\": \"search\"}") + mgr.clear("agent-1") + + assert calls == [] diff --git a/tests/unit/test_dangling_tool_calls.py b/tests/unit/test_dangling_tool_calls.py new file mode 100644 index 000000000..1e8ac3fa2 --- /dev/null +++ b/tests/unit/test_dangling_tool_calls.py @@ -0,0 +1,121 @@ +"""Regression tests for `_patch_dangling_tool_calls`. + +Reproduces the failure mode behind observatory sessions 8dd2ce30 and +59c9e678 (2026-04-25): a tool call cancelled mid-execution leaves an +orphan ``tool_use`` in history; the user types a follow-up; Bedrock +rejects the next request with HTTP 400 ``messages.N: tool_use ids were +found without tool_result blocks immediately after``. +""" + +from litellm import ChatCompletionMessageToolCall, Message + +from agent.context_manager.manager import ContextManager + + +def _tool_call(call_id: str, name: str = "research") -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall( + id=call_id, + type="function", + function={"name": name, "arguments": "{}"}, + ) + + +def _make_cm() -> ContextManager: + cm = ContextManager.__new__(ContextManager) + cm.system_prompt = "system" + cm.model_max_tokens = 100_000 + cm.compact_size = 1_000 + cm.running_context_usage = 0 + cm.untouched_messages = 5 + cm.items = [Message(role="system", content="system")] + return cm + + +def test_orphan_tool_use_followed_by_user_message_is_patched(): + cm = _make_cm() + cm.items.extend([ + Message(role="user", content="Research X"), + Message( + role="assistant", + content=None, + tool_calls=[_tool_call("call_abc", "research")], + ), + Message(role="user", content="??"), + ]) + msgs = cm.get_messages() + tool_msgs = [m for m in msgs if getattr(m, "role", None) == "tool"] + assert len(tool_msgs) == 1 + assert tool_msgs[0].tool_call_id == "call_abc" + assert "interrupted" in (tool_msgs[0].content or "").lower() or "not executed" in (tool_msgs[0].content or "").lower() + + +def test_no_orphan_means_no_stub(): + cm = _make_cm() + cm.items.extend([ + Message(role="user", content="Research X"), + Message( + role="assistant", + content=None, + tool_calls=[_tool_call("call_abc", "research")], + ), + Message(role="tool", content="ok", tool_call_id="call_abc", name="research"), + ]) + cm.get_messages() + tool_msgs = [m for m in cm.items if getattr(m, "role", None) == "tool"] + assert len(tool_msgs) == 1 + assert tool_msgs[0].content == "ok" + + +def test_multiple_dangling_tool_calls_in_one_assistant_message_are_all_patched(): + cm = _make_cm() + cm.items.extend([ + Message(role="user", content="do two things"), + Message( + role="assistant", + content=None, + tool_calls=[ + _tool_call("call_1", "research"), + _tool_call("call_2", "bash"), + ], + ), + Message(role="user", content="follow up"), + ]) + cm.get_messages() + tool_ids = { + getattr(m, "tool_call_id", None) + for m in cm.items + if getattr(m, "role", None) == "tool" + } + assert tool_ids == {"call_1", "call_2"} + + +def test_orphan_in_earlier_turn_still_gets_patched(): + """Two-turn history where the FIRST turn was interrupted. + + Old patcher stopped at the first user msg encountered while scanning + backwards, so this case never got fixed and Bedrock rejected. + """ + cm = _make_cm() + cm.items.extend([ + Message(role="user", content="turn 1"), + Message( + role="assistant", + content=None, + tool_calls=[_tool_call("call_old", "research")], + ), + Message(role="user", content="turn 2 — please retry"), + Message( + role="assistant", + content=None, + tool_calls=[_tool_call("call_new", "bash")], + ), + Message(role="tool", content="ok", tool_call_id="call_new", name="bash"), + ]) + cm.get_messages() + tool_ids = { + getattr(m, "tool_call_id", None) + for m in cm.items + if getattr(m, "role", None) == "tool" + } + assert "call_old" in tool_ids + assert "call_new" in tool_ids diff --git a/tests/unit/test_doom_loop_polling.py b/tests/unit/test_doom_loop_polling.py new file mode 100644 index 000000000..0142f4591 --- /dev/null +++ b/tests/unit/test_doom_loop_polling.py @@ -0,0 +1,96 @@ +"""Regression test for doom-loop false-positive on legitimate polling. + +Reproduces the failure mode in observatory sessions 40fcb414 ($32.59), +8e90352e ($62.63), and 403178bf ($5.71) on 2026-04-25: the agent polled a +long-running job with `bash sleep 300 && wc -l output` four times in a +row. The arguments were byte-identical, but the results moved (27210 → +36454 → 45770 → 55138 — actual progress). The detector hashed args only +and false-fired DOOM LOOP, which made the agent abandon perfectly valid +polling. + +After the fix the signature includes the tool result hash, so identical +args + different results no longer trips the detector. +""" + +from litellm import ChatCompletionMessageToolCall, Message + +from agent.core.doom_loop import check_for_doom_loop + + +def _assistant(call_id: str, name: str, args: str) -> Message: + return Message( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionMessageToolCall( + id=call_id, + type="function", + function={"name": name, "arguments": args}, + ) + ], + ) + + +def _tool(call_id: str, name: str, content: str) -> Message: + return Message(role="tool", content=content, tool_call_id=call_id, name=name) + + +_POLL_ARGS = '{"command": "sleep 300 && ls /app/images/ | wc -l"}' + + +def test_polling_with_progressing_results_does_not_fire(): + msgs = [ + Message(role="user", content="run the job"), + _assistant("c1", "bash", _POLL_ARGS), + _tool("c1", "bash", "27210"), + _assistant("c2", "bash", _POLL_ARGS), + _tool("c2", "bash", "36454"), + _assistant("c3", "bash", _POLL_ARGS), + _tool("c3", "bash", "45770"), + _assistant("c4", "bash", _POLL_ARGS), + _tool("c4", "bash", "55138"), + ] + assert check_for_doom_loop(msgs) is None + + +def test_truly_stuck_polling_with_identical_results_still_fires(): + """If the same poll returns the same number, the job is genuinely + stuck and the detector SHOULD fire.""" + msgs = [ + _assistant("c1", "bash", _POLL_ARGS), + _tool("c1", "bash", "55138"), + _assistant("c2", "bash", _POLL_ARGS), + _tool("c2", "bash", "55138"), + _assistant("c3", "bash", _POLL_ARGS), + _tool("c3", "bash", "55138"), + ] + prompt = check_for_doom_loop(msgs) + assert prompt is not None + assert "DOOM LOOP" in prompt + assert "bash" in prompt + + +def test_identical_calls_with_no_results_yet_still_fires(): + """If three identical calls have no tool results (e.g. all cancelled + or errored before a result was recorded), treat as a real loop.""" + msgs = [ + _assistant("c1", "write", '{"path": "/tmp/x", "content": "..."}'), + _assistant("c2", "write", '{"path": "/tmp/x", "content": "..."}'), + _assistant("c3", "write", '{"path": "/tmp/x", "content": "..."}'), + ] + prompt = check_for_doom_loop(msgs) + assert prompt is not None + assert "DOOM LOOP" in prompt + assert "write" in prompt + + +def test_different_args_does_not_fire(): + msgs = [ + _assistant("c1", "bash", '{"command": "ls /a"}'), + _tool("c1", "bash", "ok"), + _assistant("c2", "bash", '{"command": "ls /b"}'), + _tool("c2", "bash", "ok"), + _assistant("c3", "bash", '{"command": "ls /c"}'), + _tool("c3", "bash", "ok"), + ] + assert check_for_doom_loop(msgs) is None diff --git a/tests/unit/test_llm_error_classification.py b/tests/unit/test_llm_error_classification.py new file mode 100644 index 000000000..0b0195740 --- /dev/null +++ b/tests/unit/test_llm_error_classification.py @@ -0,0 +1,100 @@ +"""Tests for LLM error classification helpers in agent.core.agent_loop. + +Covers two regressions on 2026-04-25: + +1. Non-Anthropic context overflow (Kimi 365k > 262k) was not classified as + ``_is_context_overflow_error``, so the recovery path didn't fire and + session 62ccfdcb died with 68 wasted compaction events. + +2. Bedrock TPM rate limit (`Too many tokens, please wait before trying + again.`) needs the longer rate-limit retry schedule. The old schedule + ([5, 15, 30] = 50s) burned through 6 sessions costing >$2,400 combined + on the same day. +""" + +from agent.core.agent_loop import ( + _MAX_LLM_RETRIES, + _LLM_RATE_LIMIT_RETRY_DELAYS, + _LLM_RETRY_DELAYS, + _is_context_overflow_error, + _is_rate_limit_error, + _is_transient_error, + _retry_delay_for, +) + + +# ── context overflow ──────────────────────────────────────────────────── + + +def test_kimi_prompt_too_long_is_context_overflow(): + # Verbatim error text from session 62ccfdcb (2026-04-25, Kimi K2.6). + err = Exception( + "litellm.BadRequestError: OpenAIException - The prompt is too long: " + "365407, model maximum context length: 262143" + ) + assert _is_context_overflow_error(err) + + +def test_openai_context_length_exceeded_is_context_overflow(): + err = Exception("Error: This model's maximum context length is 8192 tokens.") + assert _is_context_overflow_error(err) + + +def test_random_error_is_not_context_overflow(): + err = Exception("connection reset by peer") + assert not _is_context_overflow_error(err) + + +# ── rate limit ────────────────────────────────────────────────────────── + + +def test_bedrock_too_many_tokens_is_rate_limit(): + # Verbatim from sessions b37a3823, c4d7a831, b63c4933 (2026-04-25). + err = Exception( + 'litellm.RateLimitError: BedrockException - {"message":"Too many ' + 'tokens, please wait before trying again."}' + ) + assert _is_rate_limit_error(err) + # Rate-limit errors are also classified as transient. + assert _is_transient_error(err) + + +def test_429_is_rate_limit(): + err = Exception("HTTP 429 Too Many Requests") + assert _is_rate_limit_error(err) + + +def test_timeout_is_transient_but_not_rate_limit(): + err = Exception("Request timed out after 600s") + assert _is_transient_error(err) + assert not _is_rate_limit_error(err) + + +# ── retry schedule selection ──────────────────────────────────────────── + + +def test_rate_limit_uses_longer_schedule(): + err = Exception("Too many tokens, please wait before trying again.") + delays = [_retry_delay_for(err, i) for i in range(len(_LLM_RATE_LIMIT_RETRY_DELAYS))] + assert delays == _LLM_RATE_LIMIT_RETRY_DELAYS + # Just past the schedule → None (stop retrying). + assert _retry_delay_for(err, len(_LLM_RATE_LIMIT_RETRY_DELAYS)) is None + + +def test_other_transient_uses_short_schedule(): + err = Exception("503 service unavailable") + delays = [_retry_delay_for(err, i) for i in range(len(_LLM_RETRY_DELAYS))] + assert delays == _LLM_RETRY_DELAYS + assert _retry_delay_for(err, len(_LLM_RETRY_DELAYS)) is None + + +def test_non_transient_returns_none(): + err = Exception("invalid request: bad parameter") + assert _retry_delay_for(err, 0) is None + + +def test_rate_limit_total_budget_covers_bedrock_bucket_recovery(): + """The whole point of the rate-limit schedule: total wait time should + exceed the ~60s Bedrock TPM bucket recovery window.""" + assert len(_LLM_RATE_LIMIT_RETRY_DELAYS) == _MAX_LLM_RETRIES - 1 + assert sum(_LLM_RATE_LIMIT_RETRY_DELAYS) > 60 diff --git a/tests/unit/test_malformed_args_recovery.py b/tests/unit/test_malformed_args_recovery.py new file mode 100644 index 000000000..3eaab91d3 --- /dev/null +++ b/tests/unit/test_malformed_args_recovery.py @@ -0,0 +1,66 @@ +"""Regression test for the malformed-JSON loop in observatory session +7750e82f (2026-04-25): GLM-5.1 produced six consecutive ``write`` calls +whose ``arguments`` strings JSON-parse-failed (truncated mid-stream by +the provider). The soft retry hint didn't move the model. The detector +in ``_detect_repeated_malformed`` looks for the streak so the agent loop +can inject a hard system-prompt forcing a different strategy. +""" + +from litellm import Message + +from agent.core.agent_loop import _detect_repeated_malformed + + +def _malformed_tool_msg(name: str, call_id: str) -> Message: + return Message( + role="tool", + content=( + f"ERROR: Tool call to '{name}' had malformed JSON arguments and " + f"was NOT executed. Retry with smaller content — for 'write', " + f"split into multiple smaller writes using 'edit'." + ), + tool_call_id=call_id, + name=name, + ) + + +def test_two_consecutive_malformed_same_tool_triggers(): + items = [ + Message(role="user", content="write a big plan"), + Message(role="assistant", content=None), + _malformed_tool_msg("write", "1"), + Message(role="assistant", content=None), + _malformed_tool_msg("write", "2"), + ] + assert _detect_repeated_malformed(items, threshold=2) == "write" + + +def test_one_malformed_does_not_trigger(): + items = [ + Message(role="user", content="write a plan"), + Message(role="assistant", content=None), + _malformed_tool_msg("write", "1"), + ] + assert _detect_repeated_malformed(items, threshold=2) is None + + +def test_two_malformed_different_tools_does_not_trigger(): + items = [ + Message(role="assistant", content=None), + _malformed_tool_msg("write", "1"), + Message(role="assistant", content=None), + _malformed_tool_msg("bash", "2"), + ] + assert _detect_repeated_malformed(items, threshold=2) is None + + +def test_streak_broken_by_successful_tool_call_does_not_trigger(): + items = [ + Message(role="assistant", content=None), + _malformed_tool_msg("write", "1"), + Message(role="assistant", content=None), + Message(role="tool", content="ok", tool_call_id="2", name="write"), + Message(role="assistant", content=None), + _malformed_tool_msg("write", "3"), + ] + assert _detect_repeated_malformed(items, threshold=2) is None diff --git a/tests/unit/test_sandbox_already_active_message.py b/tests/unit/test_sandbox_already_active_message.py new file mode 100644 index 000000000..c4e6f25de --- /dev/null +++ b/tests/unit/test_sandbox_already_active_message.py @@ -0,0 +1,47 @@ +"""Regression test for sandbox_create not surfacing the hardware lockout. + +In observatory session d6f8454c (2026-04-25) the agent called +sandbox_create 18 times across 11 distinct hardware tiers (a10g-large, +a100-large, t4-small, cpu-upgrade, cpu-basic, zero-a10g, l4x1, t4-medium, +a10g-small, l40sx1, …). Every call returned 'Sandbox already active' for +the same sandbox, but the message did not say that hardware can't be +changed by re-calling, so the agent thought "still pending, retry with a +different flavor" and burned 17 useless turns. + +The fix makes the response explicit when the requested hardware differs +from what's already active. +""" + +import asyncio +from types import SimpleNamespace + +from agent.tools.sandbox_tool import sandbox_create_handler + + +def _session_with_sandbox(): + sb = SimpleNamespace( + space_id="user/sandbox-abc123", + url="https://huggingface.co/spaces/user/sandbox-abc123", + ) + return SimpleNamespace(sandbox=sb) + + +def test_already_active_with_different_hw_warns_about_lockout(): + session = _session_with_sandbox() + out, ok = asyncio.run( + sandbox_create_handler({"hardware": "a100-large"}, session=session) + ) + assert ok is True + # The message should mention the lockout AND the requested flavor. + assert "cannot be changed" in out.lower() + assert "a100-large" in out + assert "delete" in out.lower() + + +def test_already_active_no_hw_request_just_returns_handle(): + session = _session_with_sandbox() + out, ok = asyncio.run(sandbox_create_handler({}, session=session)) + assert ok is True + assert "user/sandbox-abc123" in out + # No spurious lockout note when the agent didn't request a flavor. + assert "cannot be changed" not in out.lower() diff --git a/uv.lock b/uv.lock index c214134b5..3bddba0dc 100644 --- a/uv.lock +++ b/uv.lock @@ -1799,10 +1799,12 @@ all = [ { name = "inspect-ai" }, { name = "pandas" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "tenacity" }, ] dev = [ { name = "pytest" }, + { name = "pytest-asyncio" }, ] eval = [ { name = "datasets" }, @@ -1830,6 +1832,7 @@ requires-dist = [ { name = "prompt-toolkit", specifier = ">=3.0.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.26.0" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "requests", specifier = ">=2.33.0" }, { name = "rich", specifier = ">=13.0.0" }, @@ -2789,6 +2792,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From 1bce0eb21a6ccd6b2fbb0bb793fa1eac9b6a25d9 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Sat, 25 Apr 2026 23:04:38 +0300 Subject: [PATCH 007/120] Make the CLI startup banner show the actual CLI default model (#122) The CLI already loads direct Anthropic defaults from configs/cli_agent_config.json, but the startup banner was rendered before that config was loaded and fell back to a hardcoded Bedrock label. This made the CLI look misconfigured even when the session model was correct. Constraint: The startup banner must reflect the same model source the CLI session will actually use Rejected: Keep a hardcoded Bedrock fallback in print_banner | it misreports the active CLI default Confidence: high Scope-risk: narrow Reversibility: clean Directive: Banner/status UI should derive model labels from loaded config or session state, never from stale hardcoded defaults Tested: python -m compileall agent/main.py agent/utils/terminal_display.py Not-tested: Interactive manual CLI launch in this sandbox --- agent/main.py | 7 +++---- agent/utils/terminal_display.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/agent/main.py b/agent/main.py index fd13e8dc6..f601ab545 100644 --- a/agent/main.py +++ b/agent/main.py @@ -831,6 +831,8 @@ async def main(): if not hf_token: hf_token = await _prompt_and_save_hf_token(prompt_session) + config = load_config(CLI_CONFIG_PATH) + # Resolve username for banner hf_user = None try: @@ -839,7 +841,7 @@ async def main(): except Exception: pass - print_banner(hf_user=hf_user) + print_banner(model=config.model_name, hf_user=hf_user) # Pre-warm the HF router catalog in the background so /model switches # don't block on a network fetch. @@ -855,9 +857,6 @@ async def main(): turn_complete_event.set() ready_event = asyncio.Event() - # Start agent loop in background - config = load_config(CLI_CONFIG_PATH) - # Create tool router with local mode tool_router = ToolRouter(config.mcpServers, hf_token=hf_token, local_mode=True) diff --git a/agent/utils/terminal_display.py b/agent/utils/terminal_display.py index 3509a13ec..8ff9d5250 100644 --- a/agent/utils/terminal_display.py +++ b/agent/utils/terminal_display.py @@ -99,7 +99,7 @@ def print_banner(model: str | None = None, hf_user: str | None = None) -> None: _console.file.write("\033[2J\033[H") _console.file.flush() - model_label = model or "bedrock/us.anthropic.claude-opus-4-6-v1" + model_label = model or "unknown" user_label = hf_user or "not logged in" # Warm gold palette matching the shimmer highlight (255, 200, 80) From 4501d6981d417710794f4361d1b65fc9bba2dfbd Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Sat, 25 Apr 2026 23:23:31 +0300 Subject: [PATCH 008/120] Run Claude review for external PRs safely (#123) --- .github/workflows/claude-review.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index a38ce8c37..41bcad621 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -1,8 +1,8 @@ name: Claude PR Review on: - pull_request: - types: [opened, synchronize, ready_for_review] + pull_request_target: + types: [opened, synchronize, ready_for_review, reopened] permissions: contents: read @@ -22,6 +22,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + # On pull_request_target, keep checkout on the trusted base-repo ref. + # The Claude action can review the PR via GitHub context/API without + # executing untrusted fork code with repository secrets. + persist-credentials: false - name: Compose review prompt id: compose From 5b82e2d375bb232ac1375c845fc926a2c7f0ea38 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Sun, 26 Apr 2026 11:09:43 +0300 Subject: [PATCH 009/120] ci: bypass OIDC for Claude review on pull_request_target (#134) The GitHub App token-exchange endpoint rejects OIDC tokens minted for pull_request_target events ('401 Invalid OIDC token'), so every review has failed since the switch from pull_request in #123. Pass GITHUB_TOKEN directly to skip the exchange; comments post as github-actions[bot] instead of claude[bot], which is the documented trade-off. --- .github/workflows/claude-review.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index 41bcad621..d710bd0e1 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -62,5 +62,12 @@ jobs: - uses: anthropics/claude-code-action@v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + # Bypass the OIDC -> Claude GitHub App token exchange. That exchange + # rejects OIDC tokens minted for pull_request_target events with + # "401 Invalid OIDC token", which broke every review after the switch + # away from pull_request. Using the workflow's GITHUB_TOKEN works for + # both same-repo and fork PRs; comments post as github-actions[bot] + # instead of claude[bot], which is the documented trade-off. + github_token: ${{ secrets.GITHUB_TOKEN }} track_progress: true prompt: ${{ steps.compose.outputs.prompt }} From ff8c636fbb905c4e9a4ba230ed599ab130707c61 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Sun, 26 Apr 2026 11:32:58 +0300 Subject: [PATCH 010/120] Run jobs under user or org accounts with upgrade / org-pick UX (#132) * agent/core/agent_loop.py: jobs access rewiring * agent/core/telemetry.py: jobs access rewiring * agent/tools/jobs_tool.py: jobs access rewiring * backend/dependencies.py: jobs access rewiring * backend/models.py: jobs access rewiring * backend/routes/agent.py: jobs access rewiring * frontend/src/components/Chat/ChatInput.tsx: jobs access rewiring * frontend/src/components/ClaudeCapDialog.tsx: jobs access rewiring * frontend/src/components/SessionChat.tsx: jobs access rewiring * frontend/src/hooks/useAgentChat.ts: jobs access rewiring * frontend/src/lib/sse-chat-transport.ts: jobs access rewiring * frontend/src/store/agentStore.ts: jobs access rewiring * frontend/src/types/agent.ts: jobs access rewiring * scripts/build_kpis.py: jobs access rewiring * tests/unit/test_build_kpis.py: jobs access rewiring * agent/core/hf_access.py: jobs access rewiring * frontend/src/components/JobsUpgradeDialog.tsx: jobs access rewiring * tests/unit/test_hf_access.py: jobs access rewiring * ci: re-trigger Claude review after OIDC fix * hf_access: avoid blocking fallback whoami call * agent routes: remove dead can_run_jobs branch --- agent/core/agent_loop.py | 3 + agent/core/hf_access.py | 181 +++++++++++++++++ agent/core/telemetry.py | 38 ++++ agent/tools/jobs_tool.py | 22 +- backend/dependencies.py | 56 +---- backend/models.py | 1 + backend/routes/agent.py | 156 +++++++++++++- frontend/src/components/Chat/ChatInput.tsx | 56 ++++- frontend/src/components/ClaudeCapDialog.tsx | 3 + frontend/src/components/JobsUpgradeDialog.tsx | 191 ++++++++++++++++++ frontend/src/components/SessionChat.tsx | 4 +- frontend/src/hooks/useAgentChat.ts | 131 +++++++++++- frontend/src/lib/sse-chat-transport.ts | 26 +++ frontend/src/store/agentStore.ts | 37 ++++ frontend/src/types/agent.ts | 1 + scripts/build_kpis.py | 26 ++- tests/unit/test_build_kpis.py | 32 +++ tests/unit/test_hf_access.py | 39 ++++ 18 files changed, 944 insertions(+), 59 deletions(-) create mode 100644 agent/core/hf_access.py create mode 100644 frontend/src/components/JobsUpgradeDialog.tsx create mode 100644 tests/unit/test_hf_access.py diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index fae5465b8..26361d413 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -1137,6 +1137,9 @@ async def exec_approval(session: Session, approvals: list[dict]) -> None: tool_args["script"] = edited_script was_edited = True logger.info(f"Using user-edited script for {tool_name} ({tc.id})") + selected_namespace = approval_decision.get("namespace") + if selected_namespace and tool_name == "hf_jobs": + tool_args["namespace"] = selected_namespace approved_tasks.append((tc, tool_name, tool_args, was_edited)) else: rejected_tasks.append((tc, tool_name, approval_decision)) diff --git a/agent/core/hf_access.py b/agent/core/hf_access.py new file mode 100644 index 000000000..400db5a5a --- /dev/null +++ b/agent/core/hf_access.py @@ -0,0 +1,181 @@ +"""Helpers for Hugging Face account / org access decisions.""" + +from __future__ import annotations + +import asyncio +import os +from dataclasses import dataclass +from typing import Any + +import httpx + +OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co") + + +@dataclass(frozen=True) +class JobsAccess: + """Jobs entitlement derived from whoami-v2.""" + + username: str | None + plan: str + personal_can_run_jobs: bool + paid_org_names: list[str] + eligible_namespaces: list[str] + default_namespace: str | None + access_known: bool = True + + @property + def can_run_jobs(self) -> bool: + return bool(self.default_namespace) + + +class JobsAccessError(Exception): + """Structured jobs access error for upgrade / namespace gating.""" + + def __init__( + self, + message: str, + *, + access: JobsAccess | None = None, + upgrade_required: bool = False, + namespace_required: bool = False, + ) -> None: + super().__init__(message) + self.access = access + self.upgrade_required = upgrade_required + self.namespace_required = namespace_required + + +def _extract_username(whoami: dict[str, Any]) -> str | None: + for key in ("name", "user", "preferred_username"): + value = whoami.get(key) + if isinstance(value, str) and value: + return value + return None + + +def _normalize_personal_plan(whoami: dict[str, Any]) -> str: + plan_str = "" + for key in ("plan", "type", "accountType"): + value = whoami.get(key) + if isinstance(value, str) and value: + plan_str = value.lower() + break + + if not plan_str and (whoami.get("isPro") is True or whoami.get("is_pro") is True): + return "pro" + + if any(tag in plan_str for tag in ("pro", "enterprise", "team")): + return "pro" + return "free" + + +def _paid_org_names(whoami: dict[str, Any]) -> list[str]: + names: list[str] = [] + orgs = whoami.get("orgs") or [] + if not isinstance(orgs, list): + return names + + for org in orgs: + if not isinstance(org, dict): + continue + name = org.get("name") + if not isinstance(name, str) or not name: + continue + org_plan = str(org.get("plan") or org.get("type") or "").lower() + if any(tag in org_plan for tag in ("pro", "enterprise", "team")): + names.append(name) + return sorted(set(names)) + + +def jobs_access_from_whoami(whoami: dict[str, Any]) -> JobsAccess: + username = _extract_username(whoami) + personal_plan = _normalize_personal_plan(whoami) + paid_orgs = _paid_org_names(whoami) + personal_can_run = personal_plan == "pro" + + eligible_namespaces: list[str] = [] + if personal_can_run and username: + eligible_namespaces.append(username) + eligible_namespaces.extend(paid_orgs) + + plan = "pro" if personal_can_run else ("org" if paid_orgs else "free") + default_namespace = username if personal_can_run and username else None + + return JobsAccess( + username=username, + plan=plan, + personal_can_run_jobs=personal_can_run, + paid_org_names=paid_orgs, + eligible_namespaces=eligible_namespaces, + default_namespace=default_namespace, + ) + + +async def fetch_whoami_v2(token: str, timeout: float = 5.0) -> dict[str, Any] | None: + if not token: + return None + async with httpx.AsyncClient(timeout=timeout) as client: + try: + response = await client.get( + f"{OPENID_PROVIDER_URL}/api/whoami-v2", + headers={"Authorization": f"Bearer {token}"}, + ) + if response.status_code != 200: + return None + payload = response.json() + return payload if isinstance(payload, dict) else None + except (httpx.HTTPError, ValueError): + return None + + +async def get_jobs_access(token: str) -> JobsAccess | None: + whoami = await fetch_whoami_v2(token) + if whoami is None: + return None + return jobs_access_from_whoami(whoami) + + +async def resolve_jobs_namespace( + token: str, + requested_namespace: str | None = None, +) -> tuple[str, JobsAccess | None]: + """Return the namespace to use for jobs. + + If whoami-v2 is unavailable, fall back to the token owner's username. + """ + access = await get_jobs_access(token) + if access: + if requested_namespace: + if requested_namespace in access.eligible_namespaces: + return requested_namespace, access + raise JobsAccessError( + f"You can only run jobs under your own Pro account or a paid org you belong to. " + f"Allowed namespaces: {', '.join(access.eligible_namespaces) or '(none)'}", + access=access, + ) + if access.default_namespace: + return access.default_namespace, access + if access.paid_org_names: + raise JobsAccessError( + "Choose which paid organization should own this job run.", + access=access, + namespace_required=True, + ) + raise JobsAccessError( + "Hugging Face Jobs are available only to Pro users and Team or Enterprise organizations. " + "Upgrade to Pro, or run the job under a paid org you belong to.", + access=access, + upgrade_required=True, + ) + + # Fallback: whoami-v2 unavailable. Do not block the call pre-emptively. + from huggingface_hub import HfApi + + username = None + if token: + whoami = await asyncio.to_thread(HfApi(token=token).whoami) + username = whoami.get("name") + if not username: + raise JobsAccessError("No HF token available to resolve a jobs namespace.") + return requested_namespace or username, None diff --git a/agent/core/telemetry.py b/agent/core/telemetry.py index 11818585d..0d060dee9 100644 --- a/agent/core/telemetry.py +++ b/agent/core/telemetry.py @@ -141,6 +141,7 @@ async def record_hf_job_submit( "timeout": args.get("timeout", "30m"), "job_type": job_type, "image": image, + "namespace": args.get("namespace"), "push_to_hub": _infer_push_to_hub(script_text), }, )) @@ -239,6 +240,43 @@ async def record_feedback( logger.debug("record_feedback failed (non-fatal): %s", e) +async def record_jobs_access_blocked( + session: Any, + *, + tool_call_ids: list[str], + plan: str, + eligible_namespaces: list[str], +) -> None: + from agent.core.session import Event + try: + await session.send_event(Event( + event_type="jobs_access_blocked", + data={ + "tool_call_ids": tool_call_ids, + "plan": plan, + "eligible_namespaces": eligible_namespaces, + }, + )) + except Exception as e: + logger.debug("record_jobs_access_blocked failed (non-fatal): %s", e) + + +async def record_pro_cta_click( + session: Any, + *, + source: str, + target: str = "pro_pricing", +) -> None: + from agent.core.session import Event + try: + await session.send_event(Event( + event_type="pro_cta_click", + data={"source": source, "target": target}, + )) + except Exception as e: + logger.debug("record_pro_cta_click failed (non-fatal): %s", e) + + # ── heartbeat ────────────────────────────────────────────────────────────── # Module-level reference set for fire-and-forget heartbeat tasks. asyncio only diff --git a/agent/tools/jobs_tool.py b/agent/tools/jobs_tool.py index 474ee4cc7..c18d47e29 100644 --- a/agent/tools/jobs_tool.py +++ b/agent/tools/jobs_tool.py @@ -17,6 +17,7 @@ from huggingface_hub import HfApi from huggingface_hub.utils import HfHubHTTPError +from agent.core.hf_access import JobsAccessError, resolve_jobs_namespace from agent.core.session import Event from agent.tools.types import ToolResult @@ -298,6 +299,7 @@ def __init__( self, hf_token: Optional[str] = None, namespace: Optional[str] = None, + jobs_access: Any = None, log_callback: Optional[Callable[[str], Awaitable[None]]] = None, session: Any = None, tool_call_id: Optional[str] = None, @@ -305,6 +307,7 @@ def __init__( self.hf_token = hf_token self.api = HfApi(token=hf_token) self.namespace = namespace + self.jobs_access = jobs_access self.log_callback = log_callback self.session = session self.tool_call_id = tool_call_id @@ -565,7 +568,7 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: from agent.core import telemetry submit_ts = await telemetry.record_hf_job_submit( self.session, job, - {**args, "hardware_flavor": flavor, "timeout": timeout_str}, + {**args, "hardware_flavor": flavor, "timeout": timeout_str, "namespace": self.namespace}, image=image, job_type=job_type, ) @@ -1057,6 +1060,14 @@ async def _resume_scheduled_job(self, args: Dict[str, Any]) -> ToolResult: "type": "object", "description": "Environment variables {'KEY': 'VALUE'}. HF_TOKEN is auto-included.", }, + "namespace": { + "type": "string", + "description": ( + "Optional namespace to run the job under. Must be your own Pro account " + "or a paid org you belong to. If omitted, the tool prefers your personal " + "account when eligible, otherwise the first eligible paid org." + ), + }, "job_id": { "type": "string", "description": "Job ID. Required for: logs, inspect, cancel.", @@ -1099,11 +1110,18 @@ async def log_callback(log: str): arguments = {**arguments, "script": content} hf_token = session.hf_token if session else None - namespace = os.environ.get("HF_NAMESPACE") or (HfApi(token=hf_token).whoami().get("name") if hf_token else None) + try: + namespace, jobs_access = await resolve_jobs_namespace( + hf_token or "", + arguments.get("namespace"), + ) + except JobsAccessError as e: + return str(e), False tool = HfJobsTool( namespace=namespace, hf_token=hf_token, + jobs_access=jobs_access, log_callback=log_callback if session else None, session=session, tool_call_id=tool_call_id, diff --git a/backend/dependencies.py b/backend/dependencies.py index 97a4e2860..0f97c448d 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -12,6 +12,8 @@ import httpx from fastapi import HTTPException, Request, status +from agent.core.hf_access import fetch_whoami_v2, jobs_access_from_whoami + logger = logging.getLogger(__name__) OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL", "https://huggingface.co") @@ -80,41 +82,6 @@ def _user_from_info(user_info: dict[str, Any]) -> dict[str, Any]: } -def _normalize_plan(whoami: dict[str, Any]) -> str: - """Map an HF /api/whoami-v2 payload to one of: 'free' | 'pro' | 'org'. - - The exact field shape in whoami-v2 isn't documented for our purposes, - so we try a handful of likely keys and fall back to 'free'. The first - call logs the raw shape at DEBUG (see `_fetch_user_plan`) so we can - pin the real key post-deploy. - """ - plan_str = "" - for key in ("plan", "type", "accountType"): - val = whoami.get(key) - if isinstance(val, str) and val: - plan_str = val.lower() - break - - if not plan_str: - if whoami.get("isPro") is True or whoami.get("is_pro") is True: - return "pro" - - if "pro" in plan_str or "enterprise" in plan_str or "team" in plan_str: - return "pro" - - # Org tier: anyone in a paid / enterprise org. We don't pay for this - # right now, but the "pro" cap applies identically. - orgs = whoami.get("orgs") or [] - if isinstance(orgs, list): - for org in orgs: - if isinstance(org, dict): - org_plan = str(org.get("plan") or org.get("type") or "").lower() - if "pro" in org_plan or "enterprise" in org_plan or "team" in org_plan: - return "org" - - return "free" - - async def _fetch_user_plan(token: str) -> str: """Look up the user's HF plan via /api/whoami-v2. @@ -123,19 +90,9 @@ async def _fetch_user_plan(token: str) -> str: grant the Pro cap than over-grant it on bad data. """ global _WHOAMI_SHAPE_LOGGED - async with httpx.AsyncClient(timeout=5.0) as client: - try: - resp = await client.get( - f"{OPENID_PROVIDER_URL}/api/whoami-v2", - headers={"Authorization": f"Bearer {token}"}, - ) - if resp.status_code != 200: - return "free" - whoami = resp.json() - except httpx.HTTPError: - return "free" - except ValueError: - return "free" + whoami = await fetch_whoami_v2(token) + if whoami is None: + return "free" if not _WHOAMI_SHAPE_LOGGED: _WHOAMI_SHAPE_LOGGED = True @@ -149,7 +106,7 @@ async def _fetch_user_plan(token: str) -> str: if not isinstance(whoami, dict): return "free" - return _normalize_plan(whoami) + return jobs_access_from_whoami(whoami).plan async def _extract_user_from_token(token: str) -> dict[str, Any] | None: @@ -246,4 +203,3 @@ async def require_huggingface_org_member(request: Request) -> bool: return False return await check_org_membership(token, HF_EMPLOYEE_ORG) - diff --git a/backend/models.py b/backend/models.py index 954779f6a..952365c23 100644 --- a/backend/models.py +++ b/backend/models.py @@ -38,6 +38,7 @@ class ToolApproval(BaseModel): approved: bool feedback: str | None = None edited_script: str | None = None + namespace: str | None = None class ApprovalRequest(BaseModel): diff --git a/backend/routes/agent.py b/backend/routes/agent.py index 0224f258a..4895bbadb 100644 --- a/backend/routes/agent.py +++ b/backend/routes/agent.py @@ -32,6 +32,7 @@ import user_quotas +from agent.core.hf_access import get_jobs_access from agent.core.llm_params import _resolve_llm_params logger = logging.getLogger(__name__) @@ -136,6 +137,105 @@ async def _enforce_claude_quota( agent_session.claude_counted = True +async def _enforce_jobs_access_for_approvals( + user: dict[str, Any], + agent_session: AgentSession, + approvals: list[dict[str, Any]], +) -> None: + """Block approved hf_jobs tool calls when the user has no eligible jobs namespace.""" + pending = agent_session.session.pending_approval or {} + tool_calls = pending.get("tool_calls") or [] + if not tool_calls: + return + + approved_ids = { + a.get("tool_call_id") + for a in approvals + if a.get("approved") + } + if not approved_ids: + return + + hf_job_ids = [ + tc.id for tc in tool_calls + if tc.id in approved_ids and tc.function.name == "hf_jobs" + ] + if not hf_job_ids: + return + + token = agent_session.hf_token or agent_session.session.hf_token + if not token: + return + + access = await get_jobs_access(token) + if access is None: + return + + approval_map = {a.get("tool_call_id"): a for a in approvals} + if access.personal_can_run_jobs: + return + + if access.paid_org_names: + invalid_namespace = [ + tool_call_id + for tool_call_id in hf_job_ids + if ( + approval_map.get(tool_call_id, {}).get("namespace") + and approval_map.get(tool_call_id, {}).get("namespace") not in access.paid_org_names + ) + ] + if invalid_namespace: + raise HTTPException( + status_code=400, + detail={ + "error": "hf_jobs_invalid_namespace", + "message": ( + "The selected jobs namespace is not one of your eligible paid organizations. " + f"Allowed namespaces: {', '.join(access.paid_org_names)}" + ), + }, + ) + missing_namespace = [ + tool_call_id + for tool_call_id in hf_job_ids + if not approval_map.get(tool_call_id, {}).get("namespace") + ] + if missing_namespace: + raise HTTPException( + status_code=409, + detail={ + "error": "hf_jobs_namespace_required", + "message": "Choose which paid organization should own this job run.", + "plan": user.get("plan", "free"), + "tool_call_ids": missing_namespace, + "eligible_namespaces": access.paid_org_names, + }, + ) + return + + from agent.core import telemetry + await telemetry.record_jobs_access_blocked( + agent_session.session, + tool_call_ids=hf_job_ids, + plan=user.get("plan", "free"), + eligible_namespaces=access.eligible_namespaces, + ) + + raise HTTPException( + status_code=402, + detail={ + "error": "hf_jobs_upgrade_required", + "message": ( + "Hugging Face Jobs are available only to Pro users and Team or Enterprise organizations. " + "Upgrade to Pro, or decline the job tool call so the agent can choose another path." + ), + "plan": user.get("plan", "free"), + "tool_call_ids": hf_job_ids, + "eligible_namespaces": access.eligible_namespaces, + }, + ) + + def _check_session_access(session_id: str, user: dict[str, Any]) -> None: """Verify the user has access to the given session. Raises 403 or 404.""" info = session_manager.get_session_info(session_id) @@ -442,6 +542,27 @@ async def get_user_quota(user: dict = Depends(get_current_user)) -> dict: } +@router.get("/user/jobs-access") +async def get_jobs_access_info(request: Request, user: dict = Depends(get_current_user)) -> dict: + """Return whether the current token can run HF Jobs and under which namespaces.""" + token = None + auth_header = request.headers.get("Authorization", "") + if auth_header.startswith("Bearer "): + token = auth_header[7:] + if not token: + token = request.cookies.get("hf_access_token") + if not token: + token = os.environ.get("HF_TOKEN") + + access = await get_jobs_access(token or "") + return { + "plan": user.get("plan", "free"), + "can_run_jobs": bool(access and (access.personal_can_run_jobs or access.paid_org_names)), + "eligible_namespaces": access.eligible_namespaces if access else [], + "default_namespace": access.default_namespace if access else None, + } + + @router.get("/sessions", response_model=list[SessionInfo]) async def list_sessions(user: dict = Depends(get_current_user)) -> list[SessionInfo]: """List sessions belonging to the authenticated user.""" @@ -482,15 +603,20 @@ async def submit_approval( ) -> dict: """Submit tool approvals to a session. Only accessible by the session owner.""" _check_session_access(request.session_id, user) + agent_session = session_manager.sessions.get(request.session_id) + if agent_session is None: + raise HTTPException(status_code=404, detail="Session not found or inactive") approvals = [ { "tool_call_id": a.tool_call_id, "approved": a.approved, "feedback": a.feedback, "edited_script": a.edited_script, + "namespace": a.namespace, } for a in request.approvals ] + await _enforce_jobs_access_for_approvals(user, agent_session, approvals) success = await session_manager.submit_approval(request.session_id, approvals) if not success: raise HTTPException(status_code=404, detail="Session not found or inactive") @@ -540,9 +666,11 @@ async def chat_sse( "approved": a["approved"], "feedback": a.get("feedback"), "edited_script": a.get("edited_script"), + "namespace": a.get("namespace"), } for a in approvals ] + await _enforce_jobs_access_for_approvals(user, agent_session, formatted) success = await session_manager.submit_approval(session_id, formatted) elif text is not None: success = await session_manager.submit_user_input(session_id, text) @@ -554,6 +682,7 @@ async def chat_sse( broadcaster.unsubscribe(sub_id) raise HTTPException(status_code=404, detail="Session not found or inactive") except HTTPException: + broadcaster.unsubscribe(sub_id) raise except Exception: broadcaster.unsubscribe(sub_id) @@ -562,6 +691,31 @@ async def chat_sse( return _sse_response(broadcaster, event_queue, sub_id) +@router.post("/pro-click/{session_id}") +async def record_pro_click( + session_id: str, + body: dict, + user: dict = Depends(get_current_user), +) -> dict: + """Record a click on a Pro upgrade CTA shown from inside a session.""" + _check_session_access(session_id, user) + agent_session = session_manager.sessions.get(session_id) + if not agent_session: + raise HTTPException(status_code=404, detail="Session not found") + + from agent.core import telemetry + await telemetry.record_pro_cta_click( + agent_session.session, + source=str(body.get("source") or "unknown"), + target=str(body.get("target") or "pro_pricing"), + ) + if agent_session.session.config.save_sessions: + agent_session.session.save_and_upload_detached( + agent_session.session.config.session_dataset_repo + ) + return {"status": "ok"} + + # --------------------------------------------------------------------------- # Shared SSE helpers # --------------------------------------------------------------------------- @@ -729,5 +883,3 @@ async def submit_feedback( agent_session.session.config.session_dataset_repo ) return {"status": "ok"} - - diff --git a/frontend/src/components/Chat/ChatInput.tsx b/frontend/src/components/Chat/ChatInput.tsx index 13a6f5443..28eec9044 100644 --- a/frontend/src/components/Chat/ChatInput.tsx +++ b/frontend/src/components/Chat/ChatInput.tsx @@ -6,6 +6,7 @@ import StopIcon from '@mui/icons-material/Stop'; import { apiFetch } from '@/utils/api'; import { useUserQuota } from '@/hooks/useUserQuota'; import ClaudeCapDialog from '@/components/ClaudeCapDialog'; +import JobsUpgradeDialog from '@/components/JobsUpgradeDialog'; import { useAgentStore } from '@/store/agentStore'; import { CLAUDE_MODEL_PATH, FIRST_FREE_MODEL_PATH, isClaudePath } from '@/utils/model'; @@ -65,6 +66,8 @@ interface ChatInputProps { sessionId?: string; onSend: (text: string) => void; onStop?: () => void; + onDeclineBlockedJobs?: () => Promise; + onContinueBlockedJobsWithNamespace?: (namespace: string) => Promise; isProcessing?: boolean; disabled?: boolean; placeholder?: string; @@ -73,7 +76,7 @@ interface ChatInputProps { const isClaudeModel = (m: ModelOption) => isClaudePath(m.modelPath); const firstFreeModel = () => MODEL_OPTIONS.find(m => !isClaudeModel(m)) ?? MODEL_OPTIONS[0]; -export default function ChatInput({ sessionId, onSend, onStop, isProcessing = false, disabled = false, placeholder = 'Ask anything...' }: ChatInputProps) { +export default function ChatInput({ sessionId, onSend, onStop, onDeclineBlockedJobs, onContinueBlockedJobsWithNamespace, isProcessing = false, disabled = false, placeholder = 'Ask anything...' }: ChatInputProps) { const [input, setInput] = useState(''); const inputRef = useRef(null); const [selectedModelId, setSelectedModelId] = useState(MODEL_OPTIONS[0].id); @@ -86,6 +89,8 @@ export default function ChatInput({ sessionId, onSend, onStop, isProcessing = fa // the hook layer can flip it without threading props through. const claudeQuotaExhausted = useAgentStore((s) => s.claudeQuotaExhausted); const setClaudeQuotaExhausted = useAgentStore((s) => s.setClaudeQuotaExhausted); + const jobsUpgradeRequired = useAgentStore((s) => s.jobsUpgradeRequired); + const setJobsUpgradeRequired = useAgentStore((s) => s.setJobsUpgradeRequired); const lastSentRef = useRef(''); // Model is per-session: fetch this tab's current model every time the @@ -197,6 +202,44 @@ export default function ChatInput({ sessionId, onSend, onStop, isProcessing = fa } catch { /* ignore */ } }, [sessionId, onSend, setClaudeQuotaExhausted]); + const handleClaudeUpgradeClick = useCallback(async () => { + if (!sessionId) return; + try { + await apiFetch(`/api/pro-click/${sessionId}`, { + method: 'POST', + body: JSON.stringify({ source: 'claude_cap_dialog', target: 'pro_pricing' }), + }); + } catch { + /* tracking is best-effort */ + } + }, [sessionId]); + + const handleJobsUpgradeClose = useCallback(() => { + setJobsUpgradeRequired(null); + }, [setJobsUpgradeRequired]); + + const handleJobsUpgradeClick = useCallback(async () => { + if (!sessionId || !jobsUpgradeRequired) return; + try { + await apiFetch(`/api/pro-click/${sessionId}`, { + method: 'POST', + body: JSON.stringify({ source: 'hf_jobs_upgrade_dialog', target: 'pro_pricing' }), + }); + } catch { + /* tracking is best-effort */ + } + }, [sessionId, jobsUpgradeRequired]); + + const handleDeclineBlockedJobs = useCallback(async () => { + if (!onDeclineBlockedJobs) return; + await onDeclineBlockedJobs(); + }, [onDeclineBlockedJobs]); + + const handleContinueBlockedJobsWithNamespace = useCallback(async (namespace: string) => { + if (!onContinueBlockedJobsWithNamespace) return; + await onContinueBlockedJobsWithNamespace(namespace); + }, [onContinueBlockedJobsWithNamespace]); + // Hide the chip until the user has actually burned quota — an unused // Opus session shouldn't populate a counter. const claudeChip = (() => { @@ -435,6 +478,17 @@ export default function ChatInput({ sessionId, onSend, onStop, isProcessing = fa cap={quota?.claudeDailyCap ?? 1} onClose={handleCapDialogClose} onUseFreeModel={handleUseFreeModel} + onUpgrade={handleClaudeUpgradeClick} + /> + diff --git a/frontend/src/components/ClaudeCapDialog.tsx b/frontend/src/components/ClaudeCapDialog.tsx index 3fd4d3e0c..f959a44ca 100644 --- a/frontend/src/components/ClaudeCapDialog.tsx +++ b/frontend/src/components/ClaudeCapDialog.tsx @@ -19,6 +19,7 @@ interface ClaudeCapDialogProps { cap: number; onClose: () => void; onUseFreeModel: () => void; + onUpgrade: () => void; } export default function ClaudeCapDialog({ @@ -27,6 +28,7 @@ export default function ClaudeCapDialog({ cap, onClose, onUseFreeModel, + onUpgrade, }: ClaudeCapDialogProps) { // plan not surfaced in copy right now — Pro users see the same dialog and // can upgrade their org if they're also capped. @@ -100,6 +102,7 @@ export default function ClaudeCapDialog({ href={HF_PRICING_URL} target="_blank" rel="noopener noreferrer" + onClick={onUpgrade} variant="contained" size="small" sx={{ diff --git a/frontend/src/components/JobsUpgradeDialog.tsx b/frontend/src/components/JobsUpgradeDialog.tsx new file mode 100644 index 000000000..9a1502042 --- /dev/null +++ b/frontend/src/components/JobsUpgradeDialog.tsx @@ -0,0 +1,191 @@ +import { useEffect, useState } from 'react'; +import { + Box, + Button, + Dialog, + DialogActions, + DialogContent, + DialogContentText, + DialogTitle, + FormControl, + InputLabel, + MenuItem, + Select, + Typography, +} from '@mui/material'; + +const HF_PRICING_URL = 'https://huggingface.co/pricing'; + +interface JobsUpgradeDialogProps { + open: boolean; + mode: 'upgrade' | 'namespace'; + message: string; + eligibleNamespaces: string[]; + onUpgrade: () => void; + onDecline: () => void; + onClose: () => void; + onContinueWithNamespace: (namespace: string) => void; +} + +export default function JobsUpgradeDialog({ + open, + mode, + message, + eligibleNamespaces, + onUpgrade, + onDecline, + onClose, + onContinueWithNamespace, +}: JobsUpgradeDialogProps) { + const [selectedNamespace, setSelectedNamespace] = useState(''); + + useEffect(() => { + if (!open) return; + setSelectedNamespace(eligibleNamespaces[0] || ''); + }, [open, eligibleNamespaces]); + + return ( + + + {mode === 'namespace' ? 'Choose the org for this job' : 'Jobs need Pro or a paid org'} + + + + {message} + + {eligibleNamespaces.length > 0 && ( + + + Eligible namespaces + + {mode === 'namespace' ? ( + + Organization + + + ) : ( + + {eligibleNamespaces.join(', ')} + + )} + + )} + + If you decline, the agent will have to find another way forward without `hf_jobs`. + + + + {mode === 'namespace' ? ( + + ) : ( + + )} + + + + ); +} diff --git a/frontend/src/components/SessionChat.tsx b/frontend/src/components/SessionChat.tsx index 8f1823806..7c9167f58 100644 --- a/frontend/src/components/SessionChat.tsx +++ b/frontend/src/components/SessionChat.tsx @@ -26,7 +26,7 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess const { updateSessionTitle, sessions } = useSessionStore(); const isExpired = sessions.find((s) => s.id === sessionId)?.expired === true; - const { messages, sendMessage, stop, status, undoLastTurn, editAndRegenerate, approveTools } = useAgentChat({ + const { messages, sendMessage, stop, status, undoLastTurn, editAndRegenerate, approveTools, declineBlockedJobs, continueBlockedJobsWithNamespace } = useAgentChat({ sessionId, isActive, onReady: () => logger.log(`Session ${sessionId} ready`), @@ -114,6 +114,8 @@ export default function SessionChat({ sessionId, isActive, onSessionDead }: Sess sessionId={sessionId} onSend={handleSendMessage} onStop={handleStop} + onDeclineBlockedJobs={declineBlockedJobs} + onContinueBlockedJobsWithNamespace={continueBlockedJobsWithNamespace} isProcessing={busy} disabled={!isConnected || activityStatus.type === 'waiting-approval'} placeholder={ diff --git a/frontend/src/hooks/useAgentChat.ts b/frontend/src/hooks/useAgentChat.ts index db25c0185..a83a0ac28 100644 --- a/frontend/src/hooks/useAgentChat.ts +++ b/frontend/src/hooks/useAgentChat.ts @@ -330,6 +330,49 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD messages: UIMessage[]; }>({ setMessages: null, messages: [] }); + const hydrateFromBackend = useCallback(async () => { + try { + const [msgsRes, infoRes] = await Promise.all([ + apiFetch(`/api/session/${sessionId}/messages`), + apiFetch(`/api/session/${sessionId}`), + ]); + if (!msgsRes.ok) return null; + const data = await msgsRes.json(); + if (!Array.isArray(data) || data.length === 0) return null; + + saveBackendMessages(sessionId, data); + + let pendingIds: Set | undefined; + let info: Record | null = null; + if (infoRes.ok) { + info = await infoRes.json(); + const pendingApproval = info?.pending_approval; + if (pendingApproval && Array.isArray(pendingApproval)) { + pendingIds = new Set( + pendingApproval.map((t: { tool_call_id: string }) => t.tool_call_id), + ); + if (pendingIds.size > 0) { + setNeedsAttention(sessionId, true); + } + } + } + + const uiMsgs = llmMessagesToUIMessages(data, pendingIds, chatActionsRef.current.messages); + if (uiMsgs.length > 0) { + chatActionsRef.current.setMessages?.(uiMsgs); + saveMessages(sessionId, uiMsgs); + } + + if (pendingIds && pendingIds.size > 0) { + updateSession(sessionId, { activityStatus: { type: 'waiting-approval' }, isProcessing: false }); + } + + return { data, pendingIds, info }; + } catch { + return null; + } + }, [sessionId, setNeedsAttention]); + // -- useChat from Vercel AI SDK ----------------------------------------- const chat = useChat({ id: sessionId, @@ -354,6 +397,56 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD } return; } + if (error.message === 'HF_JOBS_UPGRADE_REQUIRED') { + const typed = error as Error & { + detail?: Record; + approvals?: Array<{ + tool_call_id: string; + approved: boolean; + feedback?: string | null; + edited_script?: string | null; + }>; + }; + void hydrateFromBackend(); + if (isActiveRef.current) { + useAgentStore.getState().setJobsUpgradeRequired({ + approvals: typed.approvals || [], + toolCallIds: (typed.detail?.tool_call_ids as string[]) || [], + message: String( + typed.detail?.message + || 'Hugging Face Jobs are available only to Pro users and Team or Enterprise organizations.', + ), + eligibleNamespaces: (typed.detail?.eligible_namespaces as string[]) || [], + plan: ((typed.detail?.plan as 'free' | 'pro' | 'org') || 'free'), + mode: 'upgrade', + }); + } + return; + } + if (error.message === 'HF_JOBS_NAMESPACE_REQUIRED') { + const typed = error as Error & { + detail?: Record; + approvals?: Array<{ + tool_call_id: string; + approved: boolean; + feedback?: string | null; + edited_script?: string | null; + namespace?: string | null; + }>; + }; + void hydrateFromBackend(); + if (isActiveRef.current) { + useAgentStore.getState().setJobsUpgradeRequired({ + approvals: typed.approvals || [], + toolCallIds: (typed.detail?.tool_call_ids as string[]) || [], + message: String(typed.detail?.message || 'Choose which organization should own this job run.'), + eligibleNamespaces: (typed.detail?.eligible_namespaces as string[]) || [], + plan: ((typed.detail?.plan as 'free' | 'pro' | 'org') || 'free'), + mode: 'namespace', + }); + } + return; + } logger.error('useChat error:', error); if (isActiveRef.current) { useAgentStore.getState().setError(error.message); @@ -672,12 +765,15 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD // -- Approve tools ------------------------------------------------------ const approveTools = useCallback( - async (approvals: Array<{ tool_call_id: string; approved: boolean; feedback?: string | null; edited_script?: string | null }>) => { + async (approvals: Array<{ tool_call_id: string; approved: boolean; feedback?: string | null; edited_script?: string | null; namespace?: string | null }>) => { // Store edited scripts so the transport can read them when sendMessages is called for (const a of approvals) { if (a.edited_script) { useAgentStore.getState().setEditedScript(a.tool_call_id, a.edited_script); } + if (a.namespace) { + useAgentStore.getState().setApprovalNamespace(a.tool_call_id, a.namespace); + } } // Update SDK tool state — this triggers sendMessages() via the transport @@ -707,6 +803,37 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD [sessionId, chat, updateSession, setNeedsAttention], ); + const declineBlockedJobs = useCallback(async () => { + const blocked = useAgentStore.getState().jobsUpgradeRequired; + if (!blocked) return false; + + const approvals = blocked.approvals.map((approval) => ({ + ...approval, + approved: blocked.toolCallIds.includes(approval.tool_call_id) ? false : approval.approved, + feedback: blocked.toolCallIds.includes(approval.tool_call_id) + ? 'Rejected because this account cannot launch Hugging Face Jobs.' + : approval.feedback, + })); + + useAgentStore.getState().setJobsUpgradeRequired(null); + return approveTools(approvals); + }, [approveTools]); + + const continueBlockedJobsWithNamespace = useCallback(async (namespace: string) => { + const blocked = useAgentStore.getState().jobsUpgradeRequired; + if (!blocked) return false; + + const approvals = blocked.approvals.map((approval) => ({ + ...approval, + namespace: blocked.toolCallIds.includes(approval.tool_call_id) + ? namespace + : approval.namespace, + })); + + useAgentStore.getState().setJobsUpgradeRequired(null); + return approveTools(approvals); + }, [approveTools]); + // -- Stop (interrupt backend agent loop, keep SSE open for events) -------- const stop = useCallback(() => { // Don't call chat.stop() — keep the SSE stream open so the backend's @@ -763,5 +890,7 @@ export function useAgentChat({ sessionId, isActive, onReady, onError, onSessionD undoLastTurn, editAndRegenerate, approveTools, + declineBlockedJobs, + continueBlockedJobsWithNamespace, }; } diff --git a/frontend/src/lib/sse-chat-transport.ts b/frontend/src/lib/sse-chat-transport.ts index 775cef619..fa59a9867 100644 --- a/frontend/src/lib/sse-chat-transport.ts +++ b/frontend/src/lib/sse-chat-transport.ts @@ -320,11 +320,13 @@ export class SSEChatTransport implements ChatTransport { const approved = p.approval?.approved ?? true; // Get edited script from agentStore if available const editedScript = useAgentStore.getState().getEditedScript(p.toolCallId); + const namespace = useAgentStore.getState().getApprovalNamespace(p.toolCallId); return { tool_call_id: p.toolCallId, approved, feedback: approved ? null : (p.approval?.reason || 'Rejected by user'), edited_script: editedScript ?? null, + namespace: namespace ?? null, }; }).filter(Boolean); body = { approvals }; @@ -362,6 +364,30 @@ export class SSEChatTransport implements ChatTransport { // instead of a generic error banner. throw new Error('CLAUDE_QUOTA_EXHAUSTED'); } + if (response.status === 402) { + const payload = await response.json().catch(() => null); + if (payload?.detail?.error === 'hf_jobs_upgrade_required') { + const err = new Error('HF_JOBS_UPGRADE_REQUIRED') as Error & { + detail?: Record; + approvals?: Array>; + }; + err.detail = payload.detail as Record; + err.approvals = (body.approvals as Array> | undefined) || []; + throw err; + } + } + if (response.status === 409) { + const payload = await response.json().catch(() => null); + if (payload?.detail?.error === 'hf_jobs_namespace_required') { + const err = new Error('HF_JOBS_NAMESPACE_REQUIRED') as Error & { + detail?: Record; + approvals?: Array>; + }; + err.detail = payload.detail as Record; + err.approvals = (body.approvals as Array> | undefined) || []; + throw err; + } + } if (!response.ok) { const errorText = await response.text().catch(() => 'Request failed'); throw new Error(`Chat request failed: ${response.status} ${errorText}`); diff --git a/frontend/src/store/agentStore.ts b/frontend/src/store/agentStore.ts index b88692526..ca32566f7 100644 --- a/frontend/src/store/agentStore.ts +++ b/frontend/src/store/agentStore.ts @@ -45,6 +45,21 @@ export interface LLMHealthError { model: string; } +export interface JobsUpgradeState { + approvals: Array<{ + tool_call_id: string; + approved: boolean; + feedback?: string | null; + edited_script?: string | null; + namespace?: string | null; + }>; + toolCallIds: string[]; + message: string; + eligibleNamespaces: string[]; + plan: 'free' | 'pro' | 'org'; + mode: 'upgrade' | 'namespace'; +} + export type ActivityStatus = | { type: 'idle' } | { type: 'thinking' } @@ -110,6 +125,7 @@ interface AgentStore { llmHealthError: LLMHealthError | null; /** Set when a Claude-send hits the daily quota — ChatInput opens the cap dialog in response. */ claudeQuotaExhausted: boolean; + jobsUpgradeRequired: JobsUpgradeState | null; // Right panel (single-artifact pattern) panelData: PanelData | null; @@ -122,6 +138,9 @@ interface AgentStore { // Edited scripts (tool_call_id -> edited content) editedScripts: Record; + // Namespace overrides chosen for hf_jobs approvals (tool_call_id -> namespace) + approvalNamespaces: Record; + // Job URLs (tool_call_id -> job URL) for HF jobs jobUrls: Record; @@ -156,6 +175,7 @@ interface AgentStore { setError: (error: string | null) => void; setLlmHealthError: (error: LLMHealthError | null) => void; setClaudeQuotaExhausted: (exhausted: boolean) => void; + setJobsUpgradeRequired: (state: JobsUpgradeState | null) => void; setPanel: (data: PanelData, view?: PanelView, editable?: boolean) => void; setPanelView: (view: PanelView) => void; @@ -170,6 +190,10 @@ interface AgentStore { getEditedScript: (toolCallId: string) => string | undefined; clearEditedScripts: () => void; + setApprovalNamespace: (toolCallId: string, namespace: string) => void; + getApprovalNamespace: (toolCallId: string) => string | undefined; + clearApprovalNamespaces: () => void; + setJobUrl: (toolCallId: string, jobUrl: string) => void; getJobUrl: (toolCallId: string) => string | undefined; @@ -251,6 +275,7 @@ export const useAgentStore = create()((set, get) => ({ error: null, llmHealthError: null, claudeQuotaExhausted: false, + jobsUpgradeRequired: null, panelData: null, panelView: 'script', @@ -259,6 +284,7 @@ export const useAgentStore = create()((set, get) => ({ plan: [], editedScripts: {}, + approvalNamespaces: {}, jobUrls: {}, jobStatuses: {}, toolErrors: loadToolErrors(), @@ -363,6 +389,7 @@ export const useAgentStore = create()((set, get) => ({ setError: (error) => set({ error }), setLlmHealthError: (error) => set({ llmHealthError: error }), setClaudeQuotaExhausted: (exhausted) => set({ claudeQuotaExhausted: exhausted }), + setJobsUpgradeRequired: (state) => set({ jobsUpgradeRequired: state }), // ── Panel (single-artifact) ─────────────────────────────────────── // Each setter also patches the active session's snapshot so that @@ -428,6 +455,16 @@ export const useAgentStore = create()((set, get) => ({ clearEditedScripts: () => set({ editedScripts: {} }), + setApprovalNamespace: (toolCallId, namespace) => { + set((state) => ({ + approvalNamespaces: { ...state.approvalNamespaces, [toolCallId]: namespace }, + })); + }, + + getApprovalNamespace: (toolCallId) => get().approvalNamespaces[toolCallId], + + clearApprovalNamespaces: () => set({ approvalNamespaces: {} }), + // ── Job URLs ──────────────────────────────────────────────────────── setJobUrl: (toolCallId, jobUrl) => { diff --git a/frontend/src/types/agent.ts b/frontend/src/types/agent.ts index 840151be5..dc7b5c836 100644 --- a/frontend/src/types/agent.ts +++ b/frontend/src/types/agent.ts @@ -27,6 +27,7 @@ export interface ToolApproval { tool_call_id: string; approved: boolean; feedback?: string | null; + namespace?: string | null; } export interface User { diff --git a/scripts/build_kpis.py b/scripts/build_kpis.py index 6fcda8753..10477288e 100644 --- a/scripts/build_kpis.py +++ b/scripts/build_kpis.py @@ -44,7 +44,8 @@ regenerate_rate — sessions with any `undo_complete` event / sessions time_to_first_action_s_p50 / _p95 — from session_start to first tool_call thumbs_up / thumbs_down - hf_jobs_submitted / _succeeded + hf_jobs_submitted / _succeeded / _blocked + pro_cta_clicks gpu_hours_by_flavor_json — JSON-serialised {flavor: gpu-hours} ================================================================================ @@ -210,7 +211,8 @@ def _session_metrics(session: dict) -> dict: "tool_calls_total": 0, "tool_calls_success": 0, "failures": 0, "regenerate_sessions": 0, "thumbs_up": 0, "thumbs_down": 0, - "hf_jobs_submitted": 0, "hf_jobs_succeeded": 0, + "hf_jobs_submitted": 0, "hf_jobs_succeeded": 0, "hf_jobs_blocked": 0, + "pro_cta_clicks": 0, "first_tool_s": -1, } events = session.get("events") or [] @@ -229,8 +231,11 @@ def _session_metrics(session: dict) -> dict: gpu_hours_by_flavor: dict[str, float] = defaultdict(float) jobs_submitted = 0 jobs_succeeded = 0 + jobs_blocked = 0 thumbs_up = 0 thumbs_down = 0 + pro_cta_clicks = 0 + pro_cta_by_source: dict[str, int] = defaultdict(int) start_dt = _parse_ts(session_start) @@ -283,6 +288,14 @@ def _session_metrics(session: dict) -> dict: if status in ("completed", "succeeded", "success"): jobs_succeeded += 1 + elif et == "jobs_access_blocked": + jobs_blocked += 1 + + elif et == "pro_cta_click": + pro_cta_clicks += 1 + source = str(data.get("source") or "unknown") + pro_cta_by_source[source] += 1 + out["tool_calls_total"] = tool_total out["tool_calls_success"] = tool_success out["failures"] = 1 if had_error else 0 @@ -291,8 +304,11 @@ def _session_metrics(session: dict) -> dict: out["thumbs_down"] = thumbs_down out["hf_jobs_submitted"] = jobs_submitted out["hf_jobs_succeeded"] = jobs_succeeded + out["hf_jobs_blocked"] = jobs_blocked + out["pro_cta_clicks"] = pro_cta_clicks out["first_tool_s"] = first_tool_ts if first_tool_ts is not None else -1 out["_gpu_hours_by_flavor"] = dict(gpu_hours_by_flavor) + out["_pro_cta_by_source"] = dict(pro_cta_by_source) out["_user"] = session.get("user_id") or session.get("session_id") return dict(out) @@ -301,9 +317,12 @@ def _aggregate(per_session: list[dict]) -> dict: """Collapse a bucket's worth of session rollups into the final KPI row.""" ttfa_values = [s["first_tool_s"] for s in per_session if s.get("first_tool_s", -1) >= 0] gpu_hours: dict[str, float] = defaultdict(float) + pro_cta_by_source: dict[str, int] = defaultdict(int) for s in per_session: for f, h in (s.get("_gpu_hours_by_flavor") or {}).items(): gpu_hours[f] += h + for source, count in (s.get("_pro_cta_by_source") or {}).items(): + pro_cta_by_source[source] += int(count) total_sessions = sum(s["sessions"] for s in per_session) total_turns = sum(s["turns"] for s in per_session) @@ -340,7 +359,10 @@ def _aggregate(per_session: list[dict]) -> dict: "thumbs_down": int(sum(s["thumbs_down"] for s in per_session)), "hf_jobs_submitted": int(sum(s["hf_jobs_submitted"] for s in per_session)), "hf_jobs_succeeded": int(sum(s["hf_jobs_succeeded"] for s in per_session)), + "hf_jobs_blocked": int(sum(s["hf_jobs_blocked"] for s in per_session)), + "pro_cta_clicks": int(sum(s["pro_cta_clicks"] for s in per_session)), "gpu_hours_by_flavor_json": json.dumps(dict(gpu_hours), sort_keys=True), + "pro_cta_by_source_json": json.dumps(dict(pro_cta_by_source), sort_keys=True), } diff --git a/tests/unit/test_build_kpis.py b/tests/unit/test_build_kpis.py index b9f744b02..5edefc572 100644 --- a/tests/unit/test_build_kpis.py +++ b/tests/unit/test_build_kpis.py @@ -88,6 +88,22 @@ def test_hf_job_gpu_hours(): assert abs(m["_gpu_hours_by_flavor"]["a100-large"] - 1.0) < 1e-6 +def test_hf_job_blocked_and_pro_clicks_are_counted(): + mod = _load() + events = [ + _ev("jobs_access_blocked", {"tool_call_ids": ["tc1"], "plan": "free"}), + _ev("pro_cta_click", {"source": "hf_jobs_upgrade_dialog"}), + _ev("pro_cta_click", {"source": "claude_cap_dialog"}), + ] + m = mod._session_metrics(_session(events)) + assert m["hf_jobs_blocked"] == 1 + assert m["pro_cta_clicks"] == 2 + assert m["_pro_cta_by_source"] == { + "hf_jobs_upgrade_dialog": 1, + "claude_cap_dialog": 1, + } + + def test_feedback_counts(): mod = _load() events = [ @@ -120,6 +136,22 @@ def test_aggregate_day_cache_hit_and_users(): assert abs(row["cost_usd"] - 1.5) < 1e-9 +def test_aggregate_day_sums_pro_click_sources(): + mod = _load() + s1 = mod._session_metrics(_session([ + _ev("pro_cta_click", {"source": "hf_jobs_upgrade_dialog"}), + _ev("pro_cta_click", {"source": "hf_jobs_upgrade_dialog"}), + ], user_id="u1")) + s2 = mod._session_metrics(_session([ + _ev("pro_cta_click", {"source": "claude_cap_dialog"}), + ], user_id="u2")) + row = mod._aggregate_day([s1, s2]) + assert row["pro_cta_clicks"] == 3 + assert row["pro_cta_by_source_json"] == ( + '{"claude_cap_dialog": 1, "hf_jobs_upgrade_dialog": 2}' + ) + + def test_failure_and_regenerate_rates(): mod = _load() s1 = mod._session_metrics(_session([_ev("error", {"error": "boom"})], user_id="a")) diff --git a/tests/unit/test_hf_access.py b/tests/unit/test_hf_access.py new file mode 100644 index 000000000..7ccb96ce7 --- /dev/null +++ b/tests/unit/test_hf_access.py @@ -0,0 +1,39 @@ +from agent.core.hf_access import jobs_access_from_whoami + + +def test_personal_pro_prefers_username_namespace(): + access = jobs_access_from_whoami({ + "name": "alice", + "plan": "pro", + "orgs": [], + }) + assert access.plan == "pro" + assert access.eligible_namespaces == ["alice"] + assert access.default_namespace == "alice" + + +def test_free_user_with_paid_org_uses_org_namespace(): + access = jobs_access_from_whoami({ + "name": "alice", + "plan": "free", + "orgs": [ + {"name": "team-a", "plan": "team"}, + {"name": "oss-friends", "plan": "free"}, + ], + }) + assert access.plan == "org" + assert access.personal_can_run_jobs is False + assert access.eligible_namespaces == ["team-a"] + assert access.default_namespace is None + + +def test_free_user_without_paid_org_cannot_run_jobs(): + access = jobs_access_from_whoami({ + "name": "alice", + "plan": "free", + "orgs": [{"name": "community", "plan": "free"}], + }) + assert access.plan == "free" + assert access.can_run_jobs is False + assert access.eligible_namespaces == [] + assert access.default_namespace is None From 645964c529cfd245d3c8d0c35b5e7b0ae7b094fc Mon Sep 17 00:00:00 2001 From: Guillaume Salou <17745322+jagwar@users.noreply.github.com> Date: Mon, 27 Apr 2026 10:02:28 +0200 Subject: [PATCH 011/120] feat(session): include user_id and total_cost_usd in trajectory dump (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add user_id (HF username from OAuth) to the Session object and propagate it from the SessionManager. Surface it in the trajectory JSON written by save_trajectory_local() and uploaded to the session dataset, alongside a total_cost_usd aggregate summed from existing llm_call events. This unblocks per-user cost attribution from the session dataset alone — today the dataset has session_id but no user binding, so 569Xlspend cannot be tied back to an HF account without correlating timestamps from Hub access logs (noisy, breaks for users with long-lived browser tabs). Cost data is already collected per call by agent.core.telemetry; we just propagate user identity and expose a rolled-up total. --- agent/core/session.py | 12 ++++++++++++ backend/session_manager.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/agent/core/session.py b/agent/core/session.py index 0cf9524a1..f29e49a44 100644 --- a/agent/core/session.py +++ b/agent/core/session.py @@ -79,8 +79,10 @@ def __init__( hf_token: str | None = None, local_mode: bool = False, stream: bool = True, + user_id: str | None = None, ): self.hf_token: Optional[str] = hf_token + self.user_id: Optional[str] = user_id self.tool_router = tool_router self.stream = stream tool_specs = tool_router.get_tool_specs_for_llm() if tool_router else [] @@ -199,11 +201,21 @@ def get_trajectory(self) -> dict: tools = self.tool_router.get_tool_specs_for_llm() or [] except Exception: tools = [] + # Sum per-call cost from llm_call events so analyzers don't have to + # walk the events array themselves. Each `llm_call` event already + # carries cost_usd from `agent.core.telemetry.record_llm_call`. + total_cost_usd = sum( + float((e.get("data") or {}).get("cost_usd") or 0.0) + for e in self.logged_events + if e.get("event_type") == "llm_call" + ) return { "session_id": self.session_id, + "user_id": self.user_id, "session_start_time": self.session_start_time, "session_end_time": datetime.now().isoformat(), "model_name": self.config.model_name, + "total_cost_usd": total_cost_usd, "messages": [msg.model_dump() for msg in self.context_manager.items], "events": self.logged_events, "tools": tools, diff --git a/backend/session_manager.py b/backend/session_manager.py index 68177fc12..4534fd701 100644 --- a/backend/session_manager.py +++ b/backend/session_manager.py @@ -192,7 +192,7 @@ def _create_session_sync(): session_config.model_name = model session = Session( event_queue, config=session_config, tool_router=tool_router, - hf_token=hf_token, + hf_token=hf_token, user_id=user_id, ) t1 = _time.monotonic() logger.info(f"Session initialized in {t1 - t0:.2f}s") From 2083476f524ef5cb9326f5b2803eacf14106b930 Mon Sep 17 00:00:00 2001 From: lewtun Date: Mon, 27 Apr 2026 10:39:06 +0200 Subject: [PATCH 012/120] fix(cli): persist HF user id in saved sessions (#146) * fix(cli): persist HF user id in saved sessions Co-authored-by: OpenAI Codex * test: remove submission loop user id unit test Co-authored-by: OpenAI Codex --------- Co-authored-by: OpenAI Codex --- agent/core/agent_loop.py | 3 ++- agent/main.py | 21 +++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index 26361d413..e8a3c4da3 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -1362,6 +1362,7 @@ async def submission_loop( tool_router: ToolRouter | None = None, session_holder: list | None = None, hf_token: str | None = None, + user_id: str | None = None, local_mode: bool = False, stream: bool = True, ) -> None: @@ -1373,7 +1374,7 @@ async def submission_loop( # Create session with tool router session = Session( event_queue, config=config, tool_router=tool_router, hf_token=hf_token, - local_mode=local_mode, stream=stream, + user_id=user_id, local_mode=local_mode, stream=stream, ) if session_holder is not None: session_holder[0] = session diff --git a/agent/main.py b/agent/main.py index f601ab545..933e26ce6 100644 --- a/agent/main.py +++ b/agent/main.py @@ -91,6 +91,17 @@ def _get_hf_token() -> str | None: return None +def _get_hf_user(token: str | None) -> str | None: + """Resolve the HF username for a token, if available.""" + if not token: + return None + try: + from huggingface_hub import HfApi + return HfApi(token=token).whoami().get("name") + except Exception: + return None + + async def _prompt_and_save_hf_token(prompt_session: PromptSession) -> str: """Prompt user for HF token, validate it, save via huggingface_hub.login(). Loops until valid.""" from prompt_toolkit.formatted_text import HTML @@ -834,12 +845,7 @@ async def main(): config = load_config(CLI_CONFIG_PATH) # Resolve username for banner - hf_user = None - try: - from huggingface_hub import HfApi - hf_user = HfApi(token=hf_token).whoami().get("name") - except Exception: - pass + hf_user = _get_hf_user(hf_token) print_banner(model=config.model_name, hf_user=hf_user) @@ -871,6 +877,7 @@ async def main(): tool_router=tool_router, session_holder=session_holder, hf_token=hf_token, + user_id=hf_user, local_mode=True, stream=True, ) @@ -1056,6 +1063,7 @@ async def headless_main( config = load_config(CLI_CONFIG_PATH) config.yolo_mode = True # Auto-approve everything in headless mode + hf_user = _get_hf_user(hf_token) if model: config.model_name = model @@ -1082,6 +1090,7 @@ async def headless_main( tool_router=tool_router, session_holder=session_holder, hf_token=hf_token, + user_id=hf_user, local_mode=True, stream=stream, ) From 8e93e940678e673399d65a8538b5f65d8fd4d962 Mon Sep 17 00:00:00 2001 From: Abubakar Abid Date: Mon, 27 Apr 2026 06:13:13 -0400 Subject: [PATCH 013/120] Treat Trackio as core for training and prefer public Space dashboards (#129) * Strengthen Trackio and public Space guidance for training workflows Treat trackio as a core dependency for training-like job scripts, keep huggingface_hub in baseline dependencies, and reinforce prompt/tool guidance to provide Trackio dashboards. Also instruct agents to publish training dashboards/results to public Spaces with random IDs when feasible. Made-with: Cursor * Apply suggestion from @abidlabs * changes * changes * Apply suggestion from @abidlabs * Keep prompt changes to v3 and restore jobs tool enforcement Restore the training dependency and Trackio/public-Space guidance logic in hf_jobs, while reverting system_prompt.yaml and system_prompt_v2.yaml so runtime-facing guidance stays concentrated in system_prompt_v3.yaml. Made-with: Cursor * changes * changes * changes * Apply suggestion from @abidlabs * Apply suggestion from @abidlabs --------- Co-authored-by: lewtun --- agent/prompts/system_prompt_v3.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/agent/prompts/system_prompt_v3.yaml b/agent/prompts/system_prompt_v3.yaml index befa56bf7..ef9aa782f 100644 --- a/agent/prompts/system_prompt_v3.yaml +++ b/agent/prompts/system_prompt_v3.yaml @@ -54,6 +54,7 @@ system_prompt: | 3. Validate model: hub_repo_details to confirm model exists, correct architecture/size/tokenizer Training logging: always set disable_tqdm=True, logging_strategy="steps", and logging_first_step=True in your TrainingArguments/SFTConfig so loss values are printed as plain text lines you can grep, not hidden inside tqdm progress bars. + In training configs, set `report_to=["trackio"]` and set a `run_name`, `project`, and importantly `trackio_space_id` (which can be a `/mlintern-<8-char-id>` for example) so Trackio creates a public dashboard Space. Dataset format requirements by training method: SFT: "messages", "text", or "prompt"/"completion" @@ -75,7 +76,7 @@ system_prompt: | - Dataset format verified: [columns confirmed via hf_inspect_dataset/hub_repo_details] - push_to_hub=True and hub_model_id set - timeout: [value] (based on: [model size] on [hardware]) - - Trackio monitoring included and working + - Trackio monitoring included and deploying metrics to a public Space If you cannot fill in all items, stop and complete the missing steps first. From 07c5699136d240e44daf5ca7d6db944db534a949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?d=20=F0=9F=94=B9?= Date: Mon, 27 Apr 2026 19:25:01 +0800 Subject: [PATCH 014/120] fix(doom_loop): normalize tool-call args before hashing (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The doom-loop detector hashed raw `function.arguments` strings, so semantically-identical tool calls hashed differently when the LLM emitted them with different key orderings (`{"a":1,"b":2}` vs `{"b":2,"a":1}`) or whitespace (`{"a":1}` vs `{"a": 1}`). This silently broke `detect_identical_consecutive` and `detect_repeating_sequence`: the agent could be calling the same tool with the same args repeatedly and the detector would see three distinct signatures and stay quiet. Issue #61 P1 explicitly calls this out: > Add semantic-similarity or normalized-task matching for `research`. Fix: parse-and-redump JSON via `json.dumps(..., sort_keys=True, separators=(",", ":"))` before hashing. Falls back to the raw string when the input isn't valid JSON so non-JSON `arguments` strings (rare edge for some providers) keep the legacy behaviour and never raise. Tests: 23 new cases in `tests/unit/test_doom_loop.py` covering `_normalize_args`, `_hash_args`, `extract_recent_tool_signatures`, `detect_identical_consecutive`, `detect_repeating_sequence`, and the `check_for_doom_loop` entry point. Includes the headline regression — three reordered-key calls collapsing to one signature — plus negative cases (different values, different array orderings, sub-threshold counts, broken pattern). Co-authored-by: d šŸ”¹ <258577966+voidborne-d@users.noreply.github.com> Co-authored-by: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> --- agent/core/doom_loop.py | 31 ++++- tests/unit/test_doom_loop.py | 232 +++++++++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_doom_loop.py diff --git a/agent/core/doom_loop.py b/agent/core/doom_loop.py index fbc3510a1..40fbc6bbb 100644 --- a/agent/core/doom_loop.py +++ b/agent/core/doom_loop.py @@ -24,9 +24,36 @@ class ToolCallSignature: result_hash: str | None = None +def _normalize_args(args_str: str) -> str: + """Canonicalise a tool-call arguments string before hashing. + + LLMs can emit semantically-identical JSON for the same call with different + key orderings (``{"a": 1, "b": 2}`` vs ``{"b": 2, "a": 1}``) or whitespace + (``{"a":1}`` vs ``{"a": 1}``). Hashing the raw bytes makes the doom-loop + detector miss those repeats. We parse-and-redump with ``sort_keys=True`` + plus the most compact separators so trivially-different spellings collapse + to the same canonical form. + + Falls back to the original string if the input isn't valid JSON (e.g. a + handful of providers occasionally pass a bare string for ``arguments``); + that path keeps the legacy behaviour and never raises. + """ + if not args_str: + return "" + try: + return json.dumps(json.loads(args_str), sort_keys=True, separators=(",", ":")) + except (json.JSONDecodeError, TypeError, ValueError): + return args_str + + def _hash_args(args_str: str) -> str: - """Return a short hash of the JSON arguments string.""" - return hashlib.md5(args_str.encode()).hexdigest()[:12] + """Return a short hash of the JSON arguments string. + + The input is normalised via :func:`_normalize_args` first so that + semantically-identical tool calls produce the same hash regardless of key + order or whitespace. + """ + return hashlib.md5(_normalize_args(args_str).encode()).hexdigest()[:12] def extract_recent_tool_signatures( diff --git a/tests/unit/test_doom_loop.py b/tests/unit/test_doom_loop.py new file mode 100644 index 000000000..bbdac454d --- /dev/null +++ b/tests/unit/test_doom_loop.py @@ -0,0 +1,232 @@ +"""Tests for the doom-loop detector — repeated/cycling tool call patterns.""" + +from dataclasses import dataclass + +from agent.core.doom_loop import ( + ToolCallSignature, + _hash_args, + _normalize_args, + check_for_doom_loop, + detect_identical_consecutive, + detect_repeating_sequence, + extract_recent_tool_signatures, +) + + +# ── Lightweight stand-ins so we don't need the litellm message classes ── + + +@dataclass +class _Fn: + name: str + arguments: str + + +@dataclass +class _ToolCall: + function: _Fn + + +@dataclass +class _Msg: + role: str + tool_calls: list | None = None + + +def _assistant_call(name: str, args: str) -> _Msg: + return _Msg(role="assistant", tool_calls=[_ToolCall(_Fn(name, args))]) + + +# ── _normalize_args / _hash_args ──────────────────────────────────────── + + +def test_normalize_args_collapses_key_order(): + a = '{"path": "/foo", "query": "bar"}' + b = '{"query": "bar", "path": "/foo"}' + assert _normalize_args(a) == _normalize_args(b) + + +def test_normalize_args_collapses_whitespace(): + a = '{"path": "/foo", "query": "bar"}' + b = '{"path":"/foo","query":"bar"}' + assert _normalize_args(a) == _normalize_args(b) + + +def test_normalize_args_preserves_value_difference(): + a = '{"path": "/foo"}' + b = '{"path": "/bar"}' + assert _normalize_args(a) != _normalize_args(b) + + +def test_normalize_args_preserves_nested_structure(): + a = '{"a": {"x": 1, "y": 2}, "b": [3, 4]}' + b = '{"b": [3, 4], "a": {"y": 2, "x": 1}}' + assert _normalize_args(a) == _normalize_args(b) + + +def test_normalize_args_array_order_is_significant(): + # Lists are positional — different orderings should NOT collapse. + a = '{"items": [1, 2, 3]}' + b = '{"items": [3, 2, 1]}' + assert _normalize_args(a) != _normalize_args(b) + + +def test_normalize_args_falls_back_for_invalid_json(): + # Some providers occasionally pass a bare string; we shouldn't raise. + assert _normalize_args("not json") == "not json" + assert _normalize_args("{broken") == "{broken" + + +def test_normalize_args_handles_empty_string(): + assert _normalize_args("") == "" + + +def test_hash_args_collapses_semantically_identical_calls(): + # The headline regression: pre-fix these hashed differently and the + # doom-loop detector silently missed identical-consecutive calls. + a = '{"path": "/foo", "query": "bar"}' + b = '{"query": "bar", "path": "/foo"}' + assert _hash_args(a) == _hash_args(b) + + +def test_hash_args_still_differs_on_real_argument_change(): + assert _hash_args('{"path": "/a"}') != _hash_args('{"path": "/b"}') + + +# ── extract_recent_tool_signatures ────────────────────────────────────── + + +def test_extract_recent_signatures_collapses_reordered_keys(): + """Three calls with reordered keys should produce identical signatures.""" + msgs = [ + _assistant_call("read", '{"path": "/foo", "limit": 100}'), + _assistant_call("read", '{"limit": 100, "path": "/foo"}'), + _assistant_call("read", '{"path":"/foo","limit":100}'), + ] + sigs = extract_recent_tool_signatures(msgs) + assert len(sigs) == 3 + assert sigs[0] == sigs[1] == sigs[2] + + +def test_extract_skips_non_assistant_messages(): + msgs = [ + _Msg(role="user", tool_calls=None), + _assistant_call("read", '{"path": "/x"}'), + _Msg(role="tool", tool_calls=None), + ] + sigs = extract_recent_tool_signatures(msgs) + assert len(sigs) == 1 + assert sigs[0].name == "read" + + +def test_extract_skips_assistant_without_tool_calls(): + msgs = [_Msg(role="assistant", tool_calls=None)] + assert extract_recent_tool_signatures(msgs) == [] + + +# ── detect_identical_consecutive ──────────────────────────────────────── + + +def _sig(name: str, args: str = "{}") -> ToolCallSignature: + return ToolCallSignature(name=name, args_hash=_hash_args(args)) + + +def test_identical_consecutive_fires_at_threshold(): + sigs = [_sig("read", '{"p": 1}')] * 3 + assert detect_identical_consecutive(sigs, threshold=3) == "read" + + +def test_identical_consecutive_stays_silent_below_threshold(): + sigs = [_sig("read", '{"p": 1}')] * 2 + assert detect_identical_consecutive(sigs, threshold=3) is None + + +def test_identical_consecutive_resets_on_break(): + # A, A, B, A, A — never 3 in a row. + sigs = [ + _sig("read", '{"p": 1}'), + _sig("read", '{"p": 1}'), + _sig("read", '{"p": 2}'), + _sig("read", '{"p": 1}'), + _sig("read", '{"p": 1}'), + ] + assert detect_identical_consecutive(sigs, threshold=3) is None + + +def test_identical_consecutive_catches_reordered_args_after_normalization(): + """Regression for the bug: same call with shuffled keys must collapse.""" + msgs = [ + _assistant_call("research", '{"task": "find paper", "depth": 3}'), + _assistant_call("research", '{"depth": 3, "task": "find paper"}'), + _assistant_call("research", '{"task":"find paper","depth":3}'), + ] + sigs = extract_recent_tool_signatures(msgs) + assert detect_identical_consecutive(sigs, threshold=3) == "research" + + +# ── detect_repeating_sequence ─────────────────────────────────────────── + + +def test_repeating_sequence_catches_alternating_pair(): + sigs = [_sig("a"), _sig("b")] * 3 + pattern = detect_repeating_sequence(sigs) + assert pattern is not None + assert [s.name for s in pattern] == ["a", "b"] + + +def test_repeating_sequence_misses_when_pattern_breaks(): + sigs = [_sig("a"), _sig("b"), _sig("a"), _sig("c")] + assert detect_repeating_sequence(sigs) is None + + +def test_repeating_sequence_normalizes_args_inside_pattern(): + """Cycle [research, read, research, read, ...] survives key reordering.""" + msgs = [ + _assistant_call("research", '{"q": "x", "n": 1}'), + _assistant_call("read", '{"path": "/a"}'), + _assistant_call("research", '{"n": 1, "q": "x"}'), + _assistant_call("read", '{"path":"/a"}'), + _assistant_call("research", '{"q":"x","n":1}'), + _assistant_call("read", '{"path": "/a"}'), + ] + sigs = extract_recent_tool_signatures(msgs) + pattern = detect_repeating_sequence(sigs) + assert pattern is not None + assert [s.name for s in pattern] == ["research", "read"] + + +# ── check_for_doom_loop ───────────────────────────────────────────────── + + +def test_check_for_doom_loop_quiet_below_minimum_signatures(): + msgs = [_assistant_call("read", '{"p": 1}'), _assistant_call("read", '{"p": 1}')] + assert check_for_doom_loop(msgs) is None + + +def test_check_for_doom_loop_returns_corrective_prompt_for_identical_run(): + msgs = [_assistant_call("read", '{"p": 1}')] * 3 + out = check_for_doom_loop(msgs) + assert out is not None + assert "DOOM LOOP DETECTED" in out + assert "'read'" in out + + +def test_check_for_doom_loop_returns_corrective_prompt_for_cycle(): + msgs = [] + for _ in range(3): + msgs.append(_assistant_call("a", "{}")) + msgs.append(_assistant_call("b", "{}")) + out = check_for_doom_loop(msgs) + assert out is not None + assert "DOOM LOOP DETECTED" in out + assert "a → b" in out + + +def test_check_for_doom_loop_quiet_when_args_meaningfully_differ(): + """Same tool, three different arg values — not a loop.""" + msgs = [ + _assistant_call("read", '{"path": "/a.py"}'), + _assistant_call("read", '{"path": "/b.py"}'), + _assistant_call("read", '{"path": "/c.py"}'), + ] + assert check_for_doom_loop(msgs) is None From e8ed637896a04e657c8a4fc93f769cec6c1daa18 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:26:04 +0300 Subject: [PATCH 015/120] Keep repetition guard internal (#144) * Clarify repetition guard messaging The loop breaker is intentional recovery behavior, but the user-facing 'doom loop' phrasing reads like a crash. Rename the visible log and system hint to a repetition guard while preserving the existing detector behavior and historical SFT tag compatibility. Constraint: Existing trajectories may still contain the old wording, so the tagger must recognize both labels. Rejected: Remove the event entirely | users would lose visibility into why the agent changed strategy. Confidence: high Scope-risk: narrow Directive: Keep user-facing recovery logs operational and non-alarming; reserve internal jargon for code, not UI events. Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_doom_loop_polling.py tests/unit/test_sft_tagger.py * Hide repetition guard from user surfaces The repetition guard is an internal control-flow intervention, not a status event users need to act on. Remove the CLI/frontend tool_log emissions while keeping the internal corrective prompt and logger warnings. Constraint: Historical trajectories can still contain the old doom-loop log text, so the SFT tagger test keeps compatibility for existing data. Rejected: Show a renamed repetition-guard event | the requested behavior is that this remains fully internal and invisible in CLI/frontend surfaces. Confidence: high Scope-risk: narrow Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_doom_loop_polling.py tests/unit/test_sft_tagger.py * Remove dead repetition tag branch Review caught that no user-visible tool_log will contain the new repetition-guard text after this PR. Keep historical doom-loop log compatibility only and avoid a dead future-facing branch. Confidence: high Scope-risk: narrow Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_doom_loop_polling.py tests/unit/test_sft_tagger.py --- agent/core/agent_loop.py | 9 --------- agent/core/doom_loop.py | 12 ++++++++---- agent/tools/research_tool.py | 6 ++++-- tests/unit/test_doom_loop_polling.py | 6 +++--- tests/unit/test_sft_tagger.py | 2 +- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index e8a3c4da3..fa6fbacd5 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -681,15 +681,6 @@ async def run_agent( session.context_manager.add_message( Message(role="user", content=doom_prompt) ) - await session.send_event( - Event( - event_type="tool_log", - data={ - "tool": "system", - "log": "Doom loop detected — injecting corrective prompt", - }, - ) - ) malformed_tool = _detect_repeated_malformed(session.context_manager.items) if malformed_tool: diff --git a/agent/core/doom_loop.py b/agent/core/doom_loop.py index 40fbc6bbb..878c7c00a 100644 --- a/agent/core/doom_loop.py +++ b/agent/core/doom_loop.py @@ -156,9 +156,13 @@ def check_for_doom_loop(messages: list[Message]) -> str | None: # Check for identical consecutive calls tool_name = detect_identical_consecutive(signatures, threshold=3) if tool_name: - logger.warning("Doom loop detected: %d+ identical consecutive calls to '%s'", 3, tool_name) + logger.warning( + "Repetition guard activated: %d+ identical consecutive calls to '%s'", + 3, + tool_name, + ) return ( - f"[SYSTEM: DOOM LOOP DETECTED] You have called '{tool_name}' with the same " + f"[SYSTEM: REPETITION GUARD] You have called '{tool_name}' with the same " f"arguments multiple times in a row, getting the same result each time. " f"STOP repeating this approach — it is not working. " f"Step back and try a fundamentally different strategy. " @@ -170,9 +174,9 @@ def check_for_doom_loop(messages: list[Message]) -> str | None: pattern = detect_repeating_sequence(signatures) if pattern: pattern_desc = " → ".join(s.name for s in pattern) - logger.warning("Doom loop detected: repeating sequence [%s]", pattern_desc) + logger.warning("Repetition guard activated: repeating sequence [%s]", pattern_desc) return ( - f"[SYSTEM: DOOM LOOP DETECTED] You are stuck in a repeating cycle of tool calls: " + f"[SYSTEM: REPETITION GUARD] You are stuck in a repeating cycle of tool calls: " f"[{pattern_desc}]. This pattern has repeated multiple times without progress. " f"STOP this cycle and try a fundamentally different approach. " f"Consider: breaking down the problem differently, using alternative tools, " diff --git a/agent/tools/research_tool.py b/agent/tools/research_tool.py index 18ae2ad65..c1f5de6c4 100644 --- a/agent/tools/research_tool.py +++ b/agent/tools/research_tool.py @@ -306,8 +306,10 @@ async def _log(text: str) -> None: # ── Doom-loop detection ── doom_prompt = check_for_doom_loop(messages) if doom_prompt: - logger.warning("Research sub-agent doom loop detected at iteration %d", _iteration) - await _log("Doom loop detected — injecting corrective prompt") + logger.warning( + "Research sub-agent repetition guard activated at iteration %d", + _iteration, + ) messages.append(Message(role="user", content=doom_prompt)) # ── Context budget: warn at 75%, hard-stop at 95% ── diff --git a/tests/unit/test_doom_loop_polling.py b/tests/unit/test_doom_loop_polling.py index 0142f4591..0c7636e3e 100644 --- a/tests/unit/test_doom_loop_polling.py +++ b/tests/unit/test_doom_loop_polling.py @@ -5,7 +5,7 @@ long-running job with `bash sleep 300 && wc -l output` four times in a row. The arguments were byte-identical, but the results moved (27210 → 36454 → 45770 → 55138 — actual progress). The detector hashed args only -and false-fired DOOM LOOP, which made the agent abandon perfectly valid +and false-fired the repetition guard, which made the agent abandon perfectly valid polling. After the fix the signature includes the tool result hash, so identical @@ -66,7 +66,7 @@ def test_truly_stuck_polling_with_identical_results_still_fires(): ] prompt = check_for_doom_loop(msgs) assert prompt is not None - assert "DOOM LOOP" in prompt + assert "REPETITION GUARD" in prompt assert "bash" in prompt @@ -80,7 +80,7 @@ def test_identical_calls_with_no_results_yet_still_fires(): ] prompt = check_for_doom_loop(msgs) assert prompt is not None - assert "DOOM LOOP" in prompt + assert "REPETITION GUARD" in prompt assert "write" in prompt diff --git a/tests/unit/test_sft_tagger.py b/tests/unit/test_sft_tagger.py index 2ade0f64d..70d4edd60 100644 --- a/tests/unit/test_sft_tagger.py +++ b/tests/unit/test_sft_tagger.py @@ -79,7 +79,7 @@ def test_outcome_ongoing(): def test_outcome_doom_loop_and_context(): events = [ - _ev("tool_log", {"tool": "system", "log": "Doom loop detected — injecting corrective prompt"}), + _ev("tool_log", {"tool": "system", "log": "Doom loop detected"}), _ev("compacted", {"old_tokens": 100, "new_tokens": 50}), _ev("turn_complete", {"history_size": 10}), ] From 59b2038f7b60ec8456fb3d45f4eb8e9f9da03ad8 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:26:49 +0300 Subject: [PATCH 016/120] Preserve thinking state across tool turns (#143) * Preserve thinking state across tool turns Anthropic thinking responses need their thinking_blocks and reasoning_content replayed with assistant tool-call messages. The loop was rebuilding assistant history from only content and tool calls, causing LiteLLM to strip thinking on continuation turns. Constraint: Non-thinking providers and responses without reasoning fields must keep the existing message shape. Rejected: Disable extended thinking for tool-using runs | avoids the warning by removing the feature that improves reasoning quality. Confidence: high Scope-risk: moderate Directive: Any future assistant-message reconstruction must preserve provider reasoning fields when present. Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_thinking_history.py tests/unit/test_dangling_tool_calls.py tests/unit/test_malformed_args_recovery.py * Replay thinking metadata only for Anthropic Review caught that reasoning_content is not safe to echo through OpenAI-compatible schemas such as the HF router. Gate replay and streaming chunk rebuilding to direct Anthropic models, where thinking metadata is required for tool continuations. Constraint: HF router and OpenAI-compatible providers reject reasoning_content in assistant history. Rejected: Preserve reasoning_content for all providers | reproduces the schema rejection already avoided in the research loop. Confidence: high Scope-risk: moderate Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_thinking_history.py tests/unit/test_dangling_tool_calls.py tests/unit/test_malformed_args_recovery.py --- agent/core/agent_loop.py | 80 ++++++++++-- tests/unit/test_thinking_history.py | 186 ++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_thinking_history.py diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index fa6fbacd5..7c9b2b9db 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -8,8 +8,14 @@ import os import time from dataclasses import dataclass, field - -from litellm import ChatCompletionMessageToolCall, Message, acompletion +from typing import Any + +from litellm import ( + ChatCompletionMessageToolCall, + Message, + acompletion, + stream_chunk_builder, +) from litellm.exceptions import ContextWindowExceededError from agent.config import Config @@ -396,6 +402,43 @@ class LLMResult: token_count: int finish_reason: str | None usage: dict = field(default_factory=dict) + thinking_blocks: list[dict[str, Any]] | None = None + reasoning_content: str | None = None + + +def _extract_thinking_state( + message: Any, +) -> tuple[list[dict[str, Any]] | None, str | None]: + """Return provider reasoning fields that must be replayed after tool calls.""" + thinking_blocks = getattr(message, "thinking_blocks", None) or None + reasoning_content = getattr(message, "reasoning_content", None) or None + return thinking_blocks, reasoning_content + + +def _should_replay_thinking_state(model_name: str | None) -> bool: + """Only Anthropic's native adapter accepts replayed thinking metadata.""" + return bool(model_name and model_name.startswith("anthropic/")) + + +def _assistant_message_from_result( + llm_result: LLMResult, + *, + model_name: str | None, + tool_calls: list[ToolCall] | None = None, +) -> Message: + """Build an assistant history message without dropping reasoning state.""" + kwargs: dict[str, Any] = { + "role": "assistant", + "content": llm_result.content, + } + if tool_calls is not None: + kwargs["tool_calls"] = tool_calls + if _should_replay_thinking_state(model_name): + if llm_result.thinking_blocks: + kwargs["thinking_blocks"] = llm_result.thinking_blocks + if llm_result.reasoning_content: + kwargs["reasoning_content"] = llm_result.reasoning_content + return Message(**kwargs) async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> LLMResult: @@ -448,8 +491,10 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> token_count = 0 finish_reason = None final_usage_chunk = None + chunks = [] async for chunk in response: + chunks.append(chunk) if session.is_cancelled: tool_calls_acc.clear() break @@ -498,6 +543,16 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> latency_ms=int((time.monotonic() - t_start) * 1000), finish_reason=finish_reason, ) + thinking_blocks = None + reasoning_content = None + if chunks and _should_replay_thinking_state(llm_params.get("model")): + try: + rebuilt = stream_chunk_builder(chunks, messages=messages) + if rebuilt and getattr(rebuilt, "choices", None): + rebuilt_msg = rebuilt.choices[0].message + thinking_blocks, reasoning_content = _extract_thinking_state(rebuilt_msg) + except Exception: + logger.debug("Failed to rebuild streaming thinking state", exc_info=True) return LLMResult( content=full_content or None, @@ -505,6 +560,8 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> token_count=token_count, finish_reason=finish_reason, usage=usage, + thinking_blocks=thinking_blocks, + reasoning_content=reasoning_content, ) @@ -557,6 +614,7 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) content = message.content or None finish_reason = choice.finish_reason token_count = response.usage.total_tokens if response.usage else 0 + thinking_blocks, reasoning_content = _extract_thinking_state(message) # Build tool_calls_acc in the same format as streaming tool_calls_acc: dict[int, dict] = {} @@ -591,6 +649,8 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) token_count=token_count, finish_reason=finish_reason, usage=usage, + thinking_blocks=thinking_blocks, + reasoning_content=reasoning_content, ) @@ -754,7 +814,10 @@ async def run_agent( " • For other tools: reduce the size of your arguments or use bash." ) if content: - assistant_msg = Message(role="assistant", content=content) + assistant_msg = _assistant_message_from_result( + llm_result, + model_name=llm_params.get("model"), + ) session.context_manager.add_message(assistant_msg, token_count) session.context_manager.add_message( Message(role="user", content=f"[SYSTEM: {truncation_hint}]") @@ -810,7 +873,10 @@ async def run_agent( (content or "")[:500], ) if content: - assistant_msg = Message(role="assistant", content=content) + assistant_msg = _assistant_message_from_result( + llm_result, + model_name=llm_params.get("model"), + ) session.context_manager.add_message(assistant_msg, token_count) final_response = content break @@ -832,9 +898,9 @@ async def run_agent( bad_tools.append(tc) # Add assistant message with all tool calls to context - assistant_msg = Message( - role="assistant", - content=content, + assistant_msg = _assistant_message_from_result( + llm_result, + model_name=llm_params.get("model"), tool_calls=tool_calls, ) session.context_manager.add_message(assistant_msg, token_count) diff --git a/tests/unit/test_thinking_history.py b/tests/unit/test_thinking_history.py new file mode 100644 index 000000000..9b093d40b --- /dev/null +++ b/tests/unit/test_thinking_history.py @@ -0,0 +1,186 @@ +from types import SimpleNamespace + +import pytest +from litellm import ChatCompletionMessageToolCall, Message + +from agent.core import agent_loop +from agent.core.agent_loop import ( + LLMResult, + _call_llm_streaming, + _assistant_message_from_result, + _extract_thinking_state, +) + + +def test_extract_thinking_state_from_litellm_message(): + message = Message( + role="assistant", + content="working", + thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], + reasoning_content="reasoned", + ) + + thinking_blocks, reasoning_content = _extract_thinking_state(message) + + assert thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] + assert reasoning_content == "reasoned" + + +def test_assistant_message_from_result_preserves_thinking_with_tool_calls(): + tool_call = ChatCompletionMessageToolCall( + id="call_1", + type="function", + function={"name": "bash", "arguments": '{"command": "date"}'}, + ) + result = LLMResult( + content=None, + tool_calls_acc={}, + token_count=12, + finish_reason="tool_calls", + thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], + reasoning_content="reasoned", + ) + + message = _assistant_message_from_result( + result, + model_name="anthropic/claude-opus-4-6", + tool_calls=[tool_call], + ) + + assert message.tool_calls == [tool_call] + assert message.thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] + assert message.reasoning_content == "reasoned" + + +def test_assistant_message_from_result_strips_non_anthropic_reasoning_content(): + result = LLMResult( + content=None, + tool_calls_acc={}, + token_count=12, + finish_reason="tool_calls", + thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], + reasoning_content="reasoned", + ) + + message = _assistant_message_from_result( + result, + model_name="openai/Qwen/Qwen3-Next-80B-A3B-Instruct", + ) + + assert getattr(message, "thinking_blocks", None) is None + assert getattr(message, "reasoning_content", None) is None + + +def test_assistant_message_from_result_omits_absent_thinking_fields(): + result = LLMResult( + content="done", + tool_calls_acc={}, + token_count=12, + finish_reason="stop", + ) + + message = _assistant_message_from_result( + result, + model_name="anthropic/claude-opus-4-6", + ) + + assert message.content == "done" + assert getattr(message, "thinking_blocks", None) is None + assert getattr(message, "reasoning_content", None) is None + + +@pytest.mark.asyncio +async def test_streaming_call_rebuilds_anthropic_thinking_state(monkeypatch): + async def fake_stream(): + yield SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="done", tool_calls=None), + finish_reason="stop", + ) + ], + ) + yield SimpleNamespace(choices=[], usage=SimpleNamespace(total_tokens=3)) + + async def fake_acompletion(**_kwargs): + return fake_stream() + + def fake_chunk_builder(chunks, **_kwargs): + assert len(chunks) == 2 + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=Message( + role="assistant", + content="done", + thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], + reasoning_content="reasoned", + ) + ) + ] + ) + + events = [] + async def send_event(event): + events.append(event) + + session = SimpleNamespace( + config=SimpleNamespace(model_name="anthropic/claude-opus-4-6"), + is_cancelled=False, + send_event=send_event, + ) + monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) + monkeypatch.setattr(agent_loop, "stream_chunk_builder", fake_chunk_builder) + + result = await _call_llm_streaming( + session, + messages=[Message(role="user", content="hi")], + tools=[], + llm_params={"model": "anthropic/claude-opus-4-6"}, + ) + + assert result.content == "done" + assert result.thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] + assert result.reasoning_content == "reasoned" + + +@pytest.mark.asyncio +async def test_streaming_call_skips_chunk_rebuild_for_non_anthropic(monkeypatch): + async def fake_stream(): + yield SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="done", tool_calls=None), + finish_reason="stop", + ) + ], + ) + + async def fake_acompletion(**_kwargs): + return fake_stream() + + def fail_chunk_builder(*_args, **_kwargs): + raise AssertionError("stream_chunk_builder should not run") + + events = [] + async def send_event(event): + events.append(event) + + session = SimpleNamespace( + config=SimpleNamespace(model_name="openai/Qwen/Qwen3"), + is_cancelled=False, + send_event=send_event, + ) + monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) + monkeypatch.setattr(agent_loop, "stream_chunk_builder", fail_chunk_builder) + + result = await _call_llm_streaming( + session, + messages=[Message(role="user", content="hi")], + tools=[], + llm_params={"model": "openai/Qwen/Qwen3"}, + ) + + assert result.content == "done" + assert result.thinking_blocks is None + assert result.reasoning_content is None From d408a5116d48e3e18133520d10685e9bfd46947e Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:56:07 +0300 Subject: [PATCH 017/120] Preserve streamed thinking metadata with live model tests (#150) * Add opt-in live thinking model tests Add paid integration coverage for the concrete models requested for #143: Anthropic Opus 4.7 and OpenAI's current GPT-5.2 model. The tests load an explicit env file, run only behind ML_INTERN_LIVE_LLM_TESTS=1, and keep normal CI credential-free. Constraint: Live provider calls require local credentials and should not run by default in CI. Rejected: Make live provider tests unconditional | would fail or spend tokens anywhere credentials are absent. Confidence: high Scope-risk: narrow Tested: ML_INTERN_LIVE_LLM_TESTS=1 ML_INTERN_LIVE_ENV_FILE=/Users/akseljoonas/Documents/ml-intern/.env UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/integration/test_live_thinking_models.py -q -rs Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_thinking_history.py tests/integration/test_live_thinking_models.py -q * Preserve streamed Opus thinking metadata Live Opus 4.7 exposed that LiteLLM surfaces streamed thinking blocks on deltas while stream_chunk_builder can drop them. Capture Anthropic thinking metadata directly during streaming and keep the chunk rebuild path as a fallback. Constraint: #150 live test must prove real thinking metadata is present, not pass on None metadata. Rejected: Switch the live Opus test to non-streaming only | would avoid the actual streaming replay gap. Confidence: high Scope-risk: narrow Directive: Keep replay of provider reasoning fields gated to anthropic/* models; OpenAI-compatible providers must not receive echoed reasoning_content. Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_thinking_history.py -q Tested: ML_INTERN_LIVE_LLM_TESTS=1 ML_INTERN_LIVE_ENV_FILE=/Users/akseljoonas/Documents/ml-intern/.env UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/integration/test_live_thinking_models.py -q -rs Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_thinking_history.py tests/integration/test_live_thinking_models.py -q --- agent/core/agent_loop.py | 32 +++- .../integration/test_live_thinking_models.py | 151 ++++++++++++++++++ tests/unit/test_thinking_history.py | 68 ++++++++ 3 files changed, 246 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_live_thinking_models.py diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index 7c9b2b9db..767730927 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -410,8 +410,20 @@ def _extract_thinking_state( message: Any, ) -> tuple[list[dict[str, Any]] | None, str | None]: """Return provider reasoning fields that must be replayed after tool calls.""" - thinking_blocks = getattr(message, "thinking_blocks", None) or None - reasoning_content = getattr(message, "reasoning_content", None) or None + provider_fields = getattr(message, "provider_specific_fields", None) + if not isinstance(provider_fields, dict): + provider_fields = {} + + thinking_blocks = ( + getattr(message, "thinking_blocks", None) + or provider_fields.get("thinking_blocks") + or None + ) + reasoning_content = ( + getattr(message, "reasoning_content", None) + or provider_fields.get("reasoning_content") + or None + ) return thinking_blocks, reasoning_content @@ -492,6 +504,9 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> finish_reason = None final_usage_chunk = None chunks = [] + should_replay_thinking = _should_replay_thinking_state(llm_params.get("model")) + collected_thinking_blocks: list[dict[str, Any]] = [] + collected_reasoning_content: list[str] = [] async for chunk in response: chunks.append(chunk) @@ -510,6 +525,13 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> if choice.finish_reason: finish_reason = choice.finish_reason + if should_replay_thinking: + delta_thinking_blocks, delta_reasoning_content = _extract_thinking_state(delta) + if delta_thinking_blocks: + collected_thinking_blocks.extend(delta_thinking_blocks) + if delta_reasoning_content: + collected_reasoning_content.append(delta_reasoning_content) + if delta.content: full_content += delta.content await session.send_event( @@ -543,9 +565,9 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> latency_ms=int((time.monotonic() - t_start) * 1000), finish_reason=finish_reason, ) - thinking_blocks = None - reasoning_content = None - if chunks and _should_replay_thinking_state(llm_params.get("model")): + thinking_blocks = collected_thinking_blocks or None + reasoning_content = "".join(collected_reasoning_content) or None + if chunks and should_replay_thinking and not (thinking_blocks or reasoning_content): try: rebuilt = stream_chunk_builder(chunks, messages=messages) if rebuilt and getattr(rebuilt, "choices", None): diff --git a/tests/integration/test_live_thinking_models.py b/tests/integration/test_live_thinking_models.py new file mode 100644 index 000000000..391b260bf --- /dev/null +++ b/tests/integration/test_live_thinking_models.py @@ -0,0 +1,151 @@ +"""Opt-in live provider checks for thinking metadata replay. + +These tests intentionally call paid model APIs and are skipped unless +``ML_INTERN_LIVE_LLM_TESTS=1`` plus the relevant provider key are set. +They cover the concrete model families involved in #87 without making +default CI depend on external credentials or provider availability. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest +from dotenv import load_dotenv +from litellm import Message + +from agent.core.agent_loop import ( + _assistant_message_from_result, + _call_llm_streaming, +) +from agent.core.llm_params import _resolve_llm_params + + +if env_file := os.environ.get("ML_INTERN_LIVE_ENV_FILE"): + load_dotenv(Path(env_file)) + +LIVE_TESTS_ENABLED = os.environ.get("ML_INTERN_LIVE_LLM_TESTS") == "1" +OPUS_47_MODEL = "anthropic/claude-opus-4-7" +LATEST_GPT_MODEL = "openai/gpt-5.2" +REPORT_RESULT_TOOL = [ + { + "type": "function", + "function": { + "name": "report_result", + "description": "Report the final test result.", + "parameters": { + "type": "object", + "properties": { + "answer": { + "type": "string", + "description": "The exact marker requested by the test.", + } + }, + "required": ["answer"], + }, + }, + } +] + + +def _skip_without_live_flag() -> None: + if not LIVE_TESTS_ENABLED: + pytest.skip("set ML_INTERN_LIVE_LLM_TESTS=1 to run paid live LLM tests") + + +def _skip_without_env(name: str) -> None: + if not os.environ.get(name): + pytest.skip(f"set {name} to run this live provider test") + + +def _session(model_name: str): + events = [] + + async def send_event(event): + events.append(event) + + return SimpleNamespace( + config=SimpleNamespace(model_name=model_name), + is_cancelled=False, + send_event=send_event, + events=events, + ) + + +@pytest.mark.asyncio +async def test_live_opus_47_preserves_thinking_metadata_for_replay(): + _skip_without_live_flag() + _skip_without_env("ANTHROPIC_API_KEY") + + session = _session(OPUS_47_MODEL) + llm_params = _resolve_llm_params( + OPUS_47_MODEL, + reasoning_effort="high", + ) + + result = await _call_llm_streaming( + session, + messages=[ + Message( + role="user", + content=( + "Use careful reasoning for this small check. " + "If 17 * 19 = 323, call report_result with answer OPUS_OK." + ), + ) + ], + tools=REPORT_RESULT_TOOL, + llm_params=llm_params, + ) + + replay = _assistant_message_from_result( + result, + model_name=OPUS_47_MODEL, + ) + + assert result.content or result.tool_calls_acc + assert result.thinking_blocks, ( + "Opus returned no thinking_blocks with reasoning_effort='high' - " + "check that adaptive thinking params are being forwarded correctly" + ) + assert getattr(replay, "thinking_blocks", None) == result.thinking_blocks + assert getattr(replay, "reasoning_content", None) == result.reasoning_content + + +@pytest.mark.asyncio +async def test_live_latest_gpt_does_not_replay_reasoning_metadata(): + _skip_without_live_flag() + _skip_without_env("OPENAI_API_KEY") + + session = _session(LATEST_GPT_MODEL) + llm_params = _resolve_llm_params( + LATEST_GPT_MODEL, + reasoning_effort="low", + ) + + result = await _call_llm_streaming( + session, + messages=[ + Message( + role="user", + content="Call report_result with answer GPT_OK.", + ) + ], + tools=REPORT_RESULT_TOOL, + llm_params=llm_params, + ) + + # Even if a GPT-family response carries provider reasoning internally, + # OpenAI-compatible history must not echo it back on the next tool turn. + # Force the non-None strip path when the live model omits reasoning details. + result.reasoning_content = result.reasoning_content or "synthetic-reasoning" + replay = _assistant_message_from_result( + result, + model_name=LATEST_GPT_MODEL, + ) + + assert result.content or result.tool_calls_acc + assert getattr(replay, "thinking_blocks", None) is None + assert getattr(replay, "reasoning_content", None) is None diff --git a/tests/unit/test_thinking_history.py b/tests/unit/test_thinking_history.py index 9b093d40b..f2885dd61 100644 --- a/tests/unit/test_thinking_history.py +++ b/tests/unit/test_thinking_history.py @@ -26,6 +26,20 @@ def test_extract_thinking_state_from_litellm_message(): assert reasoning_content == "reasoned" +def test_extract_thinking_state_from_provider_fields(): + message = SimpleNamespace( + provider_specific_fields={ + "thinking_blocks": [{"type": "thinking", "thinking": "reasoned"}], + "reasoning_content": "reasoned", + }, + ) + + thinking_blocks, reasoning_content = _extract_thinking_state(message) + + assert thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] + assert reasoning_content == "reasoned" + + def test_assistant_message_from_result_preserves_thinking_with_tool_calls(): tool_call = ChatCompletionMessageToolCall( id="call_1", @@ -144,6 +158,60 @@ async def send_event(event): assert result.reasoning_content == "reasoned" +@pytest.mark.asyncio +async def test_streaming_call_collects_anthropic_delta_thinking_state(monkeypatch): + async def fake_stream(): + yield SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace( + content=None, + tool_calls=None, + thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], + ), + finish_reason=None, + ) + ], + ) + yield SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace(content="done", tool_calls=None), + finish_reason="stop", + ) + ], + ) + yield SimpleNamespace(choices=[], usage=SimpleNamespace(total_tokens=3)) + + async def fake_acompletion(**_kwargs): + return fake_stream() + + def fail_chunk_builder(*_args, **_kwargs): + raise AssertionError("stream_chunk_builder should not run when deltas include thinking") + + events = [] + async def send_event(event): + events.append(event) + + session = SimpleNamespace( + config=SimpleNamespace(model_name="anthropic/claude-opus-4-7"), + is_cancelled=False, + send_event=send_event, + ) + monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) + monkeypatch.setattr(agent_loop, "stream_chunk_builder", fail_chunk_builder) + + result = await _call_llm_streaming( + session, + messages=[Message(role="user", content="hi")], + tools=[], + llm_params={"model": "anthropic/claude-opus-4-7"}, + ) + + assert result.content == "done" + assert result.thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] + + @pytest.mark.asyncio async def test_streaming_call_skips_chunk_rebuild_for_non_anthropic(monkeypatch): async def fake_stream(): From 98e4465bae49dbef27e6f5238d07639645b46921 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 14:57:11 +0300 Subject: [PATCH 018/120] Protect sandbox control-plane routes (#142) * Protect sandbox control-plane routes The sandbox server exposed command and file APIs on public Spaces without validating the client. This adds bearer-token validation to every non-health endpoint and provisions a per-sandbox API secret so control-plane access is not coupled to URL knowledge. Constraint: Existing sandboxes may only have HF_TOKEN configured, so the server accepts SANDBOX_API_TOKEN first and falls back to HF_TOKEN for compatibility. Rejected: Make every sandbox private by default | that changes product behavior and still leaves the embedded API without a server-side auth boundary. Confidence: high Scope-risk: narrow Directive: Do not add new sandbox API routes without the auth dependency unless the route is intentionally public health/status only. Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_sandbox_api_auth.py * Preserve sandbox reconnect with API token Review caught that Sandbox.connect would send only the HF token while newly-created sandboxes now expect SANDBOX_API_TOKEN. Add an api_token parameter, hide the field from repr, and cover the documented HF_TOKEN fallback for legacy sandboxes. Constraint: Existing connect callers must keep working when they only pass an HF token to legacy sandboxes. Confidence: high Scope-risk: narrow Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_sandbox_api_auth.py * Add live sandbox communication test Add an opt-in integration test that provisions a real public cpu-basic sandbox, verifies unauthenticated API calls are rejected, exercises authenticated bash/write/exists/read communication, verifies reconnect with the sandbox API token, and deletes the Space afterward. Constraint: Live sandbox creation requires HF_TOKEN and should not run in default CI because it creates a real Space. Rejected: Use a private Space for this test | private hf.space requests return 404 before the sandbox FastAPI auth layer is reachable, so it does not validate the public-sandbox threat model fixed here. Confidence: high Scope-risk: narrow Tested: ML_INTERN_LIVE_SANDBOX_TESTS=1 ML_INTERN_LIVE_ENV_FILE=/Users/akseljoonas/Documents/ml-intern/.env UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/integration/test_live_sandbox_auth.py -q -s Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_sandbox_api_auth.py tests/integration/test_live_sandbox_auth.py -q --- agent/tools/sandbox_client.py | 65 +++++++++++---- tests/integration/test_live_sandbox_auth.py | 90 +++++++++++++++++++++ tests/unit/test_sandbox_api_auth.py | 87 ++++++++++++++++++++ 3 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_live_sandbox_auth.py create mode 100644 tests/unit/test_sandbox_api_auth.py diff --git a/agent/tools/sandbox_client.py b/agent/tools/sandbox_client.py index 16982c76f..cbf947e3e 100644 --- a/agent/tools/sandbox_client.py +++ b/agent/tools/sandbox_client.py @@ -37,6 +37,7 @@ from __future__ import annotations import io +import secrets as secrets_lib import sys import time import uuid @@ -99,8 +100,8 @@ _SANDBOX_SERVER = '''\ """Minimal FastAPI server for sandbox operations.""" -import os, subprocess, pathlib, signal, threading, re, tempfile -from fastapi import FastAPI +import hmac, os, subprocess, pathlib, signal, threading, re, tempfile +from fastapi import Depends, FastAPI, HTTPException, Request from pydantic import BaseModel from typing import Optional import uvicorn @@ -156,6 +157,22 @@ def _atomic_write(path: pathlib.Path, content: str): app = FastAPI() +def _expected_api_token() -> str: + return os.environ.get("SANDBOX_API_TOKEN") or os.environ.get("HF_TOKEN") or "" + +def _require_auth(request: Request) -> None: + expected = _expected_api_token() + if not expected: + raise HTTPException(status_code=503, detail="Sandbox API token not configured") + auth_header = request.headers.get("authorization", "") + scheme, _, supplied = auth_header.partition(" ") + if scheme.lower() != "bearer" or not supplied: + raise HTTPException(status_code=401, detail="Missing bearer token") + if not hmac.compare_digest(supplied, expected): + raise HTTPException(status_code=401, detail="Invalid bearer token") + +_AUTH = [Depends(_require_auth)] + # Track active bash processes so they can be killed on cancel _active_procs = {} # pid -> subprocess.Popen _proc_lock = threading.Lock() @@ -344,7 +361,7 @@ def _validate_python(content, path=""): def health(): return {"status": "ok"} -@app.post("/api/bash") +@app.post("/api/bash", dependencies=_AUTH) def bash(req: BashReq): try: proc = subprocess.Popen( @@ -371,7 +388,7 @@ def bash(req: BashReq): except Exception as e: return {"success": False, "output": "", "error": str(e)} -@app.post("/api/kill") +@app.post("/api/kill", dependencies=_AUTH) def kill_all(): """Kill all active bash processes. Called when user cancels.""" with _proc_lock: @@ -389,7 +406,7 @@ def kill_all(): pass return {"success": True, "output": f"Killed {len(killed)} process(es): {killed}", "error": ""} -@app.post("/api/read") +@app.post("/api/read", dependencies=_AUTH) def read(req: ReadReq): try: p = pathlib.Path(req.path) @@ -406,7 +423,7 @@ def read(req: ReadReq): except Exception as e: return {"success": False, "output": "", "error": str(e)} -@app.post("/api/write") +@app.post("/api/write", dependencies=_AUTH) def write(req: WriteReq): try: p = pathlib.Path(req.path) @@ -420,7 +437,7 @@ def write(req: WriteReq): except Exception as e: return {"success": False, "output": "", "error": str(e)} -@app.post("/api/edit") +@app.post("/api/edit", dependencies=_AUTH) def edit(req: EditReq): try: p = pathlib.Path(req.path) @@ -447,7 +464,7 @@ def edit(req: EditReq): except Exception as e: return {"success": False, "output": "", "error": str(e)} -@app.post("/api/exists") +@app.post("/api/exists", dependencies=_AUTH) def exists(req: ExistsReq): return {"success": True, "output": str(pathlib.Path(req.path).exists()).lower(), "error": ""} @@ -482,6 +499,7 @@ class Sandbox: space_id: str token: str | None = None + api_token: str | None = field(default=None, repr=False) work_dir: str = "/app" timeout: int = DEFAULT_TIMEOUT _owns_space: bool = field(default=False, repr=False) @@ -495,9 +513,10 @@ def __post_init__(self): # Trailing slash is critical: httpx resolves relative paths against base_url. # Without it, client.get("health") resolves to /health instead of /api/health. self._base_url = f"https://{slug}.hf.space/api/" + api_token = self.api_token or self.token self._client = httpx.Client( base_url=self._base_url, - headers={"Authorization": f"Bearer {self.token}"} if self.token else {}, + headers={"Authorization": f"Bearer {api_token}"} if api_token else {}, timeout=httpx.Timeout(MAX_TIMEOUT, connect=30), follow_redirects=True, ) @@ -563,6 +582,7 @@ def _check_cancel(): base = name or "sandbox" suffix = uuid.uuid4().hex[:8] space_id = f"{owner}/{base}-{suffix}" + sandbox_api_token = secrets_lib.token_urlsafe(32) _log(f"Creating sandbox: {space_id} (from {template})...") @@ -583,8 +603,9 @@ def _check_cancel(): # Inject secrets BEFORE uploading server files (which triggers rebuild). # Secrets added after a Space is running aren't available until restart, # so they must be set before the build/start cycle. - if secrets: - for key, val in secrets.items(): + sandbox_secrets = {**(secrets or {}), "SANDBOX_API_TOKEN": sandbox_api_token} + if sandbox_secrets: + for key, val in sandbox_secrets.items(): api.add_space_secret(space_id, key, val) # Upload sandbox server and Dockerfile (triggers rebuild) @@ -617,7 +638,12 @@ def _check_cancel(): _check_cancel() # Wait for the API server to be responsive (non-fatal) - sb = cls(space_id=space_id, token=token, _owns_space=True) + sb = cls( + space_id=space_id, + token=token, + api_token=sandbox_api_token, + _owns_space=True, + ) try: sb._wait_for_api(timeout=API_WAIT_TIMEOUT, log=_log) except TimeoutError as e: @@ -648,13 +674,24 @@ def _setup_server(space_id: str, api: HfApi, *, log: Callable[[str], object] = p log("Server files uploaded, rebuild triggered.") @classmethod - def connect(cls, space_id: str, *, token: str | None = None) -> Sandbox: + def connect( + cls, + space_id: str, + *, + token: str | None = None, + api_token: str | None = None, + ) -> Sandbox: """ Connect to an existing running Space. Does a health check to verify the Space is reachable. """ - sb = cls(space_id=space_id, token=token, _owns_space=False) + sb = cls( + space_id=space_id, + token=token, + api_token=api_token, + _owns_space=False, + ) sb._wait_for_api(timeout=60) return sb diff --git a/tests/integration/test_live_sandbox_auth.py b/tests/integration/test_live_sandbox_auth.py new file mode 100644 index 000000000..b68f99904 --- /dev/null +++ b/tests/integration/test_live_sandbox_auth.py @@ -0,0 +1,90 @@ +"""Opt-in live sandbox communication test. + +This test creates a real Hugging Face Space sandbox, verifies that unauthenticated +requests are rejected, then exercises the authenticated agent client end-to-end. +It is skipped unless ``ML_INTERN_LIVE_SANDBOX_TESTS=1`` and ``HF_TOKEN`` are set. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import httpx +import pytest +from dotenv import load_dotenv +from huggingface_hub import HfApi + +from agent.tools.sandbox_client import Sandbox + + +if env_file := os.environ.get("ML_INTERN_LIVE_ENV_FILE"): + load_dotenv(Path(env_file)) + + +def _skip_without_live_sandbox() -> None: + if os.environ.get("ML_INTERN_LIVE_SANDBOX_TESTS") != "1": + pytest.skip("set ML_INTERN_LIVE_SANDBOX_TESTS=1 to create a real sandbox") + if not os.environ.get("HF_TOKEN"): + pytest.skip("set HF_TOKEN to create a real sandbox") + + +def test_live_sandbox_authenticated_agent_communication(): + _skip_without_live_sandbox() + + token = os.environ["HF_TOKEN"] + owner = HfApi(token=token).whoami()["name"] + sandbox = None + + try: + sandbox = Sandbox.create( + owner=owner, + name="ml-intern-live-auth", + hardware="cpu-basic", + private=False, + token=token, + secrets={"HF_TOKEN": token}, + wait_timeout=900, + ) + + unauthenticated = httpx.Client( + base_url=sandbox._base_url, + timeout=30, + follow_redirects=True, + ) + try: + denied = unauthenticated.post("exists", json={"path": "/tmp"}) + assert denied.status_code == 401 + finally: + unauthenticated.close() + + bash = sandbox.bash("printf sandbox-live-ok", timeout=30) + assert bash.success, bash.error + assert "sandbox-live-ok" in bash.output + + write = sandbox.write("/tmp/ml_intern_live_auth.txt", "alpha\nbeta\n") + assert write.success, write.error + + exists = sandbox._call("exists", {"path": "/tmp/ml_intern_live_auth.txt"}) + assert exists.success, exists.error + assert exists.output == "true" + + read = sandbox.read("/tmp/ml_intern_live_auth.txt") + assert read.success, read.error + assert "alpha" in read.output + assert "beta" in read.output + + reattached = Sandbox.connect( + sandbox.space_id, + token=token, + api_token=sandbox.api_token, + ) + try: + reread = reattached.read("/tmp/ml_intern_live_auth.txt") + assert reread.success, reread.error + assert "alpha" in reread.output + finally: + reattached._client.close() + finally: + if sandbox is not None: + sandbox.delete() diff --git a/tests/unit/test_sandbox_api_auth.py b/tests/unit/test_sandbox_api_auth.py new file mode 100644 index 000000000..e60dfa5b3 --- /dev/null +++ b/tests/unit/test_sandbox_api_auth.py @@ -0,0 +1,87 @@ +from fastapi.testclient import TestClient + +from agent.tools.sandbox_client import _SANDBOX_SERVER, Sandbox + + +def _sandbox_app( + monkeypatch, + token: str | None = "sandbox-secret", + *, + hf_token: str | None = None, +): + monkeypatch.delenv("SANDBOX_API_TOKEN", raising=False) + monkeypatch.delenv("HF_TOKEN", raising=False) + if token is not None: + monkeypatch.setenv("SANDBOX_API_TOKEN", token) + if hf_token is not None: + monkeypatch.setenv("HF_TOKEN", hf_token) + namespace = {} + exec(_SANDBOX_SERVER, namespace) + return namespace["app"] + + +def test_health_is_public(monkeypatch): + client = TestClient(_sandbox_app(monkeypatch)) + + response = client.get("/api/health") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_file_and_command_routes_require_bearer_token(monkeypatch): + client = TestClient(_sandbox_app(monkeypatch, "sandbox-secret")) + + response = client.post("/api/exists", json={"path": "/tmp"}) + + assert response.status_code == 401 + + +def test_file_and_command_routes_accept_valid_bearer_token(monkeypatch): + client = TestClient(_sandbox_app(monkeypatch, "sandbox-secret")) + + response = client.post( + "/api/exists", + json={"path": "/tmp"}, + headers={"Authorization": "Bearer sandbox-secret"}, + ) + + assert response.status_code == 200 + assert response.json()["success"] is True + + +def test_legacy_hf_token_fallback_is_accepted(monkeypatch): + client = TestClient(_sandbox_app(monkeypatch, token=None, hf_token="hf-secret")) + + response = client.post( + "/api/exists", + json={"path": "/tmp"}, + headers={"Authorization": "Bearer hf-secret"}, + ) + + assert response.status_code == 200 + assert response.json()["success"] is True + + +def test_protected_routes_fail_closed_without_configured_token(monkeypatch): + client = TestClient(_sandbox_app(monkeypatch, None)) + + response = client.post( + "/api/exists", + json={"path": "/tmp"}, + headers={"Authorization": "Bearer anything"}, + ) + + assert response.status_code == 503 + + +def test_sandbox_prefers_control_plane_token_for_api_headers(): + sandbox = Sandbox("owner/name", token="hf-token", api_token="sandbox-secret") + + assert sandbox._client.headers["authorization"] == "Bearer sandbox-secret" + + +def test_sandbox_api_token_is_hidden_from_repr(): + sandbox = Sandbox("owner/name", token="hf-token", api_token="sandbox-secret") + + assert "sandbox-secret" not in repr(sandbox) From f8935321daf410e628bae2ab7401061e3228ecec Mon Sep 17 00:00:00 2001 From: triscacezar-droid Date: Mon, 27 Apr 2026 13:00:04 +0100 Subject: [PATCH 019/120] fix(packaging): ship configs/ directory so non-editable installs work (#133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(packaging): ship configs/ directory so non-editable installs work CLI_CONFIG_PATH in agent.main resolves to /configs/cli_agent_config.json, but [tool.setuptools.packages.find] only included "agent*" so neither configs/ nor its JSON files were copied during a non-editable install (uv tool install, pip install without -e). Editable installs worked by accident because they reference the source tree directly. Result: ml-intern crashed at startup with FileNotFoundError on every fresh non-editable install, until the configs were manually copied into site-packages. Make configs/ a real package (empty __init__.py) and ship its JSON via [tool.setuptools.package-data]. Verified with uv tool install --reinstall. * fix(packaging): also ship agent/prompts/*.yaml + agent/README.md Same root cause as the configs/ shipping fix in the prior commit, with a worse symptom: ContextManager._load_system_prompt opens agent/prompts/system_prompt_v3.yaml at runtime, but setuptools doesn't include non-Python files inside packages without explicit package-data. Symptom in headless mode: Session() → ContextManager() raises FileNotFoundError inside submission_loop; agent_task is set to "done with exception" but headless_main awaits event_queue.get() forever waiting for "ready", so the process hangs indefinitely with no log output. The exception is recoverable only via an asyncio task dump. Editable installs ship the source tree as-is, which is why this didn't surface until non-editable reinstall. --------- Co-authored-by: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> --- configs/__init__.py | 0 pyproject.toml | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 configs/__init__.py diff --git a/configs/__init__.py b/configs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyproject.toml b/pyproject.toml index 89cadf94b..432085e09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,20 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["agent*"] +# `configs` ships the JSON files loaded by agent.main.CLI_CONFIG_PATH at +# runtime (resolves to /configs/cli_agent_config.json). +# Without it, `uv tool install` / `pip install` produce a broken install +# that imports fine but crashes at startup with FileNotFoundError. +include = ["agent*", "configs"] + +[tool.setuptools.package-data] +configs = ["*.json"] +# Agent data files: system prompts loaded by ContextManager._load_system_prompt +# at runtime (`/agent/prompts/system_prompt_v3.yaml`), plus the +# package README. Without these, headless_main hangs forever — submission_loop +# crashes with FileNotFoundError but headless_main doesn't check agent_task.done() +# and just keeps awaiting the "ready" event_queue item that will never come. +agent = ["README.md", "prompts/*.yaml"] [tool.uv] package = true From 7a76ad1940f7e142b8866fb3a5bd32e6351265f9 Mon Sep 17 00:00:00 2001 From: Abubakar Abid Date: Mon, 27 Apr 2026 08:33:11 -0400 Subject: [PATCH 020/120] Use `huggingface_hub.get_token()` for auth fallback (#126) * Use cached HF login token for router auth fallback Let HF router requests resolve credentials from huggingface_hub's local login cache when INFERENCE_TOKEN, session token, and HF_TOKEN are absent. This keeps CLI and plugin flows working after `hf auth login` without requiring explicit env exports. Made-with: Cursor * Consolidate Hugging Face token resolution Centralize HF token cleanup and lookup so router calls, CLI startup, and backend request paths do not drift after adding huggingface_hub.get_token() fallback. The backend request helper intentionally keeps browser/user token precedence separate from local cached CLI auth. Constraint: huggingface_hub.get_token() already handles HF_TOKEN, HUGGING_FACE_HUB_TOKEN, and the local hf auth login cache. Rejected: Use cached login tokens for backend request auth | would let server-local credentials stand in for the browser user. Confidence: high Scope-risk: narrow Directive: Keep router-token fallback and request-user token extraction separate; only router/CLI paths should use cached login fallback. Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_llm_params.py -q Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_hf_access.py tests/unit/test_user_quotas.py -q Tested: python -m py_compile agent/core/hf_tokens.py agent/core/llm_params.py agent/main.py backend/routes/agent.py backend/dependencies.py Tested: unset HF_TOKEN/INFERENCE_TOKEN cached-token live HfApi.whoami smoke * Preserve session user id while deduping HF token lookup The token helper cleanup should not change session identity plumbing. Restore the existing username helper and pass user_id into both interactive and headless submission_loop calls, while removing only the redundant _get_hf_token wrapper. Constraint: PR #146 relies on user_id being passed into Session for saved-session ownership. Rejected: Inline HfApi.whoami at each call site | reintroduces duplicate username lookup code and obscures the token-only cleanup. Confidence: high Scope-risk: narrow Directive: Do not remove user_id from submission_loop call sites when editing CLI auth/token code. Tested: python -m py_compile agent/main.py agent/core/hf_tokens.py agent/core/llm_params.py backend/routes/agent.py backend/dependencies.py Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_llm_params.py -q Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_cli_rendering.py tests/unit/test_hf_access.py tests/unit/test_user_quotas.py -q --------- Co-authored-by: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Co-authored-by: akseljoonas --- agent/core/hf_tokens.py | 85 +++++++++++++++++++++++++++++++++++ agent/core/llm_params.py | 19 ++++---- agent/main.py | 29 ++---------- backend/dependencies.py | 14 +++--- backend/routes/agent.py | 34 +++----------- tests/unit/test_llm_params.py | 84 +++++++++++++++++++++++++++++++++- 6 files changed, 195 insertions(+), 70 deletions(-) create mode 100644 agent/core/hf_tokens.py diff --git a/agent/core/hf_tokens.py b/agent/core/hf_tokens.py new file mode 100644 index 000000000..3e72ccc12 --- /dev/null +++ b/agent/core/hf_tokens.py @@ -0,0 +1,85 @@ +"""Hugging Face token resolution helpers.""" + +from __future__ import annotations + +import os +from typing import Any + + +def clean_hf_token(token: str | None) -> str | None: + """Normalize token strings the same way huggingface_hub does.""" + if token is None: + return None + return token.replace("\r", "").replace("\n", "").strip() or None + + +def get_cached_hf_token() -> str | None: + """Return the token from huggingface_hub's normal env/cache lookup.""" + try: + from huggingface_hub import get_token + + return get_token() + except Exception: + return None + + +def resolve_hf_token( + *candidates: str | None, + include_cached: bool = True, +) -> str | None: + """Return the first non-empty explicit token, then optionally HF cache.""" + for token in candidates: + cleaned = clean_hf_token(token) + if cleaned: + return cleaned + if include_cached: + return get_cached_hf_token() + return None + + +def resolve_hf_router_token(session_hf_token: str | None = None) -> str | None: + """Resolve the token used for Hugging Face Router LLM calls. + + App-specific precedence: + 1. INFERENCE_TOKEN: shared hosted-Space inference token. + 2. session_hf_token: the active user/session token. + 3. huggingface_hub.get_token(): HF_TOKEN/HUGGING_FACE_HUB_TOKEN or + local ``hf auth login`` cache. + """ + return resolve_hf_token(os.environ.get("INFERENCE_TOKEN"), session_hf_token) + + +def get_hf_bill_to() -> str | None: + """Return X-HF-Bill-To only when a shared inference token is active.""" + if clean_hf_token(os.environ.get("INFERENCE_TOKEN")): + return os.environ.get("HF_BILL_TO", "smolagents") + return None + + +def bearer_token_from_header(auth_header: str | None) -> str | None: + """Extract a cleaned bearer token from an Authorization header.""" + if not auth_header or not auth_header.startswith("Bearer "): + return None + return clean_hf_token(auth_header[7:]) + + +def resolve_hf_request_token( + request: Any, + *, + include_env_fallback: bool = True, +) -> str | None: + """Resolve a user token from a FastAPI request. + + This intentionally does not use the local ``hf auth login`` cache. Backend + request paths should act as the browser user from Authorization/cookie, or + fall back only to an explicit server ``HF_TOKEN`` in dev/server contexts. + """ + token = bearer_token_from_header(request.headers.get("Authorization", "")) + if token: + return token + token = clean_hf_token(request.cookies.get("hf_access_token")) + if token: + return token + if include_env_fallback: + return clean_hf_token(os.environ.get("HF_TOKEN")) + return None diff --git a/agent/core/llm_params.py b/agent/core/llm_params.py index bac507354..880886b3e 100644 --- a/agent/core/llm_params.py +++ b/agent/core/llm_params.py @@ -5,7 +5,12 @@ creating circular imports. """ -import os +from agent.core.hf_tokens import get_hf_bill_to, resolve_hf_router_token + + +def _resolve_hf_router_token(session_hf_token: str | None = None) -> str | None: + """Backward-compatible private wrapper used by tests and older imports.""" + return resolve_hf_router_token(session_hf_token) def _patch_litellm_effort_validation() -> None: @@ -129,7 +134,8 @@ def _resolve_llm_params( 1. INFERENCE_TOKEN env — shared key on the hosted Space (inference is free for users, billed to the Space owner via ``X-HF-Bill-To``). 2. session.hf_token — the user's own token (CLI / OAuth / cache file). - 3. HF_TOKEN env — belt-and-suspenders fallback for CLI users. + 3. huggingface_hub cache — ``HF_TOKEN`` / ``HUGGING_FACE_HUB_TOKEN`` / + local ``hf auth login`` cache. """ if model_name.startswith("anthropic/"): params: dict = {"model": model_name} @@ -175,18 +181,13 @@ def _resolve_llm_params( return params hf_model = model_name.removeprefix("huggingface/") - api_key = ( - os.environ.get("INFERENCE_TOKEN") - or session_hf_token - or os.environ.get("HF_TOKEN") - ) + api_key = _resolve_hf_router_token(session_hf_token) params = { "model": f"openai/{hf_model}", "api_base": "https://router.huggingface.co/v1", "api_key": api_key, } - if os.environ.get("INFERENCE_TOKEN"): - bill_to = os.environ.get("HF_BILL_TO", "smolagents") + if bill_to := get_hf_bill_to(): params["extra_headers"] = {"X-HF-Bill-To": bill_to} if reasoning_effort: hf_level = "low" if reasoning_effort == "minimal" else reasoning_effort diff --git a/agent/main.py b/agent/main.py index 933e26ce6..56e4fc4e6 100644 --- a/agent/main.py +++ b/agent/main.py @@ -23,6 +23,7 @@ from agent.config import load_config from agent.core.agent_loop import submission_loop from agent.core import model_switcher +from agent.core.hf_tokens import resolve_hf_token from agent.core.session import OpType from agent.core.tools import ToolRouter from agent.utils.reliability_checks import check_training_script_save_pattern @@ -69,28 +70,6 @@ def _safe_get_args(arguments: dict) -> dict: return args if isinstance(args, dict) else {} -def _get_hf_token() -> str | None: - """Get HF token from environment, huggingface_hub API, or cached token file.""" - token = os.environ.get("HF_TOKEN") - if token: - return token - try: - from huggingface_hub import HfApi - api = HfApi() - token = api.token - if token: - return token - except Exception: - pass - # Fallback: read the cached token file directly - token_path = Path.home() / ".cache" / "huggingface" / "token" - if token_path.exists(): - token = token_path.read_text().strip() - if token: - return token - return None - - def _get_hf_user(token: str | None) -> str | None: """Resolve the HF username for a token, if available.""" if not token: @@ -769,7 +748,7 @@ async def _handle_slash_command( normalized = arg.removeprefix("huggingface/") session = session_holder[0] if session_holder else None await model_switcher.probe_and_switch_model( - normalized, config, session, console, _get_hf_token(), + normalized, config, session, console, resolve_hf_token(), ) return None @@ -838,7 +817,7 @@ async def main(): prompt_session = PromptSession() # HF token — required, prompt if missing - hf_token = _get_hf_token() + hf_token = resolve_hf_token() if not hf_token: hf_token = await _prompt_and_save_hf_token(prompt_session) @@ -1054,7 +1033,7 @@ async def headless_main( logging.basicConfig(level=logging.WARNING) _configure_runtime_logging() - hf_token = _get_hf_token() + hf_token = resolve_hf_token() if not hf_token: print("ERROR: No HF token found. Set HF_TOKEN or run `huggingface-cli login`.", file=sys.stderr) sys.exit(1) diff --git a/backend/dependencies.py b/backend/dependencies.py index 0f97c448d..5ebc5385e 100644 --- a/backend/dependencies.py +++ b/backend/dependencies.py @@ -12,6 +12,8 @@ import httpx from fastapi import HTTPException, Request, status +from agent.core.hf_tokens import bearer_token_from_header + from agent.core.hf_access import fetch_whoami_v2, jobs_access_from_whoami logger = logging.getLogger(__name__) @@ -157,9 +159,8 @@ async def get_current_user(request: Request) -> dict[str, Any]: return DEV_USER # Try Authorization header - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - token = auth_header[7:] + token = bearer_token_from_header(request.headers.get("Authorization", "")) + if token: user = await _extract_user_from_token(token) if user: return user @@ -183,9 +184,9 @@ def _extract_token(request: Request) -> str | None: Mirrors the lookup order used by ``get_current_user``. """ - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - return auth_header[7:] + token = bearer_token_from_header(request.headers.get("Authorization", "")) + if token: + return token return request.cookies.get("hf_access_token") @@ -202,4 +203,3 @@ async def require_huggingface_org_member(request: Request) -> bool: if not token: return False return await check_org_membership(token, HF_EMPLOYEE_ORG) - diff --git a/backend/routes/agent.py b/backend/routes/agent.py index 4895bbadb..e990bb941 100644 --- a/backend/routes/agent.py +++ b/backend/routes/agent.py @@ -33,6 +33,7 @@ import user_quotas from agent.core.hf_access import get_jobs_access +from agent.core.hf_tokens import resolve_hf_request_token, resolve_hf_router_token from agent.core.llm_params import _resolve_llm_params logger = logging.getLogger(__name__) @@ -332,10 +333,8 @@ async def generate_title( reasoning model — reasoning_effort=low keeps the reasoning budget small so the 60-token output budget isn't consumed before the title is written. """ - api_key = ( - os.environ.get("INFERENCE_TOKEN") - or (user.get("hf_token") if isinstance(user, dict) else None) - or os.environ.get("HF_TOKEN") + api_key = resolve_hf_router_token( + user.get("hf_token") if isinstance(user, dict) else None ) try: response = await acompletion( @@ -391,14 +390,7 @@ async def create_session( Returns 503 if the server or user has reached the session limit. """ # Extract the user's HF token (Bearer header, HttpOnly cookie, or env var) - hf_token = None - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - hf_token = auth_header[7:] - if not hf_token: - hf_token = request.cookies.get("hf_access_token") - if not hf_token: - hf_token = os.environ.get("HF_TOKEN") + hf_token = resolve_hf_request_token(request) # Optional model override. Empty body falls back to the config default. model: str | None = None @@ -444,14 +436,7 @@ async def restore_session_summary( if not isinstance(messages, list) or not messages: raise HTTPException(status_code=400, detail="Missing 'messages' array") - hf_token = None - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - hf_token = auth_header[7:] - if not hf_token: - hf_token = request.cookies.get("hf_access_token") - if not hf_token: - hf_token = os.environ.get("HF_TOKEN") + hf_token = resolve_hf_request_token(request) model = body.get("model") valid_ids = {m["id"] for m in AVAILABLE_MODELS} @@ -545,14 +530,7 @@ async def get_user_quota(user: dict = Depends(get_current_user)) -> dict: @router.get("/user/jobs-access") async def get_jobs_access_info(request: Request, user: dict = Depends(get_current_user)) -> dict: """Return whether the current token can run HF Jobs and under which namespaces.""" - token = None - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - token = auth_header[7:] - if not token: - token = request.cookies.get("hf_access_token") - if not token: - token = os.environ.get("HF_TOKEN") + token = resolve_hf_request_token(request) access = await get_jobs_access(token or "") return { diff --git a/tests/unit/test_llm_params.py b/tests/unit/test_llm_params.py index ee6cf62c6..5234461ad 100644 --- a/tests/unit/test_llm_params.py +++ b/tests/unit/test_llm_params.py @@ -1,4 +1,9 @@ -from agent.core.llm_params import UnsupportedEffortError, _resolve_llm_params +from agent.core.hf_tokens import resolve_hf_request_token +from agent.core.llm_params import ( + UnsupportedEffortError, + _resolve_hf_router_token, + _resolve_llm_params, +) def test_openai_xhigh_effort_is_forwarded(): @@ -23,3 +28,80 @@ def test_openai_max_effort_is_still_rejected(): assert "OpenAI doesn't accept effort='max'" in str(exc) else: raise AssertionError("Expected UnsupportedEffortError for max effort") + + +def test_hf_router_token_prefers_inference_token(monkeypatch): + monkeypatch.setenv("INFERENCE_TOKEN", " inference-token ") + monkeypatch.setenv("HF_TOKEN", "hf-token") + + assert _resolve_hf_router_token("session-token") == "inference-token" + + +def test_hf_router_token_prefers_session_over_hf_cache(monkeypatch): + monkeypatch.delenv("INFERENCE_TOKEN", raising=False) + monkeypatch.setenv("HF_TOKEN", "hf-token") + + assert _resolve_hf_router_token(" session-token ") == "session-token" + + +def test_hf_router_token_uses_hf_token_env_via_huggingface_hub(monkeypatch): + monkeypatch.delenv("INFERENCE_TOKEN", raising=False) + monkeypatch.setenv("HF_TOKEN", " hf-token ") + + assert _resolve_hf_router_token(None) == "hf-token" + + +def test_hf_router_token_uses_huggingface_hub_cache(monkeypatch): + import huggingface_hub + + monkeypatch.delenv("INFERENCE_TOKEN", raising=False) + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.setattr(huggingface_hub, "get_token", lambda: "cached-token") + + assert _resolve_hf_router_token(None) == "cached-token" + + +def test_hf_router_token_swallows_huggingface_hub_errors(monkeypatch): + import huggingface_hub + + def fail(): + raise RuntimeError("cache unavailable") + + monkeypatch.delenv("INFERENCE_TOKEN", raising=False) + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.setattr(huggingface_hub, "get_token", fail) + + assert _resolve_hf_router_token(None) is None + + +def test_hf_router_params_set_bill_to_only_for_inference_token(monkeypatch): + monkeypatch.setenv("INFERENCE_TOKEN", "inference-token") + monkeypatch.setenv("HF_BILL_TO", "test-org") + + params = _resolve_llm_params("moonshotai/Kimi-K2.6") + + assert params["api_key"] == "inference-token" + assert params["extra_headers"] == {"X-HF-Bill-To": "test-org"} + + +def test_hf_request_token_keeps_browser_user_precedence(monkeypatch): + class Request: + headers = {"Authorization": "Bearer browser-token"} + cookies = {"hf_access_token": "cookie-token"} + + monkeypatch.setenv("HF_TOKEN", "server-token") + + assert resolve_hf_request_token(Request()) == "browser-token" + + +def test_hf_request_token_does_not_use_cached_login(monkeypatch): + import huggingface_hub + + class Request: + headers = {} + cookies = {} + + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.setattr(huggingface_hub, "get_token", lambda: "cached-token") + + assert resolve_hf_request_token(Request()) is None From ae86333ad3cbc0ad191b2c1cd78c981f7309974c Mon Sep 17 00:00:00 2001 From: Seunghyeon Kim Date: Mon, 27 Apr 2026 22:07:44 +0900 Subject: [PATCH 021/120] Honor interactive model override after current CLI changes (#111) The interactive CLI accepted --model but did not pass it into main(), so the REPL used the configured default while headless mode honored the flag. Reapply the override on top of the current token/session plumbing by changing only the interactive main signature, config override, and CLI dispatch. Constraint: Current main already passes user_id into submission_loop and uses shared HF token resolution; this fix must preserve both paths. Rejected: Reuse the old PR diff verbatim | it drops newer user_id/session identity changes and conflicts with current main. Confidence: high Scope-risk: narrow Directive: Keep --model behavior identical for interactive and headless paths; banner should render the resolved model. Tested: python -m py_compile agent/main.py tests/unit/test_cli_rendering.py Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_cli_rendering.py -q Tested: UV_CACHE_DIR=/tmp/uv-cache uv run --extra dev pytest tests/unit/test_llm_params.py tests/unit/test_hf_access.py tests/unit/test_user_quotas.py -q Co-authored-by: akseljoonas --- agent/main.py | 6 +++-- tests/unit/test_cli_rendering.py | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/agent/main.py b/agent/main.py index 56e4fc4e6..8459d757c 100644 --- a/agent/main.py +++ b/agent/main.py @@ -807,7 +807,7 @@ async def _handle_slash_command( return None -async def main(): +async def main(model: str | None = None): """Interactive chat with the agent""" # Clear screen @@ -822,6 +822,8 @@ async def main(): hf_token = await _prompt_and_save_hf_token(prompt_session) config = load_config(CLI_CONFIG_PATH) + if model: + config.model_name = model # Resolve username for banner hf_user = _get_hf_user(hf_token) @@ -1240,7 +1242,7 @@ def cli(): max_iter = 10_000 # effectively unlimited asyncio.run(headless_main(args.prompt, model=args.model, max_iterations=max_iter, stream=not args.no_stream)) else: - asyncio.run(main()) + asyncio.run(main(model=args.model)) except KeyboardInterrupt: print("\n\nGoodbye!") diff --git a/tests/unit/test_cli_rendering.py b/tests/unit/test_cli_rendering.py index 7704afd58..5fa0dc2c6 100644 --- a/tests/unit/test_cli_rendering.py +++ b/tests/unit/test_cli_rendering.py @@ -1,8 +1,12 @@ """Regression tests for interactive CLI rendering and research model routing.""" +import sys from io import StringIO from types import SimpleNamespace +import pytest + +import agent.main as main_mod from agent.tools.research_tool import _get_research_model from agent.utils import terminal_display @@ -42,3 +46,45 @@ def _unexpected_future(*args, **kwargs): mgr.clear("agent-1") assert calls == [] + + +def test_cli_forwards_model_flag_to_interactive_main(monkeypatch): + seen: dict[str, str | None] = {} + + async def fake_main(*, model=None): + seen["model"] = model + + monkeypatch.setattr(sys, "argv", ["ml-intern", "--model", "openai/gpt-5.5"]) + monkeypatch.setattr(main_mod, "main", fake_main) + + main_mod.cli() + + assert seen["model"] == "openai/gpt-5.5" + + +@pytest.mark.asyncio +async def test_interactive_main_applies_model_override_before_banner(monkeypatch): + class StopAfterBanner(Exception): + pass + + def fake_banner(*, model=None, hf_user=None): + assert model == "openai/gpt-5.5" + assert hf_user == "tester" + raise StopAfterBanner + + monkeypatch.setattr(main_mod.os, "system", lambda *_args, **_kwargs: 0) + monkeypatch.setattr(main_mod, "PromptSession", lambda: object()) + monkeypatch.setattr(main_mod, "resolve_hf_token", lambda: "hf-token") + monkeypatch.setattr(main_mod, "_get_hf_user", lambda _token: "tester") + monkeypatch.setattr( + main_mod, + "load_config", + lambda _path: SimpleNamespace( + model_name="moonshotai/Kimi-K2.6", + mcpServers={}, + ), + ) + monkeypatch.setattr(main_mod, "print_banner", fake_banner) + + with pytest.raises(StopAfterBanner): + await main_mod.main(model="openai/gpt-5.5") From 2d4ec200812f3b2ed09e6e55bd7e0d28d29b280d Mon Sep 17 00:00:00 2001 From: Guillaume Salou <17745322+jagwar@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:04:25 +0200 Subject: [PATCH 022/120] fix(session-uploader): forward user_id and total_cost_usd to dataset row (#153) PR #136 added user_id + total_cost_usd to Session.get_trajectory(), but the upload path builds session_row with hardcoded fields and drops anything extra. Result: rows in smolagents/ml-intern-sessions still show user_id=null even though the trajectory dict carries the values. Confirmed by kubectl exec on the running pod: session.py is patched correctly, but uploaded sessions still have null fields because they go through this hardcoded shape. Forward both fields explicitly via .get() so older sessions that lack them still upload cleanly with null. --- agent/core/session_uploader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/agent/core/session_uploader.py b/agent/core/session_uploader.py index f22b52010..d18ec6b8a 100644 --- a/agent/core/session_uploader.py +++ b/agent/core/session_uploader.py @@ -90,9 +90,11 @@ def upload_session_as_file( # across sessions with different tool rosters. session_row = { "session_id": data["session_id"], + "user_id": data.get("user_id"), "session_start_time": data["session_start_time"], "session_end_time": data["session_end_time"], "model_name": data["model_name"], + "total_cost_usd": data.get("total_cost_usd"), "messages": json.dumps(scrubbed_messages), "events": json.dumps(scrubbed_events), "tools": json.dumps(scrubbed_tools), From 6155b2652f935cff5f010f04620cc1d7fd8431c7 Mon Sep 17 00:00:00 2001 From: lewtun Date: Mon, 27 Apr 2026 17:25:03 +0200 Subject: [PATCH 023/120] Add Slack gateway (#116) * Add messaging gateway and notify tool Co-authored-by: OpenAI Codex * Handle Bedrock streaming permission denials Co-authored-by: OpenAI Codex * Revert "Handle Bedrock streaming permission denials" Co-authored-by: OpenAI Codex * Add automatic completion notifications Co-authored-by: OpenAI Codex * Auto-attach CLI notification destinations Co-authored-by: OpenAI Codex * Defer CLI completion notifications until render Co-authored-by: OpenAI Codex * Increase completion notification summary cap Co-authored-by: OpenAI Codex * Add Slack user notification defaults Co-authored-by: Codex * Increase Slack turn completion summary limit Co-authored-by: Codex * Remove legacy auto notification event upgrade Co-authored-by: Codex * Require session config instead of hard-coded model fallback Co-authored-by: Codex * Address Slack notification review findings Co-authored-by: Codex * Fix Anthropic thinking signature replay Rebuild signed Anthropic thinking blocks from streaming chunks instead of replaying raw deltas, and recover stale histories by retrying once without thinking metadata when Anthropic rejects a signature. Co-authored-by: OpenAI Codex * Format Slack notifications with mrkdwn Convert common Markdown constructs in Slack notification bodies to Slack mrkdwn before posting, while preserving code spans and fenced code blocks. Co-authored-by: OpenAI Codex --------- Co-authored-by: OpenAI Codex --- README.md | 50 +++ agent/config.py | 111 +++++- agent/core/agent_loop.py | 146 +++++++- agent/core/session.py | 130 ++++++- agent/core/tools.py | 7 + agent/main.py | 26 +- agent/messaging/__init__.py | 15 + agent/messaging/base.py | 27 ++ agent/messaging/gateway.py | 166 +++++++++ agent/messaging/models.py | 123 +++++++ agent/messaging/slack.py | 186 ++++++++++ agent/prompts/system_prompt_v3.yaml | 1 + agent/tools/notify_tool.py | 108 ++++++ backend/main.py | 5 +- backend/models.py | 9 +- backend/routes/agent.py | 22 +- backend/session_manager.py | 47 ++- configs/cli_agent_config.json | 5 + pyproject.toml | 2 +- tests/unit/test_cli_rendering.py | 2 +- tests/unit/test_config.py | 121 +++++++ tests/unit/test_messaging.py | 511 ++++++++++++++++++++++++++++ tests/unit/test_thinking_history.py | 57 +++- uv.lock | 2 +- 24 files changed, 1841 insertions(+), 38 deletions(-) create mode 100644 agent/messaging/__init__.py create mode 100644 agent/messaging/base.py create mode 100644 agent/messaging/gateway.py create mode 100644 agent/messaging/models.py create mode 100644 agent/messaging/slack.py create mode 100644 agent/tools/notify_tool.py create mode 100644 tests/unit/test_config.py create mode 100644 tests/unit/test_messaging.py diff --git a/README.md b/README.md index 8e46063a2..8a6c1ccd5 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,56 @@ ml-intern --max-iterations 100 "your prompt" ml-intern --no-stream "your prompt" ``` +## Supported Gateways + +ML Intern currently supports one-way notification gateways from CLI sessions. +These gateways send out-of-band status updates; they do not accept inbound chat +messages. + +### Slack + +Slack notifications use the Slack Web API to post messages when the agent needs +approval, hits an error, or completes a turn. Create a Slack app with a bot token +that has `chat:write`, invite the bot to the target channel, then set: + +```bash +SLACK_BOT_TOKEN=xoxb-... +SLACK_CHANNEL_ID=C... +``` + +The CLI automatically creates a `slack.default` destination when both variables +are present. Optional environment variables for the env-only default: + +```bash +ML_INTERN_SLACK_NOTIFICATIONS=false +ML_INTERN_SLACK_DESTINATION=slack.ops +ML_INTERN_SLACK_AUTO_EVENTS=approval_required,error,turn_complete +ML_INTERN_SLACK_ALLOW_AGENT_TOOL=true +ML_INTERN_SLACK_ALLOW_AUTO_EVENTS=true +``` + +For a persistent user-level config, put overrides in +`~/.config/ml-intern/cli_agent_config.json` or point `ML_INTERN_CLI_CONFIG` at a +JSON file: + +```json +{ + "messaging": { + "enabled": true, + "auto_event_types": ["approval_required", "error", "turn_complete"], + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "${SLACK_BOT_TOKEN}", + "channel": "${SLACK_CHANNEL_ID}", + "allow_agent_tool": true, + "allow_auto_events": true + } + } + } +} +``` + ## Architecture ### Component Overview diff --git a/agent/config.py b/agent/config.py index 7e696dd78..5a6a8a45f 100644 --- a/agent/config.py +++ b/agent/config.py @@ -6,6 +6,8 @@ from dotenv import load_dotenv +from agent.messaging.models import MessagingConfig + # Project root: two levels up from this file (agent/config.py -> project root) _PROJECT_ROOT = Path(__file__).resolve().parent.parent from fastmcp.mcp_config import ( @@ -47,6 +49,104 @@ class Config(BaseModel): # ``xhigh`` or ``max`` for Anthropic 4.6 / 4.7). ``None`` = thinking off. # Valid values: None | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" reasoning_effort: str | None = "max" + messaging: MessagingConfig = MessagingConfig() + + +USER_CONFIG_ENV_VAR = "ML_INTERN_CLI_CONFIG" +DEFAULT_USER_CONFIG_PATH = Path.home() / ".config" / "ml-intern" / "cli_agent_config.json" +SLACK_DEFAULT_DESTINATION = "slack.default" +SLACK_DEFAULT_AUTO_EVENT_TYPES = ["approval_required", "error", "turn_complete"] + + +def _deep_merge_config(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]: + merged = dict(base) + for key, value in override.items(): + current = merged.get(key) + if isinstance(current, dict) and isinstance(value, dict): + merged[key] = _deep_merge_config(current, value) + else: + merged[key] = value + return merged + + +def _load_json_config(path: Path) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if not isinstance(data, dict): + raise ValueError(f"Config file {path} must contain a JSON object") + return data + + +def _load_user_config() -> dict[str, Any]: + raw_path = os.environ.get(USER_CONFIG_ENV_VAR) + if raw_path: + path = Path(raw_path).expanduser() + if not path.exists(): + raise FileNotFoundError( + f"{USER_CONFIG_ENV_VAR} points to missing config file: {path}" + ) + return _load_json_config(path) + + if DEFAULT_USER_CONFIG_PATH.exists(): + return _load_json_config(DEFAULT_USER_CONFIG_PATH) + return {} + + +def _env_bool(name: str, default: bool) -> bool: + value = os.environ.get(name) + if value is None: + return default + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return default + + +def _env_list(name: str) -> list[str] | None: + value = os.environ.get(name) + if value is None: + return None + return [item.strip() for item in value.split(",") if item.strip()] + + +def apply_slack_user_defaults(raw_config: dict[str, Any]) -> dict[str, Any]: + """Enable a default Slack destination from user env vars, when present.""" + if not _env_bool("ML_INTERN_SLACK_NOTIFICATIONS", True): + return raw_config + + token = os.environ.get("SLACK_BOT_TOKEN") + channel = os.environ.get("SLACK_CHANNEL_ID") or os.environ.get("SLACK_CHANNEL") + if not token or not channel: + return raw_config + + config = dict(raw_config) + messaging = dict(config.get("messaging") or {}) + destinations = dict(messaging.get("destinations") or {}) + destination_name = ( + os.environ.get("ML_INTERN_SLACK_DESTINATION") or SLACK_DEFAULT_DESTINATION + ).strip() + + if destination_name not in destinations: + destinations[destination_name] = { + "provider": "slack", + "token": token, + "channel": channel, + "allow_agent_tool": _env_bool("ML_INTERN_SLACK_ALLOW_AGENT_TOOL", True), + "allow_auto_events": _env_bool("ML_INTERN_SLACK_ALLOW_AUTO_EVENTS", True), + } + + auto_events = _env_list("ML_INTERN_SLACK_AUTO_EVENTS") + if auto_events is not None: + messaging["auto_event_types"] = auto_events + elif "auto_event_types" not in messaging: + messaging["auto_event_types"] = SLACK_DEFAULT_AUTO_EVENT_TYPES + + messaging["enabled"] = True + messaging["destinations"] = destinations + config["messaging"] = messaging + return config def substitute_env_vars(obj: Any) -> Any: @@ -86,7 +186,10 @@ def replacer(match): return obj -def load_config(config_path: str = "config.json") -> Config: +def load_config( + config_path: str = "config.json", + include_user_defaults: bool = False, +) -> Config: """ Load configuration with environment variable substitution. @@ -98,8 +201,10 @@ def load_config(config_path: str = "config.json") -> Config: load_dotenv(_PROJECT_ROOT / ".env") load_dotenv(override=False) - with open(config_path, "r") as f: - raw_config = json.load(f) + raw_config = _load_json_config(Path(config_path)) + if include_user_defaults: + raw_config = _deep_merge_config(raw_config, _load_user_config()) + raw_config = apply_slack_user_defaults(raw_config) config_with_env = substitute_env_vars(raw_config) return Config.model_validate(config_with_env) diff --git a/agent/core/agent_loop.py b/agent/core/agent_loop.py index 767730927..8b7a4572d 100644 --- a/agent/core/agent_loop.py +++ b/agent/core/agent_loop.py @@ -19,6 +19,7 @@ from litellm.exceptions import ContextWindowExceededError from agent.config import Config +from agent.messaging.gateway import NotificationGateway from agent.core import telemetry from agent.core.doom_loop import check_for_doom_loop from agent.core.llm_params import _resolve_llm_params @@ -432,6 +433,103 @@ def _should_replay_thinking_state(model_name: str | None) -> bool: return bool(model_name and model_name.startswith("anthropic/")) +def _is_invalid_thinking_signature_error(exc: Exception) -> bool: + """Return True when Anthropic rejected replayed extended-thinking state.""" + text = str(exc) + return ( + "Invalid `signature` in `thinking` block" in text + or "Invalid signature in thinking block" in text + ) + + +def _strip_thinking_state_from_messages(messages: list[Any]) -> int: + """Remove replayed thinking metadata from assistant history messages.""" + stripped = 0 + + for message in messages: + role = ( + message.get("role") + if isinstance(message, dict) + else getattr(message, "role", None) + ) + if role != "assistant": + continue + + if isinstance(message, dict): + if message.pop("thinking_blocks", None) is not None: + stripped += 1 + if message.pop("reasoning_content", None) is not None: + stripped += 1 + provider_fields = message.get("provider_specific_fields") + content = message.get("content") + else: + if getattr(message, "thinking_blocks", None) is not None: + message.thinking_blocks = None + stripped += 1 + if getattr(message, "reasoning_content", None) is not None: + message.reasoning_content = None + stripped += 1 + provider_fields = getattr(message, "provider_specific_fields", None) + content = getattr(message, "content", None) + + if isinstance(provider_fields, dict): + cleaned_fields = dict(provider_fields) + if cleaned_fields.pop("thinking_blocks", None) is not None: + stripped += 1 + if cleaned_fields.pop("reasoning_content", None) is not None: + stripped += 1 + if cleaned_fields != provider_fields: + if isinstance(message, dict): + message["provider_specific_fields"] = cleaned_fields + else: + message.provider_specific_fields = cleaned_fields + + if isinstance(content, list): + cleaned_content = [ + block + for block in content + if not ( + isinstance(block, dict) + and block.get("type") in {"thinking", "redacted_thinking"} + ) + ] + if len(cleaned_content) != len(content): + stripped += len(content) - len(cleaned_content) + if isinstance(message, dict): + message["content"] = cleaned_content + else: + message.content = cleaned_content + + return stripped + + +async def _maybe_heal_invalid_thinking_signature( + session: Session, + messages: list[Any], + exc: Exception, + *, + already_healed: bool, +) -> bool: + if already_healed or not _is_invalid_thinking_signature_error(exc): + return False + + stripped = _strip_thinking_state_from_messages(messages) + if not stripped: + return False + + await session.send_event(Event( + event_type="tool_log", + data={ + "tool": "system", + "log": ( + "Anthropic rejected stale thinking signatures; retrying " + "without replayed thinking metadata." + ), + }, + )) + return True + + def _assistant_message_from_result( llm_result: LLMResult, *, @@ -457,6 +555,7 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> """Call the LLM with streaming, emitting assistant_chunk events.""" response = None _healed_effort = False # one-shot safety net per call + _healed_thinking_signature = False messages, tools = with_prompt_caching(messages, tools, llm_params.get("model")) t_start = time.monotonic() for _llm_attempt in range(_MAX_LLM_RETRIES): @@ -484,6 +583,14 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> data={"tool": "system", "log": "Reasoning effort not supported for this model — adjusting and retrying."}, )) continue + if await _maybe_heal_invalid_thinking_signature( + session, + messages, + e, + already_healed=_healed_thinking_signature, + ): + _healed_thinking_signature = True + continue _delay = _retry_delay_for(e, _llm_attempt) if _llm_attempt < _MAX_LLM_RETRIES - 1 and _delay is not None: logger.warning( @@ -505,8 +612,6 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> final_usage_chunk = None chunks = [] should_replay_thinking = _should_replay_thinking_state(llm_params.get("model")) - collected_thinking_blocks: list[dict[str, Any]] = [] - collected_reasoning_content: list[str] = [] async for chunk in response: chunks.append(chunk) @@ -525,13 +630,6 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> if choice.finish_reason: finish_reason = choice.finish_reason - if should_replay_thinking: - delta_thinking_blocks, delta_reasoning_content = _extract_thinking_state(delta) - if delta_thinking_blocks: - collected_thinking_blocks.extend(delta_thinking_blocks) - if delta_reasoning_content: - collected_reasoning_content.append(delta_reasoning_content) - if delta.content: full_content += delta.content await session.send_event( @@ -565,9 +663,9 @@ async def _call_llm_streaming(session: Session, messages, tools, llm_params) -> latency_ms=int((time.monotonic() - t_start) * 1000), finish_reason=finish_reason, ) - thinking_blocks = collected_thinking_blocks or None - reasoning_content = "".join(collected_reasoning_content) or None - if chunks and should_replay_thinking and not (thinking_blocks or reasoning_content): + thinking_blocks = None + reasoning_content = None + if chunks and should_replay_thinking: try: rebuilt = stream_chunk_builder(chunks, messages=messages) if rebuilt and getattr(rebuilt, "choices", None): @@ -591,6 +689,7 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) """Call the LLM without streaming, emit assistant_message at the end.""" response = None _healed_effort = False + _healed_thinking_signature = False messages, tools = with_prompt_caching(messages, tools, llm_params.get("model")) t_start = time.monotonic() for _llm_attempt in range(_MAX_LLM_RETRIES): @@ -617,6 +716,14 @@ async def _call_llm_non_streaming(session: Session, messages, tools, llm_params) data={"tool": "system", "log": "Reasoning effort not supported for this model — adjusting and retrying."}, )) continue + if await _maybe_heal_invalid_thinking_signature( + session, + messages, + e, + already_healed=_healed_thinking_signature, + ): + _healed_thinking_signature = True + continue _delay = _retry_delay_for(e, _llm_attempt) if _llm_attempt < _MAX_LLM_RETRIES - 1 and _delay is not None: logger.warning( @@ -1128,7 +1235,12 @@ async def _exec_tool( await session.send_event( Event( event_type="turn_complete", - data={"history_size": len(session.context_manager.items)}, + data={ + "history_size": len(session.context_manager.items), + "final_response": final_response + if isinstance(final_response, str) + else None, + }, ) ) @@ -1437,13 +1549,16 @@ async def process_submission(session: Session, submission) -> bool: async def submission_loop( submission_queue: asyncio.Queue, event_queue: asyncio.Queue, - config: Config | None = None, + config: Config, tool_router: ToolRouter | None = None, session_holder: list | None = None, hf_token: str | None = None, user_id: str | None = None, local_mode: bool = False, stream: bool = True, + notification_gateway: NotificationGateway | None = None, + notification_destinations: list[str] | None = None, + defer_turn_complete_notification: bool = False, ) -> None: """ Main agent loop - processes submissions and dispatches to handlers. @@ -1454,6 +1569,9 @@ async def submission_loop( session = Session( event_queue, config=config, tool_router=tool_router, hf_token=hf_token, user_id=user_id, local_mode=local_mode, stream=stream, + notification_gateway=notification_gateway, + notification_destinations=notification_destinations, + defer_turn_complete_notification=defer_turn_complete_notification, ) if session_holder is not None: session_holder[0] = session diff --git a/agent/core/session.py b/agent/core/session.py index f29e49a44..ba3a185f9 100644 --- a/agent/core/session.py +++ b/agent/core/session.py @@ -12,10 +12,13 @@ from agent.config import Config from agent.context_manager.manager import ContextManager +from agent.messaging.gateway import NotificationGateway +from agent.messaging.models import NotificationRequest logger = logging.getLogger(__name__) _DEFAULT_MAX_TOKENS = 200_000 +_TURN_COMPLETE_NOTIFICATION_CHARS = 39000 def _get_max_tokens_safe(model_name: str) -> int: @@ -73,18 +76,24 @@ class Session: def __init__( self, event_queue: asyncio.Queue, - config: Config | None = None, + config: Config, tool_router=None, context_manager: ContextManager | None = None, hf_token: str | None = None, local_mode: bool = False, stream: bool = True, + notification_gateway: NotificationGateway | None = None, + notification_destinations: list[str] | None = None, + defer_turn_complete_notification: bool = False, + session_id: str | None = None, user_id: str | None = None, ): self.hf_token: Optional[str] = hf_token self.user_id: Optional[str] = user_id self.tool_router = tool_router self.stream = stream + if config is None: + raise ValueError("Session requires a Config") tool_specs = tool_router.get_tool_specs_for_llm() if tool_router else [] self.context_manager = context_manager or ContextManager( model_max_tokens=_get_max_tokens_safe(config.model_name), @@ -95,15 +104,16 @@ def __init__( local_mode=local_mode, ) self.event_queue = event_queue - self.session_id = str(uuid.uuid4()) - self.config = config or Config( - model_name="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - ) + self.session_id = session_id or str(uuid.uuid4()) + self.config = config self.is_running = True self._cancelled = asyncio.Event() self.pending_approval: Optional[dict[str, Any]] = None self.sandbox = None self._running_job_ids: set[str] = set() # HF job IDs currently executing + self.notification_gateway = notification_gateway + self.notification_destinations = list(notification_destinations or []) + self.defer_turn_complete_notification = defer_turn_complete_notification # Session trajectory logging self.logged_events: list[dict] = [] @@ -138,11 +148,121 @@ async def send_event(self, event: Event) -> None: "data": event.data, } ) + await self._enqueue_auto_notification_requests(event) # Mid-turn heartbeat flush (owned by telemetry module). from agent.core.telemetry import HeartbeatSaver + HeartbeatSaver.maybe_fire(self) + def set_notification_destinations(self, destinations: list[str]) -> None: + """Replace the session's opted-in auto-notification destinations.""" + deduped: list[str] = [] + seen: set[str] = set() + for destination in destinations: + if destination not in seen: + deduped.append(destination) + seen.add(destination) + self.notification_destinations = deduped + + async def send_deferred_turn_complete_notification(self, event: Event) -> None: + if event.event_type != "turn_complete": + return + await self._enqueue_auto_notification_requests( + event, + include_deferred_turn_complete=True, + ) + + async def _enqueue_auto_notification_requests( + self, + event: Event, + include_deferred_turn_complete: bool = False, + ) -> None: + if self.notification_gateway is None: + return + if not self.notification_destinations: + return + auto_events = set(self.config.messaging.auto_event_types) + if event.event_type not in auto_events: + return + if ( + self.defer_turn_complete_notification + and event.event_type == "turn_complete" + and not include_deferred_turn_complete + ): + return + + requests = self._build_auto_notification_requests(event) + for request in requests: + await self.notification_gateway.enqueue(request) + + def _build_auto_notification_requests( + self, event: Event + ) -> list[NotificationRequest]: + metadata = { + "session_id": self.session_id, + "model": self.config.model_name, + "event_type": event.event_type, + } + + title: str | None = None + message: str | None = None + severity = "info" + data = event.data or {} + if event.event_type == "approval_required": + tools = data.get("tools", []) + tool_names = [] + for tool in tools if isinstance(tools, list) else []: + if isinstance(tool, dict): + tool_name = str(tool.get("tool") or "").strip() + if tool_name and tool_name not in tool_names: + tool_names.append(tool_name) + count = len(tools) if isinstance(tools, list) else 0 + title = "Agent approval required" + message = ( + f"Session {self.session_id} is waiting for approval " + f"for {count} tool call(s)." + ) + if tool_names: + message += " Tools: " + ", ".join(tool_names) + severity = "warning" + elif event.event_type == "error": + title = "Agent error" + error = str(data.get("error") or "Unknown error") + message = f"Session {self.session_id} hit an error.\n{error[:500]}" + severity = "error" + elif event.event_type == "turn_complete": + title = "Agent task complete" + summary = str(data.get("final_response") or "").strip() + if summary: + summary = summary[:_TURN_COMPLETE_NOTIFICATION_CHARS] + message = ( + f"Session {self.session_id} completed successfully.\n" + f"{summary}" + ) + else: + message = f"Session {self.session_id} completed successfully." + severity = "success" + + if message is None: + return [] + + requests: list[NotificationRequest] = [] + for destination in self.notification_destinations: + if not self.config.messaging.can_auto_send(destination): + continue + requests.append( + NotificationRequest( + destination=destination, + title=title, + message=message, + severity=severity, + metadata=metadata, + event_type=event.event_type, + ) + ) + return requests + def cancel(self) -> None: """Signal cancellation to the running agent loop.""" self._cancelled.set() diff --git a/agent/core/tools.py b/agent/core/tools.py index 9bbf91d79..f54163ccd 100644 --- a/agent/core/tools.py +++ b/agent/core/tools.py @@ -46,6 +46,7 @@ hf_repo_git_handler, ) from agent.tools.jobs_tool import HF_JOBS_TOOL_SPEC, hf_jobs_handler +from agent.tools.notify_tool import NOTIFY_TOOL_SPEC, notify_handler from agent.tools.papers_tool import HF_PAPERS_TOOL_SPEC, hf_papers_handler from agent.tools.plan_tool import PLAN_TOOL_SPEC, plan_tool_handler from agent.tools.research_tool import RESEARCH_TOOL_SPEC, research_handler @@ -324,6 +325,12 @@ def create_builtin_tools(local_mode: bool = False) -> list[ToolSpec]: parameters=PLAN_TOOL_SPEC["parameters"], handler=plan_tool_handler, ), + ToolSpec( + name=NOTIFY_TOOL_SPEC["name"], + description=NOTIFY_TOOL_SPEC["description"], + parameters=NOTIFY_TOOL_SPEC["parameters"], + handler=notify_handler, + ), ToolSpec( name=HF_JOBS_TOOL_SPEC["name"], description=HF_JOBS_TOOL_SPEC["description"], diff --git a/agent/main.py b/agent/main.py index 8459d757c..f500cc5fe 100644 --- a/agent/main.py +++ b/agent/main.py @@ -26,6 +26,7 @@ from agent.core.hf_tokens import resolve_hf_token from agent.core.session import OpType from agent.core.tools import ToolRouter +from agent.messaging.gateway import NotificationGateway from agent.utils.reliability_checks import check_training_script_save_pattern from agent.utils.terminal_display import ( get_console, @@ -332,6 +333,9 @@ def _cancel_event(): stream_buf.discard() print_turn_complete() print_plan() + session = session_holder[0] if session_holder else None + if session is not None: + await session.send_deferred_turn_complete_notification(event) turn_complete_event.set() elif event.event_type == "interrupted": shimmer.stop() @@ -821,7 +825,7 @@ async def main(model: str | None = None): if not hf_token: hf_token = await _prompt_and_save_hf_token(prompt_session) - config = load_config(CLI_CONFIG_PATH) + config = load_config(CLI_CONFIG_PATH, include_user_defaults=True) if model: config.model_name = model @@ -844,6 +848,8 @@ async def main(model: str | None = None): turn_complete_event.set() ready_event = asyncio.Event() + notification_gateway = NotificationGateway(config.messaging) + await notification_gateway.start() # Create tool router with local mode tool_router = ToolRouter(config.mcpServers, hf_token=hf_token, local_mode=True) @@ -861,6 +867,9 @@ async def main(model: str | None = None): user_id=hf_user, local_mode=True, stream=True, + notification_gateway=notification_gateway, + notification_destinations=config.messaging.default_auto_destinations(), + defer_turn_complete_notification=True, ) ) @@ -1016,6 +1025,8 @@ def _install_sigint() -> bool: agent_task.cancel() # Agent didn't shut down cleanly — close MCP explicitly await tool_router.__aexit__(None, None, None) + finally: + await notification_gateway.close() # Now safe to cancel the listener (agent is done emitting events) listener_task.cancel() @@ -1042,8 +1053,10 @@ async def headless_main( print(f"HF token loaded", file=sys.stderr) - config = load_config(CLI_CONFIG_PATH) + config = load_config(CLI_CONFIG_PATH, include_user_defaults=True) config.yolo_mode = True # Auto-approve everything in headless mode + notification_gateway = NotificationGateway(config.messaging) + await notification_gateway.start() hf_user = _get_hf_user(hf_token) if model: @@ -1074,6 +1087,9 @@ async def headless_main( user_id=hf_user, local_mode=True, stream=stream, + notification_gateway=notification_gateway, + notification_destinations=config.messaging.default_auto_destinations(), + defer_turn_complete_notification=True, ) ) @@ -1199,6 +1215,10 @@ async def headless_main( stream_buf.discard() history_size = event.data.get("history_size", "?") if event.data else "?" print(f"\n--- Agent {event.event_type} (history_size={history_size}) ---", file=sys.stderr) + if event.event_type == "turn_complete": + session = session_holder[0] if session_holder else None + if session is not None: + await session.send_deferred_turn_complete_notification(event) break # Shutdown @@ -1212,6 +1232,8 @@ async def headless_main( except asyncio.TimeoutError: agent_task.cancel() await tool_router.__aexit__(None, None, None) + finally: + await notification_gateway.close() def cli(): diff --git a/agent/messaging/__init__.py b/agent/messaging/__init__.py new file mode 100644 index 000000000..c399d254e --- /dev/null +++ b/agent/messaging/__init__.py @@ -0,0 +1,15 @@ +from agent.messaging.gateway import NotificationGateway +from agent.messaging.models import ( + MessagingConfig, + NotificationRequest, + NotificationResult, + SUPPORTED_AUTO_EVENT_TYPES, +) + +__all__ = [ + "MessagingConfig", + "NotificationGateway", + "NotificationRequest", + "NotificationResult", + "SUPPORTED_AUTO_EVENT_TYPES", +] diff --git a/agent/messaging/base.py b/agent/messaging/base.py new file mode 100644 index 000000000..bf1d73894 --- /dev/null +++ b/agent/messaging/base.py @@ -0,0 +1,27 @@ +from abc import ABC, abstractmethod + +import httpx + +from agent.messaging.models import DestinationConfig, NotificationRequest, NotificationResult + + +class NotificationError(Exception): + """Delivery failed and should not be retried.""" + + +class RetryableNotificationError(NotificationError): + """Delivery failed transiently and can be retried.""" + + +class NotificationProvider(ABC): + provider_name: str + + @abstractmethod + async def send( + self, + client: httpx.AsyncClient, + destination_name: str, + destination: DestinationConfig, + request: NotificationRequest, + ) -> NotificationResult: + """Deliver a notification to one destination.""" diff --git a/agent/messaging/gateway.py b/agent/messaging/gateway.py new file mode 100644 index 000000000..83c4704ba --- /dev/null +++ b/agent/messaging/gateway.py @@ -0,0 +1,166 @@ +import asyncio +import logging +from collections.abc import Iterable + +import httpx + +from agent.messaging.base import ( + NotificationError, + NotificationProvider, + RetryableNotificationError, +) +from agent.messaging.models import ( + MessagingConfig, + NotificationRequest, + NotificationResult, +) +from agent.messaging.slack import SlackProvider + +logger = logging.getLogger(__name__) + +_RETRY_DELAYS = (1, 2, 4) + + +class NotificationGateway: + def __init__(self, config: MessagingConfig): + self.config = config + self._providers: dict[str, NotificationProvider] = { + "slack": SlackProvider(), + } + self._queue: asyncio.Queue[NotificationRequest] = asyncio.Queue() + self._worker_task: asyncio.Task | None = None + self._client: httpx.AsyncClient | None = None + + @property + def enabled(self) -> bool: + return self.config.enabled + + async def start(self) -> None: + if not self.enabled or self._worker_task is not None: + return + self._client = httpx.AsyncClient(timeout=10.0) + self._worker_task = asyncio.create_task(self._worker(), name="notification-gateway") + + async def flush(self) -> None: + if not self.enabled: + return + await self._queue.join() + + async def close(self) -> None: + if not self.enabled: + return + await self.flush() + if self._worker_task is not None: + self._worker_task.cancel() + try: + await self._worker_task + except asyncio.CancelledError: + pass + self._worker_task = None + if self._client is not None: + await self._client.aclose() + self._client = None + + async def send(self, request: NotificationRequest) -> NotificationResult: + if not self.enabled: + return NotificationResult( + destination=request.destination, + ok=False, + provider="disabled", + error="Messaging is disabled", + ) + + destination = self.config.get_destination(request.destination) + if destination is None: + return NotificationResult( + destination=request.destination, + ok=False, + provider="unknown", + error=f"Unknown destination '{request.destination}'", + ) + + provider = self._providers.get(destination.provider) + if provider is None: + return NotificationResult( + destination=request.destination, + ok=False, + provider=destination.provider, + error=f"No provider implementation for '{destination.provider}'", + ) + return await self._send_with_retries(provider, request.destination, destination, request) + + async def send_many( + self, requests: Iterable[NotificationRequest] + ) -> list[NotificationResult]: + results: list[NotificationResult] = [] + for request in requests: + results.append(await self.send(request)) + return results + + async def enqueue(self, request: NotificationRequest) -> bool: + if not self.enabled or self._worker_task is None: + return False + await self._queue.put(request) + return True + + async def _worker(self) -> None: + while True: + request = await self._queue.get() + try: + result = await self.send(request) + if not result.ok: + logger.warning( + "Notification delivery failed for %s: %s", + request.destination, + result.error, + ) + except Exception: + logger.exception("Unexpected notification worker failure") + finally: + self._queue.task_done() + + async def _send_with_retries( + self, + provider: NotificationProvider, + destination_name: str, + destination, + request: NotificationRequest, + ) -> NotificationResult: + client = self._client or httpx.AsyncClient(timeout=10.0) + owns_client = self._client is None + try: + for attempt in range(len(_RETRY_DELAYS) + 1): + try: + return await provider.send(client, destination_name, destination, request) + except RetryableNotificationError as exc: + if attempt >= len(_RETRY_DELAYS): + return NotificationResult( + destination=destination_name, + ok=False, + provider=provider.provider_name, + error=str(exc), + ) + delay = _RETRY_DELAYS[attempt] + logger.warning( + "Retrying notification to %s in %ss after transient error: %s", + destination_name, + delay, + exc, + ) + await asyncio.sleep(delay) + except NotificationError as exc: + return NotificationResult( + destination=destination_name, + ok=False, + provider=provider.provider_name, + error=str(exc), + ) + return NotificationResult( + destination=destination_name, + ok=False, + provider=provider.provider_name, + error="Notification delivery exhausted retries", + ) + finally: + if owns_client: + await client.aclose() diff --git a/agent/messaging/models.py b/agent/messaging/models.py new file mode 100644 index 000000000..25f645fe9 --- /dev/null +++ b/agent/messaging/models.py @@ -0,0 +1,123 @@ +from typing import Annotated, Literal + +from pydantic import BaseModel, Field, field_validator, model_validator + +_DESTINATION_NAME_CHARS = set("abcdefghijklmnopqrstuvwxyz0123456789._-") +SUPPORTED_AUTO_EVENT_TYPES = {"approval_required", "error", "turn_complete"} + + +class SlackDestinationConfig(BaseModel): + provider: Literal["slack"] = "slack" + token: str + channel: str + allow_agent_tool: bool = False + allow_auto_events: bool = False + username: str | None = None + icon_emoji: str | None = None + + @field_validator("token", "channel") + @classmethod + def _require_non_empty(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("must not be empty") + return value + + +DestinationConfig = Annotated[SlackDestinationConfig, Field(discriminator="provider")] + + +class MessagingConfig(BaseModel): + enabled: bool = False + auto_event_types: list[str] = Field( + default_factory=lambda: ["approval_required", "error", "turn_complete"] + ) + destinations: dict[str, DestinationConfig] = Field(default_factory=dict) + + @field_validator("destinations") + @classmethod + def _validate_destination_names( + cls, destinations: dict[str, DestinationConfig] + ) -> dict[str, DestinationConfig]: + for name in destinations: + if not name or any(char not in _DESTINATION_NAME_CHARS for char in name): + raise ValueError( + "destination names must use lowercase letters, digits, '.', '_' or '-'" + ) + return destinations + + @field_validator("auto_event_types") + @classmethod + def _validate_auto_event_types(cls, event_types: list[str]) -> list[str]: + if not event_types: + return [] + normalized: list[str] = [] + seen: set[str] = set() + for event_type in event_types: + if event_type not in SUPPORTED_AUTO_EVENT_TYPES: + raise ValueError( + f"unsupported auto event type '{event_type}'" + ) + if event_type not in seen: + normalized.append(event_type) + seen.add(event_type) + return normalized + + @model_validator(mode="after") + def _require_destinations_when_enabled(self) -> "MessagingConfig": + if self.enabled and not self.destinations: + raise ValueError("messaging.enabled requires at least one destination") + return self + + def get_destination(self, name: str) -> DestinationConfig | None: + return self.destinations.get(name) + + def can_agent_tool_send(self, name: str) -> bool: + destination = self.get_destination(name) + return bool(destination and destination.allow_agent_tool) + + def can_auto_send(self, name: str) -> bool: + destination = self.get_destination(name) + return bool(destination and destination.allow_auto_events) + + def default_auto_destinations(self) -> list[str]: + if not self.enabled: + return [] + return [ + name + for name in self.destinations + if self.can_auto_send(name) + ] + + +class NotificationRequest(BaseModel): + destination: str + title: str | None = None + message: str + severity: Literal["info", "success", "warning", "error"] = "info" + metadata: dict[str, str] = Field(default_factory=dict) + event_type: str | None = None + + @field_validator("destination", "message") + @classmethod + def _require_text(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("must not be empty") + return value + + @field_validator("title") + @classmethod + def _normalize_title(cls, value: str | None) -> str | None: + if value is None: + return None + value = value.strip() + return value or None + + +class NotificationResult(BaseModel): + destination: str + ok: bool + provider: str + error: str | None = None + external_id: str | None = None diff --git a/agent/messaging/slack.py b/agent/messaging/slack.py new file mode 100644 index 000000000..a1fb7c18e --- /dev/null +++ b/agent/messaging/slack.py @@ -0,0 +1,186 @@ +import json +import re + +import httpx + +from agent.messaging.base import ( + NotificationError, + NotificationProvider, + RetryableNotificationError, +) +from agent.messaging.models import ( + NotificationRequest, + NotificationResult, + SlackDestinationConfig, +) + +_SEVERITY_PREFIX = { + "info": "[INFO]", + "success": "[SUCCESS]", + "warning": "[WARNING]", + "error": "[ERROR]", +} + + +def _format_slack_mrkdwn(content: str) -> str: + """Convert common Markdown constructs to Slack's mrkdwn syntax.""" + if not content: + return content + + placeholders: dict[str, str] = {} + placeholder_index = 0 + + def placeholder(value: str) -> str: + nonlocal placeholder_index + key = f"\x00SLACK{placeholder_index}\x00" + placeholder_index += 1 + placeholders[key] = value + return key + + text = content + + # Protect code before any formatting conversion. Slack's mrkdwn ignores + # formatting inside backticks, so these regions should stay byte-for-byte. + text = re.sub( + r"(```(?:[^\n]*\n)?[\s\S]*?```)", + lambda match: placeholder(match.group(0)), + text, + ) + text = re.sub(r"(`[^`\n]+`)", lambda match: placeholder(match.group(0)), text) + + def convert_markdown_link(match: re.Match[str]) -> str: + label = match.group(1) + url = match.group(2).strip() + if url.startswith("<") and url.endswith(">"): + url = url[1:-1].strip() + return placeholder(f"<{url}|{label}>") + + text = re.sub( + r"\[([^\]]+)\]\(([^()]*(?:\([^()]*\)[^()]*)*)\)", + convert_markdown_link, + text, + ) + + # Preserve existing Slack entities and manual mrkdwn links before escaping. + text = re.sub( + r"(<(?:[@#!]|(?:https?|mailto|tel):)[^>\n]+>)", + lambda match: placeholder(match.group(1)), + text, + ) + text = re.sub( + r"^(>+\s)", + lambda match: placeholder(match.group(0)), + text, + flags=re.MULTILINE, + ) + + text = text.replace("&", "&").replace("<", "<").replace(">", ">") + text = text.replace("&", "&").replace("<", "<").replace(">", ">") + + def convert_header(match: re.Match[str]) -> str: + header = match.group(1).strip() + header = re.sub(r"\*\*(.+?)\*\*", r"\1", header) + return placeholder(f"*{header}*") + + text = re.sub(r"^#{1,6}\s+(.+)$", convert_header, text, flags=re.MULTILINE) + text = re.sub( + r"\*\*\*(.+?)\*\*\*", + lambda match: placeholder(f"*_{match.group(1)}_*"), + text, + ) + text = re.sub( + r"\*\*(.+?)\*\*", + lambda match: placeholder(f"*{match.group(1)}*"), + text, + ) + text = re.sub( + r"(? str: + lines: list[str] = [] + prefix = _SEVERITY_PREFIX[request.severity] + if request.title: + lines.append(f"{prefix} {request.title}") + else: + lines.append(prefix) + lines.append(request.message) + for key, value in request.metadata.items(): + lines.append(f"{key}: {value}") + return _format_slack_mrkdwn("\n".join(lines)) + + +class SlackProvider(NotificationProvider): + provider_name = "slack" + + async def send( + self, + client: httpx.AsyncClient, + destination_name: str, + destination: SlackDestinationConfig, + request: NotificationRequest, + ) -> NotificationResult: + payload = { + "channel": destination.channel, + "text": _format_text(request), + "mrkdwn": True, + "unfurl_links": False, + "unfurl_media": False, + } + if destination.username: + payload["username"] = destination.username + if destination.icon_emoji: + payload["icon_emoji"] = destination.icon_emoji + + try: + response = await client.post( + "https://slack.com/api/chat.postMessage", + headers={ + "Authorization": f"Bearer {destination.token}", + "Content-Type": "application/json; charset=utf-8", + }, + content=json.dumps(payload), + ) + except httpx.TimeoutException as exc: + raise RetryableNotificationError("Slack request timed out") from exc + except httpx.TransportError as exc: + raise RetryableNotificationError("Slack transport error") from exc + + if response.status_code == 429 or response.status_code >= 500: + raise RetryableNotificationError( + f"Slack HTTP {response.status_code}" + ) + if response.status_code >= 400: + raise NotificationError(f"Slack HTTP {response.status_code}") + + try: + data = response.json() + except ValueError as exc: + raise RetryableNotificationError("Slack returned invalid JSON") from exc + + if not data.get("ok"): + error = str(data.get("error") or "unknown_error") + if error == "ratelimited": + raise RetryableNotificationError(error) + raise NotificationError(error) + + return NotificationResult( + destination=destination_name, + ok=True, + provider=self.provider_name, + external_id=str(data.get("ts") or ""), + error=None, + ) diff --git a/agent/prompts/system_prompt_v3.yaml b/agent/prompts/system_prompt_v3.yaml index ef9aa782f..4a628b17b 100644 --- a/agent/prompts/system_prompt_v3.yaml +++ b/agent/prompts/system_prompt_v3.yaml @@ -157,6 +157,7 @@ system_prompt: | - Always include direct Hub URLs when referencing models, datasets, Spaces, or jobs. - For errors: state what went wrong, why, and what you're doing to fix it. - Do not over-explain or present elaborate option menus for simple tasks. When the user's intent is clear, act on it. Present options only when there's genuine ambiguity. + - Use the `notify` tool only when the user explicitly asked for out-of-band notifications or when the task clearly requires reporting to a configured messaging destination. Do not use it for routine chat updates. # Tool usage diff --git a/agent/tools/notify_tool.py b/agent/tools/notify_tool.py new file mode 100644 index 000000000..f926d5a58 --- /dev/null +++ b/agent/tools/notify_tool.py @@ -0,0 +1,108 @@ +from typing import Any + +from agent.messaging.models import NotificationRequest + +NOTIFY_TOOL_SPEC = { + "name": "notify", + "description": ( + "Send an out-of-band notification to configured messaging destinations. " + "Use this only when the user explicitly asked for proactive notifications " + "or when the task requires reporting progress outside the chat. " + "Destinations must be named server-side configs such as 'slack.ops'." + ), + "parameters": { + "type": "object", + "properties": { + "destinations": { + "type": "array", + "description": "Named messaging destinations to notify.", + "items": {"type": "string"}, + "minItems": 1, + }, + "message": { + "type": "string", + "description": "Main notification body.", + }, + "title": { + "type": "string", + "description": "Optional short title line.", + }, + "severity": { + "type": "string", + "enum": ["info", "success", "warning", "error"], + "description": "Notification severity label.", + }, + }, + "required": ["destinations", "message"], + }, +} + + +async def notify_handler( + arguments: dict[str, Any], session=None, **_kwargs +) -> tuple[str, bool]: + if session is None or session.notification_gateway is None: + return "Messaging is not configured for this session.", False + + raw_destinations = arguments.get("destinations", []) + if not isinstance(raw_destinations, list) or not raw_destinations: + return "destinations must be a non-empty array of destination names.", False + + destinations: list[str] = [] + seen: set[str] = set() + for raw_name in raw_destinations: + if not isinstance(raw_name, str): + return "Each destination must be a string.", False + name = raw_name.strip() + if not name: + return "Destination names must not be empty.", False + if name not in seen: + destinations.append(name) + seen.add(name) + + disallowed = [ + name + for name in destinations + if not session.config.messaging.can_agent_tool_send(name) + ] + if disallowed: + return ( + "These destinations are unavailable for the notify tool: " + + ", ".join(disallowed) + ), False + + message = arguments.get("message", "") + if not isinstance(message, str) or not message.strip(): + return "message must be a non-empty string.", False + + title = arguments.get("title") + severity = arguments.get("severity", "info") + if title is not None and not isinstance(title, str): + return "title must be a string when provided.", False + if severity not in {"info", "success", "warning", "error"}: + return "severity must be one of: info, success, warning, error.", False + + requests = [ + NotificationRequest( + destination=name, + title=title, + message=message, + severity=severity, + metadata={ + "session_id": session.session_id, + "model": session.config.model_name, + }, + ) + for name in destinations + ] + results = await session.notification_gateway.send_many(requests) + + lines = [] + all_ok = True + for result in results: + if result.ok: + lines.append(f"{result.destination}: sent") + else: + all_ok = False + lines.append(f"{result.destination}: failed ({result.error})") + return "\n".join(lines), all_ok diff --git a/backend/main.py b/backend/main.py index 9aa939a08..9596ed2b9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,6 +11,7 @@ from fastapi.staticfiles import StaticFiles from routes.agent import router as agent_router from routes.auth import router as auth_router +from session_manager import session_manager # Load .env from project root (parent directory) load_dotenv(Path(__file__).parent.parent / ".env") @@ -27,6 +28,7 @@ async def lifespan(app: FastAPI): """Application lifespan handler.""" logger.info("Starting HF Agent backend...") + await session_manager.start() # Start in-process hourly KPI rollup. Replaces an external cron so the # rollup lives next to the data and reuses the Space's HF token. try: @@ -34,7 +36,6 @@ async def lifespan(app: FastAPI): kpis_scheduler.start() except Exception as e: logger.warning("KPI scheduler failed to start: %s", e) - yield logger.info("Shutting down HF Agent backend...") @@ -47,7 +48,6 @@ async def lifespan(app: FastAPI): # Final-flush: save every still-active session so we don't lose traces on # server restart. Uploads are detached subprocesses — this is fast. try: - from session_manager import session_manager for sid, agent_session in list(session_manager.sessions.items()): sess = agent_session.session if sess.config.save_sessions: @@ -58,6 +58,7 @@ async def lifespan(app: FastAPI): logger.warning("Failed to flush session %s: %s", sid, e) except Exception as e: logger.warning("Lifespan final-flush skipped: %s", e) + await session_manager.close() app = FastAPI( diff --git a/backend/models.py b/backend/models.py index 952365c23..aa0e1e068 100644 --- a/backend/models.py +++ b/backend/models.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, Field class OpType(str, Enum): @@ -87,6 +87,13 @@ class SessionInfo(BaseModel): user_id: str = "dev" pending_approval: list[PendingApprovalTool] | None = None model: str | None = None + notification_destinations: list[str] = Field(default_factory=list) + + +class SessionNotificationsRequest(BaseModel): + """Replace the session's auto-notification destinations.""" + + destinations: list[str] class HealthResponse(BaseModel): diff --git a/backend/routes/agent.py b/backend/routes/agent.py index e990bb941..342176eb2 100644 --- a/backend/routes/agent.py +++ b/backend/routes/agent.py @@ -24,6 +24,7 @@ HealthResponse, LLMHealthResponse, SessionInfo, + SessionNotificationsRequest, SessionResponse, SubmitRequest, TruncateRequest, @@ -513,6 +514,26 @@ async def set_session_model( return {"session_id": session_id, "model": model_id} +@router.post("/session/{session_id}/notifications") +async def set_session_notifications( + session_id: str, + body: SessionNotificationsRequest, + user: dict = Depends(get_current_user), +) -> dict: + """Replace the session's auto-notification destinations.""" + _check_session_access(session_id, user) + try: + destinations = session_manager.set_notification_destinations( + session_id, body.destinations + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + return { + "session_id": session_id, + "notification_destinations": destinations, + } + + @router.get("/user/quota") async def get_user_quota(user: dict = Depends(get_current_user)) -> dict: """Return the user's plan tier and today's Claude-session quota state.""" @@ -824,7 +845,6 @@ async def shutdown_session( raise HTTPException(status_code=404, detail="Session not found or inactive") return {"status": "shutdown_requested", "session_id": session_id} - @router.post("/feedback/{session_id}") async def submit_feedback( session_id: str, diff --git a/backend/session_manager.py b/backend/session_manager.py index 4534fd701..fc9624ba3 100644 --- a/backend/session_manager.py +++ b/backend/session_manager.py @@ -10,6 +10,7 @@ from agent.config import load_config from agent.core.agent_loop import process_submission +from agent.messaging.gateway import NotificationGateway from agent.core.session import Event, OpType, Session from agent.core.tools import ToolRouter @@ -119,9 +120,18 @@ class SessionManager: def __init__(self, config_path: str | None = None) -> None: self.config = load_config(config_path or DEFAULT_CONFIG_PATH) + self.messaging_gateway = NotificationGateway(self.config.messaging) self.sessions: dict[str, AgentSession] = {} self._lock = asyncio.Lock() + async def start(self) -> None: + """Start shared background resources.""" + await self.messaging_gateway.start() + + async def close(self) -> None: + """Flush and close shared background resources.""" + await self.messaging_gateway.close() + def _count_user_sessions(self, user_id: str) -> int: """Count active sessions owned by a specific user.""" return sum( @@ -192,7 +202,11 @@ def _create_session_sync(): session_config.model_name = model session = Session( event_queue, config=session_config, tool_router=tool_router, - hf_token=hf_token, user_id=user_id, + hf_token=hf_token, + user_id=user_id, + notification_gateway=self.messaging_gateway, + notification_destinations=[], + session_id=session_id, ) t1 = _time.monotonic() logger.info(f"Session initialized in {t1 - t0:.2f}s") @@ -518,8 +532,39 @@ def get_session_info(self, session_id: str) -> dict[str, Any] | None: "user_id": agent_session.user_id, "pending_approval": pending_approval, "model": agent_session.session.config.model_name, + "notification_destinations": list( + agent_session.session.notification_destinations + ), } + def set_notification_destinations( + self, session_id: str, destinations: list[str] + ) -> list[str]: + """Replace the session's opted-in auto-notification destinations.""" + agent_session = self.sessions.get(session_id) + if not agent_session or not agent_session.is_active: + raise ValueError("Session not found or inactive") + + normalized: list[str] = [] + seen: set[str] = set() + for raw_name in destinations: + name = raw_name.strip() + if not name: + raise ValueError("Destination names must not be empty") + destination = self.config.messaging.get_destination(name) + if destination is None: + raise ValueError(f"Unknown destination '{name}'") + if not destination.allow_auto_events: + raise ValueError( + f"Destination '{name}' is not enabled for auto events" + ) + if name not in seen: + normalized.append(name) + seen.add(name) + + agent_session.session.set_notification_destinations(normalized) + return normalized + def list_sessions(self, user_id: str | None = None) -> list[dict[str, Any]]: """List sessions, optionally filtered by user. diff --git a/configs/cli_agent_config.json b/configs/cli_agent_config.json index 99335ca71..5c6a22a35 100644 --- a/configs/cli_agent_config.json +++ b/configs/cli_agent_config.json @@ -5,6 +5,11 @@ "yolo_mode": false, "confirm_cpu_jobs": true, "auto_file_upload": true, + "messaging": { + "enabled": false, + "auto_event_types": ["approval_required", "error", "turn_complete"], + "destinations": {} + }, "mcpServers": { "hf-mcp-server": { "transport": "http", diff --git a/pyproject.toml b/pyproject.toml index 432085e09..1c6752411 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ eval = [ # Development and testing dependencies dev = [ "pytest>=9.0.2", - "pytest-asyncio>=0.26.0", + "pytest-asyncio>=1.2.0", ] # All dependencies (eval + dev) diff --git a/tests/unit/test_cli_rendering.py b/tests/unit/test_cli_rendering.py index 5fa0dc2c6..ff633c067 100644 --- a/tests/unit/test_cli_rendering.py +++ b/tests/unit/test_cli_rendering.py @@ -79,7 +79,7 @@ def fake_banner(*, model=None, hf_user=None): monkeypatch.setattr( main_mod, "load_config", - lambda _path: SimpleNamespace( + lambda _path, **_kwargs: SimpleNamespace( model_name="moonshotai/Kimi-K2.6", mcpServers={}, ), diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 000000000..71f92b2a4 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,121 @@ +import json + +from agent import config as config_module + + +def _write_json(path, data): + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_load_config_does_not_apply_slack_user_defaults_by_default(tmp_path, monkeypatch): + config_path = tmp_path / "config.json" + _write_json( + config_path, + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": False, + "destinations": {}, + }, + }, + ) + monkeypatch.setenv("SLACK_BOT_TOKEN", "xoxb-test") + monkeypatch.setenv("SLACK_CHANNEL_ID", "C123") + + config = config_module.load_config(str(config_path)) + + assert not config.messaging.enabled + assert config.messaging.destinations == {} + + +def test_load_config_applies_slack_user_defaults_from_env(tmp_path, monkeypatch): + config_path = tmp_path / "config.json" + _write_json(config_path, {"model_name": "moonshotai/Kimi-K2.6"}) + monkeypatch.delenv("ML_INTERN_CLI_CONFIG", raising=False) + monkeypatch.setattr( + config_module, + "DEFAULT_USER_CONFIG_PATH", + tmp_path / "missing-user-config.json", + ) + monkeypatch.setenv("SLACK_BOT_TOKEN", "xoxb-test") + monkeypatch.setenv("SLACK_CHANNEL_ID", "C123") + + config = config_module.load_config(str(config_path), include_user_defaults=True) + + assert config.messaging.enabled + assert config.messaging.auto_event_types == [ + "approval_required", + "error", + "turn_complete", + ] + destination = config.messaging.destinations["slack.default"] + assert destination.token == "xoxb-test" + assert destination.channel == "C123" + assert destination.allow_agent_tool + assert destination.allow_auto_events + + +def test_load_config_merges_user_config_before_env_substitution(tmp_path, monkeypatch): + config_path = tmp_path / "config.json" + user_config_path = tmp_path / "user-config.json" + _write_json(config_path, {"model_name": "moonshotai/Kimi-K2.6"}) + _write_json( + user_config_path, + { + "messaging": { + "enabled": True, + "auto_event_types": ["approval_required"], + "destinations": { + "slack.team": { + "provider": "slack", + "token": "${USER_SLACK_TOKEN}", + "channel": "C999", + "allow_agent_tool": False, + "allow_auto_events": True, + }, + }, + }, + }, + ) + monkeypatch.setenv("ML_INTERN_CLI_CONFIG", str(user_config_path)) + monkeypatch.setenv("ML_INTERN_SLACK_NOTIFICATIONS", "0") + monkeypatch.setenv("USER_SLACK_TOKEN", "xoxb-user") + + config = config_module.load_config(str(config_path), include_user_defaults=True) + + assert config.messaging.enabled + assert config.messaging.auto_event_types == ["approval_required"] + assert set(config.messaging.destinations) == {"slack.team"} + destination = config.messaging.destinations["slack.team"] + assert destination.token == "xoxb-user" + assert destination.channel == "C999" + assert not destination.allow_agent_tool + assert destination.allow_auto_events + + +def test_slack_user_defaults_can_be_disabled(tmp_path, monkeypatch): + config_path = tmp_path / "config.json" + _write_json( + config_path, + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": False, + "destinations": {}, + }, + }, + ) + monkeypatch.delenv("ML_INTERN_CLI_CONFIG", raising=False) + monkeypatch.setattr( + config_module, + "DEFAULT_USER_CONFIG_PATH", + tmp_path / "missing-user-config.json", + ) + monkeypatch.setenv("ML_INTERN_SLACK_NOTIFICATIONS", "false") + monkeypatch.setenv("SLACK_BOT_TOKEN", "xoxb-test") + monkeypatch.setenv("SLACK_CHANNEL_ID", "C123") + + config = config_module.load_config(str(config_path), include_user_defaults=True) + + assert not config.messaging.enabled + assert config.messaging.destinations == {} diff --git a/tests/unit/test_messaging.py b/tests/unit/test_messaging.py new file mode 100644 index 000000000..968622c1a --- /dev/null +++ b/tests/unit/test_messaging.py @@ -0,0 +1,511 @@ +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest +from pydantic import ValidationError + +from agent.config import Config +from agent.core.session import Event, Session +from agent.messaging.gateway import NotificationGateway +from agent.messaging.models import NotificationRequest, NotificationResult +from agent.messaging.slack import SlackProvider, _format_slack_mrkdwn +from agent.tools.notify_tool import notify_handler +from backend.session_manager import AgentSession, SessionManager + + +class DummyToolRouter: + def get_tool_specs_for_llm(self) -> list[dict]: + return [] + + +class RecordingGateway: + def __init__(self): + self.enqueued: list[NotificationRequest] = [] + self.sent: list[NotificationRequest] = [] + + async def enqueue(self, request: NotificationRequest) -> bool: + self.enqueued.append(request) + return True + + async def send_many( + self, requests: list[NotificationRequest] + ) -> list[NotificationResult]: + self.sent.extend(requests) + return [ + NotificationResult( + destination=request.destination, + ok=True, + provider="test", + ) + for request in requests + ] + + +def _config_with_messaging(**destination_overrides) -> Config: + destination = { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + **destination_overrides, + } + return Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "slack.ops": destination, + }, + }, + } + ) + + +def _test_session( + config: Config, gateway, session_id: str = "session-test" +) -> Session: + return Session( + asyncio.Queue(), + config=config, + tool_router=DummyToolRouter(), + context_manager=SimpleNamespace(items=[]), + notification_gateway=gateway, + session_id=session_id, + ) + + +def test_messaging_config_validates_destination_names(): + with pytest.raises(ValidationError): + Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "Slack Ops": { + "provider": "slack", + "token": "x", + "channel": "C123", + } + }, + }, + } + ) + + config = _config_with_messaging(allow_agent_tool=True, allow_auto_events=True) + assert config.messaging.can_agent_tool_send("slack.ops") + assert config.messaging.can_auto_send("slack.ops") + + +def test_messaging_config_default_auto_destinations_only_returns_auto_enabled(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + }, + "slack.tool": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C999", + "allow_agent_tool": True, + }, + }, + }, + } + ) + + assert config.messaging.default_auto_destinations() == ["slack.ops"] + + +def test_messaging_config_default_auto_destinations_empty_when_disabled(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": False, + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + }, + }, + }, + } + ) + + assert config.messaging.default_auto_destinations() == [] + + +def test_slack_mrkdwn_formatter_converts_common_markdown(): + formatted = _format_slack_mrkdwn( + "# Result\n" + "**Done** with *details* and ~~old text~~.\n" + "See [PR](https://github.com/huggingface/ml-intern/pull/116).\n" + "Keep `**literal**` and ```python\nx < 3\n``` untouched.\n" + "Escape & text." + ) + + assert "*Result*" in formatted + assert "*Done*" in formatted + assert "_details_" in formatted + assert "~old text~" in formatted + assert "" in formatted + assert "`**literal**`" in formatted + assert "```python\nx < 3\n```" in formatted + assert "Escape <raw> & text." in formatted + + +@pytest.mark.asyncio +async def test_slack_provider_formats_and_sends_payload(): + seen: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + seen["auth"] = request.headers["Authorization"] + seen["content_type"] = request.headers["Content-Type"] + seen["json"] = request.read().decode("utf-8") + return httpx.Response(200, json={"ok": True, "ts": "123.456"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + provider = SlackProvider() + result = await provider.send( + client, + "slack.ops", + _config_with_messaging().messaging.destinations["slack.ops"], + NotificationRequest( + destination="slack.ops", + title="Approval required", + message="A **run** is waiting. See [details](https://example.com).", + severity="warning", + metadata={"session_id": "sess-1"}, + ), + ) + + assert result.ok + assert result.external_id == "123.456" + assert seen["auth"] == "Bearer xoxb-test" + assert seen["content_type"].startswith("application/json") + payload = json.loads(str(seen["json"])) + assert payload["channel"] == "C123" + assert payload["mrkdwn"] is True + assert payload["text"] == ( + "[WARNING] Approval required\n" + "A *run* is waiting. See .\n" + "session_id: sess-1" + ) + + +@pytest.mark.asyncio +async def test_notification_gateway_retries_transient_failures(monkeypatch): + attempts = {"count": 0} + + def handler(_request: httpx.Request) -> httpx.Response: + attempts["count"] += 1 + if attempts["count"] == 1: + return httpx.Response(503, json={"ok": False}) + return httpx.Response(200, json={"ok": True, "ts": "999.1"}) + + async def fake_sleep(_delay: float) -> None: + return None + + monkeypatch.setattr("agent.messaging.gateway.asyncio.sleep", fake_sleep) + + config = _config_with_messaging(allow_agent_tool=True) + gateway = NotificationGateway(config.messaging) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client: + gateway._client = client + result = await gateway.send( + NotificationRequest( + destination="slack.ops", + message="hello", + ) + ) + gateway._client = None + + assert attempts["count"] == 2 + assert result.ok + + +@pytest.mark.asyncio +async def test_notify_tool_rejects_non_allowlisted_destinations(): + config = _config_with_messaging(allow_agent_tool=False) + gateway = RecordingGateway() + session = _test_session(config, gateway) + + output, ok = await notify_handler( + {"destinations": ["slack.ops"], "message": "done"}, + session=session, + ) + + assert not ok + assert "unavailable for the notify tool" in output + assert gateway.sent == [] + + +@pytest.mark.asyncio +async def test_notify_tool_sends_to_allowlisted_destinations(): + config = _config_with_messaging(allow_agent_tool=True) + gateway = RecordingGateway() + session = _test_session(config, gateway, session_id="sess-42") + + output, ok = await notify_handler( + { + "destinations": ["slack.ops"], + "title": "Training complete", + "message": "The run finished successfully.", + "severity": "success", + }, + session=session, + ) + + assert ok + assert output == "slack.ops: sent" + assert len(gateway.sent) == 1 + sent = gateway.sent[0] + assert sent.metadata["session_id"] == "sess-42" + assert sent.metadata["model"] == "moonshotai/Kimi-K2.6" + + +@pytest.mark.asyncio +async def test_session_auto_notifications_only_send_opted_in_auto_destinations(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + }, + "slack.tool": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C999", + "allow_agent_tool": True, + }, + }, + }, + } + ) + gateway = RecordingGateway() + session = _test_session(config, gateway, session_id="sess-auto") + session.set_notification_destinations(["slack.ops", "slack.tool"]) + + await session.send_event( + Event( + event_type="approval_required", + data={"tools": [{"tool": "hf_jobs", "tool_call_id": "tc-1"}]}, + ) + ) + await session.send_event( + Event(event_type="assistant_message", data={"content": "normal message"}) + ) + + assert len(gateway.enqueued) == 1 + request = gateway.enqueued[0] + assert request.destination == "slack.ops" + assert request.severity == "warning" + assert request.event_type == "approval_required" + assert "hf_jobs" in request.message + + +@pytest.mark.asyncio +async def test_turn_complete_auto_notification_includes_final_response_summary(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + } + }, + }, + } + ) + gateway = RecordingGateway() + session = _test_session(config, gateway, session_id="sess-done") + session.set_notification_destinations(["slack.ops"]) + + await session.send_event( + Event( + event_type="turn_complete", + data={ + "history_size": 12, + "final_response": "Evaluation finished. Accuracy: 84.2% on the validation split.", + }, + ) + ) + + assert len(gateway.enqueued) == 1 + request = gateway.enqueued[0] + assert request.destination == "slack.ops" + assert request.severity == "success" + assert request.event_type == "turn_complete" + assert "completed successfully" in request.message + assert "Accuracy: 84.2%" in request.message + + +@pytest.mark.asyncio +async def test_turn_complete_auto_notification_supports_longer_summary(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + } + }, + }, + } + ) + gateway = RecordingGateway() + session = _test_session(config, gateway, session_id="sess-long") + session.set_notification_destinations(["slack.ops"]) + + long_summary = "A" * 1200 + " END" + await session.send_event( + Event( + event_type="turn_complete", + data={ + "history_size": 12, + "final_response": long_summary, + }, + ) + ) + + assert len(gateway.enqueued) == 1 + request = gateway.enqueued[0] + assert request.event_type == "turn_complete" + assert "A" * 1200 in request.message + assert request.message.endswith("END") + + +@pytest.mark.asyncio +async def test_turn_complete_auto_notification_can_be_deferred(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + } + }, + }, + } + ) + gateway = RecordingGateway() + session = Session( + asyncio.Queue(), + config=config, + tool_router=DummyToolRouter(), + context_manager=SimpleNamespace(items=[]), + notification_gateway=gateway, + notification_destinations=["slack.ops"], + defer_turn_complete_notification=True, + session_id="sess-deferred", + ) + event = Event( + event_type="turn_complete", + data={"final_response": "Finished after the CLI drained the stream."}, + ) + + await session.send_event(event) + assert gateway.enqueued == [] + + await session.send_deferred_turn_complete_notification(event) + + assert len(gateway.enqueued) == 1 + request = gateway.enqueued[0] + assert request.destination == "slack.ops" + assert request.event_type == "turn_complete" + assert "Finished after the CLI drained the stream." in request.message + + +@pytest.mark.asyncio +async def test_turn_complete_can_be_disabled_by_custom_auto_event_config(): + config = Config.model_validate( + { + "model_name": "moonshotai/Kimi-K2.6", + "messaging": { + "enabled": True, + "auto_event_types": ["error"], + "destinations": { + "slack.ops": { + "provider": "slack", + "token": "xoxb-test", + "channel": "C123", + "allow_auto_events": True, + } + }, + }, + } + ) + gateway = RecordingGateway() + session = _test_session(config, gateway, session_id="sess-optout") + session.set_notification_destinations(["slack.ops"]) + + await session.send_event( + Event( + event_type="turn_complete", + data={"final_response": "This should not notify."}, + ) + ) + + assert gateway.enqueued == [] + + +def test_session_manager_updates_notification_destinations_in_session_info(): + config = _config_with_messaging(allow_auto_events=True) + manager = SessionManager(str(Path(__file__).resolve().parents[2] / "configs" / "cli_agent_config.json")) + manager.config = config + manager.sessions = {} + + session = _test_session(config, RecordingGateway(), session_id="sess-manager") + manager.sessions["sess-manager"] = AgentSession( + session_id="sess-manager", + session=session, + tool_router=DummyToolRouter(), + submission_queue=asyncio.Queue(), + ) + + updated = manager.set_notification_destinations( + "sess-manager", + ["slack.ops", "slack.ops"], + ) + + assert updated == ["slack.ops"] + info = manager.get_session_info("sess-manager") + assert info is not None + assert info["notification_destinations"] == ["slack.ops"] + + with pytest.raises(ValueError): + manager.set_notification_destinations("sess-manager", ["slack.unknown"]) diff --git a/tests/unit/test_thinking_history.py b/tests/unit/test_thinking_history.py index f2885dd61..9ef4b2f61 100644 --- a/tests/unit/test_thinking_history.py +++ b/tests/unit/test_thinking_history.py @@ -159,7 +159,7 @@ async def send_event(event): @pytest.mark.asyncio -async def test_streaming_call_collects_anthropic_delta_thinking_state(monkeypatch): +async def test_streaming_call_rebuilds_anthropic_delta_thinking_state(monkeypatch): async def fake_stream(): yield SimpleNamespace( choices=[ @@ -167,7 +167,31 @@ async def fake_stream(): delta=SimpleNamespace( content=None, tool_calls=None, - thinking_blocks=[{"type": "thinking", "thinking": "reasoned"}], + thinking_blocks=[ + { + "type": "thinking", + "thinking": "reasoned", + "signature": "", + } + ], + ), + finish_reason=None, + ) + ], + ) + yield SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace( + content=None, + tool_calls=None, + thinking_blocks=[ + { + "type": "thinking", + "thinking": "", + "signature": "signed", + } + ], ), finish_reason=None, ) @@ -186,8 +210,26 @@ async def fake_stream(): async def fake_acompletion(**_kwargs): return fake_stream() - def fail_chunk_builder(*_args, **_kwargs): - raise AssertionError("stream_chunk_builder should not run when deltas include thinking") + def fake_chunk_builder(chunks, **_kwargs): + assert len(chunks) == 4 + return SimpleNamespace( + choices=[ + SimpleNamespace( + message=Message( + role="assistant", + content="done", + thinking_blocks=[ + { + "type": "thinking", + "thinking": "reasoned", + "signature": "signed", + } + ], + reasoning_content="reasoned", + ) + ) + ] + ) events = [] async def send_event(event): @@ -199,7 +241,7 @@ async def send_event(event): send_event=send_event, ) monkeypatch.setattr(agent_loop, "acompletion", fake_acompletion) - monkeypatch.setattr(agent_loop, "stream_chunk_builder", fail_chunk_builder) + monkeypatch.setattr(agent_loop, "stream_chunk_builder", fake_chunk_builder) result = await _call_llm_streaming( session, @@ -209,7 +251,10 @@ async def send_event(event): ) assert result.content == "done" - assert result.thinking_blocks == [{"type": "thinking", "thinking": "reasoned"}] + assert result.thinking_blocks == [ + {"type": "thinking", "thinking": "reasoned", "signature": "signed"} + ] + assert result.reasoning_content == "reasoned" @pytest.mark.asyncio diff --git a/uv.lock b/uv.lock index 3bddba0dc..8595de7a0 100644 --- a/uv.lock +++ b/uv.lock @@ -1832,7 +1832,7 @@ requires-dist = [ { name = "prompt-toolkit", specifier = ">=3.0.0" }, { name = "pydantic", specifier = ">=2.12.3" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0.2" }, - { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.26.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.2.0" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "requests", specifier = ">=2.33.0" }, { name = "rich", specifier = ">=13.0.0" }, From 4b76ae8e2ab58335866d0588b4316a1a403cf7b8 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 20:08:19 +0300 Subject: [PATCH 024/120] Document Trackio alerts + iteration loop in system prompt (#156) The prompt previously had a one-liner about `report_to=["trackio"]` and flagged `run_name` as a wrong field (it is the correct field on TrainingArguments/SFTConfig/GRPOConfig). Replace with a focused Trackio section covering: - correct config fields (report_to, run_name, project, trackio_space_id) and the TRACKIO_PROJECT / TRACKIO_SPACE_ID env-var alternatives - trackio.alert(title, text, level) as the structured feedback channel, with ERROR/WARN/INFO semantics and an actionable-text requirement - how to wire alerts via a TrainerCallback (on_log vs on_evaluate) - CLI/Python recipes for reading alerts back between iterations - decision rules from prior alerts -> next config Also moves the dataset-format block back inside "When writing ML code" where it belongs. --- agent/prompts/system_prompt_v3.yaml | 35 +++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/agent/prompts/system_prompt_v3.yaml b/agent/prompts/system_prompt_v3.yaml index 4a628b17b..990aa2407 100644 --- a/agent/prompts/system_prompt_v3.yaml +++ b/agent/prompts/system_prompt_v3.yaml @@ -28,7 +28,7 @@ system_prompt: | # Mistakes you WILL make without research - HALLUCINATED IMPORTS: You will import from modules that were renamed or removed. Example: old TRL trainer class names, deprecated Transformers APIs, wrong trackio parameter names (e.g. `run_name` instead of `name`). Fix: read a current example script first. + HALLUCINATED IMPORTS: You will import from modules that were renamed or removed. Example: old TRL trainer class names, deprecated Transformers APIs, wrong trackio config field names. Fix: read a current example script first. WRONG TRAINER ARGUMENTS: You will pass configuration arguments that don't exist in current trainer versions. Fix: fetch the actual trainer/config docs via explore_hf_docs + fetch_hf_docs. @@ -54,13 +54,44 @@ system_prompt: | 3. Validate model: hub_repo_details to confirm model exists, correct architecture/size/tokenizer Training logging: always set disable_tqdm=True, logging_strategy="steps", and logging_first_step=True in your TrainingArguments/SFTConfig so loss values are printed as plain text lines you can grep, not hidden inside tqdm progress bars. - In training configs, set `report_to=["trackio"]` and set a `run_name`, `project`, and importantly `trackio_space_id` (which can be a `/mlintern-<8-char-id>` for example) so Trackio creates a public dashboard Space. Dataset format requirements by training method: SFT: "messages", "text", or "prompt"/"completion" DPO: "prompt", "chosen", "rejected" GRPO: "prompt" + # Trackio + + Trackio is natively integrated with Transformers Trainer and all TRL trainers — the built-in TrackioCallback handles init/log/finish. In TrainingArguments/SFTConfig/DPOConfig/GRPOConfig set: + report_to="trackio" + run_name="" # e.g. "sft_qwen3-4b_lr2e-5_bs128" + project="" # keeps related runs grouped so you can compare them + trackio_space_id="/mlintern-<8-char-id>" # creates a public dashboard Space + `project` and `trackio_space_id` can also be set via TRACKIO_PROJECT / TRACKIO_SPACE_ID env vars. + + Alerts are how iterations decide what to change. Use trackio.alert(title, text, level) at every decision point in training. Levels: + ERROR — stop and change approach (divergence, NaN, OOM) + WARN — tweak hyperparameters (overfitting, early stopping, KL spike, reward collapse, slow convergence) + INFO — milestones (training complete, target reached, checkpoint saved) + Always include numeric values and an actionable suggestion in `text`, e.g. "loss=12.4 at step 200 — lr likely too high, try Ɨ0.1". A future call must be able to parse it and act on it. + + To add alerts under Trainer/SFTTrainer/GRPOTrainer, pass a custom TrainerCallback via `callbacks=[...]` that calls trackio.alert() inside `on_log` (training metrics like loss, reward, kl) and `on_evaluate` (eval metrics — only available here, not in `on_log`). Keep each `if` simple: one metric, one threshold. Conditions stay easy to adjust between runs. + + Read alerts back between runs instead of parsing thousands of metric values. CLI — always use --json: + trackio get alerts --project

--run --json + trackio get alerts --project

--since --json # incremental polling + trackio get run --project

--run --json + trackio get metric --project

--run --metric --json + trackio list runs --project

--json + Python: api = trackio.Api(); api.alerts(

, run=, since=); api.runs(

) (each run has .name, .config, .alerts()). + + Drive the next config from prior alerts: + diverged → lr Ɨ 0.1 + overfitting → weight_decay Ɨ 10 or reduce capacity + early stopping → lr Ɨ 0.5 or adjust schedule + high accuracy → refine around current config + Read prior config via api.runs(...).config and only mutate keys the alerts justify changing. + # Data audit Before working with any dataset, audit it first. Do not assume you know what the data looks like — inspect it. From bce8a45f1f5409b549b20b050be6f822bf22be80 Mon Sep 17 00:00:00 2001 From: Darshan Thakare <143271270+DarshanCode2005@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:40:27 +0530 Subject: [PATCH 025/120] feat: add support to open links in new tab (#76) Co-authored-by: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> --- frontend/src/components/Chat/MarkdownContent.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Chat/MarkdownContent.tsx b/frontend/src/components/Chat/MarkdownContent.tsx index aaab83eb1..0d1e69171 100644 --- a/frontend/src/components/Chat/MarkdownContent.tsx +++ b/frontend/src/components/Chat/MarkdownContent.tsx @@ -1,4 +1,4 @@ -import { useMemo, useRef, useState, useEffect } from 'react'; +import { useMemo, useRef, useState, useEffect, type ComponentPropsWithoutRef } from 'react'; import { Box } from '@mui/material'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; @@ -166,9 +166,17 @@ export default function MarkdownContent({ content, sx, isStreaming = false }: Ma const remarkPlugins = useMemo(() => [remarkGfm], []); + const components = useMemo(() => ({ + a: ({ href, children, ...props }: ComponentPropsWithoutRef<'a'>) => ( + + {children} + + ), + }), []); + return ( - {displayContent} + {displayContent} ); } From 72f615f9638f21535100c7cef54f804b6cae8325 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 20:56:24 +0300 Subject: [PATCH 026/120] Add web search so the agent can cite current sources (#159) The agent had research tools for papers, docs, and repositories, but no direct current-web lookup. This ports Claw Code's WebSearch behavior into a Python tool that searches DuckDuckGo HTML, extracts citeable title/URL hits, applies domain filters, and returns the same JSON-shaped result payload for model consumption. Constraint: The implementation was prepared in a writable checkout because the primary working tree was sandbox read-only during this session. Rejected: Add a new search API dependency | the Claw implementation uses the DuckDuckGo HTML endpoint and the repo already has requests. Confidence: medium Scope-risk: narrow Directive: Keep the output schema stable unless the agent prompt/tool consumers are updated together. Tested: pytest tests/unit/test_web_search_tool.py tests/unit/test_malformed_args_recovery.py -q Tested: python -m compileall agent/tools/web_search_tool.py agent/core/tools.py agent/tools/research_tool.py agent/tools/__init__.py tests/unit/test_web_search_tool.py Not-tested: Live DuckDuckGo network response beyond mocked HTML parser coverage. --- agent/core/tools.py | 7 + agent/tools/__init__.py | 3 + agent/tools/research_tool.py | 5 +- agent/tools/web_search_tool.py | 273 +++++++++++++++++++++++++++++ tests/unit/test_web_search_tool.py | 161 +++++++++++++++++ 5 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 agent/tools/web_search_tool.py create mode 100644 tests/unit/test_web_search_tool.py diff --git a/agent/core/tools.py b/agent/core/tools.py index f54163ccd..ef2c57bc1 100644 --- a/agent/core/tools.py +++ b/agent/core/tools.py @@ -51,6 +51,7 @@ from agent.tools.plan_tool import PLAN_TOOL_SPEC, plan_tool_handler from agent.tools.research_tool import RESEARCH_TOOL_SPEC, research_handler from agent.tools.sandbox_tool import get_sandbox_tools +from agent.tools.web_search_tool import WEB_SEARCH_TOOL_SPEC, web_search_handler # NOTE: Private HF repo tool disabled - replaced by hf_repo_files and hf_repo_git # from agent.tools.private_hf_repo_tools import ( @@ -311,6 +312,12 @@ def create_builtin_tools(local_mode: bool = False) -> list[ToolSpec]: parameters=HF_PAPERS_TOOL_SPEC["parameters"], handler=hf_papers_handler, ), + ToolSpec( + name=WEB_SEARCH_TOOL_SPEC["name"], + description=WEB_SEARCH_TOOL_SPEC["description"], + parameters=WEB_SEARCH_TOOL_SPEC["parameters"], + handler=web_search_handler, + ), # Dataset inspection tool (unified) ToolSpec( name=HF_INSPECT_DATASET_TOOL_SPEC["name"], diff --git a/agent/tools/__init__.py b/agent/tools/__init__.py index 14ef45669..65c793cba 100644 --- a/agent/tools/__init__.py +++ b/agent/tools/__init__.py @@ -20,6 +20,7 @@ ) from agent.tools.jobs_tool import HF_JOBS_TOOL_SPEC, HfJobsTool, hf_jobs_handler from agent.tools.types import ToolResult +from agent.tools.web_search_tool import WEB_SEARCH_TOOL_SPEC, web_search_handler __all__ = [ "ToolResult", @@ -36,4 +37,6 @@ "github_search_code_handler", "HF_INSPECT_DATASET_TOOL_SPEC", "hf_inspect_dataset_handler", + "WEB_SEARCH_TOOL_SPEC", + "web_search_handler", ] diff --git a/agent/tools/research_tool.py b/agent/tools/research_tool.py index c1f5de6c4..11131766e 100644 --- a/agent/tools/research_tool.py +++ b/agent/tools/research_tool.py @@ -37,6 +37,7 @@ "github_find_examples", "github_list_repos", "github_read_file", + "web_search", "hf_inspect_dataset", "hf_repo_files", } @@ -102,6 +103,8 @@ - `explore_hf_docs(endpoint)`: Search docs for a library. Endpoints: trl, transformers, datasets, peft, accelerate, trackio, vllm, inference-endpoints, etc. - `fetch_hf_docs(url)`: Fetch full page content from explore results - `find_hf_api(query=..., tag=...)`: Find REST API endpoints +- `web_search(query=..., allowed_domains=[...], blocked_domains=[...])`: + Search the current web when papers/docs/GitHub are not enough. ## Hub repo inspection - `hf_repo_files`: List/read files in any HF repo (model, dataset, space) @@ -426,7 +429,7 @@ async def _log(text: str) -> None: await _log(f"ā–ø {tool_name} {args_str}") output, _success = await session.tool_router.call_tool( - tool_name, tool_args, session=session + tool_name, tool_args, session=session, tool_call_id=tc.id ) _tool_uses += 1 await _log(f"tools:{_tool_uses}") diff --git a/agent/tools/web_search_tool.py b/agent/tools/web_search_tool.py new file mode 100644 index 000000000..3e52ded03 --- /dev/null +++ b/agent/tools/web_search_tool.py @@ -0,0 +1,273 @@ +"""DuckDuckGo HTML web search tool. + +This mirrors Claw Code's Rust WebSearch behavior: fetch DuckDuckGo's HTML +endpoint, extract result links, optionally filter domains, and return a +JSON payload the model can cite. +""" + +from __future__ import annotations + +import asyncio +import html +import json +import os +import time +from dataclasses import dataclass +from html.parser import HTMLParser +from typing import Any +from urllib.parse import parse_qsl, parse_qs, urlencode, urlparse, urlunparse + +import requests + +DEFAULT_SEARCH_URL = "https://html.duckduckgo.com/html/" +WEB_SEARCH_BASE_URL_ENV = "CLAWD_WEB_SEARCH_BASE_URL" +USER_AGENT = "clawd-rust-tools/0.1" +REQUEST_TIMEOUT_SECONDS = 20 +MAX_RESULTS = 8 + + +@dataclass(frozen=True) +class SearchHit: + title: str + url: str + + def as_json(self) -> dict[str, str]: + return {"title": self.title, "url": self.url} + + +class _AnchorParser(HTMLParser): + def __init__(self, *, require_result_class: bool) -> None: + super().__init__(convert_charrefs=True) + self.require_result_class = require_result_class + self.hits: list[tuple[str, str]] = [] + self._active_href: str | None = None + self._active_text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.lower() != "a": + return + attr_map = {key.lower(): value or "" for key, value in attrs} + href = attr_map.get("href") + if not href: + return + if self.require_result_class and "result__a" not in attr_map.get("class", ""): + return + self._active_href = href + self._active_text = [] + + def handle_data(self, data: str) -> None: + if self._active_href is not None: + self._active_text.append(data) + + def handle_entityref(self, name: str) -> None: + if self._active_href is not None: + self._active_text.append(f"&{name};") + + def handle_charref(self, name: str) -> None: + if self._active_href is not None: + self._active_text.append(f"&#{name};") + + def handle_endtag(self, tag: str) -> None: + if tag.lower() != "a" or self._active_href is None: + return + title = collapse_whitespace(html.unescape("".join(self._active_text))).strip() + self.hits.append((self._active_href, title)) + self._active_href = None + self._active_text = [] + + +def build_search_url(query: str) -> str: + base = os.environ.get(WEB_SEARCH_BASE_URL_ENV, DEFAULT_SEARCH_URL) + parsed = urlparse(base) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError(f"invalid search base URL: {base}") + + query_pairs = parse_qsl(parsed.query, keep_blank_values=True) + query_pairs.append(("q", query)) + return urlunparse(parsed._replace(query=urlencode(query_pairs))) + + +def collapse_whitespace(value: str) -> str: + return " ".join(value.split()) + + +def decode_duckduckgo_redirect(url: str) -> str | None: + if url.startswith("http://") or url.startswith("https://"): + return html.unescape(url) + if url.startswith("//"): + joined = f"https:{url}" + elif url.startswith("/"): + joined = f"https://duckduckgo.com{url}" + else: + return None + + parsed = urlparse(joined) + if parsed.path in {"/l", "/l/"}: + uddg = parse_qs(parsed.query).get("uddg", []) + if uddg: + return html.unescape(uddg[0]) + return joined + + +def _extract_links(search_html: str, *, require_result_class: bool) -> list[SearchHit]: + parser = _AnchorParser(require_result_class=require_result_class) + parser.feed(search_html) + + hits: list[SearchHit] = [] + for raw_url, title in parser.hits: + if not title: + continue + decoded_url = decode_duckduckgo_redirect(raw_url) + if decoded_url and ( + decoded_url.startswith("http://") or decoded_url.startswith("https://") + ): + hits.append(SearchHit(title=title, url=decoded_url)) + return hits + + +def extract_search_hits(search_html: str) -> list[SearchHit]: + return _extract_links(search_html, require_result_class=True) + + +def extract_search_hits_from_generic_links(search_html: str) -> list[SearchHit]: + return _extract_links(search_html, require_result_class=False) + + +def normalize_domain_filter(domain: str) -> str: + trimmed = domain.strip() + parsed = urlparse(trimmed) + candidate = parsed.hostname if parsed.scheme and parsed.hostname else trimmed + return candidate.strip().lstrip(".").rstrip("/").lower() + + +def host_matches_list(url: str, domains: list[str]) -> bool: + host = urlparse(url).hostname + if not host: + return False + normalized_host = host.lower() + for domain in domains: + normalized = normalize_domain_filter(domain) + if normalized and ( + normalized_host == normalized or normalized_host.endswith(f".{normalized}") + ): + return True + return False + + +def dedupe_hits(hits: list[SearchHit]) -> list[SearchHit]: + seen: set[str] = set() + deduped: list[SearchHit] = [] + for hit in hits: + if hit.url in seen: + continue + seen.add(hit.url) + deduped.append(hit) + return deduped + + +def execute_web_search( + query: str, + allowed_domains: list[str] | None = None, + blocked_domains: list[str] | None = None, + tool_use_id: str = "web_search_1", +) -> dict[str, Any]: + started = time.monotonic() + search_url = build_search_url(query) + response = requests.get( + search_url, + headers={"User-Agent": USER_AGENT}, + timeout=REQUEST_TIMEOUT_SECONDS, + allow_redirects=True, + ) + + hits = extract_search_hits(response.text) + if not hits and urlparse(response.url or search_url).hostname: + hits = extract_search_hits_from_generic_links(response.text) + + if allowed_domains is not None: + hits = [hit for hit in hits if host_matches_list(hit.url, allowed_domains)] + if blocked_domains is not None: + hits = [hit for hit in hits if not host_matches_list(hit.url, blocked_domains)] + + hits = dedupe_hits(hits)[:MAX_RESULTS] + rendered_hits = "\n".join(f"- [{hit.title}]({hit.url})" for hit in hits) + if hits: + summary = ( + f"Search results for {query!r}. Include a Sources section in the final answer.\n" + f"{rendered_hits}" + ) + else: + summary = f"No web search results matched the query {query!r}." + + return { + "query": query, + "results": [ + summary, + { + "tool_use_id": tool_use_id, + "content": [hit.as_json() for hit in hits], + }, + ], + "durationSeconds": time.monotonic() - started, + } + + +WEB_SEARCH_TOOL_SPEC = { + "name": "web_search", + "description": "Search the web for current information and return cited results.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "minLength": 2}, + "allowed_domains": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional allowlist of domains or URLs. Subdomains match.", + }, + "blocked_domains": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional blocklist of domains or URLs. Subdomains match.", + }, + }, + "required": ["query"], + "additionalProperties": False, + }, +} + + +def _optional_string_list(arguments: dict[str, Any], key: str) -> list[str] | None: + value = arguments.get(key) + if value is None: + return None + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{key} must be an array of strings") + return value + + +async def web_search_handler( + arguments: dict[str, Any], + session: Any = None, + tool_call_id: str | None = None, + **_kw: Any, +) -> tuple[str, bool]: + query_value = arguments.get("query", "") + if not isinstance(query_value, str): + return "Error: web_search requires a query string with at least 2 characters.", False + + query = query_value.strip() + if len(query) < 2: + return "Error: web_search requires a query with at least 2 characters.", False + + try: + output = await asyncio.to_thread( + execute_web_search, + query=query, + allowed_domains=_optional_string_list(arguments, "allowed_domains"), + blocked_domains=_optional_string_list(arguments, "blocked_domains"), + tool_use_id=tool_call_id or "web_search_1", + ) + except Exception as exc: + return f"Error executing web search: {exc}", False + + return json.dumps(output, indent=2), True diff --git a/tests/unit/test_web_search_tool.py b/tests/unit/test_web_search_tool.py new file mode 100644 index 000000000..dd2434471 --- /dev/null +++ b/tests/unit/test_web_search_tool.py @@ -0,0 +1,161 @@ +import json + +import pytest + +from agent.core.tools import create_builtin_tools +from agent.tools import web_search_tool + + +class _FakeResponse: + def __init__(self, text: str, url: str = "https://html.duckduckgo.com/html/?q=x"): + self.text = text + self.url = url + + +def _content_block(output: dict): + return next(item for item in output["results"] if isinstance(item, dict))["content"] + + +def test_web_search_extracts_duckduckgo_results_and_filters_domains(monkeypatch): + seen = {} + + def fake_get(url, headers, timeout, allow_redirects): + seen.update( + { + "url": url, + "user_agent": headers["User-Agent"], + "timeout": timeout, + "allow_redirects": allow_redirects, + } + ) + return _FakeResponse( + """ + + Reqwest docs + Blocked result + + """, + url, + ) + + monkeypatch.setenv(web_search_tool.WEB_SEARCH_BASE_URL_ENV, "http://search.test/search") + monkeypatch.setattr(web_search_tool.requests, "get", fake_get) + + output = web_search_tool.execute_web_search( + "rust web search", + allowed_domains=["https://DOCS.rs/"], + blocked_domains=["HTTPS://EXAMPLE.COM"], + ) + + assert seen == { + "url": "http://search.test/search?q=rust+web+search", + "user_agent": "clawd-rust-tools/0.1", + "timeout": 20, + "allow_redirects": True, + } + assert output["query"] == "rust web search" + assert _content_block(output) == [ + {"title": "Reqwest docs", "url": "https://docs.rs/reqwest"} + ] + assert "Include a Sources section" in output["results"][0] + + +def test_web_search_decodes_duckduckgo_redirects(): + hits = web_search_tool.extract_search_hits( + """ + + Example Paper + + """ + ) + + assert hits == [ + web_search_tool.SearchHit( + title="Example Paper", + url="https://example.org/paper?x=1", + ) + ] + + +def test_web_search_generic_fallback_dedupes_and_rejects_bad_base_url(monkeypatch): + def fake_get(url, headers, timeout, allow_redirects): + return _FakeResponse( + """ + + Example One + Duplicate Example One + Tokio Docs + + """, + url, + ) + + monkeypatch.setenv(web_search_tool.WEB_SEARCH_BASE_URL_ENV, "http://search.test/fallback") + monkeypatch.setattr(web_search_tool.requests, "get", fake_get) + + output = web_search_tool.execute_web_search("generic links") + + assert _content_block(output) == [ + {"title": "Example One", "url": "https://example.com/one"}, + {"title": "Tokio Docs", "url": "https://docs.rs/tokio"}, + ] + + monkeypatch.setenv(web_search_tool.WEB_SEARCH_BASE_URL_ENV, "://bad-base-url") + with pytest.raises(ValueError): + web_search_tool.execute_web_search("generic links") + + +@pytest.mark.asyncio +async def test_web_search_handler_returns_pretty_json(monkeypatch): + to_thread_calls = [] + + async def fake_to_thread(func, /, *args, **kwargs): + to_thread_calls.append((func, args, kwargs)) + return func(*args, **kwargs) + + monkeypatch.setattr( + web_search_tool, + "execute_web_search", + lambda **kwargs: { + "query": kwargs["query"], + "results": ["No web search results matched the query 'x'.", {"content": []}], + "durationSeconds": 0.1, + }, + ) + monkeypatch.setattr(web_search_tool.asyncio, "to_thread", fake_to_thread) + + text, ok = await web_search_tool.web_search_handler({"query": "x"}) + + assert ok is False + assert "at least 2 characters" in text + + text, ok = await web_search_tool.web_search_handler( + {"query": "valid query"}, tool_call_id="call_123" + ) + + assert ok is True + parsed = json.loads(text) + assert parsed["query"] == "valid query" + assert to_thread_calls[0][0] is web_search_tool.execute_web_search + assert to_thread_calls[0][2]["tool_use_id"] == "call_123" + + text, ok = await web_search_tool.web_search_handler( + {"query": "valid query", "allowed_domains": "docs.rs"} + ) + + assert ok is False + assert "allowed_domains must be an array of strings" in text + + text, ok = await web_search_tool.web_search_handler({"query": None}) + + assert ok is False + assert "query string" in text + + +def test_web_search_is_registered_for_llm(): + tools = create_builtin_tools(local_mode=True) + specs = {tool.name: tool for tool in tools} + + assert "web_search" in specs + assert specs["web_search"].parameters["required"] == ["query"] From f9305f62019cf0e258a6c1c9300a8cb0deae6975 Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 21:00:50 +0300 Subject: [PATCH 027/120] feat(kpis): per-tool counts, research engagement, surface split (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kpis): per-tool counts, research engagement, surface split Adds intra-session telemetry to the KPI rollup so the observatory dashboard can answer "is the agent reaching for research?" and "which tools dropped out of the mix?". Production data (Apr 21->27, 21k+ sessions) shows the sessions-with-research rate dropped from ~78% on Apr 21 to ~56% on Apr 27 — without per-tool counts in the rollup, that signal was invisible to the dashboard. Also resolves the long-standing drift between this repo's scripts/build_kpis.py and the observatory's backend/build_kpis.py (both pipelines write to smolagents/ml-intern-kpis; whichever ran last dropped the other's columns). The observatory copy was the superset; this PR brings scripts/ up to it. Going forward both copies are byte-identical. Drift fix (incoming from observatory): - cost_per_session_mean / _p50 / _p95 - tool_calls_total / _succeeded / _failed counts - successful_sessions / errored_sessions / regenerated_sessions counts - sandboxes_created / _cpu / _gpu - pro_cta_by_source_json removed (was dropped by observatory; the dashboard never charted it, so no consumer to break) New fields (added in both copies via the parallel observatory PR): - research_calls, sessions_with_research - research_calls_per_session_p50/p95 (among sessions that did any) - distinct_tools_per_session_p50/p95 (vocabulary breadth) - tool_calls_per_session_p50/p95 - tool_calls_per_turn_p50/p95 - tool_calls_by_name_json, sessions_using_tool_json - sessions_by_model_json (CLI/anthropic vs frontend/bedrock) Per-tool counts come off tool_call events (data["tool"]) rather than tool_output (success-only), so the existing tool_calls_total counter is unchanged. Tests: 6 new cases in tests/unit/test_build_kpis.py covering the new private session fields, research-only-among-doers percentile, breadth/intensity aggregates, and the model split. All 19 KPI + scheduler tests pass. The matching observatory commit is e78c2d7 — visualization PRs (headline cell, "Research tool" / "Tool mix" / "Surface split" sections) live there. * fix(kpis): address review feedback Three P1s and a P2 from automated review: - Restore hf_jobs_blocked + pro_cta_clicks in _aggregate output. _session_metrics still computed both, but the aggregate had silently dropped them — dead computation and a schema regression vs the original main schema. Cheap to keep; no consumer required. - Filter zero-tool-call sessions out of distinct_tools_per_session and tool_calls_per_session percentiles, matching the existing research doer-only filter. Quiet hours full of status-check / abandoned sessions otherwise drag every median to 0. tool_calls_per_turn keeps its turns>0-only filter on purpose: a 5-turn session that did 0 tool calls is a meaningful 0 there. - Update module docstring to list every aggregate column added in this series (cost_per_session_*, tool_calls_succeeded/_failed, *_sessions outcome counts, sandboxes_*). - Drop the unreachable hardware-set check in sandbox classification: "cpu-basic"/"cpu-upgrade" both start with "cpu-", so the disjunction was dead. Simplified to a startswith check. Two new tests: - test_breadth_intensity_percentiles_exclude_zero_tool_sessions locks in the doer-only filter so it can't silently regress. - test_pro_clicks_and_blocked_jobs_in_aggregate guards the restored aggregate columns. Matching observatory commit lands its mirror of build_kpis.py. --- scripts/build_kpis.py | 149 ++++++++++++++++++++++++++++++---- tests/unit/test_build_kpis.py | 137 +++++++++++++++++++++++++++++-- 2 files changed, 262 insertions(+), 24 deletions(-) diff --git a/scripts/build_kpis.py b/scripts/build_kpis.py index 10477288e..dd62f85c0 100644 --- a/scripts/build_kpis.py +++ b/scripts/build_kpis.py @@ -38,15 +38,27 @@ llm_calls — count of llm_call events tokens_prompt / _completion / _cache_read / _cache_creation cost_usd — sum of llm_call.cost_usd + cost_per_session_mean / _p50 / _p95 — per-session cost distribution cache_hit_ratio — cache_read / (cache_read + prompt) - tool_success_rate — tool_output success=True / total tool_output - failure_rate — sessions that ended with an `error` event / sessions - regenerate_rate — sessions with any `undo_complete` event / sessions + tool_calls_total / _succeeded / _failed — per-tool_output reliability counts + tool_success_rate — succeeded / total (kept for back-compat) + successful_sessions / errored_sessions / regenerated_sessions — outcome counts + failure_rate / regenerate_rate — kept for back-compat time_to_first_action_s_p50 / _p95 — from session_start to first tool_call thumbs_up / thumbs_down hf_jobs_submitted / _succeeded / _blocked + sandboxes_created / _cpu / _gpu — sandbox_create events bucketed by hardware pro_cta_clicks gpu_hours_by_flavor_json — JSON-serialised {flavor: gpu-hours} + research_calls — total `research` tool_call events + sessions_with_research — sessions that called `research` ≄1 + research_calls_per_session_p50 / _p95 — among sessions that did any (zero-only sessions excluded) + distinct_tools_per_session_p50 / _p95 — among sessions with ≄1 named tool_call + tool_calls_per_session_p50 / _p95 — among sessions with ≄1 named tool_call + tool_calls_per_turn_p50 / _p95 — calls / turns, among sessions with turns>0 + tool_calls_by_name_json — JSON {tool: total_calls} (all tools seen) + sessions_using_tool_json — JSON {tool: distinct_sessions_using} + sessions_by_model_json — JSON {model_name: count} (CLI vs Bedrock split) ================================================================================ Usage @@ -213,6 +225,7 @@ def _session_metrics(session: dict) -> dict: "thumbs_up": 0, "thumbs_down": 0, "hf_jobs_submitted": 0, "hf_jobs_succeeded": 0, "hf_jobs_blocked": 0, "pro_cta_clicks": 0, + "sandboxes_created": 0, "sandboxes_cpu": 0, "sandboxes_gpu": 0, "first_tool_s": -1, } events = session.get("events") or [] @@ -231,11 +244,19 @@ def _session_metrics(session: dict) -> dict: gpu_hours_by_flavor: dict[str, float] = defaultdict(float) jobs_submitted = 0 jobs_succeeded = 0 - jobs_blocked = 0 thumbs_up = 0 thumbs_down = 0 + sandboxes_created = 0 + sandboxes_cpu = 0 + sandboxes_gpu = 0 + jobs_blocked = 0 pro_cta_clicks = 0 pro_cta_by_source: dict[str, int] = defaultdict(int) + # Per-tool counters from tool_call events. Counted off tool_call (which + # carries data["tool"]) rather than tool_output (which only carries + # success/output) so we can attribute calls to specific tools. + tool_calls_by_name: dict[str, int] = defaultdict(int) + total_named_tool_calls = 0 start_dt = _parse_ts(session_start) @@ -260,6 +281,10 @@ def _session_metrics(session: dict) -> dict: first_tool_ts = (ts - start_dt).total_seconds() elif et == "tool_call": + name = data.get("tool") + if name: + tool_calls_by_name[name] += 1 + total_named_tool_calls += 1 if first_tool_ts is None and ts is not None and start_dt is not None: first_tool_ts = (ts - start_dt).total_seconds() @@ -296,6 +321,19 @@ def _session_metrics(session: dict) -> dict: source = str(data.get("source") or "unknown") pro_cta_by_source[source] += 1 + elif et == "sandbox_create": + sandboxes_created += 1 + hardware = (data.get("hardware") or "").lower() + # CPU flavors are explicitly named "cpu-*". Everything else + # (including unknown/missing hardware strings) lands in the GPU + # bucket, since the auto-create default is "cpu-basic" which is + # matched here — anything that isn't is almost always an explicit + # GPU choice. + if hardware.startswith("cpu-"): + sandboxes_cpu += 1 + else: + sandboxes_gpu += 1 + out["tool_calls_total"] = tool_total out["tool_calls_success"] = tool_success out["failures"] = 1 if had_error else 0 @@ -304,12 +342,22 @@ def _session_metrics(session: dict) -> dict: out["thumbs_down"] = thumbs_down out["hf_jobs_submitted"] = jobs_submitted out["hf_jobs_succeeded"] = jobs_succeeded + out["sandboxes_created"] = sandboxes_created + out["sandboxes_cpu"] = sandboxes_cpu + out["sandboxes_gpu"] = sandboxes_gpu out["hf_jobs_blocked"] = jobs_blocked out["pro_cta_clicks"] = pro_cta_clicks out["first_tool_s"] = first_tool_ts if first_tool_ts is not None else -1 out["_gpu_hours_by_flavor"] = dict(gpu_hours_by_flavor) out["_pro_cta_by_source"] = dict(pro_cta_by_source) out["_user"] = session.get("user_id") or session.get("session_id") + # Intra-session tool fields. Underscore-prefixed = consumed by _aggregate + # only, never written to CSV directly. + out["_tool_calls_by_name"] = dict(tool_calls_by_name) + out["_research_calls"] = tool_calls_by_name.get("research", 0) + out["_distinct_tools_used"] = len(tool_calls_by_name) + out["_total_named_tool_calls"] = total_named_tool_calls + out["_model_name"] = session.get("model_name") or "unknown" return dict(out) @@ -317,12 +365,36 @@ def _aggregate(per_session: list[dict]) -> dict: """Collapse a bucket's worth of session rollups into the final KPI row.""" ttfa_values = [s["first_tool_s"] for s in per_session if s.get("first_tool_s", -1) >= 0] gpu_hours: dict[str, float] = defaultdict(float) - pro_cta_by_source: dict[str, int] = defaultdict(int) for s in per_session: for f, h in (s.get("_gpu_hours_by_flavor") or {}).items(): gpu_hours[f] += h - for source, count in (s.get("_pro_cta_by_source") or {}).items(): - pro_cta_by_source[source] += int(count) + + # Per-tool aggregates. ``sessions_using_tool`` counts each session at most + # once per tool, so the dashboard can show "how many sessions reached for + # research" alongside "how many research calls overall". + tool_calls_by_name: dict[str, int] = defaultdict(int) + sessions_using_tool: dict[str, int] = defaultdict(int) + sessions_by_model: dict[str, int] = defaultdict(int) + for s in per_session: + for name, count in (s.get("_tool_calls_by_name") or {}).items(): + tool_calls_by_name[name] += int(count) + sessions_using_tool[name] += 1 + sessions_by_model[s.get("_model_name") or "unknown"] += 1 + + # Percentile inputs. All "per session" percentiles exclude sessions that + # never reached for the relevant signal — otherwise quiet hours + # (status-check sessions, abandoned new conversations) drag every median + # to 0 and the chart tells you nothing. + research_calls_nz = [s.get("_research_calls", 0) for s in per_session if s.get("_research_calls", 0) > 0] + distinct_tools_values = [s.get("_distinct_tools_used", 0) for s in per_session if s.get("_distinct_tools_used", 0) > 0] + total_calls_values = [s.get("_total_named_tool_calls", 0) for s in per_session if s.get("_total_named_tool_calls", 0) > 0] + # Per-turn intensity: turns>0 is the natural filter here (a session with + # 5 turns and 0 tools is a meaningful 0). Don't strip those. + calls_per_turn_values = [ + s.get("_total_named_tool_calls", 0) / s["turns"] + for s in per_session + if s.get("turns", 0) > 0 + ] total_sessions = sum(s["sessions"] for s in per_session) total_turns = sum(s["turns"] for s in per_session) @@ -330,6 +402,16 @@ def _aggregate(per_session: list[dict]) -> dict: tokens_cache_read = sum(s["tokens_cache_read"] for s in per_session) tool_total = sum(s["tool_calls_total"] for s in per_session) tool_success = sum(s["tool_calls_success"] for s in per_session) + failures = int(sum(s["failures"] for s in per_session)) + regenerates = int(sum(s["regenerate_sessions"] for s in per_session)) + research_calls_total = int(sum(s.get("_research_calls", 0) for s in per_session)) + sessions_with_research = sum(1 for s in per_session if s.get("_research_calls", 0) > 0) + + # Per-session cost percentiles — chart "median session cost" alongside the + # mean so a few $700 outliers don't make you think every session is pricey. + session_costs = [float(s.get("cost_usd") or 0.0) for s in per_session] + cost_p50 = _percentile(session_costs, 0.5) + cost_p95 = _percentile(session_costs, 0.95) unique_users = {s.get("_user") for s in per_session if s.get("_user")} @@ -343,26 +425,61 @@ def _aggregate(per_session: list[dict]) -> dict: "tokens_cache_read": int(tokens_cache_read), "tokens_cache_creation": int(sum(s["tokens_cache_creation"] for s in per_session)), "cost_usd": round(sum(s["cost_usd"] for s in per_session), 4), + # Per-session cost summaries. + "cost_per_session_mean": round( + sum(s["cost_usd"] for s in per_session) / total_sessions, 6 + ) if total_sessions > 0 else 0.0, + "cost_per_session_p50": round(cost_p50, 6), + "cost_per_session_p95": round(cost_p95, 6), "cache_hit_ratio": round( tokens_cache_read / (tokens_cache_read + tokens_prompt), 4 ) if (tokens_cache_read + tokens_prompt) > 0 else 0.0, + # Raw reliability COUNTS (these are what the dashboard shows directly). + "tool_calls_total": int(tool_total), + "tool_calls_succeeded": int(tool_success), + "tool_calls_failed": int(tool_total - tool_success), + "errored_sessions": failures, + # Successful = "did not raise an error event". Mutually exclusive + # with errored_sessions; sums with errored_sessions to total sessions. + "successful_sessions": int(total_sessions - failures), + # Regenerated is an orthogonal dimension (the user retried) — a + # session can be both successful and regenerated, or both errored + # and regenerated. + "regenerated_sessions": regenerates, + # Rates kept for backwards compatibility with anything reading the + # KPI dataset directly. "tool_success_rate": round(tool_success / tool_total, 4) if tool_total > 0 else 0.0, - "failure_rate": round( - sum(s["failures"] for s in per_session) / total_sessions, 4 - ) if total_sessions > 0 else 0.0, - "regenerate_rate": round( - sum(s["regenerate_sessions"] for s in per_session) / total_sessions, 4 - ) if total_sessions > 0 else 0.0, + "failure_rate": round(failures / total_sessions, 4) if total_sessions > 0 else 0.0, + "regenerate_rate": round(regenerates / total_sessions, 4) if total_sessions > 0 else 0.0, "time_to_first_action_s_p50": round(_percentile(ttfa_values, 0.5), 2), "time_to_first_action_s_p95": round(_percentile(ttfa_values, 0.95), 2), "thumbs_up": int(sum(s["thumbs_up"] for s in per_session)), "thumbs_down": int(sum(s["thumbs_down"] for s in per_session)), "hf_jobs_submitted": int(sum(s["hf_jobs_submitted"] for s in per_session)), "hf_jobs_succeeded": int(sum(s["hf_jobs_succeeded"] for s in per_session)), - "hf_jobs_blocked": int(sum(s["hf_jobs_blocked"] for s in per_session)), - "pro_cta_clicks": int(sum(s["pro_cta_clicks"] for s in per_session)), + "sandboxes_created": int(sum(s.get("sandboxes_created", 0) for s in per_session)), + "sandboxes_cpu": int(sum(s.get("sandboxes_cpu", 0) for s in per_session)), + "sandboxes_gpu": int(sum(s.get("sandboxes_gpu", 0) for s in per_session)), + "hf_jobs_blocked": int(sum(s.get("hf_jobs_blocked", 0) for s in per_session)), + "pro_cta_clicks": int(sum(s.get("pro_cta_clicks", 0) for s in per_session)), "gpu_hours_by_flavor_json": json.dumps(dict(gpu_hours), sort_keys=True), - "pro_cta_by_source_json": json.dumps(dict(pro_cta_by_source), sort_keys=True), + # Research KPIs — answer "is the agent reaching for research?". + "research_calls": research_calls_total, + "sessions_with_research": int(sessions_with_research), + "research_calls_per_session_p50": round(_percentile(research_calls_nz, 0.5), 2), + "research_calls_per_session_p95": round(_percentile(research_calls_nz, 0.95), 2), + # Intra-session breadth + intensity. p50 + p95 over per-session values. + "distinct_tools_per_session_p50": round(_percentile(distinct_tools_values, 0.5), 2), + "distinct_tools_per_session_p95": round(_percentile(distinct_tools_values, 0.95), 2), + "tool_calls_per_session_p50": round(_percentile(total_calls_values, 0.5), 2), + "tool_calls_per_session_p95": round(_percentile(total_calls_values, 0.95), 2), + "tool_calls_per_turn_p50": round(_percentile(calls_per_turn_values, 0.5), 2), + "tool_calls_per_turn_p95": round(_percentile(calls_per_turn_values, 0.95), 2), + # JSON columns let the dashboard add/remove tools without schema churn. + "tool_calls_by_name_json": json.dumps(dict(tool_calls_by_name), sort_keys=True), + "sessions_using_tool_json": json.dumps(dict(sessions_using_tool), sort_keys=True), + # Surface split — answers "is research dropping on Bedrock specifically?". + "sessions_by_model_json": json.dumps(dict(sessions_by_model), sort_keys=True), } diff --git a/tests/unit/test_build_kpis.py b/tests/unit/test_build_kpis.py index 5edefc572..6efba2366 100644 --- a/tests/unit/test_build_kpis.py +++ b/tests/unit/test_build_kpis.py @@ -136,20 +136,141 @@ def test_aggregate_day_cache_hit_and_users(): assert abs(row["cost_usd"] - 1.5) < 1e-9 -def test_aggregate_day_sums_pro_click_sources(): +def test_per_tool_counts_in_session_metrics(): + mod = _load() + events = [ + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "research"}), + _ev("tool_call", {"tool": "read"}), + _ev("tool_call", {}), # nameless tool_call must be ignored + ] + m = mod._session_metrics(_session(events, user_id="u1")) + assert m["_tool_calls_by_name"] == {"bash": 2, "research": 1, "read": 1} + assert m["_research_calls"] == 1 + assert m["_distinct_tools_used"] == 3 + assert m["_total_named_tool_calls"] == 4 + assert m["_model_name"] == "claude-opus-4-6" + + +def test_aggregate_research_kpis_only_count_doer_sessions(): mod = _load() s1 = mod._session_metrics(_session([ - _ev("pro_cta_click", {"source": "hf_jobs_upgrade_dialog"}), - _ev("pro_cta_click", {"source": "hf_jobs_upgrade_dialog"}), + _ev("tool_call", {"tool": "research"}), + _ev("tool_call", {"tool": "research"}), + _ev("tool_call", {"tool": "research"}), ], user_id="u1")) s2 = mod._session_metrics(_session([ + _ev("tool_call", {"tool": "research"}), + ], user_id="u2")) + s3 = mod._session_metrics(_session([ + _ev("tool_call", {"tool": "bash"}), + ], user_id="u3")) + row = mod._aggregate([s1, s2, s3]) + assert row["sessions"] == 3 + assert row["sessions_with_research"] == 2 + assert row["research_calls"] == 4 + # Median among sessions that did any research = (1, 3) -> 2.0 + assert row["research_calls_per_session_p50"] == 2.0 + + +def test_aggregate_tool_breadth_and_intensity(): + import json as _json + mod = _load() + s1 = mod._session_metrics(_session([ + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "research"}), + ], user_id="u1")) + # Two user turns so calls/turn = 4/2 = 2 + s2 = _session([ + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "edit"}), + _ev("tool_call", {"tool": "edit"}), + ], user_id="u2") + s2["messages"] = [{"role": "user"}, {"role": "user"}] + s2_metrics = mod._session_metrics(s2) + row = mod._aggregate([s1, s2_metrics]) + assert _json.loads(row["tool_calls_by_name_json"]) == { + "bash": 3, "research": 1, "edit": 2, + } + assert _json.loads(row["sessions_using_tool_json"]) == { + "bash": 2, "research": 1, "edit": 1, + } + # u1: 2 distinct, u2: 2 distinct -> p50 = 2 + assert row["distinct_tools_per_session_p50"] == 2.0 + # tool_calls_per_session: u1=2, u2=4 -> p50=3 + assert row["tool_calls_per_session_p50"] == 3.0 + # u1: 2 turns(?) — _session() default has one user message, so calls/turn=2/1=2; u2=4/2=2 + assert row["tool_calls_per_turn_p50"] == 2.0 + + +def test_breadth_intensity_percentiles_exclude_zero_tool_sessions(): + """Sessions that never called a tool would otherwise crush the median.""" + mod = _load() + # Two productive sessions and three idle ones (no tool calls). Without + # the doer-only filter, median of [0,0,0,2,4] = 0, which is useless. + productive_a = mod._session_metrics(_session([ + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "research"}), + ], user_id="prod_a")) + productive_b = _session([ + _ev("tool_call", {"tool": "bash"}), + _ev("tool_call", {"tool": "edit"}), + _ev("tool_call", {"tool": "edit"}), + _ev("tool_call", {"tool": "edit"}), + ], user_id="prod_b") + productive_b["messages"] = [{"role": "user"}, {"role": "user"}] + productive_b_metrics = mod._session_metrics(productive_b) + idle = [ + mod._session_metrics(_session([], user_id="idle_a")), + mod._session_metrics(_session([], user_id="idle_b")), + mod._session_metrics(_session([], user_id="idle_c")), + ] + row = mod._aggregate([productive_a, productive_b_metrics, *idle]) + # Median of [2 distinct, 2 distinct] = 2 (idle sessions filtered). + assert row["distinct_tools_per_session_p50"] == 2.0 + # Median of [2 calls, 4 calls] = 3 (idle sessions filtered). + assert row["tool_calls_per_session_p50"] == 3.0 + + +def test_pro_clicks_and_blocked_jobs_in_aggregate(): + """The aggregate row keeps pro_cta_clicks + hf_jobs_blocked columns + even if the dashboard doesn't currently chart them — they're cheap to + keep and downstream consumers may still depend on the schema.""" + mod = _load() + s1 = mod._session_metrics(_session([ + _ev("pro_cta_click", {"source": "hf_jobs_upgrade_dialog"}), _ev("pro_cta_click", {"source": "claude_cap_dialog"}), + _ev("jobs_access_blocked", {}), + ], user_id="u1")) + s2 = mod._session_metrics(_session([ + _ev("jobs_access_blocked", {}), + _ev("jobs_access_blocked", {}), ], user_id="u2")) - row = mod._aggregate_day([s1, s2]) - assert row["pro_cta_clicks"] == 3 - assert row["pro_cta_by_source_json"] == ( - '{"claude_cap_dialog": 1, "hf_jobs_upgrade_dialog": 2}' - ) + row = mod._aggregate([s1, s2]) + assert row["pro_cta_clicks"] == 2 + assert row["hf_jobs_blocked"] == 3 + + +def test_aggregate_sessions_by_model_split(): + import json as _json + mod = _load() + s_anthropic = _session([], user_id="a") + s_anthropic["model_name"] = "anthropic/claude-opus-4-6" + s_bedrock = _session([], user_id="b") + s_bedrock["model_name"] = "bedrock/us.anthropic.claude-opus-4-6-v1" + s_bedrock2 = _session([], user_id="c") + s_bedrock2["model_name"] = "bedrock/us.anthropic.claude-opus-4-6-v1" + row = mod._aggregate([ + mod._session_metrics(s_anthropic), + mod._session_metrics(s_bedrock), + mod._session_metrics(s_bedrock2), + ]) + assert _json.loads(row["sessions_by_model_json"]) == { + "anthropic/claude-opus-4-6": 1, + "bedrock/us.anthropic.claude-opus-4-6-v1": 2, + } def test_failure_and_regenerate_rates(): From 6131fc89049efe658b501b7a5b58c4791db62c6a Mon Sep 17 00:00:00 2001 From: Aksel Joonas Reedi <125026660+akseljoonas@users.noreply.github.com> Date: Mon, 27 Apr 2026 22:40:14 +0300 Subject: [PATCH 028/120] Embed trackio dashboard in chat for jobs and sandboxes (#161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Embed trackio dashboard in chat for jobs and sandboxes Wires `trackio_space_id` / `trackio_project` through hf_jobs and sandbox_create as optional tool args, then renders the resulting dashboard inline as an embedded iframe in the chat. The plumbing turned out to need four pieces, not two: 1. Inject TRACKIO_SPACE_ID / TRACKIO_PROJECT into job + sandbox env. 2. Surface the values in tool_state_change so the frontend can render the embed when the tool call is in flight. 3. Pre-seed the Space with our own README/requirements/app.py — without this, trackio's init() against an existing empty Space leaves the iframe stuck on the default Gradio template. We omit `hf_oauth: true` from the seeded README so the embed renders without a sign-in click; per-user privacy comes from HF namespace ownership. 4. Mount the metrics bucket on the dashboard Space at /data and set TRACKIO_DIR / TRACKIO_BUCKET_ID. trackio.init() in the job creates the bucket but only attaches it to the job process — without mounting it on the Space too, the dashboard reads an empty /data and shows "No projects" even though the bucket has data. The seed helper (`agent/tools/trackio_seed.py`) is idempotent and runs synchronously before run_job / sandbox_create whenever a trackio_space_id is provided. Bumps `huggingface-hub` to >= 1.12.0 for the bucket / volume APIs (`create_bucket`, `Volume`, `set_space_volumes`, `add_space_variable`). Validated end-to-end with a real trackio job against akseljoonas/mlintern-trackio-pragmatic-01 — dashboard renders project, runs, and loss/accuracy charts with no sign-in prompt. * Brand seeded trackio dashboards with the ml-intern logo Sets TRACKIO_LOGO_LIGHT_URL / TRACKIO_LOGO_DARK_URL on every seeded Space to the smolagents.webp hosted on the smolagents/ml-intern Space, so the dashboard renders ml-intern branding instead of the trackio wordmark. Idempotent — only writes the variable when the value would change. TRACKIO_THEME deliberately not set: trackio dropped Gradio-theme support in 1.x, so the variable is a no-op there. * Show loading state in trackio embed until iframe paints A freshly-seeded HF Space takes 30–60s to build, and even after build the trackio bundle takes a few seconds to render its first frame — during that whole window the iframe was a blank white rectangle with no signal that anything was happening. Stack a 'Spinning up the dashboard…' overlay (spinner + short hint about typical build time) on top of the iframe and clear it once the iframe's load event fires. * Match trackio embed loading state to chat dark theme - Force trackio dashboard into dark mode via gradio's __theme=dark query param so the embedded charts blend with the chat instead of flashing a white panel after load. - Switch the iframe container's loading background from white to var(--code-panel-bg) so any frame between iframe paint and load uses the same dark surface as the surrounding chat. - Reword the placeholder copy to talk about the trackio dashboard rather than HF Spaces — that's the abstraction users care about. * Sandbox trackio iframe, validate space ID, persist dashboards Address PR review: - iframe gets a sandbox attribute so the embedded gradio app can't reach back into the parent (forms, scripts, same-origin, popups, downloads, modals are all the runtime needs). - spaceIdToSubdomain only runs on validated repo IDs; an unexpected value now suppresses the embed instead of building a malformed URL that could redirect to an attacker-controlled subdomain. - trackioDashboards mirrors toolErrors/rejectedTools and persists to localStorage so the dashboard survives a page refresh during a run. --- agent/tools/jobs_tool.py | 106 +++++++-- agent/tools/sandbox_tool.py | 91 +++++++- agent/tools/trackio_seed.py | 205 ++++++++++++++++++ .../src/components/Chat/ToolCallGroup.tsx | 202 ++++++++++++++++- frontend/src/lib/sse-chat-transport.ts | 5 + frontend/src/store/agentStore.ts | 49 +++++ pyproject.toml | 2 +- uv.lock | 76 ++++--- 8 files changed, 673 insertions(+), 63 deletions(-) create mode 100644 agent/tools/trackio_seed.py diff --git a/agent/tools/jobs_tool.py b/agent/tools/jobs_tool.py index c18d47e29..6518fa3cb 100644 --- a/agent/tools/jobs_tool.py +++ b/agent/tools/jobs_tool.py @@ -19,6 +19,7 @@ from agent.core.hf_access import JobsAccessError, resolve_jobs_namespace from agent.core.session import Event +from agent.tools.trackio_seed import ensure_trackio_dashboard from agent.tools.types import ToolResult logger = logging.getLogger(__name__) @@ -382,6 +383,31 @@ async def execute(self, params: Dict[str, Any]) -> ToolResult: "isError": True, } + async def _seed_trackio_dashboard(self, space_id: str) -> None: + """Idempotently install trackio dashboard files into *space_id* before + the job runs. Surfaces seed progress as tool_log events but never + raises — a seed failure should not block job submission, since trackio + often still works when the Space already has dashboard code from a + previous run. + """ + loop = asyncio.get_running_loop() + + def _log(msg: str) -> None: + if self.session is None: + return + loop.call_soon_threadsafe( + self.session.event_queue.put_nowait, + Event(event_type="tool_log", data={"tool": "hf_jobs", "log": msg}), + ) + + try: + await asyncio.to_thread( + ensure_trackio_dashboard, space_id, self.hf_token, _log + ) + except Exception as e: + logger.warning(f"trackio dashboard seed failed for {space_id}: {e}") + _log(f"trackio dashboard seed failed: {e}") + async def _wait_for_job_completion( self, job_id: str, namespace: Optional[str] = None ) -> tuple[str, list[str]]: @@ -533,11 +559,24 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: # Run the job flavor = args.get("hardware_flavor", "cpu-basic") timeout_str = args.get("timeout", "30m") + + # Trackio: agent-declared space + project become env vars on the job + # so trackio.init() picks them up automatically. We also surface them + # in tool_state_change so the frontend can embed the dashboard. + env_dict = _add_default_env(args.get("env")) + trackio_space_id = args.get("trackio_space_id") + trackio_project = args.get("trackio_project") + if trackio_space_id: + env_dict["TRACKIO_SPACE_ID"] = trackio_space_id + await self._seed_trackio_dashboard(trackio_space_id) + if trackio_project: + env_dict["TRACKIO_PROJECT"] = trackio_project + job = await _async_call( self.api.run_job, image=image, command=command, - env=_add_default_env(args.get("env")), + env=env_dict, secrets=_add_environment_variables(args.get("secrets"), self.hf_token), flavor=flavor, timeout=timeout_str, @@ -550,16 +589,18 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: # Send job URL immediately after job creation (before waiting for completion) if self.session and self.tool_call_id: + state_data: Dict[str, Any] = { + "tool_call_id": self.tool_call_id, + "tool": "hf_jobs", + "state": "running", + "jobUrl": job.url, + } + if trackio_space_id: + state_data["trackioSpaceId"] = trackio_space_id + if trackio_project: + state_data["trackioProject"] = trackio_project await self.session.send_event( - Event( - event_type="tool_state_change", - data={ - "tool_call_id": self.tool_call_id, - "tool": "hf_jobs", - "state": "running", - "jobUrl": job.url, - }, - ) + Event(event_type="tool_state_change", data=state_data) ) # Telemetry: job submission + completion (infra consumption signal). @@ -594,16 +635,18 @@ async def _run_job(self, args: Dict[str, Any]) -> ToolResult: # Notify frontend of final status if self.session and self.tool_call_id: + final_data: Dict[str, Any] = { + "tool_call_id": self.tool_call_id, + "tool": "hf_jobs", + "state": final_status.lower(), + "jobUrl": job.url, + } + if trackio_space_id: + final_data["trackioSpaceId"] = trackio_space_id + if trackio_project: + final_data["trackioProject"] = trackio_project await self.session.send_event( - Event( - event_type="tool_state_change", - data={ - "tool_call_id": self.tool_call_id, - "tool": "hf_jobs", - "state": final_status.lower(), - "jobUrl": job.url, - }, - ) + Event(event_type="tool_state_change", data=final_data) ) # Filter out UV package installation output @@ -977,7 +1020,10 @@ async def _resume_scheduled_job(self, args: Dict[str, Any]) -> ToolResult: "- You MUST have validated dataset format via hf_inspect_dataset or hub_repo_details.\n" "- Training config MUST include push_to_hub=True and hub_model_id. " "Job storage is EPHEMERAL — all files are deleted when the job ends. Without push_to_hub, trained models are lost permanently.\n" - "- Include trackio monitoring and provide the dashboard URL to the user.\n\n" + "- Include trackio monitoring and provide the dashboard URL to the user. " + "When the script uses report_to='trackio', also pass `trackio_space_id` " + "(e.g. '/mlintern-<8char>') and `trackio_project` as tool args — " + "they are injected as TRACKIO_SPACE_ID/TRACKIO_PROJECT env vars and let the UI embed the live dashboard.\n\n" "BATCH/ABLATION JOBS: Submit ONE job first. Check logs to confirm it starts training successfully. " "Only then submit the remaining jobs. Never submit all at once — if there's a bug, all jobs fail.\n\n" "Operations: run, ps, logs, inspect, cancel, scheduled run/ps/inspect/delete/suspend/resume.\n\n" @@ -1060,6 +1106,26 @@ async def _resume_scheduled_job(self, args: Dict[str, Any]) -> ToolResult: "type": "object", "description": "Environment variables {'KEY': 'VALUE'}. HF_TOKEN is auto-included.", }, + "trackio_space_id": { + "type": "string", + "description": ( + "Optional. The HF Space hosting the trackio dashboard for this run " + "(e.g. '/mlintern-<8char>', under YOUR HF namespace). " + "Injected as TRACKIO_SPACE_ID env var and used by the UI to embed " + "the live dashboard. Set this whenever the script uses " + "report_to='trackio'. The Space is auto-created and seeded with the " + "trackio dashboard before the job starts — DO NOT pre-create it via " + "hf_repo_git, that produces an empty Space that breaks the embed." + ), + }, + "trackio_project": { + "type": "string", + "description": ( + "Optional. The trackio project name to log this run under. " + "Injected as TRACKIO_PROJECT env var and used by the UI to filter " + "the embedded dashboard to this project." + ), + }, "namespace": { "type": "string", "description": ( diff --git a/agent/tools/sandbox_tool.py b/agent/tools/sandbox_tool.py index 6dfd3db19..8ff2bbcd5 100644 --- a/agent/tools/sandbox_tool.py +++ b/agent/tools/sandbox_tool.py @@ -19,6 +19,7 @@ from agent.core.session import Event from agent.tools.sandbox_client import Sandbox +from agent.tools.trackio_seed import ensure_trackio_dashboard def _looks_like_path(script: str) -> bool: @@ -62,11 +63,36 @@ async def resolve_sandbox_script( return None, f"Failed to read {script} from sandbox: {e}" +async def _seed_trackio_dashboard_safe(session: Any, space_id: str) -> None: + """Idempotently seed *space_id* with trackio dashboard files using the + session's HF token. Logs progress, swallows errors — a failed seed should + not block sandbox creation.""" + if not session or not getattr(session, "hf_token", None): + return + loop = asyncio.get_running_loop() + + def _log(msg: str) -> None: + loop.call_soon_threadsafe( + session.event_queue.put_nowait, + Event(event_type="tool_log", data={"tool": "sandbox_create", "log": msg}), + ) + + try: + await asyncio.to_thread( + ensure_trackio_dashboard, space_id, session.hf_token, _log + ) + except Exception as e: + _log(f"trackio dashboard seed failed: {e}") + + # ── Tool name mapping (short agent names → Sandbox client names) ────── async def _ensure_sandbox( - session: Any, hardware: str = "cpu-basic", **create_kwargs + session: Any, + hardware: str = "cpu-basic", + extra_secrets: dict[str, str] | None = None, + **create_kwargs, ) -> tuple[Sandbox | None, str | None]: """ Ensure a sandbox exists on the session. Auto-creates with given hardware if needed. @@ -120,11 +146,15 @@ async def _watch_cancel(): watcher_task = asyncio.create_task(_watch_cancel()) + secrets: dict[str, str] = {"HF_TOKEN": token} + if extra_secrets: + secrets.update({k: v for k, v in extra_secrets.items() if v}) + kwargs = { "owner": owner, "hardware": hardware, "token": token, - "secrets": {"HF_TOKEN": token}, + "secrets": secrets, "log": _log, "cancel_event": cancel_flag, **create_kwargs, @@ -188,6 +218,9 @@ async def _watch_cancel(): "fp32 ā‰ˆ 4 bytes/param, plus ~20% overhead for optimizer states during training.\n" "Common picks: t4-small (16GB VRAM, fits ≤1-3B), a10g-small (24GB, ≤7B), a100-large (80GB, ≤30B). " "If the model won't fit, pick larger hardware upfront — OOM on a sandbox wastes time.\n\n" + "If you intend to run a training script in this sandbox that uses report_to='trackio', " + "pass `trackio_space_id` (e.g. '/mlintern-<8char>') and `trackio_project` so they " + "are set as TRACKIO_SPACE_ID/TRACKIO_PROJECT secrets in the sandbox and the UI can embed the live dashboard.\n\n" "Hardware: " + ", ".join([e.value for e in SpaceHardware]) + ".\n" ), "parameters": { @@ -204,16 +237,49 @@ async def _watch_cancel(): "type": "boolean", "description": "If true, create a private Space", }, + "trackio_space_id": { + "type": "string", + "description": ( + "Optional. The HF Space hosting the trackio dashboard for runs in this sandbox " + "(e.g. '/mlintern-<8char>', under YOUR HF namespace). Injected as " + "TRACKIO_SPACE_ID secret and surfaced to the UI. The Space is auto-created and " + "seeded with the trackio dashboard — DO NOT pre-create it via hf_repo_git, " + "that produces an empty Space that breaks the embed." + ), + }, + "trackio_project": { + "type": "string", + "description": ( + "Optional. The trackio project name. Injected as TRACKIO_PROJECT secret and " + "used by the UI to filter the embedded dashboard to this project." + ), + }, }, }, } async def sandbox_create_handler( - args: dict[str, Any], session: Any = None + args: dict[str, Any], session: Any = None, tool_call_id: str | None = None ) -> tuple[str, bool]: """Handle sandbox_create tool calls.""" hardware = args.get("hardware", "cpu-basic") + trackio_space_id = args.get("trackio_space_id") or None + trackio_project = args.get("trackio_project") or None + + async def _emit_trackio_state(sb: Sandbox) -> None: + """Tell the frontend which trackio dashboard to embed for this sandbox.""" + if not (session and tool_call_id and trackio_space_id): + return + data: dict[str, Any] = { + "tool_call_id": tool_call_id, + "tool": "sandbox_create", + "state": "running", + "trackioSpaceId": trackio_space_id, + } + if trackio_project: + data["trackioProject"] = trackio_project + await session.send_event(Event(event_type="tool_state_change", data=data)) # If sandbox already exists, return its info if session and getattr(session, "sandbox", None): @@ -226,6 +292,7 @@ async def sandbox_create_handler( "Hardware cannot be changed by calling sandbox_create again. " "Delete the existing sandbox first if you need a different tier." ) + await _emit_trackio_state(sb) return ( f"Sandbox already active: {sb.space_id}\n" f"URL: {sb.url}\n" @@ -233,18 +300,32 @@ async def sandbox_create_handler( f"Use bash/read/write/edit to interact with it." ), True - create_kwargs = {} + create_kwargs: dict[str, Any] = {} if "private" in args: create_kwargs["private"] = args["private"] + extra_secrets: dict[str, str] = {} + if trackio_space_id: + extra_secrets["TRACKIO_SPACE_ID"] = trackio_space_id + await _seed_trackio_dashboard_safe(session, trackio_space_id) + if trackio_project: + extra_secrets["TRACKIO_PROJECT"] = trackio_project + try: - sb, error = await _ensure_sandbox(session, hardware=hardware, **create_kwargs) + sb, error = await _ensure_sandbox( + session, + hardware=hardware, + extra_secrets=extra_secrets or None, + **create_kwargs, + ) except Exception as e: return f"Failed to create sandbox: {e}", False if error: return error, False + await _emit_trackio_state(sb) + return ( f"Sandbox created: {sb.space_id}\n" f"URL: {sb.url}\n" diff --git a/agent/tools/trackio_seed.py b/agent/tools/trackio_seed.py new file mode 100644 index 000000000..1062e1b5e --- /dev/null +++ b/agent/tools/trackio_seed.py @@ -0,0 +1,205 @@ +"""Seed an HF Space with the trackio dashboard. + +Background: when the agent creates a Space via `hf_repo_git create_repo` (or +the user pre-creates one), it ships with no app.py — so the iframe shows the +default Gradio "Get started" template instead of charts. Trackio's `init()` +detects the existing Space but does NOT auto-bootstrap dashboard files into it, +so the dashboard never materializes. + +This helper writes the three files trackio's runtime expects (README.md, +requirements.txt, app.py) into the Space, idempotently, BEFORE the job that +will call `trackio.init()` runs. We deliberately omit `hf_oauth: true` from +the README so the embedded iframe in ml-intern renders without a login click — +per-user privacy is enforced by namespace ownership instead. + +Beyond the dashboard files, the helper also creates the metrics bucket and +mounts it on the Space at `/data` (with `TRACKIO_DIR` / `TRACKIO_BUCKET_ID` +Space variables). Without this, the running job writes metrics into a bucket +that the dashboard Space can't read, and the iframe shows "No projects". +""" + +from __future__ import annotations + +import io +from typing import Callable, Optional + +from huggingface_hub import ( + HfApi, + Volume, + add_space_variable, + create_bucket, + create_repo, +) +from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError + + +_README = """--- +title: Trackio Dashboard +emoji: šŸ“Š +colorFrom: pink +colorTo: gray +sdk: gradio +app_file: app.py +pinned: false +tags: + - trackio +--- + +Embedded trackio dashboard for ml-intern runs. +""" + +_REQUIREMENTS = "trackio\n" +_APP_PY = "import trackio\ntrackio.show()\n" + +# ml-intern brand mark surfaced inside the trackio dashboard. Trackio reads +# `TRACKIO_LOGO_LIGHT_URL` / `TRACKIO_LOGO_DARK_URL` from Space variables and +# renders them in place of its own logo. We point at the publicly-resolvable +# copy on the smolagents/ml-intern Space repo so any seeded dashboard inherits +# the ml-intern branding without each user having to host the asset. +_LOGO_URL = ( + "https://huggingface.co/spaces/smolagents/ml-intern/" + "resolve/main/frontend/public/smolagents.webp" +) + +_FILES = { + "README.md": _README, + "requirements.txt": _REQUIREMENTS, + "app.py": _APP_PY, +} + + +def _already_seeded(api: HfApi, space_id: str) -> bool: + """Cheap check: does the Space already have a trackio dashboard app.py? + + Avoids re-uploading the same three files on every job submission. We look + for the literal `trackio.show` call which is the load-bearing line — any + other app.py shape (the default gradio shell, a stale custom one) means + we should re-seed. + """ + try: + path = api.hf_hub_download( + repo_id=space_id, repo_type="space", filename="app.py" + ) + except (EntryNotFoundError, RepositoryNotFoundError, OSError): + return False + try: + with open(path, "r", encoding="utf-8") as f: + return "trackio.show" in f.read() + except OSError: + return False + + +def _get_space_volumes(api: HfApi, space_id: str) -> list: + """Return mounted volumes for a Space. + + `get_space_runtime()` doesn't always populate `volumes` even when the + mount exists; mirror trackio's fallback to `space_info().runtime.volumes`. + """ + runtime = api.get_space_runtime(space_id) + if getattr(runtime, "volumes", None): + return list(runtime.volumes) + info = api.space_info(space_id) + if info.runtime and getattr(info.runtime, "volumes", None): + return list(info.runtime.volumes) + return [] + + +def _ensure_bucket_mounted( + api: HfApi, + space_id: str, + bucket_id: str, + hf_token: str, + log: Optional[Callable[[str], None]] = None, +) -> None: + """Create the bucket if missing, mount it at `/data` on the Space, and + set the `TRACKIO_DIR` / `TRACKIO_BUCKET_ID` Space variables. Idempotent — + skips work that has already been done. + """ + create_bucket(bucket_id, private=True, exist_ok=True, token=hf_token) + + existing = _get_space_volumes(api, space_id) + already_mounted = any( + getattr(v, "type", None) == "bucket" + and getattr(v, "source", None) == bucket_id + and getattr(v, "mount_path", None) == "/data" + for v in existing + ) + if not already_mounted: + preserved = [ + v + for v in existing + if not ( + getattr(v, "type", None) == "bucket" + and ( + getattr(v, "source", None) == bucket_id + or getattr(v, "mount_path", None) == "/data" + ) + ) + ] + api.set_space_volumes( + space_id, + preserved + [Volume(type="bucket", source=bucket_id, mount_path="/data")], + ) + if log: + log(f"mounted bucket {bucket_id} at /data on {space_id}") + + variables = api.get_space_variables(space_id) + desired = { + "TRACKIO_DIR": "/data/trackio", + "TRACKIO_BUCKET_ID": bucket_id, + "TRACKIO_LOGO_LIGHT_URL": _LOGO_URL, + "TRACKIO_LOGO_DARK_URL": _LOGO_URL, + } + for key, value in desired.items(): + if getattr(variables.get(key), "value", None) != value: + add_space_variable(space_id, key, value, token=hf_token) + + +def ensure_trackio_dashboard( + space_id: str, + hf_token: str, + log: Optional[Callable[[str], None]] = None, +) -> bool: + """Make sure *space_id* is fully wired for trackio: + 1. Space exists with our dashboard files (README without `hf_oauth`, + `requirements.txt`, `app.py` calling `trackio.show`). + 2. Bucket `-bucket` exists, is mounted at `/data`, and the + Space has `TRACKIO_DIR` / `TRACKIO_BUCKET_ID` variables set. + + Idempotent — re-running is cheap. Returns True if any seeding happened + in step (1), False if the dashboard files were already in place. Bucket + mount is always re-checked. + """ + api = HfApi(token=hf_token) + + create_repo( + repo_id=space_id, + repo_type="space", + space_sdk="gradio", + exist_ok=True, + token=hf_token, + ) + + seeded_files = False + if _already_seeded(api, space_id): + if log: + log(f"trackio dashboard already seeded on {space_id}") + else: + if log: + log(f"seeding trackio dashboard files into {space_id}") + for path_in_repo, content in _FILES.items(): + api.upload_file( + path_or_fileobj=io.BytesIO(content.encode("utf-8")), + path_in_repo=path_in_repo, + repo_id=space_id, + repo_type="space", + commit_message=f"ml-intern: seed trackio dashboard ({path_in_repo})", + ) + seeded_files = True + + bucket_id = f"{space_id}-bucket" + _ensure_bucket_mounted(api, space_id, bucket_id, hf_token, log) + + if log: + log(f"trackio dashboard ready: https://huggingface.co/spaces/{space_id}") + return seeded_files diff --git a/frontend/src/components/Chat/ToolCallGroup.tsx b/frontend/src/components/Chat/ToolCallGroup.tsx index fc9fe35c1..657e9e368 100644 --- a/frontend/src/components/Chat/ToolCallGroup.tsx +++ b/frontend/src/components/Chat/ToolCallGroup.tsx @@ -220,6 +220,194 @@ function ResearchSteps({ steps }: { steps: string[] }) { ); } +// --------------------------------------------------------------------------- +// Trackio dashboard embed +// --------------------------------------------------------------------------- + +// HF repo IDs are `/` where each segment is alphanumerics plus +// `_`, `.`, `-`. Anything else (slashes, spaces, query params, missing owner) +// would let an attacker-controlled string redirect the embed to a different +// Space, so we refuse to render rather than build a malformed URL. +const SPACE_ID_PATTERN = /^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/; + +function isValidSpaceId(spaceId: string): boolean { + return SPACE_ID_PATTERN.test(spaceId); +} + +/** HF Space embed subdomain: 'user/space_name' → 'user-space-name'. */ +function spaceIdToSubdomain(spaceId: string): string { + return spaceId + .toLowerCase() + .replace(/[/_.]/g, '-') + .replace(/-+/g, '-') + .replace(/^-|-$/g, ''); +} + +function buildTrackioEmbedUrl(spaceId: string, project?: string): string { + // __theme=dark is gradio's standard query param to force the embedded + // dashboard into dark mode so it blends with the surrounding chat instead + // of flashing a bright white panel inside the dark UI. + const params = new URLSearchParams({ + sidebar: 'hidden', + footer: 'false', + __theme: 'dark', + }); + if (project) params.set('project', project); + return `https://${spaceIdToSubdomain(spaceId)}.hf.space/?${params.toString()}`; +} + +function buildTrackioPageUrl(spaceId: string, project?: string): string { + const qs = project ? `?${new URLSearchParams({ project }).toString()}` : ''; + return `https://huggingface.co/spaces/${spaceId}${qs}`; +} + +function TrackioEmbed({ spaceId, project }: { spaceId: string; project?: string }) { + const [expanded, setExpanded] = useState(true); + const [iframeLoaded, setIframeLoaded] = useState(false); + const embedUrl = useMemo(() => buildTrackioEmbedUrl(spaceId, project), [spaceId, project]); + const pageUrl = useMemo(() => buildTrackioPageUrl(spaceId, project), [spaceId, project]); + const label = project ? `${spaceId} Ā· ${project}` : spaceId; + + if (!isValidSpaceId(spaceId)) return null; + + return ( + + + e.stopPropagation()} + sx={{ + px: 1.25, + py: 0.5, + borderBottom: expanded ? '1px solid var(--tool-border)' : 'none', + }} + > + + trackio + + + {label} + + e.stopPropagation()} + sx={{ + display: 'inline-flex', + alignItems: 'center', + gap: 0.4, + color: 'var(--accent-yellow)', + fontSize: '0.65rem', + textDecoration: 'none', + '&:hover': { textDecoration: 'underline' }, + }} + > + + Open + + + + {expanded && ( + +