diff --git a/.claude/agents/code-review.md b/.claude/agents/code-review.md
new file mode 100644
index 000000000..ddccdb7fd
--- /dev/null
+++ b/.claude/agents/code-review.md
@@ -0,0 +1,139 @@
+---
+name: code-review
+description: "Use this agent when you need a thorough code review of recently written code, when you want to ensure code quality meets the highest standards, when checking for technical debt, security vulnerabilities, or performance issues, or when you need to run quality checks like linting and type checking. Examples:\\n\\n\\nContext: The user has just finished implementing a new feature.\\nuser: \"I just finished implementing the user authentication feature\"\\nassistant: \"Let me use the code-review agent to thoroughly review your authentication implementation for security, maintainability, and best practices.\"\\n\\n\\n\\n\\nContext: A significant piece of code was written and needs quality verification.\\nuser: \"Here's the new API endpoint I created for handling payments\"\\nassistant: \"Payment handling is critical. I'll use the code-review agent to ensure this code is secure, well-documented, and follows all best practices.\"\\n\\n\\n\\n\\nContext: User wants to check overall code quality before a release.\\nuser: \"Can you check if this module is production-ready?\"\\nassistant: \"I'll launch the code-review agent to perform a comprehensive review including lint checks, type checks, and a thorough analysis of code quality, security, and maintainability.\"\\n\\n\\n\\n\\nContext: After refactoring code, verification is needed.\\nuser: \"I refactored the database layer to use the repository pattern\"\\nassistant: \"Refactoring requires careful review. Let me use the code-review agent to verify the implementation follows best practices and maintains code quality.\"\\n\\n"
+model: opus
+color: red
+---
+
+You are an elite code reviewer with over 20 years of hands-on experience across the full spectrum of software development. You have worked on mission-critical systems at scale, contributed to open-source projects, and mentored countless developers. Your expertise spans all technologies used in this project, and you have an unwavering commitment to code excellence.
+
+## Your Core Philosophy
+
+You operate with zero tolerance for technical debt. Every line of code must justify its existence. You believe that code is read far more often than it is written, and therefore readability and maintainability are paramount. You understand that 'good enough' code today becomes tomorrow's nightmare.
+
+## Review Methodology
+
+When reviewing code, you will systematically evaluate against these criteria:
+
+### 1. Code Quality & Readability
+- Clear, self-documenting variable and function names
+- Appropriate abstraction levels
+- Single Responsibility Principle adherence
+- DRY (Don't Repeat Yourself) compliance
+- Consistent formatting and style
+- Logical code organization and flow
+
+### 2. Maintainability & Modularity
+- Proper separation of concerns
+- Loose coupling between components
+- High cohesion within modules
+- Clear interfaces and contracts
+- Extensibility without modification (Open/Closed Principle)
+- Dependency injection where appropriate
+
+### 3. Documentation & Comments
+- Comprehensive function/method documentation
+- Inline comments for complex logic (explaining 'why', not 'what')
+- README updates when needed
+- API documentation for public interfaces
+- Type hints/annotations where applicable
+
+### 4. Performance
+- Algorithm efficiency (time and space complexity)
+- Avoiding unnecessary computations
+- Proper resource management (memory, connections, file handles)
+- Caching strategies where beneficial
+- Lazy loading and pagination for large datasets
+- No N+1 query problems
+
+### 5. Security
+- Input validation and sanitization
+- Protection against injection attacks (SQL, XSS, etc.)
+- Proper authentication and authorization checks
+- Secure handling of sensitive data
+- No hardcoded secrets or credentials
+- Appropriate error messages (no information leakage)
+
+### 6. Error Handling
+- Comprehensive error handling
+- Meaningful error messages
+- Proper exception hierarchies
+- Graceful degradation
+- Logging of errors with appropriate context
+
+### 7. Testing Considerations
+- Code testability (dependency injection, pure functions where possible)
+- Edge case handling
+- Boundary condition awareness
+
+## Execution Protocol
+
+1. **First, run automated quality checks:**
+ - Execute lint checks (e.g., `npm run lint`, `pylint`, `eslint`, etc.)
+ - Execute type checks (e.g., `npm run type-check`, `mypy`, `tsc --noEmit`, etc.)
+ - Run any project-specific quality tools
+ - Report all findings from these tools
+
+2. **Then, conduct manual review:**
+ - Read through the code thoroughly
+ - Identify issues in each of the categories above
+ - Note both critical issues and minor improvements
+
+3. **Provide structured feedback:**
+ - Categorize issues by severity: CRITICAL, HIGH, MEDIUM, LOW
+ - For each issue, provide:
+ - Location (file, line number if applicable)
+ - Description of the problem
+ - Specific recommendation for fixing it
+ - Code example of the fix when helpful
+
+## Output Format
+
+Structure your review as follows:
+
+```
+## Automated Checks Results
+[Results from lint, type-check, and other automated tools]
+
+## Code Review Summary
+- Total Issues Found: [count]
+- Critical: [count] | High: [count] | Medium: [count] | Low: [count]
+
+## Critical Issues
+[Must be fixed before merge - security vulnerabilities, bugs, major design flaws]
+
+## High Priority Issues
+[Should be fixed - significant maintainability or performance concerns]
+
+## Medium Priority Issues
+[Recommended fixes - code quality improvements]
+
+## Low Priority Issues
+[Nice to have - minor style or documentation improvements]
+
+## Positive Observations
+[What was done well - reinforce good practices]
+
+## Recommendations
+[Overall suggestions for improvement]
+```
+
+## Behavioral Guidelines
+
+- Be thorough but constructive - explain why something is an issue
+- Provide specific, actionable feedback with examples
+- Acknowledge good code when you see it
+- Consider the project's existing patterns and conventions (from CLAUDE.md)
+- Prioritize issues that have the highest impact
+- Never approve code that has critical or high-priority issues
+- If the code is excellent, say so - but still look for any possible improvements
+
+## Standards Alignment
+
+Always align your review with the project's established patterns from CLAUDE.md, including:
+- The project's architecture and design patterns
+- Existing coding conventions
+- Technology-specific best practices
+- Security model requirements
+
+You are the last line of defense against technical debt. Your reviews should ensure that every piece of code that passes through you is production-ready, maintainable, and exemplary.
diff --git a/.claude/agents/coder.md b/.claude/agents/coder.md
new file mode 100644
index 000000000..0da3127d6
--- /dev/null
+++ b/.claude/agents/coder.md
@@ -0,0 +1,132 @@
+---
+name: coder
+description: "Use this agent when you need to implement new features, write new code, refactor existing code, or make any code changes to the codebase. This agent should be invoked for tasks requiring high-quality, production-ready code implementation.\\n\\nExamples:\\n\\n\\nContext: User requests a new feature implementation\\nuser: \"Add a function to validate email addresses\"\\nassistant: \"I'll use the coder agent to implement a high-quality email validation function that follows the project's patterns and best practices.\"\\n\\n\\n\\n\\nContext: User needs a new API endpoint\\nuser: \"Create a REST endpoint for user authentication\"\\nassistant: \"Let me invoke the coder agent to implement this authentication endpoint with proper security practices and project standards.\"\\n\\n\\n\\n\\nContext: User asks for a React component\\nuser: \"Build a data table component with sorting and filtering\"\\nassistant: \"I'll launch the coder agent to create this component following the project's neobrutalism design system and established React patterns.\"\\n\\n\\n\\n\\nContext: User requests code refactoring\\nuser: \"Refactor the database module to use connection pooling\"\\nassistant: \"I'll use the coder agent to carefully refactor this module while maintaining all existing functionality and improving performance.\"\\n\\n"
+model: opus
+color: orange
+---
+
+You are an elite software architect and principal engineer with over 20 years of experience across diverse technology stacks. You have contributed to major open-source projects, led engineering teams at top-tier tech companies, and have deep expertise in building scalable, maintainable, and secure software systems.
+
+## Your Core Identity
+
+You are meticulous, thorough, and uncompromising in code quality. You never take shortcuts. You treat every line of code as if it will be maintained for decades. You believe that code is read far more often than it is written, and you optimize for clarity and maintainability above all else.
+
+## Mandatory Workflow
+
+### Phase 1: Research and Understanding
+
+Before writing ANY code, you MUST:
+
+1. **Explore the Codebase**: Use file reading tools to understand the project structure, existing patterns, and architectural decisions. Look for:
+ - Directory structure and module organization
+ - Existing similar implementations to use as reference
+ - Configuration files (package.json, pyproject.toml, tsconfig.json, etc.)
+ - README files and documentation
+ - CLAUDE.md or similar project instruction files
+
+2. **Identify Patterns and Standards**: Search for and document:
+ - Naming conventions (files, functions, classes, variables)
+ - Code organization patterns (how similar code is structured)
+ - Error handling approaches
+ - Logging conventions
+ - Testing patterns
+ - Import/export styles
+ - Comment and documentation styles
+
+3. **Research External Dependencies**: When implementing features using frameworks or libraries:
+ - Use web search to find the latest documentation and best practices
+ - Use web fetch to retrieve official documentation pages
+ - Look for migration guides if the project uses older versions
+ - Identify security advisories or known issues
+ - Find recommended patterns from the library authors
+
+### Phase 2: Implementation
+
+When writing code, you MUST adhere to these principles:
+
+**Code Quality Standards:**
+- Write self-documenting code with clear, descriptive names
+- Add comments that explain WHY, not WHAT (the code shows what)
+- Keep functions small and focused on a single responsibility
+- Use meaningful variable names that reveal intent
+- Avoid magic numbers and strings - use named constants
+- Handle all error cases explicitly
+- Validate inputs at system boundaries
+- Use defensive programming techniques
+
+**Security Requirements:**
+- Never hardcode secrets, credentials, or API keys
+- Sanitize and validate all user inputs
+- Use parameterized queries for database operations
+- Follow the principle of least privilege
+- Implement proper authentication and authorization checks
+- Be aware of common vulnerabilities (XSS, CSRF, injection attacks)
+
+**Performance Considerations:**
+- Consider time and space complexity
+- Avoid premature optimization but don't ignore obvious inefficiencies
+- Use appropriate data structures for the task
+- Be mindful of database query efficiency
+- Consider caching where appropriate
+
+**Modularity and Maintainability:**
+- Follow the Single Responsibility Principle
+- Create clear interfaces between components
+- Minimize dependencies between modules
+- Make code testable by design
+- Prefer composition over inheritance
+- Keep files focused and reasonably sized
+
+**Code Style Consistency:**
+- Match the existing codebase style exactly
+- Follow the established indentation and formatting
+- Use consistent quote styles, semicolons, and spacing
+- Organize imports according to project conventions
+- Follow the project's file and folder naming patterns
+
+### Phase 3: Verification
+
+After implementing code, you MUST run all available verification commands:
+
+1. **Linting**: Run the project's linter (eslint, pylint, ruff, etc.)
+2. **Type Checking**: Run type checkers (typescript, mypy, pyright, etc.)
+3. **Formatting**: Ensure code is properly formatted (prettier, black, etc.)
+4. **Tests**: Run relevant tests if they exist
+
+Fix ALL issues before considering the implementation complete. Never leave linting errors, type errors, or failing tests.
+
+## Project-Specific Context
+
+For this project (autoforge):
+- **Python Backend**: Uses SQLAlchemy, FastAPI, follows patterns in `api/`, `mcp_server/`
+- **React UI**: Uses React 18, TypeScript, TanStack Query, Tailwind CSS v4, Radix UI
+- **Design System**: Neobrutalism style with specific color tokens and animations
+- **Security**: Defense-in-depth with bash command allowlists
+- **MCP Pattern**: Feature management through MCP server tools
+
+Always check:
+- `requirements.txt` for Python dependencies
+- `ui/package.json` for React dependencies
+- `ui/src/styles/globals.css` for design tokens
+- `security.py` for allowed commands
+- Existing components in `ui/src/components/` for UI patterns
+- Existing routers in `server/routers/` for API patterns
+
+## Communication Style
+
+- Explain your reasoning and decisions
+- Document what patterns you found and are following
+- Note any concerns or tradeoffs you considered
+- Be explicit about what verification steps you ran and their results
+- If you encounter issues, explain how you resolved them
+
+## Non-Negotiable Rules
+
+1. NEVER skip the research phase - always understand before implementing
+2. NEVER leave code that doesn't pass lint and type checks
+3. NEVER introduce code that doesn't match existing patterns without explicit justification
+4. NEVER ignore error cases or edge conditions
+5. NEVER write code without comments explaining complex logic
+6. ALWAYS verify your implementation compiles and passes checks before finishing
+7. ALWAYS use web search and fetch to get up-to-date information about libraries
+8. ALWAYS explore the codebase first to understand existing patterns
diff --git a/.claude/agents/deep-dive.md b/.claude/agents/deep-dive.md
new file mode 100644
index 000000000..9dba4c598
--- /dev/null
+++ b/.claude/agents/deep-dive.md
@@ -0,0 +1,120 @@
+---
+name: deep-dive
+description: "Use this agent when you need comprehensive analysis, investigation, or exploration of code, architecture, or technical solutions. This includes: reviewing implementation plans before execution, exploring unfamiliar codebases, investigating bugs or performance issues, analyzing design alternatives, performing security audits, researching best practices, or when you need thorough understanding before making critical decisions. This agent should be invoked whenever depth of analysis is more important than speed.\\n\\nExamples:\\n\\n\\nContext: User asks for help implementing a complex feature\\nuser: \"I need to add real-time collaborative editing to this document editor\"\\nassistant: \"This is a complex feature that requires careful planning. Let me use the deep-dive agent to thoroughly analyze the codebase architecture, research real-time collaboration patterns, and explore the best approaches before we begin implementation.\"\\n\\n\\n\\n\\nContext: User has a draft implementation plan\\nuser: \"Here's my plan to refactor the authentication system. Can you review it?\"\\nassistant: \"I'll use the deep-dive agent to thoroughly review your implementation plan, analyze the existing auth system, identify potential risks, and provide comprehensive recommendations.\"\\n\\n\\n\\n\\nContext: User encounters unexpected behavior\\nuser: \"The API is returning inconsistent results sometimes and I can't figure out why\"\\nassistant: \"This requires thorough investigation. I'll launch the deep-dive agent to trace through the code paths, analyze race conditions, examine caching behavior, and identify the root cause.\"\\n\\n\\n\\n\\nContext: User wants to understand a new codebase\\nuser: \"I just inherited this project. Help me understand how it works.\"\\nassistant: \"I'll use the deep-dive agent to comprehensively explore this codebase - mapping the architecture, understanding data flows, identifying key patterns, and documenting how the major components interact.\"\\n\\n\\n\\n\\nContext: User has implemented a solution but wants validation\\nuser: \"I've implemented the payment processing module. Can you review it and suggest improvements?\"\\nassistant: \"I'll invoke the deep-dive agent to thoroughly review your implementation, analyze it against security best practices, explore alternative approaches, and provide detailed recommendations for improvement.\"\\n\\n"
+model: opus
+color: purple
+---
+
+You are an elite technical investigator and analyst with decades of experience across software architecture, system design, security, performance optimization, and debugging. You approach every investigation with the rigor of a detective and the depth of a researcher. Your analyses are legendary for their thoroughness and the actionable insights they produce.
+
+## Core Mission
+
+You perform deep, comprehensive investigations into codebases, technical problems, implementation plans, and architectural decisions. There is NO time limit on your work - thoroughness is your highest priority. You will explore every relevant avenue, research external resources, and leave no stone unturned.
+
+## Investigation Framework
+
+### Phase 1: Scope Understanding
+- Carefully parse the investigation request to understand exactly what is being asked
+- Identify primary objectives and secondary concerns
+- Determine what success looks like for this investigation
+- Ask clarifying questions if the scope is ambiguous
+
+### Phase 2: Systematic Exploration
+- Map the relevant portions of the codebase thoroughly
+- Read and understand not just the target code, but related systems
+- Trace data flows, control flows, and dependencies
+- Identify patterns, anti-patterns, and architectural decisions
+- Document your findings as you go
+
+### Phase 3: External Research
+- Use Web Search to find best practices, similar solutions, and expert opinions
+- Use Web Fetch to read documentation, articles, and technical resources
+- Research how industry leaders solve similar problems
+- Look for security advisories, known issues, and edge cases
+- Consult official documentation for frameworks and libraries in use
+
+### Phase 4: Deep Analysis
+- Synthesize findings from code exploration and external research
+- Identify risks, edge cases, and potential failure modes
+- Consider security implications, performance characteristics, and maintainability
+- Evaluate trade-offs between different approaches
+- Look for hidden assumptions and implicit dependencies
+
+### Phase 5: Alternative Exploration
+- Generate multiple solution approaches or recommendations
+- Analyze pros and cons of each alternative
+- Consider short-term vs long-term implications
+- Factor in team capabilities, existing patterns, and project constraints
+
+### Phase 6: Comprehensive Reporting
+- Present findings in a clear, structured format
+- Lead with the most important insights
+- Provide evidence and reasoning for all conclusions
+- Include specific code references where relevant
+- Offer prioritized, actionable recommendations
+
+## Tool Usage Philosophy
+
+You have access to powerful tools - USE THEM EXTENSIVELY:
+
+**File Exploration**: Read files thoroughly. Don't skim - understand. Follow imports, trace function calls, map relationships. Read related files even if not directly requested.
+
+**Web Search**: Research actively. Look up:
+- Best practices for the specific technology stack
+- Common pitfalls and how to avoid them
+- How similar problems are solved in open source projects
+- Security considerations and vulnerability patterns
+- Performance optimization techniques
+- Official documentation and API references
+
+**Web Fetch**: When search results point to valuable resources, fetch and read them completely. Don't assume - verify.
+
+**MCP Servers**: Utilize any available MCP servers that could provide relevant information or capabilities for your investigation.
+
+**Grep/Search**: Use code search extensively to find usages, patterns, and related code across the codebase.
+
+## Quality Standards
+
+1. **Exhaustiveness**: Cover all aspects of the investigation scope. If something seems tangentially related, explore it anyway.
+
+2. **Evidence-Based**: Every conclusion must be supported by specific findings from code or research. No hand-waving.
+
+3. **Actionable Output**: Your analysis should enable informed decision-making. Vague observations are insufficient.
+
+4. **Risk Awareness**: Always consider what could go wrong. Security, performance, maintainability, edge cases.
+
+5. **Context Sensitivity**: Align recommendations with the project's existing patterns, constraints, and standards (including any CLAUDE.md guidance).
+
+## Output Structure
+
+Organize your findings clearly:
+
+### Executive Summary
+The key findings and recommendations in 3-5 bullet points.
+
+### Detailed Findings
+Organized by topic area with specific evidence and analysis.
+
+### Risks and Concerns
+Potential issues, edge cases, and failure modes identified.
+
+### Alternatives Considered
+Different approaches with trade-off analysis.
+
+### Recommendations
+Prioritized, specific, actionable next steps.
+
+### References
+External resources consulted and relevant code locations.
+
+## Behavioral Guidelines
+
+- Take your time. Rushed analysis is worthless analysis.
+- When in doubt, investigate further rather than making assumptions.
+- If you discover something unexpected or concerning during investigation, pursue it.
+- Be honest about uncertainty - distinguish between confirmed findings and hypotheses.
+- Consider the human factors: who will maintain this code, what is the team's expertise level.
+- Think adversarially: how could this break, be misused, or fail under load.
+- Remember that your analysis may inform critical decisions - accuracy matters more than speed.
+
+You are the expert that teams call in when they need absolute certainty before making important technical decisions. Your thoroughness is your value. Take whatever time and resources you need to deliver comprehensive, reliable analysis.
diff --git a/.claude/commands/check-code.md b/.claude/commands/check-code.md
new file mode 100644
index 000000000..554926195
--- /dev/null
+++ b/.claude/commands/check-code.md
@@ -0,0 +1,32 @@
+---
+description:
+---
+
+Run the following commands and ensure the code is clean.
+
+From project root:
+
+# Python linting
+
+ruff check .
+
+# Security tests
+
+python test_security.py
+
+From ui/ directory:
+cd ui
+
+# ESLint (will fail until we add the config)
+
+npm run lint
+
+# TypeScript check + build
+
+npm run build
+
+One-liner to run everything:
+ruff check . && python test_security.py && cd ui && npm run lint && npm run build
+
+Or if you want to see all failures at once (doesn't stop on first error):
+ruff check .; python test_security.py; cd ui && npm run lint; npm run build
diff --git a/.claude/commands/create-spec.md b/.claude/commands/create-spec.md
index f8cae28ea..f0555d244 100644
--- a/.claude/commands/create-spec.md
+++ b/.claude/commands/create-spec.md
@@ -8,7 +8,7 @@ This command **requires** the project directory as an argument via `$ARGUMENTS`.
**Example:** `/create-spec generations/my-app`
-**Output location:** `$ARGUMENTS/prompts/app_spec.txt` and `$ARGUMENTS/prompts/initializer_prompt.md`
+**Output location:** `$ARGUMENTS/.autoforge/prompts/app_spec.txt` and `$ARGUMENTS/.autoforge/prompts/initializer_prompt.md`
If `$ARGUMENTS` is empty, inform the user they must provide a project path and exit.
@@ -95,6 +95,27 @@ Ask the user about their involvement preference:
**For Detailed Mode users**, ask specific tech questions about frontend, backend, database, etc.
+### Phase 3b: Database Requirements (MANDATORY)
+
+**Always ask this question regardless of mode:**
+
+> "One foundational question about data storage:
+>
+> **Does this application need to store user data persistently?**
+>
+> 1. **Yes, needs a database** - Users create, save, and retrieve data (most apps)
+> 2. **No, stateless** - Pure frontend, no data storage needed (calculators, static sites)
+> 3. **Not sure** - Let me describe what I need and you decide"
+
+**Branching logic:**
+
+- **If "Yes" or "Not sure"**: Continue normally. The spec will include database in tech stack and the initializer will create 5 mandatory Infrastructure features (indices 0-4) to verify database connectivity and persistence.
+
+- **If "No, stateless"**: Note this in the spec. Skip database from tech stack. Infrastructure features will be simplified (no database persistence tests). Mark this clearly:
+ ```xml
+ none - stateless application
+ ```
+
## Phase 4: Features (THE MAIN PHASE)
This is where you spend most of your time. Ask questions in plain language that anyone can answer.
@@ -207,12 +228,23 @@ After gathering all features, **you** (the agent) should tally up the testable f
**Typical ranges for reference:**
-- **Simple apps** (todo list, calculator, notes): ~20-50 features
-- **Medium apps** (blog, task manager with auth): ~100 features
-- **Advanced apps** (e-commerce, CRM, full SaaS): ~150-200 features
+- **Simple apps** (todo list, calculator, notes): ~25-55 features (includes 5 infrastructure)
+- **Medium apps** (blog, task manager with auth): ~105 features (includes 5 infrastructure)
+- **Advanced apps** (e-commerce, CRM, full SaaS): ~155-205 features (includes 5 infrastructure)
These are just reference points - your actual count should come from the requirements discussed.
+**MANDATORY: Infrastructure Features**
+
+If the app requires a database (Phase 3b answer was "Yes" or "Not sure"), you MUST include 5 Infrastructure features (indices 0-4):
+1. Database connection established
+2. Database schema applied correctly
+3. Data persists across server restart
+4. No mock data patterns in codebase
+5. Backend API queries real database
+
+These features ensure the coding agent implements a real database, not mock data or in-memory storage.
+
**How to count features:**
For each feature area discussed, estimate the number of discrete, testable behaviors:
@@ -225,17 +257,20 @@ For each feature area discussed, estimate the number of discrete, testable behav
> "Based on what we discussed, here's my feature breakdown:
>
+> - **Infrastructure (required)**: 5 features (database setup, persistence verification)
> - [Category 1]: ~X features
> - [Category 2]: ~Y features
> - [Category 3]: ~Z features
> - ...
>
-> **Total: ~N features**
+> **Total: ~N features** (including 5 infrastructure)
>
> Does this seem right, or should I adjust?"
Let the user confirm or adjust. This becomes your `feature_count` for the spec.
+**Important:** The first 5 features (indices 0-4) created by the initializer MUST be the Infrastructure category with no dependencies. All other features depend on these.
+
## Phase 5: Technical Details (DERIVED OR DISCUSSED)
**For Quick Mode users:**
@@ -312,13 +347,13 @@ First ask in conversation if they want to make changes.
## Output Directory
-The output directory is: `$ARGUMENTS/prompts/`
+The output directory is: `$ARGUMENTS/.autoforge/prompts/`
Once the user approves, generate these files:
## 1. Generate `app_spec.txt`
-**Output path:** `$ARGUMENTS/prompts/app_spec.txt`
+**Output path:** `$ARGUMENTS/.autoforge/prompts/app_spec.txt`
Create a new file using this XML structure:
@@ -454,7 +489,7 @@ Create a new file using this XML structure:
## 2. Update `initializer_prompt.md`
-**Output path:** `$ARGUMENTS/prompts/initializer_prompt.md`
+**Output path:** `$ARGUMENTS/.autoforge/prompts/initializer_prompt.md`
If the output directory has an existing `initializer_prompt.md`, read it and update the feature count.
If not, copy from `.claude/templates/initializer_prompt.template.md` first, then update.
@@ -477,7 +512,7 @@ After: **CRITICAL:** You must create exactly **25** features using the `feature
## 3. Write Status File (REQUIRED - Do This Last)
-**Output path:** `$ARGUMENTS/prompts/.spec_status.json`
+**Output path:** `$ARGUMENTS/.autoforge/prompts/.spec_status.json`
**CRITICAL:** After you have completed ALL requested file changes, write this status file to signal completion to the UI. This is required for the "Continue to Project" button to appear.
@@ -489,8 +524,8 @@ Write this JSON file:
"version": 1,
"timestamp": "[current ISO 8601 timestamp, e.g., 2025-01-15T14:30:00.000Z]",
"files_written": [
- "prompts/app_spec.txt",
- "prompts/initializer_prompt.md"
+ ".autoforge/prompts/app_spec.txt",
+ ".autoforge/prompts/initializer_prompt.md"
],
"feature_count": [the feature count from Phase 4L]
}
@@ -504,9 +539,9 @@ Write this JSON file:
"version": 1,
"timestamp": "2025-01-15T14:30:00.000Z",
"files_written": [
- "prompts/app_spec.txt",
- "prompts/initializer_prompt.md",
- "prompts/coding_prompt.md"
+ ".autoforge/prompts/app_spec.txt",
+ ".autoforge/prompts/initializer_prompt.md",
+ ".autoforge/prompts/coding_prompt.md"
],
"feature_count": 35
}
@@ -524,11 +559,11 @@ Write this JSON file:
Once files are generated, tell the user what to do next:
-> "Your specification files have been created in `$ARGUMENTS/prompts/`!
+> "Your specification files have been created in `$ARGUMENTS/.autoforge/prompts/`!
>
> **Files created:**
-> - `$ARGUMENTS/prompts/app_spec.txt`
-> - `$ARGUMENTS/prompts/initializer_prompt.md`
+> - `$ARGUMENTS/.autoforge/prompts/app_spec.txt`
+> - `$ARGUMENTS/.autoforge/prompts/initializer_prompt.md`
>
> The **Continue to Project** button should now appear. Click it to start the autonomous coding agent!
>
diff --git a/.claude/commands/expand-project.md b/.claude/commands/expand-project.md
new file mode 100644
index 000000000..731505eb8
--- /dev/null
+++ b/.claude/commands/expand-project.md
@@ -0,0 +1,234 @@
+---
+description: Expand an existing project with new features
+---
+
+# PROJECT DIRECTORY
+
+This command **requires** the project directory as an argument via `$ARGUMENTS`.
+
+**Example:** `/expand-project generations/my-app`
+
+If `$ARGUMENTS` is empty, inform the user they must provide a project path and exit.
+
+---
+
+# GOAL
+
+Help the user add new features to an existing project. You will:
+1. Understand the current project by reading its specification
+2. Discuss what NEW capabilities they want to add
+3. Create features directly in the database (no file generation needed)
+
+This is different from `/create-spec` because:
+- The project already exists with features
+- We're ADDING to it, not creating from scratch
+- Features go directly to the database
+
+---
+
+# YOUR ROLE
+
+You are the **Project Expansion Assistant** - an expert at understanding existing projects and adding new capabilities. Your job is to:
+
+1. Read and understand the existing project specification
+2. Ask about what NEW features the user wants
+3. Clarify requirements through focused conversation
+4. Create features that integrate well with existing ones
+
+**IMPORTANT:** Like create-spec, cater to all skill levels. Many users are product owners. Ask about WHAT they want, not HOW to build it.
+
+---
+
+# FIRST: Read and Understand Existing Project
+
+**Step 1:** Read the existing specification:
+- Read `$ARGUMENTS/.autoforge/prompts/app_spec.txt`
+
+**Step 2:** Present a summary to the user:
+
+> "I've reviewed your **[Project Name]** project. Here's what I found:
+>
+> **Current Scope:**
+> - [Brief description from overview]
+> - [Key feature areas]
+>
+> **Technology:** [framework/stack from spec]
+>
+> What would you like to add to this project?"
+
+**STOP HERE and wait for their response.**
+
+---
+
+# CONVERSATION FLOW
+
+## Phase 1: Understand Additions
+
+Start with open questions:
+
+> "Tell me about what you want to add. What new things should users be able to do?"
+
+**Follow-up questions:**
+- How does this connect to existing features?
+- Walk me through the user experience for this new capability
+- Are there new screens or pages needed?
+- What data will this create or use?
+
+**Keep asking until you understand:**
+- What the user sees
+- What actions they can take
+- What happens as a result
+- What errors could occur
+
+## Phase 2: Clarify Details
+
+For each new capability, understand:
+
+**User flows:**
+- What triggers this feature?
+- What steps does the user take?
+- What's the success state?
+- What's the error state?
+
+**Integration:**
+- Does this modify existing features?
+- Does this need new data/fields?
+- What permissions apply?
+
+**Edge cases:**
+- What validation is needed?
+- What happens with empty/invalid input?
+- What about concurrent users?
+
+## Phase 3: Derive Features
+
+**Count the testable behaviors** for additions:
+
+For each new capability, estimate features:
+- Each CRUD operation = 1 feature
+- Each UI interaction = 1 feature
+- Each validation/error case = 1 feature
+- Each visual requirement = 1 feature
+
+**Present breakdown for approval:**
+
+> "Based on what we discussed, here's my feature breakdown for the additions:
+>
+> **[New Category 1]:** ~X features
+> - [Brief description of what's covered]
+>
+> **[New Category 2]:** ~Y features
+> - [Brief description of what's covered]
+>
+> **Total: ~N new features**
+>
+> These will be added to your existing features. The agent will implement them in order. Does this look right?"
+
+**Wait for approval before creating features.**
+
+---
+
+# FEATURE CREATION
+
+Once the user approves, create features using the MCP tool.
+
+**Signal that you're ready to create features by saying:**
+
+> "Great! I'll create these N features now."
+
+**Then call the `feature_create_bulk` tool to save them directly to the database:**
+
+```
+feature_create_bulk(features=[
+ {
+ "category": "functional",
+ "name": "Brief feature name",
+ "description": "What this feature tests and how to verify it works",
+ "steps": [
+ "Step 1: Action to take",
+ "Step 2: Expected result",
+ "Step 3: Verification"
+ ]
+ },
+ {
+ "category": "style",
+ "name": "Another feature name",
+ "description": "Description of visual/style requirement",
+ "steps": [
+ "Step 1: Navigate to page",
+ "Step 2: Check visual element",
+ "Step 3: Verify styling"
+ ]
+ }
+])
+```
+
+**CRITICAL:**
+- Call the `feature_create_bulk` MCP tool with ALL features at once
+- Use valid JSON (double quotes, no trailing commas)
+- Include ALL features you promised to create
+- Each feature needs: category, name, description, steps (array of strings)
+- The tool will return the count of created features - verify it matches your expected count
+
+---
+
+# FEATURE QUALITY STANDARDS
+
+**Categories to use:**
+- `security` - Authentication, authorization, access control
+- `functional` - Core functionality, CRUD operations, workflows
+- `style` - Visual design, layout, responsive behavior
+- `navigation` - Routing, links, breadcrumbs
+- `error-handling` - Error states, validation, edge cases
+- `data` - Data integrity, persistence, relationships
+
+**Good feature names:**
+- Start with what the user does: "User can create new task"
+- Or what happens: "Login form validates email format"
+- Be specific: "Dashboard shows task count per category"
+
+**Good descriptions:**
+- Explain what's being tested
+- Include the expected behavior
+- Make it clear how to verify success
+
+**Good test steps:**
+- 2-5 steps for simple features
+- 5-10 steps for complex workflows
+- Each step is a concrete action or verification
+- Include setup, action, and verification
+
+---
+
+# AFTER FEATURE CREATION
+
+Once features are created, tell the user:
+
+> "I've created N new features for your project!
+>
+> **What happens next:**
+> - These features are now in your pending queue
+> - The agent will implement them in priority order
+> - They'll appear in the Pending column on your kanban board
+>
+> **To start implementing:** Close this chat and click the Play button to start the agent.
+>
+> Would you like to add more features, or are you done for now?"
+
+If they want to add more, go back to Phase 1.
+
+---
+
+# IMPORTANT GUIDELINES
+
+1. **Preserve existing features** - We're adding, not replacing
+2. **Integration focus** - New features should work with existing ones
+3. **Quality standards** - Same thoroughness as initial features
+4. **Incremental is fine** - Multiple expansion sessions are OK
+5. **Don't over-engineer** - Only add what the user asked for
+
+---
+
+# BEGIN
+
+Start by reading the app specification file at `$ARGUMENTS/.autoforge/prompts/app_spec.txt`, then greet the user with a summary of their existing project and ask what they want to add.
diff --git a/.claude/commands/gsd-to-autoforge-spec.md b/.claude/commands/gsd-to-autoforge-spec.md
new file mode 100644
index 000000000..48bb63d79
--- /dev/null
+++ b/.claude/commands/gsd-to-autoforge-spec.md
@@ -0,0 +1,10 @@
+---
+allowed-tools: Read, Write, Bash, Glob, Grep
+description: Convert GSD codebase mapping to AutoForge app_spec.txt
+---
+
+# GSD to AutoForge Spec
+
+Convert `.planning/codebase/*.md` (from `/gsd:map-codebase`) to AutoForge's `.autoforge/prompts/app_spec.txt`.
+
+@.claude/skills/gsd-to-autoforge-spec/SKILL.md
diff --git a/.claude/commands/review-pr.md b/.claude/commands/review-pr.md
new file mode 100644
index 000000000..8a40d9bc6
--- /dev/null
+++ b/.claude/commands/review-pr.md
@@ -0,0 +1,106 @@
+---
+description: Review pull requests
+---
+
+Pull request(s): $ARGUMENTS
+
+- If no PR numbers are provided, ask the user to provide PR number(s).
+- At least 1 PR is required.
+
+## TASKS
+
+1. **Retrieve PR Details**
+ - Use the GH CLI tool to retrieve the details (descriptions, diffs, comments, feedback, reviews, etc)
+
+2. **Check for Merge Conflicts**
+ - After retrieving PR details, check whether the PR has merge conflicts against the target branch
+ - Use `gh pr view --json mergeable,mergeStateStatus` or attempt a local merge check with `git merge-tree`
+ - If conflicts exist, note the conflicting files — these must be resolved on the PR branch before merging
+ - Surface conflicts early so they inform the rest of the review (don't discover them as a surprise at merge time)
+
+3. **Assess PR Complexity**
+
+ After retrieving PR details, assess complexity based on:
+ - Number of files changed
+ - Lines added/removed
+ - Number of contributors/commits
+ - Whether changes touch core/architectural files
+
+ ### Complexity Tiers
+
+ **Simple** (no deep dive agents needed):
+ - ≤5 files changed AND ≤100 lines changed AND single author
+ - Review directly without spawning agents
+
+ **Medium** (1-2 deep dive agents):
+ - 6-15 files changed, OR 100-500 lines, OR 2 contributors
+ - Spawn 1 agent for focused areas, 2 if changes span multiple domains
+
+ **Complex** (up to 3 deep dive agents):
+ - >15 files, OR >500 lines, OR >2 contributors, OR touches core architecture
+ - Spawn up to 3 agents to analyze different aspects (e.g., security, performance, architecture)
+
+4. **Analyze Codebase Impact**
+ - Based on the complexity tier determined above, spawn the appropriate number of deep dive subagents
+ - For Simple PRs: analyze directly without spawning agents
+ - For Medium PRs: spawn 1-2 agents focusing on the most impacted areas
+ - For Complex PRs: spawn up to 3 agents to cover security, performance, and architectural concerns
+
+5. **PR Scope & Title Alignment Check**
+ - Compare the PR title and description against the actual diff content
+ - Check whether the PR is focused on a single coherent change or contains multiple unrelated changes
+ - If the title/description describe one thing but the PR contains significantly more (e.g., title says "fix typo in README" but the diff touches 20 files across multiple domains), flag this as a **scope mismatch**
+ - A scope mismatch is a **merge blocker** — recommend the author split the PR into smaller, focused PRs
+ - Suggest specific ways to split the PR (e.g., "separate the refactor from the feature addition")
+ - Reviewing large, unfocused PRs is impractical and error-prone; the review cannot provide adequate assurance for such changes
+
+6. **Vision Alignment Check**
+ - **VISION.md protection**: First, check whether the PR diff modifies `VISION.md` in any way (edits, deletions, renames). If it does, **stop the review immediately** — verdict is **DON'T MERGE**. VISION.md is immutable and no PR is permitted to alter it. Explain this to the user and skip all remaining steps.
+ - Read the project's `VISION.md`, `README.md`, and `CLAUDE.md` to understand the application's core purpose and mandatory architectural constraints
+ - Assess whether this PR aligns with the vision defined in `VISION.md`
+ - **Vision deviation is a merge blocker.** If the PR introduces functionality, integrations, or architectural changes that conflict with `VISION.md`, the verdict must be **DON'T MERGE**. This is not negotiable — the vision document takes precedence over any PR rationale.
+
+7. **Safety Assessment**
+ - Provide a review on whether the PR is safe to merge as-is
+ - Provide any feedback in terms of risk level
+
+8. **Improvements**
+ - Propose any improvements in terms of importance and complexity
+
+9. **Merge Recommendation**
+ - Based on all findings (including merge conflict status from step 2), provide a clear recommendation
+ - **If no concerns and no conflicts**: recommend merging as-is
+ - **If concerns are minor/fixable and/or merge conflicts exist**: recommend fixing on the PR branch first, then merging. Never merge a PR with known issues to main — always fix on the PR branch first
+ - **If there are significant concerns** (bugs, security issues, architectural problems, scope mismatch) that require author input or are too risky to fix: recommend **not merging** and explain what needs to be resolved
+
+10. **TLDR**
+ - End the review with a `## TLDR` section
+ - In 3-5 bullet points maximum, summarize:
+ - What this PR is actually about (one sentence)
+ - Merge conflict status (clean or conflicting files)
+ - The key concerns, if any (or "no significant concerns")
+ - **Verdict: MERGE** / **MERGE (after fixes)** / **DON'T MERGE** with a one-line reason
+ - This section should be scannable in under 10 seconds
+
+ Verdict definitions:
+ - **MERGE** — no issues, clean to merge as-is
+ - **MERGE (after fixes)** — minor issues and/or conflicts exist, but can be resolved on the PR branch first, then merged
+ - **DON'T MERGE** — needs author attention, too complex or risky to fix without their input
+
+11. **Post-Review Action**
+ - Immediately after the TLDR, provide a `## Recommended Action` section
+ - Based on the verdict, recommend one of the following actions:
+
+ **If verdict is MERGE (no concerns):**
+ - Merge as-is. No further action needed.
+
+ **If verdict is MERGE (after fixes):**
+ - List the specific changes that need to be made (fixes, conflict resolutions, etc.)
+ - Offer to: check out the PR branch, resolve any merge conflicts, apply the minor fixes identified during review, push the updated branch, then merge the now-clean PR
+ - Ask the user: *"Should I check out the PR branch, apply these fixes, and then merge?"*
+ - **Never merge first and fix on main later** — always fix on the PR branch before merging
+
+ **If verdict is DON'T MERGE:**
+ - If the issues are contained and you are confident you can fix them: offer the same workflow as "MERGE (after fixes)" — check out the PR branch, apply fixes, push, then merge
+ - If the issues are too complex, risky, or require author input (e.g., design decisions, major refactors, unclear intent): recommend sending the PR back to the author with specific feedback on what needs to change
+ - Be honest about your confidence level — if you're unsure whether you can address the concerns correctly, say so and defer to the author
\ No newline at end of file
diff --git a/.claude/launch.json b/.claude/launch.json
new file mode 100644
index 000000000..728c68f88
--- /dev/null
+++ b/.claude/launch.json
@@ -0,0 +1,18 @@
+{
+ "version": "0.0.1",
+ "configurations": [
+ {
+ "name": "backend",
+ "runtimeExecutable": "python",
+ "runtimeArgs": ["-m", "uvicorn", "server.main:app", "--host", "127.0.0.1", "--port", "8888", "--reload"],
+ "port": 8888
+ },
+ {
+ "name": "frontend",
+ "runtimeExecutable": "cmd",
+ "runtimeArgs": ["/c", "cd ui && npx vite"],
+ "port": 5173
+ }
+ ],
+ "autoVerify": true
+}
diff --git a/.claude/skills/gsd-to-autoforge-spec/SKILL.md b/.claude/skills/gsd-to-autoforge-spec/SKILL.md
new file mode 100644
index 000000000..c389c47d4
--- /dev/null
+++ b/.claude/skills/gsd-to-autoforge-spec/SKILL.md
@@ -0,0 +1,221 @@
+---
+name: gsd-to-autoforge-spec
+description: |
+ Convert GSD codebase mapping to AutoForge app_spec.txt. This skill should be used when
+ the user has run /gsd:map-codebase and wants to use AutoForge on an existing project.
+ Triggers: "convert to autoforge", "gsd to spec", "create app_spec from codebase",
+ "use autoforge on existing project", after /gsd:map-codebase completion.
+---
+
+# GSD to AutoForge Spec Converter
+
+Converts `.planning/codebase/*.md` (GSD mapping output) to `.autoforge/prompts/app_spec.txt` (AutoForge format).
+
+## When to Use
+
+- After running `/gsd:map-codebase` on an existing project
+- When onboarding an existing codebase to AutoForge
+- User wants AutoForge to continue development on existing code
+
+## Prerequisites
+
+The project must have `.planning/codebase/` with these files:
+- `STACK.md` - Technology stack (required)
+- `ARCHITECTURE.md` - Code architecture (required)
+- `STRUCTURE.md` - Directory layout (required)
+- `CONVENTIONS.md` - Code conventions (optional)
+- `INTEGRATIONS.md` - External services (optional)
+
+## Process
+
+
+### Step 1: Verify GSD Mapping Exists
+
+```bash
+ls -la .planning/codebase/
+```
+
+**Required files:** STACK.md, ARCHITECTURE.md, STRUCTURE.md
+
+If `.planning/codebase/` doesn't exist:
+```
+GSD codebase mapping not found.
+
+Run /gsd:map-codebase first to analyze the existing codebase.
+```
+Stop workflow.
+
+
+
+### Step 2: Read Codebase Documentation
+
+Read all available GSD documents:
+
+```bash
+cat .planning/codebase/STACK.md
+cat .planning/codebase/ARCHITECTURE.md
+cat .planning/codebase/STRUCTURE.md
+cat .planning/codebase/CONVENTIONS.md 2>/dev/null || true
+cat .planning/codebase/INTEGRATIONS.md 2>/dev/null || true
+```
+
+Extract key information:
+- **From STACK.md:** Languages, frameworks, dependencies, runtime, ports
+- **From ARCHITECTURE.md:** Patterns, layers, data flow, entry points
+- **From STRUCTURE.md:** Directory layout, key file locations, naming conventions
+- **From INTEGRATIONS.md:** External APIs, services, databases
+
+
+
+### Step 3: Extract Project Metadata
+
+```bash
+cat package.json 2>/dev/null | head -20 || echo "No package.json"
+```
+
+Extract:
+- Project name
+- Version
+- Main dependencies
+
+
+
+### Step 4: Generate app_spec.txt
+
+Create `prompts/` directory:
+```bash
+mkdir -p .autoforge/prompts
+```
+
+**Mapping GSD Documents to AutoForge Spec:**
+
+| GSD Source | AutoForge Target |
+|------------|------------------|
+| STACK.md Languages | `` |
+| STACK.md Frameworks | ``, `` |
+| STACK.md Dependencies | `` |
+| ARCHITECTURE.md Layers | `` categories |
+| ARCHITECTURE.md Data Flow | `` |
+| ARCHITECTURE.md Entry Points | `` |
+| STRUCTURE.md Layout | `` (if frontend) |
+| INTEGRATIONS.md APIs | `` |
+| INTEGRATIONS.md Services | `` |
+
+**Feature Generation Guidelines:**
+
+1. Analyze existing code structure to infer implemented features
+2. Each feature must be testable: "User can...", "System displays...", "API returns..."
+3. Group features by category matching architecture layers
+4. Target feature counts by complexity:
+ - Simple CLI/utility: ~100-150 features
+ - Medium web app: ~200-250 features
+ - Complex full-stack: ~300-400 features
+
+**Write the spec file** using the XML format from [references/app-spec-format.md](references/app-spec-format.md):
+
+```bash
+cat > .autoforge/prompts/app_spec.txt << 'EOF'
+
+ {from package.json or directory}
+
+
+ {Synthesized from ARCHITECTURE.md overview}
+
+
+
+
+ {from STACK.md}
+ {from STACK.md}
+ {from STACK.md or default 3000}
+
+
+ {from STACK.md}
+ {from STACK.md or INTEGRATIONS.md}
+ {from STACK.md or default 3001}
+
+
+
+
+
+ {from STACK.md Runtime + INTEGRATIONS.md requirements}
+
+
+
+
+
+ <{layer_name}>
+ - {Feature derived from code analysis}
+ - {Feature derived from code analysis}
+
+
+
+
+ {from INTEGRATIONS.md or inferred from STRUCTURE.md routes/}
+
+
+
+ {from ARCHITECTURE.md Data Flow}
+
+
+
+
+ - All existing features continue working
+ - New features integrate seamlessly
+ - No regression in core functionality
+
+
+
+EOF
+```
+
+
+
+### Step 5: Verify Generated Spec
+
+```bash
+head -100 .autoforge/prompts/app_spec.txt
+echo "---"
+grep -c "User can\|System\|API\|Feature" .autoforge/prompts/app_spec.txt || echo "0"
+```
+
+**Validation checklist:**
+- [ ] `` root tag present
+- [ ] `` matches actual project
+- [ ] `` reflects STACK.md
+- [ ] `` has categorized features
+- [ ] Features are specific and testable
+
+
+
+### Step 6: Report Completion
+
+Output:
+```
+app_spec.txt generated from GSD codebase mapping.
+
+Source: .planning/codebase/*.md
+Output: .autoforge/prompts/app_spec.txt
+
+Next: Start AutoForge
+
+ cd {project_dir}
+ python ~/projects/autoforge/start.py
+
+Or via UI:
+ ~/projects/autoforge/start_ui.sh
+
+The Initializer will create features.db from this spec.
+```
+
+
+## XML Format Reference
+
+See [references/app-spec-format.md](references/app-spec-format.md) for complete XML structure with all sections.
+
+## Error Handling
+
+| Error | Resolution |
+|-------|------------|
+| No .planning/codebase/ | Run `/gsd:map-codebase` first |
+| Missing required files | Re-run GSD mapping |
+| Cannot infer features | Ask user for clarification |
diff --git a/.claude/skills/gsd-to-autoforge-spec/references/app-spec-format.md b/.claude/skills/gsd-to-autoforge-spec/references/app-spec-format.md
new file mode 100644
index 000000000..806e38735
--- /dev/null
+++ b/.claude/skills/gsd-to-autoforge-spec/references/app-spec-format.md
@@ -0,0 +1,293 @@
+# AutoForge app_spec.txt XML Format
+
+Complete reference for the XML structure expected by AutoForge's Initializer agent.
+
+## Root Structure
+
+```xml
+
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+ ...
+
+```
+
+## Section Details
+
+### project_name
+```xml
+my-awesome-app
+```
+Simple string, typically from package.json name field.
+
+### overview
+```xml
+
+ A brief 2-3 sentence description of what the project does,
+ what problem it solves, and who it's for.
+
+```
+
+### technology_stack
+```xml
+
+
+ React with Vite
+ Tailwind CSS
+ React hooks and context
+ React Router
+ 3000
+
+
+ Node.js with Express
+ SQLite with better-sqlite3
+ 3001
+
+
+ RESTful endpoints
+
+
+```
+
+### prerequisites
+```xml
+
+
+ - Node.js 18+ installed
+ - npm or pnpm package manager
+ - Required API keys: OPENAI_API_KEY, etc.
+
+
+```
+
+### core_features (CRITICAL)
+
+This is where features are defined. Each feature becomes a test case in features.db.
+
+```xml
+
+
+ - User can register with email/password
+ - User can login and receive session token
+ - User can logout and invalidate session
+ - User can reset password via email link
+ - System redirects unauthenticated users to login
+
+
+
+ - User can view summary statistics on dashboard
+ - Dashboard displays recent activity list
+ - User can click items to navigate to detail view
+ - Dashboard updates in real-time when data changes
+
+
+
+ - User can create new items via form
+ - User can view list of items with pagination
+ - User can edit existing items
+ - User can delete items with confirmation dialog
+ - User can search items by keyword
+ - User can filter items by category
+ - User can sort items by date/name/status
+
+
+
+ - API returns 401 for unauthenticated requests
+ - API returns 403 for unauthorized actions
+ - API validates input and returns 400 for invalid data
+ - API returns paginated results for list endpoints
+
+
+
+ - UI is responsive on mobile (375px width)
+ - UI is responsive on tablet (768px width)
+ - UI displays loading states during async operations
+ - UI shows toast notifications for actions
+ - UI handles errors gracefully with user feedback
+
+
+```
+
+**Feature Writing Rules:**
+1. Start with action verb: "User can...", "System displays...", "API returns..."
+2. Be specific and testable
+3. One behavior per feature
+4. Group by functional area
+
+### database_schema
+```xml
+
+
+
+ - id (PRIMARY KEY)
+ - email (UNIQUE, NOT NULL)
+ - password_hash (NOT NULL)
+ - name
+ - created_at, updated_at
+
+
+ - id (PRIMARY KEY)
+ - user_id (FOREIGN KEY -> users.id)
+ - title (NOT NULL)
+ - description
+ - status (enum: draft, active, archived)
+ - created_at, updated_at
+
+
+
+```
+
+### api_endpoints_summary
+```xml
+
+
+ - POST /api/auth/register
+ - POST /api/auth/login
+ - POST /api/auth/logout
+ - GET /api/auth/me
+
+
+ - GET /api/items (list with pagination)
+ - POST /api/items (create)
+ - GET /api/items/:id (get single)
+ - PUT /api/items/:id (update)
+ - DELETE /api/items/:id (delete)
+
+
+```
+
+### ui_layout
+```xml
+
+
+ - Header with navigation and user menu
+ - Sidebar for navigation (collapsible on mobile)
+ - Main content area
+ - Footer (optional)
+
+
+ - Logo at top
+ - Navigation links
+ - User profile at bottom
+
+
+```
+
+### design_system
+```xml
+
+
+ - Primary: #3B82F6 (blue)
+ - Background: #FFFFFF (light), #1A1A1A (dark)
+ - Text: #1F2937 (light), #E5E5E5 (dark)
+ - Error: #EF4444
+ - Success: #10B981
+
+
+ - Font family: Inter, system-ui, sans-serif
+ - Headings: font-semibold
+ - Body: font-normal
+
+
+```
+
+### key_interactions
+```xml
+
+
+ 1. User navigates to /login
+ 2. User enters email and password
+ 3. System validates credentials
+ 4. On success: redirect to dashboard
+ 5. On failure: show error message
+
+
+ 1. User clicks "Create New" button
+ 2. Modal form opens
+ 3. User fills required fields
+ 4. User clicks save
+ 5. Item appears in list with success toast
+
+
+```
+
+### implementation_steps
+```xml
+
+
+ Project Setup
+
+ - Initialize frontend with Vite
+ - Set up Express backend
+ - Create database schema
+ - Configure environment variables
+
+
+
+ Authentication
+
+ - Implement registration
+ - Implement login/logout
+ - Add session management
+ - Create protected routes
+
+
+
+```
+
+### success_criteria
+```xml
+
+
+ - All features work as specified
+ - No console errors in browser
+ - Data persists correctly in database
+
+
+ - Responsive on all device sizes
+ - Fast load times (< 2s)
+ - Clear feedback for all actions
+
+
+ - Clean code structure
+ - Proper error handling
+ - Secure authentication
+
+
+```
+
+## Feature Count Guidelines
+
+The Initializer agent expects features distributed across categories:
+
+| Project Complexity | Total Features | Categories |
+|--------------------|----------------|------------|
+| Simple CLI/utility | 100-150 | 5-8 |
+| Medium web app | 200-250 | 10-15 |
+| Complex full-stack | 300-400 | 15-20 |
+
+## GSD to AutoForge Mapping
+
+When converting from GSD codebase mapping:
+
+| GSD Document | Maps To |
+|--------------|---------|
+| STACK.md Languages | `` |
+| STACK.md Runtime | `` |
+| STACK.md Frameworks | ``, `` |
+| ARCHITECTURE.md Pattern | `` |
+| ARCHITECTURE.md Layers | `` categories |
+| ARCHITECTURE.md Data Flow | `` |
+| ARCHITECTURE.md Entry Points | `` |
+| STRUCTURE.md Layout | Informs feature organization |
+| INTEGRATIONS.md APIs | `` |
+| INTEGRATIONS.md Services | `` |
diff --git a/.claude/skills/playwright-cli/SKILL.md b/.claude/skills/playwright-cli/SKILL.md
new file mode 100644
index 000000000..29182e763
--- /dev/null
+++ b/.claude/skills/playwright-cli/SKILL.md
@@ -0,0 +1,259 @@
+---
+name: playwright-cli
+description: Automates browser interactions for web testing, form filling, screenshots, and data extraction. Use when the user needs to navigate websites, interact with web pages, fill forms, take screenshots, test web applications, or extract information from web pages.
+allowed-tools: Bash(playwright-cli:*)
+---
+
+# Browser Automation with playwright-cli
+
+## Quick start
+
+```bash
+# open new browser
+playwright-cli open
+# navigate to a page
+playwright-cli goto https://playwright.dev
+# interact with the page using refs from the snapshot
+playwright-cli click e15
+playwright-cli type "page.click"
+playwright-cli press Enter
+# take a screenshot
+playwright-cli screenshot
+# close the browser
+playwright-cli close
+```
+
+## Commands
+
+### Core
+
+```bash
+playwright-cli open
+# open and navigate right away
+playwright-cli open https://example.com/
+playwright-cli goto https://playwright.dev
+playwright-cli type "search query"
+playwright-cli click e3
+playwright-cli dblclick e7
+playwright-cli fill e5 "user@example.com"
+playwright-cli drag e2 e8
+playwright-cli hover e4
+playwright-cli select e9 "option-value"
+playwright-cli upload ./document.pdf
+playwright-cli check e12
+playwright-cli uncheck e12
+playwright-cli snapshot
+playwright-cli snapshot --filename=after-click.yaml
+playwright-cli eval "document.title"
+playwright-cli eval "el => el.textContent" e5
+playwright-cli dialog-accept
+playwright-cli dialog-accept "confirmation text"
+playwright-cli dialog-dismiss
+playwright-cli resize 1920 1080
+playwright-cli close
+```
+
+### Navigation
+
+```bash
+playwright-cli go-back
+playwright-cli go-forward
+playwright-cli reload
+```
+
+### Keyboard
+
+```bash
+playwright-cli press Enter
+playwright-cli press ArrowDown
+playwright-cli keydown Shift
+playwright-cli keyup Shift
+```
+
+### Mouse
+
+```bash
+playwright-cli mousemove 150 300
+playwright-cli mousedown
+playwright-cli mousedown right
+playwright-cli mouseup
+playwright-cli mouseup right
+playwright-cli mousewheel 0 100
+```
+
+### Save as
+
+```bash
+playwright-cli screenshot
+playwright-cli screenshot e5
+playwright-cli screenshot --filename=page.png
+playwright-cli pdf --filename=page.pdf
+```
+
+### Tabs
+
+```bash
+playwright-cli tab-list
+playwright-cli tab-new
+playwright-cli tab-new https://example.com/page
+playwright-cli tab-close
+playwright-cli tab-close 2
+playwright-cli tab-select 0
+```
+
+### Storage
+
+```bash
+playwright-cli state-save
+playwright-cli state-save auth.json
+playwright-cli state-load auth.json
+
+# Cookies
+playwright-cli cookie-list
+playwright-cli cookie-list --domain=example.com
+playwright-cli cookie-get session_id
+playwright-cli cookie-set session_id abc123
+playwright-cli cookie-set session_id abc123 --domain=example.com --httpOnly --secure
+playwright-cli cookie-delete session_id
+playwright-cli cookie-clear
+
+# LocalStorage
+playwright-cli localstorage-list
+playwright-cli localstorage-get theme
+playwright-cli localstorage-set theme dark
+playwright-cli localstorage-delete theme
+playwright-cli localstorage-clear
+
+# SessionStorage
+playwright-cli sessionstorage-list
+playwright-cli sessionstorage-get step
+playwright-cli sessionstorage-set step 3
+playwright-cli sessionstorage-delete step
+playwright-cli sessionstorage-clear
+```
+
+### Network
+
+```bash
+playwright-cli route "**/*.jpg" --status=404
+playwright-cli route "https://api.example.com/**" --body='{"mock": true}'
+playwright-cli route-list
+playwright-cli unroute "**/*.jpg"
+playwright-cli unroute
+```
+
+### DevTools
+
+```bash
+playwright-cli console
+playwright-cli console warning
+playwright-cli network
+playwright-cli run-code "async page => await page.context().grantPermissions(['geolocation'])"
+playwright-cli tracing-start
+playwright-cli tracing-stop
+playwright-cli video-start
+playwright-cli video-stop video.webm
+```
+
+### Install
+
+```bash
+playwright-cli install --skills
+playwright-cli install-browser
+```
+
+### Configuration
+```bash
+# Use specific browser when creating session
+playwright-cli open --browser=chrome
+playwright-cli open --browser=firefox
+playwright-cli open --browser=webkit
+playwright-cli open --browser=msedge
+# Connect to browser via extension
+playwright-cli open --extension
+
+# Use persistent profile (by default profile is in-memory)
+playwright-cli open --persistent
+# Use persistent profile with custom directory
+playwright-cli open --profile=/path/to/profile
+
+# Start with config file
+playwright-cli open --config=my-config.json
+
+# Close the browser
+playwright-cli close
+# Delete user data for the default session
+playwright-cli delete-data
+```
+
+### Browser Sessions
+
+```bash
+# create new browser session named "mysession" with persistent profile
+playwright-cli -s=mysession open example.com --persistent
+# same with manually specified profile directory (use when requested explicitly)
+playwright-cli -s=mysession open example.com --profile=/path/to/profile
+playwright-cli -s=mysession click e6
+playwright-cli -s=mysession close # stop a named browser
+playwright-cli -s=mysession delete-data # delete user data for persistent session
+
+playwright-cli list
+# Close all browsers
+playwright-cli close-all
+# Forcefully kill all browser processes
+playwright-cli kill-all
+```
+
+## Example: Form submission
+
+```bash
+playwright-cli open https://example.com/form
+playwright-cli snapshot
+
+playwright-cli fill e1 "user@example.com"
+playwright-cli fill e2 "password123"
+playwright-cli click e3
+playwright-cli snapshot
+playwright-cli close
+```
+
+## Example: Multi-tab workflow
+
+```bash
+playwright-cli open https://example.com
+playwright-cli tab-new https://example.com/other
+playwright-cli tab-list
+playwright-cli tab-select 0
+playwright-cli snapshot
+playwright-cli close
+```
+
+## Example: Debugging with DevTools
+
+```bash
+playwright-cli open https://example.com
+playwright-cli click e4
+playwright-cli fill e7 "test"
+playwright-cli console
+playwright-cli network
+playwright-cli close
+```
+
+```bash
+playwright-cli open https://example.com
+playwright-cli tracing-start
+playwright-cli click e4
+playwright-cli fill e7 "test"
+playwright-cli tracing-stop
+playwright-cli close
+```
+
+## Specific tasks
+
+* **Request mocking** [references/request-mocking.md](references/request-mocking.md)
+* **Running Playwright code** [references/running-code.md](references/running-code.md)
+* **Browser session management** [references/session-management.md](references/session-management.md)
+* **Storage state (cookies, localStorage)** [references/storage-state.md](references/storage-state.md)
+* **Test generation** [references/test-generation.md](references/test-generation.md)
+* **Tracing** [references/tracing.md](references/tracing.md)
+* **Video recording** [references/video-recording.md](references/video-recording.md)
diff --git a/.claude/skills/playwright-cli/references/request-mocking.md b/.claude/skills/playwright-cli/references/request-mocking.md
new file mode 100644
index 000000000..9005fda67
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/request-mocking.md
@@ -0,0 +1,87 @@
+# Request Mocking
+
+Intercept, mock, modify, and block network requests.
+
+## CLI Route Commands
+
+```bash
+# Mock with custom status
+playwright-cli route "**/*.jpg" --status=404
+
+# Mock with JSON body
+playwright-cli route "**/api/users" --body='[{"id":1,"name":"Alice"}]' --content-type=application/json
+
+# Mock with custom headers
+playwright-cli route "**/api/data" --body='{"ok":true}' --header="X-Custom: value"
+
+# Remove headers from requests
+playwright-cli route "**/*" --remove-header=cookie,authorization
+
+# List active routes
+playwright-cli route-list
+
+# Remove a route or all routes
+playwright-cli unroute "**/*.jpg"
+playwright-cli unroute
+```
+
+## URL Patterns
+
+```
+**/api/users - Exact path match
+**/api/*/details - Wildcard in path
+**/*.{png,jpg,jpeg} - Match file extensions
+**/search?q=* - Match query parameters
+```
+
+## Advanced Mocking with run-code
+
+For conditional responses, request body inspection, response modification, or delays:
+
+### Conditional Response Based on Request
+
+```bash
+playwright-cli run-code "async page => {
+ await page.route('**/api/login', route => {
+ const body = route.request().postDataJSON();
+ if (body.username === 'admin') {
+ route.fulfill({ body: JSON.stringify({ token: 'mock-token' }) });
+ } else {
+ route.fulfill({ status: 401, body: JSON.stringify({ error: 'Invalid' }) });
+ }
+ });
+}"
+```
+
+### Modify Real Response
+
+```bash
+playwright-cli run-code "async page => {
+ await page.route('**/api/user', async route => {
+ const response = await route.fetch();
+ const json = await response.json();
+ json.isPremium = true;
+ await route.fulfill({ response, json });
+ });
+}"
+```
+
+### Simulate Network Failures
+
+```bash
+playwright-cli run-code "async page => {
+ await page.route('**/api/offline', route => route.abort('internetdisconnected'));
+}"
+# Options: connectionrefused, timedout, connectionreset, internetdisconnected
+```
+
+### Delayed Response
+
+```bash
+playwright-cli run-code "async page => {
+ await page.route('**/api/slow', async route => {
+ await new Promise(r => setTimeout(r, 3000));
+ route.fulfill({ body: JSON.stringify({ data: 'loaded' }) });
+ });
+}"
+```
diff --git a/.claude/skills/playwright-cli/references/running-code.md b/.claude/skills/playwright-cli/references/running-code.md
new file mode 100644
index 000000000..7d6d22fd0
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/running-code.md
@@ -0,0 +1,232 @@
+# Running Custom Playwright Code
+
+Use `run-code` to execute arbitrary Playwright code for advanced scenarios not covered by CLI commands.
+
+## Syntax
+
+```bash
+playwright-cli run-code "async page => {
+ // Your Playwright code here
+ // Access page.context() for browser context operations
+}"
+```
+
+## Geolocation
+
+```bash
+# Grant geolocation permission and set location
+playwright-cli run-code "async page => {
+ await page.context().grantPermissions(['geolocation']);
+ await page.context().setGeolocation({ latitude: 37.7749, longitude: -122.4194 });
+}"
+
+# Set location to London
+playwright-cli run-code "async page => {
+ await page.context().grantPermissions(['geolocation']);
+ await page.context().setGeolocation({ latitude: 51.5074, longitude: -0.1278 });
+}"
+
+# Clear geolocation override
+playwright-cli run-code "async page => {
+ await page.context().clearPermissions();
+}"
+```
+
+## Permissions
+
+```bash
+# Grant multiple permissions
+playwright-cli run-code "async page => {
+ await page.context().grantPermissions([
+ 'geolocation',
+ 'notifications',
+ 'camera',
+ 'microphone'
+ ]);
+}"
+
+# Grant permissions for specific origin
+playwright-cli run-code "async page => {
+ await page.context().grantPermissions(['clipboard-read'], {
+ origin: 'https://example.com'
+ });
+}"
+```
+
+## Media Emulation
+
+```bash
+# Emulate dark color scheme
+playwright-cli run-code "async page => {
+ await page.emulateMedia({ colorScheme: 'dark' });
+}"
+
+# Emulate light color scheme
+playwright-cli run-code "async page => {
+ await page.emulateMedia({ colorScheme: 'light' });
+}"
+
+# Emulate reduced motion
+playwright-cli run-code "async page => {
+ await page.emulateMedia({ reducedMotion: 'reduce' });
+}"
+
+# Emulate print media
+playwright-cli run-code "async page => {
+ await page.emulateMedia({ media: 'print' });
+}"
+```
+
+## Wait Strategies
+
+```bash
+# Wait for network idle
+playwright-cli run-code "async page => {
+ await page.waitForLoadState('networkidle');
+}"
+
+# Wait for specific element
+playwright-cli run-code "async page => {
+ await page.waitForSelector('.loading', { state: 'hidden' });
+}"
+
+# Wait for function to return true
+playwright-cli run-code "async page => {
+ await page.waitForFunction(() => window.appReady === true);
+}"
+
+# Wait with timeout
+playwright-cli run-code "async page => {
+ await page.waitForSelector('.result', { timeout: 10000 });
+}"
+```
+
+## Frames and Iframes
+
+```bash
+# Work with iframe
+playwright-cli run-code "async page => {
+ const frame = page.locator('iframe#my-iframe').contentFrame();
+ await frame.locator('button').click();
+}"
+
+# Get all frames
+playwright-cli run-code "async page => {
+ const frames = page.frames();
+ return frames.map(f => f.url());
+}"
+```
+
+## File Downloads
+
+```bash
+# Handle file download
+playwright-cli run-code "async page => {
+ const [download] = await Promise.all([
+ page.waitForEvent('download'),
+ page.click('a.download-link')
+ ]);
+ await download.saveAs('./downloaded-file.pdf');
+ return download.suggestedFilename();
+}"
+```
+
+## Clipboard
+
+```bash
+# Read clipboard (requires permission)
+playwright-cli run-code "async page => {
+ await page.context().grantPermissions(['clipboard-read']);
+ return await page.evaluate(() => navigator.clipboard.readText());
+}"
+
+# Write to clipboard
+playwright-cli run-code "async page => {
+ await page.evaluate(text => navigator.clipboard.writeText(text), 'Hello clipboard!');
+}"
+```
+
+## Page Information
+
+```bash
+# Get page title
+playwright-cli run-code "async page => {
+ return await page.title();
+}"
+
+# Get current URL
+playwright-cli run-code "async page => {
+ return page.url();
+}"
+
+# Get page content
+playwright-cli run-code "async page => {
+ return await page.content();
+}"
+
+# Get viewport size
+playwright-cli run-code "async page => {
+ return page.viewportSize();
+}"
+```
+
+## JavaScript Execution
+
+```bash
+# Execute JavaScript and return result
+playwright-cli run-code "async page => {
+ return await page.evaluate(() => {
+ return {
+ userAgent: navigator.userAgent,
+ language: navigator.language,
+ cookiesEnabled: navigator.cookieEnabled
+ };
+ });
+}"
+
+# Pass arguments to evaluate
+playwright-cli run-code "async page => {
+ const multiplier = 5;
+ return await page.evaluate(m => document.querySelectorAll('li').length * m, multiplier);
+}"
+```
+
+## Error Handling
+
+```bash
+# Try-catch in run-code
+playwright-cli run-code "async page => {
+ try {
+ await page.click('.maybe-missing', { timeout: 1000 });
+ return 'clicked';
+ } catch (e) {
+ return 'element not found';
+ }
+}"
+```
+
+## Complex Workflows
+
+```bash
+# Login and save state
+playwright-cli run-code "async page => {
+ await page.goto('https://example.com/login');
+ await page.fill('input[name=email]', 'user@example.com');
+ await page.fill('input[name=password]', 'secret');
+ await page.click('button[type=submit]');
+ await page.waitForURL('**/dashboard');
+ await page.context().storageState({ path: 'auth.json' });
+ return 'Login successful';
+}"
+
+# Scrape data from multiple pages
+playwright-cli run-code "async page => {
+ const results = [];
+ for (let i = 1; i <= 3; i++) {
+ await page.goto(\`https://example.com/page/\${i}\`);
+ const items = await page.locator('.item').allTextContents();
+ results.push(...items);
+ }
+ return results;
+}"
+```
diff --git a/.claude/skills/playwright-cli/references/session-management.md b/.claude/skills/playwright-cli/references/session-management.md
new file mode 100644
index 000000000..08c8c90c5
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/session-management.md
@@ -0,0 +1,169 @@
+# Browser Session Management
+
+Run multiple isolated browser sessions concurrently with state persistence.
+
+## Named Browser Sessions
+
+Use `-b` flag to isolate browser contexts:
+
+```bash
+# Browser 1: Authentication flow
+playwright-cli -s=auth open https://app.example.com/login
+
+# Browser 2: Public browsing (separate cookies, storage)
+playwright-cli -s=public open https://example.com
+
+# Commands are isolated by browser session
+playwright-cli -s=auth fill e1 "user@example.com"
+playwright-cli -s=public snapshot
+```
+
+## Browser Session Isolation Properties
+
+Each browser session has independent:
+- Cookies
+- LocalStorage / SessionStorage
+- IndexedDB
+- Cache
+- Browsing history
+- Open tabs
+
+## Browser Session Commands
+
+```bash
+# List all browser sessions
+playwright-cli list
+
+# Stop a browser session (close the browser)
+playwright-cli close # stop the default browser
+playwright-cli -s=mysession close # stop a named browser
+
+# Stop all browser sessions
+playwright-cli close-all
+
+# Forcefully kill all daemon processes (for stale/zombie processes)
+playwright-cli kill-all
+
+# Delete browser session user data (profile directory)
+playwright-cli delete-data # delete default browser data
+playwright-cli -s=mysession delete-data # delete named browser data
+```
+
+## Environment Variable
+
+Set a default browser session name via environment variable:
+
+```bash
+export PLAYWRIGHT_CLI_SESSION="mysession"
+playwright-cli open example.com # Uses "mysession" automatically
+```
+
+## Common Patterns
+
+### Concurrent Scraping
+
+```bash
+#!/bin/bash
+# Scrape multiple sites concurrently
+
+# Start all browsers
+playwright-cli -s=site1 open https://site1.com &
+playwright-cli -s=site2 open https://site2.com &
+playwright-cli -s=site3 open https://site3.com &
+wait
+
+# Take snapshots from each
+playwright-cli -s=site1 snapshot
+playwright-cli -s=site2 snapshot
+playwright-cli -s=site3 snapshot
+
+# Cleanup
+playwright-cli close-all
+```
+
+### A/B Testing Sessions
+
+```bash
+# Test different user experiences
+playwright-cli -s=variant-a open "https://app.com?variant=a"
+playwright-cli -s=variant-b open "https://app.com?variant=b"
+
+# Compare
+playwright-cli -s=variant-a screenshot
+playwright-cli -s=variant-b screenshot
+```
+
+### Persistent Profile
+
+By default, browser profile is kept in memory only. Use `--persistent` flag on `open` to persist the browser profile to disk:
+
+```bash
+# Use persistent profile (auto-generated location)
+playwright-cli open https://example.com --persistent
+
+# Use persistent profile with custom directory
+playwright-cli open https://example.com --profile=/path/to/profile
+```
+
+## Default Browser Session
+
+When `-s` is omitted, commands use the default browser session:
+
+```bash
+# These use the same default browser session
+playwright-cli open https://example.com
+playwright-cli snapshot
+playwright-cli close # Stops default browser
+```
+
+## Browser Session Configuration
+
+Configure a browser session with specific settings when opening:
+
+```bash
+# Open with config file
+playwright-cli open https://example.com --config=.playwright/my-cli.json
+
+# Open with specific browser
+playwright-cli open https://example.com --browser=firefox
+
+# Open in headed mode
+playwright-cli open https://example.com --headed
+
+# Open with persistent profile
+playwright-cli open https://example.com --persistent
+```
+
+## Best Practices
+
+### 1. Name Browser Sessions Semantically
+
+```bash
+# GOOD: Clear purpose
+playwright-cli -s=github-auth open https://github.com
+playwright-cli -s=docs-scrape open https://docs.example.com
+
+# AVOID: Generic names
+playwright-cli -s=s1 open https://github.com
+```
+
+### 2. Always Clean Up
+
+```bash
+# Stop browsers when done
+playwright-cli -s=auth close
+playwright-cli -s=scrape close
+
+# Or stop all at once
+playwright-cli close-all
+
+# If browsers become unresponsive or zombie processes remain
+playwright-cli kill-all
+```
+
+### 3. Delete Stale Browser Data
+
+```bash
+# Remove old browser data to free disk space
+playwright-cli -s=oldsession delete-data
+```
diff --git a/.claude/skills/playwright-cli/references/storage-state.md b/.claude/skills/playwright-cli/references/storage-state.md
new file mode 100644
index 000000000..c856db5e4
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/storage-state.md
@@ -0,0 +1,275 @@
+# Storage Management
+
+Manage cookies, localStorage, sessionStorage, and browser storage state.
+
+## Storage State
+
+Save and restore complete browser state including cookies and storage.
+
+### Save Storage State
+
+```bash
+# Save to auto-generated filename (storage-state-{timestamp}.json)
+playwright-cli state-save
+
+# Save to specific filename
+playwright-cli state-save my-auth-state.json
+```
+
+### Restore Storage State
+
+```bash
+# Load storage state from file
+playwright-cli state-load my-auth-state.json
+
+# Reload page to apply cookies
+playwright-cli open https://example.com
+```
+
+### Storage State File Format
+
+The saved file contains:
+
+```json
+{
+ "cookies": [
+ {
+ "name": "session_id",
+ "value": "abc123",
+ "domain": "example.com",
+ "path": "/",
+ "expires": 1735689600,
+ "httpOnly": true,
+ "secure": true,
+ "sameSite": "Lax"
+ }
+ ],
+ "origins": [
+ {
+ "origin": "https://example.com",
+ "localStorage": [
+ { "name": "theme", "value": "dark" },
+ { "name": "user_id", "value": "12345" }
+ ]
+ }
+ ]
+}
+```
+
+## Cookies
+
+### List All Cookies
+
+```bash
+playwright-cli cookie-list
+```
+
+### Filter Cookies by Domain
+
+```bash
+playwright-cli cookie-list --domain=example.com
+```
+
+### Filter Cookies by Path
+
+```bash
+playwright-cli cookie-list --path=/api
+```
+
+### Get Specific Cookie
+
+```bash
+playwright-cli cookie-get session_id
+```
+
+### Set a Cookie
+
+```bash
+# Basic cookie
+playwright-cli cookie-set session abc123
+
+# Cookie with options
+playwright-cli cookie-set session abc123 --domain=example.com --path=/ --httpOnly --secure --sameSite=Lax
+
+# Cookie with expiration (Unix timestamp)
+playwright-cli cookie-set remember_me token123 --expires=1735689600
+```
+
+### Delete a Cookie
+
+```bash
+playwright-cli cookie-delete session_id
+```
+
+### Clear All Cookies
+
+```bash
+playwright-cli cookie-clear
+```
+
+### Advanced: Multiple Cookies or Custom Options
+
+For complex scenarios like adding multiple cookies at once, use `run-code`:
+
+```bash
+playwright-cli run-code "async page => {
+ await page.context().addCookies([
+ { name: 'session_id', value: 'sess_abc123', domain: 'example.com', path: '/', httpOnly: true },
+ { name: 'preferences', value: JSON.stringify({ theme: 'dark' }), domain: 'example.com', path: '/' }
+ ]);
+}"
+```
+
+## Local Storage
+
+### List All localStorage Items
+
+```bash
+playwright-cli localstorage-list
+```
+
+### Get Single Value
+
+```bash
+playwright-cli localstorage-get token
+```
+
+### Set Value
+
+```bash
+playwright-cli localstorage-set theme dark
+```
+
+### Set JSON Value
+
+```bash
+playwright-cli localstorage-set user_settings '{"theme":"dark","language":"en"}'
+```
+
+### Delete Single Item
+
+```bash
+playwright-cli localstorage-delete token
+```
+
+### Clear All localStorage
+
+```bash
+playwright-cli localstorage-clear
+```
+
+### Advanced: Multiple Operations
+
+For complex scenarios like setting multiple values at once, use `run-code`:
+
+```bash
+playwright-cli run-code "async page => {
+ await page.evaluate(() => {
+ localStorage.setItem('token', 'jwt_abc123');
+ localStorage.setItem('user_id', '12345');
+ localStorage.setItem('expires_at', Date.now() + 3600000);
+ });
+}"
+```
+
+## Session Storage
+
+### List All sessionStorage Items
+
+```bash
+playwright-cli sessionstorage-list
+```
+
+### Get Single Value
+
+```bash
+playwright-cli sessionstorage-get form_data
+```
+
+### Set Value
+
+```bash
+playwright-cli sessionstorage-set step 3
+```
+
+### Delete Single Item
+
+```bash
+playwright-cli sessionstorage-delete step
+```
+
+### Clear sessionStorage
+
+```bash
+playwright-cli sessionstorage-clear
+```
+
+## IndexedDB
+
+### List Databases
+
+```bash
+playwright-cli run-code "async page => {
+ return await page.evaluate(async () => {
+ const databases = await indexedDB.databases();
+ return databases;
+ });
+}"
+```
+
+### Delete Database
+
+```bash
+playwright-cli run-code "async page => {
+ await page.evaluate(() => {
+ indexedDB.deleteDatabase('myDatabase');
+ });
+}"
+```
+
+## Common Patterns
+
+### Authentication State Reuse
+
+```bash
+# Step 1: Login and save state
+playwright-cli open https://app.example.com/login
+playwright-cli snapshot
+playwright-cli fill e1 "user@example.com"
+playwright-cli fill e2 "password123"
+playwright-cli click e3
+
+# Save the authenticated state
+playwright-cli state-save auth.json
+
+# Step 2: Later, restore state and skip login
+playwright-cli state-load auth.json
+playwright-cli open https://app.example.com/dashboard
+# Already logged in!
+```
+
+### Save and Restore Roundtrip
+
+```bash
+# Set up authentication state
+playwright-cli open https://example.com
+playwright-cli eval "() => { document.cookie = 'session=abc123'; localStorage.setItem('user', 'john'); }"
+
+# Save state to file
+playwright-cli state-save my-session.json
+
+# ... later, in a new session ...
+
+# Restore state
+playwright-cli state-load my-session.json
+playwright-cli open https://example.com
+# Cookies and localStorage are restored!
+```
+
+## Security Notes
+
+- Never commit storage state files containing auth tokens
+- Add `*.auth-state.json` to `.gitignore`
+- Delete state files after automation completes
+- Use environment variables for sensitive data
+- By default, sessions run in-memory mode which is safer for sensitive operations
diff --git a/.claude/skills/playwright-cli/references/test-generation.md b/.claude/skills/playwright-cli/references/test-generation.md
new file mode 100644
index 000000000..7a09df387
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/test-generation.md
@@ -0,0 +1,88 @@
+# Test Generation
+
+Generate Playwright test code automatically as you interact with the browser.
+
+## How It Works
+
+Every action you perform with `playwright-cli` generates corresponding Playwright TypeScript code.
+This code appears in the output and can be copied directly into your test files.
+
+## Example Workflow
+
+```bash
+# Start a session
+playwright-cli open https://example.com/login
+
+# Take a snapshot to see elements
+playwright-cli snapshot
+# Output shows: e1 [textbox "Email"], e2 [textbox "Password"], e3 [button "Sign In"]
+
+# Fill form fields - generates code automatically
+playwright-cli fill e1 "user@example.com"
+# Ran Playwright code:
+# await page.getByRole('textbox', { name: 'Email' }).fill('user@example.com');
+
+playwright-cli fill e2 "password123"
+# Ran Playwright code:
+# await page.getByRole('textbox', { name: 'Password' }).fill('password123');
+
+playwright-cli click e3
+# Ran Playwright code:
+# await page.getByRole('button', { name: 'Sign In' }).click();
+```
+
+## Building a Test File
+
+Collect the generated code into a Playwright test:
+
+```typescript
+import { test, expect } from '@playwright/test';
+
+test('login flow', async ({ page }) => {
+ // Generated code from playwright-cli session:
+ await page.goto('https://example.com/login');
+ await page.getByRole('textbox', { name: 'Email' }).fill('user@example.com');
+ await page.getByRole('textbox', { name: 'Password' }).fill('password123');
+ await page.getByRole('button', { name: 'Sign In' }).click();
+
+ // Add assertions
+ await expect(page).toHaveURL(/.*dashboard/);
+});
+```
+
+## Best Practices
+
+### 1. Use Semantic Locators
+
+The generated code uses role-based locators when possible, which are more resilient:
+
+```typescript
+// Generated (good - semantic)
+await page.getByRole('button', { name: 'Submit' }).click();
+
+// Avoid (fragile - CSS selectors)
+await page.locator('#submit-btn').click();
+```
+
+### 2. Explore Before Recording
+
+Take snapshots to understand the page structure before recording actions:
+
+```bash
+playwright-cli open https://example.com
+playwright-cli snapshot
+# Review the element structure
+playwright-cli click e5
+```
+
+### 3. Add Assertions Manually
+
+Generated code captures actions but not assertions. Add expectations in your test:
+
+```typescript
+// Generated action
+await page.getByRole('button', { name: 'Submit' }).click();
+
+// Manual assertion
+await expect(page.getByText('Success')).toBeVisible();
+```
diff --git a/.claude/skills/playwright-cli/references/tracing.md b/.claude/skills/playwright-cli/references/tracing.md
new file mode 100644
index 000000000..7ce7babbd
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/tracing.md
@@ -0,0 +1,139 @@
+# Tracing
+
+Capture detailed execution traces for debugging and analysis. Traces include DOM snapshots, screenshots, network activity, and console logs.
+
+## Basic Usage
+
+```bash
+# Start trace recording
+playwright-cli tracing-start
+
+# Perform actions
+playwright-cli open https://example.com
+playwright-cli click e1
+playwright-cli fill e2 "test"
+
+# Stop trace recording
+playwright-cli tracing-stop
+```
+
+## Trace Output Files
+
+When you start tracing, Playwright creates a `traces/` directory with several files:
+
+### `trace-{timestamp}.trace`
+
+**Action log** - The main trace file containing:
+- Every action performed (clicks, fills, navigations)
+- DOM snapshots before and after each action
+- Screenshots at each step
+- Timing information
+- Console messages
+- Source locations
+
+### `trace-{timestamp}.network`
+
+**Network log** - Complete network activity:
+- All HTTP requests and responses
+- Request headers and bodies
+- Response headers and bodies
+- Timing (DNS, connect, TLS, TTFB, download)
+- Resource sizes
+- Failed requests and errors
+
+### `resources/`
+
+**Resources directory** - Cached resources:
+- Images, fonts, stylesheets, scripts
+- Response bodies for replay
+- Assets needed to reconstruct page state
+
+## What Traces Capture
+
+| Category | Details |
+|----------|---------|
+| **Actions** | Clicks, fills, hovers, keyboard input, navigations |
+| **DOM** | Full DOM snapshot before/after each action |
+| **Screenshots** | Visual state at each step |
+| **Network** | All requests, responses, headers, bodies, timing |
+| **Console** | All console.log, warn, error messages |
+| **Timing** | Precise timing for each operation |
+
+## Use Cases
+
+### Debugging Failed Actions
+
+```bash
+playwright-cli tracing-start
+playwright-cli open https://app.example.com
+
+# This click fails - why?
+playwright-cli click e5
+
+playwright-cli tracing-stop
+# Open trace to see DOM state when click was attempted
+```
+
+### Analyzing Performance
+
+```bash
+playwright-cli tracing-start
+playwright-cli open https://slow-site.com
+playwright-cli tracing-stop
+
+# View network waterfall to identify slow resources
+```
+
+### Capturing Evidence
+
+```bash
+# Record a complete user flow for documentation
+playwright-cli tracing-start
+
+playwright-cli open https://app.example.com/checkout
+playwright-cli fill e1 "4111111111111111"
+playwright-cli fill e2 "12/25"
+playwright-cli fill e3 "123"
+playwright-cli click e4
+
+playwright-cli tracing-stop
+# Trace shows exact sequence of events
+```
+
+## Trace vs Video vs Screenshot
+
+| Feature | Trace | Video | Screenshot |
+|---------|-------|-------|------------|
+| **Format** | .trace file | .webm video | .png/.jpeg image |
+| **DOM inspection** | Yes | No | No |
+| **Network details** | Yes | No | No |
+| **Step-by-step replay** | Yes | Continuous | Single frame |
+| **File size** | Medium | Large | Small |
+| **Best for** | Debugging | Demos | Quick capture |
+
+## Best Practices
+
+### 1. Start Tracing Before the Problem
+
+```bash
+# Trace the entire flow, not just the failing step
+playwright-cli tracing-start
+playwright-cli open https://example.com
+# ... all steps leading to the issue ...
+playwright-cli tracing-stop
+```
+
+### 2. Clean Up Old Traces
+
+Traces can consume significant disk space:
+
+```bash
+# Remove traces older than 7 days
+find .playwright-cli/traces -mtime +7 -delete
+```
+
+## Limitations
+
+- Traces add overhead to automation
+- Large traces can consume significant disk space
+- Some dynamic content may not replay perfectly
diff --git a/.claude/skills/playwright-cli/references/video-recording.md b/.claude/skills/playwright-cli/references/video-recording.md
new file mode 100644
index 000000000..38391b37a
--- /dev/null
+++ b/.claude/skills/playwright-cli/references/video-recording.md
@@ -0,0 +1,43 @@
+# Video Recording
+
+Capture browser automation sessions as video for debugging, documentation, or verification. Produces WebM (VP8/VP9 codec).
+
+## Basic Recording
+
+```bash
+# Start recording
+playwright-cli video-start
+
+# Perform actions
+playwright-cli open https://example.com
+playwright-cli snapshot
+playwright-cli click e1
+playwright-cli fill e2 "test input"
+
+# Stop and save
+playwright-cli video-stop demo.webm
+```
+
+## Best Practices
+
+### 1. Use Descriptive Filenames
+
+```bash
+# Include context in filename
+playwright-cli video-stop recordings/login-flow-2024-01-15.webm
+playwright-cli video-stop recordings/checkout-test-run-42.webm
+```
+
+## Tracing vs Video
+
+| Feature | Video | Tracing |
+|---------|-------|---------|
+| Output | WebM file | Trace file (viewable in Trace Viewer) |
+| Shows | Visual recording | DOM snapshots, network, console, actions |
+| Use case | Demos, documentation | Debugging, analysis |
+| Size | Larger | Smaller |
+
+## Limitations
+
+- Recording adds slight overhead to automation
+- Large recordings can consume significant disk space
diff --git a/.claude/templates/auto_improve_prompt.template.md b/.claude/templates/auto_improve_prompt.template.md
new file mode 100644
index 000000000..d9ac67b93
--- /dev/null
+++ b/.claude/templates/auto_improve_prompt.template.md
@@ -0,0 +1,160 @@
+## YOUR ROLE - AUTO-IMPROVE AGENT
+
+You are running in **auto-improve mode**. Your entire job this session is to make the application **meaningfully better** in exactly ONE way. The project is already finished — all existing features pass. You are here to polish, enhance, and evolve it.
+
+This is a FRESH context window. You have no memory of previous sessions. Previous auto-improve sessions may have already added improvements. Your job is to pick ONE new improvement, implement it, and commit it.
+
+### STEP 1: GET YOUR BEARINGS
+
+Start by orienting yourself:
+
+```bash
+# Understand the project
+pwd
+ls -la
+cat app_spec.txt 2>/dev/null || cat .autoforge/prompts/app_spec.txt 2>/dev/null
+
+# See what's been done recently (previous auto-improvements, other commits)
+git log --oneline -20
+
+# See recent progress notes if they exist
+tail -200 claude-progress.txt 2>/dev/null || true
+```
+
+Then use MCP tools to check feature status:
+
+```
+Use the feature_get_stats tool
+Use the feature_get_summary tool
+```
+
+You are looking at an app that someone is running in "autopilot polish" mode. Respect what is already there. Read some of the actual source to get a feel for the codebase.
+
+### STEP 2: CHOOSE ONE MEANINGFUL IMPROVEMENT
+
+Brainstorm silently, then pick exactly ONE improvement. Valid categories:
+
+- **Performance** — cache a hot path, remove an N+1, memoize an expensive component, debounce a noisy handler
+- **UX / UI polish** — empty states, loading states, error states, keyboard shortcuts, micro-interactions, accessibility
+- **Visual design** — spacing, typography, color hierarchy, alignment, iconography
+- **Small new feature** — a natural next step that fits the app's purpose
+- **Security hardening** — input validation, authorization checks, rate limits, secret handling
+- **Refactor for clarity** — extract a confused function, rename a misleading variable, split a file that has outgrown itself
+- **Accessibility** — focus rings, aria-labels, keyboard navigation, color contrast
+- **Dependency / config** — bump a safe dep, tighten a lint rule that would catch a real class of bugs
+
+**Choose deliberately:**
+- The improvement must be genuinely useful to an end user or to future developers.
+- Prefer improvements that complement what's already there over inventing new scope.
+- If the app has obvious rough edges, fix those first before inventing new features.
+- Do NOT touch any feature on the Kanban that is currently `in_progress` — leave it alone.
+- Avoid duplicating past improvements (read `git log` to see what's already been done).
+
+### STEP 3: ADD THE IMPROVEMENT AS A FEATURE
+
+Call the `feature_create` MCP tool with:
+
+- `category`: e.g., `"Performance"`, `"UX Polish"`, `"Security"`, `"Refactor"`, `"Accessibility"`, `"New Feature"`
+- `name`: a short imperative title, e.g., `"Add empty state to project list"`
+- `description`: 1-3 sentences explaining what the change is and why it matters
+- `steps`: 3-5 concrete acceptance steps (what must be true when this is done)
+
+**Record the returned feature ID.** You will use it in later steps. Then mark it in progress:
+
+```
+Use the feature_mark_in_progress tool with feature_id={your_new_id}
+```
+
+### STEP 4: IMPLEMENT THE IMPROVEMENT
+
+Implement the change fully. Keep scope tight:
+
+- Edit only the files you need to change.
+- Don't add speculative abstractions or "while I'm here" refactors.
+- Don't add comments/docstrings to code you didn't touch.
+- Don't rename things that don't need renaming.
+- If you discover a bug that is NOT your chosen improvement, leave it alone (or note it in `claude-progress.txt` for a future session).
+
+If your improvement is a UI change, actually look at the result — take a screenshot with `playwright-cli` if the dev server is running, or at minimum open the relevant component and verify your edit makes sense.
+
+### STEP 5: VERIFY WITH LINT / TYPECHECK / BUILD
+
+**Mandatory.** Before committing, confirm the code still compiles cleanly. Pick the right commands based on the project type (check `package.json`, `pyproject.toml`, `Cargo.toml`, etc.).
+
+Typical command sets:
+
+- **Node / TypeScript / Vite / Next**: `npm run lint && npm run build`
+ (or `npm run typecheck` if it exists as a separate script)
+- **Python**: `ruff check . && mypy .` (or whatever is configured in `pyproject.toml`)
+- **Rust**: `cargo check && cargo clippy`
+- **Go**: `go vet ./... && go build ./...`
+
+**Resolve any issues your change introduced.** If lint/typecheck/build was already failing before your change (unrelated breakage), do NOT "fix" the unrelated failures — that's scope creep. Revert your change and pick a different improvement if the codebase is in a broken baseline state.
+
+### STEP 6: MARK THE FEATURE PASSING
+
+Call the feature MCP tool:
+
+```
+Use the feature_mark_passing tool with feature_id={your_new_id}
+```
+
+### STEP 7: CREATE A COMMIT
+
+Stage your changes and commit with a **short, concise, TLDR-style message**. One line for the subject, optionally one or two more for the "why". No verbose bullet lists, no trailing summaries.
+
+```bash
+git status
+git add
+git commit -m "Add empty state to project list when no projects exist"
+```
+
+Good commit message examples:
+- `"Cache project stats query to cut dashboard load time"`
+- `"Add keyboard shortcut (Cmd+K) to open command palette"`
+- `"Harden upload endpoint against oversized files"`
+- `"Extract confused session handling into its own module"`
+
+Bad commit message examples:
+- `"Various improvements"` (too vague)
+- `"Made the app better by implementing several changes to improve UX including..."` (too long)
+
+### STEP 8: EXIT THIS SESSION
+
+When the commit is created successfully, your work for this session is done. Do NOT try to find a second improvement — one per session is the rule. Stop and let the next scheduled tick handle the next improvement.
+
+---
+
+## GUARDRAILS (READ CAREFULLY)
+
+1. **One improvement per session.** If you finish early, don't start another. Exit cleanly.
+2. **Never skip lint / typecheck / build.** If they fail, fix or revert.
+3. **Never commit broken code.** A commit with failing lint/build is worse than no commit.
+4. **Don't touch features other agents are working on** (anything with `in_progress=True`).
+5. **Don't bypass the feature MCP tools.** Create a real Kanban feature for your change so it shows up in the UI.
+6. **Keep commit messages under 72 characters for the subject line.**
+7. **Don't add dependencies you don't need.** If the improvement needs a new package, be sure it's justified.
+8. **Respect the existing architecture.** Don't rewrite patterns the project has already committed to.
+
+---
+
+## BROWSER AUTOMATION (OPTIONAL)
+
+If your improvement is visual and the dev server is running, you may use `playwright-cli` to verify it renders correctly:
+
+- Open: `playwright-cli open http://localhost:PORT`
+- Screenshot: `playwright-cli screenshot`
+- Read the screenshot file to verify visual appearance
+- Close: `playwright-cli close`
+
+Browser verification is **optional** in auto-improve mode. Lint + typecheck + build is mandatory; visual verification is a bonus when relevant.
+
+---
+
+## SUCCESS CRITERIA
+
+A successful auto-improve session ends with:
+1. One new feature on the Kanban, marked passing.
+2. A clean git commit with a short TLDR message.
+3. No lint / typecheck / build errors introduced.
+4. The agent exits cleanly without starting a second improvement.
diff --git a/.claude/templates/coding_prompt.template.md b/.claude/templates/coding_prompt.template.md
index 6da10a2f8..832eb5996 100644
--- a/.claude/templates/coding_prompt.template.md
+++ b/.claude/templates/coding_prompt.template.md
@@ -17,8 +17,8 @@ ls -la
# 3. Read the project specification to understand what you're building
cat app_spec.txt
-# 4. Read progress notes from previous sessions
-cat claude-progress.txt
+# 4. Read progress notes from previous sessions (last 500 lines to avoid context overflow)
+tail -500 claude-progress.txt
# 5. Check recent git history
git log --oneline -20
@@ -29,9 +29,6 @@ Then use MCP tools to check feature status:
```
# 6. Get progress statistics (passing/total counts)
Use the feature_get_stats tool
-
-# 7. Get the next feature to work on
-Use the feature_get_next tool
```
Understanding the `app_spec.txt` is critical - it contains the full requirements
@@ -48,89 +45,25 @@ chmod +x init.sh
Otherwise, start servers manually and document the process.
-### STEP 3: VERIFICATION TEST (CRITICAL!)
-
-**MANDATORY BEFORE NEW WORK:**
-
-The previous session may have introduced bugs. Before implementing anything
-new, you MUST run verification tests.
-
-Run 1-2 of the features marked as passing that are most core to the app's functionality to verify they still work.
-
-To get passing features for regression testing:
-
-```
-Use the feature_get_for_regression tool (returns up to 3 random passing features)
-```
-
-For example, if this were a chat app, you should perform a test that logs into the app, sends a message, and gets a response.
-
-**If you find ANY issues (functional or visual):**
-
-- Mark that feature as "passes": false immediately
-- Add issues to a list
-- Fix all issues BEFORE moving to new features
-- This includes UI bugs like:
- - White-on-white text or poor contrast
- - Random characters displayed
- - Incorrect timestamps
- - Layout issues or overflow
- - Buttons too close together
- - Missing hover states
- - Console errors
-
-### STEP 4: CHOOSE ONE FEATURE TO IMPLEMENT
+### STEP 3: GET YOUR ASSIGNED FEATURE
#### TEST-DRIVEN DEVELOPMENT MINDSET (CRITICAL)
-Features are **test cases** that drive development. This is test-driven development:
-
-- **If you can't test a feature because functionality doesn't exist → BUILD IT**
-- You are responsible for implementing ALL required functionality
-- Never assume another process will build it later
-- "Missing functionality" is NOT a blocker - it's your job to create it
+Features are **test cases** that drive development. If functionality doesn't exist, **BUILD IT** -- you are responsible for implementing ALL required functionality. Missing pages, endpoints, database tables, or components are NOT blockers; they are your job to create.
-**Example:** Feature says "User can filter flashcards by difficulty level"
-- WRONG: "Flashcard page doesn't exist yet" → skip feature
-- RIGHT: "Flashcard page doesn't exist yet" → build flashcard page → implement filter → test feature
-
-Get the next feature to implement:
+**Note:** Your feature has been pre-assigned by the orchestrator. Use `feature_get_by_id` with your assigned feature ID to get the details. Then mark it as in-progress:
```
-# Get the highest-priority pending feature
-Use the feature_get_next tool
+Use the feature_mark_in_progress tool with feature_id={your_assigned_id}
```
-Once you've retrieved the feature, **immediately mark it as in-progress**:
-
-```
-# Mark feature as in-progress to prevent other sessions from working on it
-Use the feature_mark_in_progress tool with feature_id=42
-```
+If you get "already in-progress" error, that's OK - continue with implementation.
-Focus on completing one feature perfectly and completing its testing steps in this session before moving on to other features.
-It's ok if you only complete one feature in this session, as there will be more sessions later that continue to make progress.
+Focus on completing one feature perfectly in this session. It's ok if you only complete one feature, as more sessions will follow.
#### When to Skip a Feature (EXTREMELY RARE)
-**Skipping should almost NEVER happen.** Only skip for truly external blockers you cannot control:
-
-- **External API not configured**: Third-party service credentials missing (e.g., Stripe keys, OAuth secrets)
-- **External service unavailable**: Dependency on service that's down or inaccessible
-- **Environment limitation**: Hardware or system requirement you cannot fulfill
-
-**NEVER skip because:**
-
-| Situation | Wrong Action | Correct Action |
-|-----------|--------------|----------------|
-| "Page doesn't exist" | Skip | Create the page |
-| "API endpoint missing" | Skip | Implement the endpoint |
-| "Database table not ready" | Skip | Create the migration |
-| "Component not built" | Skip | Build the component |
-| "No data to test with" | Skip | Create test data or build data entry flow |
-| "Feature X needs to be done first" | Skip | Build feature X as part of this feature |
-
-If a feature requires building other functionality first, **build that functionality**. You are the coding agent - your job is to make the feature work, not to defer it.
+Only skip for truly external blockers: missing third-party credentials (Stripe keys, OAuth secrets), unavailable external services, or unfulfillable environment requirements. **NEVER** skip because a page, endpoint, component, or data doesn't exist yet -- build it. If a feature requires other functionality first, build that functionality as part of this feature.
If you must skip (truly external blocker only):
@@ -140,119 +73,69 @@ Use the feature_skip tool with feature_id={id}
Document the SPECIFIC external blocker in `claude-progress.txt`. "Functionality not built" is NEVER a valid reason.
-### STEP 5: IMPLEMENT THE FEATURE
+### STEP 4: IMPLEMENT THE FEATURE
Implement the chosen feature thoroughly:
1. Write the code (frontend and/or backend as needed)
-2. Test manually using browser automation (see Step 6)
+2. Test manually using browser automation (see Step 5)
3. Fix any issues discovered
4. Verify the feature works end-to-end
-### STEP 6: VERIFY WITH BROWSER AUTOMATION
+### STEP 5: VERIFY WITH BROWSER AUTOMATION
**CRITICAL:** You MUST verify features through the actual UI.
-Use browser automation tools:
+Use `playwright-cli` for browser automation:
-- Navigate to the app in a real browser
-- Interact like a human user (click, type, scroll)
-- Take screenshots at each step
-- Verify both functionality AND visual appearance
+- Open the browser: `playwright-cli open http://localhost:PORT`
+- Take a snapshot to see page elements: `playwright-cli snapshot`
+- Read the snapshot YAML file to see element refs
+- Click elements by ref: `playwright-cli click e5`
+- Type text: `playwright-cli type "search query"`
+- Fill form fields: `playwright-cli fill e3 "value"`
+- Take screenshots: `playwright-cli screenshot`
+- Read the screenshot file to verify visual appearance
+- Check console errors: `playwright-cli console`
+- Close browser when done: `playwright-cli close`
-**DO:**
+**Token-efficient workflow:** `playwright-cli screenshot` and `snapshot` save files
+to `.playwright-cli/`. You will see a file link in the output. Read the file only
+when you need to verify visual appearance or find element refs.
+**DO:**
- Test through the UI with clicks and keyboard input
-- Take screenshots to verify visual appearance
-- Check for console errors in browser
+- Take screenshots and read them to verify visual appearance
+- Check for console errors with `playwright-cli console`
- Verify complete user workflows end-to-end
+- Always run `playwright-cli close` when finished testing
**DON'T:**
-
-- Only test with curl commands (backend testing alone is insufficient)
-- Use JavaScript evaluation to bypass UI (no shortcuts)
+- Only test with curl commands
+- Use JavaScript evaluation to bypass UI (`eval` and `run-code` are blocked)
- Skip visual verification
- Mark tests passing without thorough verification
-### STEP 6.5: MANDATORY VERIFICATION CHECKLIST (BEFORE MARKING ANY TEST PASSING)
-
-**You MUST complete ALL of these checks before marking any feature as "passes": true**
-
-#### Security Verification (for protected features)
+### STEP 5.5: MANDATORY VERIFICATION CHECKLIST (BEFORE MARKING ANY TEST PASSING)
-- [ ] Feature respects user role permissions
-- [ ] Unauthenticated access is blocked (redirects to login)
-- [ ] API endpoint checks authorization (returns 401/403 appropriately)
-- [ ] Cannot access other users' data by manipulating URLs
+**Complete ALL applicable checks before marking any feature as passing:**
-#### Real Data Verification (CRITICAL - NO MOCK DATA)
+- **Security:** Feature respects role permissions; unauthenticated access blocked; API checks auth (401/403); no cross-user data leaks via URL manipulation
+- **Real Data:** Create unique test data via UI, verify it appears, refresh to confirm persistence, delete and verify removal. No unexplained data (indicates mocks). Dashboard counts reflect real numbers
+- **Mock Data Grep:** Run STEP 5.6 grep checks - no hits in src/ (excluding tests). No globalThis, devStore, or dev-store patterns
+- **Server Restart:** For data features, run STEP 5.7 - data persists across server restart
+- **Navigation:** All buttons link to existing routes, no 404s, back button works, edit/view/delete links have correct IDs
+- **Integration:** Zero JS console errors, no 500s in network tab, API data matches UI, loading/error states work
-- [ ] Created unique test data via UI (e.g., "TEST_12345_VERIFY_ME")
-- [ ] Verified the EXACT data I created appears in UI
-- [ ] Refreshed page - data persists (proves database storage)
-- [ ] Deleted the test data - verified it's gone everywhere
-- [ ] NO unexplained data appeared (would indicate mock data)
-- [ ] Dashboard/counts reflect real numbers after my changes
+### STEP 5.6: MOCK DATA DETECTION (Before marking passing)
-#### Navigation Verification
+Before marking a feature passing, grep for mock/placeholder data patterns in src/ (excluding test files): `globalThis`, `devStore`, `dev-store`, `mockDb`, `mockData`, `fakeData`, `sampleData`, `dummyData`, `testData`, `TODO.*real`, `TODO.*database`, `STUB`, `MOCK`, `isDevelopment`, `isDev`. Any hits in production code must be investigated and fixed. Also create unique test data (e.g., "TEST_12345"), verify it appears in UI, then delete and confirm removal - unexplained data indicates mock implementations.
-- [ ] All buttons on this page link to existing routes
-- [ ] No 404 errors when clicking any interactive element
-- [ ] Back button returns to correct previous page
-- [ ] Related links (edit, view, delete) have correct IDs in URLs
+### STEP 5.7: SERVER RESTART PERSISTENCE TEST (MANDATORY for data features)
-#### Integration Verification
-
-- [ ] Console shows ZERO JavaScript errors
-- [ ] Network tab shows successful API calls (no 500s)
-- [ ] Data returned from API matches what UI displays
-- [ ] Loading states appeared during API calls
-- [ ] Error states handle failures gracefully
-
-### STEP 6.6: MOCK DATA DETECTION SWEEP
-
-**Run this sweep AFTER EVERY FEATURE before marking it as passing:**
-
-#### 1. Code Pattern Search
-
-Search the codebase for forbidden patterns:
-
-```bash
-# Search for mock data patterns
-grep -r "mockData\|fakeData\|sampleData\|dummyData\|testData" --include="*.js" --include="*.ts" --include="*.jsx" --include="*.tsx"
-grep -r "// TODO\|// FIXME\|// STUB\|// MOCK" --include="*.js" --include="*.ts" --include="*.jsx" --include="*.tsx"
-grep -r "hardcoded\|placeholder" --include="*.js" --include="*.ts" --include="*.jsx" --include="*.tsx"
-```
+For any feature involving CRUD or data persistence: create unique test data (e.g., "RESTART_TEST_12345"), verify it exists, then fully stop and restart the dev server. After restart, verify the test data still exists. If data is gone, the implementation uses in-memory storage -- run STEP 5.6 greps, find the mock pattern, and replace with real database queries. Clean up test data after verification. This test catches in-memory stores like `globalThis.devStore` that pass all other tests but lose data on restart.
-**If ANY matches found related to your feature - FIX THEM before proceeding.**
-
-#### 2. Runtime Verification
-
-For ANY data displayed in UI:
-
-1. Create NEW data with UNIQUE content (e.g., "TEST_12345_DELETE_ME")
-2. Verify that EXACT content appears in the UI
-3. Delete the record
-4. Verify it's GONE from the UI
-5. **If you see data that wasn't created during testing - IT'S MOCK DATA. Fix it.**
-
-#### 3. Database Verification
-
-Check that:
-
-- Database tables contain only data you created during tests
-- Counts/statistics match actual database record counts
-- No seed data is masquerading as user data
-
-#### 4. API Response Verification
-
-For API endpoints used by this feature:
-
-- Call the endpoint directly
-- Verify response contains actual database data
-- Empty database = empty response (not pre-populated mock data)
-
-### STEP 7: UPDATE FEATURE STATUS (CAREFULLY!)
+### STEP 6: UPDATE FEATURE STATUS (CAREFULLY!)
**YOU CAN ONLY MODIFY ONE FIELD: "passes"**
@@ -271,24 +154,30 @@ Use the feature_mark_passing tool with feature_id=42
- Combine or consolidate features
- Reorder features
-**ONLY MARK A FEATURE AS PASSING AFTER VERIFICATION WITH SCREENSHOTS.**
+**ONLY MARK A FEATURE AS PASSING AFTER VERIFICATION WITH BROWSER AUTOMATION.**
+
+### STEP 7: COMMIT YOUR PROGRESS
-### STEP 8: COMMIT YOUR PROGRESS
+Make a descriptive git commit.
-Make a descriptive git commit:
+**Git Commit Rules:**
+- ALWAYS use simple `-m` flag for commit messages
+- NEVER use heredocs (`cat <
-
-
-## YOLO MODE - Rapid Prototyping (Testing Disabled)
-
-**WARNING:** This mode skips all browser testing and regression tests.
-Features are marked as passing after lint/type-check succeeds.
-Use for rapid prototyping only - not for production-quality development.
-
----
-
-## YOUR ROLE - CODING AGENT (YOLO MODE)
-
-You are continuing work on a long-running autonomous development task.
-This is a FRESH context window - you have no memory of previous sessions.
-
-### STEP 1: GET YOUR BEARINGS (MANDATORY)
-
-Start by orienting yourself:
-
-```bash
-# 1. See your working directory
-pwd
-
-# 2. List files to understand project structure
-ls -la
-
-# 3. Read the project specification to understand what you're building
-cat app_spec.txt
-
-# 4. Read progress notes from previous sessions
-cat claude-progress.txt
-
-# 5. Check recent git history
-git log --oneline -20
-```
-
-Then use MCP tools to check feature status:
-
-```
-# 6. Get progress statistics (passing/total counts)
-Use the feature_get_stats tool
-
-# 7. Get the next feature to work on
-Use the feature_get_next tool
-```
-
-Understanding the `app_spec.txt` is critical - it contains the full requirements
-for the application you're building.
-
-### STEP 2: START SERVERS (IF NOT RUNNING)
-
-If `init.sh` exists, run it:
-
-```bash
-chmod +x init.sh
-./init.sh
-```
-
-Otherwise, start servers manually and document the process.
-
-### STEP 3: CHOOSE ONE FEATURE TO IMPLEMENT
-
-Get the next feature to implement:
-
-```
-# Get the highest-priority pending feature
-Use the feature_get_next tool
-```
-
-Once you've retrieved the feature, **immediately mark it as in-progress**:
-
-```
-# Mark feature as in-progress to prevent other sessions from working on it
-Use the feature_mark_in_progress tool with feature_id=42
-```
-
-Focus on completing one feature in this session before moving on to other features.
-It's ok if you only complete one feature in this session, as there will be more sessions later that continue to make progress.
-
-#### When to Skip a Feature (EXTREMELY RARE)
-
-**Skipping should almost NEVER happen.** Only skip for truly external blockers you cannot control:
-
-- **External API not configured**: Third-party service credentials missing (e.g., Stripe keys, OAuth secrets)
-- **External service unavailable**: Dependency on service that's down or inaccessible
-- **Environment limitation**: Hardware or system requirement you cannot fulfill
-
-**NEVER skip because:**
-
-| Situation | Wrong Action | Correct Action |
-|-----------|--------------|----------------|
-| "Page doesn't exist" | Skip | Create the page |
-| "API endpoint missing" | Skip | Implement the endpoint |
-| "Database table not ready" | Skip | Create the migration |
-| "Component not built" | Skip | Build the component |
-| "No data to test with" | Skip | Create test data or build data entry flow |
-| "Feature X needs to be done first" | Skip | Build feature X as part of this feature |
-
-If a feature requires building other functionality first, **build that functionality**. You are the coding agent - your job is to make the feature work, not to defer it.
-
-If you must skip (truly external blocker only):
-
-```
-Use the feature_skip tool with feature_id={id}
-```
-
-Document the SPECIFIC external blocker in `claude-progress.txt`. "Functionality not built" is NEVER a valid reason.
-
-### STEP 4: IMPLEMENT THE FEATURE
-
-Implement the chosen feature thoroughly:
-
-1. Write the code (frontend and/or backend as needed)
-2. Ensure proper error handling
-3. Follow existing code patterns in the codebase
-
-### STEP 5: VERIFY WITH LINT AND TYPE CHECK (YOLO MODE)
-
-**In YOLO mode, verification is done through static analysis only.**
-
-Run the appropriate lint and type-check commands for your project:
-
-**For TypeScript/JavaScript projects:**
-```bash
-npm run lint
-npm run typecheck # or: npx tsc --noEmit
-```
-
-**For Python projects:**
-```bash
-ruff check .
-mypy .
-```
-
-**If lint/type-check passes:** Proceed to mark the feature as passing.
-
-**If lint/type-check fails:** Fix the errors before proceeding.
-
-### STEP 6: UPDATE FEATURE STATUS
-
-**YOU CAN ONLY MODIFY ONE FIELD: "passes"**
-
-After lint/type-check passes, mark the feature as passing:
-
-```
-# Mark feature #42 as passing (replace 42 with the actual feature ID)
-Use the feature_mark_passing tool with feature_id=42
-```
-
-**NEVER:**
-
-- Delete features
-- Edit feature descriptions
-- Modify feature steps
-- Combine or consolidate features
-- Reorder features
-
-### STEP 7: COMMIT YOUR PROGRESS
-
-Make a descriptive git commit:
-
-```bash
-git add .
-git commit -m "Implement [feature name] - YOLO mode
-
-- Added [specific changes]
-- Lint/type-check passing
-- Marked feature #X as passing
-"
-```
-
-### STEP 8: UPDATE PROGRESS NOTES
-
-Update `claude-progress.txt` with:
-
-- What you accomplished this session
-- Which feature(s) you completed
-- Any issues discovered or fixed
-- What should be worked on next
-- Current completion status (e.g., "45/200 features passing")
-
-### STEP 9: END SESSION CLEANLY
-
-Before context fills up:
-
-1. Commit all working code
-2. Update claude-progress.txt
-3. Mark features as passing if lint/type-check verified
-4. Ensure no uncommitted changes
-5. Leave app in working state
-
----
-
-## FEATURE TOOL USAGE RULES (CRITICAL - DO NOT VIOLATE)
-
-The feature tools exist to reduce token usage. **DO NOT make exploratory queries.**
-
-### ALLOWED Feature Tools (ONLY these):
-
-```
-# 1. Get progress stats (passing/in_progress/total counts)
-feature_get_stats
-
-# 2. Get the NEXT feature to work on (one feature only)
-feature_get_next
-
-# 3. Mark a feature as in-progress (call immediately after feature_get_next)
-feature_mark_in_progress with feature_id={id}
-
-# 4. Mark a feature as passing (after lint/type-check succeeds)
-feature_mark_passing with feature_id={id}
-
-# 5. Skip a feature (moves to end of queue) - ONLY when blocked by dependency
-feature_skip with feature_id={id}
-
-# 6. Clear in-progress status (when abandoning a feature)
-feature_clear_in_progress with feature_id={id}
-```
-
-### RULES:
-
-- Do NOT try to fetch lists of all features
-- Do NOT query features by category
-- Do NOT list all pending features
-
-**You do NOT need to see all features.** The feature_get_next tool tells you exactly what to work on. Trust it.
-
----
-
-## EMAIL INTEGRATION (DEVELOPMENT MODE)
-
-When building applications that require email functionality (password resets, email verification, notifications, etc.), you typically won't have access to a real email service or the ability to read email inboxes.
-
-**Solution:** Configure the application to log emails to the terminal instead of sending them.
-
-- Password reset links should be printed to the console
-- Email verification links should be printed to the console
-- Any notification content should be logged to the terminal
-
-**During testing:**
-
-1. Trigger the email action (e.g., click "Forgot Password")
-2. Check the terminal/server logs for the generated link
-3. Use that link directly to verify the functionality works
-
-This allows you to fully test email-dependent flows without needing external email services.
-
----
-
-## IMPORTANT REMINDERS (YOLO MODE)
-
-**Your Goal:** Rapidly prototype the application with all features implemented
-
-**This Session's Goal:** Complete at least one feature
-
-**Quality Bar (YOLO Mode):**
-
-- Code compiles without errors (lint/type-check passing)
-- Follows existing code patterns
-- Basic error handling in place
-- Features are implemented according to spec
-
-**Note:** Browser testing and regression testing are SKIPPED in YOLO mode.
-Features may have bugs that would be caught by manual testing.
-Use standard mode for production-quality verification.
-
-**You have unlimited time.** Take as long as needed to implement features correctly.
-The most important thing is that you leave the code base in a clean state before
-terminating the session (Step 9).
-
----
-
-Begin by running Step 1 (Get Your Bearings).
diff --git a/.claude/templates/initializer_prompt.template.md b/.claude/templates/initializer_prompt.template.md
index 312cd1792..bb914e2d4 100644
--- a/.claude/templates/initializer_prompt.template.md
+++ b/.claude/templates/initializer_prompt.template.md
@@ -26,63 +26,182 @@ which is the single source of truth for what needs to be built.
**Creating Features:**
-Use the feature_create_bulk tool to add all features at once:
-
-```
-Use the feature_create_bulk tool with features=[
- {
- "category": "functional",
- "name": "Brief feature name",
- "description": "Brief description of the feature and what this test verifies",
- "steps": [
- "Step 1: Navigate to relevant page",
- "Step 2: Perform action",
- "Step 3: Verify expected result"
- ]
- },
- {
- "category": "style",
- "name": "Brief feature name",
- "description": "Brief description of UI/UX requirement",
- "steps": [
- "Step 1: Navigate to page",
- "Step 2: Take screenshot",
- "Step 3: Verify visual requirements"
- ]
- }
-]
-```
+Use the feature_create_bulk tool to add all features at once. You can create features in batches if there are many (e.g., 50 at a time).
**Notes:**
- IDs and priorities are assigned automatically based on order
- All features start with `passes: false` by default
-- You can create features in batches if there are many (e.g., 50 at a time)
**Requirements for features:**
- Feature count must match the `feature_count` specified in app_spec.txt
- Reference tiers for other projects:
- - **Simple apps**: ~150 tests
- - **Medium apps**: ~250 tests
- - **Complex apps**: ~400+ tests
+ - **Simple apps**: ~165 tests (includes 5 infrastructure)
+ - **Medium apps**: ~265 tests (includes 5 infrastructure)
+ - **Advanced apps**: ~405+ tests (includes 5 infrastructure)
- Both "functional" and "style" categories
- Mix of narrow tests (2-5 steps) and comprehensive tests (10+ steps)
- At least 25 tests MUST have 10+ steps each (more for complex apps)
- Order features by priority: fundamental features first (the API assigns priority based on order)
-- All features start with `passes: false` automatically
- Cover every feature in the spec exhaustively
- **MUST include tests from ALL 20 mandatory categories below**
---
+## FEATURE DEPENDENCIES (MANDATORY)
+
+Dependencies enable **parallel execution** of independent features. When specified correctly, multiple agents can work on unrelated features simultaneously, dramatically speeding up development.
+
+**Why this matters:** Without dependencies, features execute in random order, causing logical issues (e.g., "Edit user" before "Create user") and preventing efficient parallelization.
+
+### Dependency Rules
+
+1. **Use `depends_on_indices`** (0-based array indices) to reference dependencies
+2. **Can only depend on EARLIER features** (index must be less than current position)
+3. **No circular dependencies** allowed
+4. **Maximum 20 dependencies** per feature
+5. **Infrastructure features (indices 0-4)** have NO dependencies - they run FIRST
+6. **ALL features after index 4** MUST depend on `[0, 1, 2, 3, 4]` (infrastructure)
+7. **60% of features after index 10** should have additional dependencies beyond infrastructure
+
+### Dependency Types
+
+| Type | Example |
+|------|---------|
+| Data | "Edit item" depends on "Create item" |
+| Auth | "View dashboard" depends on "User can log in" |
+| Navigation | "Modal close works" depends on "Modal opens" |
+| UI | "Filter results" depends on "Display results list" |
+
+### Wide Graph Pattern (REQUIRED)
+
+Create WIDE dependency graphs, not linear chains:
+- **BAD:** A -> B -> C -> D -> E (linear chain, only 1 feature runs at a time)
+- **GOOD:** A -> B, A -> C, A -> D, B -> E, C -> E (wide graph, parallel execution)
+
+### Complete Example
+
+```json
+[
+ // INFRASTRUCTURE TIER (indices 0-4, no dependencies) - MUST run first
+ { "name": "Database connection established", "category": "functional" },
+ { "name": "Database schema applied correctly", "category": "functional" },
+ { "name": "Data persists across server restart", "category": "functional" },
+ { "name": "No mock data patterns in codebase", "category": "functional" },
+ { "name": "Backend API queries real database", "category": "functional" },
+
+ // FOUNDATION TIER (indices 5-7, depend on infrastructure)
+ { "name": "App loads without errors", "category": "functional", "depends_on_indices": [0, 1, 2, 3, 4] },
+ { "name": "Navigation bar displays", "category": "style", "depends_on_indices": [0, 1, 2, 3, 4] },
+ { "name": "Homepage renders correctly", "category": "functional", "depends_on_indices": [0, 1, 2, 3, 4] },
+
+ // AUTH TIER (indices 8-10, depend on foundation + infrastructure)
+ { "name": "User can register", "depends_on_indices": [0, 1, 2, 3, 4, 5] },
+ { "name": "User can login", "depends_on_indices": [0, 1, 2, 3, 4, 5, 8] },
+ { "name": "User can logout", "depends_on_indices": [0, 1, 2, 3, 4, 9] },
+
+ // CORE CRUD TIER (indices 11-14) - WIDE GRAPH: all 4 depend on login
+ { "name": "User can create todo", "depends_on_indices": [0, 1, 2, 3, 4, 9] },
+ { "name": "User can view todos", "depends_on_indices": [0, 1, 2, 3, 4, 9] },
+ { "name": "User can edit todo", "depends_on_indices": [0, 1, 2, 3, 4, 9, 11] },
+ { "name": "User can delete todo", "depends_on_indices": [0, 1, 2, 3, 4, 9, 11] },
+
+ // ADVANCED TIER (indices 15-16) - both depend on view, not each other
+ { "name": "User can filter todos", "depends_on_indices": [0, 1, 2, 3, 4, 12] },
+ { "name": "User can search todos", "depends_on_indices": [0, 1, 2, 3, 4, 12] }
+]
+```
+
+**Result:** With 3 parallel agents, this project completes efficiently with proper database validation first.
+
+---
+
+## MANDATORY INFRASTRUCTURE FEATURES (Indices 0-4)
+
+**CRITICAL:** Create these FIRST, before any functional features. These features ensure the application uses a real database, not mock data or in-memory storage.
+
+| Index | Name | Test Steps |
+|-------|------|------------|
+| 0 | Database connection established | Start server → check logs for DB connection → health endpoint returns DB status |
+| 1 | Database schema applied correctly | Connect to DB directly → list tables → verify schema matches spec |
+| 2 | Data persists across server restart | Create via API → STOP server completely → START server → query API → data still exists |
+| 3 | No mock data patterns in codebase | Run grep for prohibited patterns → must return empty |
+| 4 | Backend API queries real database | Check server logs → SQL/DB queries appear for API calls |
+
+**ALL other features MUST depend on indices [0, 1, 2, 3, 4].**
+
+### Infrastructure Feature Descriptions
+
+**Feature 0 - Database connection established:**
+```text
+Steps:
+1. Start the development server
+2. Check server logs for database connection message
+3. Call health endpoint (e.g., GET /api/health)
+4. Verify response includes database status: connected
+```
+
+**Feature 1 - Database schema applied correctly:**
+```text
+Steps:
+1. Connect to database directly (sqlite3, psql, etc.)
+2. List all tables in the database
+3. Verify tables match what's defined in app_spec.txt
+4. Verify key columns exist on each table
+```
+
+**Feature 2 - Data persists across server restart (CRITICAL):**
+```text
+Steps:
+1. Create unique test data via API (e.g., POST /api/items with name "RESTART_TEST_12345")
+2. Verify data appears in API response (GET /api/items)
+3. STOP the server completely (kill by port to avoid killing unrelated Node processes):
+ - Unix/macOS: lsof -ti :$PORT | xargs kill -9 2>/dev/null || true && sleep 5
+ - Windows: FOR /F "tokens=5" %a IN ('netstat -aon ^| find ":$PORT"') DO taskkill /F /PID %a 2>nul
+ - Note: Replace $PORT with actual port (e.g., 3000)
+4. Verify server is stopped: lsof -ti :$PORT returns nothing (or netstat on Windows)
+5. RESTART the server: ./init.sh & sleep 15
+6. Query API again: GET /api/items
+7. Verify "RESTART_TEST_12345" still exists
+8. If data is GONE → CRITICAL FAILURE (in-memory storage detected)
+9. Clean up test data
+```
+
+**Feature 3 - No mock data patterns in codebase:**
+```text
+Steps:
+1. Run: grep -r "globalThis\." --include="*.ts" --include="*.tsx" --include="*.js" src/
+2. Run: grep -r "dev-store\|devStore\|DevStore\|mock-db\|mockDb" --include="*.ts" --include="*.tsx" --include="*.js" src/
+3. Run: grep -r "mockData\|testData\|fakeData\|sampleData\|dummyData" --include="*.ts" --include="*.tsx" --include="*.js" src/
+4. Run: grep -r "TODO.*real\|TODO.*database\|TODO.*API\|STUB\|MOCK" --include="*.ts" --include="*.tsx" --include="*.js" src/
+5. Run: grep -r "isDevelopment\|isDev\|process\.env\.NODE_ENV.*development" --include="*.ts" --include="*.tsx" --include="*.js" src/
+6. Run: grep -r "new Map\(\)\|new Set\(\)" --include="*.ts" --include="*.tsx" --include="*.js" src/ 2>/dev/null
+7. Run: grep -E "json-server|miragejs|msw" package.json
+8. ALL grep commands must return empty (exit code 1)
+9. If any returns results → investigate and fix before passing
+```
+
+**Feature 4 - Backend API queries real database:**
+```text
+Steps:
+1. Start server with verbose logging
+2. Make API call (e.g., GET /api/items)
+3. Check server logs
+4. Verify SQL query appears (SELECT, INSERT, etc.) or ORM query log
+5. If no DB queries in logs → implementation is using mock data
+```
+
+---
+
## MANDATORY TEST CATEGORIES
-The feature_list.json **MUST** include tests from ALL of these categories. The minimum counts scale by complexity tier.
+The feature_list.json **MUST** include tests from ALL 20 categories. Minimum counts scale by complexity tier.
### Category Distribution by Complexity Tier
-| Category | Simple | Medium | Complex |
+| Category | Simple | Medium | Advanced |
| -------------------------------- | ------- | ------- | -------- |
+| **0. Infrastructure (REQUIRED)** | 5 | 5 | 5 |
| A. Security & Access Control | 5 | 20 | 40 |
| B. Navigation Integrity | 15 | 25 | 40 |
| C. Real Data Verification | 20 | 30 | 50 |
@@ -103,335 +222,53 @@ The feature_list.json **MUST** include tests from ALL of these categories. The m
| R. Concurrency & Race Conditions | 5 | 8 | 15 |
| S. Export/Import | 5 | 6 | 10 |
| T. Performance | 5 | 5 | 10 |
-| **TOTAL** | **150** | **250** | **400+** |
+| **TOTAL** | **165** | **265** | **405+** |
---
-### A. Security & Access Control Tests
-
-Test that unauthorized access is blocked and permissions are enforced.
-
-**Required tests (examples):**
-
-- Unauthenticated user cannot access protected routes (redirect to login)
-- Regular user cannot access admin-only pages (403 or redirect)
-- API endpoints return 401 for unauthenticated requests
-- API endpoints return 403 for unauthorized role access
-- Session expires after configured inactivity period
-- Logout clears all session data and tokens
-- Invalid/expired tokens are rejected
-- Each role can ONLY see their permitted menu items
-- Direct URL access to unauthorized pages is blocked
-- Sensitive operations require confirmation or re-authentication
-- Cannot access another user's data by manipulating IDs in URL
-- Password reset flow works securely
-- Failed login attempts are handled (no information leakage)
-
-### B. Navigation Integrity Tests
-
-Test that every button, link, and menu item goes to the correct place.
-
-**Required tests (examples):**
-
-- Every button in sidebar navigates to correct page
-- Every menu item links to existing route
-- All CRUD action buttons (Edit, Delete, View) go to correct URLs with correct IDs
-- Back button works correctly after each navigation
-- Deep linking works (direct URL access to any page with auth)
-- Breadcrumbs reflect actual navigation path
-- 404 page shown for non-existent routes (not crash)
-- After login, user redirected to intended destination (or dashboard)
-- After logout, user redirected to login page
-- Pagination links work and preserve current filters
-- Tab navigation within pages works correctly
-- Modal close buttons return to previous state
-- Cancel buttons on forms return to previous page
-
-### C. Real Data Verification Tests
-
-Test that data is real (not mocked) and persists correctly.
-
-**Required tests (examples):**
-
-- Create a record via UI with unique content → verify it appears in list
-- Create a record → refresh page → record still exists
-- Create a record → log out → log in → record still exists
-- Edit a record → verify changes persist after refresh
-- Delete a record → verify it's gone from list AND database
-- Delete a record → verify it's gone from related dropdowns
-- Filter/search → results match actual data created in test
-- Dashboard statistics reflect real record counts (create 3 items, count shows 3)
-- Reports show real aggregated data
-- Export functionality exports actual data you created
-- Related records update when parent changes
-- Timestamps are real and accurate (created_at, updated_at)
-- Data created by User A is not visible to User B (unless shared)
-- Empty state shows correctly when no data exists
-
-### D. Workflow Completeness Tests
-
-Test that every workflow can be completed end-to-end through the UI.
-
-**Required tests (examples):**
-
-- Every entity has working Create operation via UI form
-- Every entity has working Read/View operation (detail page loads)
-- Every entity has working Update operation (edit form saves)
-- Every entity has working Delete operation (with confirmation dialog)
-- Every status/state has a UI mechanism to transition to next state
-- Multi-step processes (wizards) can be completed end-to-end
-- Bulk operations (select all, delete selected) work
-- Cancel/Undo operations work where applicable
-- Required fields prevent submission when empty
-- Form validation shows errors before submission
-- Successful submission shows success feedback
-- Backend workflow (e.g., user→customer conversion) has UI trigger
-
-### E. Error Handling Tests
-
-Test graceful handling of errors and edge cases.
-
-**Required tests (examples):**
-
-- Network failure shows user-friendly error message, not crash
-- Invalid form input shows field-level errors
-- API errors display meaningful messages to user
-- 404 responses handled gracefully (show not found page)
-- 500 responses don't expose stack traces or technical details
-- Empty search results show "no results found" message
-- Loading states shown during all async operations
-- Timeout doesn't hang the UI indefinitely
-- Submitting form with server error keeps user data in form
-- File upload errors (too large, wrong type) show clear message
-- Duplicate entry errors (e.g., email already exists) are clear
-
-### F. UI-Backend Integration Tests
-
-Test that frontend and backend communicate correctly.
-
-**Required tests (examples):**
-
-- Frontend request format matches what backend expects
-- Backend response format matches what frontend parses
-- All dropdown options come from real database data (not hardcoded)
-- Related entity selectors (e.g., "choose category") populated from DB
-- Changes in one area reflect in related areas after refresh
-- Deleting parent handles children correctly (cascade or block)
-- Filters work with actual data attributes from database
-- Sort functionality sorts real data correctly
-- Pagination returns correct page of real data
-- API error responses are parsed and displayed correctly
-- Loading spinners appear during API calls
-- Optimistic updates (if used) rollback on failure
-
-### G. State & Persistence Tests
-
-Test that state is maintained correctly across sessions and tabs.
-
-**Required tests (examples):**
-
-- Refresh page mid-form - appropriate behavior (data kept or cleared)
-- Close browser, reopen - session state handled correctly
-- Same user in two browser tabs - changes sync or handled gracefully
-- Browser back after form submit - no duplicate submission
-- Bookmark a page, return later - works (with auth check)
-- LocalStorage/cookies cleared - graceful re-authentication
-- Unsaved changes warning when navigating away from dirty form
-
-### H. URL & Direct Access Tests
-
-Test direct URL access and URL manipulation security.
-
-**Required tests (examples):**
-
-- Change entity ID in URL - cannot access others' data
-- Access /admin directly as regular user - blocked
-- Malformed URL parameters - handled gracefully (no crash)
-- Very long URL - handled correctly
-- URL with SQL injection attempt - rejected/sanitized
-- Deep link to deleted entity - shows "not found", not crash
-- Query parameters for filters are reflected in UI
-- Sharing a URL with filters preserves those filters
-
-### I. Double-Action & Idempotency Tests
-
-Test that rapid or duplicate actions don't cause issues.
-
-**Required tests (examples):**
-
-- Double-click submit button - only one record created
-- Rapid multiple clicks on delete - only one deletion occurs
-- Submit form, hit back, submit again - appropriate behavior
-- Multiple simultaneous API calls - server handles correctly
-- Refresh during save operation - data not corrupted
-- Click same navigation link twice quickly - no issues
-- Submit button disabled during processing
-
-### J. Data Cleanup & Cascade Tests
-
-Test that deleting data cleans up properly everywhere.
-
-**Required tests (examples):**
-
-- Delete parent entity - children removed from all views
-- Delete item - removed from search results immediately
-- Delete item - statistics/counts updated immediately
-- Delete item - related dropdowns updated
-- Delete item - cached views refreshed
-- Soft delete (if applicable) - item hidden but recoverable
-- Hard delete - item completely removed from database
-
-### K. Default & Reset Tests
-
-Test that defaults and reset functionality work correctly.
-
-**Required tests (examples):**
-
-- New form shows correct default values
-- Date pickers default to sensible dates (today, not 1970)
-- Dropdowns default to correct option (or placeholder)
-- Reset button clears to defaults, not just empty
-- Clear filters button resets all filters to default
-- Pagination resets to page 1 when filters change
-- Sorting resets when changing views
-
-### L. Search & Filter Edge Cases
-
-Test search and filter functionality thoroughly.
-
-**Required tests (examples):**
-
-- Empty search shows all results (or appropriate message)
-- Search with only spaces - handled correctly
-- Search with special characters (!@#$%^&\*) - no errors
-- Search with quotes - handled correctly
-- Search with very long string - handled correctly
-- Filter combinations that return zero results - shows message
-- Filter + search + sort together - all work correctly
-- Filter persists after viewing detail and returning to list
-- Clear individual filter - works correctly
-- Search is case-insensitive (or clearly case-sensitive)
-
-### M. Form Validation Tests
-
-Test all form validation rules exhaustively.
-
-**Required tests (examples):**
-
-- Required field empty - shows error, blocks submit
-- Email field with invalid email formats - shows error
-- Password field - enforces complexity requirements
-- Numeric field with letters - rejected
-- Date field with invalid date - rejected
-- Min/max length enforced on text fields
-- Min/max values enforced on numeric fields
-- Duplicate unique values rejected (e.g., duplicate email)
-- Error messages are specific (not just "invalid")
-- Errors clear when user fixes the issue
-- Server-side validation matches client-side
-- Whitespace-only input rejected for required fields
-
-### N. Feedback & Notification Tests
+### Category Descriptions
-Test that users get appropriate feedback for all actions.
+**0. Infrastructure (REQUIRED - Priority 0)** - Database connectivity, schema existence, data persistence across server restart, absence of mock patterns. These features MUST pass before any functional features can begin. All tiers require exactly 5 infrastructure features (indices 0-4).
-**Required tests (examples):**
+**A. Security & Access Control** - Test unauthorized access blocking, permission enforcement, session management, role-based access, and data isolation between users.
-- Every successful save/create shows success feedback
-- Every failed action shows error feedback
-- Loading spinner during every async operation
-- Disabled state on buttons during form submission
-- Progress indicator for long operations (file upload)
-- Toast/notification disappears after appropriate time
-- Multiple notifications don't overlap incorrectly
-- Success messages are specific (not just "Success")
+**B. Navigation Integrity** - Test all buttons, links, menus, breadcrumbs, deep links, back button behavior, 404 handling, and post-login/logout redirects.
-### O. Responsive & Layout Tests
+**C. Real Data Verification** - Test data persistence across refreshes and sessions, CRUD operations with unique test data, related record updates, and empty states.
-Test that the UI works on different screen sizes.
+**D. Workflow Completeness** - Test end-to-end CRUD for every entity, state transitions, multi-step wizards, bulk operations, and form submission feedback.
-**Required tests (examples):**
+**E. Error Handling** - Test network failures, invalid input, API errors, 404/500 responses, loading states, timeouts, and user-friendly error messages.
-- Desktop layout correct at 1920px width
-- Tablet layout correct at 768px width
-- Mobile layout correct at 375px width
-- No horizontal scroll on any standard viewport
-- Touch targets large enough on mobile (44px min)
-- Modals fit within viewport on mobile
-- Long text truncates or wraps correctly (no overflow)
-- Tables scroll horizontally if needed on mobile
-- Navigation collapses appropriately on mobile
+**F. UI-Backend Integration** - Test request/response format matching, database-driven dropdowns, cascading updates, filters/sorts with real data, and API error display.
-### P. Accessibility Tests
+**G. State & Persistence** - Test refresh mid-form, session recovery, multi-tab behavior, back-button after submit, and unsaved changes warnings.
-Test basic accessibility compliance.
+**H. URL & Direct Access** - Test URL manipulation security, direct route access by role, malformed parameters, deep links to deleted entities, and shareable filter URLs.
-**Required tests (examples):**
+**I. Double-Action & Idempotency** - Test double-click submit, rapid delete clicks, back-and-resubmit, button disabled during processing, and concurrent submissions.
-- Tab navigation works through all interactive elements
-- Focus ring visible on all focused elements
-- Screen reader can navigate main content areas
-- ARIA labels on icon-only buttons
-- Color contrast meets WCAG AA (4.5:1 for text)
-- No information conveyed by color alone
-- Form fields have associated labels
-- Error messages announced to screen readers
-- Skip link to main content (if applicable)
-- Images have alt text
+**J. Data Cleanup & Cascade** - Test parent deletion effects on children, removal from search/lists/dropdowns, statistics updates, and soft vs hard delete behavior.
-### Q. Temporal & Timezone Tests
+**K. Default & Reset** - Test form defaults, sensible date picker defaults, dropdown placeholders, reset button behavior, and filter/pagination reset on context change.
-Test date/time handling.
+**L. Search & Filter Edge Cases** - Test empty search, whitespace-only, special characters, quotes, long strings, zero-result combinations, and filter persistence.
-**Required tests (examples):**
+**M. Form Validation** - Test required fields, email/password/numeric/date formats, min/max constraints, uniqueness, specific error messages, and server-side validation.
-- Dates display in user's local timezone
-- Created/updated timestamps accurate and formatted correctly
-- Date picker allows only valid date ranges
-- Overdue items identified correctly (timezone-aware)
-- "Today", "This Week" filters work correctly for user's timezone
-- Recurring items generate at correct times (if applicable)
-- Date sorting works correctly across months/years
+**N. Feedback & Notification** - Test success/error feedback for all actions, loading spinners, disabled buttons during submit, progress indicators, and toast behavior.
-### R. Concurrency & Race Condition Tests
+**O. Responsive & Layout** - Test layouts at desktop (1920px), tablet (768px), and mobile (375px), no horizontal scroll, touch targets, modal fit, and text overflow.
-Test multi-user and race condition scenarios.
+**P. Accessibility** - Test tab navigation, focus rings, screen reader compatibility, ARIA labels, color contrast, labels on form fields, and error announcements.
-**Required tests (examples):**
+**Q. Temporal & Timezone** - Test timezone-aware display, accurate timestamps, date picker constraints, overdue detection, and date sorting across boundaries.
-- Two users edit same record - last save wins or conflict shown
-- Record deleted while another user viewing - graceful handling
-- List updates while user on page 2 - pagination still works
-- Rapid navigation between pages - no stale data displayed
-- API response arrives after user navigated away - no crash
-- Concurrent form submissions from same user handled
+**R. Concurrency & Race Conditions** - Test concurrent edits, viewing deleted records, pagination during updates, rapid navigation, and late API response handling.
-### S. Export/Import Tests (if applicable)
+**S. Export/Import** - Test full/filtered export, import with valid/duplicate/malformed files, and round-trip data integrity.
-Test data export and import functionality.
-
-**Required tests (examples):**
-
-- Export all data - file contains all records
-- Export filtered data - only filtered records included
-- Import valid file - all records created correctly
-- Import duplicate data - handled correctly (skip/update/error)
-- Import malformed file - error message, no partial import
-- Export then import - data integrity preserved exactly
-
-### T. Performance Tests
-
-Test basic performance requirements.
-
-**Required tests (examples):**
-
-- Page loads in <3s with 100 records
-- Page loads in <5s with 1000 records
-- Search responds in <1s
-- Infinite scroll doesn't degrade with many items
-- Large file upload shows progress
-- Memory doesn't leak on long sessions
-- No console errors during normal operation
+**T. Performance** - Test page load with 100/1000 records, search response time, infinite scroll stability, upload progress, and memory/console errors.
---
@@ -455,6 +292,16 @@ The feature_list.json must include tests that **actively verify real data** and
- `setTimeout` simulating API delays with static data
- Static returns instead of database queries
+**Additional prohibited patterns (in-memory stores):**
+
+- `globalThis.` (in-memory storage pattern)
+- `dev-store`, `devStore`, `DevStore` (development stores)
+- `json-server`, `mirage`, `msw` (mock backends)
+- `Map()` or `Set()` used as primary data store
+- Environment checks like `if (process.env.NODE_ENV === 'development')` for data routing
+
+**Why this matters:** In-memory stores (like `globalThis.devStore`) will pass simple tests because data persists during a single server run. But data is LOST on server restart, which is unacceptable for production. The Infrastructure features (0-4) specifically test for this by requiring data to survive a full server restart.
+
---
**CRITICAL INSTRUCTION:**
@@ -492,32 +339,16 @@ Set up the basic project structure based on what's specified in `app_spec.txt`.
This typically includes directories for frontend, backend, and any other
components mentioned in the spec.
-### OPTIONAL: Start Implementation
-
-If you have time remaining in this session, you may begin implementing
-the highest-priority features. Get the next feature with:
-
-```
-Use the feature_get_next tool
-```
-
-Remember:
-- Work on ONE feature at a time
-- Test thoroughly before marking as passing
-- Commit your progress before session ends
-
### ENDING THIS SESSION
-Before your context fills up:
-
-1. Commit all work with descriptive messages
-2. Create `claude-progress.txt` with a summary of what you accomplished
-3. Verify features were created using the feature_get_stats tool
-4. Leave the environment in a clean, working state
+Once you have completed the four tasks above:
-The next agent will continue from here with a fresh context window.
-
----
+1. Commit all work with a descriptive message
+2. Verify features were created using the feature_get_stats tool
+3. Leave the environment in a clean, working state
+4. Exit cleanly
-**Remember:** You have unlimited time across many sessions. Focus on
-quality over speed. Production-ready is the goal.
+**IMPORTANT:** Do NOT attempt to implement any features. Your job is setup only.
+Feature implementation will be handled by parallel coding agents that spawn after
+you complete initialization. Starting implementation here would create a bottleneck
+and defeat the purpose of the parallel architecture.
diff --git a/.claude/templates/testing_prompt.template.md b/.claude/templates/testing_prompt.template.md
new file mode 100644
index 000000000..ee6a08fa2
--- /dev/null
+++ b/.claude/templates/testing_prompt.template.md
@@ -0,0 +1,150 @@
+## YOUR ROLE - TESTING AGENT
+
+You are a **testing agent** responsible for **regression testing** previously-passing features. If you find a regression, you must fix it.
+
+## ASSIGNED FEATURES FOR REGRESSION TESTING
+
+You are assigned to test the following features: {{TESTING_FEATURE_IDS}}
+
+### Workflow for EACH feature:
+1. Call `feature_get_by_id` with the feature ID
+2. Read the feature's verification steps
+3. Test the feature in the browser
+4. Call `feature_mark_passing` or `feature_mark_failing`
+5. Move to the next feature
+
+---
+
+### STEP 1: GET YOUR ASSIGNED FEATURE(S)
+
+Your features have been pre-assigned by the orchestrator. For each feature ID listed above, use `feature_get_by_id` to get the details:
+
+```
+Use the feature_get_by_id tool with feature_id=
+```
+
+### STEP 2: VERIFY THE FEATURE
+
+**CRITICAL:** You MUST verify the feature through the actual UI using browser automation.
+
+For the feature returned:
+1. Read and understand the feature's verification steps
+2. Navigate to the relevant part of the application
+3. Execute each verification step using browser automation
+4. Take screenshots and read them to verify visual appearance
+5. Check for console errors
+
+### Browser Automation (Playwright CLI)
+
+**Navigation & Screenshots:**
+- `playwright-cli open ` - Open browser and navigate
+- `playwright-cli goto ` - Navigate to URL
+- `playwright-cli screenshot` - Save screenshot to `.playwright-cli/`
+- `playwright-cli snapshot` - Save page snapshot with element refs to `.playwright-cli/`
+
+**Element Interaction:**
+- `playwright-cli click [` - Click elements (ref from snapshot)
+- `playwright-cli type ` - Type text
+- `playwright-cli fill ][ ` - Fill form fields
+- `playwright-cli select ][ ` - Select dropdown
+- `playwright-cli press ` - Keyboard input
+
+**Debugging:**
+- `playwright-cli console` - Check for JS errors
+- `playwright-cli network` - Monitor API calls
+
+**Cleanup:**
+- `playwright-cli close` - Close browser when done (ALWAYS do this)
+
+**Note:** Screenshots and snapshots save to files. Read the file to see the content.
+
+### STEP 3: HANDLE RESULTS
+
+#### If the feature PASSES:
+
+The feature still works correctly. **DO NOT** call feature_mark_passing again -- it's already passing. End your session.
+
+#### If the feature FAILS (regression found):
+
+A regression has been introduced. You MUST fix it:
+
+1. **Mark the feature as failing:**
+ ```
+ Use the feature_mark_failing tool with feature_id={id}
+ ```
+
+2. **Investigate the root cause:**
+ - Check console errors
+ - Review network requests
+ - Examine recent git commits that might have caused the regression
+
+3. **Fix the regression:**
+ - Make the necessary code changes
+ - Test your fix using browser automation
+ - Ensure the feature works correctly again
+
+4. **Verify the fix:**
+ - Run through all verification steps again
+ - Take screenshots and read them to confirm the fix
+
+5. **Mark as passing after fix:**
+ ```
+ Use the feature_mark_passing tool with feature_id={id}
+ ```
+
+6. **Commit the fix:**
+ ```bash
+ git add .
+ git commit -m "Fix regression in [feature name]
+
+ - [Describe what was broken]
+ - [Describe the fix]
+ - Verified with browser automation"
+ ```
+
+---
+
+## AVAILABLE TOOLS
+
+### Feature Management
+- `feature_get_stats` - Get progress overview (passing/in_progress/total counts)
+- `feature_get_by_id` - Get your assigned feature details
+- `feature_mark_failing` - Mark a feature as failing (when you find a regression)
+- `feature_mark_passing` - Mark a feature as passing (after fixing a regression)
+
+### Browser Automation (Playwright CLI)
+Use `playwright-cli` commands for browser interaction. Key commands:
+- `playwright-cli open ` - Open browser
+- `playwright-cli goto ` - Navigate to URL
+- `playwright-cli screenshot` - Take screenshot (saved to `.playwright-cli/`)
+- `playwright-cli snapshot` - Get page snapshot with element refs
+- `playwright-cli click ][` - Click element
+- `playwright-cli type ` - Type text
+- `playwright-cli fill ][ ` - Fill form field
+- `playwright-cli console` - Check for JS errors
+- `playwright-cli close` - Close browser (always do this when done)
+
+---
+
+## IMPORTANT REMINDERS
+
+**Your Goal:** Test each assigned feature thoroughly. Verify it still works, and fix any regression found. Process ALL features in your list before ending your session.
+
+**Quality Bar:**
+- Zero console errors
+- All verification steps pass
+- Visual appearance correct
+- API calls succeed
+
+**If you find a regression:**
+1. Mark the feature as failing immediately
+2. Fix the issue
+3. Verify the fix with browser automation
+4. Mark as passing only after thorough verification
+5. Commit the fix
+
+**You have one iteration.** Test all assigned features before ending.
+
+---
+
+Begin by running Step 1 for the first feature in your assigned list.
diff --git a/.env.example b/.env.example
index fe59407e0..ed163bacf 100644
--- a/.env.example
+++ b/.env.example
@@ -1,2 +1,63 @@
# Optional: N8N webhook for progress notifications
# PROGRESS_N8N_WEBHOOK_URL=https://your-n8n-instance.com/webhook/...
+
+# Playwright Browser Configuration
+#
+# PLAYWRIGHT_BROWSER: Which browser to use for testing
+# - firefox: Lower CPU usage, recommended (default)
+# - chrome: Google Chrome
+# - webkit: Safari engine
+# - msedge: Microsoft Edge
+# PLAYWRIGHT_BROWSER=firefox
+
+# Extra Read Paths (Optional)
+# Comma-separated list of absolute paths for read-only access to external directories.
+# The agent can read files from these paths but cannot write to them.
+# Useful for referencing documentation, shared libraries, or other projects.
+# Example: EXTRA_READ_PATHS=/Volumes/Data/dev,/Users/shared/libs
+# EXTRA_READ_PATHS=
+
+# Google Cloud Vertex AI Configuration (Optional)
+# To use Claude via Vertex AI on Google Cloud Platform, uncomment and set these variables.
+# Requires: gcloud CLI installed and authenticated (run: gcloud auth application-default login)
+# Note: Use @ instead of - in model names for date-suffixed models (e.g., claude-sonnet-4-5@20250929)
+#
+# CLAUDE_CODE_USE_VERTEX=1
+# CLOUD_ML_REGION=us-east5
+# ANTHROPIC_VERTEX_PROJECT_ID=your-gcp-project-id
+# ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-4-6
+# ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4-5@20250929
+# ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku@20241022
+
+# ===================
+# Alternative API Providers (Azure, GLM, Ollama, Kimi, Custom)
+# ===================
+# Configure via Settings UI (recommended) or set env vars below.
+# When both are set, env vars take precedence.
+#
+# Azure Anthropic (Claude):
+# ANTHROPIC_BASE_URL=https://your-resource.services.ai.azure.com/anthropic
+# ANTHROPIC_API_KEY=your-azure-api-key
+# ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-4-6
+# ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4-5
+# ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-haiku-4-5
+#
+# GLM (Zhipu AI):
+# ANTHROPIC_BASE_URL=https://api.z.ai/api/anthropic
+# ANTHROPIC_AUTH_TOKEN=your-glm-api-key
+# ANTHROPIC_DEFAULT_OPUS_MODEL=glm-4.7
+# ANTHROPIC_DEFAULT_SONNET_MODEL=glm-4.7
+# ANTHROPIC_DEFAULT_HAIKU_MODEL=glm-4.7
+#
+# Ollama (Local):
+# ANTHROPIC_BASE_URL=http://localhost:11434
+# ANTHROPIC_DEFAULT_OPUS_MODEL=qwen3-coder
+# ANTHROPIC_DEFAULT_SONNET_MODEL=qwen3-coder
+# ANTHROPIC_DEFAULT_HAIKU_MODEL=qwen3-coder
+#
+# Kimi (Moonshot):
+# ANTHROPIC_BASE_URL=https://api.kimi.com/coding/
+# ANTHROPIC_API_KEY=your-kimi-api-key
+# ANTHROPIC_DEFAULT_OPUS_MODEL=kimi-k2.5
+# ANTHROPIC_DEFAULT_SONNET_MODEL=kimi-k2.5
+# ANTHROPIC_DEFAULT_HAIKU_MODEL=kimi-k2.5
diff --git a/.gitignore b/.gitignore
index d14182cd2..d63e64ec6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,18 @@
# Agent-generated output directories
generations/
+automaker/
+temp/
+temp-docs/
+
+nul
+issues/
+
+# Browser profiles for parallel agent execution
+.browser-profiles/
+
+# Playwright CLI daemon artifacts
+.playwright-cli/
+.playwright/
# Log files
logs/
@@ -61,12 +74,20 @@ coverage.xml
.hypothesis/
.pytest_cache/
nosetests.xml
+ui/playwright-report/
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
+.ruff_cache/
+
+# ===================
+# Claude Code
+# ===================
+.claude/settings.local.json
+
# ===================
# IDE / Editors
# ===================
@@ -98,6 +119,7 @@ Desktop.ini
ui/dist/
ui/.vite/
.vite/
+*.tgz
# ===================
# Environment files
@@ -125,6 +147,11 @@ pnpm-lock.yaml
poetry.lock
Pipfile.lock
+# ===================
+# TypeScript
+# ===================
+*.tsbuildinfo
+
# ===================
# Misc
# ===================
@@ -133,3 +160,5 @@ Pipfile.lock
*.temp
.tmp/
.temp/
+tmpclaude-*-cwd
+ui/test-results/
diff --git a/.npmignore b/.npmignore
new file mode 100644
index 000000000..6bf112b0c
--- /dev/null
+++ b/.npmignore
@@ -0,0 +1,31 @@
+venv/
+**/__pycache__/
+**/*.pyc
+.git/
+.github/
+node_modules/
+test_*.py
+tests/
+generations/
+*.db
+.env
+requirements.txt
+CLAUDE.md
+LICENSE.md
+README.md
+ui/src/
+ui/node_modules/
+ui/tsconfig*.json
+ui/vite.config.ts
+ui/eslint.config.js
+ui/index.html
+ui/public/
+ui/playwright.config.ts
+ui/tests/
+start.bat
+start_ui.bat
+start.sh
+start_ui.sh
+start_ui.py
+.claude/agents/
+.claude/settings.json
diff --git a/CLAUDE.md b/CLAUDE.md
index 51c094939..6f59910be 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -2,6 +2,12 @@
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+## Prerequisites
+
+- Python 3.11+
+- Node.js 20+ (for UI development)
+- Claude Code CLI
+
## Project Overview
This is an autonomous coding agent system with a React-based UI. It uses the Claude Agent SDK to build complete applications over multiple sessions using a two-agent pattern:
@@ -11,18 +17,28 @@ This is an autonomous coding agent system with a React-based UI. It uses the Cla
## Commands
-### Quick Start (Recommended)
+### npm Global Install (Recommended)
```bash
-# Windows - launches CLI menu
-start.bat
+npm install -g autoforge-ai
+autoforge # Start server (first run sets up Python venv)
+autoforge config # Edit ~/.autoforge/.env in $EDITOR
+autoforge config --show # Print active configuration
+autoforge --port 9999 # Custom port
+autoforge --no-browser # Don't auto-open browser
+autoforge --repair # Delete and recreate ~/.autoforge/venv/
+```
-# macOS/Linux
-./start.sh
+### From Source (Development)
+```bash
# Launch Web UI (serves pre-built React app)
start_ui.bat # Windows
./start_ui.sh # macOS/Linux
+
+# CLI menu
+start.bat # Windows
+./start.sh # macOS/Linux
```
### Python Backend (Manual)
@@ -45,6 +61,15 @@ python autonomous_agent_demo.py --project-dir my-app # if registered
# YOLO mode: rapid prototyping without browser testing
python autonomous_agent_demo.py --project-dir my-app --yolo
+
+# Parallel mode: run multiple agents concurrently (1-5 agents)
+python autonomous_agent_demo.py --project-dir my-app --parallel --max-concurrency 3
+
+# Batch mode: implement multiple features per agent session (1-15)
+python autonomous_agent_demo.py --project-dir my-app --batch-size 3
+
+# Batch specific features by ID
+python autonomous_agent_demo.py --project-dir my-app --batch-features 1,2,3
```
### YOLO Mode (Rapid Prototyping)
@@ -59,8 +84,8 @@ python autonomous_agent_demo.py --project-dir my-app --yolo
```
**What's different in YOLO mode:**
-- No regression testing (skips `feature_get_for_regression`)
-- No Playwright MCP server (browser automation disabled)
+- No regression testing
+- No Playwright CLI (browser automation disabled)
- Features marked passing after lint/type-check succeeds
- Faster iteration for prototyping
@@ -83,23 +108,78 @@ npm run lint # Run ESLint
**Note:** The `start_ui.bat` script serves the pre-built UI from `ui/dist/`. After making UI changes, run `npm run build` in the `ui/` directory.
+## Testing
+
+### Python
+
+```bash
+ruff check . # Lint
+mypy . # Type check
+python test_security.py # Security unit tests (12 tests)
+python test_security_integration.py # Integration tests (9 tests)
+python -m pytest test_client.py # Client tests (20 tests)
+python -m pytest test_dependency_resolver.py # Dependency resolver tests (12 tests)
+python -m pytest test_rate_limit_utils.py # Rate limit tests (22 tests)
+```
+
+### React UI
+
+```bash
+cd ui
+npm run lint # ESLint
+npm run build # Type check + build (Vite 7)
+npm run test:e2e # Playwright end-to-end tests
+npm run test:e2e:ui # Playwright tests with UI
+```
+
+### CI/CD
+
+GitHub Actions (`.github/workflows/ci.yml`) runs on push/PR to master:
+- **Python job**: ruff lint + security tests
+- **UI job**: ESLint + TypeScript build
+
+### Code Quality
+
+Configuration in `pyproject.toml`:
+- ruff: Line length 120, Python 3.11 target
+- mypy: Strict return type checking, ignores missing imports
+
## Architecture
+### npm CLI (bin/, lib/)
+
+The `autoforge` command is a Node.js wrapper that manages the Python environment and server lifecycle:
+- `bin/autoforge.js` - Entry point (shebang script)
+- `lib/cli.js` - Main CLI logic: Python 3.11+ detection (cross-platform), venv management at `~/.autoforge/venv/` with composite marker (requirements hash + Python version), `.env` config loading from `~/.autoforge/.env`, uvicorn server startup with PID file, and signal handling
+- `package.json` - npm package config (`autoforge-ai` on npm), `files` whitelist with `__pycache__` exclusions, `prepublishOnly` builds the UI
+- `requirements-prod.txt` - Runtime-only Python deps (excludes ruff, mypy, pytest)
+- `.npmignore` - Excludes dev files, tests, UI source from the published tarball
+
+Publishing: `npm publish` (triggers `prepublishOnly` which builds UI, then publishes ~600KB tarball with 84 files)
+
### Core Python Modules
- `start.py` - CLI launcher with project creation/selection menu
-- `autonomous_agent_demo.py` - Entry point for running the agent
+- `autonomous_agent_demo.py` - Entry point for running the agent (supports `--yolo`, `--parallel`, `--batch-size`, `--batch-features`)
+- `autoforge_paths.py` - Central path resolution with dual-path backward compatibility and migration
- `agent.py` - Agent session loop using Claude Agent SDK
-- `client.py` - ClaudeSDKClient configuration with security hooks and MCP servers
+- `client.py` - ClaudeSDKClient configuration with security hooks, feature MCP server, and Vertex AI support
- `security.py` - Bash command allowlist validation (ALLOWED_COMMANDS whitelist)
-- `prompts.py` - Prompt template loading with project-specific fallback
+- `prompts.py` - Prompt template loading with project-specific fallback and batch feature prompts
- `progress.py` - Progress tracking, database queries, webhook notifications
-- `registry.py` - Project registry for mapping names to paths (cross-platform)
+- `registry.py` - Project registry for mapping names to paths (cross-platform), global settings model
+- `parallel_orchestrator.py` - Concurrent agent execution with dependency-aware scheduling
+- `auth.py` - Authentication error detection for Claude CLI
+- `env_constants.py` - Shared environment variable constants (API_ENV_VARS) used by client.py and chat sessions
+- `rate_limit_utils.py` - Rate limit detection, retry parsing, exponential backoff with jitter
+- `api/database.py` - SQLAlchemy models (Feature, Schedule, ScheduleOverride)
+- `api/dependency_resolver.py` - Cycle detection (Kahn's algorithm + DFS) and dependency validation
+- `api/migration.py` - JSON-to-SQLite migration utility
### Project Registry
Projects can be stored in any directory. The registry maps project names to paths using SQLite:
-- **All platforms**: `~/.autocoder/registry.db`
+- **All platforms**: `~/.autoforge/registry.db`
The registry uses:
- SQLite database with SQLAlchemy ORM
@@ -108,72 +188,282 @@ The registry uses:
### Server API (server/)
-The FastAPI server provides REST endpoints for the UI:
-
-- `server/routers/projects.py` - Project CRUD with registry integration
-- `server/routers/features.py` - Feature management
-- `server/routers/agent.py` - Agent control (start/stop/pause/resume)
-- `server/routers/filesystem.py` - Filesystem browser API with security controls
-- `server/routers/spec_creation.py` - WebSocket for interactive spec creation
+The FastAPI server provides REST and WebSocket endpoints for the UI:
+
+**Routers** (`server/routers/`):
+- `projects.py` - Project CRUD with registry integration
+- `features.py` - Feature management
+- `agent.py` - Agent control (start/stop/pause/resume)
+- `filesystem.py` - Filesystem browser API with security controls
+- `spec_creation.py` - WebSocket for interactive spec creation
+- `expand_project.py` - Interactive project expansion via natural language
+- `assistant_chat.py` - Read-only project assistant chat (WebSocket/REST)
+- `terminal.py` - Interactive terminal I/O with PTY support (WebSocket bidirectional)
+- `devserver.py` - Dev server control (start/stop) and config
+- `schedules.py` - CRUD for time-based agent scheduling
+- `settings.py` - Global settings management (model selection, YOLO, batch size, headless browser)
+
+**Services** (`server/services/`):
+- `process_manager.py` - Agent process lifecycle management
+- `project_config.py` - Project type detection and dev command management
+- `terminal_manager.py` - Terminal session management with PTY (`pywinpty` on Windows)
+- `scheduler_service.py` - APScheduler-based automated agent scheduling
+- `dev_server_manager.py` - Dev server lifecycle management
+- `assistant_chat_session.py` / `assistant_database.py` - Assistant chat sessions with SQLite persistence
+- `spec_chat_session.py` - Spec creation chat sessions
+- `expand_chat_session.py` - Expand project chat sessions
+- `chat_constants.py` - Shared constants for chat services
+
+**Utilities** (`server/utils/`):
+- `process_utils.py` - Process management utilities
+- `project_helpers.py` - Project path resolution helpers
+- `validation.py` - Project name validation
### Feature Management
Features are stored in SQLite (`features.db`) via SQLAlchemy. The agent interacts with features through an MCP server:
- `mcp_server/feature_mcp.py` - MCP server exposing feature management tools
-- `api/database.py` - SQLAlchemy models (Feature table with priority, category, name, description, steps, passes)
+- `api/database.py` - SQLAlchemy models (Feature table with priority, category, name, description, steps, passes, dependencies)
MCP tools available to the agent:
- `feature_get_stats` - Progress statistics
-- `feature_get_next` - Get highest-priority pending feature
-- `feature_get_for_regression` - Random passing features for regression testing
+- `feature_get_by_id` - Get a single feature by ID
+- `feature_get_summary` - Get summary of all features
+- `feature_get_ready` - Get features ready to work on (dependencies met)
+- `feature_get_blocked` - Get features blocked by unmet dependencies
+- `feature_get_graph` - Get full dependency graph
+- `feature_claim_and_get` - Atomically claim next available feature (for parallel mode)
+- `feature_mark_in_progress` - Mark feature as in progress
- `feature_mark_passing` - Mark feature complete
+- `feature_mark_failing` - Mark feature as failing
- `feature_skip` - Move feature to end of queue
+- `feature_clear_in_progress` - Clear in-progress status
- `feature_create_bulk` - Initialize all features (used by initializer)
+- `feature_create` - Create a single feature
+- `feature_add_dependency` - Add dependency between features (with cycle detection)
+- `feature_remove_dependency` - Remove a dependency
+- `feature_set_dependencies` - Set all dependencies for a feature at once
### React UI (ui/)
-- Tech stack: React 18, TypeScript, TanStack Query, Tailwind CSS v4, Radix UI
+- Tech stack: React 19, TypeScript, Vite 7, TanStack Query, Tailwind CSS v4, Radix UI, dagre (graph layout), xterm.js (terminal)
- `src/App.tsx` - Main app with project selection, kanban board, agent controls
-- `src/hooks/useWebSocket.ts` - Real-time updates via WebSocket
+- `src/hooks/useWebSocket.ts` - Real-time updates via WebSocket (progress, agent status, logs, agent updates)
- `src/hooks/useProjects.ts` - React Query hooks for API calls
- `src/lib/api.ts` - REST API client
- `src/lib/types.ts` - TypeScript type definitions
-- `src/components/FolderBrowser.tsx` - Server-side filesystem browser for project folder selection
-- `src/components/NewProjectModal.tsx` - Multi-step project creation wizard
+
+Key components:
+- `AgentMissionControl.tsx` - Dashboard showing active agents with mascots (Spark, Fizz, Octo, Hoot, Buzz)
+- `DependencyGraph.tsx` - Interactive node graph visualization with dagre layout
+- `CelebrationOverlay.tsx` - Confetti animation on feature completion
+- `FolderBrowser.tsx` - Server-side filesystem browser for project folder selection
+- `Terminal.tsx` / `TerminalTabs.tsx` - xterm.js-based multi-tab terminal
+- `AssistantPanel.tsx` / `AssistantChat.tsx` - AI assistant for project Q&A
+- `ExpandProjectModal.tsx` / `ExpandProjectChat.tsx` - Add features via natural language
+- `DevServerControl.tsx` - Dev server start/stop control
+- `ScheduleModal.tsx` - Schedule management UI
+- `SettingsModal.tsx` - Global settings panel
+
+In-app documentation (`/#/docs` route):
+- `src/components/docs/sections/` - Content for each doc section (GettingStarted.tsx, AgentSystem.tsx, etc.)
+- `src/components/docs/docsData.ts` - Sidebar structure, subsection IDs, search keywords
+- `src/components/docs/DocsPage.tsx` - Page layout; `DocsContent.tsx` - section renderer with scroll tracking
+
+Keyboard shortcuts (press `?` for help):
+- `D` - Toggle debug panel
+- `G` - Toggle Kanban/Graph view
+- `N` - Add new feature
+- `A` - Toggle AI assistant
+- `,` - Open settings
### Project Structure for Generated Apps
-Projects can be stored in any directory (registered in `~/.autocoder/registry.db`). Each project contains:
-- `prompts/app_spec.txt` - Application specification (XML format)
-- `prompts/initializer_prompt.md` - First session prompt
-- `prompts/coding_prompt.md` - Continuation session prompt
-- `features.db` - SQLite database with feature test cases
-- `.agent.lock` - Lock file to prevent multiple agent instances
+Projects can be stored in any directory (registered in `~/.autoforge/registry.db`). Each project contains:
+- `.autoforge/prompts/app_spec.txt` - Application specification (XML format)
+- `.autoforge/prompts/initializer_prompt.md` - First session prompt
+- `.autoforge/prompts/coding_prompt.md` - Continuation session prompt
+- `.autoforge/features.db` - SQLite database with feature test cases
+- `.autoforge/.agent.lock` - Lock file to prevent multiple agent instances
+- `.autoforge/allowed_commands.yaml` - Project-specific bash command allowlist (optional)
+- `.autoforge/.gitignore` - Ignores runtime files
+- `.claude/skills/playwright-cli/` - Playwright CLI skill for browser automation
+- `.playwright/cli.config.json` - Browser configuration (headless, viewport, etc.)
+- `.playwright-cli/` - Playwright CLI daemon artifacts (screenshots, snapshots) - gitignored
+- `CLAUDE.md` - Stays at project root (SDK convention)
+- `app_spec.txt` - Root copy for agent template compatibility
+
+Legacy projects with files at root level (e.g., `features.db`, `prompts/`) are auto-migrated to `.autoforge/` on next agent start. Dual-path resolution ensures old and new layouts work transparently.
### Security Model
Defense-in-depth approach configured in `client.py`:
1. OS-level sandbox for bash commands
2. Filesystem restricted to project directory only
-3. Bash commands validated against `ALLOWED_COMMANDS` in `security.py`
+3. Bash commands validated using hierarchical allowlist system
+
+#### Extra Read Paths (Cross-Project File Access)
+
+The agent can optionally read files from directories outside the project folder via the `EXTRA_READ_PATHS` environment variable. This enables referencing documentation, shared libraries, or other projects.
+
+**Configuration:**
+
+```bash
+# Single path
+EXTRA_READ_PATHS=/Users/me/docs
+
+# Multiple paths (comma-separated)
+EXTRA_READ_PATHS=/Users/me/docs,/opt/shared-libs,/Volumes/Data/reference
+```
+
+**Security Controls:**
+
+All paths are validated before being granted read access:
+- Must be absolute paths (not relative)
+- Must exist and be directories
+- Paths are canonicalized via `Path.resolve()` to prevent `..` traversal attacks
+- Sensitive directories are blocked (see blocklist below)
+- Only Read, Glob, and Grep operations are allowed (no Write/Edit)
+
+**Blocked Sensitive Directories:**
+
+The following directories (relative to home) are always blocked:
+- `.ssh`, `.aws`, `.azure`, `.kube` - Cloud/SSH credentials
+- `.gnupg`, `.gpg`, `.password-store` - Encryption keys
+- `.docker`, `.config/gcloud` - Container/cloud configs
+- `.npmrc`, `.pypirc`, `.netrc` - Package manager credentials
+
+#### Per-Project Allowed Commands
+
+The agent's bash command access is controlled through a hierarchical configuration system:
+
+**Command Hierarchy (highest to lowest priority):**
+1. **Hardcoded Blocklist** (`security.py`) - NEVER allowed (dd, sudo, shutdown, etc.)
+2. **Org Blocklist** (`~/.autoforge/config.yaml`) - Cannot be overridden by projects
+3. **Org Allowlist** (`~/.autoforge/config.yaml`) - Available to all projects
+4. **Global Allowlist** (`security.py`) - Default commands (npm, git, curl, etc.)
+5. **Project Allowlist** (`.autoforge/allowed_commands.yaml`) - Project-specific commands
+
+**Project Configuration:**
+
+Each project can define custom allowed commands in `.autoforge/allowed_commands.yaml`:
+
+```yaml
+version: 1
+commands:
+ # Exact command names
+ - name: swift
+ description: Swift compiler
+
+ # Prefix wildcards (matches swiftc, swiftlint, swiftformat)
+ - name: swift*
+ description: All Swift development tools
+
+ # Local project scripts
+ - name: ./scripts/build.sh
+ description: Project build script
+```
+
+**Organization Configuration:**
+
+System administrators can set org-wide policies in `~/.autoforge/config.yaml`:
+
+```yaml
+version: 1
+
+# Commands available to ALL projects
+allowed_commands:
+ - name: jq
+ description: JSON processor
+
+# Commands blocked across ALL projects (cannot be overridden)
+blocked_commands:
+ - aws # Prevent accidental cloud operations
+ - kubectl # Block production deployments
+```
+
+**Pattern Matching:**
+- Exact: `swift` matches only `swift`
+- Wildcard: `swift*` matches `swift`, `swiftc`, `swiftlint`, etc.
+- Scripts: `./scripts/build.sh` matches the script by name from any directory
+
+**Limits:**
+- Maximum 100 commands per project config
+- Blocklisted commands (sudo, dd, shutdown, etc.) can NEVER be allowed
+- Org-level blocked commands cannot be overridden by project configs
+
+**Files:**
+- `security.py` - Command validation logic and hardcoded blocklist
+- `test_security.py` - Unit tests for security system
+- `test_security_integration.py` - Integration tests with real hooks
+- `examples/project_allowed_commands.yaml` - Project config example (all commented by default)
+- `examples/org_config.yaml` - Org config example (all commented by default)
+- `examples/README.md` - Comprehensive guide with use cases, testing, and troubleshooting
+
+### Vertex AI Configuration (Optional)
+
+Run coding agents via Google Cloud Vertex AI:
+
+1. Install and authenticate gcloud CLI: `gcloud auth application-default login`
+2. Configure `.env`:
+ ```
+ CLAUDE_CODE_USE_VERTEX=1
+ CLOUD_ML_REGION=us-east5
+ ANTHROPIC_VERTEX_PROJECT_ID=your-gcp-project-id
+ ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-4-6
+ ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4-5@20250929
+ ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku@20241022
+ ```
+
+**Note:** Use `@` instead of `-` in model names for Vertex AI.
+
+### Alternative API Providers (GLM, Ollama, Kimi, Custom)
+
+Alternative providers are configured via the **Settings UI** (gear icon > API Provider section). Select a provider, set the base URL, auth token, and model — no `.env` changes needed.
+
+**Available providers:** Claude (default), GLM (Zhipu AI), Ollama (local models), Kimi (Moonshot), Custom
+
+**Ollama notes:**
+- Requires Ollama v0.14.0+ with Anthropic API compatibility
+- Install: https://ollama.com → `ollama serve` → `ollama pull qwen3-coder`
+- Recommended models: `qwen3-coder`, `deepseek-coder-v2`, `codellama`
+- Performance depends on local hardware (GPU recommended)
## Claude Code Integration
-- `.claude/commands/create-spec.md` - `/create-spec` slash command for interactive spec creation
-- `.claude/skills/frontend-design/SKILL.md` - Skill for distinctive UI design
+**Slash commands** (`.claude/commands/`):
+- `/create-spec` - Interactive spec creation for new projects
+- `/expand-project` - Expand existing project with new features
+- `/gsd-to-autoforge-spec` - Convert GSD codebase mapping to app_spec.txt
+- `/check-code` - Run lint and type-check for code quality
+- `/checkpoint` - Create comprehensive checkpoint commit
+- `/review-pr` - Review pull requests
+
+**Custom agents** (`.claude/agents/`):
+- `coder.md` - Elite software architect agent for code implementation (Opus)
+- `code-review.md` - Code review agent for quality/security/performance analysis (Opus)
+- `deep-dive.md` - Technical investigator for deep analysis and debugging (Opus)
+
+**Skills** (`.claude/skills/`):
+- `frontend-design` - Distinctive, production-grade UI design
+- `gsd-to-autoforge-spec` - Convert GSD codebase mapping to AutoForge app_spec format
+- `playwright-cli` - Browser automation via Playwright CLI (copied to each project)
+
+**Other:**
- `.claude/templates/` - Prompt templates copied to new projects
+- `examples/` - Configuration examples and documentation for security settings
## Key Patterns
### Prompt Loading Fallback Chain
-1. Project-specific: `{project_dir}/prompts/{name}.md`
+1. Project-specific: `{project_dir}/.autoforge/prompts/{name}.md` (or legacy `{project_dir}/prompts/{name}.md`)
2. Base template: `.claude/templates/{name}.template.md`
### Agent Session Flow
-1. Check if `features.db` has features (determines initializer vs coding agent)
+1. Check if `.autoforge/features.db` has features (determines initializer vs coding agent)
2. Create ClaudeSDKClient with security settings
3. Send prompt and stream response
4. Auto-continue with 3-second delay between sessions
@@ -181,10 +471,38 @@ Defense-in-depth approach configured in `client.py`:
### Real-time UI Updates
The UI receives updates via WebSocket (`/ws/projects/{project_name}`):
-- `progress` - Test pass counts
+- `progress` - Test pass counts (passing, in_progress, total)
- `agent_status` - Running/paused/stopped/crashed
-- `log` - Agent output lines (streamed from subprocess stdout)
+- `log` - Agent output lines with optional featureId/agentIndex for attribution
- `feature_update` - Feature status changes
+- `agent_update` - Multi-agent state updates (thinking/working/testing/success/error) with mascot names
+
+### Parallel Mode
+
+When running with `--parallel`, the orchestrator:
+1. Spawns multiple Claude agents as subprocesses (up to `--max-concurrency`)
+2. Each agent claims features atomically via `feature_claim_and_get`
+3. Features blocked by unmet dependencies are skipped
+4. Browser sessions are isolated per agent via `PLAYWRIGHT_CLI_SESSION` environment variable
+5. AgentTracker parses output and emits `agent_update` messages for UI
+
+### Process Limits (Parallel Mode)
+
+The orchestrator enforces strict bounds on concurrent processes:
+- `MAX_PARALLEL_AGENTS = 5` - Maximum concurrent coding agents
+- `MAX_TOTAL_AGENTS = 10` - Hard limit on total agents (coding + testing)
+- Testing agents are capped at `max_concurrency` (same as coding agents)
+- Total process count never exceeds 11 Python processes (1 orchestrator + 5 coding + 5 testing)
+
+### Multi-Feature Batching
+
+Agents can implement multiple features per session using `--batch-size` (1-15, default: 3):
+- `--batch-size N` - Max features per coding agent batch
+- `--testing-batch-size N` - Features per testing batch (1-15, default: 3)
+- `--batch-features 1,2,3` - Specific feature IDs for batch implementation
+- `--testing-batch-features 1,2,3` - Specific feature IDs for batch regression testing
+- `prompts.py` provides `get_batch_feature_prompt()` for multi-feature prompt generation
+- Configurable in UI via settings panel
### Design System
diff --git a/README.md b/README.md
index a5f623166..0ad71c941 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,30 @@
-# AutoCoder
+# AutoForge
[](https://www.buymeacoffee.com/leonvanzyl)
A long-running autonomous coding agent powered by the Claude Agent SDK. This tool can build complete applications over multiple sessions using a two-agent pattern (initializer + coding agent). Includes a React-based UI for monitoring progress in real-time.
+> [!WARNING]
+> **Authentication:** Anthropic's policy states that third-party developers may not offer `claude.ai` login or subscription-based rate limits for their products (including agents built on the Claude Agent SDK) unless previously approved. Using your Claude subscription with AutoForge may risk account suspension. We recommend using an API key from [console.anthropic.com](https://console.anthropic.com/) instead.
+
+> [!NOTE]
+> **This repository is no longer actively maintained.** Most agent coding tools now ship their own long-running harnesses, making this project less necessary. Feel free to fork and continue development on your own!
+
## Video Tutorial
-[](https://youtu.be/lGWFlpffWk4)
+[](https://youtu.be/nKiPOxDpcJY)
-> **[Watch the setup and usage guide →](https://youtu.be/lGWFlpffWk4)**
+> **[Watch the setup and usage guide →](https://youtu.be/nKiPOxDpcJY)**
---
## Prerequisites
-### Claude Code CLI (Required)
+- **Node.js 20+** - Required for the CLI
+- **Python 3.11+** - Auto-detected on first run ([download](https://www.python.org/downloads/))
+- **Claude Code CLI** - Install and authenticate (see below)
-This project requires the Claude Code CLI to be installed. Install it using one of these methods:
+### Claude Code CLI (Required)
**macOS / Linux:**
```bash
@@ -32,46 +40,74 @@ irm https://claude.ai/install.ps1 | iex
You need one of the following:
-- **Claude Pro/Max Subscription** - Use `claude login` to authenticate (recommended)
-- **Anthropic API Key** - Pay-per-use from https://console.anthropic.com/
+- **Anthropic API Key** (recommended) - Pay-per-use from https://console.anthropic.com/
+- **Claude Pro/Max Subscription** - Use `claude login` to authenticate (see warning above)
---
## Quick Start
-### Option 1: Web UI (Recommended)
+### Option 1: npm Install (Recommended)
-**Windows:**
-```cmd
-start_ui.bat
+```bash
+npm install -g autoforge-ai
+autoforge
```
-**macOS / Linux:**
+On first run, AutoForge automatically:
+1. Checks for Python 3.11+
+2. Creates a virtual environment at `~/.autoforge/venv/`
+3. Installs Python dependencies
+4. Copies a default config file to `~/.autoforge/.env`
+5. Starts the server and opens your browser
+
+### CLI Commands
+
+```
+autoforge Start the server (default)
+autoforge config Open ~/.autoforge/.env in $EDITOR
+autoforge config --path Print config file path
+autoforge config --show Show active configuration values
+autoforge --port PORT Custom port (default: auto from 8888)
+autoforge --host HOST Custom host (default: 127.0.0.1)
+autoforge --no-browser Don't auto-open browser
+autoforge --repair Delete and recreate virtual environment
+autoforge --version Print version
+autoforge --help Show help
+```
+
+### Option 2: From Source (Development)
+
+Clone the repository and use the start scripts directly. This is the recommended path if you want to contribute or modify AutoForge itself.
+
```bash
-./start_ui.sh
+git clone https://github.com/leonvanzyl/autoforge.git
+cd autoforge
```
+**Web UI:**
+
+| Platform | Command |
+|---|---|
+| Windows | `start_ui.bat` |
+| macOS / Linux | `./start_ui.sh` |
+
This launches the React-based web UI at `http://localhost:5173` with:
- Project selection and creation
- Kanban board view of features
- Real-time agent output streaming
- Start/pause/stop controls
-### Option 2: CLI Mode
+**CLI Mode:**
-**Windows:**
-```cmd
-start.bat
-```
-
-**macOS / Linux:**
-```bash
-./start.sh
-```
+| Platform | Command |
+|---|---|
+| Windows | `start.bat` |
+| macOS / Linux | `./start.sh` |
The start script will:
1. Check if Claude CLI is installed
-2. Check if you're authenticated (prompt to run `claude login` if not)
+2. Check if you're authenticated (prompt to configure authentication if not)
3. Create a Python virtual environment
4. Install dependencies
5. Launch the main menu
@@ -130,44 +166,43 @@ Features are stored in SQLite via SQLAlchemy and managed through an MCP server t
## Project Structure
```
-autonomous-coding/
-├── start.bat # Windows CLI start script
-├── start.sh # macOS/Linux CLI start script
-├── start_ui.bat # Windows Web UI start script
-├── start_ui.sh # macOS/Linux Web UI start script
-├── start.py # CLI menu and project management
-├── start_ui.py # Web UI backend (FastAPI server launcher)
-├── autonomous_agent_demo.py # Agent entry point
-├── agent.py # Agent session logic
-├── client.py # Claude SDK client configuration
-├── security.py # Bash command allowlist and validation
-├── progress.py # Progress tracking utilities
-├── prompts.py # Prompt loading utilities
+autoforge/
+├── bin/ # npm CLI entry point
+├── lib/ # CLI bootstrap and setup logic
+├── start.py # CLI menu and project management
+├── start_ui.py # Web UI backend (FastAPI server launcher)
+├── autonomous_agent_demo.py # Agent entry point
+├── agent.py # Agent session logic
+├── client.py # Claude SDK client configuration
+├── security.py # Bash command allowlist and validation
+├── progress.py # Progress tracking utilities
+├── prompts.py # Prompt loading utilities
├── api/
-│ └── database.py # SQLAlchemy models (Feature table)
+│ └── database.py # SQLAlchemy models (Feature table)
├── mcp_server/
-│ └── feature_mcp.py # MCP server for feature management tools
+│ └── feature_mcp.py # MCP server for feature management tools
├── server/
-│ ├── main.py # FastAPI REST API server
-│ ├── websocket.py # WebSocket handler for real-time updates
-│ ├── schemas.py # Pydantic schemas
-│ ├── routers/ # API route handlers
-│ └── services/ # Business logic services
-├── ui/ # React frontend
+│ ├── main.py # FastAPI REST API server
+│ ├── websocket.py # WebSocket handler for real-time updates
+│ ├── schemas.py # Pydantic schemas
+│ ├── routers/ # API route handlers
+│ └── services/ # Business logic services
+├── ui/ # React frontend
│ ├── src/
-│ │ ├── App.tsx # Main app component
-│ │ ├── hooks/ # React Query and WebSocket hooks
-│ │ └── lib/ # API client and types
+│ │ ├── App.tsx # Main app component
+│ │ ├── hooks/ # React Query and WebSocket hooks
+│ │ └── lib/ # API client and types
│ ├── package.json
│ └── vite.config.ts
├── .claude/
│ ├── commands/
-│ │ └── create-spec.md # /create-spec slash command
-│ ├── skills/ # Claude Code skills
-│ └── templates/ # Prompt templates
-├── generations/ # Generated projects go here
-├── requirements.txt # Python dependencies
-└── .env # Optional configuration (N8N webhook)
+│ │ └── create-spec.md # /create-spec slash command
+│ ├── skills/ # Claude Code skills
+│ └── templates/ # Prompt templates
+├── requirements.txt # Python dependencies (development)
+├── requirements-prod.txt # Python dependencies (npm install)
+├── package.json # npm package definition
+└── .env # Optional configuration
```
---
@@ -264,11 +299,20 @@ The UI receives live updates via WebSocket (`/ws/projects/{project_name}`):
---
-## Configuration (Optional)
+## Configuration
+
+AutoForge reads configuration from a `.env` file. The file location depends on how you installed AutoForge:
+
+| Install method | Config file location | Edit command |
+|---|---|---|
+| npm (global) | `~/.autoforge/.env` | `autoforge config` |
+| From source | `.env` in the project root | Edit directly |
+
+A default config file is created automatically on first run. Use `autoforge config` to open it in your editor, or `autoforge config --show` to print the active values.
### N8N Webhook Integration
-The agent can send progress notifications to an N8N webhook. Create a `.env` file:
+Add to your `.env` to send progress notifications to an N8N webhook:
```bash
# Optional: N8N webhook for progress notifications
@@ -288,6 +332,29 @@ When test progress increases, the agent sends:
}
```
+### Alternative API Providers (GLM, Ollama, Kimi, Custom)
+
+Alternative providers are configured via the **Settings UI** (gear icon > API Provider). Select your provider, set the base URL, auth token, and model directly in the UI — no `.env` changes needed.
+
+Available providers: **Claude** (default), **GLM** (Zhipu AI), **Ollama** (local models), **Kimi** (Moonshot), **Custom**
+
+For Ollama, install [Ollama v0.14.0+](https://ollama.com), run `ollama serve`, and pull a coding model (e.g., `ollama pull qwen3-coder`). Then select "Ollama" in the Settings UI.
+
+### Using Vertex AI
+
+Add these variables to your `.env` file to run agents via Google Cloud Vertex AI:
+
+```bash
+CLAUDE_CODE_USE_VERTEX=1
+CLOUD_ML_REGION=us-east5
+ANTHROPIC_VERTEX_PROJECT_ID=your-gcp-project-id
+ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-4-6
+ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4-5@20250929
+ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku@20241022
+```
+
+Requires `gcloud auth application-default login` first. Note the `@` separator (not `-`) in Vertex AI model names.
+
---
## Customization
@@ -310,7 +377,7 @@ Edit `security.py` to add or remove commands from `ALLOWED_COMMANDS`.
Install the Claude Code CLI using the instructions in the Prerequisites section.
**"Not authenticated with Claude"**
-Run `claude login` to authenticate. The start script will prompt you to do this automatically.
+Set your API key via `ANTHROPIC_API_KEY` environment variable or the Settings UI. Alternatively, run `claude login` to use subscription credentials, but note that Anthropic's policy may not permit subscription-based auth for third-party agents.
**"Appears to hang on first run"**
This is normal. The initializer agent is generating detailed test cases, which takes significant time. Watch for `[Tool: ...]` output to confirm the agent is working.
@@ -318,6 +385,18 @@ This is normal. The initializer agent is generating detailed test cases, which t
**"Command blocked by security hook"**
The agent tried to run a command not in the allowlist. This is the security system working as intended. If needed, add the command to `ALLOWED_COMMANDS` in `security.py`.
+**"Python 3.11+ required but not found"**
+Install Python 3.11 or later from [python.org](https://www.python.org/downloads/). Make sure `python3` (or `python` on Windows) is on your PATH.
+
+**"Python venv module not available"**
+On Debian/Ubuntu, the venv module is packaged separately. Install it with `sudo apt install python3.XX-venv` (replace `XX` with your Python minor version, e.g., `python3.12-venv`).
+
+**"AutoForge is already running"**
+A server instance is already active. Use the browser URL shown in the terminal, or stop the existing instance with Ctrl+C first.
+
+**Virtual environment issues after a Python upgrade**
+Run `autoforge --repair` to delete and recreate the virtual environment from scratch.
+
---
## License
diff --git a/SAMPLE_PROMPT.md b/SAMPLE_PROMPT.md
deleted file mode 100644
index 284a4bf68..000000000
--- a/SAMPLE_PROMPT.md
+++ /dev/null
@@ -1,22 +0,0 @@
-Let's call it Simple Todo. This is a really simple web app that I can use to track my to-do items using a Kanban
-board. I should be able to add to-dos and then drag and drop them through the Kanban board. The different columns in
-the Kanban board are:
-
-- To Do
-- In Progress
-- Done
-
-The app should use a neobrutalism design.
-
-There is no need for user authentication either. All the to-dos will be stored in local storage, so each user has
-access to all of their to-dos when they open their browser. So do not worry about implementing a backend with user
-authentication or a database. Simply store everything in local storage. As for the design, please try to avoid AI
-slop, so use your front-end design skills to design something beautiful and practical. As for the content of the
-to-dos, we should store:
-
-- The name or the title at the very least
-- Optionally, we can also set tags, due dates, and priorities which should be represented as beautiful little badges
- on the to-do card Users should have the ability to easily clear out all the completed To-Dos. They should also be
- able to filter and search for To-Dos as well.
-
-You choose the rest. Keep it simple. Should be 25 features.
diff --git a/VISION.md b/VISION.md
new file mode 100644
index 000000000..3ae6975d4
--- /dev/null
+++ b/VISION.md
@@ -0,0 +1,22 @@
+# VISION
+
+This document defines the mandatory project vision for AutoForge. All contributions must align with these principles. PRs that deviate from this vision will be rejected. This file itself is immutable via PR — any PR that modifies VISION.md will be rejected outright.
+
+## Claude Agent SDK Exclusivity
+
+AutoForge is a wrapper around the **Claude Agent SDK**. This is a foundational architectural decision, not a preference.
+
+**What this means:**
+
+- AutoForge only supports providers, models, and integrations that work through the Claude Agent SDK.
+- We will not integrate with, accommodate, or add support for other AI SDKs, CLIs, or coding agent platforms (e.g., Codex, OpenCode, Aider, Continue, Cursor agents, or similar tools).
+
+**Why:**
+
+Each platform has its own approach to MCP tools, skills, context management, and feature integration. Attempting to support multiple agent frameworks creates an unsustainable maintenance burden and dilutes the quality of the core experience. By committing to the Claude Agent SDK exclusively, we can build deep, reliable integration rather than shallow compatibility across many targets.
+
+**In practice:**
+
+- PRs adding support for non-Claude agent frameworks will be rejected.
+- PRs introducing abstractions designed to make AutoForge "agent-agnostic" will be rejected.
+- Alternative API providers (e.g., Vertex AI, AWS Bedrock) are acceptable only when accessed through the Claude Agent SDK's own configuration.
diff --git a/agent.py b/agent.py
index e4d0de494..668391f7b 100644
--- a/agent.py
+++ b/agent.py
@@ -7,25 +7,43 @@
import asyncio
import io
+import re
import sys
+from datetime import datetime, timedelta
from pathlib import Path
from typing import Optional
+from zoneinfo import ZoneInfo
from claude_agent_sdk import ClaudeSDKClient
# Fix Windows console encoding for Unicode characters (emoji, etc.)
# Without this, print() crashes when Claude outputs emoji like ✅
if sys.platform == "win32":
- sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
- sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
+ sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace", line_buffering=True)
+ sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace", line_buffering=True)
from client import create_client
-from progress import has_features, print_progress_summary, print_session_header
+from progress import (
+ count_passing_tests,
+ has_features,
+ print_progress_summary,
+ print_session_header,
+)
from prompts import (
copy_spec_to_project,
+ get_auto_improve_prompt,
+ get_batch_feature_prompt,
get_coding_prompt,
- get_coding_prompt_yolo,
get_initializer_prompt,
+ get_single_feature_prompt,
+ get_testing_prompt,
+)
+from rate_limit_utils import (
+ calculate_error_backoff,
+ calculate_rate_limit_backoff,
+ clamp_retry_delay,
+ is_rate_limit_error,
+ parse_retry_after,
)
# Configuration
@@ -57,53 +75,83 @@ async def run_agent_session(
await client.query(message)
# Collect response text and show tool use
+ # Retry receive_response() on MessageParseError — the SDK raises this for
+ # unknown CLI message types (e.g. "rate_limit_event") which kills the async
+ # generator. The subprocess is still alive so we restart to read remaining
+ # messages from the buffered channel.
response_text = ""
- async for msg in client.receive_response():
- msg_type = type(msg).__name__
-
- # Handle AssistantMessage (text and tool use)
- if msg_type == "AssistantMessage" and hasattr(msg, "content"):
- for block in msg.content:
- block_type = type(block).__name__
-
- if block_type == "TextBlock" and hasattr(block, "text"):
- response_text += block.text
- print(block.text, end="", flush=True)
- elif block_type == "ToolUseBlock" and hasattr(block, "name"):
- print(f"\n[Tool: {block.name}]", flush=True)
- if hasattr(block, "input"):
- input_str = str(block.input)
- if len(input_str) > 200:
- print(f" Input: {input_str[:200]}...", flush=True)
- else:
- print(f" Input: {input_str}", flush=True)
-
- # Handle UserMessage (tool results)
- elif msg_type == "UserMessage" and hasattr(msg, "content"):
- for block in msg.content:
- block_type = type(block).__name__
-
- if block_type == "ToolResultBlock":
- result_content = getattr(block, "content", "")
- is_error = getattr(block, "is_error", False)
-
- # Check if command was blocked by security hook
- if "blocked" in str(result_content).lower():
- print(f" [BLOCKED] {result_content}", flush=True)
- elif is_error:
- # Show errors (truncated)
- error_str = str(result_content)[:500]
- print(f" [Error] {error_str}", flush=True)
- else:
- # Tool succeeded - just show brief confirmation
- print(" [Done]", flush=True)
+ max_parse_retries = 50
+ parse_retries = 0
+ while True:
+ try:
+ async for msg in client.receive_response():
+ msg_type = type(msg).__name__
+
+ # Handle AssistantMessage (text and tool use)
+ if msg_type == "AssistantMessage" and hasattr(msg, "content"):
+ for block in msg.content:
+ block_type = type(block).__name__
+
+ if block_type == "TextBlock" and hasattr(block, "text"):
+ response_text += block.text
+ print(block.text, end="", flush=True)
+ elif block_type == "ToolUseBlock" and hasattr(block, "name"):
+ print(f"\n[Tool: {block.name}]", flush=True)
+ if hasattr(block, "input"):
+ input_str = str(block.input)
+ if len(input_str) > 200:
+ print(f" Input: {input_str[:200]}...", flush=True)
+ else:
+ print(f" Input: {input_str}", flush=True)
+
+ # Handle UserMessage (tool results)
+ elif msg_type == "UserMessage" and hasattr(msg, "content"):
+ for block in msg.content:
+ block_type = type(block).__name__
+
+ if block_type == "ToolResultBlock":
+ result_content = getattr(block, "content", "")
+ is_error = getattr(block, "is_error", False)
+
+ # Check if command was blocked by security hook
+ if "blocked" in str(result_content).lower():
+ print(f" [BLOCKED] {result_content}", flush=True)
+ elif is_error:
+ # Show errors (truncated)
+ error_str = str(result_content)[:500]
+ print(f" [Error] {error_str}", flush=True)
+ else:
+ # Tool succeeded - just show brief confirmation
+ print(" [Done]", flush=True)
+
+ break # Normal completion
+ except Exception as inner_exc:
+ if type(inner_exc).__name__ == "MessageParseError":
+ parse_retries += 1
+ if parse_retries > max_parse_retries:
+ print(f"Too many unrecognized CLI messages ({parse_retries}), stopping")
+ break
+ print(f"Ignoring unrecognized message from Claude CLI: {inner_exc}")
+ continue
+ raise # Re-raise to outer except
print("\n" + "-" * 70 + "\n")
return "continue", response_text
except Exception as e:
- print(f"Error during agent session: {e}")
- return "error", str(e)
+ error_str = str(e)
+ print(f"Error during agent session: {error_str}")
+
+ # Detect rate limit errors from exception message
+ if is_rate_limit_error(error_str):
+ # Try to extract retry-after time from error
+ retry_seconds = parse_retry_after(error_str)
+ if retry_seconds is not None:
+ return "rate_limit", str(retry_seconds)
+ else:
+ return "rate_limit", "unknown"
+
+ return "error", error_str
async def run_autonomous_agent(
@@ -111,6 +159,12 @@ async def run_autonomous_agent(
model: str,
max_iterations: Optional[int] = None,
yolo_mode: bool = False,
+ feature_id: Optional[int] = None,
+ feature_ids: Optional[list[int]] = None,
+ agent_type: Optional[str] = None,
+ testing_feature_id: Optional[int] = None,
+ testing_feature_ids: Optional[list[int]] = None,
+ auto_improve: bool = False,
) -> None:
"""
Run the autonomous agent loop.
@@ -119,17 +173,31 @@ async def run_autonomous_agent(
project_dir: Directory for the project
model: Claude model to use
max_iterations: Maximum number of iterations (None for unlimited)
- yolo_mode: If True, skip browser testing and use YOLO prompt
+ yolo_mode: If True, skip browser testing in coding agent prompts
+ feature_id: If set, work only on this specific feature (used by orchestrator for coding agents)
+ feature_ids: If set, work on these features in batch (used by orchestrator for batch mode)
+ agent_type: Type of agent: "initializer", "coding", "testing", or None (auto-detect)
+ testing_feature_id: For testing agents, the pre-claimed feature ID to test (legacy single mode)
+ testing_feature_ids: For testing agents, list of feature IDs to batch test
+ auto_improve: If True, run in auto-improve mode (agent creates one
+ improvement feature, implements it, commits, and exits). Takes
+ precedence over other prompt selection branches.
"""
print("\n" + "=" * 70)
- print(" AUTONOMOUS CODING AGENT DEMO")
+ print(" AUTONOMOUS CODING AGENT")
print("=" * 70)
print(f"\nProject directory: {project_dir}")
print(f"Model: {model}")
+ if agent_type:
+ print(f"Agent type: {agent_type}")
+ if auto_improve:
+ print("Mode: AUTO-IMPROVE (one improvement + commit per session)")
if yolo_mode:
- print("Mode: YOLO (testing disabled)")
- else:
- print("Mode: Standard (full testing)")
+ print("Mode: YOLO (testing agents disabled)")
+ if feature_ids and len(feature_ids) > 1:
+ print(f"Feature batch: {', '.join(f'#{fid}' for fid in feature_ids)}")
+ elif feature_id:
+ print(f"Feature assignment: #{feature_id}")
if max_iterations:
print(f"Max iterations: {max_iterations}")
else:
@@ -139,32 +207,56 @@ async def run_autonomous_agent(
# Create project directory
project_dir.mkdir(parents=True, exist_ok=True)
- # Check if this is a fresh start or continuation
- # Uses has_features() which checks if the database actually has features,
- # not just if the file exists (empty db should still trigger initializer)
- is_first_run = not has_features(project_dir)
+ # Determine agent type if not explicitly set
+ if agent_type is None:
+ # Auto-detect based on whether we have features
+ # (This path is for legacy compatibility - orchestrator should always set agent_type)
+ is_first_run = not has_features(project_dir)
+ if is_first_run:
+ agent_type = "initializer"
+ else:
+ agent_type = "coding"
+
+ is_initializer = agent_type == "initializer"
- if is_first_run:
- print("Fresh start - will use initializer agent")
+ if is_initializer:
+ print("Running as INITIALIZER agent")
print()
print("=" * 70)
- print(" NOTE: First session takes 10-20+ minutes!")
- print(" The agent is generating 200 detailed test cases.")
+ print(" NOTE: Initialization takes 10-20+ minutes!")
+ print(" The agent is generating detailed test cases.")
print(" This may appear to hang - it's working. Watch for [Tool: ...] output.")
print("=" * 70)
print()
# Copy the app spec into the project directory for the agent to read
copy_spec_to_project(project_dir)
+ elif agent_type == "testing":
+ print("Running as TESTING agent (regression testing)")
+ print_progress_summary(project_dir)
else:
- print("Continuing existing project")
+ print("Running as CODING agent")
print_progress_summary(project_dir)
# Main loop
iteration = 0
+ rate_limit_retries = 0 # Track consecutive rate limit errors for exponential backoff
+ error_retries = 0 # Track consecutive non-rate-limit errors
while True:
iteration += 1
+ # Check if all features are already complete (before starting a new session)
+ # Skip this check if running as initializer (needs to create features first)
+ # or auto-improve mode (intentionally runs against finished projects)
+ if not is_initializer and not auto_improve and iteration == 1:
+ passing, in_progress, total, _nhi = count_passing_tests(project_dir)
+ if total > 0 and passing == total:
+ print("\n" + "=" * 70)
+ print(" ALL FEATURES ALREADY COMPLETE!")
+ print("=" * 70)
+ print(f"\nAll {total} features are passing. Nothing left to do.")
+ break
+
# Check max iterations
if max_iterations and iteration > max_iterations:
print(f"\nReached max iterations ({max_iterations})")
@@ -172,37 +264,180 @@ async def run_autonomous_agent(
break
# Print session header
- print_session_header(iteration, is_first_run)
+ print_session_header(iteration, is_initializer)
# Create client (fresh context)
- client = create_client(project_dir, model, yolo_mode=yolo_mode)
-
- # Choose prompt based on session type
- # Pass project_dir to enable project-specific prompts
- if is_first_run:
+ client = create_client(project_dir, model, yolo_mode=yolo_mode, agent_type=agent_type)
+
+ # Choose prompt based on agent type
+ # auto_improve takes precedence over other branches — it's a distinct
+ # mode where the agent creates its own feature before implementing it.
+ if auto_improve:
+ prompt = get_auto_improve_prompt(project_dir, yolo_mode=yolo_mode)
+ elif agent_type == "initializer":
prompt = get_initializer_prompt(project_dir)
- is_first_run = False # Only use initializer once
+ elif agent_type == "testing":
+ prompt = get_testing_prompt(project_dir, testing_feature_id, testing_feature_ids)
+ elif feature_ids and len(feature_ids) > 1:
+ # Batch mode (used by orchestrator for multi-feature coding agents)
+ prompt = get_batch_feature_prompt(feature_ids, project_dir, yolo_mode)
+ elif feature_id or (feature_ids is not None and len(feature_ids) == 1):
+ # Single-feature mode (used by orchestrator for coding agents)
+ fid = feature_id if feature_id is not None else feature_ids[0] # type: ignore[index]
+ prompt = get_single_feature_prompt(fid, project_dir, yolo_mode)
else:
- # Use YOLO prompt if in YOLO mode
- if yolo_mode:
- prompt = get_coding_prompt_yolo(project_dir)
- else:
- prompt = get_coding_prompt(project_dir)
+ # General coding prompt (legacy path)
+ prompt = get_coding_prompt(project_dir, yolo_mode=yolo_mode)
# Run session with async context manager
- async with client:
- status, response = await run_agent_session(client, prompt, project_dir)
+ # Wrap in try/except to handle MCP server startup failures gracefully
+ try:
+ async with client:
+ status, response = await run_agent_session(client, prompt, project_dir)
+ except Exception as e:
+ print(f"Client/MCP server error: {e}")
+ # Don't crash - return error status so the loop can retry
+ status, response = "error", str(e)
+
+ # Check for project completion - EXIT when all features pass
+ if "all features are passing" in response.lower() or "no more work to do" in response.lower():
+ print("\n" + "=" * 70)
+ print(" 🎉 PROJECT COMPLETE - ALL FEATURES PASSING!")
+ print("=" * 70)
+ print_progress_summary(project_dir)
+ break
# Handle status
if status == "continue":
- print(f"\nAgent will auto-continue in {AUTO_CONTINUE_DELAY_SECONDS}s...")
+ # Reset error retries on success; rate-limit retries reset only if no signal
+ error_retries = 0
+ reset_rate_limit_retries = True
+
+ delay_seconds = AUTO_CONTINUE_DELAY_SECONDS
+ target_time_str = None
+
+ # Check for rate limit indicators in response text
+ if is_rate_limit_error(response):
+ print("Claude Agent SDK indicated rate limit reached.")
+ reset_rate_limit_retries = False
+
+ # Try to extract retry-after from response text first
+ retry_seconds = parse_retry_after(response)
+ if retry_seconds is not None:
+ delay_seconds = clamp_retry_delay(retry_seconds)
+ else:
+ # Use exponential backoff when retry-after unknown
+ delay_seconds = calculate_rate_limit_backoff(rate_limit_retries)
+ rate_limit_retries += 1
+
+ # Try to parse reset time from response (more specific format)
+ match = re.search(
+ r"(?i)\bresets(?:\s+at)?\s+(\d+)(?::(\d+))?\s*(am|pm)\s*\(([^)]+)\)",
+ response,
+ )
+ if match:
+ hour = int(match.group(1))
+ minute = int(match.group(2)) if match.group(2) else 0
+ period = match.group(3).lower()
+ tz_name = match.group(4).strip()
+
+ # Convert to 24-hour format
+ if period == "pm" and hour != 12:
+ hour += 12
+ elif period == "am" and hour == 12:
+ hour = 0
+
+ try:
+ tz = ZoneInfo(tz_name)
+ now = datetime.now(tz)
+ target = now.replace(
+ hour=hour, minute=minute, second=0, microsecond=0
+ )
+
+ # If target time has already passed today, wait until tomorrow
+ if target <= now:
+ target += timedelta(days=1)
+
+ delta = target - now
+ delay_seconds = min(max(int(delta.total_seconds()), 1), 24 * 60 * 60)
+ target_time_str = target.strftime("%B %d, %Y at %I:%M %p %Z")
+
+ except Exception as e:
+ print(f"Error parsing reset time: {e}, using default delay")
+
+ if target_time_str:
+ print(
+ f"\nClaude Code Limit Reached. Agent will auto-continue in {delay_seconds:.0f}s ({target_time_str})...",
+ flush=True,
+ )
+ else:
+ print(
+ f"\nAgent will auto-continue in {delay_seconds:.0f}s...", flush=True
+ )
+
+ sys.stdout.flush() # this should allow the pause to be displayed before sleeping
print_progress_summary(project_dir)
- await asyncio.sleep(AUTO_CONTINUE_DELAY_SECONDS)
+
+ # Check if all features are complete - exit gracefully if done
+ passing, in_progress, total, _nhi = count_passing_tests(project_dir)
+ if total > 0 and passing == total:
+ print("\n" + "=" * 70)
+ print(" ALL FEATURES COMPLETE!")
+ print("=" * 70)
+ print(f"\nCongratulations! All {total} features are passing.")
+ print("The autonomous agent has finished its work.")
+ break
+
+ # Single-feature mode, batch mode, or testing agent: exit after one session
+ if feature_ids and len(feature_ids) > 1:
+ print(f"\nBatch mode: Features {', '.join(f'#{fid}' for fid in feature_ids)} session complete.")
+ break
+ elif feature_id is not None or (feature_ids is not None and len(feature_ids) == 1):
+ fid = feature_id if feature_id is not None else feature_ids[0] # type: ignore[index]
+ if agent_type == "testing":
+ print("\nTesting agent complete. Terminating session.")
+ else:
+ print(f"\nSingle-feature mode: Feature #{fid} session complete.")
+ break
+ elif agent_type == "testing":
+ print("\nTesting agent complete. Terminating session.")
+ break
+
+ # Reset rate limit retries only if no rate limit signal was detected
+ if reset_rate_limit_retries:
+ rate_limit_retries = 0
+
+ await asyncio.sleep(delay_seconds)
+
+ elif status == "rate_limit":
+ # Smart rate limit handling with exponential backoff
+ # Reset error counter so mixed events don't inflate delays
+ error_retries = 0
+ if response != "unknown":
+ try:
+ delay_seconds = clamp_retry_delay(int(response))
+ except (ValueError, TypeError):
+ # Malformed value - fall through to exponential backoff
+ response = "unknown"
+ if response == "unknown":
+ # Use exponential backoff when retry-after unknown or malformed
+ delay_seconds = calculate_rate_limit_backoff(rate_limit_retries)
+ rate_limit_retries += 1
+ print(f"\nRate limit hit. Backoff wait: {delay_seconds} seconds (attempt #{rate_limit_retries})...")
+ else:
+ print(f"\nRate limit hit. Waiting {delay_seconds} seconds before retry...")
+
+ await asyncio.sleep(delay_seconds)
elif status == "error":
+ # Non-rate-limit errors: linear backoff capped at 5 minutes
+ # Reset rate limit counter so mixed events don't inflate delays
+ rate_limit_retries = 0
+ error_retries += 1
+ delay_seconds = calculate_error_backoff(error_retries)
print("\nSession encountered an error")
- print("Will retry with a fresh session...")
- await asyncio.sleep(AUTO_CONTINUE_DELAY_SECONDS)
+ print(f"Will retry in {delay_seconds}s (attempt #{error_retries})...")
+ await asyncio.sleep(delay_seconds)
# Small delay between sessions
if max_iterations is None or iteration < max_iterations:
diff --git a/api/database.py b/api/database.py
index a74b857aa..523ea2243 100644
--- a/api/database.py
+++ b/api/database.py
@@ -5,15 +5,37 @@
SQLite database schema for feature storage using SQLAlchemy.
"""
+import sys
+from datetime import datetime, timezone
from pathlib import Path
-from typing import Optional
-
-from sqlalchemy import Boolean, Column, Integer, String, Text, create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import Session, sessionmaker
+from typing import Generator, Optional
+
+
+def _utc_now() -> datetime:
+ """Return current UTC time. Replacement for deprecated _utc_now()."""
+ return datetime.now(timezone.utc)
+
+from sqlalchemy import (
+ Boolean,
+ CheckConstraint,
+ Column,
+ DateTime,
+ ForeignKey,
+ Index,
+ Integer,
+ String,
+ Text,
+ create_engine,
+ event,
+ text,
+)
+from sqlalchemy.orm import DeclarativeBase, Session, relationship, sessionmaker
from sqlalchemy.types import JSON
-Base = declarative_base()
+
+class Base(DeclarativeBase):
+ """SQLAlchemy 2.0 style declarative base."""
+ pass
class Feature(Base):
@@ -21,14 +43,28 @@ class Feature(Base):
__tablename__ = "features"
+ # Composite index for common status query pattern (passes, in_progress, needs_human_input)
+ # Used by feature_get_stats, get_ready_features, and other status queries
+ __table_args__ = (
+ Index('ix_feature_status', 'passes', 'in_progress', 'needs_human_input'),
+ )
+
id = Column(Integer, primary_key=True, index=True)
priority = Column(Integer, nullable=False, default=999, index=True)
category = Column(String(100), nullable=False)
name = Column(String(255), nullable=False)
description = Column(Text, nullable=False)
steps = Column(JSON, nullable=False) # Stored as JSON array
- passes = Column(Boolean, default=False, index=True)
- in_progress = Column(Boolean, default=False, index=True)
+ passes = Column(Boolean, nullable=False, default=False, index=True)
+ in_progress = Column(Boolean, nullable=False, default=False, index=True)
+ # Dependencies: list of feature IDs that must be completed before this feature
+ # NULL/empty = no dependencies (backwards compatible)
+ dependencies = Column(JSON, nullable=True, default=None)
+
+ # Human input: agent can request structured input from a human
+ needs_human_input = Column(Boolean, nullable=False, default=False, index=True)
+ human_input_request = Column(JSON, nullable=True, default=None) # Agent's structured request
+ human_input_response = Column(JSON, nullable=True, default=None) # Human's response
def to_dict(self) -> dict:
"""Convert feature to dictionary for JSON serialization."""
@@ -39,14 +75,125 @@ def to_dict(self) -> dict:
"name": self.name,
"description": self.description,
"steps": self.steps,
- "passes": self.passes,
- "in_progress": self.in_progress,
+ # Handle legacy NULL values gracefully - treat as False
+ "passes": self.passes if self.passes is not None else False,
+ "in_progress": self.in_progress if self.in_progress is not None else False,
+ # Dependencies: NULL/empty treated as empty list for backwards compat
+ "dependencies": self.dependencies if self.dependencies else [],
+ # Human input fields
+ "needs_human_input": self.needs_human_input if self.needs_human_input is not None else False,
+ "human_input_request": self.human_input_request,
+ "human_input_response": self.human_input_response,
+ }
+
+ def get_dependencies_safe(self) -> list[int]:
+ """Safely extract dependencies, handling NULL and malformed data."""
+ if self.dependencies is None:
+ return []
+ if isinstance(self.dependencies, list):
+ return [d for d in self.dependencies if isinstance(d, int)]
+ return []
+
+
+class Schedule(Base):
+ """Time-based schedule for automated agent start/stop."""
+
+ __tablename__ = "schedules"
+
+ # Database-level CHECK constraints for data integrity
+ __table_args__ = (
+ CheckConstraint('duration_minutes >= 1 AND duration_minutes <= 1440', name='ck_schedule_duration'),
+ CheckConstraint('days_of_week >= 0 AND days_of_week <= 127', name='ck_schedule_days'),
+ CheckConstraint('max_concurrency >= 1 AND max_concurrency <= 5', name='ck_schedule_concurrency'),
+ CheckConstraint('crash_count >= 0', name='ck_schedule_crash_count'),
+ )
+
+ id = Column(Integer, primary_key=True, index=True)
+ project_name = Column(String(50), nullable=False, index=True)
+
+ # Timing (stored in UTC)
+ start_time = Column(String(5), nullable=False) # "HH:MM" format
+ duration_minutes = Column(Integer, nullable=False) # 1-1440
+
+ # Day filtering (bitfield: Mon=1, Tue=2, Wed=4, Thu=8, Fri=16, Sat=32, Sun=64)
+ days_of_week = Column(Integer, nullable=False, default=127) # 127 = all days
+
+ # State
+ enabled = Column(Boolean, nullable=False, default=True, index=True)
+
+ # Agent configuration for scheduled runs
+ yolo_mode = Column(Boolean, nullable=False, default=False)
+ model = Column(String(50), nullable=True) # None = use global default
+ max_concurrency = Column(Integer, nullable=False, default=3) # 1-5 concurrent agents
+
+ # Crash recovery tracking
+ crash_count = Column(Integer, nullable=False, default=0) # Resets at window start
+
+ # Metadata
+ created_at = Column(DateTime, nullable=False, default=_utc_now)
+
+ # Relationships
+ overrides = relationship(
+ "ScheduleOverride", back_populates="schedule", cascade="all, delete-orphan"
+ )
+
+ def to_dict(self) -> dict:
+ """Convert schedule to dictionary for JSON serialization."""
+ return {
+ "id": self.id,
+ "project_name": self.project_name,
+ "start_time": self.start_time,
+ "duration_minutes": self.duration_minutes,
+ "days_of_week": self.days_of_week,
+ "enabled": self.enabled,
+ "yolo_mode": self.yolo_mode,
+ "model": self.model,
+ "max_concurrency": self.max_concurrency,
+ "crash_count": self.crash_count,
+ "created_at": self.created_at.isoformat() if self.created_at else None,
+ }
+
+ def is_active_on_day(self, weekday: int) -> bool:
+ """Check if schedule is active on given weekday (0=Monday, 6=Sunday)."""
+ day_bit = 1 << weekday
+ return bool(self.days_of_week & day_bit)
+
+
+class ScheduleOverride(Base):
+ """Persisted manual override for a schedule window."""
+
+ __tablename__ = "schedule_overrides"
+
+ id = Column(Integer, primary_key=True, index=True)
+ schedule_id = Column(
+ Integer, ForeignKey("schedules.id", ondelete="CASCADE"), nullable=False
+ )
+
+ # Override details
+ override_type = Column(String(10), nullable=False) # "start" or "stop"
+ expires_at = Column(DateTime, nullable=False) # When this window ends (UTC)
+
+ # Metadata
+ created_at = Column(DateTime, nullable=False, default=_utc_now)
+
+ # Relationships
+ schedule = relationship("Schedule", back_populates="overrides")
+
+ def to_dict(self) -> dict:
+ """Convert override to dictionary for JSON serialization."""
+ return {
+ "id": self.id,
+ "schedule_id": self.schedule_id,
+ "override_type": self.override_type,
+ "expires_at": self.expires_at.isoformat() if self.expires_at else None,
+ "created_at": self.created_at.isoformat() if self.created_at else None,
}
def get_database_path(project_dir: Path) -> Path:
"""Return the path to the SQLite database for a project."""
- return project_dir / "features.db"
+ from autoforge_paths import get_features_db_path
+ return get_features_db_path(project_dir)
def get_database_url(project_dir: Path) -> str:
@@ -60,8 +207,6 @@ def get_database_url(project_dir: Path) -> str:
def _migrate_add_in_progress_column(engine) -> None:
"""Add in_progress column to existing databases that don't have it."""
- from sqlalchemy import text
-
with engine.connect() as conn:
# Check if column exists
result = conn.execute(text("PRAGMA table_info(features)"))
@@ -73,30 +218,276 @@ def _migrate_add_in_progress_column(engine) -> None:
conn.commit()
+def _migrate_fix_null_boolean_fields(engine) -> None:
+ """Fix NULL values in passes and in_progress columns."""
+ with engine.connect() as conn:
+ # Fix NULL passes values
+ conn.execute(text("UPDATE features SET passes = 0 WHERE passes IS NULL"))
+ # Fix NULL in_progress values
+ conn.execute(text("UPDATE features SET in_progress = 0 WHERE in_progress IS NULL"))
+ conn.commit()
+
+
+def _migrate_add_dependencies_column(engine) -> None:
+ """Add dependencies column to existing databases that don't have it.
+
+ Uses NULL default for backwards compatibility - existing features
+ without dependencies will have NULL which is treated as empty list.
+ """
+ with engine.connect() as conn:
+ # Check if column exists
+ result = conn.execute(text("PRAGMA table_info(features)"))
+ columns = [row[1] for row in result.fetchall()]
+
+ if "dependencies" not in columns:
+ # Use TEXT for SQLite JSON storage, NULL default for backwards compat
+ conn.execute(text("ALTER TABLE features ADD COLUMN dependencies TEXT DEFAULT NULL"))
+ conn.commit()
+
+
+def _migrate_add_testing_columns(engine) -> None:
+ """Legacy migration - no longer adds testing columns.
+
+ The testing_in_progress and last_tested_at columns were removed from the
+ Feature model as part of simplifying the testing agent architecture.
+ Multiple testing agents can now test the same feature concurrently
+ without coordination.
+
+ This function is kept for backwards compatibility but does nothing.
+ Existing databases with these columns will continue to work - the columns
+ are simply ignored.
+ """
+ pass
+
+
+def _is_network_path(path: Path) -> bool:
+ """Detect if path is on a network filesystem.
+
+ WAL mode doesn't work reliably on network filesystems (NFS, SMB, CIFS)
+ and can cause database corruption. This function detects common network
+ path patterns so we can fall back to DELETE mode.
+
+ Args:
+ path: The path to check
+
+ Returns:
+ True if the path appears to be on a network filesystem
+ """
+ path_str = str(path.resolve())
+
+ if sys.platform == "win32":
+ # Windows UNC paths: \\server\share or \\?\UNC\server\share
+ if path_str.startswith("\\\\"):
+ return True
+ # Mapped network drives - check if the drive is a network drive
+ try:
+ import ctypes
+ drive = path_str[:2] # e.g., "Z:"
+ if len(drive) == 2 and drive[1] == ":":
+ # DRIVE_REMOTE = 4
+ drive_type = ctypes.windll.kernel32.GetDriveTypeW(drive + "\\")
+ if drive_type == 4: # DRIVE_REMOTE
+ return True
+ except (AttributeError, OSError):
+ pass
+ else:
+ # Unix: Check mount type via /proc/mounts or mount command
+ try:
+ with open("/proc/mounts", "r") as f:
+ mounts = f.read()
+ # Check each mount point to find which one contains our path
+ for line in mounts.splitlines():
+ parts = line.split()
+ if len(parts) >= 3:
+ mount_point = parts[1]
+ fs_type = parts[2]
+ # Check if path is under this mount point and if it's a network FS
+ if path_str.startswith(mount_point):
+ if fs_type in ("nfs", "nfs4", "cifs", "smbfs", "fuse.sshfs"):
+ return True
+ except (FileNotFoundError, PermissionError):
+ pass
+
+ return False
+
+
+def _migrate_add_human_input_columns(engine) -> None:
+ """Add human input columns to existing databases that don't have them."""
+ with engine.connect() as conn:
+ result = conn.execute(text("PRAGMA table_info(features)"))
+ columns = [row[1] for row in result.fetchall()]
+
+ if "needs_human_input" not in columns:
+ conn.execute(text("ALTER TABLE features ADD COLUMN needs_human_input BOOLEAN DEFAULT 0"))
+ if "human_input_request" not in columns:
+ conn.execute(text("ALTER TABLE features ADD COLUMN human_input_request TEXT DEFAULT NULL"))
+ if "human_input_response" not in columns:
+ conn.execute(text("ALTER TABLE features ADD COLUMN human_input_response TEXT DEFAULT NULL"))
+ conn.commit()
+
+
+def _migrate_add_schedules_tables(engine) -> None:
+ """Create schedules and schedule_overrides tables if they don't exist."""
+ from sqlalchemy import inspect
+
+ inspector = inspect(engine)
+ existing_tables = inspector.get_table_names()
+
+ # Create schedules table if missing
+ if "schedules" not in existing_tables:
+ Schedule.__table__.create(bind=engine) # type: ignore[attr-defined]
+
+ # Create schedule_overrides table if missing
+ if "schedule_overrides" not in existing_tables:
+ ScheduleOverride.__table__.create(bind=engine) # type: ignore[attr-defined]
+
+ # Add crash_count column if missing (for upgrades)
+ if "schedules" in existing_tables:
+ columns = [c["name"] for c in inspector.get_columns("schedules")]
+ if "crash_count" not in columns:
+ with engine.connect() as conn:
+ conn.execute(
+ text("ALTER TABLE schedules ADD COLUMN crash_count INTEGER DEFAULT 0")
+ )
+ conn.commit()
+
+ # Add max_concurrency column if missing (for upgrades)
+ if "max_concurrency" not in columns:
+ with engine.connect() as conn:
+ conn.execute(
+ text("ALTER TABLE schedules ADD COLUMN max_concurrency INTEGER DEFAULT 3")
+ )
+ conn.commit()
+
+
+def _configure_sqlite_immediate_transactions(engine) -> None:
+ """Configure engine for IMMEDIATE transactions via event hooks.
+
+ Per SQLAlchemy docs: https://docs.sqlalchemy.org/en/20/dialects/sqlite.html
+
+ This replaces fragile pysqlite implicit transaction handling with explicit
+ BEGIN IMMEDIATE at transaction start. Benefits:
+ - Acquires write lock immediately, preventing stale reads
+ - Works correctly regardless of prior ORM operations
+ - Future-proof: won't break when pysqlite legacy mode is removed in Python 3.16
+ """
+ @event.listens_for(engine, "connect")
+ def do_connect(dbapi_connection, connection_record):
+ # Disable pysqlite's implicit transaction handling
+ dbapi_connection.isolation_level = None
+
+ # Set busy_timeout on raw connection before any transactions
+ cursor = dbapi_connection.cursor()
+ try:
+ cursor.execute("PRAGMA busy_timeout=30000")
+ finally:
+ cursor.close()
+
+ @event.listens_for(engine, "begin")
+ def do_begin(conn):
+ # Use IMMEDIATE for all transactions to prevent stale reads
+ conn.exec_driver_sql("BEGIN IMMEDIATE")
+
+
def create_database(project_dir: Path) -> tuple:
"""
Create database and return engine + session maker.
+ Uses a cache to avoid creating new engines for each request, which improves
+ performance by reusing database connections.
+
Args:
project_dir: Directory containing the project
Returns:
Tuple of (engine, SessionLocal)
"""
+ cache_key = project_dir.as_posix()
+
+ if cache_key in _engine_cache:
+ return _engine_cache[cache_key]
+
db_url = get_database_url(project_dir)
- engine = create_engine(db_url, connect_args={"check_same_thread": False})
+
+ # Ensure parent directory exists (for .autoforge/ layout)
+ db_path = get_database_path(project_dir)
+ db_path.parent.mkdir(parents=True, exist_ok=True)
+
+ # Choose journal mode based on filesystem type
+ # WAL mode doesn't work reliably on network filesystems and can cause corruption
+ is_network = _is_network_path(project_dir)
+ journal_mode = "DELETE" if is_network else "WAL"
+
+ engine = create_engine(db_url, connect_args={
+ "check_same_thread": False,
+ "timeout": 30 # Wait up to 30s for locks
+ })
+
+ # Set journal mode BEFORE configuring event hooks
+ # PRAGMA journal_mode must run outside of a transaction, and our event hooks
+ # start a transaction with BEGIN IMMEDIATE on every operation
+ with engine.connect() as conn:
+ # Get raw DBAPI connection to execute PRAGMA outside transaction
+ raw_conn = conn.connection.dbapi_connection
+ if raw_conn is None:
+ raise RuntimeError("Failed to get raw DBAPI connection")
+ cursor = raw_conn.cursor()
+ try:
+ cursor.execute(f"PRAGMA journal_mode={journal_mode}")
+ cursor.execute("PRAGMA busy_timeout=30000")
+ finally:
+ cursor.close()
+
+ # Configure IMMEDIATE transactions via event hooks AFTER setting PRAGMAs
+ # This must happen before create_all() and migrations run
+ _configure_sqlite_immediate_transactions(engine)
+
Base.metadata.create_all(bind=engine)
- # Migrate existing databases to add in_progress column
+ # Migrate existing databases
_migrate_add_in_progress_column(engine)
+ _migrate_fix_null_boolean_fields(engine)
+ _migrate_add_dependencies_column(engine)
+ _migrate_add_testing_columns(engine)
+ _migrate_add_human_input_columns(engine)
+
+ # Migrate to add schedules tables
+ _migrate_add_schedules_tables(engine)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+ # Cache the engine and session maker
+ _engine_cache[cache_key] = (engine, SessionLocal)
+
return engine, SessionLocal
+def dispose_engine(project_dir: Path) -> bool:
+ """Dispose of and remove the cached engine for a project.
+
+ This closes all database connections, releasing file locks on Windows.
+ Should be called before deleting the database file.
+
+ Returns:
+ True if an engine was disposed, False if no engine was cached.
+ """
+ cache_key = project_dir.as_posix()
+
+ if cache_key in _engine_cache:
+ engine, _ = _engine_cache.pop(cache_key)
+ engine.dispose()
+ return True
+
+ return False
+
+
# Global session maker - will be set when server starts
_session_maker: Optional[sessionmaker] = None
+# Engine cache to avoid creating new engines for each request
+# Key: project directory path (as posix string), Value: (engine, SessionLocal)
+_engine_cache: dict[str, tuple] = {}
+
def set_session_maker(session_maker: sessionmaker) -> None:
"""Set the global session maker."""
@@ -104,7 +495,7 @@ def set_session_maker(session_maker: sessionmaker) -> None:
_session_maker = session_maker
-def get_db() -> Session:
+def get_db() -> Generator[Session, None, None]:
"""
Dependency for FastAPI to get database session.
@@ -116,5 +507,55 @@ def get_db() -> Session:
db = _session_maker()
try:
yield db
+ except Exception:
+ db.rollback()
+ raise
finally:
db.close()
+
+
+# =============================================================================
+# Atomic Transaction Helpers for Parallel Mode
+# =============================================================================
+# These helpers prevent database corruption when multiple processes access the
+# same SQLite database concurrently. They use IMMEDIATE transactions which
+# acquire write locks at the start (preventing stale reads) and atomic
+# UPDATE ... WHERE clauses (preventing check-then-modify races).
+
+
+from contextlib import contextmanager
+
+
+@contextmanager
+def atomic_transaction(session_maker):
+ """Context manager for atomic SQLite transactions.
+
+ Acquires a write lock immediately via BEGIN IMMEDIATE (configured by
+ engine event hooks), preventing stale reads in read-modify-write patterns.
+ This is essential for preventing race conditions in parallel mode.
+
+ Args:
+ session_maker: SQLAlchemy sessionmaker
+
+ Yields:
+ SQLAlchemy session with automatic commit/rollback
+
+ Example:
+ with atomic_transaction(session_maker) as session:
+ # All reads in this block are protected by write lock
+ feature = session.query(Feature).filter(...).first()
+ feature.priority = new_priority
+ # Commit happens automatically on exit
+ """
+ session = session_maker()
+ try:
+ yield session
+ session.commit()
+ except Exception:
+ try:
+ session.rollback()
+ except Exception:
+ pass # Don't let rollback failure mask original error
+ raise
+ finally:
+ session.close()
diff --git a/api/dependency_resolver.py b/api/dependency_resolver.py
new file mode 100644
index 000000000..9cc8082d3
--- /dev/null
+++ b/api/dependency_resolver.py
@@ -0,0 +1,449 @@
+"""
+Dependency Resolver
+===================
+
+Provides dependency resolution using Kahn's algorithm for topological sorting.
+Includes cycle detection, validation, and helper functions for dependency management.
+"""
+
+import heapq
+from collections import deque
+from typing import TypedDict
+
+# Security: Prevent DoS via excessive dependencies
+MAX_DEPENDENCIES_PER_FEATURE = 20
+MAX_DEPENDENCY_DEPTH = 50 # Prevent stack overflow in cycle detection
+
+
+class DependencyResult(TypedDict):
+ """Result from dependency resolution."""
+
+ ordered_features: list[dict]
+ circular_dependencies: list[list[int]]
+ blocked_features: dict[int, list[int]] # feature_id -> [blocking_ids]
+ missing_dependencies: dict[int, list[int]] # feature_id -> [missing_ids]
+
+
+def resolve_dependencies(features: list[dict]) -> DependencyResult:
+ """Topological sort using Kahn's algorithm with priority-aware ordering.
+
+ Returns ordered features respecting dependencies, plus metadata about
+ cycles, blocked features, and missing dependencies.
+
+ Args:
+ features: List of feature dicts with id, priority, passes, and dependencies fields
+
+ Returns:
+ DependencyResult with ordered_features, circular_dependencies,
+ blocked_features, and missing_dependencies
+ """
+ feature_map = {f["id"]: f for f in features}
+ in_degree = {f["id"]: 0 for f in features}
+ adjacency: dict[int, list[int]] = {f["id"]: [] for f in features}
+ blocked: dict[int, list[int]] = {}
+ missing: dict[int, list[int]] = {}
+
+ # Build graph
+ for feature in features:
+ deps = feature.get("dependencies") or []
+ for dep_id in deps:
+ if dep_id not in feature_map:
+ missing.setdefault(feature["id"], []).append(dep_id)
+ else:
+ adjacency[dep_id].append(feature["id"])
+ in_degree[feature["id"]] += 1
+ # Track blocked features
+ dep = feature_map[dep_id]
+ if not dep.get("passes"):
+ blocked.setdefault(feature["id"], []).append(dep_id)
+
+ # Kahn's algorithm with priority-aware selection using a heap
+ # Heap entries are tuples: (priority, id, feature_dict) for stable ordering
+ heap = [
+ (f.get("priority", 999), f["id"], f)
+ for f in features
+ if in_degree[f["id"]] == 0
+ ]
+ heapq.heapify(heap)
+ ordered: list[dict] = []
+
+ while heap:
+ _, _, current = heapq.heappop(heap)
+ ordered.append(current)
+ for dependent_id in adjacency[current["id"]]:
+ in_degree[dependent_id] -= 1
+ if in_degree[dependent_id] == 0:
+ dep_feature = feature_map[dependent_id]
+ heapq.heappush(
+ heap,
+ (dep_feature.get("priority", 999), dependent_id, dep_feature)
+ )
+
+ # Detect cycles (features not in ordered = part of cycle)
+ cycles: list[list[int]] = []
+ if len(ordered) < len(features):
+ remaining = [f for f in features if f not in ordered]
+ cycles = _detect_cycles(remaining, feature_map)
+ ordered.extend(remaining) # Add cyclic features at end
+
+ return {
+ "ordered_features": ordered,
+ "circular_dependencies": cycles,
+ "blocked_features": blocked,
+ "missing_dependencies": missing,
+ }
+
+
+def are_dependencies_satisfied(
+ feature: dict,
+ all_features: list[dict],
+ passing_ids: set[int] | None = None,
+) -> bool:
+ """Check if all dependencies have passes=True.
+
+ Args:
+ feature: Feature dict to check
+ all_features: List of all feature dicts
+ passing_ids: Optional pre-computed set of passing feature IDs.
+ If None, will be computed from all_features. Pass this when
+ calling in a loop to avoid O(n^2) complexity.
+
+ Returns:
+ True if all dependencies are satisfied (or no dependencies)
+ """
+ deps = feature.get("dependencies") or []
+ if not deps:
+ return True
+ if passing_ids is None:
+ passing_ids = {f["id"] for f in all_features if f.get("passes")}
+ return all(dep_id in passing_ids for dep_id in deps)
+
+
+def get_blocking_dependencies(
+ feature: dict,
+ all_features: list[dict],
+ passing_ids: set[int] | None = None,
+) -> list[int]:
+ """Get list of incomplete dependency IDs.
+
+ Args:
+ feature: Feature dict to check
+ all_features: List of all feature dicts
+ passing_ids: Optional pre-computed set of passing feature IDs.
+ If None, will be computed from all_features. Pass this when
+ calling in a loop to avoid O(n^2) complexity.
+
+ Returns:
+ List of feature IDs that are blocking this feature
+ """
+ deps = feature.get("dependencies") or []
+ if passing_ids is None:
+ passing_ids = {f["id"] for f in all_features if f.get("passes")}
+ return [dep_id for dep_id in deps if dep_id not in passing_ids]
+
+
+def would_create_circular_dependency(
+ features: list[dict], source_id: int, target_id: int
+) -> bool:
+ """Check if adding a dependency from target to source would create a cycle.
+
+ Uses DFS with visited set for efficient cycle detection.
+
+ Args:
+ features: List of all feature dicts
+ source_id: The feature that would gain the dependency
+ target_id: The feature that would become a dependency
+
+ Returns:
+ True if adding the dependency would create a cycle
+ """
+ if source_id == target_id:
+ return True # Self-reference is a cycle
+
+ feature_map = {f["id"]: f for f in features}
+ source = feature_map.get(source_id)
+ if not source:
+ return False
+
+ # Check if target already depends on source (direct or indirect)
+ target = feature_map.get(target_id)
+ if not target:
+ return False
+
+ # DFS from target to see if we can reach source
+ visited: set[int] = set()
+
+ def can_reach(current_id: int, depth: int = 0) -> bool:
+ # Security: Prevent stack overflow with depth limit
+ if depth > MAX_DEPENDENCY_DEPTH:
+ return True # Assume cycle if too deep (fail-safe)
+ if current_id == source_id:
+ return True
+ if current_id in visited:
+ return False
+ visited.add(current_id)
+
+ current = feature_map.get(current_id)
+ if not current:
+ return False
+
+ deps = current.get("dependencies") or []
+ for dep_id in deps:
+ if can_reach(dep_id, depth + 1):
+ return True
+ return False
+
+ return can_reach(target_id)
+
+
+def validate_dependencies(
+ feature_id: int, dependency_ids: list[int], all_feature_ids: set[int]
+) -> tuple[bool, str]:
+ """Validate dependency list.
+
+ Args:
+ feature_id: ID of the feature being validated
+ dependency_ids: List of proposed dependency IDs
+ all_feature_ids: Set of all valid feature IDs
+
+ Returns:
+ Tuple of (is_valid, error_message)
+ """
+ # Security: Check limits
+ if len(dependency_ids) > MAX_DEPENDENCIES_PER_FEATURE:
+ return False, f"Maximum {MAX_DEPENDENCIES_PER_FEATURE} dependencies allowed"
+
+ # Check self-reference
+ if feature_id in dependency_ids:
+ return False, "A feature cannot depend on itself"
+
+ # Check all dependencies exist
+ missing = [d for d in dependency_ids if d not in all_feature_ids]
+ if missing:
+ return False, f"Dependencies not found: {missing}"
+
+ # Check for duplicates
+ if len(dependency_ids) != len(set(dependency_ids)):
+ return False, "Duplicate dependencies not allowed"
+
+ return True, ""
+
+
+def _detect_cycles(features: list[dict], feature_map: dict) -> list[list[int]]:
+ """Detect cycles using DFS with recursion tracking.
+
+ Args:
+ features: List of features to check for cycles
+ feature_map: Map of feature_id -> feature dict
+
+ Returns:
+ List of cycles, where each cycle is a list of feature IDs
+ """
+ cycles: list[list[int]] = []
+ visited: set[int] = set()
+ rec_stack: set[int] = set()
+ path: list[int] = []
+
+ def dfs(fid: int) -> bool:
+ visited.add(fid)
+ rec_stack.add(fid)
+ path.append(fid)
+
+ feature = feature_map.get(fid)
+ if feature:
+ for dep_id in feature.get("dependencies") or []:
+ if dep_id not in visited:
+ if dfs(dep_id):
+ return True
+ elif dep_id in rec_stack:
+ cycle_start = path.index(dep_id)
+ cycles.append(path[cycle_start:])
+ return True
+
+ path.pop()
+ rec_stack.remove(fid)
+ return False
+
+ for f in features:
+ if f["id"] not in visited:
+ dfs(f["id"])
+
+ return cycles
+
+
+def compute_scheduling_scores(features: list[dict]) -> dict[int, float]:
+ """Compute scheduling scores for all features.
+
+ Higher scores mean higher priority for scheduling. The algorithm considers:
+ 1. Unblocking potential - Features that unblock more downstream work score higher
+ 2. Depth in graph - Features with no dependencies (roots) are "shovel-ready"
+ 3. User priority - Existing priority field as tiebreaker
+
+ Score formula: (1000 * unblock) + (100 * depth_score) + (10 * priority_factor)
+
+ Args:
+ features: List of feature dicts with id, priority, dependencies fields
+
+ Returns:
+ Dict mapping feature_id -> score (higher = schedule first)
+ """
+ if not features:
+ return {}
+
+ # Build adjacency lists
+ children: dict[int, list[int]] = {f["id"]: [] for f in features} # who depends on me
+ parents: dict[int, list[int]] = {f["id"]: [] for f in features} # who I depend on
+
+ for f in features:
+ for dep_id in (f.get("dependencies") or []):
+ if dep_id in children: # Only valid deps
+ children[dep_id].append(f["id"])
+ parents[f["id"]].append(dep_id)
+
+ # Calculate depths via BFS from roots
+ # Use visited set to prevent infinite loops from circular dependencies
+ # Use deque for O(1) popleft instead of list.pop(0) which is O(n)
+ depths: dict[int, int] = {}
+ visited: set[int] = set()
+ roots = [f["id"] for f in features if not parents[f["id"]]]
+ bfs_queue: deque[tuple[int, int]] = deque((root, 0) for root in roots)
+ while bfs_queue:
+ node_id, depth = bfs_queue.popleft()
+ if node_id in visited:
+ continue # Skip already visited nodes (handles cycles)
+ visited.add(node_id)
+ depths[node_id] = depth
+ for child_id in children[node_id]:
+ if child_id not in visited:
+ bfs_queue.append((child_id, depth + 1))
+
+ # Handle orphaned nodes (shouldn't happen but be safe)
+ for f in features:
+ if f["id"] not in depths:
+ depths[f["id"]] = 0
+
+ # Calculate transitive downstream counts (reverse topo order)
+ downstream: dict[int, int] = {f["id"]: 0 for f in features}
+ # Process in reverse depth order (leaves first)
+ for fid in sorted(depths.keys(), key=lambda x: -depths[x]):
+ for parent_id in parents[fid]:
+ downstream[parent_id] += 1 + downstream[fid]
+
+ # Normalize and compute scores
+ max_depth = max(depths.values()) if depths else 0
+ max_downstream = max(downstream.values()) if downstream else 0
+
+ scores: dict[int, float] = {}
+ for f in features:
+ fid = f["id"]
+
+ # Unblocking score: 0-1, higher = unblocks more
+ unblock = downstream[fid] / max_downstream if max_downstream > 0 else 0
+
+ # Depth score: 0-1, higher = closer to root (no deps)
+ depth_score = 1 - (depths[fid] / max_depth) if max_depth > 0 else 1
+
+ # Priority factor: 0-1, lower priority number = higher factor
+ priority = f.get("priority", 999)
+ priority_factor = (10 - min(priority, 10)) / 10
+
+ scores[fid] = (1000 * unblock) + (100 * depth_score) + (10 * priority_factor)
+
+ return scores
+
+
+def get_ready_features(features: list[dict], limit: int = 10) -> list[dict]:
+ """Get features that are ready to be worked on.
+
+ A feature is ready if:
+ - It is not passing
+ - It is not in progress
+ - All its dependencies are satisfied
+
+ Args:
+ features: List of all feature dicts
+ limit: Maximum number of features to return
+
+ Returns:
+ List of ready features, sorted by priority
+ """
+ passing_ids = {f["id"] for f in features if f.get("passes")}
+
+ ready = []
+ for f in features:
+ if f.get("passes") or f.get("in_progress"):
+ continue
+ deps = f.get("dependencies") or []
+ if all(dep_id in passing_ids for dep_id in deps):
+ ready.append(f)
+
+ # Sort by scheduling score (higher = first), then priority, then id
+ scores = compute_scheduling_scores(features)
+ ready.sort(key=lambda f: (-scores.get(f["id"], 0), f.get("priority", 999), f["id"]))
+
+ return ready[:limit]
+
+
+def get_blocked_features(features: list[dict]) -> list[dict]:
+ """Get features that are blocked by unmet dependencies.
+
+ Args:
+ features: List of all feature dicts
+
+ Returns:
+ List of blocked features with 'blocked_by' field added
+ """
+ passing_ids = {f["id"] for f in features if f.get("passes")}
+
+ blocked = []
+ for f in features:
+ if f.get("passes"):
+ continue
+ deps = f.get("dependencies") or []
+ blocking = [d for d in deps if d not in passing_ids]
+ if blocking:
+ blocked.append({**f, "blocked_by": blocking})
+
+ return blocked
+
+
+def build_graph_data(features: list[dict]) -> dict:
+ """Build graph data structure for visualization.
+
+ Args:
+ features: List of all feature dicts
+
+ Returns:
+ Dict with 'nodes' and 'edges' for graph visualization
+ """
+ passing_ids = {f["id"] for f in features if f.get("passes")}
+
+ nodes = []
+ edges = []
+
+ for f in features:
+ deps = f.get("dependencies") or []
+ blocking = [d for d in deps if d not in passing_ids]
+
+ if f.get("passes"):
+ status = "done"
+ elif blocking:
+ status = "blocked"
+ elif f.get("in_progress"):
+ status = "in_progress"
+ else:
+ status = "pending"
+
+ nodes.append({
+ "id": f["id"],
+ "name": f["name"],
+ "category": f["category"],
+ "status": status,
+ "priority": f.get("priority", 999),
+ "dependencies": deps,
+ })
+
+ for dep_id in deps:
+ edges.append({"source": dep_id, "target": f["id"]})
+
+ return {"nodes": nodes, "edges": edges}
diff --git a/api/migration.py b/api/migration.py
index 7f9bfb898..930945617 100644
--- a/api/migration.py
+++ b/api/migration.py
@@ -82,6 +82,8 @@ def migrate_json_to_sqlite(
description=feature_dict.get("description", ""),
steps=feature_dict.get("steps", []),
passes=feature_dict.get("passes", False),
+ in_progress=feature_dict.get("in_progress", False),
+ dependencies=feature_dict.get("dependencies"),
)
session.add(feature)
imported_count += 1
diff --git a/auth.py b/auth.py
new file mode 100644
index 000000000..d8150981c
--- /dev/null
+++ b/auth.py
@@ -0,0 +1,93 @@
+"""
+Authentication Error Detection
+==============================
+
+Shared utilities for detecting Claude CLI authentication errors.
+Used by both CLI (start.py) and server (process_manager.py) to provide
+consistent error detection and messaging.
+"""
+
+import re
+
+# Patterns that indicate authentication errors from Claude CLI
+AUTH_ERROR_PATTERNS = [
+ r"not\s+logged\s+in",
+ r"not\s+authenticated",
+ r"authentication\s+(failed|required|error)",
+ r"login\s+required",
+ r"please\s+(run\s+)?['\"]?claude\s+login",
+ r"unauthorized",
+ r"invalid\s+(token|credential|api.?key)",
+ r"expired\s+(token|session|credential)",
+ r"could\s+not\s+authenticate",
+ r"sign\s+in\s+(to|required)",
+]
+
+
+def is_auth_error(text: str) -> bool:
+ """
+ Check if text contains Claude CLI authentication error messages.
+
+ Uses case-insensitive pattern matching against known error messages.
+
+ Args:
+ text: Output text to check
+
+ Returns:
+ True if any auth error pattern matches, False otherwise
+ """
+ if not text:
+ return False
+ text_lower = text.lower()
+ for pattern in AUTH_ERROR_PATTERNS:
+ if re.search(pattern, text_lower):
+ return True
+ return False
+
+
+# CLI-style help message (for terminal output)
+AUTH_ERROR_HELP_CLI = """
+==================================================
+ Authentication Error Detected
+==================================================
+
+Claude CLI requires authentication to work.
+
+Option 1 (Recommended): Set an API key
+ export ANTHROPIC_API_KEY=your-key-here
+ Get a key at: https://console.anthropic.com/
+
+Option 2: Use subscription login
+ claude login
+
+ Note: Anthropic's policy may not permit using
+ subscription auth with third-party agents.
+ API key authentication is recommended.
+==================================================
+"""
+
+# Server-style help message (for WebSocket streaming)
+AUTH_ERROR_HELP_SERVER = """
+================================================================================
+ AUTHENTICATION ERROR DETECTED
+================================================================================
+
+Claude CLI requires authentication to work.
+
+Option 1 (Recommended): Set an API key
+ export ANTHROPIC_API_KEY=your-key-here
+ Get a key at: https://console.anthropic.com/
+
+Option 2: Use subscription login
+ claude login
+
+ Note: Anthropic's policy may not permit using
+ subscription auth with third-party agents.
+ API key authentication is recommended.
+================================================================================
+"""
+
+
+def print_auth_error_help() -> None:
+ """Print helpful message when authentication error is detected (CLI version)."""
+ print(AUTH_ERROR_HELP_CLI)
diff --git a/autoforge_paths.py b/autoforge_paths.py
new file mode 100644
index 000000000..e720dfb47
--- /dev/null
+++ b/autoforge_paths.py
@@ -0,0 +1,326 @@
+"""
+AutoForge Path Resolution
+=========================
+
+Central module for resolving paths to autoforge-generated files within a project.
+
+Implements a tri-path resolution strategy for backward compatibility:
+
+ 1. Check ``project_dir / ".autoforge" / X`` (current layout)
+ 2. Check ``project_dir / ".autocoder" / X`` (legacy layout)
+ 3. Check ``project_dir / X`` (legacy root-level layout)
+ 4. Default to the new location for fresh projects
+
+This allows existing projects with root-level ``features.db``, ``.agent.lock``,
+etc. to keep working while new projects store everything under ``.autoforge/``.
+Projects using the old ``.autocoder/`` directory are auto-migrated on next start.
+
+The ``migrate_project_layout`` function can move an old-layout project to the
+new layout safely, with full integrity checks for SQLite databases.
+"""
+
+import logging
+import shutil
+import sqlite3
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# .gitignore content written into every .autoforge/ directory
+# ---------------------------------------------------------------------------
+_GITIGNORE_CONTENT = """\
+# AutoForge runtime files
+features.db
+features.db-wal
+features.db-shm
+assistant.db
+assistant.db-wal
+assistant.db-shm
+.agent.lock
+.devserver.lock
+.pause_drain
+.claude_settings.json
+.claude_assistant_settings.json
+.claude_settings.expand.*.json
+.progress_cache
+.migration_version
+"""
+
+
+# ---------------------------------------------------------------------------
+# Private helpers
+# ---------------------------------------------------------------------------
+
+def _resolve_path(project_dir: Path, filename: str) -> Path:
+ """Resolve a file path using tri-path strategy.
+
+ Checks the new ``.autoforge/`` location first, then the legacy
+ ``.autocoder/`` location, then the root-level location. If none exist,
+ returns the new location so that newly-created files land in ``.autoforge/``.
+ """
+ new = project_dir / ".autoforge" / filename
+ if new.exists():
+ return new
+ legacy = project_dir / ".autocoder" / filename
+ if legacy.exists():
+ return legacy
+ old = project_dir / filename
+ if old.exists():
+ return old
+ return new # default for new projects
+
+
+def _resolve_dir(project_dir: Path, dirname: str) -> Path:
+ """Resolve a directory path using tri-path strategy.
+
+ Same logic as ``_resolve_path`` but intended for directories such as
+ ``prompts/``.
+ """
+ new = project_dir / ".autoforge" / dirname
+ if new.exists():
+ return new
+ legacy = project_dir / ".autocoder" / dirname
+ if legacy.exists():
+ return legacy
+ old = project_dir / dirname
+ if old.exists():
+ return old
+ return new
+
+
+# ---------------------------------------------------------------------------
+# .autoforge directory management
+# ---------------------------------------------------------------------------
+
+def get_autoforge_dir(project_dir: Path) -> Path:
+ """Return the ``.autoforge`` directory path. Does NOT create it."""
+ return project_dir / ".autoforge"
+
+
+def ensure_autoforge_dir(project_dir: Path) -> Path:
+ """Create the ``.autoforge/`` directory (if needed) and write its ``.gitignore``.
+
+ Returns:
+ The path to the ``.autoforge`` directory.
+ """
+ autoforge_dir = get_autoforge_dir(project_dir)
+ autoforge_dir.mkdir(parents=True, exist_ok=True)
+
+ gitignore_path = autoforge_dir / ".gitignore"
+ gitignore_path.write_text(_GITIGNORE_CONTENT, encoding="utf-8")
+
+ return autoforge_dir
+
+
+# ---------------------------------------------------------------------------
+# Dual-path file helpers
+# ---------------------------------------------------------------------------
+
+def get_features_db_path(project_dir: Path) -> Path:
+ """Resolve the path to ``features.db``."""
+ return _resolve_path(project_dir, "features.db")
+
+
+def get_assistant_db_path(project_dir: Path) -> Path:
+ """Resolve the path to ``assistant.db``."""
+ return _resolve_path(project_dir, "assistant.db")
+
+
+def get_agent_lock_path(project_dir: Path) -> Path:
+ """Resolve the path to ``.agent.lock``."""
+ return _resolve_path(project_dir, ".agent.lock")
+
+
+def get_devserver_lock_path(project_dir: Path) -> Path:
+ """Resolve the path to ``.devserver.lock``."""
+ return _resolve_path(project_dir, ".devserver.lock")
+
+
+def get_claude_settings_path(project_dir: Path) -> Path:
+ """Resolve the path to ``.claude_settings.json``."""
+ return _resolve_path(project_dir, ".claude_settings.json")
+
+
+def get_claude_assistant_settings_path(project_dir: Path) -> Path:
+ """Resolve the path to ``.claude_assistant_settings.json``."""
+ return _resolve_path(project_dir, ".claude_assistant_settings.json")
+
+
+def get_pause_drain_path(project_dir: Path) -> Path:
+ """Return the path to the ``.pause_drain`` signal file.
+
+ This file is created to request a graceful pause (drain mode).
+ Always uses the new location since it's a transient signal file.
+ """
+ return project_dir / ".autoforge" / ".pause_drain"
+
+
+def get_progress_cache_path(project_dir: Path) -> Path:
+ """Resolve the path to ``.progress_cache``."""
+ return _resolve_path(project_dir, ".progress_cache")
+
+
+def get_prompts_dir(project_dir: Path) -> Path:
+ """Resolve the path to the ``prompts/`` directory."""
+ return _resolve_dir(project_dir, "prompts")
+
+
+# ---------------------------------------------------------------------------
+# Non-dual-path helpers (always use new location)
+# ---------------------------------------------------------------------------
+
+def get_expand_settings_path(project_dir: Path, uuid_hex: str) -> Path:
+ """Return the path for an ephemeral expand-session settings file.
+
+ These files are short-lived and always stored in ``.autoforge/``.
+ """
+ return project_dir / ".autoforge" / f".claude_settings.expand.{uuid_hex}.json"
+
+
+# ---------------------------------------------------------------------------
+# Lock-file safety check
+# ---------------------------------------------------------------------------
+
+def has_agent_running(project_dir: Path) -> bool:
+ """Check whether any agent or dev-server lock file exists at either location.
+
+ Inspects the legacy root-level paths, the old ``.autocoder/`` paths, and
+ the new ``.autoforge/`` paths so that a running agent is detected
+ regardless of project layout.
+
+ Returns:
+ ``True`` if any ``.agent.lock`` or ``.devserver.lock`` exists.
+ """
+ lock_names = (".agent.lock", ".devserver.lock")
+ for name in lock_names:
+ if (project_dir / name).exists():
+ return True
+ # Check both old and new directory names for backward compatibility
+ if (project_dir / ".autocoder" / name).exists():
+ return True
+ if (project_dir / ".autoforge" / name).exists():
+ return True
+ return False
+
+
+# ---------------------------------------------------------------------------
+# Migration
+# ---------------------------------------------------------------------------
+
+def migrate_project_layout(project_dir: Path) -> list[str]:
+ """Migrate a project from the legacy root-level layout to ``.autoforge/``.
+
+ The migration is incremental and safe:
+
+ * If the agent is running (lock files present) the migration is skipped
+ entirely to avoid corrupting in-use databases.
+ * Each file/directory is migrated independently. If any single step
+ fails the error is logged and migration continues with the remaining
+ items. Partial migration is safe because the dual-path resolution
+ strategy will find files at whichever location they ended up in.
+
+ Returns:
+ A list of human-readable descriptions of what was migrated, e.g.
+ ``["prompts/ -> .autoforge/prompts/", "features.db -> .autoforge/features.db"]``.
+ An empty list means nothing was migrated (either everything is
+ already migrated, or the agent is running).
+ """
+ # Safety: refuse to migrate while an agent is running
+ if has_agent_running(project_dir):
+ logger.warning("Migration skipped: agent or dev-server is running for %s", project_dir)
+ return []
+
+ # --- 0. Migrate .autocoder/ → .autoforge/ directory -------------------
+ old_autocoder_dir = project_dir / ".autocoder"
+ new_autoforge_dir = project_dir / ".autoforge"
+ if old_autocoder_dir.exists() and old_autocoder_dir.is_dir() and not new_autoforge_dir.exists():
+ try:
+ old_autocoder_dir.rename(new_autoforge_dir)
+ logger.info("Migrated .autocoder/ -> .autoforge/")
+ migrated: list[str] = [".autocoder/ -> .autoforge/"]
+ except Exception:
+ logger.warning("Failed to migrate .autocoder/ -> .autoforge/", exc_info=True)
+ migrated = []
+ else:
+ migrated = []
+
+ autoforge_dir = ensure_autoforge_dir(project_dir)
+
+ # --- 1. Migrate prompts/ directory -----------------------------------
+ try:
+ old_prompts = project_dir / "prompts"
+ new_prompts = autoforge_dir / "prompts"
+ if old_prompts.exists() and old_prompts.is_dir() and not new_prompts.exists():
+ shutil.copytree(str(old_prompts), str(new_prompts))
+ shutil.rmtree(str(old_prompts))
+ migrated.append("prompts/ -> .autoforge/prompts/")
+ logger.info("Migrated prompts/ -> .autoforge/prompts/")
+ except Exception:
+ logger.warning("Failed to migrate prompts/ directory", exc_info=True)
+
+ # --- 2. Migrate SQLite databases (features.db, assistant.db) ---------
+ db_names = ("features.db", "assistant.db")
+ for db_name in db_names:
+ try:
+ old_db = project_dir / db_name
+ new_db = autoforge_dir / db_name
+ if old_db.exists() and not new_db.exists():
+ # Flush WAL to ensure all data is in the main database file
+ conn = sqlite3.connect(str(old_db))
+ try:
+ cursor = conn.cursor()
+ cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+ finally:
+ conn.close()
+
+ # Copy the main database file (WAL is now flushed)
+ shutil.copy2(str(old_db), str(new_db))
+
+ # Verify the copy is intact
+ verify_conn = sqlite3.connect(str(new_db))
+ try:
+ verify_cursor = verify_conn.cursor()
+ result = verify_cursor.execute("PRAGMA integrity_check").fetchone()
+ if result is None or result[0] != "ok":
+ logger.error(
+ "Integrity check failed for migrated %s: %s",
+ db_name, result,
+ )
+ # Remove the broken copy; old file stays in place
+ new_db.unlink(missing_ok=True)
+ continue
+ finally:
+ verify_conn.close()
+
+ # Remove old database files (.db, .db-wal, .db-shm)
+ old_db.unlink(missing_ok=True)
+ for suffix in ("-wal", "-shm"):
+ wal_file = project_dir / f"{db_name}{suffix}"
+ wal_file.unlink(missing_ok=True)
+
+ migrated.append(f"{db_name} -> .autoforge/{db_name}")
+ logger.info("Migrated %s -> .autoforge/%s", db_name, db_name)
+ except Exception:
+ logger.warning("Failed to migrate %s", db_name, exc_info=True)
+
+ # --- 3. Migrate simple files -----------------------------------------
+ simple_files = (
+ ".agent.lock",
+ ".devserver.lock",
+ ".claude_settings.json",
+ ".claude_assistant_settings.json",
+ ".progress_cache",
+ )
+ for filename in simple_files:
+ try:
+ old_file = project_dir / filename
+ new_file = autoforge_dir / filename
+ if old_file.exists() and not new_file.exists():
+ shutil.move(str(old_file), str(new_file))
+ migrated.append(f"{filename} -> .autoforge/{filename}")
+ logger.info("Migrated %s -> .autoforge/%s", filename, filename)
+ except Exception:
+ logger.warning("Failed to migrate %s", filename, exc_info=True)
+
+ return migrated
diff --git a/autonomous_agent_demo.py b/autonomous_agent_demo.py
index f240cc288..787cdf6e2 100644
--- a/autonomous_agent_demo.py
+++ b/autonomous_agent_demo.py
@@ -4,8 +4,10 @@
============================
A minimal harness demonstrating long-running autonomous coding with Claude.
-This script implements the two-agent pattern (initializer + coding agent) and
-incorporates all the strategies from the long-running agents guide.
+This script implements a unified orchestrator pattern that handles:
+- Initialization (creating features from app_spec)
+- Coding agents (implementing features)
+- Testing agents (regression testing)
Example Usage:
# Using absolute path directly
@@ -14,11 +16,22 @@
# Using registered project name (looked up from registry)
python autonomous_agent_demo.py --project-dir my-app
- # Limit iterations for testing
+ # Limit iterations for testing (when running as subprocess)
python autonomous_agent_demo.py --project-dir my-app --max-iterations 5
- # YOLO mode: rapid prototyping without browser testing
+ # YOLO mode: rapid prototyping without testing agents
python autonomous_agent_demo.py --project-dir my-app --yolo
+
+ # Parallel execution with 3 concurrent coding agents
+ python autonomous_agent_demo.py --project-dir my-app --concurrency 3
+
+ # Single agent mode (orchestrator with concurrency=1, the default)
+ python autonomous_agent_demo.py --project-dir my-app
+
+ # Run as specific agent type (used by orchestrator to spawn subprocesses)
+ python autonomous_agent_demo.py --project-dir my-app --agent-type initializer
+ python autonomous_agent_demo.py --project-dir my-app --agent-type coding --feature-id 42
+ python autonomous_agent_demo.py --project-dir my-app --agent-type testing
"""
import argparse
@@ -31,39 +44,40 @@
# IMPORTANT: Must be called BEFORE importing other modules that read env vars at load time
load_dotenv()
-from agent import run_autonomous_agent
-from registry import get_project_path
+import os
-# Configuration
-# DEFAULT_MODEL = "claude-sonnet-4-5-20250929"
-DEFAULT_MODEL = "claude-opus-4-5-20251101"
+from agent import run_autonomous_agent
+from registry import DEFAULT_MODEL, get_effective_sdk_env, get_project_path
def parse_args() -> argparse.Namespace:
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
- description="Autonomous Coding Agent Demo - Long-running agent harness",
+ description="Autonomous Coding Agent Demo - Unified orchestrator pattern",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
- # Use absolute path directly
+ # Use absolute path directly (single agent, default)
python autonomous_agent_demo.py --project-dir C:/Projects/my-app
# Use registered project name (looked up from registry)
python autonomous_agent_demo.py --project-dir my-app
- # Use a specific model
- python autonomous_agent_demo.py --project-dir my-app --model claude-sonnet-4-5-20250929
-
- # Limit iterations for testing
- python autonomous_agent_demo.py --project-dir my-app --max-iterations 5
+ # Parallel execution with 3 concurrent agents
+ python autonomous_agent_demo.py --project-dir my-app --concurrency 3
- # YOLO mode: rapid prototyping without browser testing
+ # YOLO mode: rapid prototyping without testing agents
python autonomous_agent_demo.py --project-dir my-app --yolo
+ # Configure testing agent ratio (2 testing agents per coding agent)
+ python autonomous_agent_demo.py --project-dir my-app --testing-ratio 2
+
+ # Disable testing agents (similar to YOLO but with verification)
+ python autonomous_agent_demo.py --project-dir my-app --testing-ratio 0
+
Authentication:
- Uses Claude CLI credentials from ~/.claude/.credentials.json
- Run 'claude login' to authenticate (handled by start.bat/start.sh)
+ Uses Claude CLI authentication. API key (ANTHROPIC_API_KEY) is recommended.
+ Alternatively run 'claude login', but note Anthropic's policy may restrict subscription auth.
""",
)
@@ -78,7 +92,7 @@ def parse_args() -> argparse.Namespace:
"--max-iterations",
type=int,
default=None,
- help="Maximum number of agent iterations (default: unlimited)",
+ help="Maximum number of agent iterations (default: unlimited, typically 1 for subprocesses)",
)
parser.add_argument(
@@ -92,7 +106,95 @@ def parse_args() -> argparse.Namespace:
"--yolo",
action="store_true",
default=False,
- help="Enable YOLO mode: rapid prototyping without browser testing",
+ help="Enable YOLO mode: skip testing agents for rapid prototyping",
+ )
+
+ # Unified orchestrator mode (replaces --parallel)
+ parser.add_argument(
+ "--concurrency", "-c",
+ type=int,
+ default=1,
+ help="Number of concurrent coding agents (default: 1, max: 5)",
+ )
+
+ # Backward compatibility: --parallel is deprecated alias for --concurrency
+ parser.add_argument(
+ "--parallel", "-p",
+ type=int,
+ nargs="?",
+ const=3,
+ default=None,
+ metavar="N",
+ help="DEPRECATED: Use --concurrency instead. Alias for --concurrency.",
+ )
+
+ parser.add_argument(
+ "--feature-id",
+ type=int,
+ default=None,
+ help="Work on a specific feature ID only (used by orchestrator for coding agents)",
+ )
+
+ parser.add_argument(
+ "--feature-ids",
+ type=str,
+ default=None,
+ help="Comma-separated feature IDs to implement in batch (e.g., '5,8,12')",
+ )
+
+ # Agent type for subprocess mode
+ parser.add_argument(
+ "--agent-type",
+ choices=["initializer", "coding", "testing"],
+ default=None,
+ help="Agent type (used by orchestrator to spawn specialized subprocesses)",
+ )
+
+ parser.add_argument(
+ "--testing-feature-id",
+ type=int,
+ default=None,
+ help="Feature ID to regression test (used by orchestrator for testing agents, legacy single mode)",
+ )
+
+ parser.add_argument(
+ "--testing-feature-ids",
+ type=str,
+ default=None,
+ help="Comma-separated feature IDs to regression test in batch (e.g., '5,12,18')",
+ )
+
+ # Testing agent configuration
+ parser.add_argument(
+ "--testing-ratio",
+ type=int,
+ default=1,
+ help="Testing agents per coding agent (0-3, default: 1). Set to 0 to disable testing agents.",
+ )
+
+ parser.add_argument(
+ "--testing-batch-size",
+ type=int,
+ default=3,
+ help="Number of features per testing batch (1-15, default: 3)",
+ )
+
+ parser.add_argument(
+ "--batch-size",
+ type=int,
+ default=3,
+ help="Max features per coding agent batch (1-15, default: 3)",
+ )
+
+ parser.add_argument(
+ "--auto-improve",
+ action="store_true",
+ default=False,
+ help=(
+ "Run in auto-improve mode: a single agent session that analyses "
+ "the codebase, creates one improvement feature, implements it, "
+ "verifies with lint/typecheck/build, commits, and exits."
+ ),
)
return parser.parse_args()
@@ -100,11 +202,25 @@ def parse_args() -> argparse.Namespace:
def main() -> None:
"""Main entry point."""
+ print("[ENTRY] autonomous_agent_demo.py starting...", flush=True)
args = parse_args()
# Note: Authentication is handled by start.bat/start.sh before this script runs.
# The Claude SDK auto-detects credentials from ~/.claude/.credentials.json
+ # Apply UI-configured provider settings to this process's environment.
+ # This ensures CLI-launched agents respect Settings UI provider config (GLM, Ollama, etc.).
+ # Uses setdefault so explicit env vars / .env file take precedence.
+ sdk_overrides = get_effective_sdk_env()
+ for key, value in sdk_overrides.items():
+ if value: # Only set non-empty values (empty values are used to clear conflicts)
+ os.environ.setdefault(key, value)
+
+ # Handle deprecated --parallel flag
+ if args.parallel is not None:
+ print("WARNING: --parallel is deprecated. Use --concurrency instead.", flush=True)
+ args.concurrency = args.parallel
+
# Resolve project directory:
# 1. If absolute path, use as-is
# 2. Otherwise, look up from registry by name
@@ -126,16 +242,98 @@ def main() -> None:
print("Use an absolute path or register the project first.")
return
+ # Migrate project layout to .autoforge/ if needed (idempotent, safe)
+ from autoforge_paths import migrate_project_layout
+ migrated = migrate_project_layout(project_dir)
+ if migrated:
+ print(f"Migrated project files to .autoforge/: {', '.join(migrated)}", flush=True)
+
+ # Migrate project to current AutoForge version (idempotent, safe)
+ from prompts import migrate_project_to_current
+ version_migrated = migrate_project_to_current(project_dir)
+ if version_migrated:
+ print(f"Upgraded project: {', '.join(version_migrated)}", flush=True)
+
+ # Parse batch testing feature IDs (comma-separated string -> list[int])
+ testing_feature_ids: list[int] | None = None
+ if args.testing_feature_ids:
+ try:
+ testing_feature_ids = [int(x.strip()) for x in args.testing_feature_ids.split(",") if x.strip()]
+ except ValueError:
+ print(f"Error: --testing-feature-ids must be comma-separated integers, got: {args.testing_feature_ids}")
+ return
+
+ # Parse batch coding feature IDs (comma-separated string -> list[int])
+ coding_feature_ids: list[int] | None = None
+ if args.feature_ids:
+ try:
+ coding_feature_ids = [int(x.strip()) for x in args.feature_ids.split(",") if x.strip()]
+ except ValueError:
+ print(f"Error: --feature-ids must be comma-separated integers, got: {args.feature_ids}")
+ return
+
try:
- # Run the agent (MCP server handles feature database)
- asyncio.run(
- run_autonomous_agent(
- project_dir=project_dir,
- model=args.model,
- max_iterations=args.max_iterations,
- yolo_mode=args.yolo,
+ if args.auto_improve:
+ # Auto-improve mode: single agent session, one improvement per run.
+ # Bypasses the parallel orchestrator entirely — auto-improve is
+ # always single-agent, single-feature, and exits after one commit.
+ print("[AUTO-IMPROVE] Starting single-session improvement run...", flush=True)
+ asyncio.run(
+ run_autonomous_agent(
+ project_dir=project_dir,
+ model=args.model,
+ max_iterations=1,
+ yolo_mode=args.yolo,
+ agent_type="coding",
+ auto_improve=True,
+ )
+ )
+ elif args.agent_type:
+ # Subprocess mode - spawned by orchestrator for a specific role
+ asyncio.run(
+ run_autonomous_agent(
+ project_dir=project_dir,
+ model=args.model,
+ max_iterations=args.max_iterations or 1,
+ yolo_mode=args.yolo,
+ feature_id=args.feature_id,
+ feature_ids=coding_feature_ids,
+ agent_type=args.agent_type,
+ testing_feature_id=args.testing_feature_id,
+ testing_feature_ids=testing_feature_ids,
+ )
+ )
+ else:
+ # Entry point mode - always use unified orchestrator
+ # Clean up stale temp files before starting (prevents temp folder bloat)
+ from temp_cleanup import cleanup_stale_temp
+ cleanup_stats = cleanup_stale_temp()
+ if cleanup_stats["dirs_deleted"] > 0 or cleanup_stats["files_deleted"] > 0:
+ mb_freed = cleanup_stats["bytes_freed"] / (1024 * 1024)
+ print(
+ f"[CLEANUP] Removed {cleanup_stats['dirs_deleted']} dirs, "
+ f"{cleanup_stats['files_deleted']} files ({mb_freed:.1f} MB freed)",
+ flush=True,
+ )
+
+ from parallel_orchestrator import run_parallel_orchestrator
+
+ # Clamp concurrency to valid range (1-5)
+ concurrency = max(1, min(args.concurrency, 5))
+ if concurrency != args.concurrency:
+ print(f"Clamping concurrency to valid range: {concurrency}", flush=True)
+
+ asyncio.run(
+ run_parallel_orchestrator(
+ project_dir=project_dir,
+ max_concurrency=concurrency,
+ model=args.model,
+ yolo_mode=args.yolo,
+ testing_agent_ratio=args.testing_ratio,
+ testing_batch_size=args.testing_batch_size,
+ batch_size=args.batch_size,
+ )
)
- )
except KeyboardInterrupt:
print("\n\nInterrupted by user")
print("To resume, run the same command again")
diff --git a/bin/autoforge.js b/bin/autoforge.js
new file mode 100755
index 000000000..8a03b1600
--- /dev/null
+++ b/bin/autoforge.js
@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { run } from '../lib/cli.js';
+run(process.argv.slice(2));
diff --git a/client.py b/client.py
index 0f68e5ef6..26a710f50 100644
--- a/client.py
+++ b/client.py
@@ -7,60 +7,192 @@
import json
import os
+import re
import shutil
import sys
from pathlib import Path
from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient
-from claude_agent_sdk.types import HookMatcher
+from claude_agent_sdk.types import HookContext, HookInput, HookMatcher, SyncHookJSONOutput
+from dotenv import load_dotenv
-from security import bash_security_hook
+from security import SENSITIVE_DIRECTORIES, bash_security_hook
-# Feature MCP tools for feature/test management
-FEATURE_MCP_TOOLS = [
+# Load environment variables from .env file if present
+load_dotenv()
+
+# Extra read paths for cross-project file access (read-only)
+# Set EXTRA_READ_PATHS environment variable with comma-separated absolute paths
+# Example: EXTRA_READ_PATHS=/Volumes/Data/dev,/Users/shared/libs
+EXTRA_READ_PATHS_VAR = "EXTRA_READ_PATHS"
+
+# Sensitive directories that should never be allowed via EXTRA_READ_PATHS.
+# Delegates to the canonical SENSITIVE_DIRECTORIES set in security.py so that
+# this blocklist and the filesystem browser API share a single source of truth.
+EXTRA_READ_PATHS_BLOCKLIST = SENSITIVE_DIRECTORIES
+
+
+def convert_model_for_vertex(model: str) -> str:
+ """
+ Convert model name format for Vertex AI compatibility.
+
+ Vertex AI uses @ to separate model name from version (e.g., claude-sonnet-4-5@20250929)
+ while the Anthropic API uses - (e.g., claude-sonnet-4-5-20250929).
+ Models without a date suffix (e.g., claude-opus-4-7) pass through unchanged.
+
+ Args:
+ model: Model name in Anthropic format (with hyphens)
+
+ Returns:
+ Model name in Vertex AI format (with @ before date) if Vertex AI is enabled,
+ otherwise returns the model unchanged.
+ """
+ # Only convert if Vertex AI is enabled
+ if os.getenv("CLAUDE_CODE_USE_VERTEX") != "1":
+ return model
+
+ # Pattern: claude-{name}-{version}-{date} -> claude-{name}-{version}@{date}
+ # Example: claude-sonnet-4-5-20250929 -> claude-sonnet-4-5@20250929
+ # The date is always 8 digits at the end
+ match = re.match(r'^(claude-.+)-(\d{8})$', model)
+ if match:
+ base_name, date = match.groups()
+ return f"{base_name}@{date}"
+
+ # If already in @ format or doesn't match expected pattern, return as-is
+ return model
+
+
+def get_extra_read_paths() -> list[Path]:
+ """
+ Get extra read-only paths from EXTRA_READ_PATHS environment variable.
+
+ Parses comma-separated absolute paths and validates each one:
+ - Must be an absolute path
+ - Must exist and be a directory
+ - Cannot be or contain sensitive directories (e.g., .ssh, .aws)
+
+ Returns:
+ List of validated, canonicalized Path objects.
+ """
+ raw_value = os.getenv(EXTRA_READ_PATHS_VAR, "").strip()
+ if not raw_value:
+ return []
+
+ validated_paths: list[Path] = []
+ home_dir = Path.home()
+
+ for path_str in raw_value.split(","):
+ path_str = path_str.strip()
+ if not path_str:
+ continue
+
+ # Parse and canonicalize the path
+ try:
+ path = Path(path_str).resolve()
+ except (OSError, ValueError) as e:
+ print(f" - Warning: Invalid EXTRA_READ_PATHS path '{path_str}': {e}")
+ continue
+
+ # Must be absolute (resolve() makes it absolute, but check original input)
+ if not Path(path_str).is_absolute():
+ print(f" - Warning: EXTRA_READ_PATHS requires absolute paths, skipping: {path_str}")
+ continue
+
+ # Must exist
+ if not path.exists():
+ print(f" - Warning: EXTRA_READ_PATHS path does not exist, skipping: {path_str}")
+ continue
+
+ # Must be a directory
+ if not path.is_dir():
+ print(f" - Warning: EXTRA_READ_PATHS path is not a directory, skipping: {path_str}")
+ continue
+
+ # Check against sensitive directory blocklist
+ is_blocked = False
+ for sensitive in EXTRA_READ_PATHS_BLOCKLIST:
+ sensitive_path = (home_dir / sensitive).resolve()
+ try:
+ # Block if path IS the sensitive dir or is INSIDE it
+ if path == sensitive_path or path.is_relative_to(sensitive_path):
+ print(f" - Warning: EXTRA_READ_PATHS blocked sensitive path: {path_str}")
+ is_blocked = True
+ break
+ # Also block if sensitive dir is INSIDE the requested path
+ if sensitive_path.is_relative_to(path):
+ print(f" - Warning: EXTRA_READ_PATHS path contains sensitive directory ({sensitive}): {path_str}")
+ is_blocked = True
+ break
+ except (OSError, ValueError):
+ # is_relative_to can raise on some edge cases
+ continue
+
+ if is_blocked:
+ continue
+
+ validated_paths.append(path)
+
+ return validated_paths
+
+
+# Per-agent-type MCP tool lists.
+# Only expose the tools each agent type actually needs, reducing tool schema
+# overhead and preventing agents from calling tools meant for other roles.
+#
+# Tools intentionally omitted from ALL agent lists (UI/orchestrator only):
+# feature_remove_dependency
+#
+# The ghost tool "feature_release_testing" was removed entirely -- it was
+# listed here but never implemented in mcp_server/feature_mcp.py.
+
+CODING_AGENT_TOOLS = [
"mcp__features__feature_get_stats",
- "mcp__features__feature_get_next",
- "mcp__features__feature_get_for_regression",
+ "mcp__features__feature_get_by_id",
+ "mcp__features__feature_get_summary",
+ "mcp__features__feature_get_ready",
+ "mcp__features__feature_get_blocked",
+ "mcp__features__feature_get_graph",
+ "mcp__features__feature_claim_and_get",
"mcp__features__feature_mark_in_progress",
"mcp__features__feature_mark_passing",
+ "mcp__features__feature_mark_failing",
"mcp__features__feature_skip",
- "mcp__features__feature_create_bulk",
+ "mcp__features__feature_clear_in_progress",
]
-# Playwright MCP tools for browser automation
-PLAYWRIGHT_TOOLS = [
- # Core navigation & screenshots
- "mcp__playwright__browser_navigate",
- "mcp__playwright__browser_navigate_back",
- "mcp__playwright__browser_take_screenshot",
- "mcp__playwright__browser_snapshot",
-
- # Element interaction
- "mcp__playwright__browser_click",
- "mcp__playwright__browser_type",
- "mcp__playwright__browser_fill_form",
- "mcp__playwright__browser_select_option",
- "mcp__playwright__browser_hover",
- "mcp__playwright__browser_drag",
- "mcp__playwright__browser_press_key",
-
- # JavaScript & debugging
- "mcp__playwright__browser_evaluate",
- # "mcp__playwright__browser_run_code", # REMOVED - causes Playwright MCP server crash
- "mcp__playwright__browser_console_messages",
- "mcp__playwright__browser_network_requests",
-
- # Browser management
- "mcp__playwright__browser_close",
- "mcp__playwright__browser_resize",
- "mcp__playwright__browser_tabs",
- "mcp__playwright__browser_wait_for",
- "mcp__playwright__browser_handle_dialog",
- "mcp__playwright__browser_file_upload",
- "mcp__playwright__browser_install",
+TESTING_AGENT_TOOLS = [
+ "mcp__features__feature_get_stats",
+ "mcp__features__feature_get_by_id",
+ "mcp__features__feature_get_summary",
+ "mcp__features__feature_get_ready",
+ "mcp__features__feature_get_blocked",
+ "mcp__features__feature_get_graph",
+ "mcp__features__feature_mark_passing",
+ "mcp__features__feature_mark_failing",
]
-# Built-in tools
+INITIALIZER_AGENT_TOOLS = [
+ "mcp__features__feature_get_stats",
+ "mcp__features__feature_get_ready",
+ "mcp__features__feature_get_blocked",
+ "mcp__features__feature_get_graph",
+ "mcp__features__feature_create_bulk",
+ "mcp__features__feature_create",
+ "mcp__features__feature_add_dependency",
+ "mcp__features__feature_set_dependencies",
+]
+
+# Union of all agent tool lists -- used for permissions (all tools remain
+# *permitted* so the MCP server can respond, but only the agent-type-specific
+# list is included in allowed_tools, which controls what the LLM sees).
+ALL_FEATURE_MCP_TOOLS = sorted(
+ set(CODING_AGENT_TOOLS) | set(TESTING_AGENT_TOOLS) | set(INITIALIZER_AGENT_TOOLS)
+)
+
+# Built-in tools available to agents.
+# WebFetch and WebSearch are included so coding agents can look up current
+# documentation for frameworks and libraries they are implementing.
BUILTIN_TOOLS = [
"Read",
"Write",
@@ -73,14 +205,21 @@
]
-def create_client(project_dir: Path, model: str, yolo_mode: bool = False):
+def create_client(
+ project_dir: Path,
+ model: str,
+ yolo_mode: bool = False,
+ agent_type: str = "coding",
+):
"""
Create a Claude Agent SDK client with multi-layered security.
Args:
project_dir: Directory for the project
model: Claude model to use
- yolo_mode: If True, skip Playwright MCP server for rapid prototyping
+ yolo_mode: If True, skip browser testing for rapid prototyping
+ agent_type: One of "coding", "testing", or "initializer". Controls which
+ MCP tools are exposed and the max_turns limit.
Returns:
Configured ClaudeSDKClient (from claude_agent_sdk)
@@ -92,15 +231,33 @@ def create_client(project_dir: Path, model: str, yolo_mode: bool = False):
(see security.py for ALLOWED_COMMANDS)
Note: Authentication is handled by start.bat/start.sh before this runs.
- The Claude SDK auto-detects credentials from ~/.claude/.credentials.json
+ The Claude SDK auto-detects credentials from the Claude CLI configuration
"""
- # Build allowed tools list based on mode
- # In YOLO mode, exclude Playwright tools for faster prototyping
- allowed_tools = [*BUILTIN_TOOLS, *FEATURE_MCP_TOOLS]
- if not yolo_mode:
- allowed_tools.extend(PLAYWRIGHT_TOOLS)
+ # Select the feature MCP tools appropriate for this agent type
+ feature_tools_map = {
+ "coding": CODING_AGENT_TOOLS,
+ "testing": TESTING_AGENT_TOOLS,
+ "initializer": INITIALIZER_AGENT_TOOLS,
+ }
+ feature_tools = feature_tools_map.get(agent_type, CODING_AGENT_TOOLS)
+
+ # Select max_turns based on agent type:
+ # - coding/initializer: 300 turns (complex multi-step implementation)
+ # - testing: 100 turns (focused verification of a single feature)
+ max_turns_map = {
+ "coding": 300,
+ "testing": 100,
+ "initializer": 300,
+ }
+ max_turns = max_turns_map.get(agent_type, 300)
- # Build permissions list
+ # Build allowed tools list based on agent type.
+ allowed_tools = [*BUILTIN_TOOLS, *feature_tools]
+
+ # Build permissions list.
+ # We permit ALL feature MCP tools at the security layer (so the MCP server
+ # can respond if called), but the LLM only *sees* the agent-type-specific
+ # subset via allowed_tools above.
permissions_list = [
# Allow all file operations within the project directory
"Read(./**)",
@@ -111,15 +268,21 @@ def create_client(project_dir: Path, model: str, yolo_mode: bool = False):
# Bash permission granted here, but actual commands are validated
# by the bash_security_hook (see security.py for allowed commands)
"Bash(*)",
- # Allow web tools for documentation lookup
- "WebFetch",
- "WebSearch",
+ # Allow web tools for looking up framework/library documentation
+ "WebFetch(*)",
+ "WebSearch(*)",
# Allow Feature MCP tools for feature management
- *FEATURE_MCP_TOOLS,
+ *ALL_FEATURE_MCP_TOOLS,
]
- if not yolo_mode:
- # Allow Playwright MCP tools for browser automation (standard mode only)
- permissions_list.extend(PLAYWRIGHT_TOOLS)
+
+ # Add extra read paths from environment variable (read-only access)
+ # Paths are validated, canonicalized, and checked against sensitive blocklist
+ extra_read_paths = get_extra_read_paths()
+ for path in extra_read_paths:
+ # Add read-only permissions for each validated path
+ permissions_list.append(f"Read({path}/**)")
+ permissions_list.append(f"Glob({path}/**)")
+ permissions_list.append(f"Grep({path}/**)")
# Create comprehensive security settings
# Note: Using relative paths ("./**") restricts access to project directory
@@ -136,18 +299,22 @@ def create_client(project_dir: Path, model: str, yolo_mode: bool = False):
project_dir.mkdir(parents=True, exist_ok=True)
# Write settings to a file in the project directory
- settings_file = project_dir / ".claude_settings.json"
+ from autoforge_paths import get_claude_settings_path
+ settings_file = get_claude_settings_path(project_dir)
+ settings_file.parent.mkdir(parents=True, exist_ok=True)
with open(settings_file, "w") as f:
json.dump(security_settings, f, indent=2)
print(f"Created security settings at {settings_file}")
print(" - Sandbox enabled (OS-level bash isolation)")
print(f" - Filesystem restricted to: {project_dir.resolve()}")
+ if extra_read_paths:
+ print(f" - Extra read paths (validated): {', '.join(str(p) for p in extra_read_paths)}")
print(" - Bash commands restricted to allowlist (see security.py)")
if yolo_mode:
- print(" - MCP servers: features (database) - YOLO MODE (no Playwright)")
+ print(" - MCP servers: features (database) - YOLO MODE (no browser testing)")
else:
- print(" - MCP servers: playwright (browser), features (database)")
+ print(" - MCP servers: features (database)")
print(" - Project settings enabled (skills, commands, CLAUDE.md)")
print()
@@ -156,7 +323,7 @@ def create_client(project_dir: Path, model: str, yolo_mode: bool = False):
if system_cli:
print(f" - Using system CLI: {system_cli}")
else:
- print(" - Warning: System Claude CLI not found, using bundled CLI")
+ print(" - Warning: System 'claude' CLI not found, using bundled CLI")
# Build MCP servers config - features is always included, playwright only in standard mode
mcp_servers = {
@@ -164,37 +331,172 @@ def create_client(project_dir: Path, model: str, yolo_mode: bool = False):
"command": sys.executable, # Use the same Python that's running this script
"args": ["-m", "mcp_server.feature_mcp"],
"env": {
- # Inherit parent environment (PATH, ANTHROPIC_API_KEY, etc.)
- **os.environ,
- # Add custom variables
+ # Only specify variables the MCP server needs
+ # (subprocess inherits parent environment automatically)
"PROJECT_DIR": str(project_dir.resolve()),
"PYTHONPATH": str(Path(__file__).parent.resolve()),
},
},
}
- if not yolo_mode:
- # Include Playwright MCP server for browser automation (standard mode only)
- mcp_servers["playwright"] = {
- "command": "npx",
- "args": ["@playwright/mcp@latest", "--viewport-size", "1280x720"],
- }
+ # Build environment overrides for API endpoint configuration
+ # Uses get_effective_sdk_env() which reads provider settings from the database,
+ # ensuring UI-configured alternative providers (GLM, Ollama, Kimi, Custom) propagate
+ # correctly to the Claude CLI subprocess
+ from registry import get_effective_sdk_env, get_effort_setting
+ sdk_env = get_effective_sdk_env()
+ effort = get_effort_setting()
+ print(f" - Reasoning effort: {effort}")
+
+ # Detect alternative API mode (Ollama, GLM, or Vertex AI)
+ base_url = sdk_env.get("ANTHROPIC_BASE_URL", "")
+ is_vertex = sdk_env.get("CLAUDE_CODE_USE_VERTEX") == "1"
+ is_alternative_api = bool(base_url) or is_vertex
+ is_ollama = "localhost:11434" in base_url or "127.0.0.1:11434" in base_url
+ is_azure = "services.ai.azure.com" in base_url
+ model = convert_model_for_vertex(model)
+ if sdk_env:
+ print(f" - API overrides: {', '.join(sdk_env.keys())}")
+ if is_vertex:
+ project_id = sdk_env.get("ANTHROPIC_VERTEX_PROJECT_ID", "unknown")
+ region = sdk_env.get("CLOUD_ML_REGION", "unknown")
+ print(f" - Vertex AI Mode: Using GCP project '{project_id}' with model '{model}' in region '{region}'")
+ elif is_ollama:
+ print(" - Ollama Mode: Using local models")
+ elif is_azure:
+ print(f" - Azure Mode: Using {base_url}")
+ elif "ANTHROPIC_BASE_URL" in sdk_env:
+ print(f" - Alternative API: Using {sdk_env['ANTHROPIC_BASE_URL']}")
+
+ # Create a wrapper for bash_security_hook that passes project_dir via context
+ async def bash_hook_with_context(input_data, tool_use_id=None, context=None):
+ """Wrapper that injects project_dir into context for security hook."""
+ if context is None:
+ context = {}
+ context["project_dir"] = str(project_dir.resolve())
+ return await bash_security_hook(input_data, tool_use_id, context)
+
+ # PreCompact hook for logging and customizing context compaction.
+ # Compaction is handled automatically by Claude Code CLI when context approaches limits.
+ # This hook provides custom instructions that guide the summarizer to preserve
+ # critical workflow state while discarding verbose/redundant content.
+ async def pre_compact_hook(
+ input_data: HookInput,
+ tool_use_id: str | None,
+ context: HookContext,
+ ) -> SyncHookJSONOutput:
+ """
+ Hook called before context compaction occurs.
+
+ Compaction triggers:
+ - "auto": Automatic compaction when context approaches token limits
+ - "manual": User-initiated compaction via /compact command
+
+ Returns custom instructions that guide the compaction summarizer to:
+ 1. Preserve critical workflow state (feature ID, modified files, test results)
+ 2. Discard verbose content (screenshots, long grep outputs, repeated reads)
+ """
+ trigger = input_data.get("trigger", "auto")
+ custom_instructions = input_data.get("custom_instructions")
+ if trigger == "auto":
+ print("[Context] Auto-compaction triggered (context approaching limit)")
+ else:
+ print("[Context] Manual compaction requested")
+
+ if custom_instructions:
+ print(f"[Context] Custom instructions provided: {custom_instructions}")
+
+ # Build compaction instructions that preserve workflow-critical context
+ # while discarding verbose content that inflates token usage.
+ #
+ # The summarizer receives these instructions and uses them to decide
+ # what to keep vs. discard during context compaction.
+ compaction_guidance = "\n".join([
+ "## PRESERVE (critical workflow state)",
+ "- Current feature ID, feature name, and feature status (pending/in_progress/passing/failing)",
+ "- List of all files created or modified during this session, with their paths",
+ "- Last test/lint/type-check results: command run, pass/fail status, and key error messages",
+ "- Current step in the workflow (e.g., implementing, testing, fixing lint errors)",
+ "- Any dependency information (which features block this one)",
+ "- Git operations performed (commits, branches created)",
+ "- MCP tool call results (feature_claim_and_get, feature_mark_passing, etc.)",
+ "- Key architectural decisions made during this session",
+ "",
+ "## DISCARD (verbose content safe to drop)",
+ "- Full screenshot base64 data (just note that a screenshot was taken and what it showed)",
+ "- Long grep/find/glob output listings (summarize to: searched for X, found Y relevant files)",
+ "- Repeated file reads of the same file (keep only the latest read or a summary of changes)",
+ "- Full file contents from Read tool (summarize to: read file X, key sections were Y)",
+ "- Verbose npm/pip install output (just note: dependencies installed successfully/failed)",
+ "- Full lint/type-check output when passing (just note: lint passed with no errors)",
+ "- Browser console message dumps (summarize to: N errors found, key error was X)",
+ "- Redundant tool result confirmations ([Done] markers)",
+ ])
+
+ print("[Context] Applying custom compaction instructions (preserve workflow state, discard verbose content)")
+
+ # The SDK's HookSpecificOutput union type does not yet include a
+ # PreCompactHookSpecificOutput variant, but the CLI protocol accepts
+ # {"hookEventName": "PreCompact", "customInstructions": "..."}.
+ # The dict is serialized to JSON and sent to the CLI process directly,
+ # so the runtime behavior is correct despite the type mismatch.
+ return SyncHookJSONOutput(
+ hookSpecificOutput={ # type: ignore[typeddict-item]
+ "hookEventName": "PreCompact",
+ "customInstructions": compaction_guidance,
+ }
+ )
+
+ # PROMPT CACHING: The Claude Code CLI applies cache_control breakpoints internally.
+ # Our system_prompt benefits from automatic caching without explicit configuration.
+ # If explicit cache_control is needed, the SDK would need to accept content blocks
+ # with cache_control fields (not currently supported in v0.1.x).
return ClaudeSDKClient(
options=ClaudeAgentOptions(
model=model,
+ # SDK 0.1.61's effort Literal omits "xhigh" but the CLI's
+ # --effort flag accepts it; the SDK forwards the string unchanged.
+ effort=effort, # type: ignore[arg-type]
cli_path=system_cli, # Use system CLI to avoid bundled Bun crash (exit code 3)
system_prompt="You are an expert full-stack developer building a production-quality web application.",
setting_sources=["project"], # Enable skills, commands, and CLAUDE.md from project dir
max_buffer_size=10 * 1024 * 1024, # 10MB for large Playwright screenshots
allowed_tools=allowed_tools,
- mcp_servers=mcp_servers,
+ mcp_servers=mcp_servers, # type: ignore[arg-type] # SDK accepts dict config at runtime
hooks={
"PreToolUse": [
- HookMatcher(matcher="Bash", hooks=[bash_security_hook]),
+ HookMatcher(matcher="Bash", hooks=[bash_hook_with_context]),
+ ],
+ # PreCompact hook for context management during long sessions.
+ # Compaction is automatic when context approaches token limits.
+ # This hook logs compaction events and can customize summarization.
+ "PreCompact": [
+ HookMatcher(hooks=[pre_compact_hook]),
],
},
- max_turns=1000,
+ max_turns=max_turns,
cwd=str(project_dir.resolve()),
settings=str(settings_file.resolve()), # Use absolute path
+ env=sdk_env, # Pass API configuration overrides to CLI subprocess
+ # Enable extended context beta for better handling of long sessions.
+ # This provides up to 1M tokens of context with automatic compaction.
+ # See: https://docs.anthropic.com/en/api/beta-headers
+ # Disabled for alternative APIs (Ollama, GLM, Vertex AI) as they don't support this beta.
+ betas=[] if is_alternative_api else ["context-1m-2025-08-07"],
+ # Note on context management:
+ # The Claude Agent SDK handles context management automatically through the
+ # underlying Claude Code CLI. When context approaches limits, the CLI
+ # automatically compacts/summarizes previous messages.
+ #
+ # The SDK does NOT expose explicit compaction_control or context_management
+ # parameters. Instead, context is managed via:
+ # 1. betas=["context-1m-2025-08-07"] - Extended context window
+ # 2. PreCompact hook - Intercept and customize compaction behavior
+ # 3. max_turns - Limit conversation turns (per agent type: coding=300, testing=100)
+ #
+ # Future SDK versions may add explicit compaction controls. When available,
+ # consider adding:
+ # - compaction_control={"enabled": True, "context_token_threshold": 80000}
+ # - context_management={"edits": [...]} for tool use clearing
)
)
diff --git a/env_constants.py b/env_constants.py
new file mode 100644
index 000000000..45737c415
--- /dev/null
+++ b/env_constants.py
@@ -0,0 +1,28 @@
+"""
+Shared Environment Variable Constants
+======================================
+
+Single source of truth for environment variables forwarded to Claude CLI
+subprocesses. Imported by both ``client.py`` (agent sessions) and
+``server/services/chat_constants.py`` (chat sessions) to avoid maintaining
+duplicate lists.
+
+These allow autoforge to use alternative API endpoints (Ollama, GLM,
+Vertex AI) without affecting the user's global Claude Code settings.
+"""
+
+API_ENV_VARS: list[str] = [
+ # Core API configuration
+ "ANTHROPIC_BASE_URL", # Custom API endpoint (e.g., https://api.z.ai/api/anthropic)
+ "ANTHROPIC_AUTH_TOKEN", # API authentication token
+ "ANTHROPIC_API_KEY", # API key (used by Kimi and other providers)
+ "API_TIMEOUT_MS", # Request timeout in milliseconds
+ # Model tier overrides
+ "ANTHROPIC_DEFAULT_SONNET_MODEL", # Model override for Sonnet
+ "ANTHROPIC_DEFAULT_OPUS_MODEL", # Model override for Opus
+ "ANTHROPIC_DEFAULT_HAIKU_MODEL", # Model override for Haiku
+ # Vertex AI configuration
+ "CLAUDE_CODE_USE_VERTEX", # Enable Vertex AI mode (set to "1")
+ "CLOUD_ML_REGION", # GCP region (e.g., us-east5)
+ "ANTHROPIC_VERTEX_PROJECT_ID", # GCP project ID
+]
diff --git a/examples/OPTIMIZE_CONFIG.md b/examples/OPTIMIZE_CONFIG.md
new file mode 100644
index 000000000..57236bdf4
--- /dev/null
+++ b/examples/OPTIMIZE_CONFIG.md
@@ -0,0 +1,230 @@
+# How to Optimize Your allowed_commands.yaml
+
+## The Problem
+
+Your config might have redundant commands like this:
+
+```yaml
+commands:
+ - name: flutter
+ - name: flutter* # ← This already covers EVERYTHING below!
+ - name: flutter test # ← Redundant
+ - name: flutter test --coverage # ← Redundant
+ - name: flutter build apk # ← Redundant
+ - name: flutter build ios # ← Redundant
+ # ... 20+ more flutter commands
+```
+
+**Result:** 65 commands when you only need ~10-15
+
+## How Wildcards Work
+
+When you have `flutter*`, it matches:
+- ✅ `flutter` (the base command)
+- ✅ `flutter test`
+- ✅ `flutter test --coverage`
+- ✅ `flutter build apk`
+- ✅ `flutter build ios`
+- ✅ `flutter run`
+- ✅ **ANY command starting with "flutter"**
+
+**You don't need to list every subcommand separately!**
+
+## Example: GOD-APP Optimization
+
+### Before (65 commands)
+```yaml
+commands:
+ # Flutter
+ - name: flutter
+ - name: flutter*
+ - name: flutter test
+ - name: flutter test --coverage
+ - name: flutter test --exclude-tags=golden,integration
+ - name: flutter test --tags=golden
+ - name: flutter test --tags=golden --update-goldens
+ - name: flutter test --verbose
+ - name: flutter drive
+ - name: flutter test integration_test/
+ - name: flutter build apk
+ - name: flutter build apk --debug
+ - name: flutter build apk --release
+ - name: flutter build appbundle
+ - name: flutter build ios
+ - name: flutter build ios --debug
+ - name: flutter build ipa
+ - name: flutter build web
+ - name: flutter pub get
+ - name: flutter pub upgrade
+ - name: flutter doctor
+ - name: flutter clean
+ # ... and more
+
+ # Dart
+ - name: dart
+ - name: dartfmt
+ - name: dartanalyzer
+ - name: dart format
+ - name: dart analyze
+ - name: dart fix
+ - name: dart pub
+```
+
+### After (15 commands) ✨
+```yaml
+commands:
+ # Flutter & Dart (wildcards cover all subcommands)
+ - name: flutter*
+ description: All Flutter SDK commands
+
+ - name: dart*
+ description: All Dart language tools
+
+ # Testing tools
+ - name: patrol
+ description: Patrol integration testing (if needed separately)
+
+ # Coverage tools
+ - name: lcov
+ description: Code coverage tool
+
+ - name: genhtml
+ description: Generate HTML coverage reports
+
+ # Android tools
+ - name: adb*
+ description: Android Debug Bridge commands
+
+ - name: gradle*
+ description: Gradle build system
+
+ # iOS tools (macOS only)
+ - name: xcrun*
+ description: Xcode developer tools
+
+ - name: xcodebuild
+ description: Xcode build system
+
+ - name: simctl
+ description: iOS Simulator control
+
+ - name: ios-deploy
+ description: Deploy to iOS devices
+
+ # Project scripts
+ - name: ./scripts/*.sh
+ description: All project build/test scripts
+```
+
+**Reduced from 65 → 15 commands (77% reduction!)**
+
+## Optimization Checklist
+
+For each group of commands, ask:
+
+### ❓ "Do I have the base command AND a wildcard?"
+```yaml
+# Bad (redundant)
+- name: flutter
+- name: flutter*
+
+# Good (just the wildcard)
+- name: flutter*
+```
+
+The wildcard already matches the base command!
+
+### ❓ "Am I listing subcommands individually?"
+```yaml
+# Bad (verbose)
+- name: flutter test
+- name: flutter test --coverage
+- name: flutter build apk
+- name: flutter run
+
+# Good (one wildcard)
+- name: flutter*
+```
+
+### ❓ "Can I group multiple scripts?"
+```yaml
+# If you can't use wildcards for scripts, at least group them logically
+- name: ./scripts/test.sh
+- name: ./scripts/build.sh
+- name: ./scripts/integration_test.sh
+```
+
+These are fine - scripts need explicit paths. But if you have 20+ scripts, consider if they're all necessary.
+
+## Common Wildcards
+
+| Instead of... | Use... | Covers |
+|---------------|--------|--------|
+| flutter, flutter test, flutter build, flutter run | `flutter*` | All flutter commands |
+| dart, dart format, dart analyze, dart pub | `dart*` | All dart commands |
+| npm, npm install, npm run, npm test | `npm*` | All npm commands |
+| cargo, cargo build, cargo test, cargo run | `cargo*` | All cargo commands |
+| git, git status, git commit, git push | Just `git` | Git is in global defaults |
+
+## When NOT to Optimize
+
+Keep separate entries when:
+1. **Different base commands:** `swift` and `swiftc` are different (though `swift*` covers both)
+2. **Documentation clarity:** Sometimes listing helps future developers understand what's needed
+3. **Argument restrictions (Phase 3):** If you'll add argument validation later
+
+## Quick Optimization Script
+
+To see what you can reduce:
+
+```bash
+# Count commands by prefix
+grep "^ - name:" .autoforge/allowed_commands.yaml | \
+ sed 's/^ - name: //' | \
+ cut -d' ' -f1 | \
+ sort | uniq -c | sort -rn
+```
+
+If you see multiple commands with the same prefix, use a wildcard!
+
+## UI Feedback (Future Enhancement)
+
+Great suggestion! Here's what a UI could show:
+
+```
+⚠️ Config Optimization Available
+
+Your config has 65 commands. We detected opportunities to reduce it:
+
+• 25 flutter commands → Use flutter* (saves 24 entries)
+• 8 dart commands → Use dart* (saves 7 entries)
+• 12 adb commands → Use adb* (saves 11 entries)
+
+[Optimize Automatically] [Keep As-Is]
+
+Potential reduction: 65 → 23 commands
+```
+
+Or during config editing:
+
+```
+📊 Command Usage Stats
+
+flutter* : 25 subcommands detected
+dart* : 8 subcommands detected
+./scripts/*.sh : 7 scripts detected
+
+💡 Tip: Using wildcards covers all subcommands automatically
+```
+
+**This would be a great addition to Phase 3 or beyond!**
+
+## Your GOD-APP Optimization
+
+Here's what you could do:
+
+**Current:** 65 commands (exceeds 50 limit, config rejected!)
+
+**Optimized:** ~15 commands using wildcards
+
+Would you like me to create an optimized version of your GOD-APP config?
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 000000000..21ba3c80b
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,531 @@
+# AutoForge Security Configuration Examples
+
+This directory contains example configuration files for controlling which bash commands the autonomous coding agent can execute.
+
+## Table of Contents
+
+- [Quick Start](#quick-start)
+- [Project-Level Configuration](#project-level-configuration)
+- [Organization-Level Configuration](#organization-level-configuration)
+- [Command Hierarchy](#command-hierarchy)
+- [Pattern Matching](#pattern-matching)
+- [Common Use Cases](#common-use-cases)
+- [Security Best Practices](#security-best-practices)
+
+---
+
+## Quick Start
+
+### For a Single Project (Most Common)
+
+When you create a new project with AutoForge, it automatically creates:
+
+```text
+my-project/
+ .autoforge/
+ allowed_commands.yaml ← Automatically created from template
+```
+
+**Edit this file** to add project-specific commands (Swift tools, Rust compiler, etc.).
+
+### For All Projects (Organization-Wide)
+
+If you want commands available across **all projects**, manually create:
+
+```bash
+# Copy the example to your home directory
+cp examples/org_config.yaml ~/.autoforge/config.yaml
+
+# Edit it to add org-wide commands
+nano ~/.autoforge/config.yaml
+```
+
+---
+
+## Project-Level Configuration
+
+**File:** `{project_dir}/.autoforge/allowed_commands.yaml`
+
+**Purpose:** Define commands needed for THIS specific project.
+
+**Example** (iOS project):
+
+```yaml
+version: 1
+commands:
+ - name: swift
+ description: Swift compiler
+
+ - name: xcodebuild
+ description: Xcode build system
+
+ - name: swift*
+ description: All Swift tools (swiftc, swiftlint, swiftformat)
+
+ - name: ./scripts/build.sh
+ description: Project build script
+```
+
+**When to use:**
+- ✅ Project uses a specific language toolchain (Swift, Rust, Go)
+- ✅ Project has custom build scripts
+- ✅ Temporary tools needed during development
+
+**Limits:**
+- Maximum 100 commands per project
+- Cannot override org-level blocked commands
+- Cannot allow hardcoded blocklist commands (sudo, dd, etc.)
+
+**See:** `examples/project_allowed_commands.yaml` for full example with Rust, Python, iOS, etc.
+
+---
+
+## Organization-Level Configuration
+
+**File:** `~/.autoforge/config.yaml`
+
+**Purpose:** Define commands and policies for ALL projects.
+
+**Example** (startup team):
+
+```yaml
+version: 1
+
+# Available to all projects
+allowed_commands:
+ - name: jq
+ description: JSON processor
+
+ - name: python3
+ description: Python interpreter
+
+# Blocked across all projects (cannot be overridden)
+blocked_commands:
+ - aws
+ - kubectl
+ - terraform
+```
+
+**When to use:**
+- ✅ Multiple projects need the same tools (jq, python3, etc.)
+- ✅ Enforce organization-wide security policies
+- ✅ Block dangerous commands across all projects
+
+**See:** `examples/org_config.yaml` for full example with enterprise/startup configurations.
+
+---
+
+## Command Hierarchy
+
+When the agent tries to run a command, the system checks in this order:
+
+```text
+┌─────────────────────────────────────────────────────┐
+│ 1. HARDCODED BLOCKLIST (highest priority) │
+│ sudo, dd, shutdown, reboot, chown, etc. │
+│ ❌ NEVER allowed, even with user approval │
+└─────────────────────────────────────────────────────┘
+ ↓
+┌─────────────────────────────────────────────────────┐
+│ 2. ORG BLOCKLIST (~/.autoforge/config.yaml) │
+│ Commands you block organization-wide │
+│ ❌ Projects CANNOT override these │
+└─────────────────────────────────────────────────────┘
+ ↓
+┌─────────────────────────────────────────────────────┐
+│ 3. ORG ALLOWLIST (~/.autoforge/config.yaml) │
+│ Commands available to all projects │
+│ ✅ Automatically available │
+└─────────────────────────────────────────────────────┘
+ ↓
+┌─────────────────────────────────────────────────────┐
+│ 4. GLOBAL ALLOWLIST (security.py) │
+│ Default commands: npm, git, curl, ls, cat, etc. │
+│ ✅ Always available │
+└─────────────────────────────────────────────────────┘
+ ↓
+┌─────────────────────────────────────────────────────┐
+│ 5. PROJECT ALLOWLIST (.autoforge/allowed_commands) │
+│ Project-specific commands │
+│ ✅ Available only to this project │
+└─────────────────────────────────────────────────────┘
+```
+
+**Key Rules:**
+- If a command is BLOCKED at any level above, it cannot be allowed below
+- If a command is ALLOWED at any level, it's available (unless blocked above)
+- Blocklist always wins over allowlist
+
+---
+
+## Pattern Matching
+
+You can use patterns to match multiple commands:
+
+### Exact Match
+```yaml
+- name: swift
+ description: Swift compiler only
+```
+Matches: `swift`
+Does NOT match: `swiftc`, `swiftlint`
+
+### Prefix Wildcard
+```yaml
+- name: swift*
+ description: All Swift tools
+```
+Matches: `swift`, `swiftc`, `swiftlint`, `swiftformat`
+Does NOT match: `npm`, `rustc`
+
+### Local Scripts
+```yaml
+- name: ./scripts/build.sh
+ description: Build script
+```
+Matches:
+- `./scripts/build.sh`
+- `scripts/build.sh`
+- `/full/path/to/scripts/build.sh`
+- Running `build.sh` from any directory (matched by filename)
+
+---
+
+## Common Use Cases
+
+### iOS Development
+
+**Project config** (`.autoforge/allowed_commands.yaml`):
+```yaml
+version: 1
+commands:
+ - name: swift*
+ description: All Swift tools
+ - name: xcodebuild
+ description: Xcode build system
+ - name: xcrun
+ description: Xcode tools runner
+ - name: simctl
+ description: iOS Simulator control
+```
+
+### Rust CLI Project
+
+**Project config**:
+```yaml
+version: 1
+commands:
+ - name: cargo
+ description: Rust package manager
+ - name: rustc
+ description: Rust compiler
+ - name: rustfmt
+ description: Rust formatter
+ - name: clippy
+ description: Rust linter
+ - name: ./target/debug/my-cli
+ description: Debug build
+ - name: ./target/release/my-cli
+ description: Release build
+```
+
+### API Testing Project
+
+**Project config**:
+```yaml
+version: 1
+commands:
+ - name: jq
+ description: JSON processor
+ - name: httpie
+ description: HTTP client
+ - name: ./scripts/test-api.sh
+ description: API test runner
+```
+
+### Enterprise Organization (Restrictive)
+
+**Org config** (`~/.autoforge/config.yaml`):
+```yaml
+version: 1
+
+allowed_commands:
+ - name: jq
+ description: JSON processor
+
+blocked_commands:
+ - aws # No cloud access
+ - gcloud
+ - az
+ - kubectl # No k8s access
+ - terraform # No infrastructure changes
+ - psql # No production DB access
+ - mysql
+```
+
+### Startup Team (Permissive)
+
+**Org config** (`~/.autoforge/config.yaml`):
+```yaml
+version: 1
+
+allowed_commands:
+ - name: python3
+ description: Python interpreter
+ - name: jq
+ description: JSON processor
+ - name: pytest
+ description: Python tests
+
+blocked_commands: [] # Rely on hardcoded blocklist only
+```
+
+---
+
+## Security Best Practices
+
+### ✅ DO
+
+1. **Start restrictive, add as needed**
+ - Begin with default commands only
+ - Add project-specific tools when required
+ - Review the agent's blocked command errors to understand what's needed
+
+2. **Use org-level config for shared tools**
+ - If 3+ projects need `jq`, add it to org config
+ - Reduces duplication across project configs
+
+3. **Block dangerous commands at org level**
+ - Prevent accidental production deployments (`kubectl`, `terraform`)
+ - Block cloud CLIs if appropriate (`aws`, `gcloud`, `az`)
+
+4. **Use descriptive command names**
+ - Good: `description: "Swift compiler for iOS builds"`
+ - Bad: `description: "Compiler"`
+
+5. **Prefer patterns for tool families**
+ - `swift*` instead of listing `swift`, `swiftc`, `swiftlint` separately
+ - Automatically includes future tools (e.g., new Swift utilities)
+
+### ❌ DON'T
+
+1. **Don't add commands "just in case"**
+ - Only add when the agent actually needs them
+ - Empty config is fine - defaults are usually enough
+
+2. **Don't try to allow blocklisted commands**
+ - Commands like `sudo`, `dd`, `shutdown` can NEVER be allowed
+ - The system will reject these in validation
+
+3. **Don't use org config for project-specific tools**
+ - Bad: Adding `xcodebuild` to org config when only one project uses it
+ - Good: Add `xcodebuild` to that project's config
+
+4. **Don't exceed the 100 command limit per project**
+ - If you need more, you're probably listing subcommands unnecessarily
+ - Use wildcards instead: `flutter*` covers all flutter commands, not just the base
+
+5. **Don't ignore validation errors**
+ - If your YAML is rejected, fix the structure
+ - Common issues: missing `version`, malformed lists, over 100 commands
+
+---
+
+## Default Allowed Commands
+
+These commands are **always available** to all projects:
+
+**File Operations:**
+- `ls`, `cat`, `head`, `tail`, `wc`, `grep`, `cp`, `mkdir`, `mv`, `rm`, `touch`
+
+**Shell:**
+- `pwd`, `echo`, `sh`, `bash`, `sleep`
+
+**Version Control:**
+- `git`
+
+**Process Management:**
+- `ps`, `lsof`, `kill`, `pkill` (dev processes only: node, npm, vite)
+
+**Network:**
+- `curl`
+
+**Node.js:**
+- `npm`, `npx`, `pnpm`, `node`
+
+**Docker:**
+- `docker`
+
+**Special:**
+- `chmod` (only `+x` mode for making scripts executable)
+
+---
+
+## Hardcoded Blocklist
+
+These commands are **NEVER allowed**, even with user approval:
+
+**Disk Operations:**
+- `dd`, `mkfs`, `fdisk`, `parted`
+
+**System Control:**
+- `shutdown`, `reboot`, `poweroff`, `halt`, `init`
+
+**Privilege Escalation:**
+- `sudo`, `su`, `doas`
+
+**System Services:**
+- `systemctl`, `service`, `launchctl`
+
+**Network Security:**
+- `iptables`, `ufw`
+
+**Ownership Changes:**
+- `chown`, `chgrp`
+
+**Dangerous Commands** (Phase 3 will add approval):
+- `aws`, `gcloud`, `az`, `kubectl`, `docker-compose`
+
+---
+
+## Troubleshooting
+
+### Error: "Command 'X' is not allowed"
+
+**Solution:** Add the command to your project config:
+```yaml
+# In .autoforge/allowed_commands.yaml
+commands:
+ - name: X
+ description: What this command does
+```
+
+### Error: "Command 'X' is blocked at organization level"
+
+**Cause:** The command is in the org blocklist or hardcoded blocklist.
+
+**Solution:**
+- If in org blocklist: Edit `~/.autoforge/config.yaml` to remove it
+- If in hardcoded blocklist: Cannot be allowed (by design)
+
+### Error: "Could not parse YAML config"
+
+**Cause:** YAML syntax error.
+
+**Solution:** Check for:
+- Missing colons after keys
+- Incorrect indentation (use 2 spaces, not tabs)
+- Missing quotes around special characters
+
+### Config not taking effect
+
+**Solution:**
+1. Restart the agent (changes are loaded on startup)
+2. Verify file location:
+ - Project: `{project}/.autoforge/allowed_commands.yaml`
+ - Org: `~/.autoforge/config.yaml` (must be manually created)
+3. Check YAML is valid (run through a YAML validator)
+
+---
+
+## Testing
+
+### Running the Tests
+
+AutoForge has comprehensive tests for the security system:
+
+**Unit Tests** (136 tests - fast):
+```bash
+source venv/bin/activate
+python test_security.py
+```
+
+Tests:
+- Pattern matching (exact, wildcards, scripts)
+- YAML loading and validation
+- Blocklist enforcement
+- Project and org config hierarchy
+- All existing security validations
+
+**Integration Tests** (9 tests - uses real security hooks):
+```bash
+source venv/bin/activate
+python test_security_integration.py
+```
+
+Tests:
+- Blocked commands are rejected (sudo, shutdown, etc.)
+- Default commands work (ls, git, npm, etc.)
+- Non-allowed commands are blocked (wget, python, etc.)
+- Project config allows commands (swift, xcodebuild, etc.)
+- Pattern matching works (swift* matches swiftlint)
+- Org blocklist cannot be overridden
+- Org allowlist is inherited by projects
+- Invalid YAML is safely ignored
+- 50 command limit is enforced
+
+### Manual Testing
+
+To manually test the security system:
+
+**1. Create a test project:**
+```bash
+python start.py
+# Choose "Create new project"
+# Name it "security-test"
+```
+
+**2. Edit the project config:**
+```bash
+# Navigate to the project directory
+cd path/to/security-test
+
+# Edit the config
+nano .autoforge/allowed_commands.yaml
+```
+
+**3. Add a test command (e.g., Swift):**
+```yaml
+version: 1
+commands:
+ - name: swift
+ description: Swift compiler
+```
+
+**4. Run the agent and observe:**
+- Try a blocked command: `"Run sudo apt install nginx"` → Should be blocked
+- Try an allowed command: `"Run ls -la"` → Should work
+- Try your config command: `"Run swift --version"` → Should work
+- Try a non-allowed command: `"Run wget https://example.com"` → Should be blocked
+
+**5. Check the agent output:**
+
+The agent will show security hook messages like:
+```text
+Command 'sudo' is blocked at organization level and cannot be approved.
+```
+
+Or:
+```text
+Command 'wget' is not allowed.
+To allow this command:
+ 1. Add to .autoforge/allowed_commands.yaml for this project, OR
+ 2. Request mid-session approval (the agent can ask)
+```
+
+---
+
+## Files Reference
+
+- **`examples/project_allowed_commands.yaml`** - Full project config template
+- **`examples/org_config.yaml`** - Full org config template
+- **`security.py`** - Implementation and hardcoded blocklist
+- **`test_security.py`** - Unit tests (136 tests)
+- **`test_security_integration.py`** - Integration tests (9 tests)
+- **`CLAUDE.md`** - Full system documentation
+
+---
+
+## Questions?
+
+See the main documentation in `CLAUDE.md` for architecture details and implementation specifics.
diff --git a/examples/org_config.yaml b/examples/org_config.yaml
new file mode 100644
index 000000000..efdd60069
--- /dev/null
+++ b/examples/org_config.yaml
@@ -0,0 +1,172 @@
+# Organization-Level AutoForge Configuration
+# ============================================
+# Location: ~/.autoforge/config.yaml
+#
+# IMPORTANT: This file is OPTIONAL and must be manually created by you.
+# It does NOT exist by default.
+#
+# Org-level config applies to ALL projects and provides:
+# 1. Organization-wide allowed commands (available to all projects)
+# 2. Organization-wide blocked commands (cannot be overridden by projects)
+# 3. Global settings (approval timeout, etc.)
+#
+# Use this to:
+# - Add commands that ALL your projects need (jq, python3, etc.)
+# - Block dangerous commands across ALL projects (aws, kubectl, etc.)
+# - Enforce organization-wide security policies
+
+version: 1
+
+
+# ==========================================
+# Organization-Wide Allowed Commands
+# ==========================================
+# These commands become available to ALL projects automatically.
+# Projects don't need to add them to their own .autoforge/allowed_commands.yaml
+#
+# By default, this is empty. Uncomment and add commands as needed.
+
+allowed_commands: []
+
+ # Common development utilities
+ # - name: jq
+ # description: JSON processor for API responses
+
+ # - name: python3
+ # description: Python 3 interpreter
+
+ # - name: pip3
+ # description: Python package installer
+
+ # - name: pytest
+ # description: Python testing framework
+
+ # - name: black
+ # description: Python code formatter
+
+ # Database CLIs (if safe in your environment)
+ # - name: psql
+ # description: PostgreSQL client
+
+ # - name: mysql
+ # description: MySQL client
+
+
+# ==========================================
+# Organization-Wide Blocked Commands
+# ==========================================
+# Commands listed here are BLOCKED across ALL projects.
+# Projects CANNOT override these blocks - this is the final word.
+#
+# Use this to enforce security policies, such as:
+# - Preventing accidental production deployments
+# - Blocking cloud CLI tools to avoid infrastructure changes
+# - Preventing access to production databases
+#
+# By default, this is empty. Uncomment commands you want to block.
+
+blocked_commands: []
+
+ # Block cloud CLIs to prevent accidental production changes
+ # - aws
+ # - gcloud
+ # - az
+
+ # Block container orchestration to prevent production deployments
+ # - kubectl
+ # - docker-compose
+
+ # Block infrastructure-as-code tools
+ # - terraform
+ # - pulumi
+
+ # Block database CLIs to prevent production data access
+ # - psql
+ # - mysql
+ # - mongosh
+
+ # Block other potentially dangerous tools
+ # - ansible
+ # - chef
+ # - puppet
+
+
+# ==========================================
+# Global Settings (Phase 3 feature)
+# ==========================================
+# These settings control approval behavior when agents request
+# commands that aren't in the allowlist.
+
+# How long to wait for user approval before denying a command request
+approval_timeout_minutes: 5
+
+
+# ==========================================
+# Command Hierarchy (for reference)
+# ==========================================
+# When the agent tries to run a bash command, the system checks in this order:
+#
+# 1. Hardcoded Blocklist (in security.py) - HIGHEST PRIORITY
+# Commands like: sudo, dd, shutdown, reboot, etc.
+# These can NEVER be allowed, even with user approval.
+#
+# 2. Org Blocked Commands (this file)
+# Commands you specify in "blocked_commands:" above.
+# Projects cannot override these.
+#
+# 3. Org Allowed Commands (this file)
+# Commands you specify in "allowed_commands:" above.
+# Available to all projects automatically.
+#
+# 4. Global Allowed Commands (in security.py)
+# Default commands: npm, git, curl, ls, cat, etc.
+# Always available to all projects.
+#
+# 5. Project Allowed Commands (.autoforge/allowed_commands.yaml)
+# Project-specific commands defined in each project.
+# LOWEST PRIORITY (can't override blocks above).
+#
+# If a command is in BOTH allowed and blocked lists, BLOCKED wins.
+
+
+# ==========================================
+# Example Configurations by Organization Type
+# ==========================================
+
+# Startup / Small Team (permissive):
+# allowed_commands:
+# - name: python3
+# - name: jq
+# blocked_commands: [] # Empty - rely on hardcoded blocklist only
+
+# Enterprise / Regulated (restrictive):
+# allowed_commands: [] # Empty - projects must explicitly request each tool
+# blocked_commands:
+# - aws
+# - gcloud
+# - az
+# - kubectl
+# - terraform
+# - psql
+# - mysql
+# - mongosh
+
+# Development Team (balanced):
+# allowed_commands:
+# - name: jq
+# - name: python3
+# - name: pytest
+# blocked_commands:
+# - aws # Block production access
+# - kubectl # Block deployments
+# - terraform
+
+
+# ==========================================
+# To Create This File
+# ==========================================
+# 1. Copy this example to: ~/.autoforge/config.yaml
+# 2. Uncomment and customize the sections you need
+# 3. Leave empty lists if you don't need org-level controls
+#
+# To learn more, see: examples/README.md
diff --git a/examples/project_allowed_commands.yaml b/examples/project_allowed_commands.yaml
new file mode 100644
index 000000000..af956cf73
--- /dev/null
+++ b/examples/project_allowed_commands.yaml
@@ -0,0 +1,139 @@
+# Project-Specific Allowed Commands
+# ==================================
+# Location: {project_dir}/.autoforge/allowed_commands.yaml
+#
+# This file defines bash commands that the autonomous coding agent can use
+# for THIS SPECIFIC PROJECT, beyond the default allowed commands.
+#
+# When you create a new project, AutoForge automatically creates this file
+# in your project's .autoforge/ directory. You can customize it for your
+# project's specific needs (iOS, Rust, Python, etc.).
+
+version: 1
+
+# Uncomment the commands you need for your specific project.
+# By default, this file has NO commands enabled - you must explicitly add them.
+
+commands: []
+
+ # ==========================================
+ # iOS Development Example
+ # ==========================================
+ # Uncomment these if building an iOS app:
+
+ # - name: xcodebuild
+ # description: Xcode build system for compiling iOS apps
+
+ # - name: swift
+ # description: Swift compiler and REPL
+
+ # - name: swiftc
+ # description: Swift compiler command-line interface
+
+ # - name: xcrun
+ # description: Run Xcode developer tools
+
+ # - name: simctl
+ # description: iOS Simulator control tool
+
+ # Pattern matching with wildcard
+ # This matches: swift, swiftc, swiftformat, swiftlint, etc.
+ # - name: swift*
+ # description: All Swift development tools
+
+
+ # ==========================================
+ # Rust Development Example
+ # ==========================================
+ # Uncomment these if building a Rust project:
+
+ # - name: cargo
+ # description: Rust package manager and build tool
+
+ # - name: rustc
+ # description: Rust compiler
+
+ # - name: rustfmt
+ # description: Rust code formatter
+
+ # - name: clippy
+ # description: Rust linter
+
+
+ # ==========================================
+ # Python Development Example
+ # ==========================================
+ # Uncomment these if building a Python project:
+
+ # - name: python3
+ # description: Python 3 interpreter
+
+ # - name: pip3
+ # description: Python package installer
+
+ # - name: pytest
+ # description: Python testing framework
+
+
+ # ==========================================
+ # Database Tools Example
+ # ==========================================
+ # Uncomment these if you need database access:
+
+ # - name: psql
+ # description: PostgreSQL command-line client
+
+ # - name: sqlite3
+ # description: SQLite database CLI
+
+
+ # ==========================================
+ # Project-Specific Scripts
+ # ==========================================
+ # Local scripts are matched by filename, so these work from any directory
+ # Uncomment and customize for your project:
+
+ # - name: ./scripts/build.sh
+ # description: Project build script
+
+ # - name: ./scripts/test.sh
+ # description: Run all project tests
+
+ # - name: ./scripts/deploy-staging.sh
+ # description: Deploy to staging environment
+
+
+# ==========================================
+# Notes and Best Practices
+# ==========================================
+#
+# Pattern Matching:
+# - Exact: "swift" matches only "swift"
+# - Wildcard: "swift*" matches "swift", "swiftc", "swiftlint", etc.
+# - Scripts: "./scripts/build.sh" matches the script by name
+#
+# Limits:
+# - Maximum 100 commands per project
+# - Commands in the blocklist (sudo, dd, shutdown, etc.) can NEVER be allowed
+# - Org-level blocked commands (see ~/.autoforge/config.yaml) cannot be overridden
+#
+# Default Allowed Commands (always available):
+# File operations: ls, cat, head, tail, wc, grep, cp, mkdir, mv, rm, touch
+# Shell: pwd, echo, sh, bash, sleep
+# Version control: git
+# Process management: ps, lsof, kill, pkill (dev processes only)
+# Network: curl
+# Node.js: npm, npx, pnpm, node
+# Docker: docker
+# chmod: Only +x mode (making scripts executable)
+#
+# Hardcoded Blocklist (NEVER allowed):
+# Disk operations: dd, mkfs, fdisk, parted
+# System control: shutdown, reboot, poweroff, halt, init
+# Privilege escalation: sudo, su, doas
+# System services: systemctl, service, launchctl
+# Network security: iptables, ufw
+# Ownership changes: chown, chgrp
+# Dangerous commands: aws, gcloud, az, kubectl (unless org allows)
+#
+# To learn more, see: examples/README.md
diff --git a/lib/cli.js b/lib/cli.js
new file mode 100644
index 000000000..682ba849e
--- /dev/null
+++ b/lib/cli.js
@@ -0,0 +1,834 @@
+/**
+ * AutoForge CLI
+ * =============
+ *
+ * Main CLI module for the AutoForge npm global package.
+ * Handles Python detection, virtual environment management,
+ * config loading, and uvicorn server lifecycle.
+ *
+ * Uses only Node.js built-in modules -- no external dependencies.
+ */
+
+import { execFileSync, spawn, execSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync, mkdirSync, unlinkSync, rmSync, copyFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { createServer } from 'node:net';
+import { homedir, platform } from 'node:os';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+// ---------------------------------------------------------------------------
+// Path constants
+// ---------------------------------------------------------------------------
+
+/** Root of the autoforge npm package (one level up from lib/) */
+const PKG_DIR = dirname(dirname(fileURLToPath(import.meta.url)));
+
+/** User config home: ~/.autoforge/ */
+const CONFIG_HOME = join(homedir(), '.autoforge');
+
+/** Virtual-environment directory managed by the CLI */
+const VENV_DIR = join(CONFIG_HOME, 'venv');
+
+/** Composite marker written after a successful pip install */
+const DEPS_MARKER = join(VENV_DIR, '.deps-installed');
+
+/** PID file for the running server */
+const PID_FILE = join(CONFIG_HOME, 'server.pid');
+
+/** Path to the production requirements file inside the package */
+const REQUIREMENTS_FILE = join(PKG_DIR, 'requirements-prod.txt');
+
+/** Path to the .env example shipped with the package */
+const ENV_EXAMPLE = join(PKG_DIR, '.env.example');
+
+/** User .env config file */
+const ENV_FILE = join(CONFIG_HOME, '.env');
+
+const IS_WIN = platform() === 'win32';
+
+// ---------------------------------------------------------------------------
+// Package version (read lazily via createRequire)
+// ---------------------------------------------------------------------------
+
+const require = createRequire(import.meta.url);
+const { version: VERSION } = require(join(PKG_DIR, 'package.json'));
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/** Indented console output matching the spec format. */
+function log(msg = '') {
+ console.log(` ${msg}`);
+}
+
+/** Print a fatal error and exit. */
+function die(msg) {
+ console.error(`\n Error: ${msg}\n`);
+ process.exit(1);
+}
+
+/**
+ * Parse a Python version string like "Python 3.13.6" and return
+ * { major, minor, patch, raw } or null on failure.
+ */
+function parsePythonVersion(raw) {
+ const m = raw.match(/Python\s+(\d+)\.(\d+)\.(\d+)/);
+ if (!m) return null;
+ return {
+ major: Number(m[1]),
+ minor: Number(m[2]),
+ patch: Number(m[3]),
+ raw: `${m[1]}.${m[2]}.${m[3]}`,
+ };
+}
+
+/**
+ * Try a single Python candidate. Returns { exe, version } or null.
+ * `candidate` is either a bare name or an array of args (e.g. ['py', '-3']).
+ */
+function tryPythonCandidate(candidate) {
+ const args = Array.isArray(candidate) ? candidate : [candidate];
+ const exe = args[0];
+ const extraArgs = args.slice(1);
+
+ try {
+ const out = execFileSync(exe, [...extraArgs, '--version'], {
+ encoding: 'utf8',
+ timeout: 10_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+
+ const ver = parsePythonVersion(out);
+ if (!ver) return null;
+
+ // Require 3.11+
+ if (ver.major < 3 || (ver.major === 3 && ver.minor < 11)) {
+ return { exe: args.join(' '), version: ver, tooOld: true };
+ }
+
+ return { exe: args.join(' '), version: ver, tooOld: false };
+ } catch {
+ return null;
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Python detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Find a suitable Python >= 3.11 interpreter.
+ *
+ * Search order is platform-dependent:
+ * Windows: python -> py -3 -> python3
+ * macOS/Linux: python3 -> python
+ *
+ * The AUTOFORGE_PYTHON env var overrides automatic detection.
+ *
+ * After finding a candidate we also verify that the venv module is
+ * available (Debian/Ubuntu strip it out of the base package).
+ */
+function findPython() {
+ // Allow explicit override via environment variable
+ const override = process.env.AUTOFORGE_PYTHON;
+ if (override) {
+ const result = tryPythonCandidate(override);
+ if (!result) {
+ die(`AUTOFORGE_PYTHON is set to "${override}" but it could not be executed.`);
+ }
+ if (result.tooOld) {
+ die(
+ `Python ${result.version.raw} found (via AUTOFORGE_PYTHON), but 3.11+ required.\n` +
+ ' Install Python 3.11+ from https://python.org'
+ );
+ }
+ return result;
+ }
+
+ // Platform-specific candidate order
+ const candidates = IS_WIN
+ ? ['python', ['py', '-3'], 'python3']
+ : ['python3', 'python'];
+
+ let bestTooOld = null;
+
+ for (const candidate of candidates) {
+ const result = tryPythonCandidate(candidate);
+ if (!result) continue;
+
+ if (result.tooOld) {
+ // Remember the first "too old" result for a better error message
+ if (!bestTooOld) bestTooOld = result;
+ continue;
+ }
+
+ // Verify venv module is available (Debian/Ubuntu may need python3-venv)
+ try {
+ const exeParts = result.exe.split(' ');
+ execFileSync(exeParts[0], [...exeParts.slice(1), '-c', 'import ensurepip'], {
+ encoding: 'utf8',
+ timeout: 10_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+ } catch {
+ die(
+ `Python venv module not available.\n` +
+ ` Run: sudo apt install python3.${result.version.minor}-venv`
+ );
+ }
+
+ return result;
+ }
+
+ // Provide the most helpful error message we can
+ if (bestTooOld) {
+ die(
+ `Python ${bestTooOld.version.raw} found, but 3.11+ required.\n` +
+ ' Install Python 3.11+ from https://python.org'
+ );
+ }
+ die(
+ 'Python 3.11+ required but not found.\n' +
+ ' Install from https://python.org'
+ );
+}
+
+// ---------------------------------------------------------------------------
+// Venv management
+// ---------------------------------------------------------------------------
+
+/** Return the path to the Python executable inside the venv. */
+function venvPython() {
+ return IS_WIN
+ ? join(VENV_DIR, 'Scripts', 'python.exe')
+ : join(VENV_DIR, 'bin', 'python');
+}
+
+/** SHA-256 hash of the requirements-prod.txt file contents. */
+function requirementsHash() {
+ const content = readFileSync(REQUIREMENTS_FILE, 'utf8');
+ return createHash('sha256').update(content).digest('hex');
+}
+
+/**
+ * Read the composite deps marker. Returns the parsed JSON object
+ * or null if the file is missing / corrupt.
+ */
+function readMarker() {
+ try {
+ return JSON.parse(readFileSync(DEPS_MARKER, 'utf8'));
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * Ensure the virtual environment exists and dependencies are installed.
+ * Returns true if all setup steps were already satisfied (fast path).
+ *
+ * @param {object} python - The result of findPython()
+ * @param {boolean} forceRecreate - If true, delete and recreate the venv
+ */
+function ensureVenv(python, forceRecreate) {
+ mkdirSync(CONFIG_HOME, { recursive: true });
+
+ const marker = readMarker();
+ const reqHash = requirementsHash();
+ const pyExe = venvPython();
+
+ // Determine if the venv itself needs to be (re)created
+ let needsCreate = forceRecreate || !existsSync(pyExe);
+
+ if (!needsCreate && marker) {
+ // Recreate if Python major.minor changed
+ const markerMinor = marker.python_version;
+ const currentMinor = `${python.version.major}.${python.version.minor}`;
+ if (markerMinor && markerMinor !== currentMinor) {
+ needsCreate = true;
+ }
+
+ // Recreate if the recorded python path no longer exists
+ if (marker.python_path && !existsSync(marker.python_path)) {
+ needsCreate = true;
+ }
+ }
+
+ let depsUpToDate = false;
+ if (!needsCreate && marker && marker.requirements_hash === reqHash) {
+ depsUpToDate = true;
+ }
+
+ // Fast path: nothing to do
+ if (!needsCreate && depsUpToDate) {
+ return true;
+ }
+
+ // --- Slow path: show setup progress ---
+
+ log('[2/3] Setting up environment...');
+
+ if (needsCreate) {
+ if (existsSync(VENV_DIR)) {
+ log(' Removing old virtual environment...');
+ rmSync(VENV_DIR, { recursive: true, force: true });
+ }
+
+ log(` Creating virtual environment at ~/.autoforge/venv/`);
+ const exeParts = python.exe.split(' ');
+ try {
+ execFileSync(exeParts[0], [...exeParts.slice(1), '-m', 'venv', VENV_DIR], {
+ encoding: 'utf8',
+ timeout: 120_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+ } catch (err) {
+ die(`Failed to create virtual environment: ${err.message}`);
+ }
+ }
+
+ // Install / update dependencies
+ log(' Installing dependencies...');
+ try {
+ execFileSync(pyExe, ['-m', 'pip', 'install', '-q', '--upgrade', 'pip'], {
+ encoding: 'utf8',
+ timeout: 300_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+
+ execFileSync(pyExe, ['-m', 'pip', 'install', '-q', '-r', REQUIREMENTS_FILE], {
+ encoding: 'utf8',
+ timeout: 600_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+ } catch (err) {
+ die(`Failed to install dependencies: ${err.message}`);
+ }
+
+ // Write marker only after pip succeeds to prevent partial state
+ const markerData = {
+ requirements_hash: reqHash,
+ python_version: `${python.version.major}.${python.version.minor}`,
+ python_path: pyExe,
+ created_at: new Date().toISOString(),
+ };
+ writeFileSync(DEPS_MARKER, JSON.stringify(markerData, null, 2), 'utf8');
+
+ log(' Done');
+ return false;
+}
+
+// ---------------------------------------------------------------------------
+// Config (.env) management
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a .env file into a plain object.
+ * Handles comments, blank lines, and quoted values.
+ */
+function parseEnvFile(filePath) {
+ const env = {};
+ if (!existsSync(filePath)) return env;
+
+ const lines = readFileSync(filePath, 'utf8').split('\n');
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed || trimmed.startsWith('#')) continue;
+
+ const eqIdx = trimmed.indexOf('=');
+ if (eqIdx === -1) continue;
+
+ const key = trimmed.slice(0, eqIdx).trim();
+ let value = trimmed.slice(eqIdx + 1).trim();
+
+ // Strip matching quotes (single or double)
+ if (
+ (value.startsWith('"') && value.endsWith('"')) ||
+ (value.startsWith("'") && value.endsWith("'"))
+ ) {
+ value = value.slice(1, -1);
+ }
+
+ if (key) {
+ env[key] = value;
+ }
+ }
+ return env;
+}
+
+/**
+ * Ensure ~/.autoforge/.env exists. On first run, copy .env.example
+ * from the package directory and print a notice.
+ *
+ * Returns true if the file was newly created.
+ */
+function ensureEnvFile() {
+ if (existsSync(ENV_FILE)) return false;
+
+ mkdirSync(CONFIG_HOME, { recursive: true });
+
+ if (existsSync(ENV_EXAMPLE)) {
+ copyFileSync(ENV_EXAMPLE, ENV_FILE);
+ } else {
+ // Fallback: create a minimal placeholder
+ writeFileSync(ENV_FILE, '# AutoForge configuration\n# See documentation for available options.\n', 'utf8');
+ }
+ return true;
+}
+
+// ---------------------------------------------------------------------------
+// Port detection
+// ---------------------------------------------------------------------------
+
+/**
+ * Find an available TCP port starting from `start`.
+ * Tries by actually binding a socket (most reliable cross-platform approach).
+ */
+function findAvailablePort(start = 8888, maxAttempts = 20) {
+ for (let port = start; port < start + maxAttempts; port++) {
+ try {
+ const server = createServer();
+ // Use a synchronous-like approach: try to listen, then close immediately
+ const result = new Promise((resolve, reject) => {
+ server.once('error', reject);
+ server.listen(port, '127.0.0.1', () => {
+ server.close(() => resolve(port));
+ });
+ });
+ // We cannot await here (sync context), so use the blocking approach:
+ // Try to bind synchronously using a different technique.
+ server.close();
+ } catch {
+ // fall through
+ }
+ }
+ // Synchronous fallback: try to connect; if connection refused, port is free.
+ for (let port = start; port < start + maxAttempts; port++) {
+ try {
+ execFileSync(process.execPath, [
+ '-e',
+ `const s=require("net").createServer();` +
+ `s.listen(${port},"127.0.0.1",()=>{s.close();process.exit(0)});` +
+ `s.on("error",()=>process.exit(1))`,
+ ], { timeout: 3000, stdio: 'pipe' });
+ return port;
+ } catch {
+ continue;
+ }
+ }
+ die(`No available ports found in range ${start}-${start + maxAttempts - 1}`);
+}
+
+// ---------------------------------------------------------------------------
+// PID file management
+// ---------------------------------------------------------------------------
+
+/** Read PID from the PID file. Returns the PID number or null. */
+function readPid() {
+ try {
+ const content = readFileSync(PID_FILE, 'utf8').trim();
+ const pid = Number(content);
+ return Number.isFinite(pid) && pid > 0 ? pid : null;
+ } catch {
+ return null;
+ }
+}
+
+/** Check whether a process with the given PID is still running. */
+function isProcessAlive(pid) {
+ try {
+ process.kill(pid, 0); // signal 0 = existence check
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Write the PID file. */
+function writePid(pid) {
+ mkdirSync(CONFIG_HOME, { recursive: true });
+ writeFileSync(PID_FILE, String(pid), 'utf8');
+}
+
+/** Remove the PID file. */
+function removePid() {
+ try {
+ unlinkSync(PID_FILE);
+ } catch {
+ // Ignore -- file may already be gone
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Browser opening
+// ---------------------------------------------------------------------------
+
+/** Open a URL in the user's default browser (best-effort). */
+function openBrowser(url) {
+ try {
+ if (IS_WIN) {
+ // "start" is a cmd built-in; the empty title string avoids
+ // issues when the URL contains special characters.
+ execSync(`start "" "${url}"`, { stdio: 'ignore' });
+ } else if (platform() === 'darwin') {
+ execFileSync('open', [url], { stdio: 'ignore' });
+ } else {
+ // Linux: only attempt if a display server is available and
+ // we are not in an SSH session.
+ const hasDisplay = process.env.DISPLAY || process.env.WAYLAND_DISPLAY;
+ const isSSH = !!process.env.SSH_TTY;
+ if (hasDisplay && !isSSH) {
+ execFileSync('xdg-open', [url], { stdio: 'ignore' });
+ }
+ }
+ } catch {
+ // Non-fatal: user can open the URL manually
+ }
+}
+
+/** Detect headless / CI environments where opening a browser is pointless. */
+function isHeadless() {
+ if (process.env.CI) return true;
+ if (process.env.CODESPACES) return true;
+ if (process.env.SSH_TTY) return true;
+ // Linux without a display server
+ if (!IS_WIN && platform() !== 'darwin' && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY) {
+ return true;
+ }
+ return false;
+}
+
+// ---------------------------------------------------------------------------
+// Process cleanup
+// ---------------------------------------------------------------------------
+
+/** Kill a process tree. On Windows uses taskkill; elsewhere sends SIGTERM. */
+function killProcess(pid) {
+ try {
+ if (IS_WIN) {
+ execSync(`taskkill /pid ${pid} /t /f`, { stdio: 'ignore' });
+ } else {
+ process.kill(pid, 'SIGTERM');
+ }
+ } catch {
+ // Process may already be gone
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Playwright CLI
+// ---------------------------------------------------------------------------
+
+/**
+ * Ensure playwright-cli is available globally for browser automation.
+ * Returns true if available (already installed or freshly installed).
+ *
+ * @param {boolean} showProgress - If true, print install progress
+ */
+function ensurePlaywrightCli(showProgress) {
+ try {
+ execSync('playwright-cli --version', {
+ timeout: 10_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+ return true;
+ } catch {
+ // Not installed — try to install
+ }
+
+ if (showProgress) {
+ log(' Installing playwright-cli for browser automation...');
+ }
+ try {
+ execSync('npm install -g @playwright/cli', {
+ timeout: 120_000,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+// ---------------------------------------------------------------------------
+// CLI commands
+// ---------------------------------------------------------------------------
+
+function printVersion() {
+ console.log(`autoforge v${VERSION}`);
+}
+
+function printHelp() {
+ console.log(`
+ AutoForge v${VERSION}
+ Autonomous coding agent with web UI
+
+ Usage:
+ autoforge Start the server (default)
+ autoforge config Open ~/.autoforge/.env in $EDITOR
+ autoforge config --path Print config file path
+ autoforge config --show Show effective configuration
+
+ Options:
+ --port PORT Custom port (default: auto from 8888)
+ --host HOST Custom host (default: 127.0.0.1)
+ --no-browser Don't auto-open browser
+ --repair Delete and recreate virtual environment
+ --dev Development mode (requires cloned repo)
+ --version Print version
+ --help Show this help
+`);
+}
+
+function handleConfig(args) {
+ ensureEnvFile();
+
+ if (args.includes('--path')) {
+ console.log(ENV_FILE);
+ return;
+ }
+
+ if (args.includes('--show')) {
+ if (!existsSync(ENV_FILE)) {
+ log('No configuration file found.');
+ return;
+ }
+ const lines = readFileSync(ENV_FILE, 'utf8').split('\n');
+ const active = lines.filter(l => {
+ const t = l.trim();
+ return t && !t.startsWith('#');
+ });
+ if (active.length === 0) {
+ log('No active configuration. All lines are commented out.');
+ log(`Edit: ${ENV_FILE}`);
+ } else {
+ for (const line of active) {
+ console.log(line);
+ }
+ }
+ return;
+ }
+
+ // Open in editor
+ const editor = process.env.EDITOR || process.env.VISUAL || (IS_WIN ? 'notepad' : 'vi');
+ try {
+ execFileSync(editor, [ENV_FILE], { stdio: 'inherit' });
+ } catch {
+ log(`Could not open editor "${editor}".`);
+ log(`Edit the file manually: ${ENV_FILE}`);
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Main server start
+// ---------------------------------------------------------------------------
+
+function startServer(opts) {
+ const { port: requestedPort, host, noBrowser, repair } = opts;
+
+ // Step 1: Find Python
+ const fastPath = !repair && existsSync(venvPython()) && readMarker()?.requirements_hash === requirementsHash();
+
+ let python;
+ if (fastPath) {
+ // Skip the Python search header on fast path -- we already have a working venv
+ python = null;
+ } else {
+ log(`[1/3] Checking Python...`);
+ python = findPython();
+ log(` Found Python ${python.version.raw} at ${python.exe}`);
+ }
+
+ // Step 2: Ensure venv and deps
+ if (!python) {
+ // Fast path still needs a python reference for potential repair
+ python = findPython();
+ }
+ const wasAlreadyReady = ensureVenv(python, repair);
+
+ // Ensure playwright-cli for browser automation (quick check, installs once)
+ if (!ensurePlaywrightCli(!wasAlreadyReady)) {
+ log('');
+ log(' Note: playwright-cli not available (browser automation will be limited)');
+ log(' Install manually: npm install -g @playwright/cli');
+ log('');
+ }
+
+ // Step 3: Config file
+ const configCreated = ensureEnvFile();
+
+ // Load .env into process.env for the spawned server
+ const dotenvVars = parseEnvFile(ENV_FILE);
+
+ // Determine port
+ const port = requestedPort || findAvailablePort();
+
+ // Check for already-running instance
+ const existingPid = readPid();
+ if (existingPid && isProcessAlive(existingPid)) {
+ log(`AutoForge is already running at http://${host}:${port}`);
+ log('Opening browser...');
+ if (!noBrowser && !isHeadless()) {
+ openBrowser(`http://${host}:${port}`);
+ }
+ return;
+ }
+
+ // Clean up stale PID file
+ if (existingPid) {
+ removePid();
+ }
+
+ // Show server startup step only on slow path
+ if (!wasAlreadyReady) {
+ log('[3/3] Starting server...');
+ }
+
+ if (configCreated) {
+ log(` Created config file: ~/.autoforge/.env`);
+ log(' Edit this file to configure API providers (Ollama, Vertex AI, z.ai)');
+ log('');
+ }
+
+ // Security warning for non-localhost host
+ if (host !== '127.0.0.1') {
+ console.log('');
+ console.log(' !! SECURITY WARNING !!');
+ console.log(` Remote access enabled on host: ${host}`);
+ console.log(' The AutoForge UI will be accessible from other machines.');
+ console.log(' Ensure you understand the security implications.');
+ console.log('');
+ }
+
+ // Build environment for uvicorn
+ const serverEnv = { ...process.env, ...dotenvVars, PYTHONPATH: PKG_DIR };
+
+ // Enable remote access flag for the FastAPI server
+ if (host !== '127.0.0.1') {
+ serverEnv.AUTOFORGE_ALLOW_REMOTE = '1';
+ }
+
+ // Spawn uvicorn
+ const pyExe = venvPython();
+ const child = spawn(
+ pyExe,
+ [
+ '-m', 'uvicorn',
+ 'server.main:app',
+ '--host', host,
+ '--port', String(port),
+ ],
+ {
+ cwd: PKG_DIR,
+ env: serverEnv,
+ stdio: 'inherit',
+ }
+ );
+
+ writePid(child.pid);
+
+ // Open browser after a short delay to let the server start
+ if (!noBrowser && !isHeadless()) {
+ setTimeout(() => openBrowser(`http://${host}:${port}`), 2000);
+ }
+
+ const url = `http://${host}:${port}`;
+ console.log('');
+ log(`Server running at ${url}`);
+ log('Press Ctrl+C to stop');
+
+ // Graceful shutdown handlers
+ const cleanup = () => {
+ killProcess(child.pid);
+ removePid();
+ };
+
+ process.on('SIGINT', () => {
+ console.log('');
+ cleanup();
+ process.exit(0);
+ });
+
+ process.on('SIGTERM', () => {
+ cleanup();
+ process.exit(0);
+ });
+
+ // If the child exits on its own, clean up and propagate the exit code
+ child.on('exit', (code) => {
+ removePid();
+ process.exit(code ?? 1);
+ });
+}
+
+// ---------------------------------------------------------------------------
+// Entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Main CLI entry point.
+ *
+ * @param {string[]} args - Command-line arguments (process.argv.slice(2))
+ */
+export function run(args) {
+ // --version / -v
+ if (args.includes('--version') || args.includes('-v')) {
+ printVersion();
+ return;
+ }
+
+ // --help / -h
+ if (args.includes('--help') || args.includes('-h')) {
+ printHelp();
+ return;
+ }
+
+ // --dev guard: this only works from a cloned repository
+ if (args.includes('--dev')) {
+ die(
+ 'Dev mode requires a cloned repository.\n' +
+ ' Clone from https://github.com/paperlinguist/autocoder and run start_ui.sh'
+ );
+ return;
+ }
+
+ // "config" subcommand
+ if (args[0] === 'config') {
+ handleConfig(args.slice(1));
+ return;
+ }
+
+ // Parse flags for server start
+ const host = getFlagValue(args, '--host') || '127.0.0.1';
+ const portStr = getFlagValue(args, '--port');
+ const port = portStr ? Number(portStr) : null;
+ const noBrowser = args.includes('--no-browser');
+ const repair = args.includes('--repair');
+
+ if (port !== null && (!Number.isFinite(port) || port < 1 || port > 65535)) {
+ die('Invalid port number. Must be between 1 and 65535.');
+ }
+
+ // Print banner
+ console.log('');
+ log(`AutoForge v${VERSION}`);
+ console.log('');
+
+ startServer({ port, host, noBrowser, repair });
+}
+
+// ---------------------------------------------------------------------------
+// Argument parsing helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract the value following a flag from the args array.
+ * E.g. getFlagValue(['--port', '9000', '--host', '0.0.0.0'], '--port') => '9000'
+ */
+function getFlagValue(args, flag) {
+ const idx = args.indexOf(flag);
+ if (idx === -1 || idx + 1 >= args.length) return null;
+ return args[idx + 1];
+}
diff --git a/mcp_server/feature_mcp.py b/mcp_server/feature_mcp.py
old mode 100644
new mode 100755
index 8c5f3c83c..71e217d41
--- a/mcp_server/feature_mcp.py
+++ b/mcp_server/feature_mcp.py
@@ -3,18 +3,28 @@
MCP Server for Feature Management
==================================
-Provides tools to manage features in the autonomous coding system,
-replacing the previous FastAPI-based REST API.
+Provides tools to manage features in the autonomous coding system.
Tools:
- feature_get_stats: Get progress statistics
-- feature_get_next: Get next feature to implement
-- feature_get_for_regression: Get random passing features for testing
+- feature_get_by_id: Get a specific feature by ID
+- feature_get_summary: Get minimal feature info (id, name, status, deps)
- feature_mark_passing: Mark a feature as passing
+- feature_mark_failing: Mark a feature as failing (regression detected)
- feature_skip: Skip a feature (move to end of queue)
- feature_mark_in_progress: Mark a feature as in-progress
+- feature_claim_and_get: Atomically claim and get feature details
- feature_clear_in_progress: Clear in-progress status
- feature_create_bulk: Create multiple features at once
+- feature_create: Create a single feature
+- feature_add_dependency: Add a dependency between features
+- feature_remove_dependency: Remove a dependency
+- feature_get_ready: Get features ready to implement
+- feature_get_blocked: Get features blocked by dependencies (with limit)
+- feature_get_graph: Get the dependency graph
+
+Note: Feature selection (which feature to work on) is handled by the
+orchestrator, not by agents. Agents receive pre-assigned feature IDs.
"""
import json
@@ -26,12 +36,17 @@
from mcp.server.fastmcp import FastMCP
from pydantic import BaseModel, Field
-from sqlalchemy.sql.expression import func
+from sqlalchemy import text
# Add parent directory to path so we can import from api module
sys.path.insert(0, str(Path(__file__).parent.parent))
-from api.database import Feature, create_database
+from api.database import Feature, atomic_transaction, create_database
+from api.dependency_resolver import (
+ MAX_DEPENDENCIES_PER_FEATURE,
+ compute_scheduling_scores,
+ would_create_circular_dependency,
+)
from api.migration import migrate_json_to_sqlite
# Configuration from environment
@@ -81,6 +96,10 @@ class BulkCreateInput(BaseModel):
_session_maker = None
_engine = None
+# NOTE: The old threading.Lock() was removed because it only worked per-process,
+# not cross-process. In parallel mode, multiple MCP servers run in separate
+# processes, so the lock was useless. We now use atomic SQL operations instead.
+
@asynccontextmanager
async def server_lifespan(server: FastMCP):
@@ -124,81 +143,90 @@ def feature_get_stats() -> str:
Returns:
JSON with: passing (int), in_progress (int), total (int), percentage (float)
"""
+ from sqlalchemy import case, func
+
session = get_session()
try:
- total = session.query(Feature).count()
- passing = session.query(Feature).filter(Feature.passes == True).count()
- in_progress = session.query(Feature).filter(Feature.in_progress == True).count()
+ # Single aggregate query instead of 3 separate COUNT queries
+ result = session.query(
+ func.count(Feature.id).label('total'),
+ func.sum(case((Feature.passes == True, 1), else_=0)).label('passing'),
+ func.sum(case((Feature.in_progress == True, 1), else_=0)).label('in_progress'),
+ func.sum(case((Feature.needs_human_input == True, 1), else_=0)).label('needs_human_input')
+ ).first()
+
+ total = result.total or 0
+ passing = int(result.passing or 0)
+ in_progress = int(result.in_progress or 0)
+ needs_human_input = int(result.needs_human_input or 0)
percentage = round((passing / total) * 100, 1) if total > 0 else 0.0
return json.dumps({
"passing": passing,
"in_progress": in_progress,
+ "needs_human_input": needs_human_input,
"total": total,
"percentage": percentage
- }, indent=2)
+ })
finally:
session.close()
@mcp.tool()
-def feature_get_next() -> str:
- """Get the highest-priority pending feature to work on.
+def feature_get_by_id(
+ feature_id: Annotated[int, Field(description="The ID of the feature to retrieve", ge=1)]
+) -> str:
+ """Get a specific feature by its ID.
- Returns the feature with the lowest priority number that has passes=false.
- Use this at the start of each coding session to determine what to implement next.
+ Returns the full details of a feature including its name, description,
+ verification steps, and current status.
+
+ Args:
+ feature_id: The ID of the feature to retrieve
Returns:
- JSON with feature details (id, priority, category, name, description, steps, passes, in_progress)
- or error message if all features are passing.
+ JSON with feature details, or error if not found.
"""
session = get_session()
try:
- feature = (
- session.query(Feature)
- .filter(Feature.passes == False)
- .order_by(Feature.priority.asc(), Feature.id.asc())
- .first()
- )
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
if feature is None:
- return json.dumps({"error": "All features are passing! No more work to do."})
+ return json.dumps({"error": f"Feature with ID {feature_id} not found"})
- return json.dumps(feature.to_dict(), indent=2)
+ return json.dumps(feature.to_dict())
finally:
session.close()
@mcp.tool()
-def feature_get_for_regression(
- limit: Annotated[int, Field(default=3, ge=1, le=10, description="Maximum number of passing features to return")] = 3
+def feature_get_summary(
+ feature_id: Annotated[int, Field(description="The ID of the feature", ge=1)]
) -> str:
- """Get random passing features for regression testing.
+ """Get minimal feature info: id, name, status, and dependencies only.
- Returns a random selection of features that are currently passing.
- Use this to verify that previously implemented features still work
- after making changes.
+ Use this instead of feature_get_by_id when you only need status info,
+ not the full description and steps. This reduces response size significantly.
Args:
- limit: Maximum number of features to return (1-10, default 3)
+ feature_id: The ID of the feature to retrieve
Returns:
- JSON with: features (list of feature objects), count (int)
+ JSON with: id, name, passes, in_progress, dependencies
"""
session = get_session()
try:
- features = (
- session.query(Feature)
- .filter(Feature.passes == True)
- .order_by(func.random())
- .limit(limit)
- .all()
- )
-
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if feature is None:
+ return json.dumps({"error": f"Feature with ID {feature_id} not found"})
return json.dumps({
- "features": [f.to_dict() for f in features],
- "count": len(features)
- }, indent=2)
+ "id": feature.id,
+ "name": feature.name,
+ "passes": feature.passes,
+ "in_progress": feature.in_progress,
+ "needs_human_input": feature.needs_human_input if feature.needs_human_input is not None else False,
+ "dependencies": feature.dependencies or []
+ })
finally:
session.close()
@@ -216,21 +244,84 @@ def feature_mark_passing(
feature_id: The ID of the feature to mark as passing
Returns:
- JSON with the updated feature details, or error if not found.
+ JSON with success confirmation: {success, feature_id, name}
"""
session = get_session()
try:
+ # Atomic update with state guard - prevents double-pass in parallel mode
+ result = session.execute(text("""
+ UPDATE features
+ SET passes = 1, in_progress = 0
+ WHERE id = :id AND passes = 0
+ """), {"id": feature_id})
+ session.commit()
+
+ if result.rowcount == 0:
+ # Check why the update didn't match
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if feature is None:
+ return json.dumps({"error": f"Feature with ID {feature_id} not found"})
+ if feature.passes:
+ return json.dumps({"error": f"Feature with ID {feature_id} is already passing"})
+ return json.dumps({"error": "Failed to mark feature passing for unknown reason"})
+
+ # Get the feature name for the response
feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ return json.dumps({"success": True, "feature_id": feature_id, "name": feature.name})
+ except Exception as e:
+ session.rollback()
+ return json.dumps({"error": f"Failed to mark feature passing: {str(e)}"})
+ finally:
+ session.close()
+
+
+@mcp.tool()
+def feature_mark_failing(
+ feature_id: Annotated[int, Field(description="The ID of the feature to mark as failing", ge=1)]
+) -> str:
+ """Mark a feature as failing after finding a regression.
+
+ Updates the feature's passes field to false and clears the in_progress flag.
+ Use this when a testing agent discovers that a previously-passing feature
+ no longer works correctly (regression detected).
+ After marking as failing, you should:
+ 1. Investigate the root cause
+ 2. Fix the regression
+ 3. Verify the fix
+ 4. Call feature_mark_passing once fixed
+
+ Args:
+ feature_id: The ID of the feature to mark as failing
+
+ Returns:
+ JSON with the updated feature details, or error if not found.
+ """
+ session = get_session()
+ try:
+ # Check if feature exists first
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
if feature is None:
return json.dumps({"error": f"Feature with ID {feature_id} not found"})
- feature.passes = True
- feature.in_progress = False
+ # Atomic update for parallel safety
+ session.execute(text("""
+ UPDATE features
+ SET passes = 0, in_progress = 0
+ WHERE id = :id
+ """), {"id": feature_id})
session.commit()
+
+ # Refresh to get updated state
session.refresh(feature)
- return json.dumps(feature.to_dict(), indent=2)
+ return json.dumps({
+ "message": f"Feature #{feature_id} marked as failing - regression detected",
+ "feature": feature.to_dict()
+ })
+ except Exception as e:
+ session.rollback()
+ return json.dumps({"error": f"Failed to mark feature failing: {str(e)}"})
finally:
session.close()
@@ -267,23 +358,32 @@ def feature_skip(
return json.dumps({"error": "Cannot skip a feature that is already passing"})
old_priority = feature.priority
-
- # Get max priority and set this feature to max + 1
- max_priority_result = session.query(Feature.priority).order_by(Feature.priority.desc()).first()
- new_priority = (max_priority_result[0] + 1) if max_priority_result else 1
-
- feature.priority = new_priority
- feature.in_progress = False
+ name = feature.name
+
+ # Atomic update: set priority to max+1 in a single statement
+ # This prevents race conditions where two features get the same priority
+ session.execute(text("""
+ UPDATE features
+ SET priority = (SELECT COALESCE(MAX(priority), 0) + 1 FROM features),
+ in_progress = 0
+ WHERE id = :id
+ """), {"id": feature_id})
session.commit()
+
+ # Refresh to get new priority
session.refresh(feature)
+ new_priority = feature.priority
return json.dumps({
- "id": feature.id,
- "name": feature.name,
+ "id": feature_id,
+ "name": name,
"old_priority": old_priority,
"new_priority": new_priority,
- "message": f"Feature '{feature.name}' moved to end of queue"
- }, indent=2)
+ "message": f"Feature '{name}' moved to end of queue"
+ })
+ except Exception as e:
+ session.rollback()
+ return json.dumps({"error": f"Failed to skip feature: {str(e)}"})
finally:
session.close()
@@ -292,10 +392,10 @@ def feature_skip(
def feature_mark_in_progress(
feature_id: Annotated[int, Field(description="The ID of the feature to mark as in-progress", ge=1)]
) -> str:
- """Mark a feature as in-progress. Call immediately after feature_get_next().
+ """Mark a feature as in-progress.
This prevents other agent sessions from working on the same feature.
- Use this as soon as you retrieve a feature to work on.
+ Call this after getting your assigned feature details with feature_get_by_id.
Args:
feature_id: The ID of the feature to mark as in-progress
@@ -305,22 +405,89 @@ def feature_mark_in_progress(
"""
session = get_session()
try:
+ # Atomic claim: only succeeds if feature is not already claimed, passing, or blocked for human input
+ result = session.execute(text("""
+ UPDATE features
+ SET in_progress = 1
+ WHERE id = :id AND passes = 0 AND in_progress = 0 AND needs_human_input = 0
+ """), {"id": feature_id})
+ session.commit()
+
+ if result.rowcount == 0:
+ # Check why the claim failed
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if feature is None:
+ return json.dumps({"error": f"Feature with ID {feature_id} not found"})
+ if feature.passes:
+ return json.dumps({"error": f"Feature with ID {feature_id} is already passing"})
+ if feature.in_progress:
+ return json.dumps({"error": f"Feature with ID {feature_id} is already in-progress"})
+ if getattr(feature, 'needs_human_input', False):
+ return json.dumps({"error": f"Feature with ID {feature_id} is blocked waiting for human input"})
+ return json.dumps({"error": "Failed to mark feature in-progress for unknown reason"})
+
+ # Fetch the claimed feature
feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ return json.dumps(feature.to_dict())
+ except Exception as e:
+ session.rollback()
+ return json.dumps({"error": f"Failed to mark feature in-progress: {str(e)}"})
+ finally:
+ session.close()
+
+@mcp.tool()
+def feature_claim_and_get(
+ feature_id: Annotated[int, Field(description="The ID of the feature to claim", ge=1)]
+) -> str:
+ """Atomically claim a feature (mark in-progress) and return its full details.
+
+ Combines feature_mark_in_progress + feature_get_by_id into a single operation.
+ If already in-progress, still returns the feature details (idempotent).
+
+ Args:
+ feature_id: The ID of the feature to claim and retrieve
+
+ Returns:
+ JSON with feature details including claimed status, or error if not found.
+ """
+ session = get_session()
+ try:
+ # First check if feature exists
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
if feature is None:
return json.dumps({"error": f"Feature with ID {feature_id} not found"})
if feature.passes:
return json.dumps({"error": f"Feature with ID {feature_id} is already passing"})
- if feature.in_progress:
- return json.dumps({"error": f"Feature with ID {feature_id} is already in-progress"})
+ if getattr(feature, 'needs_human_input', False):
+ return json.dumps({"error": f"Feature with ID {feature_id} is blocked waiting for human input"})
- feature.in_progress = True
+ # Try atomic claim: only succeeds if not already claimed and not blocked for human input
+ result = session.execute(text("""
+ UPDATE features
+ SET in_progress = 1
+ WHERE id = :id AND passes = 0 AND in_progress = 0 AND needs_human_input = 0
+ """), {"id": feature_id})
session.commit()
- session.refresh(feature)
- return json.dumps(feature.to_dict(), indent=2)
+ # Determine if we claimed it or it was already claimed
+ already_claimed = result.rowcount == 0
+ if already_claimed:
+ # Verify it's in_progress (not some other failure condition)
+ session.refresh(feature)
+ if not feature.in_progress:
+ return json.dumps({"error": f"Failed to claim feature {feature_id} for unknown reason"})
+
+ # Refresh to get current state
+ session.refresh(feature)
+ result_dict = feature.to_dict()
+ result_dict["already_claimed"] = already_claimed
+ return json.dumps(result_dict)
+ except Exception as e:
+ session.rollback()
+ return json.dumps({"error": f"Failed to claim feature: {str(e)}"})
finally:
session.close()
@@ -342,16 +509,24 @@ def feature_clear_in_progress(
"""
session = get_session()
try:
+ # Check if feature exists
feature = session.query(Feature).filter(Feature.id == feature_id).first()
-
if feature is None:
return json.dumps({"error": f"Feature with ID {feature_id} not found"})
- feature.in_progress = False
+ # Atomic update - idempotent, safe in parallel mode
+ session.execute(text("""
+ UPDATE features
+ SET in_progress = 0
+ WHERE id = :id
+ """), {"id": feature_id})
session.commit()
- session.refresh(feature)
- return json.dumps(feature.to_dict(), indent=2)
+ session.refresh(feature)
+ return json.dumps(feature.to_dict())
+ except Exception as e:
+ session.rollback()
+ return json.dumps({"error": f"Failed to clear in-progress status: {str(e)}"})
finally:
session.close()
@@ -374,44 +549,593 @@ def feature_create_bulk(
- name (str): Feature name
- description (str): Detailed description
- steps (list[str]): Implementation/test steps
+ - depends_on_indices (list[int], optional): Array indices (0-based) of
+ features in THIS batch that this feature depends on. Use this instead
+ of 'dependencies' since IDs aren't known until after creation.
+ Example: [0, 2] means this feature depends on features at index 0 and 2.
Returns:
- JSON with: created (int) - number of features created
+ JSON with: created (int) - number of features created, with_dependencies (int)
"""
- session = get_session()
try:
- # Get the starting priority
- max_priority_result = session.query(Feature.priority).order_by(Feature.priority.desc()).first()
- start_priority = (max_priority_result[0] + 1) if max_priority_result else 1
-
- created_count = 0
- for i, feature_data in enumerate(features):
- # Validate required fields
- if not all(key in feature_data for key in ["category", "name", "description", "steps"]):
- return json.dumps({
- "error": f"Feature at index {i} missing required fields (category, name, description, steps)"
- })
+ # Use atomic transaction for bulk inserts to prevent priority conflicts
+ with atomic_transaction(_session_maker) as session:
+ # Get the starting priority atomically within the transaction
+ result = session.execute(text("""
+ SELECT COALESCE(MAX(priority), 0) FROM features
+ """)).fetchone()
+ start_priority = (result[0] or 0) + 1
+
+ # First pass: validate all features and their index-based dependencies
+ for i, feature_data in enumerate(features):
+ # Validate required fields
+ if not all(key in feature_data for key in ["category", "name", "description", "steps"]):
+ return json.dumps({
+ "error": f"Feature at index {i} missing required fields (category, name, description, steps)"
+ })
+
+ # Validate depends_on_indices
+ indices = feature_data.get("depends_on_indices", [])
+ if indices:
+ # Check max dependencies
+ if len(indices) > MAX_DEPENDENCIES_PER_FEATURE:
+ return json.dumps({
+ "error": f"Feature at index {i} has {len(indices)} dependencies, max is {MAX_DEPENDENCIES_PER_FEATURE}"
+ })
+ # Check for duplicates
+ if len(indices) != len(set(indices)):
+ return json.dumps({
+ "error": f"Feature at index {i} has duplicate dependencies"
+ })
+ # Check for forward references (can only depend on earlier features)
+ for idx in indices:
+ if not isinstance(idx, int) or idx < 0:
+ return json.dumps({
+ "error": f"Feature at index {i} has invalid dependency index: {idx}"
+ })
+ if idx >= i:
+ return json.dumps({
+ "error": f"Feature at index {i} cannot depend on feature at index {idx} (forward reference not allowed)"
+ })
+
+ # Second pass: create all features with reserved priorities
+ created_features: list[Feature] = []
+ for i, feature_data in enumerate(features):
+ db_feature = Feature(
+ priority=start_priority + i,
+ category=feature_data["category"],
+ name=feature_data["name"],
+ description=feature_data["description"],
+ steps=feature_data["steps"],
+ passes=False,
+ in_progress=False,
+ )
+ session.add(db_feature)
+ created_features.append(db_feature)
+
+ # Flush to get IDs assigned
+ session.flush()
+
+ # Third pass: resolve index-based dependencies to actual IDs
+ deps_count = 0
+ for i, feature_data in enumerate(features):
+ indices = feature_data.get("depends_on_indices", [])
+ if indices:
+ # Convert indices to actual feature IDs
+ dep_ids = [created_features[idx].id for idx in indices]
+ created_features[i].dependencies = sorted(dep_ids) # type: ignore[assignment] # SQLAlchemy JSON Column accepts list at runtime
+ deps_count += 1
+
+ # Commit happens automatically on context manager exit
+ return json.dumps({
+ "created": len(created_features),
+ "with_dependencies": deps_count
+ })
+ except Exception as e:
+ return json.dumps({"error": str(e)})
+
+
+@mcp.tool()
+def feature_create(
+ category: Annotated[str, Field(min_length=1, max_length=100, description="Feature category (e.g., 'Authentication', 'API', 'UI')")],
+ name: Annotated[str, Field(min_length=1, max_length=255, description="Feature name")],
+ description: Annotated[str, Field(min_length=1, description="Detailed description of the feature")],
+ steps: Annotated[list[str], Field(min_length=1, description="List of implementation/verification steps")]
+) -> str:
+ """Create a single feature in the project backlog.
+
+ Use this when the user asks to add a new feature, capability, or test case.
+ The feature will be added with the next available priority number.
+
+ Args:
+ category: Feature category for grouping (e.g., 'Authentication', 'API', 'UI')
+ name: Descriptive name for the feature
+ description: Detailed description of what this feature should do
+ steps: List of steps to implement or verify the feature
+
+ Returns:
+ JSON with the created feature details including its ID
+ """
+ try:
+ # Use atomic transaction to prevent priority collisions
+ with atomic_transaction(_session_maker) as session:
+ # Get the next priority atomically within the transaction
+ result = session.execute(text("""
+ SELECT COALESCE(MAX(priority), 0) + 1 FROM features
+ """)).fetchone()
+ next_priority = result[0]
db_feature = Feature(
- priority=start_priority + i,
- category=feature_data["category"],
- name=feature_data["name"],
- description=feature_data["description"],
- steps=feature_data["steps"],
+ priority=next_priority,
+ category=category,
+ name=name,
+ description=description,
+ steps=steps,
passes=False,
+ in_progress=False,
)
session.add(db_feature)
- created_count += 1
+ session.flush() # Get the ID
+
+ feature_dict = db_feature.to_dict()
+ # Commit happens automatically on context manager exit
+
+ return json.dumps({
+ "success": True,
+ "message": f"Created feature: {name}",
+ "feature": feature_dict
+ })
+ except Exception as e:
+ return json.dumps({"error": str(e)})
+
+
+@mcp.tool()
+def feature_add_dependency(
+ feature_id: Annotated[int, Field(ge=1, description="Feature to add dependency to")],
+ dependency_id: Annotated[int, Field(ge=1, description="ID of the dependency feature")]
+) -> str:
+ """Add a dependency relationship between features.
+
+ The dependency_id feature must be completed before feature_id can be started.
+ Validates: self-reference, existence, circular dependencies, max limit.
+
+ Args:
+ feature_id: The ID of the feature that will depend on another feature
+ dependency_id: The ID of the feature that must be completed first
+
+ Returns:
+ JSON with success status and updated dependencies list, or error message
+ """
+ try:
+ # Security: Self-reference check (can do before transaction)
+ if feature_id == dependency_id:
+ return json.dumps({"error": "A feature cannot depend on itself"})
+
+ # Use atomic transaction for consistent cycle detection
+ with atomic_transaction(_session_maker) as session:
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ dependency = session.query(Feature).filter(Feature.id == dependency_id).first()
+
+ if not feature:
+ return json.dumps({"error": f"Feature {feature_id} not found"})
+ if not dependency:
+ return json.dumps({"error": f"Dependency feature {dependency_id} not found"})
+
+ current_deps = feature.dependencies or []
+
+ # Security: Max dependencies limit
+ if len(current_deps) >= MAX_DEPENDENCIES_PER_FEATURE:
+ return json.dumps({"error": f"Maximum {MAX_DEPENDENCIES_PER_FEATURE} dependencies allowed per feature"})
+
+ # Check if already exists
+ if dependency_id in current_deps:
+ return json.dumps({"error": "Dependency already exists"})
+
+ # Security: Circular dependency check
+ # Within IMMEDIATE transaction, snapshot is protected by write lock
+ all_features = [f.to_dict() for f in session.query(Feature).all()]
+ if would_create_circular_dependency(all_features, feature_id, dependency_id):
+ return json.dumps({"error": "Cannot add: would create circular dependency"})
+
+ # Add dependency atomically
+ new_deps = sorted(current_deps + [dependency_id])
+ feature.dependencies = new_deps
+ # Commit happens automatically on context manager exit
+
+ return json.dumps({
+ "success": True,
+ "feature_id": feature_id,
+ "dependencies": new_deps
+ })
+ except Exception as e:
+ return json.dumps({"error": f"Failed to add dependency: {str(e)}"})
+
+
+@mcp.tool()
+def feature_remove_dependency(
+ feature_id: Annotated[int, Field(ge=1, description="Feature to remove dependency from")],
+ dependency_id: Annotated[int, Field(ge=1, description="ID of dependency to remove")]
+) -> str:
+ """Remove a dependency from a feature.
+
+ Args:
+ feature_id: The ID of the feature to remove a dependency from
+ dependency_id: The ID of the dependency to remove
+
+ Returns:
+ JSON with success status and updated dependencies list, or error message
+ """
+ try:
+ # Use atomic transaction for consistent read-modify-write
+ with atomic_transaction(_session_maker) as session:
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if not feature:
+ return json.dumps({"error": f"Feature {feature_id} not found"})
+
+ current_deps = feature.dependencies or []
+ if dependency_id not in current_deps:
+ return json.dumps({"error": "Dependency does not exist"})
+
+ # Remove dependency atomically
+ new_deps = [d for d in current_deps if d != dependency_id]
+ feature.dependencies = new_deps if new_deps else None
+ # Commit happens automatically on context manager exit
+
+ return json.dumps({
+ "success": True,
+ "feature_id": feature_id,
+ "dependencies": new_deps
+ })
+ except Exception as e:
+ return json.dumps({"error": f"Failed to remove dependency: {str(e)}"})
+
+
+@mcp.tool()
+def feature_get_ready(
+ limit: Annotated[int, Field(default=10, ge=1, le=50, description="Max features to return")] = 10
+) -> str:
+ """Get all features ready to start (dependencies satisfied, not in progress).
+
+ Useful for parallel execution - returns multiple features that can run simultaneously.
+ A feature is ready if it is not passing, not in progress, and all dependencies are passing.
+
+ Args:
+ limit: Maximum number of features to return (1-50, default 10)
+
+ Returns:
+ JSON with: features (list), count (int), total_ready (int)
+ """
+ session = get_session()
+ try:
+ all_features = session.query(Feature).all()
+ passing_ids = {f.id for f in all_features if f.passes}
+
+ ready = []
+ all_dicts = [f.to_dict() for f in all_features]
+ for f in all_features:
+ if f.passes or f.in_progress:
+ continue
+ if getattr(f, 'needs_human_input', False):
+ continue
+ deps = f.dependencies or []
+ if all(dep_id in passing_ids for dep_id in deps):
+ ready.append(f.to_dict())
+
+ # Sort by scheduling score (higher = first), then priority, then id
+ scores = compute_scheduling_scores(all_dicts)
+ ready.sort(key=lambda f: (-scores.get(f["id"], 0), f["priority"], f["id"]))
+
+ return json.dumps({
+ "features": ready[:limit],
+ "count": len(ready[:limit]),
+ "total_ready": len(ready)
+ })
+ finally:
+ session.close()
+
+
+@mcp.tool()
+def feature_get_blocked(
+ limit: Annotated[int, Field(default=20, ge=1, le=100, description="Max features to return")] = 20
+) -> str:
+ """Get features that are blocked by unmet dependencies.
+
+ Returns features that have dependencies which are not yet passing.
+ Each feature includes a 'blocked_by' field listing the blocking feature IDs.
+
+ Args:
+ limit: Maximum number of features to return (1-100, default 20)
+
+ Returns:
+ JSON with: features (list with blocked_by field), count (int), total_blocked (int)
+ """
+ session = get_session()
+ try:
+ all_features = session.query(Feature).all()
+ passing_ids = {f.id for f in all_features if f.passes}
+
+ blocked = []
+ for f in all_features:
+ if f.passes:
+ continue
+ deps = f.dependencies or []
+ blocking = [d for d in deps if d not in passing_ids]
+ if blocking:
+ blocked.append({
+ **f.to_dict(),
+ "blocked_by": blocking
+ })
+
+ return json.dumps({
+ "features": blocked[:limit],
+ "count": len(blocked[:limit]),
+ "total_blocked": len(blocked)
+ })
+ finally:
+ session.close()
+
+
+@mcp.tool()
+def feature_get_graph() -> str:
+ """Get dependency graph data for visualization.
+ Returns nodes (features) and edges (dependencies) for rendering a graph.
+ Each node includes status: 'pending', 'in_progress', 'done', or 'blocked'.
+
+ Returns:
+ JSON with: nodes (list), edges (list of {source, target})
+ """
+ session = get_session()
+ try:
+ all_features = session.query(Feature).all()
+ passing_ids = {f.id for f in all_features if f.passes}
+
+ nodes = []
+ edges = []
+
+ for f in all_features:
+ deps = f.dependencies or []
+ blocking = [d for d in deps if d not in passing_ids]
+
+ if f.passes:
+ status = "done"
+ elif getattr(f, 'needs_human_input', False):
+ status = "needs_human_input"
+ elif blocking:
+ status = "blocked"
+ elif f.in_progress:
+ status = "in_progress"
+ else:
+ status = "pending"
+
+ nodes.append({
+ "id": f.id,
+ "name": f.name,
+ "category": f.category,
+ "status": status,
+ "priority": f.priority,
+ "dependencies": deps
+ })
+
+ for dep_id in deps:
+ edges.append({"source": dep_id, "target": f.id})
+
+ return json.dumps({
+ "nodes": nodes,
+ "edges": edges
+ })
+ finally:
+ session.close()
+
+
+@mcp.tool()
+def feature_set_dependencies(
+ feature_id: Annotated[int, Field(ge=1, description="Feature to set dependencies for")],
+ dependency_ids: Annotated[list[int], Field(description="List of dependency feature IDs")]
+) -> str:
+ """Set all dependencies for a feature at once, replacing any existing dependencies.
+
+ Validates: self-reference, existence of all dependencies, circular dependencies, max limit.
+
+ Args:
+ feature_id: The ID of the feature to set dependencies for
+ dependency_ids: List of feature IDs that must be completed first
+
+ Returns:
+ JSON with success status and updated dependencies list, or error message
+ """
+ try:
+ # Security: Self-reference check (can do before transaction)
+ if feature_id in dependency_ids:
+ return json.dumps({"error": "A feature cannot depend on itself"})
+
+ # Security: Max dependencies limit
+ if len(dependency_ids) > MAX_DEPENDENCIES_PER_FEATURE:
+ return json.dumps({"error": f"Maximum {MAX_DEPENDENCIES_PER_FEATURE} dependencies allowed"})
+
+ # Check for duplicates
+ if len(dependency_ids) != len(set(dependency_ids)):
+ return json.dumps({"error": "Duplicate dependencies not allowed"})
+
+ # Use atomic transaction for consistent cycle detection
+ with atomic_transaction(_session_maker) as session:
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if not feature:
+ return json.dumps({"error": f"Feature {feature_id} not found"})
+
+ # Validate all dependencies exist
+ all_feature_ids = {f.id for f in session.query(Feature).all()}
+ missing = [d for d in dependency_ids if d not in all_feature_ids]
+ if missing:
+ return json.dumps({"error": f"Dependencies not found: {missing}"})
+
+ # Check for circular dependencies
+ # Within IMMEDIATE transaction, snapshot is protected by write lock
+ all_features = [f.to_dict() for f in session.query(Feature).all()]
+ test_features = []
+ for f in all_features:
+ if f["id"] == feature_id:
+ test_features.append({**f, "dependencies": dependency_ids})
+ else:
+ test_features.append(f)
+
+ for dep_id in dependency_ids:
+ if would_create_circular_dependency(test_features, feature_id, dep_id):
+ return json.dumps({"error": f"Cannot add dependency {dep_id}: would create circular dependency"})
+
+ # Set dependencies atomically
+ sorted_deps = sorted(dependency_ids) if dependency_ids else None
+ feature.dependencies = sorted_deps
+ # Commit happens automatically on context manager exit
+
+ return json.dumps({
+ "success": True,
+ "feature_id": feature_id,
+ "dependencies": sorted_deps or []
+ })
+ except Exception as e:
+ return json.dumps({"error": f"Failed to set dependencies: {str(e)}"})
+
+
+@mcp.tool()
+def feature_request_human_input(
+ feature_id: Annotated[int, Field(description="The ID of the feature that needs human input", ge=1)],
+ prompt: Annotated[str, Field(min_length=1, description="Explain what you need from the human and why")],
+ fields: Annotated[list[dict], Field(min_length=1, description="List of input fields to collect")]
+) -> str:
+ """Request structured input from a human for a feature that is blocked.
+
+ Use this ONLY when the feature genuinely cannot proceed without human intervention:
+ - Creating API keys or external accounts
+ - Choosing between design approaches that require human preference
+ - Configuring external services the agent cannot access
+ - Providing credentials or secrets
+
+ Do NOT use this for issues you can solve yourself (debugging, reading docs, etc.).
+
+ The feature will be moved out of in_progress and into a "needs human input" state.
+ Once the human provides their response, the feature returns to the pending queue
+ and will include the human's response when you pick it up again.
+
+ Args:
+ feature_id: The ID of the feature that needs human input
+ prompt: A clear explanation of what you need and why
+ fields: List of input fields, each with:
+ - id (str): Unique field identifier
+ - label (str): Human-readable label
+ - type (str): "text", "textarea", "select", or "boolean" (default: "text")
+ - required (bool): Whether the field is required (default: true)
+ - placeholder (str, optional): Placeholder text
+ - options (list, optional): For select type: [{value, label}]
+
+ Returns:
+ JSON with success confirmation or error message
+ """
+ # Validate fields
+ VALID_FIELD_TYPES = {"text", "textarea", "select", "boolean"}
+ seen_ids: set[str] = set()
+ for i, field in enumerate(fields):
+ if "id" not in field or "label" not in field:
+ return json.dumps({"error": f"Field at index {i} missing required 'id' or 'label'"})
+ fid = field["id"]
+ flabel = field["label"]
+ if not isinstance(fid, str) or not fid.strip():
+ return json.dumps({"error": f"Field at index {i} has empty or invalid 'id'"})
+ if not isinstance(flabel, str) or not flabel.strip():
+ return json.dumps({"error": f"Field at index {i} has empty or invalid 'label'"})
+ if fid in seen_ids:
+ return json.dumps({"error": f"Duplicate field id '{fid}' at index {i}"})
+ seen_ids.add(fid)
+ ftype = field.get("type", "text")
+ if ftype not in VALID_FIELD_TYPES:
+ return json.dumps({"error": f"Field at index {i} has invalid type '{ftype}'. Must be one of: {', '.join(sorted(VALID_FIELD_TYPES))}"})
+ if ftype == "select":
+ options = field.get("options")
+ if not options or not isinstance(options, list):
+ return json.dumps({"error": f"Field at index {i} is type 'select' but missing or invalid 'options' array"})
+ for j, opt in enumerate(options):
+ if not isinstance(opt, dict):
+ return json.dumps({"error": f"Field at index {i}, option {j} must be an object with 'value' and 'label'"})
+ if "value" not in opt or "label" not in opt:
+ return json.dumps({"error": f"Field at index {i}, option {j} missing required 'value' or 'label'"})
+ if not isinstance(opt["value"], str) or not opt["value"].strip():
+ return json.dumps({"error": f"Field at index {i}, option {j} has empty or invalid 'value'"})
+ if not isinstance(opt["label"], str) or not opt["label"].strip():
+ return json.dumps({"error": f"Field at index {i}, option {j} has empty or invalid 'label'"})
+ elif field.get("options"):
+ return json.dumps({"error": f"Field at index {i} has 'options' but type is '{ftype}' (only 'select' uses options)"})
+
+ request_data = {
+ "prompt": prompt,
+ "fields": fields,
+ }
+
+ session = get_session()
+ try:
+ # Atomically set needs_human_input, clear in_progress, store request, clear previous response
+ result = session.execute(text("""
+ UPDATE features
+ SET needs_human_input = 1,
+ in_progress = 0,
+ human_input_request = :request,
+ human_input_response = NULL
+ WHERE id = :id AND passes = 0 AND in_progress = 1
+ """), {"id": feature_id, "request": json.dumps(request_data)})
session.commit()
- return json.dumps({"created": created_count}, indent=2)
+ if result.rowcount == 0:
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if feature is None:
+ return json.dumps({"error": f"Feature with ID {feature_id} not found"})
+ if feature.passes:
+ return json.dumps({"error": f"Feature with ID {feature_id} is already passing"})
+ if not feature.in_progress:
+ return json.dumps({"error": f"Feature with ID {feature_id} is not in progress"})
+ return json.dumps({"error": "Failed to request human input for unknown reason"})
+
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ return json.dumps({
+ "success": True,
+ "feature_id": feature_id,
+ "name": feature.name,
+ "message": f"Feature '{feature.name}' is now blocked waiting for human input"
+ })
except Exception as e:
session.rollback()
- return json.dumps({"error": str(e)})
+ return json.dumps({"error": f"Failed to request human input: {str(e)}"})
finally:
session.close()
+@mcp.tool()
+def ask_user(
+ questions: Annotated[list[dict], Field(description="List of questions to ask, each with question, header, options (list of {label, description}), and multiSelect (bool)")]
+) -> str:
+ """Ask the user structured questions with selectable options.
+
+ Use this when you need clarification or want to offer choices to the user.
+ Each question has a short header, the question text, and 2-4 clickable options.
+ The user's selections will be returned as your next message.
+
+ Args:
+ questions: List of questions, each with:
+ - question (str): The question to ask
+ - header (str): Short label (max 12 chars)
+ - options (list): Each with label (str) and description (str)
+ - multiSelect (bool): Allow multiple selections (default false)
+
+ Returns:
+ Acknowledgment that questions were presented to the user
+ """
+ # Validate input
+ for i, q in enumerate(questions):
+ if not all(key in q for key in ["question", "header", "options"]):
+ return json.dumps({"error": f"Question at index {i} missing required fields"})
+ if len(q["options"]) < 2 or len(q["options"]) > 4:
+ return json.dumps({"error": f"Question at index {i} must have 2-4 options"})
+
+ return "Questions presented to the user. Their response will arrive as your next message."
+
+
if __name__ == "__main__":
mcp.run()
diff --git a/package.json b/package.json
new file mode 100644
index 000000000..9c547ddfc
--- /dev/null
+++ b/package.json
@@ -0,0 +1,55 @@
+{
+ "name": "autoforge-ai",
+ "version": "0.1.22",
+ "description": "Autonomous coding agent with web UI - build complete apps with AI",
+ "license": "AGPL-3.0",
+ "bin": {
+ "autoforge": "./bin/autoforge.js"
+ },
+ "type": "module",
+ "engines": {
+ "node": ">=20"
+ },
+ "files": [
+ "bin/",
+ "lib/",
+ "api/",
+ "server/",
+ "mcp_server/",
+ "ui/dist/",
+ "ui/package.json",
+ ".claude/commands/",
+ ".claude/skills/",
+ ".claude/templates/",
+ "examples/",
+ "start.py",
+ "agent.py",
+ "auth.py",
+ "autoforge_paths.py",
+ "autonomous_agent_demo.py",
+ "client.py",
+ "env_constants.py",
+ "parallel_orchestrator.py",
+ "progress.py",
+ "prompts.py",
+ "registry.py",
+ "rate_limit_utils.py",
+ "security.py",
+ "temp_cleanup.py",
+ "requirements-prod.txt",
+ "pyproject.toml",
+ ".env.example",
+ "!**/__pycache__/",
+ "!**/*.pyc"
+ ],
+ "keywords": [
+ "ai",
+ "coding-agent",
+ "claude",
+ "autonomous",
+ "code-generation"
+ ],
+ "scripts": {
+ "prepublishOnly": "npm --prefix ui install && npm --prefix ui run build"
+ }
+}
diff --git a/parallel_orchestrator.py b/parallel_orchestrator.py
new file mode 100644
index 000000000..e39ef7995
--- /dev/null
+++ b/parallel_orchestrator.py
@@ -0,0 +1,1870 @@
+"""
+Parallel Orchestrator
+=====================
+
+Unified orchestrator that handles all agent lifecycle:
+- Initialization: Creates features from app_spec if needed
+- Coding agents: Implement features one at a time
+- Testing agents: Regression test passing features (optional)
+
+Uses dependency-aware scheduling to ensure features are only started when their
+dependencies are satisfied.
+
+Usage:
+ # Entry point (always uses orchestrator)
+ python autonomous_agent_demo.py --project-dir my-app --concurrency 3
+
+ # Direct orchestrator usage
+ python parallel_orchestrator.py --project-dir my-app --max-concurrency 3
+"""
+
+import asyncio
+import atexit
+import logging
+import os
+import re
+import signal
+import subprocess
+import sys
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable, Literal
+
+from sqlalchemy import text
+
+from api.database import Feature, create_database
+from api.dependency_resolver import are_dependencies_satisfied, compute_scheduling_scores
+from progress import has_features
+from server.utils.process_utils import kill_process_tree
+
+logger = logging.getLogger(__name__)
+
+# Root directory of autoforge (where this script and autonomous_agent_demo.py live)
+AUTOFORGE_ROOT = Path(__file__).parent.resolve()
+
+# Debug log file path
+DEBUG_LOG_FILE = AUTOFORGE_ROOT / "orchestrator_debug.log"
+
+
+class DebugLogger:
+ """Thread-safe debug logger that writes to a file."""
+
+ def __init__(self, log_file: Path = DEBUG_LOG_FILE):
+ self.log_file = log_file
+ self._lock = threading.Lock()
+ self._session_started = False
+ # DON'T clear on import - only mark session start when run_loop begins
+
+ def start_session(self):
+ """Mark the start of a new orchestrator session. Clears previous logs."""
+ with self._lock:
+ self._session_started = True
+ with open(self.log_file, "w") as f:
+ f.write(f"=== Orchestrator Debug Log Started: {datetime.now().isoformat()} ===\n")
+ f.write(f"=== PID: {os.getpid()} ===\n\n")
+
+ def log(self, category: str, message: str, **kwargs):
+ """Write a timestamped log entry."""
+ timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+ with self._lock:
+ with open(self.log_file, "a") as f:
+ f.write(f"[{timestamp}] [{category}] {message}\n")
+ for key, value in kwargs.items():
+ f.write(f" {key}: {value}\n")
+ f.write("\n")
+
+ def section(self, title: str):
+ """Write a section header."""
+ with self._lock:
+ with open(self.log_file, "a") as f:
+ f.write(f"\n{'='*60}\n")
+ f.write(f" {title}\n")
+ f.write(f"{'='*60}\n\n")
+
+
+# Global debug logger instance
+debug_log = DebugLogger()
+
+
+def _dump_database_state(feature_dicts: list[dict], label: str = ""):
+ """Helper to dump full database state to debug log.
+
+ Args:
+ feature_dicts: Pre-fetched list of feature dicts.
+ label: Optional label for the dump entry.
+ """
+ passing = [f for f in feature_dicts if f.get("passes")]
+ in_progress = [f for f in feature_dicts if f.get("in_progress") and not f.get("passes")]
+ pending = [f for f in feature_dicts if not f.get("passes") and not f.get("in_progress")]
+
+ debug_log.log("DB_DUMP", f"Full database state {label}",
+ total_features=len(feature_dicts),
+ passing_count=len(passing),
+ passing_ids=[f["id"] for f in passing],
+ in_progress_count=len(in_progress),
+ in_progress_ids=[f["id"] for f in in_progress],
+ pending_count=len(pending),
+ pending_ids=[f["id"] for f in pending[:10]]) # First 10 pending only
+
+# =============================================================================
+# Process Limits
+# =============================================================================
+# These constants bound the number of concurrent agent processes to prevent
+# resource exhaustion (memory, CPU, API rate limits).
+#
+# MAX_PARALLEL_AGENTS: Max concurrent coding agents (each is a Claude session)
+# MAX_TOTAL_AGENTS: Hard limit on total child processes (coding + testing)
+#
+# Expected process count during normal operation:
+# - 1 orchestrator process (this script)
+# - Up to MAX_PARALLEL_AGENTS coding agents
+# - Up to max_concurrency testing agents
+# - Total never exceeds MAX_TOTAL_AGENTS + 1 (including orchestrator)
+#
+# Stress test verification:
+# 1. Note baseline: tasklist | findstr python | find /c /v ""
+# 2. Run: python autonomous_agent_demo.py --project-dir test --parallel --max-concurrency 5
+# 3. During run: count should never exceed baseline + 11 (1 orchestrator + 10 agents)
+# 4. After stop: should return to baseline
+# =============================================================================
+MAX_PARALLEL_AGENTS = 5
+MAX_TOTAL_AGENTS = 10
+DEFAULT_CONCURRENCY = 3
+DEFAULT_TESTING_BATCH_SIZE = 3 # Number of features per testing batch (1-15)
+POLL_INTERVAL = 5 # seconds between checking for ready features
+MAX_FEATURE_RETRIES = 3 # Maximum times to retry a failed feature
+INITIALIZER_TIMEOUT = 1800 # 30 minutes timeout for initializer
+
+
+class ParallelOrchestrator:
+ """Orchestrates parallel execution of independent features.
+
+ Process bounds:
+ - Up to MAX_PARALLEL_AGENTS (5) coding agents concurrently
+ - Up to max_concurrency testing agents concurrently
+ - Hard limit of MAX_TOTAL_AGENTS (10) total child processes
+ """
+
+ def __init__(
+ self,
+ project_dir: Path,
+ max_concurrency: int = DEFAULT_CONCURRENCY,
+ model: str | None = None,
+ yolo_mode: bool = False,
+ testing_agent_ratio: int = 1,
+ testing_batch_size: int = DEFAULT_TESTING_BATCH_SIZE,
+ batch_size: int = 3,
+ on_output: Callable[[int, str], None] | None = None,
+ on_status: Callable[[int, str], None] | None = None,
+ ):
+ """Initialize the orchestrator.
+
+ Args:
+ project_dir: Path to the project directory
+ max_concurrency: Maximum number of concurrent coding agents (1-5).
+ Also caps testing agents at the same limit.
+ model: Claude model to use (or None for default)
+ yolo_mode: Whether to run in YOLO mode (skip testing agents entirely)
+ testing_agent_ratio: Number of regression testing agents to maintain (0-3).
+ 0 = disabled, 1-3 = maintain that many testing agents running independently.
+ testing_batch_size: Number of features to include per testing session (1-15).
+ Each testing agent receives this many features to regression test.
+ on_output: Callback for agent output (feature_id, line)
+ on_status: Callback for agent status changes (feature_id, status)
+ """
+ self.project_dir = project_dir
+ self.max_concurrency = min(max(max_concurrency, 1), MAX_PARALLEL_AGENTS)
+ self.model = model
+ self.yolo_mode = yolo_mode
+ self.testing_agent_ratio = min(max(testing_agent_ratio, 0), 3) # Clamp 0-3
+ self.testing_batch_size = min(max(testing_batch_size, 1), 15) # Clamp 1-15
+ self.batch_size = min(max(batch_size, 1), 15) # Clamp 1-15
+ self.on_output = on_output
+ self.on_status = on_status
+
+ # Thread-safe state
+ self._lock = threading.Lock()
+ # Coding agents: feature_id -> process
+ # Safe to key by feature_id because start_feature() checks for duplicates before spawning
+ self.running_coding_agents: dict[int, subprocess.Popen] = {}
+ # Testing agents: pid -> (feature_id, process)
+ # Keyed by PID (not feature_id) because multiple agents can test the same feature
+ self.running_testing_agents: dict[int, tuple[int, subprocess.Popen]] = {}
+ # Legacy alias for backward compatibility
+ self.running_agents = self.running_coding_agents
+ self.abort_events: dict[int, threading.Event] = {}
+ self._testing_session_counter = 0
+ self.is_running = False
+
+ # Track feature failures to prevent infinite retry loops
+ self._failure_counts: dict[int, int] = {}
+
+ # Track recently tested feature IDs to avoid redundant re-testing.
+ # Cleared when all passing features have been covered at least once.
+ self._recently_tested: set[int] = set()
+
+ # Batch tracking: primary feature_id -> all feature IDs in batch
+ self._batch_features: dict[int, list[int]] = {}
+ # Reverse mapping: any feature_id -> primary feature_id
+ self._feature_to_primary: dict[int, int] = {}
+
+ # Shutdown flag for async-safe signal handling
+ # Signal handlers only set this flag; cleanup happens in the main loop
+ self._shutdown_requested = False
+
+ # Graceful pause (drain mode) flag
+ self._drain_requested = False
+
+ # Session tracking for logging/debugging
+ self.session_start_time: datetime | None = None
+
+ # Event signaled when any agent completes, allowing the main loop to wake
+ # immediately instead of waiting for the full POLL_INTERVAL timeout.
+ # This reduces latency when spawning the next feature after completion.
+ self._agent_completed_event: asyncio.Event | None = None # Created in run_loop
+ self._event_loop: asyncio.AbstractEventLoop | None = None # Stored for thread-safe signaling
+
+ # Database session for this orchestrator
+ self._engine, self._session_maker = create_database(project_dir)
+
+ def get_session(self):
+ """Get a new database session."""
+ return self._session_maker()
+
+ def _get_random_passing_feature(self) -> int | None:
+ """Get a random passing feature for regression testing (no claim needed).
+
+ Testing agents can test the same feature concurrently - it doesn't matter.
+ This simplifies the architecture by removing unnecessary coordination.
+
+ Returns the feature ID if available, None if no passing features exist.
+
+ Note: Prefer _get_test_batch() for batch testing mode. This method is
+ retained for backward compatibility.
+ """
+ from sqlalchemy.sql.expression import func
+
+ session = self.get_session()
+ try:
+ # Find a passing feature that's not currently being coded
+ # Multiple testing agents can test the same feature - that's fine
+ feature = (
+ session.query(Feature)
+ .filter(Feature.passes == True)
+ .filter(Feature.in_progress == False) # Don't test while coding
+ .order_by(func.random())
+ .first()
+ )
+ return feature.id if feature else None
+ finally:
+ session.close()
+
+ def _get_test_batch(self, batch_size: int = 3) -> list[int]:
+ """Select a prioritized batch of passing features for regression testing.
+
+ Uses weighted scoring to prioritize features that:
+ 1. Haven't been tested recently in this orchestrator session
+ 2. Are depended on by many other features (higher impact if broken)
+ 3. Have more dependencies themselves (complex integration points)
+
+ When all passing features have been recently tested, the tracking set
+ is cleared so the cycle starts fresh.
+
+ Args:
+ batch_size: Maximum number of feature IDs to return (1-5).
+
+ Returns:
+ List of feature IDs to test, may be shorter than batch_size if
+ fewer passing features are available. Empty list if none available.
+ """
+ session = self.get_session()
+ try:
+ session.expire_all()
+ passing = (
+ session.query(Feature)
+ .filter(Feature.passes == True)
+ .filter(Feature.in_progress == False) # Don't test while coding
+ .all()
+ )
+
+ # Extract data from ORM objects before closing the session to avoid
+ # DetachedInstanceError when accessing attributes after session.close().
+ passing_data: list[dict] = []
+ for f in passing:
+ passing_data.append({
+ 'id': f.id,
+ 'dependencies': f.get_dependencies_safe() if hasattr(f, 'get_dependencies_safe') else [],
+ })
+ finally:
+ session.close()
+
+ if not passing_data:
+ return []
+
+ # Build a reverse dependency map: feature_id -> count of features that depend on it.
+ # The Feature model stores dependencies (what I depend ON), so we invert to find
+ # dependents (what depends ON me).
+ dependent_counts: dict[int, int] = {}
+ for fd in passing_data:
+ for dep_id in fd['dependencies']:
+ dependent_counts[dep_id] = dependent_counts.get(dep_id, 0) + 1
+
+ # Exclude features that are already being tested by running testing agents
+ # to avoid redundant concurrent testing of the same features.
+ # running_testing_agents is dict[pid, (primary_feature_id, process)]
+ with self._lock:
+ currently_testing_ids: set[int] = set()
+ for _pid, (feat_id, _proc) in self.running_testing_agents.items():
+ currently_testing_ids.add(feat_id)
+
+ # If all passing features have been recently tested, reset the tracker
+ # so we cycle through them again rather than returning empty batches.
+ passing_ids = {fd['id'] for fd in passing_data}
+ if passing_ids.issubset(self._recently_tested):
+ self._recently_tested.clear()
+
+ # Score each feature by testing priority
+ scored: list[tuple[int, int]] = []
+ for fd in passing_data:
+ f_id = fd['id']
+
+ # Skip features already being tested by a running testing agent
+ if f_id in currently_testing_ids:
+ continue
+
+ score = 0
+
+ # Weight 1: Features depended on by many others are higher impact
+ # if they regress, so test them more often
+ score += dependent_counts.get(f_id, 0) * 2
+
+ # Weight 2: Strongly prefer features not tested recently
+ if f_id not in self._recently_tested:
+ score += 5
+
+ # Weight 3: Features with more dependencies are integration points
+ # that are more likely to regress when other code changes
+ dep_count = len(fd['dependencies'])
+ score += min(dep_count, 3) # Cap at 3 to avoid over-weighting
+
+ scored.append((f_id, score))
+
+ # Sort by score descending (highest priority first)
+ scored.sort(key=lambda x: x[1], reverse=True)
+ selected = [fid for fid, _ in scored[:batch_size]]
+
+ # Track what we've tested to avoid re-testing the same features next batch
+ self._recently_tested.update(selected)
+
+ debug_log.log("TEST_BATCH", f"Selected {len(selected)} features for testing batch",
+ selected_ids=selected,
+ recently_tested_count=len(self._recently_tested),
+ total_passing=len(passing_data))
+
+ return selected
+
+ def build_feature_batches(
+ self,
+ ready: list[dict],
+ all_features: list[dict],
+ scheduling_scores: dict[int, float],
+ ) -> list[list[dict]]:
+ """Build dependency-aware feature batches for coding agents.
+
+ Each batch contains up to `batch_size` features. The algorithm:
+ 1. Start with a ready feature (sorted by scheduling score)
+ 2. Chain extension: find dependents whose deps are satisfied if earlier batch features pass
+ 3. Same-category fill: fill remaining slots with ready features from the same category
+
+ Args:
+ ready: Ready features (sorted by scheduling score)
+ all_features: All features for dependency checking
+ scheduling_scores: Pre-computed scheduling scores
+
+ Returns:
+ List of batches, each batch is a list of feature dicts
+ """
+ if self.batch_size <= 1:
+ # No batching - return each feature as a single-item batch
+ return [[f] for f in ready]
+
+ # Build children adjacency: parent_id -> [child_ids]
+ children: dict[int, list[int]] = {f["id"]: [] for f in all_features}
+ feature_map: dict[int, dict] = {f["id"]: f for f in all_features}
+ for f in all_features:
+ for dep_id in (f.get("dependencies") or []):
+ if dep_id in children:
+ children[dep_id].append(f["id"])
+
+ # Pre-compute passing IDs
+ passing_ids = {f["id"] for f in all_features if f.get("passes")}
+
+ used_ids: set[int] = set() # Features already assigned to a batch
+ batches: list[list[dict]] = []
+
+ for feature in ready:
+ if feature["id"] in used_ids:
+ continue
+
+ batch = [feature]
+ used_ids.add(feature["id"])
+ # Simulate passing set = real passing + batch features
+ simulated_passing = passing_ids | {feature["id"]}
+
+ # Phase 1: Chain extension - find dependents whose deps are met
+ for _ in range(self.batch_size - 1):
+ best_candidate = None
+ best_score = -1.0
+ # Check children of all features currently in the batch
+ candidate_ids: set[int] = set()
+ for bf in batch:
+ for child_id in children.get(bf["id"], []):
+ if child_id not in used_ids and child_id not in simulated_passing:
+ candidate_ids.add(child_id)
+
+ for cid in candidate_ids:
+ cf = feature_map.get(cid)
+ if not cf or cf.get("passes") or cf.get("in_progress"):
+ continue
+ # Check if ALL deps are satisfied by simulated passing set
+ deps = cf.get("dependencies") or []
+ if all(d in simulated_passing for d in deps):
+ score = scheduling_scores.get(cid, 0)
+ if score > best_score:
+ best_score = score
+ best_candidate = cf
+
+ if best_candidate:
+ batch.append(best_candidate)
+ used_ids.add(best_candidate["id"])
+ simulated_passing.add(best_candidate["id"])
+ else:
+ break
+
+ # Phase 2: Same-category fill
+ if len(batch) < self.batch_size:
+ category = feature.get("category", "")
+ for rf in ready:
+ if len(batch) >= self.batch_size:
+ break
+ if rf["id"] in used_ids:
+ continue
+ if rf.get("category", "") == category:
+ batch.append(rf)
+ used_ids.add(rf["id"])
+
+ batches.append(batch)
+
+ debug_log.log("BATCH", f"Built {len(batches)} batches from {len(ready)} ready features",
+ batch_sizes=[len(b) for b in batches],
+ batch_ids=[[f['id'] for f in b] for b in batches[:5]])
+
+ return batches
+
+ def get_resumable_features(
+ self,
+ feature_dicts: list[dict] | None = None,
+ scheduling_scores: dict[int, float] | None = None,
+ ) -> list[dict]:
+ """Get features that were left in_progress from a previous session.
+
+ These are features where in_progress=True but passes=False, and they're
+ not currently being worked on by this orchestrator. This handles the case
+ where a previous session was interrupted before completing the feature.
+
+ Args:
+ feature_dicts: Pre-fetched list of feature dicts. If None, queries the database.
+ scheduling_scores: Pre-computed scheduling scores. If None, computed from feature_dicts.
+ """
+ if feature_dicts is None:
+ session = self.get_session()
+ try:
+ session.expire_all()
+ all_features = session.query(Feature).all()
+ feature_dicts = [f.to_dict() for f in all_features]
+ finally:
+ session.close()
+
+ # Snapshot running IDs once (include all batch feature IDs)
+ with self._lock:
+ running_ids = set(self.running_coding_agents.keys())
+ for batch_ids in self._batch_features.values():
+ running_ids.update(batch_ids)
+
+ resumable = []
+ for fd in feature_dicts:
+ if not fd.get("in_progress") or fd.get("passes"):
+ continue
+ # Skip if blocked for human input
+ if fd.get("needs_human_input"):
+ continue
+ # Skip if already running in this orchestrator instance
+ if fd["id"] in running_ids:
+ continue
+ # Skip if feature has failed too many times
+ if self._failure_counts.get(fd["id"], 0) >= MAX_FEATURE_RETRIES:
+ continue
+ resumable.append(fd)
+
+ # Sort by scheduling score (higher = first), then priority, then id
+ if scheduling_scores is None:
+ scheduling_scores = compute_scheduling_scores(feature_dicts)
+ resumable.sort(key=lambda f: (-scheduling_scores.get(f["id"], 0), f["priority"], f["id"]))
+ return resumable
+
+ def get_ready_features(
+ self,
+ feature_dicts: list[dict] | None = None,
+ scheduling_scores: dict[int, float] | None = None,
+ ) -> list[dict]:
+ """Get features with satisfied dependencies, not already running.
+
+ Args:
+ feature_dicts: Pre-fetched list of feature dicts. If None, queries the database.
+ scheduling_scores: Pre-computed scheduling scores. If None, computed from feature_dicts.
+ """
+ if feature_dicts is None:
+ session = self.get_session()
+ try:
+ session.expire_all()
+ all_features = session.query(Feature).all()
+ feature_dicts = [f.to_dict() for f in all_features]
+ finally:
+ session.close()
+
+ # Pre-compute passing_ids once to avoid O(n^2) in the loop
+ passing_ids = {fd["id"] for fd in feature_dicts if fd.get("passes")}
+
+ # Snapshot running IDs once (include all batch feature IDs)
+ with self._lock:
+ running_ids = set(self.running_coding_agents.keys())
+ for batch_ids in self._batch_features.values():
+ running_ids.update(batch_ids)
+
+ ready = []
+ skipped_reasons = {"passes": 0, "in_progress": 0, "running": 0, "failed": 0, "deps": 0, "needs_human_input": 0}
+ for fd in feature_dicts:
+ if fd.get("passes"):
+ skipped_reasons["passes"] += 1
+ continue
+ if fd.get("needs_human_input"):
+ skipped_reasons["needs_human_input"] += 1
+ continue
+ if fd.get("in_progress"):
+ skipped_reasons["in_progress"] += 1
+ continue
+ # Skip if already running in this orchestrator
+ if fd["id"] in running_ids:
+ skipped_reasons["running"] += 1
+ continue
+ # Skip if feature has failed too many times
+ if self._failure_counts.get(fd["id"], 0) >= MAX_FEATURE_RETRIES:
+ skipped_reasons["failed"] += 1
+ continue
+ # Check dependencies (pass pre-computed passing_ids)
+ if are_dependencies_satisfied(fd, feature_dicts, passing_ids):
+ ready.append(fd)
+ else:
+ skipped_reasons["deps"] += 1
+
+ # Sort by scheduling score (higher = first), then priority, then id
+ if scheduling_scores is None:
+ scheduling_scores = compute_scheduling_scores(feature_dicts)
+ ready.sort(key=lambda f: (-scheduling_scores.get(f["id"], 0), f["priority"], f["id"]))
+
+ # Summary counts for logging
+ passing = skipped_reasons["passes"]
+ in_progress = skipped_reasons["in_progress"]
+ total = len(feature_dicts)
+
+ debug_log.log("READY", "get_ready_features() called",
+ ready_count=len(ready),
+ ready_ids=[f['id'] for f in ready[:5]], # First 5 only
+ passing=passing,
+ in_progress=in_progress,
+ total=total,
+ skipped=skipped_reasons)
+
+ return ready
+
+ def get_all_complete(self, feature_dicts: list[dict] | None = None) -> bool:
+ """Check if all features are complete or permanently failed.
+
+ Returns False if there are no features (initialization needed).
+
+ Args:
+ feature_dicts: Pre-fetched list of feature dicts. If None, queries the database.
+ """
+ if feature_dicts is None:
+ session = self.get_session()
+ try:
+ session.expire_all()
+ all_features = session.query(Feature).all()
+ feature_dicts = [f.to_dict() for f in all_features]
+ finally:
+ session.close()
+
+ # No features = NOT complete, need initialization
+ if len(feature_dicts) == 0:
+ return False
+
+ passing_count = 0
+ failed_count = 0
+ pending_count = 0
+ for fd in feature_dicts:
+ if fd.get("passes"):
+ passing_count += 1
+ continue # Completed successfully
+ if self._failure_counts.get(fd["id"], 0) >= MAX_FEATURE_RETRIES:
+ failed_count += 1
+ continue # Permanently failed, count as "done"
+ pending_count += 1
+
+ total = len(feature_dicts)
+ is_complete = pending_count == 0
+ debug_log.log("COMPLETE_CHECK", f"get_all_complete: {passing_count}/{total} passing, "
+ f"{failed_count} failed, {pending_count} pending -> {is_complete}")
+ return is_complete
+
+ def get_passing_count(self, feature_dicts: list[dict] | None = None) -> int:
+ """Get the number of passing features.
+
+ Args:
+ feature_dicts: Pre-fetched list of feature dicts. If None, queries the database.
+ """
+ if feature_dicts is None:
+ session = self.get_session()
+ try:
+ session.expire_all()
+ count: int = session.query(Feature).filter(Feature.passes == True).count()
+ return count
+ finally:
+ session.close()
+ return sum(1 for fd in feature_dicts if fd.get("passes"))
+
+ def _maintain_testing_agents(self, feature_dicts: list[dict] | None = None) -> None:
+ """Maintain the desired count of testing agents independently.
+
+ This runs every loop iteration and spawns testing agents as needed to maintain
+ the configured testing_agent_ratio. Testing agents run independently from
+ coding agents and continuously re-test passing features to catch regressions.
+
+ Multiple testing agents can test the same feature concurrently - this is
+ intentional and simplifies the architecture by removing claim coordination.
+
+ Stops spawning when:
+ - YOLO mode is enabled
+ - testing_agent_ratio is 0
+ - No passing features exist yet
+
+ Args:
+ feature_dicts: Pre-fetched list of feature dicts. If None, queries the database.
+ """
+ # Skip if testing is disabled
+ if self.yolo_mode or self.testing_agent_ratio == 0:
+ return
+
+ # No testing until there are passing features
+ passing_count = self.get_passing_count(feature_dicts)
+ if passing_count == 0:
+ return
+
+ # Don't spawn testing agents if all features are already complete
+ if self.get_all_complete(feature_dicts):
+ return
+
+ # Spawn testing agents one at a time, re-checking limits each time
+ # This avoids TOCTOU race by holding lock during the decision
+ while True:
+ # Check limits and decide whether to spawn (atomically)
+ with self._lock:
+ current_testing = len(self.running_testing_agents)
+ desired = self.testing_agent_ratio
+ total_agents = len(self.running_coding_agents) + current_testing
+
+ # Check if we need more testing agents
+ if current_testing >= desired:
+ return # Already at desired count
+
+ # Check hard limit on total agents
+ if total_agents >= MAX_TOTAL_AGENTS:
+ return # At max total agents
+
+ # We're going to spawn - log while still holding lock
+ spawn_index = current_testing + 1
+ debug_log.log("TESTING", f"Spawning testing agent ({spawn_index}/{desired})",
+ passing_count=passing_count)
+
+ # Spawn outside lock (I/O bound operation)
+ logger.debug("Spawning testing agent (%d/%d)", spawn_index, desired)
+ success, msg = self._spawn_testing_agent()
+ if not success:
+ debug_log.log("TESTING", f"Spawn failed, stopping: {msg}")
+ return
+
+ def start_feature(self, feature_id: int, resume: bool = False) -> tuple[bool, str]:
+ """Start a single coding agent for a feature.
+
+ Args:
+ feature_id: ID of the feature to start
+ resume: If True, resume a feature that's already in_progress from a previous session
+
+ Returns:
+ Tuple of (success, message)
+ """
+ with self._lock:
+ if feature_id in self.running_coding_agents:
+ return False, "Feature already running"
+ if len(self.running_coding_agents) >= self.max_concurrency:
+ return False, "At max concurrency"
+ # Enforce hard limit on total agents (coding + testing)
+ total_agents = len(self.running_coding_agents) + len(self.running_testing_agents)
+ if total_agents >= MAX_TOTAL_AGENTS:
+ return False, f"At max total agents ({total_agents}/{MAX_TOTAL_AGENTS})"
+
+ # Mark as in_progress in database (or verify it's resumable)
+ session = self.get_session()
+ try:
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if not feature:
+ return False, "Feature not found"
+ if feature.passes:
+ return False, "Feature already complete"
+
+ if resume:
+ # Resuming: feature should already be in_progress
+ if not feature.in_progress:
+ return False, "Feature not in progress, cannot resume"
+ else:
+ # Starting fresh: feature should not be in_progress
+ if feature.in_progress:
+ return False, "Feature already in progress"
+ feature.in_progress = True
+ session.commit()
+ finally:
+ session.close()
+
+ # Start coding agent subprocess
+ success, message = self._spawn_coding_agent(feature_id)
+ if not success:
+ return False, message
+
+ # NOTE: Testing agents are now maintained independently via _maintain_testing_agents()
+ # called in the main loop, rather than being spawned when coding agents start.
+
+ return True, f"Started feature {feature_id}"
+
+ def start_feature_batch(self, feature_ids: list[int], resume: bool = False) -> tuple[bool, str]:
+ """Start a coding agent for a batch of features.
+
+ Args:
+ feature_ids: List of feature IDs to implement in batch
+ resume: If True, resume features already in_progress
+
+ Returns:
+ Tuple of (success, message)
+ """
+ if not feature_ids:
+ return False, "No features to start"
+
+ # Single feature falls back to start_feature
+ if len(feature_ids) == 1:
+ return self.start_feature(feature_ids[0], resume=resume)
+
+ with self._lock:
+ # Check if any feature in batch is already running
+ for fid in feature_ids:
+ if fid in self.running_coding_agents or fid in self._feature_to_primary:
+ return False, f"Feature {fid} already running"
+ if len(self.running_coding_agents) >= self.max_concurrency:
+ return False, "At max concurrency"
+ total_agents = len(self.running_coding_agents) + len(self.running_testing_agents)
+ if total_agents >= MAX_TOTAL_AGENTS:
+ return False, f"At max total agents ({total_agents}/{MAX_TOTAL_AGENTS})"
+
+ # Mark all features as in_progress in a single transaction
+ session = self.get_session()
+ try:
+ features_to_mark = []
+ for fid in feature_ids:
+ feature = session.query(Feature).filter(Feature.id == fid).first()
+ if not feature:
+ return False, f"Feature {fid} not found"
+ if feature.passes:
+ return False, f"Feature {fid} already complete"
+ if not resume:
+ if feature.in_progress:
+ return False, f"Feature {fid} already in progress"
+ features_to_mark.append(feature)
+ else:
+ if not feature.in_progress:
+ return False, f"Feature {fid} not in progress, cannot resume"
+
+ for feature in features_to_mark:
+ feature.in_progress = True
+ session.commit()
+ finally:
+ session.close()
+
+ # Spawn batch coding agent
+ success, message = self._spawn_coding_agent_batch(feature_ids)
+ if not success:
+ # Clear in_progress on failure
+ session = self.get_session()
+ try:
+ for fid in feature_ids:
+ feature = session.query(Feature).filter(Feature.id == fid).first()
+ if feature and not resume:
+ feature.in_progress = False
+ session.commit()
+ finally:
+ session.close()
+ return False, message
+
+ return True, f"Started batch [{', '.join(str(fid) for fid in feature_ids)}]"
+
+ def _spawn_coding_agent(self, feature_id: int) -> tuple[bool, str]:
+ """Spawn a coding agent subprocess for a specific feature."""
+ # Create abort event
+ abort_event = threading.Event()
+
+ # Start subprocess for this feature
+ cmd = [
+ sys.executable,
+ "-u", # Force unbuffered stdout/stderr
+ str(AUTOFORGE_ROOT / "autonomous_agent_demo.py"),
+ "--project-dir", str(self.project_dir),
+ "--max-iterations", "1",
+ "--agent-type", "coding",
+ "--feature-id", str(feature_id),
+ ]
+ if self.model:
+ cmd.extend(["--model", self.model])
+ if self.yolo_mode:
+ cmd.append("--yolo")
+
+ try:
+ # CREATE_NO_WINDOW on Windows prevents console window pop-ups
+ # stdin=DEVNULL prevents blocking on stdin reads
+ # encoding="utf-8" and errors="replace" fix Windows CP1252 issues
+ popen_kwargs: dict[str, Any] = {
+ "stdin": subprocess.DEVNULL,
+ "stdout": subprocess.PIPE,
+ "stderr": subprocess.STDOUT,
+ "text": True,
+ "encoding": "utf-8",
+ "errors": "replace",
+ "cwd": str(self.project_dir), # Run from project dir so CLI creates .claude/ in project
+ "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{feature_id}"},
+ }
+ if sys.platform == "win32":
+ popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+
+ proc = subprocess.Popen(cmd, **popen_kwargs)
+ except Exception as e:
+ # Reset in_progress on failure
+ session = self.get_session()
+ try:
+ feature = session.query(Feature).filter(Feature.id == feature_id).first()
+ if feature:
+ feature.in_progress = False
+ session.commit()
+ finally:
+ session.close()
+ return False, f"Failed to start agent: {e}"
+
+ with self._lock:
+ self.running_coding_agents[feature_id] = proc
+ self.abort_events[feature_id] = abort_event
+
+ # Start output reader thread
+ threading.Thread(
+ target=self._read_output,
+ args=(feature_id, proc, abort_event, "coding"),
+ daemon=True
+ ).start()
+
+ if self.on_status is not None:
+ self.on_status(feature_id, "running")
+
+ print(f"Started coding agent for feature #{feature_id}", flush=True)
+ return True, f"Started feature {feature_id}"
+
+ def _spawn_coding_agent_batch(self, feature_ids: list[int]) -> tuple[bool, str]:
+ """Spawn a coding agent subprocess for a batch of features."""
+ primary_id = feature_ids[0]
+ abort_event = threading.Event()
+
+ cmd = [
+ sys.executable,
+ "-u",
+ str(AUTOFORGE_ROOT / "autonomous_agent_demo.py"),
+ "--project-dir", str(self.project_dir),
+ "--max-iterations", "1",
+ "--agent-type", "coding",
+ "--feature-ids", ",".join(str(fid) for fid in feature_ids),
+ ]
+ if self.model:
+ cmd.extend(["--model", self.model])
+ if self.yolo_mode:
+ cmd.append("--yolo")
+
+ try:
+ popen_kwargs: dict[str, Any] = {
+ "stdin": subprocess.DEVNULL,
+ "stdout": subprocess.PIPE,
+ "stderr": subprocess.STDOUT,
+ "text": True,
+ "encoding": "utf-8",
+ "errors": "replace",
+ "cwd": str(self.project_dir), # Run from project dir so CLI creates .claude/ in project
+ "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"coding-{primary_id}"},
+ }
+ if sys.platform == "win32":
+ popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+
+ proc = subprocess.Popen(cmd, **popen_kwargs)
+ except Exception as e:
+ # Reset in_progress on failure
+ session = self.get_session()
+ try:
+ for fid in feature_ids:
+ feature = session.query(Feature).filter(Feature.id == fid).first()
+ if feature:
+ feature.in_progress = False
+ session.commit()
+ finally:
+ session.close()
+ return False, f"Failed to start batch agent: {e}"
+
+ with self._lock:
+ self.running_coding_agents[primary_id] = proc
+ self.abort_events[primary_id] = abort_event
+ self._batch_features[primary_id] = list(feature_ids)
+ for fid in feature_ids:
+ self._feature_to_primary[fid] = primary_id
+
+ # Start output reader thread
+ threading.Thread(
+ target=self._read_output,
+ args=(primary_id, proc, abort_event, "coding"),
+ daemon=True
+ ).start()
+
+ if self.on_status is not None:
+ for fid in feature_ids:
+ self.on_status(fid, "running")
+
+ ids_str = ", ".join(f"#{fid}" for fid in feature_ids)
+ print(f"Started coding agent for features {ids_str}", flush=True)
+ return True, f"Started batch [{ids_str}]"
+
+ def _spawn_testing_agent(self) -> tuple[bool, str]:
+ """Spawn a testing agent subprocess for batch regression testing.
+
+ Selects a prioritized batch of passing features using weighted scoring
+ (via _get_test_batch) and passes them as --testing-feature-ids to the
+ subprocess. Falls back to single --testing-feature-id for batches of one.
+
+ Multiple testing agents can test the same feature concurrently - this is
+ intentional and simplifies the architecture by removing claim coordination.
+ """
+ # Check limits first (under lock)
+ with self._lock:
+ current_testing_count = len(self.running_testing_agents)
+ if current_testing_count >= self.max_concurrency:
+ debug_log.log("TESTING", f"Skipped spawn - at max testing agents ({current_testing_count}/{self.max_concurrency})")
+ return False, f"At max testing agents ({current_testing_count})"
+ total_agents = len(self.running_coding_agents) + len(self.running_testing_agents)
+ if total_agents >= MAX_TOTAL_AGENTS:
+ debug_log.log("TESTING", f"Skipped spawn - at max total agents ({total_agents}/{MAX_TOTAL_AGENTS})")
+ return False, f"At max total agents ({total_agents})"
+
+ # Select a weighted batch of passing features for regression testing
+ batch = self._get_test_batch(self.testing_batch_size)
+ if not batch:
+ debug_log.log("TESTING", "No features available for testing")
+ return False, "No features available for testing"
+
+ # Use the first feature ID as the representative for logging/tracking
+ primary_feature_id = batch[0]
+ batch_str = ",".join(str(fid) for fid in batch)
+ debug_log.log("TESTING", f"Selected batch for testing: [{batch_str}]")
+
+ # Spawn the testing agent
+ with self._lock:
+ # Re-check limits in case another thread spawned while we were selecting
+ current_testing_count = len(self.running_testing_agents)
+ if current_testing_count >= self.max_concurrency:
+ return False, f"At max testing agents ({current_testing_count})"
+
+ cmd = [
+ sys.executable,
+ "-u",
+ str(AUTOFORGE_ROOT / "autonomous_agent_demo.py"),
+ "--project-dir", str(self.project_dir),
+ "--max-iterations", "1",
+ "--agent-type", "testing",
+ "--testing-feature-ids", batch_str,
+ ]
+ if self.model:
+ cmd.extend(["--model", self.model])
+
+ try:
+ # CREATE_NO_WINDOW on Windows prevents console window pop-ups
+ # stdin=DEVNULL prevents blocking on stdin reads
+ # encoding="utf-8" and errors="replace" fix Windows CP1252 issues
+ popen_kwargs: dict[str, Any] = {
+ "stdin": subprocess.DEVNULL,
+ "stdout": subprocess.PIPE,
+ "stderr": subprocess.STDOUT,
+ "text": True,
+ "encoding": "utf-8",
+ "errors": "replace",
+ "cwd": str(self.project_dir), # Run from project dir so CLI creates .claude/ in project
+ "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": "", "PLAYWRIGHT_CLI_SESSION": f"testing-{self._testing_session_counter}"},
+ }
+ self._testing_session_counter += 1
+ if sys.platform == "win32":
+ popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+
+ proc = subprocess.Popen(cmd, **popen_kwargs)
+ except Exception as e:
+ debug_log.log("TESTING", f"FAILED to spawn testing agent: {e}")
+ return False, f"Failed to start testing agent: {e}"
+
+ # Register process by PID (not feature_id) to avoid overwrites
+ # when multiple agents test the same feature
+ self.running_testing_agents[proc.pid] = (primary_feature_id, proc)
+ testing_count = len(self.running_testing_agents)
+
+ # Start output reader thread with primary feature ID for log attribution
+ threading.Thread(
+ target=self._read_output,
+ args=(primary_feature_id, proc, threading.Event(), "testing"),
+ daemon=True
+ ).start()
+
+ print(f"Started testing agent for features [{batch_str}] (PID {proc.pid})", flush=True)
+ debug_log.log("TESTING", f"Successfully spawned testing agent for batch [{batch_str}]",
+ pid=proc.pid,
+ feature_ids=batch,
+ total_testing_agents=testing_count)
+ return True, f"Started testing agent for features [{batch_str}]"
+
+ async def _run_initializer(self) -> bool:
+ """Run initializer agent as blocking subprocess.
+
+ Returns True if initialization succeeded (features were created).
+ """
+ debug_log.section("INITIALIZER PHASE")
+ debug_log.log("INIT", "Starting initializer subprocess",
+ project_dir=str(self.project_dir))
+
+ cmd = [
+ sys.executable, "-u",
+ str(AUTOFORGE_ROOT / "autonomous_agent_demo.py"),
+ "--project-dir", str(self.project_dir),
+ "--agent-type", "initializer",
+ "--max-iterations", "1",
+ ]
+ if self.model:
+ cmd.extend(["--model", self.model])
+
+ print("Running initializer agent...", flush=True)
+
+ # CREATE_NO_WINDOW on Windows prevents console window pop-ups
+ # stdin=DEVNULL prevents blocking on stdin reads
+ # encoding="utf-8" and errors="replace" fix Windows CP1252 issues
+ popen_kwargs: dict[str, Any] = {
+ "stdin": subprocess.DEVNULL,
+ "stdout": subprocess.PIPE,
+ "stderr": subprocess.STDOUT,
+ "text": True,
+ "encoding": "utf-8",
+ "errors": "replace",
+ "cwd": str(AUTOFORGE_ROOT),
+ "env": {**os.environ, "PYTHONUNBUFFERED": "1", "NODE_COMPILE_CACHE": ""},
+ }
+ if sys.platform == "win32":
+ popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+
+ proc = subprocess.Popen(cmd, **popen_kwargs)
+
+ debug_log.log("INIT", "Initializer subprocess started", pid=proc.pid)
+
+ # Stream output with timeout
+ loop = asyncio.get_running_loop()
+ try:
+ async def stream_output():
+ while True:
+ line = await loop.run_in_executor(None, proc.stdout.readline)
+ if not line:
+ break
+ print(line.rstrip(), flush=True)
+ if self.on_output is not None:
+ self.on_output(0, line.rstrip()) # Use 0 as feature_id for initializer
+ proc.wait()
+
+ await asyncio.wait_for(stream_output(), timeout=INITIALIZER_TIMEOUT)
+
+ except asyncio.TimeoutError:
+ print(f"ERROR: Initializer timed out after {INITIALIZER_TIMEOUT // 60} minutes", flush=True)
+ debug_log.log("INIT", "TIMEOUT - Initializer exceeded time limit",
+ timeout_minutes=INITIALIZER_TIMEOUT // 60)
+ result = kill_process_tree(proc)
+ debug_log.log("INIT", "Killed timed-out initializer process tree",
+ status=result.status, children_found=result.children_found)
+ return False
+
+ debug_log.log("INIT", "Initializer subprocess completed",
+ return_code=proc.returncode,
+ success=proc.returncode == 0)
+
+ if proc.returncode != 0:
+ print(f"ERROR: Initializer failed with exit code {proc.returncode}", flush=True)
+ return False
+
+ return True
+
+ # Pattern to detect when a batch agent claims a new feature
+ _CLAIM_FEATURE_PATTERN = re.compile(
+ r"feature_claim_and_get\b.*?['\"]?feature_id['\"]?\s*[:=]\s*(\d+)"
+ )
+
+ def _read_output(
+ self,
+ feature_id: int | None,
+ proc: subprocess.Popen,
+ abort: threading.Event,
+ agent_type: Literal["coding", "testing"] = "coding",
+ ):
+ """Read output from subprocess and emit events."""
+ current_feature_id = feature_id
+ try:
+ if proc.stdout is None:
+ proc.wait()
+ return
+ for line in proc.stdout:
+ if abort.is_set():
+ break
+ line = line.rstrip()
+ # Detect when a batch agent claims a new feature
+ claim_match = self._CLAIM_FEATURE_PATTERN.search(line)
+ if claim_match:
+ claimed_id = int(claim_match.group(1))
+ if claimed_id != current_feature_id:
+ current_feature_id = claimed_id
+ if self.on_output is not None:
+ self.on_output(current_feature_id or 0, line)
+ else:
+ # Both coding and testing agents now use [Feature #X] format
+ print(f"[Feature #{current_feature_id}] {line}", flush=True)
+ proc.wait()
+ finally:
+ # CRITICAL: Kill the process tree to clean up any child processes (e.g., Claude CLI)
+ # This prevents zombie processes from accumulating
+ try:
+ kill_process_tree(proc, timeout=2.0)
+ except Exception as e:
+ debug_log.log("CLEANUP", f"Error killing process tree for {agent_type} agent", error=str(e))
+ self._on_agent_complete(feature_id, proc.returncode, agent_type, proc)
+
+ def _run_inter_session_cleanup(self):
+ """Run lightweight cleanup between agent sessions.
+
+ Removes stale temp files and project screenshots to prevent
+ disk space accumulation during long overnight runs.
+ """
+ try:
+ from temp_cleanup import cleanup_project_screenshots, cleanup_stale_temp
+ cleanup_stale_temp()
+ cleanup_project_screenshots(self.project_dir)
+ except Exception as e:
+ debug_log.log("CLEANUP", f"Inter-session cleanup failed (non-fatal): {e}")
+
+ def _signal_agent_completed(self):
+ """Signal that an agent has completed, waking the main loop.
+
+ This method is safe to call from any thread. It schedules the event.set()
+ call to run on the event loop thread to avoid cross-thread issues with
+ asyncio.Event.
+ """
+ if self._agent_completed_event is not None and self._event_loop is not None:
+ try:
+ # Use the stored event loop reference to schedule the set() call
+ # This is necessary because asyncio.Event is not thread-safe and
+ # asyncio.get_event_loop() fails in threads without an event loop
+ if self._event_loop.is_running():
+ self._event_loop.call_soon_threadsafe(self._agent_completed_event.set)
+ else:
+ # Fallback: set directly if loop isn't running (shouldn't happen during normal operation)
+ self._agent_completed_event.set()
+ except RuntimeError:
+ # Event loop closed, ignore (orchestrator may be shutting down)
+ pass
+
+ async def _wait_for_agent_completion(self, timeout: float = POLL_INTERVAL):
+ """Wait for an agent to complete or until timeout expires.
+
+ This replaces fixed `asyncio.sleep(POLL_INTERVAL)` calls with event-based
+ waiting. When an agent completes, _signal_agent_completed() sets the event,
+ causing this method to return immediately. If no agent completes within
+ the timeout, we return anyway to check for ready features.
+
+ Args:
+ timeout: Maximum seconds to wait (default: POLL_INTERVAL)
+ """
+ if self._agent_completed_event is None:
+ # Fallback if event not initialized (shouldn't happen in normal operation)
+ await asyncio.sleep(timeout)
+ return
+
+ try:
+ await asyncio.wait_for(self._agent_completed_event.wait(), timeout=timeout)
+ # Event was set - an agent completed. Clear it for the next wait cycle.
+ self._agent_completed_event.clear()
+ debug_log.log("EVENT", "Woke up immediately - agent completed")
+ except asyncio.TimeoutError:
+ # Timeout reached without agent completion - this is normal, just check anyway
+ pass
+
+ def _on_agent_complete(
+ self,
+ feature_id: int | None,
+ return_code: int,
+ agent_type: Literal["coding", "testing"],
+ proc: subprocess.Popen,
+ ):
+ """Handle agent completion.
+
+ For coding agents:
+ - ALWAYS clears in_progress when agent exits, regardless of success/failure.
+ - This prevents features from getting stuck if an agent crashes or is killed.
+ - The agent marks features as passing BEFORE clearing in_progress, so this
+ is safe.
+
+ For testing agents:
+ - Remove from running dict (no claim to release - concurrent testing is allowed).
+ """
+ if agent_type == "testing":
+ with self._lock:
+ # Remove by PID
+ self.running_testing_agents.pop(proc.pid, None)
+
+ status = "completed" if return_code == 0 else "failed"
+ print(f"Feature #{feature_id} testing {status}", flush=True)
+ debug_log.log("COMPLETE", f"Testing agent for feature #{feature_id} finished",
+ pid=proc.pid,
+ feature_id=feature_id,
+ status=status)
+ # Run lightweight cleanup between sessions
+ self._run_inter_session_cleanup()
+ # Signal main loop that an agent slot is available
+ self._signal_agent_completed()
+ return
+
+ # feature_id is required for coding agents (always passed from start_feature)
+ assert feature_id is not None, "feature_id must not be None for coding agents"
+
+ # Coding agent completion - handle both single and batch features
+ batch_ids = None
+ with self._lock:
+ batch_ids = self._batch_features.pop(feature_id, None)
+ if batch_ids:
+ # Clean up reverse mapping
+ for fid in batch_ids:
+ self._feature_to_primary.pop(fid, None)
+ self.running_coding_agents.pop(feature_id, None)
+ self.abort_events.pop(feature_id, None)
+
+ all_feature_ids = batch_ids or [feature_id]
+
+ debug_log.log("COMPLETE", f"Coding agent for feature(s) {all_feature_ids} finished",
+ return_code=return_code,
+ status="success" if return_code == 0 else "failed",
+ batch_size=len(all_feature_ids))
+
+ # Refresh session cache to see subprocess commits
+ session = self.get_session()
+ try:
+ session.expire_all()
+ for fid in all_feature_ids:
+ feature = session.query(Feature).filter(Feature.id == fid).first()
+ feature_passes = feature.passes if feature else None
+ feature_in_progress = feature.in_progress if feature else None
+ debug_log.log("DB", f"Feature #{fid} state after session.expire_all()",
+ passes=feature_passes,
+ in_progress=feature_in_progress)
+ if feature and feature.in_progress and not feature.passes:
+ feature.in_progress = False
+ session.commit()
+ debug_log.log("DB", f"Cleared in_progress for feature #{fid} (agent failed)")
+ finally:
+ session.close()
+
+ # Track failures for features still in_progress at exit
+ if return_code != 0:
+ with self._lock:
+ for fid in all_feature_ids:
+ self._failure_counts[fid] = self._failure_counts.get(fid, 0) + 1
+ failure_count = self._failure_counts[fid]
+ if failure_count >= MAX_FEATURE_RETRIES:
+ print(f"Feature #{fid} has failed {failure_count} times, will not retry", flush=True)
+ debug_log.log("COMPLETE", f"Feature #{fid} exceeded max retries",
+ failure_count=failure_count)
+
+ status = "completed" if return_code == 0 else "failed"
+ if self.on_status is not None:
+ for fid in all_feature_ids:
+ self.on_status(fid, status)
+
+ # CRITICAL: Print triggers WebSocket to emit agent_update
+ if batch_ids and len(batch_ids) > 1:
+ ids_str = ", ".join(f"#{fid}" for fid in batch_ids)
+ print(f"Features {ids_str} {status}", flush=True)
+ else:
+ print(f"Feature #{feature_id} {status}", flush=True)
+
+ # Run lightweight cleanup between sessions
+ self._run_inter_session_cleanup()
+ # Signal main loop that an agent slot is available
+ self._signal_agent_completed()
+
+ def stop_feature(self, feature_id: int) -> tuple[bool, str]:
+ """Stop a running coding agent and all its child processes."""
+ with self._lock:
+ # Check if this feature is part of a batch
+ primary_id = self._feature_to_primary.get(feature_id, feature_id)
+ if primary_id not in self.running_coding_agents:
+ return False, "Feature not running"
+
+ abort = self.abort_events.get(primary_id)
+ proc = self.running_coding_agents.get(primary_id)
+
+ if abort:
+ abort.set()
+ if proc:
+ result = kill_process_tree(proc, timeout=5.0)
+ debug_log.log("STOP", f"Killed feature {feature_id} (primary {primary_id}) process tree",
+ status=result.status, children_found=result.children_found,
+ children_terminated=result.children_terminated, children_killed=result.children_killed)
+
+ return True, f"Stopped feature {feature_id}"
+
+ def stop_all(self) -> None:
+ """Stop all running agents (coding and testing)."""
+ self.is_running = False
+
+ # Stop coding agents
+ with self._lock:
+ feature_ids = list(self.running_coding_agents.keys())
+
+ for fid in feature_ids:
+ self.stop_feature(fid)
+
+ # Stop testing agents (no claim to release - concurrent testing is allowed)
+ with self._lock:
+ testing_items = list(self.running_testing_agents.items())
+
+ for pid, (feature_id, proc) in testing_items:
+ result = kill_process_tree(proc, timeout=5.0)
+ debug_log.log("STOP", f"Killed testing agent for feature #{feature_id} (PID {pid})",
+ status=result.status, children_found=result.children_found,
+ children_terminated=result.children_terminated, children_killed=result.children_killed)
+
+ # Clear dict so get_status() doesn't report stale agents while
+ # _on_agent_complete callbacks are still in flight.
+ with self._lock:
+ self.running_testing_agents.clear()
+
+ async def run_loop(self):
+ """Main orchestration loop."""
+ self.is_running = True
+
+ # Initialize the agent completion event for this run
+ # Must be created in the async context where it will be used
+ self._agent_completed_event = asyncio.Event()
+ # Store the event loop reference for thread-safe signaling from output reader threads
+ self._event_loop = asyncio.get_running_loop()
+
+ # Track session start for regression testing (UTC for consistency with last_tested_at)
+ self.session_start_time = datetime.now(timezone.utc)
+
+ # Start debug logging session FIRST (clears previous logs)
+ # Must happen before any debug_log.log() calls
+ debug_log.start_session()
+
+ # Clear any stale drain signal from a previous session
+ self._clear_drain_signal()
+
+ # Log startup to debug file
+ debug_log.section("ORCHESTRATOR STARTUP")
+ debug_log.log("STARTUP", "Orchestrator run_loop starting",
+ project_dir=str(self.project_dir),
+ max_concurrency=self.max_concurrency,
+ yolo_mode=self.yolo_mode,
+ testing_agent_ratio=self.testing_agent_ratio,
+ session_start_time=self.session_start_time.isoformat())
+
+ print("=" * 70, flush=True)
+ print(" UNIFIED ORCHESTRATOR SETTINGS", flush=True)
+ print("=" * 70, flush=True)
+ print(f"Project: {self.project_dir}", flush=True)
+ print(f"Max concurrency: {self.max_concurrency} coding agents", flush=True)
+ print(f"YOLO mode: {self.yolo_mode}", flush=True)
+ print(f"Regression agents: {self.testing_agent_ratio} (maintained independently)", flush=True)
+ print(f"Batch size: {self.batch_size} features per agent", flush=True)
+ print("=" * 70, flush=True)
+ print(flush=True)
+
+ # Phase 1: Check if initialization needed
+ if not has_features(self.project_dir):
+ print("=" * 70, flush=True)
+ print(" INITIALIZATION PHASE", flush=True)
+ print("=" * 70, flush=True)
+ print("No features found - running initializer agent first...", flush=True)
+ print("NOTE: This may take 10-20+ minutes to generate features.", flush=True)
+ print(flush=True)
+
+ success = await self._run_initializer()
+
+ if not success or not has_features(self.project_dir):
+ print("ERROR: Initializer did not create features. Exiting.", flush=True)
+ return
+
+ print(flush=True)
+ print("=" * 70, flush=True)
+ print(" INITIALIZATION COMPLETE - Starting feature loop", flush=True)
+ print("=" * 70, flush=True)
+ print(flush=True)
+
+ # CRITICAL: Recreate database connection after initializer subprocess commits
+ # The initializer runs as a subprocess and commits to the database file.
+ # SQLAlchemy may have stale connections or cached state. Disposing the old
+ # engine and creating a fresh engine/session_maker ensures we see all the
+ # newly created features.
+ debug_log.section("INITIALIZATION COMPLETE")
+ debug_log.log("INIT", "Disposing old database engine and creating fresh connection")
+ logger.debug("Recreating database connection after initialization")
+ if self._engine is not None:
+ self._engine.dispose()
+ self._engine, self._session_maker = create_database(self.project_dir)
+
+ # Debug: Show state immediately after initialization
+ logger.debug("Post-initialization state check")
+ logger.debug("Post-initialization state: max_concurrency=%d, yolo_mode=%s, testing_agent_ratio=%d",
+ self.max_concurrency, self.yolo_mode, self.testing_agent_ratio)
+
+ # Verify features were created and are visible
+ session = self.get_session()
+ try:
+ feature_count = session.query(Feature).count()
+ all_features = session.query(Feature).all()
+ feature_names = [f"{f.id}: {f.name}" for f in all_features[:10]]
+ logger.debug("Features in database: %d", feature_count)
+ debug_log.log("INIT", "Post-initialization database state",
+ max_concurrency=self.max_concurrency,
+ yolo_mode=self.yolo_mode,
+ testing_agent_ratio=self.testing_agent_ratio,
+ feature_count=feature_count,
+ first_10_features=feature_names)
+ finally:
+ session.close()
+
+ # Phase 2: Feature loop
+ # Check for features to resume from previous session
+ resumable = self.get_resumable_features()
+ if resumable:
+ print(f"Found {len(resumable)} feature(s) to resume from previous session:", flush=True)
+ for f in resumable:
+ print(f" - Feature #{f['id']}: {f['name']}", flush=True)
+ print(flush=True)
+
+ debug_log.section("FEATURE LOOP STARTING")
+ loop_iteration = 0
+ while self.is_running and not self._shutdown_requested:
+ loop_iteration += 1
+ if loop_iteration <= 3:
+ logger.debug("=== Loop iteration %d ===", loop_iteration)
+
+ # Query all features ONCE per iteration and build reusable snapshot.
+ # Every sub-method receives this snapshot instead of re-querying the DB.
+ session = self.get_session()
+ session.expire_all()
+ all_features = session.query(Feature).all()
+ feature_dicts = [f.to_dict() for f in all_features]
+ session.close()
+
+ # Pre-compute scheduling scores once (BFS + reverse topo sort)
+ scheduling_scores = compute_scheduling_scores(feature_dicts)
+
+ # Log every iteration to debug file (first 10, then every 5th)
+ if loop_iteration <= 10 or loop_iteration % 5 == 0:
+ with self._lock:
+ running_ids = list(self.running_coding_agents.keys())
+ testing_count = len(self.running_testing_agents)
+ debug_log.log("LOOP", f"Iteration {loop_iteration}",
+ running_coding_agents=running_ids,
+ running_testing_agents=testing_count,
+ max_concurrency=self.max_concurrency)
+
+ # Full database dump every 5 iterations
+ if loop_iteration == 1 or loop_iteration % 5 == 0:
+ _dump_database_state(feature_dicts, f"(iteration {loop_iteration})")
+
+ try:
+ # Check if all complete
+ if self.get_all_complete(feature_dicts):
+ print("\nAll features complete!", flush=True)
+ break
+
+ # --- Graceful pause (drain mode) ---
+ if not self._drain_requested and self._check_drain_signal():
+ self._drain_requested = True
+ print("Graceful pause requested - draining running agents...", flush=True)
+ debug_log.log("DRAIN", "Graceful pause requested, draining running agents")
+
+ if self._drain_requested:
+ with self._lock:
+ coding_count = len(self.running_coding_agents)
+ testing_count = len(self.running_testing_agents)
+
+ if coding_count == 0 and testing_count == 0:
+ print("All agents drained - paused.", flush=True)
+ debug_log.log("DRAIN", "All agents drained, entering paused state")
+ # Wait until signal file is removed (resume) or shutdown
+ while self._check_drain_signal() and self.is_running and not self._shutdown_requested:
+ await asyncio.sleep(1)
+ if not self.is_running or self._shutdown_requested:
+ break
+ self._drain_requested = False
+ print("Resuming from graceful pause...", flush=True)
+ debug_log.log("DRAIN", "Resuming from graceful pause")
+ continue
+ else:
+ debug_log.log("DRAIN", f"Waiting for agents to finish: coding={coding_count}, testing={testing_count}")
+ await self._wait_for_agent_completion()
+ continue
+
+ # Maintain testing agents independently (runs every iteration)
+ self._maintain_testing_agents(feature_dicts)
+
+ # Check capacity
+ with self._lock:
+ current = len(self.running_coding_agents)
+ current_testing = len(self.running_testing_agents)
+ running_ids = list(self.running_coding_agents.keys())
+
+ debug_log.log("CAPACITY", "Checking capacity",
+ current_coding=current,
+ current_testing=current_testing,
+ running_coding_ids=running_ids,
+ max_concurrency=self.max_concurrency,
+ at_capacity=(current >= self.max_concurrency))
+
+ if current >= self.max_concurrency:
+ debug_log.log("CAPACITY", "At max capacity, waiting for agent completion...")
+ await self._wait_for_agent_completion()
+ continue
+
+ # Priority 1: Resume features from previous session
+ resumable = self.get_resumable_features(feature_dicts, scheduling_scores)
+ if resumable:
+ slots = self.max_concurrency - current
+ for feature in resumable[:slots]:
+ print(f"Resuming feature #{feature['id']}: {feature['name']}", flush=True)
+ self.start_feature(feature["id"], resume=True)
+ await asyncio.sleep(0.5) # Brief delay for subprocess to claim feature before re-querying
+ continue
+
+ # Priority 2: Start new ready features
+ ready = self.get_ready_features(feature_dicts, scheduling_scores)
+ if not ready:
+ # Wait for running features to complete
+ if current > 0:
+ await self._wait_for_agent_completion()
+ continue
+ else:
+ # No ready features and nothing running
+ # Force a fresh database check before declaring blocked
+ # This handles the case where subprocess commits weren't visible yet
+ session = self.get_session()
+ try:
+ session.expire_all()
+ fresh_dicts = [f.to_dict() for f in session.query(Feature).all()]
+ finally:
+ session.close()
+
+ # Recheck if all features are now complete
+ if self.get_all_complete(fresh_dicts):
+ print("\nAll features complete!", flush=True)
+ break
+
+ # Still have pending features but all are blocked by dependencies
+ print("No ready features available. All remaining features may be blocked by dependencies.", flush=True)
+ await self._wait_for_agent_completion(timeout=POLL_INTERVAL * 2)
+ continue
+
+ # Build dependency-aware batches from ready features
+ slots = self.max_concurrency - current
+ batches = self.build_feature_batches(ready, feature_dicts, scheduling_scores)
+
+ logger.debug("Spawning loop: %d ready, %d slots available, %d batches built",
+ len(ready), slots, len(batches))
+
+ debug_log.log("SPAWN", "Starting feature batches",
+ ready_count=len(ready),
+ slots_available=slots,
+ batch_count=len(batches),
+ batches=[[f['id'] for f in b] for b in batches[:slots]])
+
+ for batch in batches[:slots]:
+ batch_ids = [f["id"] for f in batch]
+ batch_names = [f"{f['id']}:{f['name']}" for f in batch]
+ logger.debug("Starting batch: %s", batch_ids)
+ success, msg = self.start_feature_batch(batch_ids)
+ if not success:
+ logger.debug("Failed to start batch %s: %s", batch_ids, msg)
+ debug_log.log("SPAWN", f"FAILED to start batch {batch_ids}",
+ batch_names=batch_names,
+ error=msg)
+ else:
+ logger.debug("Successfully started batch %s", batch_ids)
+ with self._lock:
+ running_count = len(self.running_coding_agents)
+ logger.debug("Running coding agents after start: %d", running_count)
+ debug_log.log("SPAWN", f"Successfully started batch {batch_ids}",
+ batch_names=batch_names,
+ running_coding_agents=running_count)
+
+ await asyncio.sleep(0.5)
+
+ except Exception as e:
+ print(f"Orchestrator error: {e}", flush=True)
+ await self._wait_for_agent_completion()
+
+ # Wait for remaining agents to complete
+ print("Waiting for running agents to complete...", flush=True)
+ while True:
+ with self._lock:
+ coding_done = len(self.running_coding_agents) == 0
+ testing_done = len(self.running_testing_agents) == 0
+ if coding_done and testing_done:
+ break
+ # Use short timeout since we're just waiting for final agents to finish
+ await self._wait_for_agent_completion(timeout=1.0)
+
+ print("Orchestrator finished.", flush=True)
+
+ def get_status(self) -> dict:
+ """Get current orchestrator status."""
+ with self._lock:
+ return {
+ "running_features": list(self.running_coding_agents.keys()),
+ "coding_agent_count": len(self.running_coding_agents),
+ "testing_agent_count": len(self.running_testing_agents),
+ "count": len(self.running_coding_agents), # Legacy compatibility
+ "max_concurrency": self.max_concurrency,
+ "testing_agent_ratio": self.testing_agent_ratio,
+ "is_running": self.is_running,
+ "yolo_mode": self.yolo_mode,
+ }
+
+ def _check_drain_signal(self) -> bool:
+ """Check if the graceful pause (drain) signal file exists."""
+ from autoforge_paths import get_pause_drain_path
+ return get_pause_drain_path(self.project_dir).exists()
+
+ def _clear_drain_signal(self) -> None:
+ """Delete the drain signal file and reset the flag."""
+ from autoforge_paths import get_pause_drain_path
+ get_pause_drain_path(self.project_dir).unlink(missing_ok=True)
+ self._drain_requested = False
+
+ def cleanup(self) -> None:
+ """Clean up database resources. Safe to call multiple times.
+
+ Forces WAL checkpoint to flush pending writes to main database file,
+ then disposes engine to close all connections. Prevents stale cache
+ issues when the orchestrator restarts.
+ """
+ # Atomically grab and clear the engine reference to prevent re-entry
+ engine = self._engine
+ self._engine = None
+
+ if engine is None:
+ return # Already cleaned up
+
+ try:
+ debug_log.log("CLEANUP", "Forcing WAL checkpoint before dispose")
+ with engine.connect() as conn:
+ conn.execute(text("PRAGMA wal_checkpoint(FULL)"))
+ conn.commit()
+ debug_log.log("CLEANUP", "WAL checkpoint completed, disposing engine")
+ except Exception as e:
+ debug_log.log("CLEANUP", f"WAL checkpoint failed (non-fatal): {e}")
+
+ try:
+ engine.dispose()
+ debug_log.log("CLEANUP", "Engine disposed successfully")
+ except Exception as e:
+ debug_log.log("CLEANUP", f"Engine dispose failed: {e}")
+
+
+async def run_parallel_orchestrator(
+ project_dir: Path,
+ max_concurrency: int = DEFAULT_CONCURRENCY,
+ model: str | None = None,
+ yolo_mode: bool = False,
+ testing_agent_ratio: int = 1,
+ testing_batch_size: int = DEFAULT_TESTING_BATCH_SIZE,
+ batch_size: int = 3,
+) -> None:
+ """Run the unified orchestrator.
+
+ Args:
+ project_dir: Path to the project directory
+ max_concurrency: Maximum number of concurrent coding agents
+ model: Claude model to use
+ yolo_mode: Whether to run in YOLO mode (skip testing agents)
+ testing_agent_ratio: Number of regression agents to maintain (0-3)
+ testing_batch_size: Number of features per testing batch (1-5)
+ batch_size: Max features per coding agent batch (1-3)
+ """
+ print(f"[ORCHESTRATOR] run_parallel_orchestrator called with max_concurrency={max_concurrency}", flush=True)
+ orchestrator = ParallelOrchestrator(
+ project_dir=project_dir,
+ max_concurrency=max_concurrency,
+ model=model,
+ yolo_mode=yolo_mode,
+ testing_agent_ratio=testing_agent_ratio,
+ testing_batch_size=testing_batch_size,
+ batch_size=batch_size,
+ )
+
+ # Set up cleanup to run on exit (handles normal exit, exceptions)
+ def cleanup_handler():
+ debug_log.log("CLEANUP", "atexit cleanup handler invoked")
+ orchestrator.cleanup()
+
+ atexit.register(cleanup_handler)
+
+ # Set up async-safe signal handler for graceful shutdown
+ # Only sets flags - everything else is unsafe in signal context
+ def signal_handler(signum, frame):
+ orchestrator._shutdown_requested = True
+ orchestrator.is_running = False
+
+ # Register SIGTERM handler for process termination signals
+ # Note: On Windows, SIGTERM handlers only fire from os.kill() calls within Python.
+ # External termination (Task Manager, taskkill, Popen.terminate()) uses
+ # TerminateProcess() which bypasses signal handlers entirely.
+ signal.signal(signal.SIGTERM, signal_handler)
+
+ # Note: We intentionally do NOT register SIGINT handler
+ # Let Python raise KeyboardInterrupt naturally so the except block works
+
+ try:
+ await orchestrator.run_loop()
+ except KeyboardInterrupt:
+ print("\n\nInterrupted by user. Stopping agents...", flush=True)
+ orchestrator.stop_all()
+ finally:
+ # CRITICAL: Always clean up database resources on exit
+ # This forces WAL checkpoint and disposes connections
+ orchestrator.cleanup()
+
+
+def main():
+ """Main entry point for parallel orchestration."""
+ import argparse
+
+ from dotenv import load_dotenv
+
+ from registry import DEFAULT_MODEL, get_project_path
+
+ load_dotenv()
+
+ parser = argparse.ArgumentParser(
+ description="Parallel Feature Orchestrator - Run multiple agent instances",
+ )
+ parser.add_argument(
+ "--project-dir",
+ type=str,
+ required=True,
+ help="Project directory path (absolute) or registered project name",
+ )
+ parser.add_argument(
+ "--max-concurrency",
+ "-p",
+ type=int,
+ default=DEFAULT_CONCURRENCY,
+ help=f"Maximum concurrent agents (1-{MAX_PARALLEL_AGENTS}, default: {DEFAULT_CONCURRENCY})",
+ )
+ parser.add_argument(
+ "--model",
+ type=str,
+ default=DEFAULT_MODEL,
+ help=f"Claude model to use (default: {DEFAULT_MODEL})",
+ )
+ parser.add_argument(
+ "--yolo",
+ action="store_true",
+ default=False,
+ help="Enable YOLO mode: rapid prototyping without browser testing",
+ )
+ parser.add_argument(
+ "--testing-agent-ratio",
+ type=int,
+ default=1,
+ help="Number of regression testing agents (0-3, default: 1). Set to 0 to disable testing agents.",
+ )
+ parser.add_argument(
+ "--testing-batch-size",
+ type=int,
+ default=DEFAULT_TESTING_BATCH_SIZE,
+ help=f"Number of features per testing batch (1-5, default: {DEFAULT_TESTING_BATCH_SIZE})",
+ )
+ parser.add_argument(
+ "--batch-size",
+ type=int,
+ default=3,
+ help="Max features per coding agent batch (1-5, default: 3)",
+ )
+
+ args = parser.parse_args()
+
+ # Resolve project directory
+ project_dir_input = args.project_dir
+ project_dir = Path(project_dir_input)
+
+ if project_dir.is_absolute():
+ if not project_dir.exists():
+ print(f"Error: Project directory does not exist: {project_dir}", flush=True)
+ sys.exit(1)
+ else:
+ registered_path = get_project_path(project_dir_input)
+ if registered_path:
+ project_dir = registered_path
+ else:
+ print(f"Error: Project '{project_dir_input}' not found in registry", flush=True)
+ sys.exit(1)
+
+ try:
+ asyncio.run(run_parallel_orchestrator(
+ project_dir=project_dir,
+ max_concurrency=args.max_concurrency,
+ model=args.model,
+ yolo_mode=args.yolo,
+ testing_agent_ratio=args.testing_agent_ratio,
+ testing_batch_size=args.testing_batch_size,
+ batch_size=args.batch_size,
+ ))
+ except KeyboardInterrupt:
+ print("\n\nInterrupted by user", flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/progress.py b/progress.py
index dfb700b44..e2d847e34 100644
--- a/progress.py
+++ b/progress.py
@@ -10,12 +10,21 @@
import os
import sqlite3
import urllib.request
-from datetime import datetime
+from contextlib import closing
+from datetime import datetime, timezone
from pathlib import Path
WEBHOOK_URL = os.environ.get("PROGRESS_N8N_WEBHOOK_URL")
PROGRESS_CACHE_FILE = ".progress_cache"
+# SQLite connection settings for parallel mode safety
+SQLITE_TIMEOUT = 30 # seconds to wait for locks
+
+
+def _get_connection(db_file: Path) -> sqlite3.Connection:
+ """Get a SQLite connection with proper timeout settings for parallel mode."""
+ return sqlite3.connect(db_file, timeout=SQLITE_TIMEOUT)
+
def has_features(project_dir: Path) -> bool:
"""
@@ -31,62 +40,93 @@ def has_features(project_dir: Path) -> bool:
Returns False if no features exist (initializer needs to run).
"""
- import sqlite3
-
# Check legacy JSON file first
json_file = project_dir / "feature_list.json"
if json_file.exists():
return True
# Check SQLite database
- db_file = project_dir / "features.db"
+ from autoforge_paths import get_features_db_path
+ db_file = get_features_db_path(project_dir)
if not db_file.exists():
return False
try:
- conn = sqlite3.connect(db_file)
- cursor = conn.cursor()
- cursor.execute("SELECT COUNT(*) FROM features")
- count = cursor.fetchone()[0]
- conn.close()
- return count > 0
+ with closing(_get_connection(db_file)) as conn:
+ cursor = conn.cursor()
+ cursor.execute("SELECT COUNT(*) FROM features")
+ count: int = cursor.fetchone()[0]
+ return bool(count > 0)
except Exception:
# Database exists but can't be read or has no features table
return False
-def count_passing_tests(project_dir: Path) -> tuple[int, int, int]:
+def count_passing_tests(project_dir: Path) -> tuple[int, int, int, int]:
"""
- Count passing, in_progress, and total tests via direct database access.
+ Count passing, in_progress, total, and needs_human_input tests via direct database access.
Args:
project_dir: Directory containing the project
Returns:
- (passing_count, in_progress_count, total_count)
+ (passing_count, in_progress_count, total_count, needs_human_input_count)
"""
- db_file = project_dir / "features.db"
+ from autoforge_paths import get_features_db_path
+ db_file = get_features_db_path(project_dir)
if not db_file.exists():
- return 0, 0, 0
+ return 0, 0, 0, 0
try:
- conn = sqlite3.connect(db_file)
- cursor = conn.cursor()
- cursor.execute("SELECT COUNT(*) FROM features")
- total = cursor.fetchone()[0]
- cursor.execute("SELECT COUNT(*) FROM features WHERE passes = 1")
- passing = cursor.fetchone()[0]
- # Handle case where in_progress column doesn't exist yet
- try:
- cursor.execute("SELECT COUNT(*) FROM features WHERE in_progress = 1")
- in_progress = cursor.fetchone()[0]
- except sqlite3.OperationalError:
- in_progress = 0
- conn.close()
- return passing, in_progress, total
+ with closing(_get_connection(db_file)) as conn:
+ cursor = conn.cursor()
+ # Single aggregate query instead of separate COUNT queries
+ # Handle case where columns don't exist yet (legacy DBs)
+ try:
+ cursor.execute("""
+ SELECT
+ COUNT(*) as total,
+ SUM(CASE WHEN passes = 1 THEN 1 ELSE 0 END) as passing,
+ SUM(CASE WHEN in_progress = 1 THEN 1 ELSE 0 END) as in_progress,
+ SUM(CASE WHEN needs_human_input = 1 THEN 1 ELSE 0 END) as needs_human_input
+ FROM features
+ """)
+ row = cursor.fetchone()
+ total = row[0] or 0
+ passing = row[1] or 0
+ in_progress = row[2] or 0
+ needs_human_input = row[3] or 0
+ except sqlite3.OperationalError:
+ # Fallback for databases without newer columns
+ try:
+ cursor.execute("""
+ SELECT
+ COUNT(*) as total,
+ SUM(CASE WHEN passes = 1 THEN 1 ELSE 0 END) as passing,
+ SUM(CASE WHEN in_progress = 1 THEN 1 ELSE 0 END) as in_progress
+ FROM features
+ """)
+ row = cursor.fetchone()
+ total = row[0] or 0
+ passing = row[1] or 0
+ in_progress = row[2] or 0
+ needs_human_input = 0
+ except sqlite3.OperationalError:
+ cursor.execute("""
+ SELECT
+ COUNT(*) as total,
+ SUM(CASE WHEN passes = 1 THEN 1 ELSE 0 END) as passing
+ FROM features
+ """)
+ row = cursor.fetchone()
+ total = row[0] or 0
+ passing = row[1] or 0
+ in_progress = 0
+ needs_human_input = 0
+ return passing, in_progress, total, needs_human_input
except Exception as e:
print(f"[Database error in count_passing_tests: {e}]")
- return 0, 0, 0
+ return 0, 0, 0, 0
def get_all_passing_features(project_dir: Path) -> list[dict]:
@@ -99,22 +139,22 @@ def get_all_passing_features(project_dir: Path) -> list[dict]:
Returns:
List of dicts with id, category, name for each passing feature
"""
- db_file = project_dir / "features.db"
+ from autoforge_paths import get_features_db_path
+ db_file = get_features_db_path(project_dir)
if not db_file.exists():
return []
try:
- conn = sqlite3.connect(db_file)
- cursor = conn.cursor()
- cursor.execute(
- "SELECT id, category, name FROM features WHERE passes = 1 ORDER BY priority ASC"
- )
- features = [
- {"id": row[0], "category": row[1], "name": row[2]}
- for row in cursor.fetchall()
- ]
- conn.close()
- return features
+ with closing(_get_connection(db_file)) as conn:
+ cursor = conn.cursor()
+ cursor.execute(
+ "SELECT id, category, name FROM features WHERE passes = 1 ORDER BY priority ASC"
+ )
+ features = [
+ {"id": row[0], "category": row[1], "name": row[2]}
+ for row in cursor.fetchall()
+ ]
+ return features
except Exception:
return []
@@ -124,7 +164,8 @@ def send_progress_webhook(passing: int, total: int, project_dir: Path) -> None:
if not WEBHOOK_URL:
return # Webhook not configured
- cache_file = project_dir / PROGRESS_CACHE_FILE
+ from autoforge_paths import get_progress_cache_path
+ cache_file = get_progress_cache_path(project_dir)
previous = 0
previous_passing_ids = set()
@@ -171,7 +212,7 @@ def send_progress_webhook(passing: int, total: int, project_dir: Path) -> None:
"tests_completed_this_session": passing - previous,
"completed_tests": completed_tests,
"project": project_dir.name,
- "timestamp": datetime.utcnow().isoformat() + "Z",
+ "timestamp": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
}
try:
@@ -210,7 +251,7 @@ def print_session_header(session_num: int, is_initializer: bool) -> None:
def print_progress_summary(project_dir: Path) -> None:
"""Print a summary of current progress."""
- passing, in_progress, total = count_passing_tests(project_dir)
+ passing, in_progress, total, _needs_human_input = count_passing_tests(project_dir)
if total > 0:
percentage = (passing / total) * 100
diff --git a/prompts.py b/prompts.py
index 0fc403b53..6031080d2 100644
--- a/prompts.py
+++ b/prompts.py
@@ -9,16 +9,21 @@
2. Base template: .claude/templates/{name}.template.md
"""
+import re
import shutil
from pathlib import Path
# Base templates location (generic templates)
TEMPLATES_DIR = Path(__file__).parent / ".claude" / "templates"
+# Migration version — bump when adding new migration steps
+CURRENT_MIGRATION_VERSION = 1
+
def get_project_prompts_dir(project_dir: Path) -> Path:
"""Get the prompts directory for a specific project."""
- return project_dir / "prompts"
+ from autoforge_paths import get_prompts_dir
+ return get_prompts_dir(project_dir)
def load_prompt(name: str, project_dir: Path | None = None) -> str:
@@ -69,14 +74,220 @@ def get_initializer_prompt(project_dir: Path | None = None) -> str:
return load_prompt("initializer_prompt", project_dir)
-def get_coding_prompt(project_dir: Path | None = None) -> str:
- """Load the coding agent prompt (project-specific if available)."""
- return load_prompt("coding_prompt", project_dir)
+def _strip_browser_testing_sections(prompt: str) -> str:
+ """Strip browser automation and Playwright testing instructions from prompt.
+
+ Used in YOLO mode where browser testing is skipped entirely. Replaces
+ browser-related sections with a brief YOLO-mode note while preserving
+ all non-testing instructions (implementation, git, progress notes, etc.).
+
+ Args:
+ prompt: The full coding prompt text.
+
+ Returns:
+ The prompt with browser testing sections replaced by YOLO guidance.
+ """
+ original_prompt = prompt
+
+ # Replace STEP 5 (browser automation verification) with YOLO note
+ prompt = re.sub(
+ r"### STEP 5: VERIFY WITH BROWSER AUTOMATION.*?(?=### STEP 5\.5:)",
+ "### STEP 5: VERIFY FEATURE (YOLO MODE)\n\n"
+ "**YOLO mode is active.** Skip browser automation testing. "
+ "Instead, verify your feature works by ensuring:\n"
+ "- Code compiles without errors (lint and type-check pass)\n"
+ "- Server starts without errors after your changes\n"
+ "- No obvious runtime errors in server logs\n\n",
+ prompt,
+ flags=re.DOTALL,
+ )
+
+ # Replace the marking rule with YOLO-appropriate wording
+ prompt = prompt.replace(
+ "**ONLY MARK A FEATURE AS PASSING AFTER VERIFICATION WITH BROWSER AUTOMATION.**",
+ "**YOLO mode: Mark a feature as passing after lint/type-check succeeds and server starts cleanly.**",
+ )
+
+ # Replace the BROWSER AUTOMATION reference section
+ prompt = re.sub(
+ r"## BROWSER AUTOMATION\n\n.*?(?=---)",
+ "## VERIFICATION (YOLO MODE)\n\n"
+ "Browser automation is disabled in YOLO mode. "
+ "Verify features by running lint, type-check, and confirming the dev server starts without errors.\n\n",
+ prompt,
+ flags=re.DOTALL,
+ )
+
+ # In STEP 4, replace browser automation reference with YOLO guidance
+ prompt = prompt.replace(
+ "2. Test manually using browser automation (see Step 5)",
+ "2. Verify code compiles (lint and type-check pass)",
+ )
+
+ if prompt == original_prompt:
+ print("[YOLO] Warning: No browser testing sections found to strip. "
+ "Project-specific prompt may need manual YOLO adaptation.")
+
+ return prompt
+
+
+def get_coding_prompt(project_dir: Path | None = None, yolo_mode: bool = False) -> str:
+ """Load the coding agent prompt (project-specific if available).
+
+ Args:
+ project_dir: Optional project directory for project-specific prompts
+ yolo_mode: If True, strip browser automation / Playwright testing
+ instructions and replace with YOLO-mode guidance. This reduces
+ prompt tokens since YOLO mode skips all browser testing anyway.
+
+ Returns:
+ The coding prompt, optionally stripped of testing instructions.
+ """
+ prompt = load_prompt("coding_prompt", project_dir)
+
+ if yolo_mode:
+ prompt = _strip_browser_testing_sections(prompt)
+
+ return prompt
+
+
+def get_auto_improve_prompt(project_dir: Path | None = None, yolo_mode: bool = False) -> str:
+ """Load the auto-improve agent prompt (project-specific if available).
+
+ The auto-improve prompt instructs the agent to analyze an already-finished
+ project, pick ONE meaningful improvement, create a feature on the Kanban,
+ implement it, verify with lint/typecheck/build, mark passing, and commit.
+
+ Args:
+ project_dir: Optional project directory for project-specific prompts
+ yolo_mode: If True, strip browser automation sections for YOLO-mode
+ token savings. Browser verification is already optional in
+ auto-improve mode, so this is a small adjustment.
+
+ Returns:
+ The auto-improve prompt, optionally stripped of browser testing.
+ """
+ prompt = load_prompt("auto_improve_prompt", project_dir)
+
+ if yolo_mode:
+ prompt = _strip_browser_testing_sections(prompt)
+
+ return prompt
+
+
+def get_testing_prompt(
+ project_dir: Path | None = None,
+ testing_feature_id: int | None = None,
+ testing_feature_ids: list[int] | None = None,
+) -> str:
+ """Load the testing agent prompt (project-specific if available).
+
+ Supports both single-feature and multi-feature testing modes. When
+ testing_feature_ids is provided, the template's {{TESTING_FEATURE_IDS}}
+ placeholder is replaced with the comma-separated list. Falls back to
+ the legacy single-feature header when only testing_feature_id is given.
+
+ Args:
+ project_dir: Optional project directory for project-specific prompts
+ testing_feature_id: If provided, the pre-assigned feature ID to test (legacy single mode).
+ testing_feature_ids: If provided, a list of feature IDs to test (batch mode).
+ Takes precedence over testing_feature_id when both are set.
+
+ Returns:
+ The testing prompt, with feature assignment instructions populated.
+ """
+ base_prompt = load_prompt("testing_prompt", project_dir)
+
+ # Batch mode: replace the {{TESTING_FEATURE_IDS}} placeholder in the template
+ if testing_feature_ids is not None and len(testing_feature_ids) > 0:
+ ids_str = ", ".join(str(fid) for fid in testing_feature_ids)
+ return base_prompt.replace("{{TESTING_FEATURE_IDS}}", ids_str)
+
+ # Legacy single-feature mode: prepend header and replace placeholder
+ if testing_feature_id is not None:
+ # Replace the placeholder with the single ID for template consistency
+ base_prompt = base_prompt.replace("{{TESTING_FEATURE_IDS}}", str(testing_feature_id))
+ return base_prompt
+
+ # No feature assignment -- return template with placeholder cleared
+ return base_prompt.replace("{{TESTING_FEATURE_IDS}}", "(none assigned)")
+
+
+def get_single_feature_prompt(feature_id: int, project_dir: Path | None = None, yolo_mode: bool = False) -> str:
+ """Prepend single-feature assignment header to base coding prompt.
+
+ Used in parallel mode to assign a specific feature to an agent.
+ The base prompt already contains the full workflow - this just
+ identifies which feature to work on.
+
+ Args:
+ feature_id: The specific feature ID to work on
+ project_dir: Optional project directory for project-specific prompts
+ yolo_mode: If True, strip browser testing instructions from the base
+ coding prompt for reduced token usage in YOLO mode.
+
+ Returns:
+ The prompt with single-feature header prepended
+ """
+ base_prompt = get_coding_prompt(project_dir, yolo_mode=yolo_mode)
+
+ # Minimal header - the base prompt already contains the full workflow
+ single_feature_header = f"""## ASSIGNED FEATURE: #{feature_id}
+
+Work ONLY on this feature. Other agents are handling other features.
+Use `feature_claim_and_get` with ID {feature_id} to claim it and get details.
+If blocked, use `feature_skip` and document the blocker.
+
+---
+
+"""
+ return single_feature_header + base_prompt
+
+
+def get_batch_feature_prompt(
+ feature_ids: list[int],
+ project_dir: Path | None = None,
+ yolo_mode: bool = False,
+) -> str:
+ """Prepend batch-feature assignment header to base coding prompt.
+
+ Used in parallel mode to assign multiple features to an agent.
+ Features should be implemented sequentially in the given order.
+
+ Args:
+ feature_ids: List of feature IDs to implement in order
+ project_dir: Optional project directory for project-specific prompts
+ yolo_mode: If True, strip browser testing instructions from the base prompt
+
+ Returns:
+ The prompt with batch-feature header prepended
+ """
+ base_prompt = get_coding_prompt(project_dir, yolo_mode=yolo_mode)
+ ids_str = ", ".join(f"#{fid}" for fid in feature_ids)
+
+ batch_header = f"""## ASSIGNED FEATURES (BATCH): {ids_str}
+
+You have been assigned {len(feature_ids)} features to implement sequentially.
+Process them IN ORDER: {ids_str}
+### Workflow for each feature:
+1. Call `feature_claim_and_get` with the feature ID to get its details
+2. Implement the feature fully
+3. Verify it works (browser testing if applicable)
+4. Call `feature_mark_passing` to mark it complete
+5. Git commit the changes
+6. Move to the next feature
-def get_coding_prompt_yolo(project_dir: Path | None = None) -> str:
- """Load the YOLO mode coding agent prompt (project-specific if available)."""
- return load_prompt("coding_prompt_yolo", project_dir)
+### Important:
+- Complete each feature fully before starting the next
+- Mark each feature passing individually as you go
+- If blocked on a feature, use `feature_skip` and move to the next one
+- Other agents are handling other features - focus only on yours
+
+---
+
+"""
+ return batch_header + base_prompt
def get_app_spec(project_dir: Path) -> str:
@@ -131,12 +342,16 @@ def scaffold_project_prompts(project_dir: Path) -> Path:
project_prompts = get_project_prompts_dir(project_dir)
project_prompts.mkdir(parents=True, exist_ok=True)
+ # Create .autoforge directory with .gitignore for runtime files
+ from autoforge_paths import ensure_autoforge_dir
+ autoforge_dir = ensure_autoforge_dir(project_dir)
+
# Define template mappings: (source_template, destination_name)
templates = [
("app_spec.template.txt", "app_spec.txt"),
("coding_prompt.template.md", "coding_prompt.md"),
- ("coding_prompt_yolo.template.md", "coding_prompt_yolo.md"),
("initializer_prompt.template.md", "initializer_prompt.md"),
+ ("testing_prompt.template.md", "testing_prompt.md"),
]
copied_files = []
@@ -152,8 +367,80 @@ def scaffold_project_prompts(project_dir: Path) -> Path:
except (OSError, PermissionError) as e:
print(f" Warning: Could not copy {dest_name}: {e}")
+ # Copy allowed_commands.yaml template to .autoforge/
+ examples_dir = Path(__file__).parent / "examples"
+ allowed_commands_template = examples_dir / "project_allowed_commands.yaml"
+ allowed_commands_dest = autoforge_dir / "allowed_commands.yaml"
+ if allowed_commands_template.exists() and not allowed_commands_dest.exists():
+ try:
+ shutil.copy(allowed_commands_template, allowed_commands_dest)
+ copied_files.append(".autoforge/allowed_commands.yaml")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not copy allowed_commands.yaml: {e}")
+
+ # Copy Playwright CLI skill for browser automation
+ skills_src = Path(__file__).parent / ".claude" / "skills" / "playwright-cli"
+ skills_dest = project_dir / ".claude" / "skills" / "playwright-cli"
+ if skills_src.exists() and not skills_dest.exists():
+ try:
+ shutil.copytree(skills_src, skills_dest)
+ copied_files.append(".claude/skills/playwright-cli/")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not copy playwright-cli skill: {e}")
+
+ # Ensure .playwright-cli/ and .playwright/ are in project .gitignore
+ project_gitignore = project_dir / ".gitignore"
+ entries_to_add = [".playwright-cli/", ".playwright/"]
+ existing_lines: list[str] = []
+ if project_gitignore.exists():
+ try:
+ existing_lines = project_gitignore.read_text(encoding="utf-8").splitlines()
+ except (OSError, PermissionError):
+ pass
+ missing_entries = [e for e in entries_to_add if e not in existing_lines]
+ if missing_entries:
+ try:
+ with open(project_gitignore, "a", encoding="utf-8") as f:
+ # Add newline before entries if file doesn't end with one
+ if existing_lines and existing_lines[-1].strip():
+ f.write("\n")
+ for entry in missing_entries:
+ f.write(f"{entry}\n")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not update .gitignore: {e}")
+
+ # Scaffold .playwright/cli.config.json for browser settings
+ playwright_config_dir = project_dir / ".playwright"
+ playwright_config_file = playwright_config_dir / "cli.config.json"
+ if not playwright_config_file.exists():
+ try:
+ playwright_config_dir.mkdir(parents=True, exist_ok=True)
+ import json
+ config = {
+ "browser": {
+ "browserName": "chromium",
+ "launchOptions": {
+ "channel": "chrome",
+ "headless": True,
+ },
+ "contextOptions": {
+ "viewport": {"width": 1280, "height": 720},
+ },
+ "isolated": True,
+ },
+ }
+ with open(playwright_config_file, "w", encoding="utf-8") as f:
+ json.dump(config, f, indent=2)
+ f.write("\n")
+ copied_files.append(".playwright/cli.config.json")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not create playwright config: {e}")
+
if copied_files:
- print(f" Created prompt files: {', '.join(copied_files)}")
+ print(f" Created project files: {', '.join(copied_files)}")
+
+ # Stamp new projects at the current migration version so they never trigger migration
+ _set_migration_version(project_dir, CURRENT_MIGRATION_VERSION)
return project_prompts
@@ -226,3 +513,330 @@ def copy_spec_to_project(project_dir: Path) -> None:
return
print("Warning: No app_spec.txt found to copy to project directory")
+
+
+# ---------------------------------------------------------------------------
+# Project version migration
+# ---------------------------------------------------------------------------
+
+# Replacement content: coding_prompt.md STEP 5 section (Playwright CLI)
+_CLI_STEP5_CONTENT = """\
+### STEP 5: VERIFY WITH BROWSER AUTOMATION
+
+**CRITICAL:** You MUST verify features through the actual UI.
+
+Use `playwright-cli` for browser automation:
+
+- Open the browser: `playwright-cli open http://localhost:PORT`
+- Take a snapshot to see page elements: `playwright-cli snapshot`
+- Read the snapshot YAML file to see element refs
+- Click elements by ref: `playwright-cli click e5`
+- Type text: `playwright-cli type "search query"`
+- Fill form fields: `playwright-cli fill e3 "value"`
+- Take screenshots: `playwright-cli screenshot`
+- Read the screenshot file to verify visual appearance
+- Check console errors: `playwright-cli console`
+- Close browser when done: `playwright-cli close`
+
+**Token-efficient workflow:** `playwright-cli screenshot` and `snapshot` save files
+to `.playwright-cli/`. You will see a file link in the output. Read the file only
+when you need to verify visual appearance or find element refs.
+
+**DO:**
+- Test through the UI with clicks and keyboard input
+- Take screenshots and read them to verify visual appearance
+- Check for console errors with `playwright-cli console`
+- Verify complete user workflows end-to-end
+- Always run `playwright-cli close` when finished testing
+
+**DON'T:**
+- Only test with curl commands
+- Use JavaScript evaluation to bypass UI (`eval` and `run-code` are blocked)
+- Skip visual verification
+- Mark tests passing without thorough verification
+
+"""
+
+# Replacement content: coding_prompt.md BROWSER AUTOMATION reference section
+_CLI_BROWSER_SECTION = """\
+## BROWSER AUTOMATION
+
+Use `playwright-cli` commands for UI verification. Key commands: `open`, `goto`,
+`snapshot`, `click`, `type`, `fill`, `screenshot`, `console`, `close`.
+
+**How it works:** `playwright-cli` uses a persistent browser daemon. `open` starts it,
+subsequent commands interact via socket, `close` shuts it down. Screenshots and snapshots
+save to `.playwright-cli/` -- read the files when you need to verify content.
+
+Test like a human user with mouse and keyboard. Use `playwright-cli console` to detect
+JS errors. Don't bypass UI with JavaScript evaluation.
+
+"""
+
+# Replacement content: testing_prompt.md STEP 2 section (Playwright CLI)
+_CLI_TESTING_STEP2 = """\
+### STEP 2: VERIFY THE FEATURE
+
+**CRITICAL:** You MUST verify the feature through the actual UI using browser automation.
+
+For the feature returned:
+1. Read and understand the feature's verification steps
+2. Navigate to the relevant part of the application
+3. Execute each verification step using browser automation
+4. Take screenshots and read them to verify visual appearance
+5. Check for console errors
+
+### Browser Automation (Playwright CLI)
+
+**Navigation & Screenshots:**
+- `playwright-cli open ` - Open browser and navigate
+- `playwright-cli goto ` - Navigate to URL
+- `playwright-cli screenshot` - Save screenshot to `.playwright-cli/`
+- `playwright-cli snapshot` - Save page snapshot with element refs to `.playwright-cli/`
+
+**Element Interaction:**
+- `playwright-cli click ][` - Click elements (ref from snapshot)
+- `playwright-cli type ` - Type text
+- `playwright-cli fill ][ ` - Fill form fields
+- `playwright-cli select ][ ` - Select dropdown
+- `playwright-cli press ` - Keyboard input
+
+**Debugging:**
+- `playwright-cli console` - Check for JS errors
+- `playwright-cli network` - Monitor API calls
+
+**Cleanup:**
+- `playwright-cli close` - Close browser when done (ALWAYS do this)
+
+**Note:** Screenshots and snapshots save to files. Read the file to see the content.
+
+"""
+
+# Replacement content: testing_prompt.md AVAILABLE TOOLS browser subsection
+_CLI_TESTING_TOOLS = """\
+### Browser Automation (Playwright CLI)
+Use `playwright-cli` commands for browser interaction. Key commands:
+- `playwright-cli open ` - Open browser
+- `playwright-cli goto ` - Navigate to URL
+- `playwright-cli screenshot` - Take screenshot (saved to `.playwright-cli/`)
+- `playwright-cli snapshot` - Get page snapshot with element refs
+- `playwright-cli click ][` - Click element
+- `playwright-cli type ` - Type text
+- `playwright-cli fill ][ ` - Fill form field
+- `playwright-cli console` - Check for JS errors
+- `playwright-cli close` - Close browser (always do this when done)
+
+"""
+
+
+def _get_migration_version(project_dir: Path) -> int:
+ """Read the migration version from .autoforge/.migration_version."""
+ from autoforge_paths import get_autoforge_dir
+ version_file = get_autoforge_dir(project_dir) / ".migration_version"
+ if not version_file.exists():
+ return 0
+ try:
+ return int(version_file.read_text().strip())
+ except (ValueError, OSError):
+ return 0
+
+
+def _set_migration_version(project_dir: Path, version: int) -> None:
+ """Write the migration version to .autoforge/.migration_version."""
+ from autoforge_paths import get_autoforge_dir
+ version_file = get_autoforge_dir(project_dir) / ".migration_version"
+ version_file.parent.mkdir(parents=True, exist_ok=True)
+ version_file.write_text(str(version))
+
+
+def _migrate_coding_prompt_to_cli(content: str) -> str:
+ """Replace MCP-based Playwright sections with CLI-based content in coding prompt."""
+ # Replace STEP 5 section (from header to just before STEP 5.5)
+ content = re.sub(
+ r"### STEP 5: VERIFY WITH BROWSER AUTOMATION.*?(?=### STEP 5\.5:)",
+ _CLI_STEP5_CONTENT,
+ content,
+ count=1,
+ flags=re.DOTALL,
+ )
+
+ # Replace BROWSER AUTOMATION reference section (from header to next ---)
+ content = re.sub(
+ r"## BROWSER AUTOMATION\n\n.*?(?=---)",
+ _CLI_BROWSER_SECTION,
+ content,
+ count=1,
+ flags=re.DOTALL,
+ )
+
+ # Replace inline screenshot rule
+ content = content.replace(
+ "**ONLY MARK A FEATURE AS PASSING AFTER VERIFICATION WITH SCREENSHOTS.**",
+ "**ONLY MARK A FEATURE AS PASSING AFTER VERIFICATION WITH BROWSER AUTOMATION.**",
+ )
+
+ # Replace inline screenshot references (various phrasings from old templates)
+ for old_phrase in (
+ "(inline only -- do NOT save to disk)",
+ "(inline only, never save to disk)",
+ "(inline mode only -- never save to disk)",
+ ):
+ content = content.replace(old_phrase, "(saved to `.playwright-cli/`)")
+
+ return content
+
+
+def _migrate_testing_prompt_to_cli(content: str) -> str:
+ """Replace MCP-based Playwright sections with CLI-based content in testing prompt."""
+ # Replace AVAILABLE TOOLS browser subsection FIRST (before STEP 2, to avoid
+ # matching the new CLI subsection header that the STEP 2 replacement inserts).
+ # In old prompts, ### Browser Automation (Playwright) only exists in AVAILABLE TOOLS.
+ content = re.sub(
+ r"### Browser Automation \(Playwright[^)]*\)\n.*?(?=---)",
+ _CLI_TESTING_TOOLS,
+ content,
+ count=1,
+ flags=re.DOTALL,
+ )
+
+ # Replace STEP 2 verification section (from header to just before STEP 3)
+ content = re.sub(
+ r"### STEP 2: VERIFY THE FEATURE.*?(?=### STEP 3:)",
+ _CLI_TESTING_STEP2,
+ content,
+ count=1,
+ flags=re.DOTALL,
+ )
+
+ # Replace inline screenshot references (various phrasings from old templates)
+ for old_phrase in (
+ "(inline only -- do NOT save to disk)",
+ "(inline only, never save to disk)",
+ "(inline mode only -- never save to disk)",
+ ):
+ content = content.replace(old_phrase, "(saved to `.playwright-cli/`)")
+
+ return content
+
+
+def _migrate_v0_to_v1(project_dir: Path) -> list[str]:
+ """Migrate from v0 (MCP-based Playwright) to v1 (Playwright CLI).
+
+ Four idempotent sub-steps:
+ A. Copy playwright-cli skill to project
+ B. Scaffold .playwright/cli.config.json
+ C. Update .gitignore with .playwright-cli/ and .playwright/
+ D. Update coding_prompt.md and testing_prompt.md
+ """
+ import json
+
+ migrated: list[str] = []
+
+ # A. Copy Playwright CLI skill
+ skills_src = Path(__file__).parent / ".claude" / "skills" / "playwright-cli"
+ skills_dest = project_dir / ".claude" / "skills" / "playwright-cli"
+ if skills_src.exists() and not skills_dest.exists():
+ try:
+ shutil.copytree(skills_src, skills_dest)
+ migrated.append("Copied playwright-cli skill")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not copy playwright-cli skill: {e}")
+
+ # B. Scaffold .playwright/cli.config.json
+ playwright_config_dir = project_dir / ".playwright"
+ playwright_config_file = playwright_config_dir / "cli.config.json"
+ if not playwright_config_file.exists():
+ try:
+ playwright_config_dir.mkdir(parents=True, exist_ok=True)
+ config = {
+ "browser": {
+ "browserName": "chromium",
+ "launchOptions": {
+ "channel": "chrome",
+ "headless": True,
+ },
+ "contextOptions": {
+ "viewport": {"width": 1280, "height": 720},
+ },
+ "isolated": True,
+ },
+ }
+ with open(playwright_config_file, "w", encoding="utf-8") as f:
+ json.dump(config, f, indent=2)
+ f.write("\n")
+ migrated.append("Created .playwright/cli.config.json")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not create playwright config: {e}")
+
+ # C. Update .gitignore
+ project_gitignore = project_dir / ".gitignore"
+ entries_to_add = [".playwright-cli/", ".playwright/"]
+ existing_lines: list[str] = []
+ if project_gitignore.exists():
+ try:
+ existing_lines = project_gitignore.read_text(encoding="utf-8").splitlines()
+ except (OSError, PermissionError):
+ pass
+ missing_entries = [e for e in entries_to_add if e not in existing_lines]
+ if missing_entries:
+ try:
+ with open(project_gitignore, "a", encoding="utf-8") as f:
+ if existing_lines and existing_lines[-1].strip():
+ f.write("\n")
+ for entry in missing_entries:
+ f.write(f"{entry}\n")
+ migrated.append(f"Added {', '.join(missing_entries)} to .gitignore")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not update .gitignore: {e}")
+
+ # D. Update prompts
+ prompts_dir = get_project_prompts_dir(project_dir)
+
+ # D1. Update coding_prompt.md
+ coding_prompt_path = prompts_dir / "coding_prompt.md"
+ if coding_prompt_path.exists():
+ try:
+ content = coding_prompt_path.read_text(encoding="utf-8")
+ if "Playwright MCP" in content or "browser_navigate" in content or "browser_take_screenshot" in content:
+ updated = _migrate_coding_prompt_to_cli(content)
+ if updated != content:
+ coding_prompt_path.write_text(updated, encoding="utf-8")
+ migrated.append("Updated coding_prompt.md to Playwright CLI")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not update coding_prompt.md: {e}")
+
+ # D2. Update testing_prompt.md
+ testing_prompt_path = prompts_dir / "testing_prompt.md"
+ if testing_prompt_path.exists():
+ try:
+ content = testing_prompt_path.read_text(encoding="utf-8")
+ if "browser_navigate" in content or "browser_take_screenshot" in content:
+ updated = _migrate_testing_prompt_to_cli(content)
+ if updated != content:
+ testing_prompt_path.write_text(updated, encoding="utf-8")
+ migrated.append("Updated testing_prompt.md to Playwright CLI")
+ except (OSError, PermissionError) as e:
+ print(f" Warning: Could not update testing_prompt.md: {e}")
+
+ return migrated
+
+
+def migrate_project_to_current(project_dir: Path) -> list[str]:
+ """Migrate an existing project to the current AutoForge version.
+
+ Idempotent — safe to call on every agent start. Returns list of
+ human-readable descriptions of what was migrated.
+ """
+ current = _get_migration_version(project_dir)
+ if current >= CURRENT_MIGRATION_VERSION:
+ return []
+
+ migrated: list[str] = []
+
+ if current < 1:
+ migrated.extend(_migrate_v0_to_v1(project_dir))
+
+ # Future: if current < 2: migrated.extend(_migrate_v1_to_v2(project_dir))
+
+ _set_migration_version(project_dir, CURRENT_MIGRATION_VERSION)
+ return migrated
diff --git a/rate_limit_utils.py b/rate_limit_utils.py
new file mode 100644
index 000000000..7fe77ead4
--- /dev/null
+++ b/rate_limit_utils.py
@@ -0,0 +1,132 @@
+"""
+Rate Limit Utilities
+====================
+
+Shared utilities for detecting and handling API rate limits.
+Used by both agent.py (production) and test_rate_limit_utils.py (tests).
+"""
+
+import random
+import re
+from typing import Optional
+
+# Regex patterns for rate limit detection (used in both exception messages and response text)
+# These patterns use word boundaries to avoid false positives like "PR #429" or "please wait while I..."
+RATE_LIMIT_REGEX_PATTERNS = [
+ r"\brate[_\s]?limit", # "rate limit", "rate_limit", "ratelimit"
+ r"\btoo\s+many\s+requests", # "too many requests"
+ r"\bhttp\s*429\b", # "http 429", "http429"
+ r"\bstatus\s*429\b", # "status 429", "status429"
+ r"\berror\s*429\b", # "error 429", "error429"
+ r"\b429\s+too\s+many", # "429 too many"
+ r"\b(?:server|api|system)\s+(?:is\s+)?overloaded\b", # "server is overloaded", "api overloaded"
+ r"\bquota\s*exceeded\b", # "quota exceeded"
+]
+
+# Compiled regex for efficient matching
+_RATE_LIMIT_REGEX = re.compile(
+ "|".join(RATE_LIMIT_REGEX_PATTERNS),
+ re.IGNORECASE
+)
+
+
+def parse_retry_after(error_message: str) -> Optional[int]:
+ """
+ Extract retry-after seconds from various error message formats.
+
+ Handles common formats:
+ - "Retry-After: 60"
+ - "retry after 60 seconds"
+ - "try again in 5 seconds"
+ - "30 seconds remaining"
+
+ Args:
+ error_message: The error message to parse
+
+ Returns:
+ Seconds to wait, or None if not parseable.
+ """
+ # Patterns require explicit "seconds" or "s" unit, OR no unit at all (end of string/sentence)
+ # This prevents matching "30 minutes" or "1 hour" since those have non-seconds units
+ patterns = [
+ r"retry.?after[:\s]+(\d+)\s*(?:seconds?|s\b)", # Requires seconds unit
+ r"retry.?after[:\s]+(\d+)(?:\s*$|\s*[,.])", # Or end of string/sentence
+ r"try again in\s+(\d+)\s*(?:seconds?|s\b)", # Requires seconds unit
+ r"try again in\s+(\d+)(?:\s*$|\s*[,.])", # Or end of string/sentence
+ r"(\d+)\s*seconds?\s*(?:remaining|left|until)",
+ ]
+
+ for pattern in patterns:
+ match = re.search(pattern, error_message, re.IGNORECASE)
+ if match:
+ return int(match.group(1))
+
+ return None
+
+
+def is_rate_limit_error(error_message: str) -> bool:
+ """
+ Detect if an error message indicates a rate limit.
+
+ Uses regex patterns with word boundaries to avoid false positives
+ like "PR #429", "please wait while I...", or "Node v14.29.0".
+
+ Args:
+ error_message: The error message to check
+
+ Returns:
+ True if the message indicates a rate limit, False otherwise.
+ """
+ return bool(_RATE_LIMIT_REGEX.search(error_message))
+
+
+def calculate_rate_limit_backoff(retries: int) -> int:
+ """
+ Calculate exponential backoff with jitter for rate limits.
+
+ Base formula: min(15 * 2^retries, 3600)
+ Jitter: adds 0-30% random jitter to prevent thundering herd.
+ Base sequence: ~15-20s, ~30-40s, ~60-78s, ~120-156s, ...
+
+ The lower starting delay (15s vs 60s) allows faster recovery from
+ transient rate limits, while jitter prevents synchronized retries
+ when multiple agents hit limits simultaneously.
+
+ Args:
+ retries: Number of consecutive rate limit retries (0-indexed)
+
+ Returns:
+ Delay in seconds (clamped to 1-3600 range, with jitter)
+ """
+ base = int(min(max(15 * (2 ** retries), 1), 3600))
+ jitter = random.uniform(0, base * 0.3)
+ return int(base + jitter)
+
+
+def calculate_error_backoff(retries: int) -> int:
+ """
+ Calculate linear backoff for non-rate-limit errors.
+
+ Formula: min(30 * retries, 300) - caps at 5 minutes
+ Sequence: 30s, 60s, 90s, 120s, ... 300s
+
+ Args:
+ retries: Number of consecutive error retries (1-indexed)
+
+ Returns:
+ Delay in seconds (clamped to 1-300 range)
+ """
+ return min(max(30 * retries, 1), 300)
+
+
+def clamp_retry_delay(delay_seconds: int) -> int:
+ """
+ Clamp a retry delay to a safe range (1-3600 seconds).
+
+ Args:
+ delay_seconds: The raw delay value
+
+ Returns:
+ Delay clamped to 1-3600 seconds
+ """
+ return min(max(delay_seconds, 1), 3600)
diff --git a/registry.py b/registry.py
index 5d48a1c84..6255d431d 100644
--- a/registry.py
+++ b/registry.py
@@ -3,25 +3,92 @@
=======================
Cross-platform project registry for storing project name to path mappings.
-Uses SQLite database stored at ~/.autocoder/registry.db.
+Uses SQLite database stored at ~/.autoforge/registry.db.
"""
import logging
import os
import re
+import threading
+import time
from contextlib import contextmanager
from datetime import datetime
from pathlib import Path
-from typing import Any
+from typing import Any, Literal, cast
-from sqlalchemy import Column, DateTime, String, create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy import Boolean, Column, DateTime, Integer, String, create_engine, text
+from sqlalchemy.orm import DeclarativeBase, sessionmaker
# Module logger
logger = logging.getLogger(__name__)
+def _migrate_registry_dir() -> None:
+ """Migrate ~/.autocoder/ to ~/.autoforge/ if needed.
+
+ Provides backward compatibility by automatically renaming the old
+ config directory to the new location on first access.
+ """
+ old_dir = Path.home() / ".autocoder"
+ new_dir = Path.home() / ".autoforge"
+ if old_dir.exists() and not new_dir.exists():
+ try:
+ old_dir.rename(new_dir)
+ logger.info("Migrated registry directory: ~/.autocoder/ -> ~/.autoforge/")
+ except Exception:
+ logger.warning("Failed to migrate ~/.autocoder/ to ~/.autoforge/", exc_info=True)
+
+
+# =============================================================================
+# Model Configuration (Single Source of Truth)
+# =============================================================================
+
+# Available models with display names
+# To add a new model: add an entry here with {"id": "model-id", "name": "Display Name"}
+AVAILABLE_MODELS = [
+ {"id": "claude-opus-4-7", "name": "Claude Opus"},
+ {"id": "claude-sonnet-4-6", "name": "Claude Sonnet"},
+]
+
+# Map legacy model IDs to their current replacements.
+# Used by get_all_settings() to auto-migrate stale values on first read after upgrade.
+LEGACY_MODEL_MAP = {
+ "claude-opus-4-5-20251101": "claude-opus-4-7",
+ "claude-opus-4-6": "claude-opus-4-7",
+ "claude-sonnet-4-5": "claude-sonnet-4-6",
+ "claude-sonnet-4-5-20250929": "claude-sonnet-4-6",
+}
+
+# List of valid model IDs (derived from AVAILABLE_MODELS)
+VALID_MODELS = [m["id"] for m in AVAILABLE_MODELS]
+
+# Default model and settings
+# Respect ANTHROPIC_DEFAULT_OPUS_MODEL env var for Foundry/custom deployments
+# Guard against empty/whitespace values by trimming and falling back when blank
+_env_default_model = os.getenv("ANTHROPIC_DEFAULT_OPUS_MODEL")
+if _env_default_model is not None:
+ _env_default_model = _env_default_model.strip()
+# Auto-remap stale env-provided values (e.g. user's .env still pins 4.6)
+if _env_default_model and _env_default_model in LEGACY_MODEL_MAP:
+ logging.getLogger(__name__).warning(
+ "ANTHROPIC_DEFAULT_OPUS_MODEL=%s is legacy; remapping to %s. "
+ "Update your .env to silence this warning.",
+ _env_default_model, LEGACY_MODEL_MAP[_env_default_model],
+ )
+ _env_default_model = LEGACY_MODEL_MAP[_env_default_model]
+DEFAULT_MODEL = _env_default_model or "claude-opus-4-7"
+
+# Ensure env-provided DEFAULT_MODEL is in VALID_MODELS for validation consistency
+# (idempotent: only adds if missing, doesn't alter AVAILABLE_MODELS semantics)
+if DEFAULT_MODEL and DEFAULT_MODEL not in VALID_MODELS:
+ VALID_MODELS.append(DEFAULT_MODEL)
+DEFAULT_YOLO_MODE = False
+
+# SQLite connection settings
+SQLITE_TIMEOUT = 30 # seconds to wait for database lock
+SQLITE_MAX_RETRIES = 3 # number of retry attempts on busy database
+
+
# =============================================================================
# Exceptions
# =============================================================================
@@ -50,7 +117,9 @@ class RegistryPermissionDenied(RegistryError):
# SQLAlchemy Model
# =============================================================================
-Base = declarative_base()
+class Base(DeclarativeBase):
+ """SQLAlchemy 2.0 style declarative base."""
+ pass
class Project(Base):
@@ -60,25 +129,41 @@ class Project(Base):
name = Column(String(50), primary_key=True, index=True)
path = Column(String, nullable=False) # POSIX format for cross-platform
created_at = Column(DateTime, nullable=False)
+ default_concurrency = Column(Integer, nullable=False, default=3)
+ auto_improve_enabled = Column(Boolean, nullable=False, default=False)
+ auto_improve_interval_minutes = Column(Integer, nullable=False, default=10)
+
+
+class Settings(Base):
+ """SQLAlchemy model for global settings (key-value store)."""
+ __tablename__ = "settings"
+
+ key = Column(String(50), primary_key=True)
+ value = Column(String(500), nullable=False)
+ updated_at = Column(DateTime, nullable=False)
# =============================================================================
# Database Connection
# =============================================================================
-# Module-level singleton for database engine
+# Module-level singleton for database engine with thread-safe initialization
_engine = None
_SessionLocal = None
+_engine_lock = threading.Lock()
def get_config_dir() -> Path:
"""
- Get the config directory: ~/.autocoder/
+ Get the config directory: ~/.autoforge/
+
+ Automatically migrates from ~/.autocoder/ if needed.
Returns:
- Path to ~/.autocoder/ (created if it doesn't exist)
+ Path to ~/.autoforge/ (created if it doesn't exist)
"""
- config_dir = Path.home() / ".autocoder"
+ _migrate_registry_dir()
+ config_dir = Path.home() / ".autoforge"
config_dir.mkdir(parents=True, exist_ok=True)
return config_dir
@@ -90,29 +175,74 @@ def get_registry_path() -> Path:
def _get_engine():
"""
- Get or create the database engine (singleton pattern).
+ Get or create the database engine (thread-safe singleton pattern).
Returns:
Tuple of (engine, SessionLocal)
"""
global _engine, _SessionLocal
+ # Double-checked locking for thread safety
if _engine is None:
- db_path = get_registry_path()
- db_url = f"sqlite:///{db_path.as_posix()}"
- _engine = create_engine(db_url, connect_args={"check_same_thread": False})
- Base.metadata.create_all(bind=_engine)
- _SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=_engine)
- logger.debug("Initialized registry database at: %s", db_path)
+ with _engine_lock:
+ if _engine is None:
+ db_path = get_registry_path()
+ db_url = f"sqlite:///{db_path.as_posix()}"
+ _engine = create_engine(
+ db_url,
+ connect_args={
+ "check_same_thread": False,
+ "timeout": SQLITE_TIMEOUT,
+ }
+ )
+ Base.metadata.create_all(bind=_engine)
+ _migrate_add_default_concurrency(_engine)
+ _migrate_add_auto_improve(_engine)
+ _SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=_engine)
+ logger.debug("Initialized registry database at: %s", db_path)
return _engine, _SessionLocal
+def _migrate_add_default_concurrency(engine) -> None:
+ """Add default_concurrency column if missing (for existing databases)."""
+ with engine.connect() as conn:
+ result = conn.execute(text("PRAGMA table_info(projects)"))
+ columns = [row[1] for row in result.fetchall()]
+ if "default_concurrency" not in columns:
+ conn.execute(text(
+ "ALTER TABLE projects ADD COLUMN default_concurrency INTEGER DEFAULT 3"
+ ))
+ conn.commit()
+ logger.info("Migrated projects table: added default_concurrency column")
+
+
+def _migrate_add_auto_improve(engine) -> None:
+ """Add auto-improve columns if missing (for existing databases)."""
+ with engine.connect() as conn:
+ result = conn.execute(text("PRAGMA table_info(projects)"))
+ columns = [row[1] for row in result.fetchall()]
+ if "auto_improve_enabled" not in columns:
+ conn.execute(text(
+ "ALTER TABLE projects ADD COLUMN auto_improve_enabled INTEGER NOT NULL DEFAULT 0"
+ ))
+ conn.commit()
+ logger.info("Migrated projects table: added auto_improve_enabled column")
+ if "auto_improve_interval_minutes" not in columns:
+ conn.execute(text(
+ "ALTER TABLE projects ADD COLUMN auto_improve_interval_minutes INTEGER NOT NULL DEFAULT 10"
+ ))
+ conn.commit()
+ logger.info("Migrated projects table: added auto_improve_interval_minutes column")
+
+
@contextmanager
def _get_session():
"""
Context manager for database sessions with automatic commit/rollback.
+ Includes retry logic for SQLite busy database errors.
+
Yields:
SQLAlchemy session
"""
@@ -128,6 +258,40 @@ def _get_session():
session.close()
+def _with_retry(func, *args, **kwargs):
+ """
+ Execute a database operation with retry logic for busy database.
+
+ Args:
+ func: Function to execute
+ *args, **kwargs: Arguments to pass to the function
+
+ Returns:
+ Result of the function
+
+ Raises:
+ Last exception if all retries fail
+ """
+ last_error = None
+ for attempt in range(SQLITE_MAX_RETRIES):
+ try:
+ return func(*args, **kwargs)
+ except Exception as e:
+ last_error = e
+ error_str = str(e).lower()
+ if "database is locked" in error_str or "sqlite_busy" in error_str:
+ if attempt < SQLITE_MAX_RETRIES - 1:
+ wait_time = (2 ** attempt) * 0.1 # Exponential backoff: 0.1s, 0.2s, 0.4s
+ logger.warning(
+ "Database busy, retrying in %.1fs (attempt %d/%d)",
+ wait_time, attempt + 1, SQLITE_MAX_RETRIES
+ )
+ time.sleep(wait_time)
+ continue
+ raise
+ raise last_error
+
+
# =============================================================================
# Project CRUD Functions
# =============================================================================
@@ -227,7 +391,12 @@ def list_registered_projects() -> dict[str, dict[str, Any]]:
return {
p.name: {
"path": p.path,
- "created_at": p.created_at.isoformat() if p.created_at else None
+ "created_at": p.created_at.isoformat() if p.created_at else None,
+ "default_concurrency": getattr(p, 'default_concurrency', 3) or 3,
+ "auto_improve_enabled": bool(getattr(p, 'auto_improve_enabled', False)),
+ "auto_improve_interval_minutes": int(
+ getattr(p, 'auto_improve_interval_minutes', 10) or 10
+ ),
}
for p in projects
}
@@ -253,7 +422,12 @@ def get_project_info(name: str) -> dict[str, Any] | None:
return None
return {
"path": project.path,
- "created_at": project.created_at.isoformat() if project.created_at else None
+ "created_at": project.created_at.isoformat() if project.created_at else None,
+ "default_concurrency": getattr(project, 'default_concurrency', 3) or 3,
+ "auto_improve_enabled": bool(getattr(project, 'auto_improve_enabled', False)),
+ "auto_improve_interval_minutes": int(
+ getattr(project, 'auto_improve_interval_minutes', 10) or 10
+ ),
}
finally:
session.close()
@@ -282,6 +456,120 @@ def update_project_path(name: str, new_path: Path) -> bool:
return True
+def get_project_concurrency(name: str) -> int:
+ """
+ Get project's default concurrency (1-5).
+
+ Args:
+ name: The project name.
+
+ Returns:
+ The default concurrency value (defaults to 3 if not set or project not found).
+ """
+ _, SessionLocal = _get_engine()
+ session = SessionLocal()
+ try:
+ project = session.query(Project).filter(Project.name == name).first()
+ if project is None:
+ return 3
+ return getattr(project, 'default_concurrency', 3) or 3
+ finally:
+ session.close()
+
+
+def set_project_concurrency(name: str, concurrency: int) -> bool:
+ """
+ Set project's default concurrency (1-5).
+
+ Args:
+ name: The project name.
+ concurrency: The concurrency value (1-5).
+
+ Returns:
+ True if updated, False if project wasn't found.
+
+ Raises:
+ ValueError: If concurrency is not between 1 and 5.
+ """
+ if concurrency < 1 or concurrency > 5:
+ raise ValueError("concurrency must be between 1 and 5")
+
+ with _get_session() as session:
+ project = session.query(Project).filter(Project.name == name).first()
+ if not project:
+ return False
+
+ project.default_concurrency = concurrency
+
+ logger.info("Set project '%s' default_concurrency to %d", name, concurrency)
+ return True
+
+
+def get_project_auto_improve(name: str) -> tuple[bool, int]:
+ """
+ Get a project's auto-improve configuration.
+
+ Args:
+ name: The project name.
+
+ Returns:
+ Tuple of (enabled, interval_minutes). Defaults to (False, 10) if
+ the project is not found or the columns are missing.
+ """
+ _, SessionLocal = _get_engine()
+ session = SessionLocal()
+ try:
+ project = session.query(Project).filter(Project.name == name).first()
+ if project is None:
+ return (False, 10)
+ enabled = bool(getattr(project, "auto_improve_enabled", False))
+ interval = int(getattr(project, "auto_improve_interval_minutes", 10) or 10)
+ return (enabled, interval)
+ finally:
+ session.close()
+
+
+def set_project_auto_improve(
+ name: str,
+ enabled: bool | None = None,
+ interval_minutes: int | None = None,
+) -> bool:
+ """
+ Update a project's auto-improve configuration.
+
+ Either field can be updated independently by passing None for the other.
+
+ Args:
+ name: The project name.
+ enabled: If provided, set the enabled flag.
+ interval_minutes: If provided, set the interval in minutes (1-1440).
+
+ Returns:
+ True if updated, False if the project wasn't found.
+
+ Raises:
+ ValueError: If interval_minutes is outside the 1-1440 range.
+ """
+ if interval_minutes is not None and (interval_minutes < 1 or interval_minutes > 1440):
+ raise ValueError("interval_minutes must be between 1 and 1440")
+
+ with _get_session() as session:
+ project = session.query(Project).filter(Project.name == name).first()
+ if not project:
+ return False
+
+ if enabled is not None:
+ project.auto_improve_enabled = bool(enabled)
+ if interval_minutes is not None:
+ project.auto_improve_interval_minutes = int(interval_minutes)
+
+ logger.info(
+ "Set project '%s' auto_improve: enabled=%s, interval=%s",
+ name, enabled, interval_minutes,
+ )
+ return True
+
+
# =============================================================================
# Validation Functions
# =============================================================================
@@ -364,3 +652,263 @@ def list_valid_projects() -> list[dict[str, Any]]:
return valid
finally:
session.close()
+
+
+# =============================================================================
+# Settings CRUD Functions
+# =============================================================================
+
+def get_setting(key: str, default: str | None = None) -> str | None:
+ """
+ Get a setting value by key.
+
+ Args:
+ key: The setting key.
+ default: Default value if setting doesn't exist or on DB error.
+
+ Returns:
+ The setting value, or default if not found or on error.
+ """
+ try:
+ _, SessionLocal = _get_engine()
+ session = SessionLocal()
+ try:
+ setting = session.query(Settings).filter(Settings.key == key).first()
+ return setting.value if setting else default
+ finally:
+ session.close()
+ except Exception as e:
+ logger.warning("Failed to read setting '%s': %s", key, e)
+ return default
+
+
+# Valid Claude Code reasoning/effort levels. Must match the CLI's --effort
+# choices (low, medium, high, xhigh, max) — note: the SDK's Literal type at
+# 0.1.61 omits "xhigh", but the string is forwarded to the CLI as-is and
+# accepted there.
+EffortLevel = Literal["low", "medium", "high", "xhigh", "max"]
+VALID_EFFORT_LEVELS: tuple[EffortLevel, ...] = ("low", "medium", "high", "xhigh", "max")
+DEFAULT_EFFORT: EffortLevel = "xhigh"
+
+
+def get_effort_setting() -> EffortLevel:
+ """
+ Read the global reasoning-effort setting, falling back to ``xhigh``.
+
+ Unknown/invalid stored values are treated as missing so a DB corruption or
+ schema drift can't force the CLI into an unsupported mode.
+ """
+ value = get_setting("effort")
+ if value in VALID_EFFORT_LEVELS:
+ return cast(EffortLevel, value)
+ return DEFAULT_EFFORT
+
+
+def set_setting(key: str, value: str) -> None:
+ """
+ Set a setting value (creates or updates).
+
+ Args:
+ key: The setting key.
+ value: The setting value.
+ """
+ with _get_session() as session:
+ setting = session.query(Settings).filter(Settings.key == key).first()
+ if setting:
+ setting.value = value
+ setting.updated_at = datetime.now()
+ else:
+ setting = Settings(
+ key=key,
+ value=value,
+ updated_at=datetime.now()
+ )
+ session.add(setting)
+
+ logger.debug("Set setting '%s' = '%s'", key, value)
+
+
+def get_all_settings() -> dict[str, str]:
+ """
+ Get all settings as a dictionary.
+
+ Automatically migrates legacy model IDs (e.g. claude-opus-4-6 -> claude-opus-4-7)
+ on first read after upgrade. This is a one-time silent migration.
+
+ Returns:
+ Dictionary mapping setting keys to values.
+ """
+ try:
+ _, SessionLocal = _get_engine()
+ session = SessionLocal()
+ try:
+ settings = session.query(Settings).all()
+ result = {s.key: s.value for s in settings}
+
+ # Auto-migrate legacy model IDs
+ migrated = False
+ for key in ("model", "api_model"):
+ old_id = result.get(key)
+ if old_id and old_id in LEGACY_MODEL_MAP:
+ new_id = LEGACY_MODEL_MAP[old_id]
+ setting = session.query(Settings).filter(Settings.key == key).first()
+ if setting:
+ setting.value = new_id
+ setting.updated_at = datetime.now()
+ result[key] = new_id
+ migrated = True
+ logger.info("Migrated setting '%s': %s -> %s", key, old_id, new_id)
+
+ if migrated:
+ session.commit()
+
+ return result
+ finally:
+ session.close()
+ except Exception as e:
+ logger.warning("Failed to read settings: %s", e)
+ return {}
+
+
+# =============================================================================
+# API Provider Definitions
+# =============================================================================
+
+API_PROVIDERS: dict[str, dict[str, Any]] = {
+ "claude": {
+ "name": "Claude (Anthropic)",
+ "base_url": None,
+ "requires_auth": False,
+ "models": [
+ {"id": "claude-opus-4-7", "name": "Claude Opus"},
+ {"id": "claude-sonnet-4-6", "name": "Claude Sonnet"},
+ ],
+ "default_model": "claude-opus-4-7",
+ },
+ "kimi": {
+ "name": "Kimi K2.5 (Moonshot)",
+ "base_url": "https://api.kimi.com/coding/",
+ "requires_auth": True,
+ "auth_env_var": "ANTHROPIC_API_KEY",
+ "models": [{"id": "kimi-k2.5", "name": "Kimi K2.5"}],
+ "default_model": "kimi-k2.5",
+ },
+ "glm": {
+ "name": "GLM (Zhipu AI)",
+ "base_url": "https://api.z.ai/api/anthropic",
+ "requires_auth": True,
+ "auth_env_var": "ANTHROPIC_AUTH_TOKEN",
+ "models": [
+ {"id": "glm-5", "name": "GLM 5"},
+ {"id": "glm-4.7", "name": "GLM 4.7"},
+ {"id": "glm-4.5-air", "name": "GLM 4.5 Air"},
+ ],
+ "default_model": "glm-4.7",
+ },
+ "azure": {
+ "name": "Azure Anthropic (Claude)",
+ "base_url": "",
+ "requires_auth": True,
+ "auth_env_var": "ANTHROPIC_API_KEY",
+ "models": [
+ {"id": "claude-opus-4-7", "name": "Claude Opus"},
+ {"id": "claude-sonnet-4-6", "name": "Claude Sonnet"},
+ {"id": "claude-haiku-4-5", "name": "Claude Haiku"},
+ ],
+ "default_model": "claude-opus-4-7",
+ },
+ "ollama": {
+ "name": "Ollama (Local)",
+ "base_url": "http://localhost:11434",
+ "requires_auth": False,
+ "models": [
+ {"id": "qwen3-coder", "name": "Qwen3 Coder"},
+ {"id": "deepseek-coder-v2", "name": "DeepSeek Coder V2"},
+ ],
+ "default_model": "qwen3-coder",
+ },
+ "custom": {
+ "name": "Custom Provider",
+ "base_url": "",
+ "requires_auth": True,
+ "auth_env_var": "ANTHROPIC_AUTH_TOKEN",
+ "models": [],
+ "default_model": "",
+ },
+}
+
+
+def get_effective_sdk_env() -> dict[str, str]:
+ """Build environment variable dict for Claude SDK based on current API provider settings.
+
+ When api_provider is "claude" (or unset), falls back to existing env vars (current behavior).
+ For other providers, builds env dict from stored settings (api_base_url, api_auth_token, api_model).
+
+ Returns:
+ Dict ready to merge into subprocess env or pass to SDK.
+ """
+ all_settings = get_all_settings()
+ provider_id = all_settings.get("api_provider", "claude")
+
+ if provider_id == "claude":
+ # Default behavior: forward existing env vars
+ from env_constants import API_ENV_VARS
+ sdk_env: dict[str, str] = {}
+ for var in API_ENV_VARS:
+ value = os.getenv(var)
+ if value:
+ sdk_env[var] = value
+ return sdk_env
+
+ # Alternative provider: build env from settings
+ provider = API_PROVIDERS.get(provider_id)
+ if not provider:
+ logger.warning("Unknown API provider '%s', falling back to claude", provider_id)
+ from env_constants import API_ENV_VARS
+ sdk_env = {}
+ for var in API_ENV_VARS:
+ value = os.getenv(var)
+ if value:
+ sdk_env[var] = value
+ return sdk_env
+
+ sdk_env = {}
+
+ # Explicitly clear credentials that could leak from the server process env.
+ # For providers using ANTHROPIC_AUTH_TOKEN (GLM, Custom), clear ANTHROPIC_API_KEY.
+ # For providers using ANTHROPIC_API_KEY (Kimi), clear ANTHROPIC_AUTH_TOKEN.
+ # This prevents the Claude CLI from using the wrong credentials.
+ auth_env_var = provider.get("auth_env_var", "ANTHROPIC_AUTH_TOKEN")
+ if auth_env_var == "ANTHROPIC_AUTH_TOKEN":
+ sdk_env["ANTHROPIC_API_KEY"] = ""
+ elif auth_env_var == "ANTHROPIC_API_KEY":
+ sdk_env["ANTHROPIC_AUTH_TOKEN"] = ""
+
+ # Clear Vertex AI vars when using non-Vertex alternative providers
+ sdk_env["CLAUDE_CODE_USE_VERTEX"] = ""
+ sdk_env["CLOUD_ML_REGION"] = ""
+ sdk_env["ANTHROPIC_VERTEX_PROJECT_ID"] = ""
+
+ # Base URL
+ base_url = all_settings.get("api_base_url") or provider.get("base_url")
+ if base_url:
+ sdk_env["ANTHROPIC_BASE_URL"] = base_url
+
+ # Auth token
+ auth_token = all_settings.get("api_auth_token")
+ if auth_token:
+ sdk_env[auth_env_var] = auth_token
+
+ # Model - set all three tier overrides to the same model
+ model = all_settings.get("api_model") or provider.get("default_model")
+ if model:
+ sdk_env["ANTHROPIC_DEFAULT_OPUS_MODEL"] = model
+ sdk_env["ANTHROPIC_DEFAULT_SONNET_MODEL"] = model
+ sdk_env["ANTHROPIC_DEFAULT_HAIKU_MODEL"] = model
+
+ # Timeout
+ timeout = all_settings.get("api_timeout_ms")
+ if timeout:
+ sdk_env["API_TIMEOUT_MS"] = timeout
+
+ return sdk_env
diff --git a/requirements-prod.txt b/requirements-prod.txt
new file mode 100644
index 000000000..12d5d327c
--- /dev/null
+++ b/requirements-prod.txt
@@ -0,0 +1,18 @@
+# Production runtime dependencies only
+# For development, use requirements.txt (includes ruff, mypy, pytest)
+claude-agent-sdk>=0.1.39,<0.2.0
+python-dotenv>=1.0.0
+sqlalchemy>=2.0.0
+fastapi>=0.115.0
+uvicorn[standard]>=0.32.0
+websockets>=13.0
+python-multipart>=0.0.17
+psutil>=6.0.0
+aiofiles>=24.0.0
+apscheduler>=3.10.0,<4.0.0
+pywinpty>=2.0.0; sys_platform == "win32"
+pyyaml>=6.0.0
+python-docx>=1.1.0
+openpyxl>=3.1.0
+PyPDF2>=3.0.0
+python-pptx>=1.0.0
diff --git a/requirements.txt b/requirements.txt
index a12673dda..23c9afdbd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-claude-agent-sdk>=0.1.0
+claude-agent-sdk>=0.1.39,<0.2.0
python-dotenv>=1.0.0
sqlalchemy>=2.0.0
fastapi>=0.115.0
@@ -7,8 +7,16 @@ websockets>=13.0
python-multipart>=0.0.17
psutil>=6.0.0
aiofiles>=24.0.0
+apscheduler>=3.10.0,<4.0.0
+pywinpty>=2.0.0; sys_platform == "win32"
+pyyaml>=6.0.0
+python-docx>=1.1.0
+openpyxl>=3.1.0
+PyPDF2>=3.0.0
+python-pptx>=1.0.0
# Dev dependencies
ruff>=0.8.0
mypy>=1.13.0
pytest>=8.0.0
+types-PyYAML>=6.0.0
diff --git a/security.py b/security.py
index 4e03117e0..9d928b5a9 100644
--- a/security.py
+++ b/security.py
@@ -6,8 +6,21 @@
Uses an allowlist approach - only explicitly permitted commands can run.
"""
+import logging
import os
+import re
import shlex
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+# Logger for security-related events (fallback parsing, validation failures, etc.)
+logger = logging.getLogger(__name__)
+
+# Regex pattern for valid pkill process names (no regex metacharacters allowed)
+# Matches alphanumeric names with dots, underscores, and hyphens
+VALID_PROCESS_NAME_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$")
# Allowed commands for development tasks
# Minimal set needed for the autonomous coding demo
@@ -53,10 +66,79 @@
"bash",
# Script execution
"init.sh", # Init scripts; validated separately
+ # Browser automation
+ "playwright-cli", # Playwright CLI for browser testing; validated separately
}
# Commands that need additional validation even when in the allowlist
-COMMANDS_NEEDING_EXTRA_VALIDATION = {"pkill", "chmod", "init.sh"}
+COMMANDS_NEEDING_EXTRA_VALIDATION = {"pkill", "chmod", "init.sh", "playwright-cli"}
+
+# Commands that are NEVER allowed, even with user approval
+# These commands can cause permanent system damage or security breaches
+BLOCKED_COMMANDS = {
+ # Disk operations
+ "dd",
+ "mkfs",
+ "fdisk",
+ "parted",
+ # System control
+ "shutdown",
+ "reboot",
+ "poweroff",
+ "halt",
+ "init",
+ # Ownership changes
+ "chown",
+ "chgrp",
+ # System services
+ "systemctl",
+ "service",
+ "launchctl",
+ # Network security
+ "iptables",
+ "ufw",
+}
+
+# Sensitive directories (relative to home) that should never be exposed.
+# Used by both the EXTRA_READ_PATHS validator (client.py) and the filesystem
+# browser API (server/routers/filesystem.py) to block credential/key directories.
+# This is the single source of truth -- import from here in both places.
+#
+# SENSITIVE_DIRECTORIES is the union of the previous filesystem browser blocklist
+# (filesystem.py) and the previous EXTRA_READ_PATHS blocklist (client.py).
+# Some entries are new to each consumer -- this is intentional for defense-in-depth.
+SENSITIVE_DIRECTORIES = {
+ ".ssh",
+ ".aws",
+ ".azure",
+ ".kube",
+ ".gnupg",
+ ".gpg",
+ ".password-store",
+ ".docker",
+ ".config/gcloud",
+ ".config/gh",
+ ".npmrc",
+ ".pypirc",
+ ".netrc",
+ ".terraform",
+}
+
+# Commands that trigger emphatic warnings but CAN be approved (Phase 3)
+# For now, these are blocked like BLOCKED_COMMANDS until Phase 3 implements approval
+DANGEROUS_COMMANDS = {
+ # Privilege escalation
+ "sudo",
+ "su",
+ "doas",
+ # Cloud CLIs (can modify production infrastructure)
+ "aws",
+ "gcloud",
+ "az",
+ # Container and orchestration
+ "kubectl",
+ "docker-compose",
+}
def split_command_segments(command_string: str) -> list[str]:
@@ -89,6 +171,45 @@ def split_command_segments(command_string: str) -> list[str]:
return result
+def _extract_primary_command(segment: str) -> str | None:
+ """
+ Fallback command extraction when shlex fails.
+
+ Extracts the first word that looks like a command, handling cases
+ like complex docker exec commands with nested quotes.
+
+ Args:
+ segment: The command segment to parse
+
+ Returns:
+ The primary command name, or None if extraction fails
+ """
+ # Remove leading whitespace
+ segment = segment.lstrip()
+
+ if not segment:
+ return None
+
+ # Skip env var assignments at start (VAR=value cmd)
+ words = segment.split()
+ while words and "=" in words[0] and not words[0].startswith("="):
+ words = words[1:]
+
+ if not words:
+ return None
+
+ # Extract first token (the command)
+ first_word = words[0]
+
+ # Match valid command characters (alphanumeric, dots, underscores, hyphens, slashes)
+ match = re.match(r"^([a-zA-Z0-9_./-]+)", first_word)
+ if match:
+ cmd = match.group(1)
+ return os.path.basename(cmd)
+
+ return None
+
+
def extract_commands(command_string: str) -> list[str]:
"""
Extract command names from a shell command string.
@@ -105,7 +226,6 @@ def extract_commands(command_string: str) -> list[str]:
commands = []
# shlex doesn't treat ; as a separator, so we need to pre-process
- import re
# Split on semicolons that aren't inside quotes (simple heuristic)
# This handles common cases like "echo hello; ls"
@@ -120,8 +240,21 @@ def extract_commands(command_string: str) -> list[str]:
tokens = shlex.split(segment)
except ValueError:
# Malformed command (unclosed quotes, etc.)
- # Return empty to trigger block (fail-safe)
- return []
+ # Try fallback extraction instead of blocking entirely
+ fallback_cmd = _extract_primary_command(segment)
+ if fallback_cmd:
+ logger.debug(
+ "shlex fallback used: segment=%r -> command=%r",
+ segment,
+ fallback_cmd,
+ )
+ commands.append(fallback_cmd)
+ else:
+ logger.debug(
+ "shlex fallback failed: segment=%r (no command extracted)",
+ segment,
+ )
+ continue
if not tokens:
continue
@@ -173,23 +306,37 @@ def extract_commands(command_string: str) -> list[str]:
return commands
-def validate_pkill_command(command_string: str) -> tuple[bool, str]:
+# Default pkill process names (hardcoded baseline, always available)
+DEFAULT_PKILL_PROCESSES = {
+ "node",
+ "npm",
+ "npx",
+ "vite",
+ "next",
+}
+
+
+def validate_pkill_command(
+ command_string: str,
+ extra_processes: Optional[set[str]] = None
+) -> tuple[bool, str]:
"""
Validate pkill commands - only allow killing dev-related processes.
Uses shlex to parse the command, avoiding regex bypass vulnerabilities.
+ Args:
+ command_string: The pkill command to validate
+ extra_processes: Optional set of additional process names to allow
+ (from org/project config pkill_processes)
+
Returns:
Tuple of (is_allowed, reason_if_blocked)
"""
- # Allowed process names for pkill
- allowed_process_names = {
- "node",
- "npm",
- "npx",
- "vite",
- "next",
- }
+ # Merge default processes with any extra configured processes
+ allowed_process_names = DEFAULT_PKILL_PROCESSES.copy()
+ if extra_processes:
+ allowed_process_names |= extra_processes
try:
tokens = shlex.split(command_string)
@@ -208,17 +355,19 @@ def validate_pkill_command(command_string: str) -> tuple[bool, str]:
if not args:
return False, "pkill requires a process name"
- # The target is typically the last non-flag argument
- target = args[-1]
-
- # For -f flag (full command line match), extract the first word as process name
- # e.g., "pkill -f 'node server.js'" -> target is "node server.js", process is "node"
- if " " in target:
- target = target.split()[0]
-
- if target in allowed_process_names:
+ # Validate every non-flag argument (pkill accepts multiple patterns on BSD)
+ # This defensively ensures no disallowed process can be targeted
+ targets = []
+ for arg in args:
+ # For -f flag (full command line match), take the first word as process name
+ # e.g., "pkill -f 'node server.js'" -> target is "node server.js", process is "node"
+ t = arg.split()[0] if " " in arg else arg
+ targets.append(t)
+
+ disallowed = [t for t in targets if t not in allowed_process_names]
+ if not disallowed:
return True, ""
- return False, f"pkill only allowed for dev processes: {allowed_process_names}"
+ return False, f"pkill only allowed for processes: {sorted(allowed_process_names)}"
def validate_chmod_command(command_string: str) -> tuple[bool, str]:
@@ -291,34 +440,469 @@ def validate_init_script(command_string: str) -> tuple[bool, str]:
return False, f"Only ./init.sh is allowed, got: {script}"
-def get_command_for_validation(cmd: str, segments: list[str]) -> str:
+def validate_playwright_command(command_string: str) -> tuple[bool, str]:
+ """
+ Validate playwright-cli commands - block dangerous subcommands.
+
+ Blocks `run-code` (arbitrary Node.js execution) and `eval` (arbitrary JS
+ evaluation) which bypass the security sandbox.
+
+ Returns:
+ Tuple of (is_allowed, reason_if_blocked)
+ """
+ try:
+ tokens = shlex.split(command_string)
+ except ValueError:
+ return False, "Could not parse playwright-cli command"
+
+ if not tokens:
+ return False, "Empty command"
+
+ BLOCKED_SUBCOMMANDS = {"run-code", "eval"}
+
+ # Find the subcommand: first non-flag token after 'playwright-cli'
+ for token in tokens[1:]:
+ if token.startswith("-"):
+ continue # skip flags like -s=agent-1
+ if token in BLOCKED_SUBCOMMANDS:
+ return False, f"playwright-cli '{token}' is not allowed"
+ break # first non-flag token is the subcommand
+
+ return True, ""
+
+
+def matches_pattern(command: str, pattern: str) -> bool:
"""
- Find the specific command segment that contains the given command.
+ Check if a command matches a pattern.
+
+ Supports:
+ - Exact match: "swift"
+ - Prefix wildcard: "swift*" matches "swift", "swiftc", "swiftformat"
+ - Local script paths: "./scripts/build.sh" or "scripts/test.sh"
Args:
- cmd: The command name to find
- segments: List of command segments
+ command: The command to check
+ pattern: The pattern to match against
Returns:
- The segment containing the command, or empty string if not found
+ True if command matches pattern
"""
- for segment in segments:
- segment_commands = extract_commands(segment)
- if cmd in segment_commands:
- return segment
- return ""
+ # Reject bare wildcards - security measure to prevent matching everything
+ if pattern == "*":
+ return False
+
+ # Exact match
+ if command == pattern:
+ return True
+
+ # Prefix wildcard (e.g., "swift*" matches "swiftc", "swiftlint")
+ if pattern.endswith("*"):
+ prefix = pattern[:-1]
+ # Also reject if prefix is empty (would be bare "*")
+ if not prefix:
+ return False
+ return command.startswith(prefix)
+
+ # Path patterns (./scripts/build.sh, scripts/test.sh, etc.)
+ if "/" in pattern:
+ # Extract the script name from the pattern
+ pattern_name = os.path.basename(pattern)
+ return command == pattern or command == pattern_name or command.endswith("/" + pattern_name)
+
+ return False
+
+
+def _validate_command_list(commands: list, config_path: Path, field_name: str) -> bool:
+ """
+ Validate a list of command entries from a YAML config.
+
+ Each entry must be a dict with a non-empty string 'name' field.
+ Used by both load_org_config() and load_project_commands() to avoid
+ duplicating the same validation logic.
+
+ Args:
+ commands: List of command entries to validate
+ config_path: Path to the config file (for log messages)
+ field_name: Name of the YAML field being validated (e.g., 'allowed_commands', 'commands')
+
+ Returns:
+ True if all entries are valid, False otherwise
+ """
+ if not isinstance(commands, list):
+ logger.warning(f"Config at {config_path}: '{field_name}' must be a list")
+ return False
+ for i, cmd in enumerate(commands):
+ if not isinstance(cmd, dict):
+ logger.warning(f"Config at {config_path}: {field_name}[{i}] must be a dict")
+ return False
+ if "name" not in cmd:
+ logger.warning(f"Config at {config_path}: {field_name}[{i}] missing 'name'")
+ return False
+ if not isinstance(cmd["name"], str) or cmd["name"].strip() == "":
+ logger.warning(f"Config at {config_path}: {field_name}[{i}] has invalid 'name'")
+ return False
+ return True
+
+
+def _validate_pkill_processes(config: dict, config_path: Path) -> Optional[list[str]]:
+ """
+ Validate and normalize pkill_processes from a YAML config.
+
+ Each entry must be a non-empty string matching VALID_PROCESS_NAME_PATTERN
+ (alphanumeric, dots, underscores, hyphens only -- no regex metacharacters).
+ Used by both load_org_config() and load_project_commands().
+
+ Args:
+ config: Parsed YAML config dict that may contain 'pkill_processes'
+ config_path: Path to the config file (for log messages)
+
+ Returns:
+ Normalized list of process names, or None if validation fails.
+ Returns an empty list if 'pkill_processes' is not present.
+ """
+ if "pkill_processes" not in config:
+ return []
+
+ processes = config["pkill_processes"]
+ if not isinstance(processes, list):
+ logger.warning(f"Config at {config_path}: 'pkill_processes' must be a list")
+ return None
+
+ normalized = []
+ for i, proc in enumerate(processes):
+ if not isinstance(proc, str):
+ logger.warning(f"Config at {config_path}: pkill_processes[{i}] must be a string")
+ return None
+ proc = proc.strip()
+ if not proc or not VALID_PROCESS_NAME_PATTERN.fullmatch(proc):
+ logger.warning(f"Config at {config_path}: pkill_processes[{i}] has invalid value '{proc}'")
+ return None
+ normalized.append(proc)
+ return normalized
+
+
+def get_org_config_path() -> Path:
+ """
+ Get the organization-level config file path.
+
+ Returns:
+ Path to ~/.autoforge/config.yaml (falls back to ~/.autocoder/config.yaml)
+ """
+ new_path = Path.home() / ".autoforge" / "config.yaml"
+ if new_path.exists():
+ return new_path
+ # Backward compatibility: check old location
+ old_path = Path.home() / ".autocoder" / "config.yaml"
+ if old_path.exists():
+ return old_path
+ return new_path
+
+
+def load_org_config() -> Optional[dict]:
+ """
+ Load organization-level config from ~/.autoforge/config.yaml.
+
+ Falls back to ~/.autocoder/config.yaml for backward compatibility.
+
+ Returns:
+ Dict with parsed org config, or None if file doesn't exist or is invalid
+ """
+ config_path = get_org_config_path()
+
+ if not config_path.exists():
+ return None
+
+ try:
+ with open(config_path, "r", encoding="utf-8") as f:
+ config = yaml.safe_load(f)
+
+ if not config:
+ logger.warning(f"Org config at {config_path} is empty")
+ return None
+
+ # Validate structure
+ if not isinstance(config, dict):
+ logger.warning(f"Org config at {config_path} must be a YAML dictionary")
+ return None
+
+ if "version" not in config:
+ logger.warning(f"Org config at {config_path} missing required 'version' field")
+ return None
+
+ # Validate allowed_commands if present
+ if "allowed_commands" in config:
+ if not _validate_command_list(config["allowed_commands"], config_path, "allowed_commands"):
+ return None
+
+ # Validate blocked_commands if present
+ if "blocked_commands" in config:
+ blocked = config["blocked_commands"]
+ if not isinstance(blocked, list):
+ logger.warning(f"Org config at {config_path}: 'blocked_commands' must be a list")
+ return None
+ for i, cmd in enumerate(blocked):
+ if not isinstance(cmd, str):
+ logger.warning(f"Org config at {config_path}: blocked_commands[{i}] must be a string")
+ return None
+
+ # Validate pkill_processes if present
+ normalized = _validate_pkill_processes(config, config_path)
+ if normalized is None:
+ return None
+ if normalized:
+ config["pkill_processes"] = normalized
+
+ return config
+
+ except yaml.YAMLError as e:
+ logger.warning(f"Failed to parse org config at {config_path}: {e}")
+ return None
+ except (IOError, OSError) as e:
+ logger.warning(f"Failed to read org config at {config_path}: {e}")
+ return None
+
+
+def load_project_commands(project_dir: Path) -> Optional[dict]:
+ """
+ Load allowed commands from project-specific YAML config.
+
+ Args:
+ project_dir: Path to the project directory
+
+ Returns:
+ Dict with parsed YAML config, or None if file doesn't exist or is invalid
+ """
+ # Check new location first, fall back to old for backward compatibility
+ config_path = project_dir.resolve() / ".autoforge" / "allowed_commands.yaml"
+ if not config_path.exists():
+ config_path = project_dir.resolve() / ".autocoder" / "allowed_commands.yaml"
+
+ if not config_path.exists():
+ return None
+
+ try:
+ with open(config_path, "r", encoding="utf-8") as f:
+ config = yaml.safe_load(f)
+
+ if not config:
+ logger.warning(f"Project config at {config_path} is empty")
+ return None
+
+ # Validate structure
+ if not isinstance(config, dict):
+ logger.warning(f"Project config at {config_path} must be a YAML dictionary")
+ return None
+
+ if "version" not in config:
+ logger.warning(f"Project config at {config_path} missing required 'version' field")
+ return None
+
+ commands = config.get("commands", [])
+
+ # Enforce 100 command limit
+ if isinstance(commands, list) and len(commands) > 100:
+ logger.warning(f"Project config at {config_path} exceeds 100 command limit ({len(commands)} commands)")
+ return None
+
+ # Validate each command entry using shared helper
+ if not _validate_command_list(commands, config_path, "commands"):
+ return None
+
+ # Validate pkill_processes if present
+ normalized = _validate_pkill_processes(config, config_path)
+ if normalized is None:
+ return None
+ if normalized:
+ config["pkill_processes"] = normalized
+
+ return config
+
+ except yaml.YAMLError as e:
+ logger.warning(f"Failed to parse project config at {config_path}: {e}")
+ return None
+ except (IOError, OSError) as e:
+ logger.warning(f"Failed to read project config at {config_path}: {e}")
+ return None
+
+
+def validate_project_command(cmd_config: dict) -> tuple[bool, str]:
+ """
+ Validate a single command entry from project config.
+
+ Checks that the command has a valid name and is not in any blocklist.
+ Called during hierarchy resolution to gate each project command before
+ it is added to the effective allowed set.
+
+ Args:
+ cmd_config: Dict with command configuration (name, description)
+
+ Returns:
+ Tuple of (is_valid, error_message)
+ """
+ if not isinstance(cmd_config, dict):
+ return False, "Command must be a dict"
+
+ if "name" not in cmd_config:
+ return False, "Command must have 'name' field"
+
+ name = cmd_config["name"]
+ if not isinstance(name, str) or not name:
+ return False, "Command name must be a non-empty string"
+
+ # Reject bare wildcard - security measure to prevent matching all commands
+ if name == "*":
+ return False, "Bare wildcard '*' is not allowed (security risk: matches all commands)"
+
+ # Check if command is in the blocklist or dangerous commands
+ base_cmd = os.path.basename(name.rstrip("*"))
+ if base_cmd in BLOCKED_COMMANDS:
+ return False, f"Command '{name}' is in the blocklist and cannot be allowed"
+ if base_cmd in DANGEROUS_COMMANDS:
+ return False, f"Command '{name}' is in the blocklist and cannot be allowed"
+
+ # Description is optional
+ if "description" in cmd_config and not isinstance(cmd_config["description"], str):
+ return False, "Description must be a string"
+
+ return True, ""
+
+
+def get_effective_commands(project_dir: Optional[Path]) -> tuple[set[str], set[str]]:
+ """
+ Get effective allowed and blocked commands after hierarchy resolution.
+
+ Hierarchy (highest to lowest priority):
+ 1. BLOCKED_COMMANDS (hardcoded) - always blocked
+ 2. Org blocked_commands - cannot be unblocked
+ 3. Org allowed_commands - adds to global
+ 4. Project allowed_commands - adds to global + org
+
+ Args:
+ project_dir: Path to the project directory, or None
+
+ Returns:
+ Tuple of (allowed_commands, blocked_commands)
+ """
+ # Start with global allowed commands
+ allowed = ALLOWED_COMMANDS.copy()
+ blocked = BLOCKED_COMMANDS.copy()
+
+ # Add dangerous commands to blocked (Phase 3 will add approval flow)
+ blocked |= DANGEROUS_COMMANDS
+
+ # Load org config and apply
+ org_config = load_org_config()
+ if org_config:
+ # Add org-level blocked commands (cannot be overridden)
+ org_blocked = org_config.get("blocked_commands", [])
+ blocked |= set(org_blocked)
+
+ # Add org-level allowed commands
+ for cmd_config in org_config.get("allowed_commands", []):
+ if isinstance(cmd_config, dict) and "name" in cmd_config:
+ allowed.add(cmd_config["name"])
+
+ # Load project config and apply
+ if project_dir:
+ project_config = load_project_commands(project_dir)
+ if project_config:
+ # Add project-specific commands
+ for cmd_config in project_config.get("commands", []):
+ valid, error = validate_project_command(cmd_config)
+ if valid:
+ allowed.add(cmd_config["name"])
+
+ # Remove blocked commands from allowed (blocklist takes precedence)
+ allowed -= blocked
+
+ return allowed, blocked
+
+
+def get_project_allowed_commands(project_dir: Optional[Path]) -> set[str]:
+ """
+ Get the set of allowed commands for a project.
+
+ Uses hierarchy resolution from get_effective_commands().
+
+ Args:
+ project_dir: Path to the project directory, or None
+
+ Returns:
+ Set of allowed command names (including patterns)
+ """
+ allowed, blocked = get_effective_commands(project_dir)
+ return allowed
+
+
+def get_effective_pkill_processes(project_dir: Optional[Path]) -> set[str]:
+ """
+ Get effective pkill process names after hierarchy resolution.
+
+ Merges processes from:
+ 1. DEFAULT_PKILL_PROCESSES (hardcoded baseline)
+ 2. Org config pkill_processes
+ 3. Project config pkill_processes
+
+ Args:
+ project_dir: Path to the project directory, or None
+
+ Returns:
+ Set of allowed process names for pkill
+ """
+ # Start with default processes
+ processes = DEFAULT_PKILL_PROCESSES.copy()
+
+ # Add org-level pkill_processes
+ org_config = load_org_config()
+ if org_config:
+ org_processes = org_config.get("pkill_processes", [])
+ if isinstance(org_processes, list):
+ processes |= {p for p in org_processes if isinstance(p, str) and p.strip()}
+
+ # Add project-level pkill_processes
+ if project_dir:
+ project_config = load_project_commands(project_dir)
+ if project_config:
+ proj_processes = project_config.get("pkill_processes", [])
+ if isinstance(proj_processes, list):
+ processes |= {p for p in proj_processes if isinstance(p, str) and p.strip()}
+
+ return processes
+
+
+def is_command_allowed(command: str, allowed_commands: set[str]) -> bool:
+ """
+ Check if a command is allowed (supports patterns).
+
+ Args:
+ command: The command to check
+ allowed_commands: Set of allowed commands (may include patterns)
+
+ Returns:
+ True if command is allowed
+ """
+ # Check exact match first
+ if command in allowed_commands:
+ return True
+
+ # Check pattern matches
+ for pattern in allowed_commands:
+ if matches_pattern(command, pattern):
+ return True
+
+ return False
async def bash_security_hook(input_data, tool_use_id=None, context=None):
"""
Pre-tool-use hook that validates bash commands using an allowlist.
- Only commands in ALLOWED_COMMANDS are permitted.
+ Only commands in ALLOWED_COMMANDS and project-specific commands are permitted.
Args:
input_data: Dict containing tool_name and tool_input
tool_use_id: Optional tool use ID
- context: Optional context
+ context: Optional context dict with 'project_dir' key
Returns:
Empty dict to allow, or {"decision": "block", "reason": "..."} to block
@@ -340,26 +924,60 @@ async def bash_security_hook(input_data, tool_use_id=None, context=None):
"reason": f"Could not parse command for security validation: {command}",
}
+ # Get project directory from context
+ project_dir = None
+ if context and isinstance(context, dict):
+ project_dir_str = context.get("project_dir")
+ if project_dir_str:
+ project_dir = Path(project_dir_str)
+
+ # Get effective commands using hierarchy resolution
+ allowed_commands, blocked_commands = get_effective_commands(project_dir)
+
+ # Get effective pkill processes (includes org/project config)
+ pkill_processes = get_effective_pkill_processes(project_dir)
+
# Split into segments for per-command validation
segments = split_command_segments(command)
- # Check each command against the allowlist
+ # Check each command against the blocklist and allowlist
for cmd in commands:
- if cmd not in ALLOWED_COMMANDS:
+ # Check blocklist first (highest priority)
+ if cmd in blocked_commands:
+ return {
+ "decision": "block",
+ "reason": f"Command '{cmd}' is blocked at organization level and cannot be approved.",
+ }
+
+ # Check allowlist (with pattern matching)
+ if not is_command_allowed(cmd, allowed_commands):
+ # Provide helpful error message with config hint
+ error_msg = f"Command '{cmd}' is not allowed.\n"
+ error_msg += "To allow this command:\n"
+ error_msg += " 1. Add to .autoforge/allowed_commands.yaml for this project, OR\n"
+ error_msg += " 2. Request mid-session approval (the agent can ask)\n"
+ error_msg += "Note: Some commands are blocked at org-level and cannot be overridden."
return {
"decision": "block",
- "reason": f"Command '{cmd}' is not in the allowed commands list",
+ "reason": error_msg,
}
# Additional validation for sensitive commands
if cmd in COMMANDS_NEEDING_EXTRA_VALIDATION:
- # Find the specific segment containing this command
- cmd_segment = get_command_for_validation(cmd, segments)
+ # Find the specific segment containing this command by searching
+ # each segment's extracted commands for a match
+ cmd_segment = ""
+ for segment in segments:
+ if cmd in extract_commands(segment):
+ cmd_segment = segment
+ break
if not cmd_segment:
cmd_segment = command # Fallback to full command
if cmd == "pkill":
- allowed, reason = validate_pkill_command(cmd_segment)
+ # Pass configured extra processes (beyond defaults)
+ extra_procs = pkill_processes - DEFAULT_PKILL_PROCESSES
+ allowed, reason = validate_pkill_command(cmd_segment, extra_procs if extra_procs else None)
if not allowed:
return {"decision": "block", "reason": reason}
elif cmd == "chmod":
@@ -370,5 +988,9 @@ async def bash_security_hook(input_data, tool_use_id=None, context=None):
allowed, reason = validate_init_script(cmd_segment)
if not allowed:
return {"decision": "block", "reason": reason}
+ elif cmd == "playwright-cli":
+ allowed, reason = validate_playwright_command(cmd_segment)
+ if not allowed:
+ return {"decision": "block", "reason": reason}
return {}
diff --git a/server/__init__.py b/server/__init__.py
index 6db079369..e2558b49d 100644
--- a/server/__init__.py
+++ b/server/__init__.py
@@ -6,3 +6,12 @@
Provides REST API and WebSocket endpoints for project management,
feature tracking, and agent control.
"""
+
+# Fix Windows asyncio subprocess support - MUST be before any other imports
+# that might create an event loop
+import sys
+
+if sys.platform == "win32":
+ import asyncio
+
+ asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
diff --git a/server/main.py b/server/main.py
index f48e9f2ef..1ea369764 100644
--- a/server/main.py
+++ b/server/main.py
@@ -6,10 +6,35 @@
Provides REST API, WebSocket, and static file serving.
"""
+import asyncio
+import logging
+import os
import shutil
+import sys
from contextlib import asynccontextmanager
from pathlib import Path
+# Fail fast on unsupported Python versions. Older interpreters surface as
+# opaque Claude SDK errors (e.g. "Control request timeout: initialize").
+if sys.version_info < (3, 11):
+ sys.exit(
+ "ERROR: AutoForge requires Python 3.11 or newer "
+ f"(you are running Python {sys.version.split()[0]}). "
+ "Older versions cause opaque Claude SDK errors such as "
+ "'Control request timeout: initialize'. "
+ "Recreate your virtual environment with Python 3.11+ "
+ "(python3.11 -m venv venv && pip install -r requirements.txt)."
+ )
+
+# Fix for Windows subprocess support in asyncio
+if sys.platform == "win32":
+ asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
+from dotenv import load_dotenv
+
+# Load environment variables from .env file if present
+load_dotenv()
+
from fastapi import FastAPI, HTTPException, Request, WebSocket
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse
@@ -18,29 +43,67 @@
from .routers import (
agent_router,
assistant_chat_router,
+ devserver_router,
+ expand_project_router,
features_router,
filesystem_router,
projects_router,
+ scaffold_router,
+ schedules_router,
+ settings_router,
spec_creation_router,
+ terminal_router,
)
from .schemas import SetupStatus
from .services.assistant_chat_session import cleanup_all_sessions as cleanup_assistant_sessions
-from .services.process_manager import cleanup_all_managers
+from .services.chat_constants import ROOT_DIR
+from .services.dev_server_manager import (
+ cleanup_all_devservers,
+ cleanup_orphaned_devserver_locks,
+)
+from .services.expand_chat_session import cleanup_all_expand_sessions
+from .services.process_manager import cleanup_all_managers, cleanup_orphaned_locks
+from .services.scheduler_service import cleanup_scheduler, get_scheduler
+from .services.terminal_manager import cleanup_all_terminals
+from .utils.ws_security import WebSocketOriginMiddleware
from .websocket import project_websocket
# Paths
-ROOT_DIR = Path(__file__).parent.parent
UI_DIST_DIR = ROOT_DIR / "ui" / "dist"
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Lifespan context manager for startup and shutdown."""
- # Startup
+ # Startup - clean up stale temp files (Playwright profiles, .node cache, etc.)
+ try:
+ from temp_cleanup import cleanup_stale_temp
+ stats = cleanup_stale_temp()
+ if stats["dirs_deleted"] > 0 or stats["files_deleted"] > 0:
+ mb_freed = stats["bytes_freed"] / (1024 * 1024)
+ logger.info("Startup temp cleanup: %d dirs, %d files, %.1f MB freed",
+ stats["dirs_deleted"], stats["files_deleted"], mb_freed)
+ except Exception as e:
+ logger.warning("Startup temp cleanup failed (non-fatal): %s", e)
+
+ # Startup - clean up orphaned lock files from previous runs
+ cleanup_orphaned_locks()
+ cleanup_orphaned_devserver_locks()
+
+ # Start the scheduler service
+ scheduler = get_scheduler()
+ await scheduler.start()
+
yield
- # Shutdown - cleanup all running agents and assistant sessions
+
+ # Shutdown - cleanup scheduler first to stop triggering new starts
+ await cleanup_scheduler()
+ # Then cleanup all running agents, sessions, terminals, and dev servers
await cleanup_all_managers()
await cleanup_assistant_sessions()
+ await cleanup_all_expand_sessions()
+ await cleanup_all_terminals()
+ await cleanup_all_devservers()
# Create FastAPI app
@@ -51,35 +114,65 @@ async def lifespan(app: FastAPI):
lifespan=lifespan,
)
-# CORS - allow only localhost origins for security
-app.add_middleware(
- CORSMiddleware,
- allow_origins=[
- "http://localhost:5173", # Vite dev server
- "http://127.0.0.1:5173",
- "http://localhost:8888", # Production
- "http://127.0.0.1:8888",
- ],
- allow_credentials=True,
- allow_methods=["*"],
- allow_headers=["*"],
-)
+# Module logger
+logger = logging.getLogger(__name__)
+
+# Check if remote access is enabled via environment variable
+# Set by start_ui.py when --host is not 127.0.0.1
+ALLOW_REMOTE = os.environ.get("AUTOFORGE_ALLOW_REMOTE", "").lower() in ("1", "true", "yes")
+
+if ALLOW_REMOTE:
+ logger.warning(
+ "ALLOW_REMOTE is enabled. Terminal WebSocket is exposed without sandboxing. "
+ "Only use this in trusted network environments."
+ )
+
+# CORS - allow all origins when remote access is enabled, otherwise localhost only
+if ALLOW_REMOTE:
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"], # Allow all origins for remote access
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+else:
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=[
+ "http://localhost:5173", # Vite dev server
+ "http://127.0.0.1:5173",
+ "http://localhost:8888", # Production
+ "http://127.0.0.1:8888",
+ ],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
# ============================================================================
# Security Middleware
# ============================================================================
-@app.middleware("http")
-async def require_localhost(request: Request, call_next):
- """Only allow requests from localhost."""
- client_host = request.client.host if request.client else None
+# WebSocket Origin validation (CSWSH protection). Starlette's @app.middleware("http")
+# never runs for WebSocket handshakes, so the require_localhost guard below does not
+# cover WS routes. Browsers do not enforce same-origin on WebSocket connects either,
+# so without this check any web page could hijack the terminal/chat sockets.
+# Registered unconditionally so it applies even when AUTOFORGE_ALLOW_REMOTE=1.
+app.add_middleware(WebSocketOriginMiddleware, allow_remote=ALLOW_REMOTE)
- # Allow localhost connections
- if client_host not in ("127.0.0.1", "::1", "localhost", None):
- raise HTTPException(status_code=403, detail="Localhost access only")
+if not ALLOW_REMOTE:
+ @app.middleware("http")
+ async def require_localhost(request: Request, call_next):
+ """Only allow requests from localhost (disabled when AUTOFORGE_ALLOW_REMOTE=1)."""
+ client_host = request.client.host if request.client else None
- return await call_next(request)
+ # Allow localhost connections
+ if client_host not in ("127.0.0.1", "::1", "localhost", None):
+ raise HTTPException(status_code=403, detail="Localhost access only")
+
+ return await call_next(request)
# ============================================================================
@@ -89,9 +182,15 @@ async def require_localhost(request: Request, call_next):
app.include_router(projects_router)
app.include_router(features_router)
app.include_router(agent_router)
+app.include_router(schedules_router)
+app.include_router(devserver_router)
app.include_router(spec_creation_router)
+app.include_router(expand_project_router)
app.include_router(filesystem_router)
app.include_router(assistant_chat_router)
+app.include_router(settings_router)
+app.include_router(terminal_router)
+app.include_router(scaffold_router)
# ============================================================================
@@ -120,9 +219,15 @@ async def setup_status():
# Check for Claude CLI
claude_cli = shutil.which("claude") is not None
- # Check for credentials file
- credentials_path = Path.home() / ".claude" / ".credentials.json"
- credentials = credentials_path.exists()
+ # Check for CLI configuration directory
+ # Note: CLI no longer stores credentials in ~/.claude/.credentials.json
+ # The existence of ~/.claude indicates the CLI has been configured
+ claude_dir = Path.home() / ".claude"
+ has_claude_config = claude_dir.exists() and claude_dir.is_dir()
+
+ # If GLM mode is configured via .env, we have alternative credentials
+ glm_configured = bool(os.getenv("ANTHROPIC_BASE_URL") and os.getenv("ANTHROPIC_AUTH_TOKEN"))
+ credentials = has_claude_config or glm_configured
# Check for Node.js and npm
node = shutil.which("node") is not None
@@ -160,7 +265,14 @@ async def serve_spa(path: str):
raise HTTPException(status_code=404)
# Try to serve the file directly
- file_path = UI_DIST_DIR / path
+ file_path = (UI_DIST_DIR / path).resolve()
+
+ # Ensure resolved path is within UI_DIST_DIR (prevent path traversal)
+ try:
+ file_path.relative_to(UI_DIST_DIR.resolve())
+ except ValueError:
+ raise HTTPException(status_code=404)
+
if file_path.exists() and file_path.is_file():
return FileResponse(file_path)
diff --git a/server/routers/__init__.py b/server/routers/__init__.py
index 48b4f8048..58f2d00dd 100644
--- a/server/routers/__init__.py
+++ b/server/routers/__init__.py
@@ -7,16 +7,28 @@
from .agent import router as agent_router
from .assistant_chat import router as assistant_chat_router
+from .devserver import router as devserver_router
+from .expand_project import router as expand_project_router
from .features import router as features_router
from .filesystem import router as filesystem_router
from .projects import router as projects_router
+from .scaffold import router as scaffold_router
+from .schedules import router as schedules_router
+from .settings import router as settings_router
from .spec_creation import router as spec_creation_router
+from .terminal import router as terminal_router
__all__ = [
"projects_router",
"features_router",
"agent_router",
+ "schedules_router",
+ "devserver_router",
"spec_creation_router",
+ "expand_project_router",
"filesystem_router",
"assistant_chat_router",
+ "settings_router",
+ "terminal_router",
+ "scaffold_router",
]
diff --git a/server/routers/agent.py b/server/routers/agent.py
index d5631fa73..d27d803cd 100644
--- a/server/routers/agent.py
+++ b/server/routers/agent.py
@@ -6,40 +6,54 @@
Uses project registry for path lookups.
"""
-import re
from pathlib import Path
from fastapi import APIRouter, HTTPException
from ..schemas import AgentActionResponse, AgentStartRequest, AgentStatus
+from ..services.chat_constants import ROOT_DIR
from ..services.process_manager import get_manager
+from ..utils.project_helpers import get_project_path as _get_project_path
+from ..utils.validation import validate_project_name
-def _get_project_path(project_name: str) -> Path:
- """Get project path from registry."""
+def _get_settings_defaults() -> tuple[bool, str, int, int, int]:
+ """Get defaults from global settings.
+
+ Returns:
+ Tuple of (yolo_mode, model, testing_agent_ratio, batch_size, testing_batch_size)
+ """
import sys
root = Path(__file__).parent.parent.parent
if str(root) not in sys.path:
sys.path.insert(0, str(root))
- from registry import get_project_path
- return get_project_path(project_name)
+ from registry import DEFAULT_MODEL, get_all_settings
+ settings = get_all_settings()
+ yolo_mode = (settings.get("yolo_mode") or "false").lower() == "true"
+ model = settings.get("api_model") or settings.get("model", DEFAULT_MODEL)
-router = APIRouter(prefix="/api/projects/{project_name}/agent", tags=["agent"])
+ # Parse testing agent settings with defaults
+ try:
+ testing_agent_ratio = int(settings.get("testing_agent_ratio", "1"))
+ except (ValueError, TypeError):
+ testing_agent_ratio = 1
-# Root directory for process manager
-ROOT_DIR = Path(__file__).parent.parent.parent
+ try:
+ batch_size = int(settings.get("batch_size", "3"))
+ except (ValueError, TypeError):
+ batch_size = 3
+ try:
+ testing_batch_size = int(settings.get("testing_batch_size", "3"))
+ except (ValueError, TypeError):
+ testing_batch_size = 3
-def validate_project_name(name: str) -> str:
- """Validate and sanitize project name to prevent path traversal."""
- if not re.match(r'^[a-zA-Z0-9_-]{1,50}$', name):
- raise HTTPException(
- status_code=400,
- detail="Invalid project name"
- )
- return name
+ return yolo_mode, model, testing_agent_ratio, batch_size, testing_batch_size
+
+
+router = APIRouter(prefix="/api/projects/{project_name}/agent", tags=["agent"])
def get_project_manager(project_name: str):
@@ -67,8 +81,12 @@ async def get_agent_status(project_name: str):
return AgentStatus(
status=manager.status,
pid=manager.pid,
- started_at=manager.started_at,
+ started_at=manager.started_at.isoformat() if manager.started_at else None,
yolo_mode=manager.yolo_mode,
+ model=manager.model,
+ parallel_mode=manager.parallel_mode,
+ max_concurrency=manager.max_concurrency,
+ testing_agent_ratio=manager.testing_agent_ratio,
)
@@ -80,7 +98,34 @@ async def start_agent(
"""Start the agent for a project."""
manager = get_project_manager(project_name)
- success, message = await manager.start(yolo_mode=request.yolo_mode)
+ # Get defaults from global settings if not provided in request
+ default_yolo, default_model, default_testing_ratio, default_batch_size, default_testing_batch_size = _get_settings_defaults()
+
+ yolo_mode = request.yolo_mode if request.yolo_mode is not None else default_yolo
+ model = request.model if request.model else default_model
+ max_concurrency = request.max_concurrency or 1
+ testing_agent_ratio = request.testing_agent_ratio if request.testing_agent_ratio is not None else default_testing_ratio
+
+ batch_size = default_batch_size
+ testing_batch_size = default_testing_batch_size
+
+ # Always run headless - the embedded browser view panel replaces desktop windows
+ success, message = await manager.start(
+ yolo_mode=yolo_mode,
+ model=model,
+ max_concurrency=max_concurrency,
+ testing_agent_ratio=testing_agent_ratio,
+ playwright_headless=True,
+ batch_size=batch_size,
+ testing_batch_size=testing_batch_size,
+ )
+
+ # Notify scheduler of manual start (to prevent auto-stop during scheduled window)
+ if success:
+ from ..services.scheduler_service import get_scheduler
+ project_dir = _get_project_path(project_name)
+ if project_dir:
+ get_scheduler().notify_manual_start(project_name, project_dir)
return AgentActionResponse(
success=success,
@@ -96,6 +141,13 @@ async def stop_agent(project_name: str):
success, message = await manager.stop()
+ # Notify scheduler of manual stop (to prevent auto-start during scheduled window)
+ if success:
+ from ..services.scheduler_service import get_scheduler
+ project_dir = _get_project_path(project_name)
+ if project_dir:
+ get_scheduler().notify_manual_stop(project_name, project_dir)
+
return AgentActionResponse(
success=success,
status=manager.status,
@@ -129,3 +181,31 @@ async def resume_agent(project_name: str):
status=manager.status,
message=message,
)
+
+
+@router.post("/graceful-pause", response_model=AgentActionResponse)
+async def graceful_pause_agent(project_name: str):
+ """Request a graceful pause (drain mode) - finish current work then pause."""
+ manager = get_project_manager(project_name)
+
+ success, message = await manager.graceful_pause()
+
+ return AgentActionResponse(
+ success=success,
+ status=manager.status,
+ message=message,
+ )
+
+
+@router.post("/graceful-resume", response_model=AgentActionResponse)
+async def graceful_resume_agent(project_name: str):
+ """Resume from a graceful pause."""
+ manager = get_project_manager(project_name)
+
+ success, message = await manager.graceful_resume()
+
+ return AgentActionResponse(
+ success=success,
+ status=manager.status,
+ message=message,
+ )
diff --git a/server/routers/assistant_chat.py b/server/routers/assistant_chat.py
index dae53b4a8..1c3ece5c8 100644
--- a/server/routers/assistant_chat.py
+++ b/server/routers/assistant_chat.py
@@ -7,8 +7,6 @@
import json
import logging
-import re
-from pathlib import Path
from typing import Optional
from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect
@@ -27,30 +25,13 @@
get_conversation,
get_conversations,
)
+from ..utils.project_helpers import get_project_path as _get_project_path
+from ..utils.validation import validate_project_name
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/assistant", tags=["assistant-chat"])
-# Root directory
-ROOT_DIR = Path(__file__).parent.parent.parent
-
-
-def _get_project_path(project_name: str) -> Optional[Path]:
- """Get project path from registry."""
- import sys
- root = Path(__file__).parent.parent.parent
- if str(root) not in sys.path:
- sys.path.insert(0, str(root))
-
- from registry import get_project_path
- return get_project_path(project_name)
-
-
-def validate_project_name(name: str) -> bool:
- """Validate project name to prevent path traversal."""
- return bool(re.match(r'^[a-zA-Z0-9_-]{1,50}$', name))
-
# ============================================================================
# Pydantic Models
@@ -145,9 +126,9 @@ async def create_project_conversation(project_name: str):
conversation = create_conversation(project_dir, project_name)
return ConversationSummary(
- id=conversation.id,
- project_name=conversation.project_name,
- title=conversation.title,
+ id=int(conversation.id),
+ project_name=str(conversation.project_name),
+ title=str(conversation.title) if conversation.title else None,
created_at=conversation.created_at.isoformat() if conversation.created_at else None,
updated_at=conversation.updated_at.isoformat() if conversation.updated_at else None,
message_count=0,
@@ -226,30 +207,38 @@ async def assistant_chat_websocket(websocket: WebSocket, project_name: str):
Client -> Server:
- {"type": "start", "conversation_id": int | null} - Start/resume session
- {"type": "message", "content": "..."} - Send user message
+ - {"type": "answer", "answers": {...}} - Answer to structured questions
- {"type": "ping"} - Keep-alive ping
Server -> Client:
- {"type": "conversation_created", "conversation_id": int} - New conversation created
- {"type": "text", "content": "..."} - Text chunk from Claude
- {"type": "tool_call", "tool": "...", "input": {...}} - Tool being called
+ - {"type": "question", "questions": [...]} - Structured questions for user
- {"type": "response_done"} - Response complete
- {"type": "error", "content": "..."} - Error message
- {"type": "pong"} - Keep-alive pong
"""
- if not validate_project_name(project_name):
+ # Always accept WebSocket first to avoid opaque 403 errors
+ await websocket.accept()
+
+ try:
+ project_name = validate_project_name(project_name)
+ except HTTPException:
+ await websocket.send_json({"type": "error", "content": "Invalid project name"})
await websocket.close(code=4000, reason="Invalid project name")
return
project_dir = _get_project_path(project_name)
if not project_dir:
+ await websocket.send_json({"type": "error", "content": "Project not found in registry"})
await websocket.close(code=4004, reason="Project not found in registry")
return
if not project_dir.exists():
+ await websocket.send_json({"type": "error", "content": "Project directory not found"})
await websocket.close(code=4004, reason="Project directory not found")
return
-
- await websocket.accept()
logger.info(f"Assistant WebSocket connected for project: {project_name}")
session: Optional[AssistantChatSession] = None
@@ -260,7 +249,7 @@ async def assistant_chat_websocket(websocket: WebSocket, project_name: str):
data = await websocket.receive_text()
message = json.loads(data)
msg_type = message.get("type")
- logger.info(f"Assistant received message type: {msg_type}")
+ logger.debug(f"Assistant received message type: {msg_type}")
if msg_type == "ping":
await websocket.send_json({"type": "pong"})
@@ -269,18 +258,24 @@ async def assistant_chat_websocket(websocket: WebSocket, project_name: str):
elif msg_type == "start":
# Get optional conversation_id to resume
conversation_id = message.get("conversation_id")
+ logger.debug(f"Processing start message with conversation_id={conversation_id}")
try:
# Create a new session
+ logger.debug(f"Creating session for {project_name}")
session = await create_session(
project_name,
project_dir,
conversation_id=conversation_id,
)
+ logger.debug("Session created, starting...")
# Stream the initial greeting
async for chunk in session.start():
+ if logger.isEnabledFor(logging.DEBUG):
+ logger.debug(f"Sending chunk: {chunk.get('type')}")
await websocket.send_json(chunk)
+ logger.debug("Session start complete")
except Exception as e:
logger.exception(f"Error starting assistant session for {project_name}")
await websocket.send_json({
@@ -310,6 +305,34 @@ async def assistant_chat_websocket(websocket: WebSocket, project_name: str):
async for chunk in session.send_message(user_content):
await websocket.send_json(chunk)
+ elif msg_type == "answer":
+ # User answered a structured question
+ if not session:
+ session = get_session(project_name)
+ if not session:
+ await websocket.send_json({
+ "type": "error",
+ "content": "No active session. Send 'start' first."
+ })
+ continue
+
+ # Format the answers as a natural response
+ answers = message.get("answers", {})
+ if isinstance(answers, dict):
+ response_parts = []
+ for question_idx, answer_value in answers.items():
+ if isinstance(answer_value, list):
+ response_parts.append(", ".join(answer_value))
+ else:
+ response_parts.append(str(answer_value))
+ user_response = "; ".join(response_parts) if response_parts else "OK"
+ else:
+ user_response = str(answers)
+
+ # Stream Claude's response
+ async for chunk in session.send_message(user_response):
+ await websocket.send_json(chunk)
+
else:
await websocket.send_json({
"type": "error",
diff --git a/server/routers/devserver.py b/server/routers/devserver.py
new file mode 100644
index 000000000..bc4029c2c
--- /dev/null
+++ b/server/routers/devserver.py
@@ -0,0 +1,436 @@
+"""
+Dev Server Router
+=================
+
+API endpoints for dev server control (start/stop) and configuration.
+Uses project registry for path lookups and project_config for command detection.
+"""
+
+import logging
+import shlex
+import sys
+from pathlib import Path
+
+from fastapi import APIRouter, HTTPException
+
+from ..schemas import (
+ DevServerActionResponse,
+ DevServerConfigResponse,
+ DevServerConfigUpdate,
+ DevServerStartRequest,
+ DevServerStatus,
+)
+from ..services.dev_server_manager import get_devserver_manager
+from ..services.project_config import (
+ clear_dev_command,
+ get_dev_command,
+ get_project_config,
+ set_dev_command,
+)
+from ..utils.project_helpers import get_project_path as _get_project_path
+from ..utils.validation import validate_project_name
+
+# Add root to path for security module import
+_root = Path(__file__).parent.parent.parent
+if str(_root) not in sys.path:
+ sys.path.insert(0, str(_root))
+
+from security import extract_commands, get_effective_commands, is_command_allowed
+
+logger = logging.getLogger(__name__)
+
+
+router = APIRouter(prefix="/api/projects/{project_name}/devserver", tags=["devserver"])
+
+
+def get_project_dir(project_name: str) -> Path:
+ """
+ Get the validated project directory for a project name.
+
+ Args:
+ project_name: Name of the project
+
+ Returns:
+ Path to the project directory
+
+ Raises:
+ HTTPException: If project is not found or directory does not exist
+ """
+ project_name = validate_project_name(project_name)
+ project_dir = _get_project_path(project_name)
+
+ if not project_dir:
+ raise HTTPException(
+ status_code=404,
+ detail=f"Project '{project_name}' not found in registry"
+ )
+
+ if not project_dir.exists():
+ raise HTTPException(
+ status_code=404,
+ detail=f"Project directory not found: {project_dir}"
+ )
+
+ return project_dir
+
+ALLOWED_RUNNERS = {
+ "npm", "pnpm", "yarn", "npx",
+ "uvicorn", "python", "python3",
+ "flask", "poetry",
+ "cargo", "go",
+}
+
+ALLOWED_NPM_SCRIPTS = {"dev", "start", "serve", "develop", "server", "preview"}
+
+# Allowed Python -m modules for dev servers
+ALLOWED_PYTHON_MODULES = {"uvicorn", "flask", "gunicorn", "http.server"}
+
+BLOCKED_SHELLS = {"sh", "bash", "zsh", "cmd", "powershell", "pwsh", "cmd.exe"}
+
+
+def validate_custom_command_strict(cmd: str) -> None:
+ """
+ Strict allowlist validation for dev server commands.
+ Prevents arbitrary command execution (no sh -c, no cmd /c, no python -c, etc.)
+ """
+ if not isinstance(cmd, str) or not cmd.strip():
+ raise ValueError("custom_command cannot be empty")
+
+ argv = shlex.split(cmd, posix=(sys.platform != "win32"))
+ if not argv:
+ raise ValueError("custom_command could not be parsed")
+
+ base = Path(argv[0]).name.lower()
+
+ # Block direct shells / interpreters commonly used for command injection
+ if base in BLOCKED_SHELLS:
+ raise ValueError(f"custom_command runner not allowed: {base}")
+
+ if base not in ALLOWED_RUNNERS:
+ raise ValueError(
+ f"custom_command runner not allowed: {base}. "
+ f"Allowed: {', '.join(sorted(ALLOWED_RUNNERS))}"
+ )
+
+ # Block one-liner execution for python
+ lowered = [a.lower() for a in argv]
+ if base in {"python", "python3"}:
+ if "-c" in lowered:
+ raise ValueError("python -c is not allowed")
+ if len(argv) >= 3 and argv[1] == "-m":
+ # Allow: python -m ...
+ if argv[2] not in ALLOWED_PYTHON_MODULES:
+ raise ValueError(
+ f"python -m {argv[2]} is not allowed. "
+ f"Allowed modules: {', '.join(sorted(ALLOWED_PYTHON_MODULES))}"
+ )
+ elif len(argv) >= 2 and argv[1].endswith(".py"):
+ # Allow: python manage.py runserver, python app.py, etc.
+ pass
+ else:
+ raise ValueError(
+ "Python commands must use 'python -m ...' or 'python ]