Files
coding-agent-gitea/core/dispatcher.py
T
michael c0c9278d75 feat: add web_search and fetch_url research tools (#2)
- Add ResearchTools class in gitea/tools/research_tools.py
  - web_search: DuckDuckGo HTML search, no API key required
  - fetch_url: httpx-based URL fetcher with HTML stripping + truncation
- Wire both tools into planning_tools and coding_tools_list in dispatcher.py
- Add 19 unit tests covering all methods, error paths, and edge cases

---------

Co-authored-by: Michael <michael@example.com>
Reviewed-on: #2
2026-06-28 19:01:55 +02:00

515 lines
26 KiB
Python

"""Dispatches work to a single CodingAgent, one repo at a time."""
import logging
import re
from typing import Any
from core.coding_agent import CodingAgent
from core.queue import WorkItem
from gitea.tools.coding_tools import CodingTools
from gitea.tools.research_tools import ResearchTools
from gitea.tools.gitea_tools import GiteaTools
from gitea.client import GiteaClient
from core.coding_prompt import CODING_AGENT_SYSTEM_PROMPT
from gitea.config import AGENT_MODEL_ID
from gitea.workspace import WorkspaceManager
from gitea.models import CommentModel, PullRequestFileModel, PullRequestModel, IssueModel
logger: logging.Logger = logging.getLogger("agent-dispatcher")
AGENT_USERNAMES: frozenset[str] = frozenset({"meeks-ai", "agent-bot"})
CLOSE_KEYWORDS_PATTERN = re.compile(
rf"\b(?:close|closes|closed|fix|fixes|fixed|resolve|resolves|resolved)\s+#(\d+)\b",
re.IGNORECASE
)
class AgentDispatcher:
"""Dispatches work to a single CodingAgent, one repo at a time."""
def __init__(
self,
client: GiteaClient,
tools: GiteaTools,
model_name: str = AGENT_MODEL_ID,
max_retries: int = 2,
) -> None:
self._client = client
self._tools = tools
self._model_name = model_name
self._max_retries = max_retries
def _find_pr_for_issue(self, repo_full_name: str, issue_number: int) -> PullRequestModel | None:
"""Find an open pull request that addresses the given issue number."""
owner, repo_name = repo_full_name.split("/")
try:
prs = self._client.list_repo_pull_requests(owner, repo_name)
for pr in prs:
ref = pr.head.get("ref", "") if pr.head else ""
if re.search(rf"(?<!\d){issue_number}(?!\d)", ref):
return pr
body = pr.body or ""
title = pr.title or ""
matches = CLOSE_KEYWORDS_PATTERN.findall(body) + CLOSE_KEYWORDS_PATTERN.findall(title)
if any(int(m) == issue_number for m in matches):
return pr
issue_ref_pattern = re.compile(rf"(?<!\w)#{issue_number}\b")
if issue_ref_pattern.search(title) or issue_ref_pattern.search(body):
return pr
except Exception as e:
logger.warning(f"Error checking PRs for issue #{issue_number} in {repo_full_name}: {e}")
return None
def _find_issues_for_pr(self, pr_body: str) -> list[int]:
"""Extract referenced issue numbers from the PR body."""
matches = CLOSE_KEYWORDS_PATTERN.findall(pr_body)
return list(set(int(m) for m in matches))
def _is_awaiting_reply(self, comments: list[CommentModel]) -> bool:
"""Return True if the agent's most recent comment contains the
awaiting-reply marker AND no human has commented after it.
The agent itself embeds the marker when it needs human input.
"""
if not comments:
return False
# Find the last agent comment index
last_agent_idx: int = -1
for i, c in enumerate(comments):
if c.user and c.user.login in AGENT_USERNAMES:
last_agent_idx = i
if last_agent_idx == -1:
return False
last_agent_comment = comments[last_agent_idx]
body = (last_agent_comment.body or "")
# The agent explicitly embeds this marker when it is waiting for input
if "<!-- agent:awaiting-reply -->" not in body:
return False
# Check if any human replied AFTER the last agent comment
for c in comments[last_agent_idx + 1:]:
if c.user and c.user.login not in AGENT_USERNAMES:
return False # Human replied — we can proceed
return True # Agent signalled wait, no human replied yet
async def dispatch(
self,
repo: str,
work_items: list[WorkItem],
) -> list[str]:
"""Dispatch all work for a single repo to a fresh agent, then discard it."""
workspace = WorkspaceManager()
repo_path = workspace.get_repo_path(repo)
coding_tools = CodingTools(str(repo_path))
research_tools = ResearchTools()
planning_tools: list[Any] = [
self._tools.get_issue,
self._tools.get_pull_request,
self._tools.list_issues,
self._tools.list_pull_requests,
self._tools.get_file_content,
self._tools.get_issue_comments,
self._tools.get_pull_request_comments,
self._tools.get_pull_request_diff,
self._tools.get_pull_request_patch,
coding_tools.list_files,
coding_tools.read_file,
coding_tools.grep_search,
coding_tools.get_working_directory,
coding_tools.run_command,
research_tools.web_search,
research_tools.fetch_url,
]
coding_tools_list: list[Any] = [
self._tools.get_issue,
self._tools.get_pull_request,
self._tools.list_issues,
self._tools.list_pull_requests,
self._tools.get_file_content,
self._tools.create_pull_request,
self._tools.add_label_to_issue,
self._tools.add_label_to_pr,
self._tools.create_branch,
self._tools.commit_file,
self._tools.create_issue,
self._tools.add_comment_to_issue,
self._tools.close_issue,
self._tools.close_pull_request,
self._tools.get_issue_comments,
self._tools.get_pull_request_comments,
self._tools.add_comment,
self._tools.add_label,
self._tools.update_file,
self._tools.get_pull_request_diff,
self._tools.get_pull_request_patch,
self._tools.approve_pull_request,
self._tools.request_changes,
coding_tools.list_files,
coding_tools.read_file,
coding_tools.write_file,
coding_tools.edit_file,
coding_tools.run_command,
coding_tools.grep_search,
coding_tools.get_working_directory,
research_tools.web_search,
research_tools.fetch_url,
]
results: list[str] = []
import os
original_cwd = os.getcwd()
changed_dir = False
if os.path.isdir(str(repo_path)):
os.chdir(str(repo_path))
changed_dir = True
try:
for item in work_items:
if item.task_type == "issue":
existing_pr = self._find_pr_for_issue(repo, item.task_number)
if existing_pr:
logger.info(f"Issue #{item.task_number} already has open PR #{existing_pr.number}. Skipping.")
results.append(f"SKIP: A pull request (PR #{existing_pr.number}) addressing issue #{item.task_number} already exists.")
continue
# Check if we're waiting for a human reply before acting
owner, repo_name = repo.split("/")
issue_comments = []
try:
issue_comments = self._client.get_issue_comments(owner, repo_name, item.task_number)
except Exception:
pass
if self._is_awaiting_reply(issue_comments):
logger.info(f"Issue #{item.task_number}: agent asked a question and is awaiting a human reply. Skipping.")
results.append(f"SKIP: Awaiting human reply on issue #{item.task_number}.")
continue
elif item.task_type == "pr":
# Check if we're waiting for a human reply before acting on a PR
owner, repo_name = repo.split("/")
pr_comments = []
try:
pr_comments = self._client.get_pull_request_comments(owner, repo_name, item.task_number)
except Exception:
pass
if self._is_awaiting_reply(pr_comments):
logger.info(f"PR #{item.task_number}: agent asked a question and is awaiting a human reply. Skipping.")
results.append(f"SKIP: Awaiting human reply on PR #{item.task_number}.")
continue
for attempt in range(1, self._max_retries + 1):
try:
if item.task_type == "issue":
base_mission = self._build_issue_mission(item)
else:
base_mission = self._build_pr_mission(item)
if base_mission.startswith("SKIP:"):
logger.info(f"Skipping task #{item.task_number}: {base_mission}")
results.append(base_mission)
break
# Step 1: Planning Phase
logger.info(f"Starting Planning Phase for {item.task_type} #{item.task_number} (attempt {attempt})")
planning_mission = (
f"PHASE 1: PLANNING PHASE\n\n"
f"Your task is to research the problem, analyse the repository structure, and produce a detailed implementation plan.\n"
f"Original Mission details:\n{base_mission}\n\n"
f"CRITICAL RULES:\n"
f"1. You are ONLY generating an implementation plan. DO NOT write files, DO NOT edit files, DO NOT commit, DO NOT push, and DO NOT create branches or PRs.\n"
f"2. RESEARCH FIRST (mandatory before writing the plan):\n"
f" - Use web_search to find relevant documentation, known solutions, library APIs, error explanations, and best practices.\n"
f" - Use fetch_url to read specific documentation pages, changelogs, or Stack Overflow answers in full.\n"
f" - Use web_search with time_range='month' or time_range='year' for recency-sensitive queries.\n"
f" - Use web_search with categories='it' for technical/programming topics.\n"
f"3. EXPLORE the codebase using read_file, list_files, grep_search, or run_command (read-only queries like find/grep).\n"
f"4. Output your final plan clearly, describing the exact changes to be made and which files to modify.\n"
)
planning_agent = CodingAgent(self._model_name)
plan = await planning_agent.run_with_tools(planning_mission, planning_tools)
logger.info(f"Generated Plan:\n{plan}")
# Step 2: Coding Phase
logger.info(f"Starting Execution/Coding Phase for {item.task_type} #{item.task_number} (attempt {attempt})")
coding_mission = (
f"PHASE 2: EXECUTION/CODING PHASE\n\n"
f"You must now implement the changes based on the following plan generated in Phase 1:\n"
f"--- PLAN ---\n{plan}\n--- PLAN END ---\n\n"
f"Original Mission details:\n{base_mission}\n\n"
f"AVAILABLE RESEARCH TOOLS (use whenever uncertain about an API, syntax, or behaviour):\n"
f" - web_search(query, time_range, categories) — search the web via SearXNG/DuckDuckGo\n"
f" - fetch_url(url) — read any documentation page, README, or Stack Overflow answer in full\n\n"
f"Follow the repository workflow: implement changes, run verification, commit, push, and create a PR.\n"
)
coding_agent = CodingAgent(self._model_name)
response = await coding_agent.run_with_tools(coding_mission, coding_tools_list)
logger.info(f"Agent response for {item.task_type} #{item.task_number}: {response}")
results.append(response)
break
except Exception as e:
logger.error(f"Error processing {item.task_type} #{item.task_number} (attempt {attempt}/{self._max_retries}): {e}")
if attempt == self._max_retries:
results.append(f"FAILED after {self._max_retries} attempts: {str(e)}")
finally:
if changed_dir:
os.chdir(original_cwd)
return results
def _build_issue_mission(self, item: WorkItem) -> str:
issue_info = item.task_info
assert isinstance(issue_info, IssueModel)
repo_full_name: str = item.repo_full_name
issue_number: int = item.task_number
issue_body: str = issue_info.body or "No description provided."
issue_labels: list[str] = [lbl.name for lbl in issue_info.labels]
issue_user: str = issue_info.user.login if issue_info.user else "unknown"
issue_created: str = issue_info.created_at or "unknown"
title: str = issue_info.title
owner: str = repo_full_name.split("/")[0]
repo_name: str = repo_full_name.split("/")[1]
comments: list[CommentModel] = []
try:
comments = self._client.get_issue_comments(owner, repo_name, issue_number)
except Exception:
pass
labels_str: str = f"Labels: {', '.join(issue_labels)}" if issue_labels else "Labels: none"
comments_str: str = "\n".join([
f"- @{c.user.login} ({c.created_at}): {c.body}"
for c in comments
]) if comments else "No comments yet."
clean_title = re.sub(r'[^a-zA-Z0-9\s-]', '', title).strip().lower()
title_words = clean_title.split()[:5]
desc_suffix = "-".join(title_words)
if not desc_suffix:
desc_suffix = "fix-issue"
branch_name: str = f"fix/issue-{issue_number}-{desc_suffix}"
workspace = WorkspaceManager()
repo_path = workspace.get_repo_path(repo_full_name)
return (
f"Your mission is to resolve issue #{issue_number} in {repo_full_name}.\n\n"
f"Issue: {title}\n"
f"Author: @{issue_user} (created {issue_created})\n"
f"{labels_str}\n\n"
f"Description:\n{issue_body}\n\n"
f"Comments ({len(comments)}):\n{comments_str}\n\n"
f"Branch name: {branch_name}.\n\n"
"BEFORE WRITING ANY CODE:\n"
" - Search online for relevant documentation, known solutions, library APIs, and platform-specific behavior.\n"
" - If ANY part of the issue is unclear, ambiguous, or has multiple valid approaches:\n"
" → Post a comment on the issue using `add_comment_to_issue` with your specific question(s).\n"
" → List the approaches you are considering.\n"
" → End the comment with the marker: <!-- agent:awaiting-reply --> on its own line.\n"
" → STOP. Do NOT proceed until a human replies. The system will re-dispatch you once a human responds.\n"
" - Never assume or guess. Always prefer asking over guessing.\n\n"
"CRITICAL INSTRUCTIONS:\n"
f"1. The repo is already cloned locally at '{repo_path}'. DO NOT create a new repository.\n"
f" The repository directory is your current working directory (cwd). You can verify this via the `get_working_directory` tool.\n"
"2. Always start from master: `git checkout master && git pull origin master`\n"
"3. Branch from master: `git checkout -b <type>/issue-<number>-<descriptive-name>`\n"
" Branch names MUST include a descriptive name (words/hyphens), not just the issue number.\n"
" Types: feat, fix, chore, docs, style, refactor, test, build, ci, perf\n"
"4. Use `edit_file`/`write_file` for code changes, then `git add` and `git commit` via `run_command`.\n"
"5. Push: `git push origin <branch>` via `run_command`.\n"
"6. Create PR: Use the `create_pull_request` tool (do NOT use Gitea's `tea` CLI or GitHub's `gh` CLI in run_command, as they can hang/freeze interactively).\n"
" PR description MUST include the Gitea automation template with 'closes #<ISSUE>'.\n"
"7. IMPORTANT: After successfully creating the pull request, you MUST comment on the issue (using the `add_comment_to_issue` tool) with the PR number, PR link, and summary.\n"
"8. Check AGENTS.md in repo root for project conventions and verification steps.\n"
"9. Grade severity: Critical/High = fix, Medium = review/fix, Low = skip.\n"
"An issue is DONE when the connected PR is merged (you cannot merge yourself).\n"
"DO NOT edit .git files unless explicitly resolving a git issue.\n"
"DO NOT work on non-meeks organization repos."
)
def _build_pr_mission(self, item: WorkItem) -> str:
pr_info = item.task_info
assert isinstance(pr_info, PullRequestModel)
repo_full_name: str = item.repo_full_name
pr_number: int = item.task_number
owner: str = repo_full_name.split("/")[0]
repo_name: str = repo_full_name.split("/")[1]
pr_model = self._client.get_pull_request(owner, repo_name, pr_number)
pr_details: str = pr_model.model_dump_json(indent=2)
pr_diff: str = ""
try:
pr_diff = self._client.get_pull_request_diff(owner, repo_name, pr_number)
except Exception as e:
logger.warning(f"Could not fetch PR diff for #{pr_number}: {e}")
pr_diff = f"Error fetching diff: {e}"
pr_files: list[PullRequestFileModel] = []
try:
pr_files = self._client.get_pull_request_files(owner, repo_name, pr_number)
except Exception:
pass
files_summary: str = "\n".join([f"- {f.filename}" for f in pr_files]) if pr_files else "No files available."
comments: list[CommentModel] = []
try:
comments = self._client.get_pull_request_comments(owner, repo_name, pr_number)
if not isinstance(comments, list):
comments = []
except Exception:
pass
reviews: list[dict[str, Any]] = []
try:
reviews = self._client.get_pr_reviews(owner, repo_name, pr_number)
if not isinstance(reviews, list):
reviews = []
except Exception:
pass
ai_username = "meeks-ai"
try:
user = self._client.get_authenticated_user()
if user:
ai_username = user.login
except Exception:
pass
# Create a combined, sorted timeline of timeline comments and reviews
timeline: list[dict[str, Any]] = []
for c in comments:
timeline.append({
"timestamp": c.created_at or "",
"user": c.user.login,
"type": "comment",
"body": c.body,
"by_ai": "Reviewed by AI Agent" in c.body or c.user.login == ai_username
})
for r in reviews:
r_user = (r.get("user") or {}).get("login", "unknown")
r_body = r.get("body", "")
r_state = r.get("state", "")
timeline.append({
"timestamp": r.get("submitted_at") or r.get("updated_at") or "",
"user": r_user,
"type": "review",
"body": f"[{r_state}] {r_body}",
"by_ai": r_user == ai_username
})
timeline.sort(key=lambda x: x["timestamp"])
last_action_by_ai = False
if timeline:
last_action_by_ai = timeline[-1]["by_ai"]
pr_author: str = pr_info.user.login if pr_info.user else "unknown"
is_own_pr = (pr_author == ai_username)
# Skip if the latest action is already by AI (waiting for human turn)
if last_action_by_ai:
logger.info(f"PR #{pr_number} already addressed by AI. Skipping.")
return f"SKIP: PR #{pr_number} has already been addressed by AI. No new action needed."
comments_str: str = "\n".join([
f"- @{c.user.login} ({c.created_at}): {c.body}"
for c in comments
]) if comments else "No comments yet."
reviews_str: str = "\n".join([
f"- @{(r.get('user') or {}).get('login')} ({r.get('submitted_at')}): [{r.get('state')}] {r.get('body')}"
for r in reviews
]) if reviews else "No reviews yet."
connected_issues_ctx = ""
pr_body = pr_info.body or ""
linked_issues = self._find_issues_for_pr(pr_body)
if linked_issues:
issues_details = []
for issue_num in linked_issues:
try:
issue = self._client.get_issue(owner, repo_name, issue_num)
issue_comments = self._client.get_issue_comments(owner, repo_name, issue_num)
comments_list = "\n".join([
f" - @{c.user.login} ({c.created_at}): {c.body}"
for c in issue_comments
]) if issue_comments else " No comments yet."
issues_details.append(
f"### Connected Issue #{issue_num}: {issue.title}\n"
f"Author: @{issue.user.login} (created {issue.created_at})\n"
f"Description:\n{issue.body or 'No description'}\n"
f"Discussion:\n{comments_list}"
)
except Exception as e:
logger.warning(f"Could not fetch connected issue #{issue_num}: {e}")
if issues_details:
connected_issues_ctx = "\n---\n\n## 📋 CONNECTED ISSUE CONTEXT\n" + "\n\n".join(issues_details)
workspace = WorkspaceManager()
repo_path = workspace.get_repo_path(repo_full_name)
pr_head_branch: str = pr_info.head.get('ref', 'unknown') if pr_info.head else "unknown"
pr_base_branch: str = pr_info.base.get('ref', 'unknown') if pr_info.base else "unknown"
pr_state: str = pr_info.state
pr_created: str = pr_info.created_at or "unknown"
# Dynamically determine instructions based on ownership/comments
is_fixing_pr = is_own_pr or any(r.get("state") == "REQUEST_CHANGES" for r in reviews)
if is_fixing_pr:
instructions = (
f" Note: The repository is located locally at '{repo_path}'. This repository directory is your current working directory (cwd). You can verify this via the `get_working_directory` tool.\n"
f" Your task is to FIX/UPDATE this PR by addressing comments/change requests.\n"
f" DO NOT create a new branch or PR. Follow this exact workflow:\n"
f" 1. Checkout the PR's head branch: `git checkout {pr_head_branch}`\n"
f" 2. Implement the requested fixes or changes on this branch.\n"
f" 3. Verify your fixes and run verification/tests.\n"
f" 4. Commit and push the changes directly: `git add <files> && git commit -m \"fix: address feedback\" && git push origin {pr_head_branch}`\n"
f" 5. After pushing, comment on the PR (using the `add_comment` tool) with a summary of the fixes implemented."
)
else:
instructions = (
f" Note: The repository is located locally at '{repo_path}'. This repository directory is your current working directory (cwd). You can verify this via the `get_working_directory` tool.\n"
" CRITICAL: You are ONLY reviewing this PR. DO NOT edit files, DO NOT make commits, DO NOT push branches, and DO NOT create any new PRs.\n"
"1. Read the PR diff carefully.\n"
"2. Analyze the changes for correctness, quality, and potential issues.\n"
"3. Check for: code quality, security issues, edge cases, test coverage.\n"
"4. If the PR is good: approve it (using approve_pull_request tool) with a meaningful comment.\n"
"5. If the PR has issues: request changes (using request_changes tool) with specific feedback.\n"
"6. Post your review comment on the PR (using add_comment tool).\n"
"IMPORTANT: Never merge the PR yourself - that is handled by humans."
)
return (
f"Your mission is to process PR #{pr_number} in {repo_full_name}.\n\n"
f"PR: {pr_info.title}\n"
f"Author: @{pr_author}\n"
f"Branch: {pr_head_branch}{pr_base_branch}\n"
f"State: {pr_state} (created {pr_created})\n\n"
f"Description:\n{pr_info.body or 'No description'}\n\n"
f"Files Changed ({len(pr_files)}):\n{files_summary}\n\n"
f"Timeline Comments:\n{comments_str}\n\n"
f"Reviews:\n{reviews_str}\n\n"
f"{connected_issues_ctx}\n\n"
"BEFORE MAKING ANY CHANGES:\n"
" - Search online for any technology, API, or behavior you are not 100% certain about.\n"
" - Read ALL review comments and change requests carefully.\n"
" - If any review comment is ambiguous or unclear:\n"
" → Post a clarifying comment on the PR (using the `add_comment` tool) with your specific question(s).\n"
" → End the comment with the marker: <!-- agent:awaiting-reply --> on its own line.\n"
" → STOP. Do NOT implement anything until a human replies. The system will re-dispatch you once a human responds.\n"
" - Never assume or guess what a reviewer meant. Always prefer asking over guessing.\n\n"
f"Instructions:\n{instructions}"
)