From bf13a979c27f44bf190ed61f87a60bdbf2d07d8b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jun 2026 19:57:14 +0200 Subject: [PATCH] feat: add output truncation and offset paging to all large-output tools - coding_tools: list_files (max_entries=200, sorted), run_command (max_chars=8000, output_offset), grep_search (max_lines=100, offset); added _truncate_output helper for run_command paging - pr_tools: get_pull_request_diff / get_pull_request_patch now truncate at 15k chars with hunk-boundary awareness and char_offset paging; get_pull_request_comments gains limit/offset paging - issue_tools: get_issue_comments gains limit/offset paging - file_tools: get_file_content / get_file_content_with_ref now paginate by line (offset=1, limit=250), matching existing read_file convention; added _paginate_lines helper - research_tools: fetch_url gains char_offset parameter; _smart_truncate now slices from an offset and embeds next char_offset in the footer - gitea_tools facade: all new params threaded through --- gitea/tools/coding_tools.py | 136 +++++++++++++++++++++++++++++----- gitea/tools/file_tools.py | 69 +++++++++++++++-- gitea/tools/gitea_tools.py | 79 +++++++++++++++----- gitea/tools/issue_tools.py | 30 +++++++- gitea/tools/pr_tools.py | 92 +++++++++++++++++++++-- gitea/tools/research_tools.py | 46 ++++++++---- 6 files changed, 383 insertions(+), 69 deletions(-) diff --git a/gitea/tools/coding_tools.py b/gitea/tools/coding_tools.py index 6382ce6..4615d46 100644 --- a/gitea/tools/coding_tools.py +++ b/gitea/tools/coding_tools.py @@ -21,12 +21,26 @@ class CodingTools: return path return os.path.abspath(os.path.join(self.repo_path, path)) - def list_files(self, path: str = ".") -> str: - """List all files and directories at the given path (relative to repo root or absolute).""" + def list_files(self, path: str = ".", max_entries: int = 200) -> str: + """List files and directories at the given path (relative to repo root or absolute). + + Args: + path: Directory to list (relative to repo root or absolute). + max_entries: Maximum number of entries to return (default 200). + Pass a subdirectory path to narrow results when a directory is very large. + """ resolved: str = self._resolve_path(path) try: - items: list[str] = os.listdir(resolved) - return "\n".join(items) + items: list[str] = sorted(os.listdir(resolved)) + total: int = len(items) + shown: list[str] = items[:max_entries] + result: str = "\n".join(shown) + if total > max_entries: + result += ( + f"\n\n[{total} entries total — showing first {max_entries}. " + "Pass a subdirectory path to narrow results.]" + ) + return result except Exception as e: return f"Error listing files: {str(e)}" @@ -147,16 +161,58 @@ class CodingTools: return False, "\n".join(log_output) return True, "\n".join(log_output) - def run_command(self, command: str, timeout: int = 120) -> str: - """Execute a shell command in the repository workspace and return stdout and stderr output. Args: command, timeout (default 120 seconds).""" + def _truncate_output( + self, + text: str, + max_chars: int, + offset: int, + label: str = "Output", + ) -> str: + """Slice [offset : offset+max_chars] from text and append a paging footer if truncated.""" + total: int = len(text) + chunk: str = text[offset : offset + max_chars] + if offset > 0 or (offset + max_chars) < total: + next_offset: int = offset + len(chunk) + chunk += ( + f"\n\n[{label} truncated — {total} chars total. " + f"Showing chars {offset}–{next_offset}. " + f"Re-run with output_offset={next_offset} to read more.]" + ) + return chunk + + def run_command( + self, + command: str, + timeout: int = 120, + max_chars: int = 8000, + output_offset: int = 0, + ) -> str: + """Execute a shell command in the repository workspace and return stdout/stderr. + + Args: + command: Shell command to run. + timeout: Seconds before the command is killed (default 120). + max_chars: Maximum characters of combined output to return (default 8000). + Tail (most recent lines) is preferred because errors appear there. + output_offset: Character offset into the full output to start reading from + (default 0). Increment by max_chars to page through large output. + """ if "tea pr create" in command: success, log_msg = self.run_verification() if not success: - return f"Verification failed! You cannot create a pull request because the tests/checks are failing:\n\n{log_msg}\n\nPlease fix the issues and try again." + return ( + f"Verification failed! You cannot create a pull request because the " + f"tests/checks are failing:\n\n{log_msg}\n\nPlease fix the issues and try again." + ) try: process: subprocess.Popen[str] = subprocess.Popen( - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=self.repo_path + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=self.repo_path, ) stdout: str stderr: str @@ -165,7 +221,11 @@ class CodingTools: except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() - return f"Command timed out after {timeout} seconds.\n--- STDOUT ---\n{stdout}\n--- STDERR ---\n{stderr}" + raw: str = ( + f"Command timed out after {timeout} seconds.\n" + f"--- STDOUT ---\n{stdout}\n--- STDERR ---\n{stderr}" + ) + return self._truncate_output(raw, max_chars, output_offset, "Output") output: str = "" if stdout: @@ -173,20 +233,44 @@ class CodingTools: if stderr: output += f"\n--- STDERR ---\n{stderr}" - if process.returncode != 0: - return f"Command failed with exit code {process.returncode}:\n{output}" + if not output: + return "Command executed successfully (no output)." - return output if output else "Command executed successfully (no output)." + prefix: str = ( + f"Command failed with exit code {process.returncode}:\n" + if process.returncode != 0 + else "" + ) + full: str = prefix + output + return self._truncate_output(full, max_chars, output_offset, "Output") except Exception as e: return f"Error running command: {str(e)}" - def grep_search(self, pattern: str, path: str = ".") -> str: - """Search for pattern in files under path using grep (case-insensitive).""" + def grep_search( + self, + pattern: str, + path: str = ".", + max_lines: int = 100, + offset: int = 0, + ) -> str: + """Search for pattern in files under path using grep (case-insensitive). + + Args: + pattern: Regular-expression / literal pattern to search for. + path: Directory or file to search (relative to repo root or absolute). + max_lines: Maximum number of matching lines to return (default 100). + offset: Line offset into the full result set for paging (default 0). + """ resolved: str = self._resolve_path(path) try: command: str = f"grep -ri '{pattern}' {resolved}" process: subprocess.Popen[str] = subprocess.Popen( - command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=self.repo_path + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=self.repo_path, ) stdout: str stderr: str @@ -195,14 +279,28 @@ class CodingTools: except subprocess.TimeoutExpired: process.kill() stdout, stderr = process.communicate() - return f"Grep search timed out after 30 seconds.\nStdout: {stdout}\nStderr: {stderr}" + return ( + f"Grep search timed out after 30 seconds.\n" + f"Stdout: {stdout}\nStderr: {stderr}" + ) if process.returncode != 0 and not stdout: return f"No matches found for '{pattern}'." - output: str = stdout + all_lines: list[str] = stdout.splitlines() + total: int = len(all_lines) + page: list[str] = all_lines[offset : offset + max_lines] + result: str = "\n".join(page) + + if total > offset + max_lines: + next_offset: int = offset + max_lines + result += ( + f"\n\n[{total} matches total — showing lines {offset}–{offset + len(page)}. " + f"Use offset={next_offset} to see more.]" + ) + if stderr: - output += f"\nError: {stderr}" - return output + result += f"\nError: {stderr}" + return result except Exception as e: return f"Error during grep search: {str(e)}" diff --git a/gitea/tools/file_tools.py b/gitea/tools/file_tools.py index bea4e3e..9821988 100644 --- a/gitea/tools/file_tools.py +++ b/gitea/tools/file_tools.py @@ -8,21 +8,74 @@ class FileTools: def __init__(self, client: GiteaClient) -> None: self._client = client - def get_file_content(self, owner: str, repo: str, path: str) -> str: + def _paginate_lines( + self, + content: str, + offset: int, + limit: int, + ) -> str: + """Return lines[offset-1 : offset-1+limit] with a paging footer if truncated. + + Uses the same 1-indexed convention as CodingTools.read_file. + """ + lines: list[str] = content.splitlines() + total: int = len(lines) + start: int = offset - 1 # convert to 0-indexed + page: list[str] = lines[start : start + limit] + formatted: list[str] = [ + f"{start + i + 1}: {line}" for i, line in enumerate(page) + ] + result: str = "\n".join(formatted) + end_line: int = start + len(page) + if end_line < total: + next_offset: int = end_line + 1 + result += ( + f"\n\n[{total} lines total — showing lines {offset}–{end_line}. " + f"Re-call with offset={next_offset} to read more.]" + ) + return result + + def get_file_content( + self, + owner: str, + repo: str, + path: str, + offset: int = 1, + limit: int = 250, + ) -> str: + """Get the content of a file from a Gitea repository with line paging. + + Args: + offset: 1-indexed line to start from (default 1). + limit: Maximum number of lines to return (default 250). + """ try: content = self._client.get_file_content(owner, repo, path) - if isinstance(content, list): - return "\n".join(content) - return content + raw: str = "\n".join(content) if isinstance(content, list) else content + return self._paginate_lines(raw, offset, limit) except Exception as e: return f"Error getting file content: {str(e)}" - def get_file_content_with_ref(self, owner: str, repo: str, path: str, ref: str = "master") -> str: + def get_file_content_with_ref( + self, + owner: str, + repo: str, + path: str, + ref: str = "master", + offset: int = 1, + limit: int = 250, + ) -> str: + """Get file content at a specific git ref with line paging. + + Args: + ref: Branch, tag, or commit SHA (default 'master'). + offset: 1-indexed line to start from (default 1). + limit: Maximum number of lines to return (default 250). + """ try: content = self._client.get_file_content(owner, repo, path, ref) - if isinstance(content, list): - return "\n".join(content) - return content + raw: str = "\n".join(content) if isinstance(content, list) else content + return self._paginate_lines(raw, offset, limit) except Exception as e: return f"Error getting file content: {str(e)}" diff --git a/gitea/tools/gitea_tools.py b/gitea/tools/gitea_tools.py index 40197dc..ae7cb06 100644 --- a/gitea/tools/gitea_tools.py +++ b/gitea/tools/gitea_tools.py @@ -33,13 +33,27 @@ class GiteaTools: """Close a pull request. Args: owner (repo owner), repo (repo name), pull_number (PR ID).""" return self.pr_tools.close_pull_request(owner, repo, pull_number) - def get_issue_comments(self, owner: str, repo: str, issue_number: int) -> str: - """Get all comments on an issue. Args: owner (repo owner), repo (repo name), issue_number (issue ID).""" - return self.issue_tools.get_issue_comments(owner, repo, issue_number) + def get_issue_comments( + self, + owner: str, + repo: str, + issue_number: int, + limit: int = 20, + offset: int = 0, + ) -> str: + """Get all comments on an issue. Args: owner, repo, issue_number, limit (default 20), offset (default 0).""" + return self.issue_tools.get_issue_comments(owner, repo, issue_number, limit=limit, offset=offset) - def get_pull_request_comments(self, owner: str, repo: str, pull_number: int) -> str: - """Get all comments on a pull request. Args: owner (repo owner), repo (repo name), pull_number (PR ID).""" - return self.pr_tools.get_pull_request_comments(owner, repo, pull_number) + def get_pull_request_comments( + self, + owner: str, + repo: str, + pull_number: int, + limit: int = 20, + offset: int = 0, + ) -> str: + """Get all comments on a pull request. Args: owner, repo, pull_number, limit (default 20), offset (default 0).""" + return self.pr_tools.get_pull_request_comments(owner, repo, pull_number, limit=limit, offset=offset) def list_assigned_issues(self) -> list[dict]: """List all issues assigned to the authenticated user across all repos.""" @@ -57,9 +71,16 @@ class GiteaTools: """List pull requests in a repository. Args: owner (repo owner), repo (repo name), state (open, closed, or all).""" return self.pr_tools.list_pull_requests(owner, repo, state) - def get_file_content(self, owner: str, repo: str, path: str) -> str: - """Get the content of a file from a repository. Args: owner (repo owner), repo (repo name), path (file path).""" - return self.file_tools.get_file_content(owner, repo, path) + def get_file_content( + self, + owner: str, + repo: str, + path: str, + offset: int = 1, + limit: int = 250, + ) -> str: + """Get the content of a file from a repository. Args: owner, repo, path, offset (line, default 1), limit (default 250).""" + return self.file_tools.get_file_content(owner, repo, path, offset=offset, limit=limit) def create_pull_request(self, owner: str, repo: str, head: str, base: str, title: str, description: str = "") -> str: """Create a new pull request. Args: owner, repo, head (source branch), base (target branch), title, description.""" @@ -89,13 +110,27 @@ class GiteaTools: """Add a comment to an issue. Args: owner, repo, issue_number, body.""" return self.issue_tools.add_comment_to_issue(owner, repo, issue_number, body) - def get_pull_request_diff(self, owner: str, repo: str, pull_number: int) -> str: - """Get the diff of a pull request. Args: owner, repo, pull_number.""" - return self.pr_tools.get_pull_request_diff(owner, repo, pull_number) + def get_pull_request_diff( + self, + owner: str, + repo: str, + pull_number: int, + max_chars: int = 15000, + char_offset: int = 0, + ) -> str: + """Get the diff of a pull request. Args: owner, repo, pull_number, max_chars (default 15000), char_offset (default 0).""" + return self.pr_tools.get_pull_request_diff(owner, repo, pull_number, max_chars=max_chars, char_offset=char_offset) - def get_pull_request_patch(self, owner: str, repo: str, pull_number: int) -> str: - """Get the patch of a pull request. Args: owner, repo, pull_number.""" - return self.pr_tools.get_pull_request_patch(owner, repo, pull_number) + def get_pull_request_patch( + self, + owner: str, + repo: str, + pull_number: int, + max_chars: int = 15000, + char_offset: int = 0, + ) -> str: + """Get the patch of a pull request. Args: owner, repo, pull_number, max_chars (default 15000), char_offset (default 0).""" + return self.pr_tools.get_pull_request_patch(owner, repo, pull_number, max_chars=max_chars, char_offset=char_offset) def approve_pull_request(self, owner: str, repo: str, pull_number: int, comment: str) -> str: """Approve a pull request. Args: owner, repo, pull_number, comment.""" @@ -113,9 +148,17 @@ class GiteaTools: """Add a label to an issue. Args: owner, repo, issue_number, label.""" return self.issue_tools.add_label(owner, repo, issue_number, label) - def get_file_content_with_ref(self, owner: str, repo: str, path: str, ref: str = "master") -> str: - """Get file content with a specific git ref. Args: owner, repo, path, ref (branch/tag).""" - return self.file_tools.get_file_content_with_ref(owner, repo, path, ref) + def get_file_content_with_ref( + self, + owner: str, + repo: str, + path: str, + ref: str = "master", + offset: int = 1, + limit: int = 250, + ) -> str: + """Get file content with a specific git ref. Args: owner, repo, path, ref (branch/tag), offset (line, default 1), limit (default 250).""" + return self.file_tools.get_file_content_with_ref(owner, repo, path, ref=ref, offset=offset, limit=limit) def update_file(self, owner: str, repo: str, path: str, message: str, content: str, branch: str) -> str: """Update a file in a repository. Args: owner, repo, path, message, content, branch.""" diff --git a/gitea/tools/issue_tools.py b/gitea/tools/issue_tools.py index 6d13b10..3a29f9c 100644 --- a/gitea/tools/issue_tools.py +++ b/gitea/tools/issue_tools.py @@ -26,10 +26,34 @@ class IssueTools: except Exception as e: return f"Error closing issue: {str(e)}" - def get_issue_comments(self, owner: str, repo: str, issue_number: int) -> str: + def get_issue_comments( + self, + owner: str, + repo: str, + issue_number: int, + limit: int = 20, + offset: int = 0, + ) -> str: + """Get comments on an issue with optional paging. + + Args: + limit: Maximum number of comments to return (default 20). + offset: Zero-based comment index to start from (default 0). + """ try: - comments: list[CommentModel] = self._client.get_issue_comments(owner, repo, issue_number) - return json.dumps([c.model_dump() for c in comments], indent=2) + comments: list[CommentModel] = self._client.get_issue_comments( + owner, repo, issue_number + ) + total: int = len(comments) + page: list[CommentModel] = comments[offset : offset + limit] + result: str = json.dumps([c.model_dump() for c in page], indent=2) + if total > offset + limit: + next_offset: int = offset + limit + result += ( + f"\n\n[{total} comments total — showing {offset}–{offset + len(page)}. " + f"Re-call with offset={next_offset} to see more.]" + ) + return result except Exception as e: return f"Error getting issue comments: {str(e)}" diff --git a/gitea/tools/pr_tools.py b/gitea/tools/pr_tools.py index 855b430..820c5c0 100644 --- a/gitea/tools/pr_tools.py +++ b/gitea/tools/pr_tools.py @@ -5,6 +5,30 @@ from typing import Any from gitea.client import GiteaClient from gitea.models import PullRequestModel, CommentModel +_MAX_DIFF_CHARS: int = 15_000 + + +def _truncate_diff( + text: str, + max_chars: int = _MAX_DIFF_CHARS, + char_offset: int = 0, +) -> str: + """Slice [char_offset : char_offset+max_chars] from text, cutting at a hunk boundary.""" + total: int = len(text) + chunk: str = text[char_offset : char_offset + max_chars] + if char_offset > 0 or (char_offset + max_chars) < total: + # Try to cut at a diff hunk boundary (@@) for coherence + hunk_boundary: int = chunk.rfind("\n@@") + if hunk_boundary > int(len(chunk) * 0.6): + chunk = chunk[:hunk_boundary] + next_offset: int = char_offset + len(chunk) + chunk += ( + f"\n\n[Diff truncated — {total} chars total. " + f"Showing chars {char_offset}–{next_offset}. " + f"Re-call with char_offset={next_offset} to read more.]" + ) + return chunk + class PRTools: """Tools for Gitea pull request operations.""" @@ -26,10 +50,34 @@ class PRTools: except Exception as e: return f"Error closing pull request: {str(e)}" - def get_pull_request_comments(self, owner: str, repo: str, pull_number: int) -> str: + def get_pull_request_comments( + self, + owner: str, + repo: str, + pull_number: int, + limit: int = 20, + offset: int = 0, + ) -> str: + """Get comments on a pull request with optional paging. + + Args: + limit: Maximum number of comments to return (default 20). + offset: Zero-based comment index to start from (default 0). + """ try: - comments: list[CommentModel] = self._client.get_pull_request_comments(owner, repo, pull_number) - return json.dumps([c.model_dump() for c in comments], indent=2) + comments: list[CommentModel] = self._client.get_pull_request_comments( + owner, repo, pull_number + ) + total: int = len(comments) + page: list[CommentModel] = comments[offset : offset + limit] + result: str = json.dumps([c.model_dump() for c in page], indent=2) + if total > offset + limit: + next_offset: int = offset + limit + result += ( + f"\n\n[{total} comments total — showing {offset}–{offset + len(page)}. " + f"Re-call with offset={next_offset} to see more.]" + ) + return result except Exception as e: return f"Error getting PR comments: {str(e)}" @@ -72,15 +120,45 @@ class PRTools: except Exception as e: return f"Error adding label to PR #{pr_number}: {e}" - def get_pull_request_diff(self, owner: str, repo: str, pull_number: int) -> str: + def get_pull_request_diff( + self, + owner: str, + repo: str, + pull_number: int, + max_chars: int = _MAX_DIFF_CHARS, + char_offset: int = 0, + ) -> str: + """Get the diff of a pull request, with truncation and offset paging. + + Args: + max_chars: Maximum characters to return (default 15 000). + char_offset: Character offset to start reading from (default 0). + Increment by max_chars to page through a large diff. + """ try: - return self._client.get_pull_request_diff(owner, repo, pull_number) + diff: str = self._client.get_pull_request_diff(owner, repo, pull_number) + return _truncate_diff(diff, max_chars, char_offset) except Exception as e: return f"Error getting PR diff: {str(e)}" - def get_pull_request_patch(self, owner: str, repo: str, pull_number: int) -> str: + def get_pull_request_patch( + self, + owner: str, + repo: str, + pull_number: int, + max_chars: int = _MAX_DIFF_CHARS, + char_offset: int = 0, + ) -> str: + """Get the patch of a pull request, with truncation and offset paging. + + Args: + max_chars: Maximum characters to return (default 15 000). + char_offset: Character offset to start reading from (default 0). + Increment by max_chars to page through a large patch. + """ try: - return self._client.get_pull_request_patch(owner, repo, pull_number) + patch: str = self._client.get_pull_request_patch(owner, repo, pull_number) + return _truncate_diff(patch, max_chars, char_offset) except Exception as e: return f"Error getting PR patch: {str(e)}" diff --git a/gitea/tools/research_tools.py b/gitea/tools/research_tools.py index c7de041..273da07 100644 --- a/gitea/tools/research_tools.py +++ b/gitea/tools/research_tools.py @@ -42,20 +42,34 @@ class ResearchTools: # Internal helpers # ------------------------------------------------------------------ # - def _smart_truncate(self, text: str, max_chars: int = _MAX_CONTENT_CHARS) -> str: - """Truncate at a paragraph boundary to preserve coherence. + def _smart_truncate( + self, + text: str, + max_chars: int = _MAX_CONTENT_CHARS, + char_offset: int = 0, + ) -> str: + """Slice [char_offset : char_offset+max_chars] and truncate at a paragraph boundary. - Prefers cutting at a blank-line paragraph boundary rather than - mid-sentence so the LLM receives a coherent chunk. + Prefers cutting at a blank-line paragraph boundary rather than mid-sentence + so the LLM receives a coherent chunk. """ - if len(text) <= max_chars: - return text - truncated = text[:max_chars] - last_para: int = truncated.rfind("\n\n") - tail = "\n\n[Content truncated — use fetch_url with a more specific URL or anchor]" - if last_para > int(max_chars * 0.7): - return truncated[:last_para] + tail - return truncated + tail + total: int = len(text) + chunk: str = text[char_offset : char_offset + max_chars] + if char_offset == 0 and len(chunk) <= max_chars and total <= max_chars: + return text # Common fast-path: content fits entirely + if len(chunk) >= max_chars: + last_para: int = chunk.rfind("\n\n") + if last_para > int(max_chars * 0.7): + chunk = chunk[:last_para] + next_offset: int = char_offset + len(chunk) + if next_offset < total: + tail = ( + f"\n\n[Content truncated — {total} chars total. " + f"Showing chars {char_offset}–{next_offset}. " + f"Re-call fetch_url with char_offset={next_offset} to read more.]" + ) + return chunk + tail + return chunk def _html_to_markdown(self, html: str) -> str: """Convert HTML to clean Markdown. @@ -352,6 +366,7 @@ class ResearchTools: url: str, extract_text: bool = True, max_chars: int = _MAX_CONTENT_CHARS, + char_offset: int = 0, ) -> str: """Fetch the content of a URL and return it as clean, readable Markdown. @@ -371,8 +386,11 @@ class ResearchTools: extract_text: If True (default), extract and clean the main content as Markdown, stripping navigation, ads, and boilerplate. Set to False to get raw HTML/JSON (useful for schemas). - max_chars: Maximum characters to return (default 20,000). + max_chars: Maximum characters to return per call (default 20,000). The tool cuts at a paragraph boundary when truncating. + char_offset: Character offset into the extracted content to start + reading from (default 0). Increment by max_chars to + page through content larger than max_chars. Returns: Clean Markdown text of the main content (HTML pages), pretty-printed @@ -425,7 +443,7 @@ class ResearchTools: else: text = raw - return self._smart_truncate(text, max_chars) + return self._smart_truncate(text, max_chars, char_offset) except httpx.HTTPStatusError as exc: return (