feat: add output truncation and offset paging to all large-output tools

- coding_tools: list_files (max_entries=200, sorted), run_command
  (max_chars=8000, output_offset), grep_search (max_lines=100, offset);
  added _truncate_output helper for run_command paging
- pr_tools: get_pull_request_diff / get_pull_request_patch now truncate
  at 15k chars with hunk-boundary awareness and char_offset paging;
  get_pull_request_comments gains limit/offset paging
- issue_tools: get_issue_comments gains limit/offset paging
- file_tools: get_file_content / get_file_content_with_ref now paginate
  by line (offset=1, limit=250), matching existing read_file convention;
  added _paginate_lines helper
- research_tools: fetch_url gains char_offset parameter; _smart_truncate
  now slices from an offset and embeds next char_offset in the footer
- gitea_tools facade: all new params threaded through
This commit is contained in:
Michael
2026-06-28 19:57:14 +02:00
parent c0c9278d75
commit bf13a979c2
6 changed files with 383 additions and 69 deletions
+32 -14
View File
@@ -42,20 +42,34 @@ class ResearchTools:
# Internal helpers
# ------------------------------------------------------------------ #
def _smart_truncate(self, text: str, max_chars: int = _MAX_CONTENT_CHARS) -> str:
"""Truncate at a paragraph boundary to preserve coherence.
def _smart_truncate(
self,
text: str,
max_chars: int = _MAX_CONTENT_CHARS,
char_offset: int = 0,
) -> str:
"""Slice [char_offset : char_offset+max_chars] and truncate at a paragraph boundary.
Prefers cutting at a blank-line paragraph boundary rather than
mid-sentence so the LLM receives a coherent chunk.
Prefers cutting at a blank-line paragraph boundary rather than mid-sentence
so the LLM receives a coherent chunk.
"""
if len(text) <= max_chars:
return text
truncated = text[:max_chars]
last_para: int = truncated.rfind("\n\n")
tail = "\n\n[Content truncated — use fetch_url with a more specific URL or anchor]"
if last_para > int(max_chars * 0.7):
return truncated[:last_para] + tail
return truncated + tail
total: int = len(text)
chunk: str = text[char_offset : char_offset + max_chars]
if char_offset == 0 and len(chunk) <= max_chars and total <= max_chars:
return text # Common fast-path: content fits entirely
if len(chunk) >= max_chars:
last_para: int = chunk.rfind("\n\n")
if last_para > int(max_chars * 0.7):
chunk = chunk[:last_para]
next_offset: int = char_offset + len(chunk)
if next_offset < total:
tail = (
f"\n\n[Content truncated — {total} chars total. "
f"Showing chars {char_offset}{next_offset}. "
f"Re-call fetch_url with char_offset={next_offset} to read more.]"
)
return chunk + tail
return chunk
def _html_to_markdown(self, html: str) -> str:
"""Convert HTML to clean Markdown.
@@ -352,6 +366,7 @@ class ResearchTools:
url: str,
extract_text: bool = True,
max_chars: int = _MAX_CONTENT_CHARS,
char_offset: int = 0,
) -> str:
"""Fetch the content of a URL and return it as clean, readable Markdown.
@@ -371,8 +386,11 @@ class ResearchTools:
extract_text: If True (default), extract and clean the main content
as Markdown, stripping navigation, ads, and boilerplate.
Set to False to get raw HTML/JSON (useful for schemas).
max_chars: Maximum characters to return (default 20,000).
max_chars: Maximum characters to return per call (default 20,000).
The tool cuts at a paragraph boundary when truncating.
char_offset: Character offset into the extracted content to start
reading from (default 0). Increment by max_chars to
page through content larger than max_chars.
Returns:
Clean Markdown text of the main content (HTML pages), pretty-printed
@@ -425,7 +443,7 @@ class ResearchTools:
else:
text = raw
return self._smart_truncate(text, max_chars)
return self._smart_truncate(text, max_chars, char_offset)
except httpx.HTTPStatusError as exc:
return (