feat: add output truncation and offset paging to all large-output tools
- coding_tools: list_files (max_entries=200, sorted), run_command (max_chars=8000, output_offset), grep_search (max_lines=100, offset); added _truncate_output helper for run_command paging - pr_tools: get_pull_request_diff / get_pull_request_patch now truncate at 15k chars with hunk-boundary awareness and char_offset paging; get_pull_request_comments gains limit/offset paging - issue_tools: get_issue_comments gains limit/offset paging - file_tools: get_file_content / get_file_content_with_ref now paginate by line (offset=1, limit=250), matching existing read_file convention; added _paginate_lines helper - research_tools: fetch_url gains char_offset parameter; _smart_truncate now slices from an offset and embeds next char_offset in the footer - gitea_tools facade: all new params threaded through
This commit is contained in:
@@ -42,20 +42,34 @@ class ResearchTools:
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def _smart_truncate(self, text: str, max_chars: int = _MAX_CONTENT_CHARS) -> str:
|
||||
"""Truncate at a paragraph boundary to preserve coherence.
|
||||
def _smart_truncate(
|
||||
self,
|
||||
text: str,
|
||||
max_chars: int = _MAX_CONTENT_CHARS,
|
||||
char_offset: int = 0,
|
||||
) -> str:
|
||||
"""Slice [char_offset : char_offset+max_chars] and truncate at a paragraph boundary.
|
||||
|
||||
Prefers cutting at a blank-line paragraph boundary rather than
|
||||
mid-sentence so the LLM receives a coherent chunk.
|
||||
Prefers cutting at a blank-line paragraph boundary rather than mid-sentence
|
||||
so the LLM receives a coherent chunk.
|
||||
"""
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
truncated = text[:max_chars]
|
||||
last_para: int = truncated.rfind("\n\n")
|
||||
tail = "\n\n[Content truncated — use fetch_url with a more specific URL or anchor]"
|
||||
if last_para > int(max_chars * 0.7):
|
||||
return truncated[:last_para] + tail
|
||||
return truncated + tail
|
||||
total: int = len(text)
|
||||
chunk: str = text[char_offset : char_offset + max_chars]
|
||||
if char_offset == 0 and len(chunk) <= max_chars and total <= max_chars:
|
||||
return text # Common fast-path: content fits entirely
|
||||
if len(chunk) >= max_chars:
|
||||
last_para: int = chunk.rfind("\n\n")
|
||||
if last_para > int(max_chars * 0.7):
|
||||
chunk = chunk[:last_para]
|
||||
next_offset: int = char_offset + len(chunk)
|
||||
if next_offset < total:
|
||||
tail = (
|
||||
f"\n\n[Content truncated — {total} chars total. "
|
||||
f"Showing chars {char_offset}–{next_offset}. "
|
||||
f"Re-call fetch_url with char_offset={next_offset} to read more.]"
|
||||
)
|
||||
return chunk + tail
|
||||
return chunk
|
||||
|
||||
def _html_to_markdown(self, html: str) -> str:
|
||||
"""Convert HTML to clean Markdown.
|
||||
@@ -352,6 +366,7 @@ class ResearchTools:
|
||||
url: str,
|
||||
extract_text: bool = True,
|
||||
max_chars: int = _MAX_CONTENT_CHARS,
|
||||
char_offset: int = 0,
|
||||
) -> str:
|
||||
"""Fetch the content of a URL and return it as clean, readable Markdown.
|
||||
|
||||
@@ -371,8 +386,11 @@ class ResearchTools:
|
||||
extract_text: If True (default), extract and clean the main content
|
||||
as Markdown, stripping navigation, ads, and boilerplate.
|
||||
Set to False to get raw HTML/JSON (useful for schemas).
|
||||
max_chars: Maximum characters to return (default 20,000).
|
||||
max_chars: Maximum characters to return per call (default 20,000).
|
||||
The tool cuts at a paragraph boundary when truncating.
|
||||
char_offset: Character offset into the extracted content to start
|
||||
reading from (default 0). Increment by max_chars to
|
||||
page through content larger than max_chars.
|
||||
|
||||
Returns:
|
||||
Clean Markdown text of the main content (HTML pages), pretty-printed
|
||||
@@ -425,7 +443,7 @@ class ResearchTools:
|
||||
else:
|
||||
text = raw
|
||||
|
||||
return self._smart_truncate(text, max_chars)
|
||||
return self._smart_truncate(text, max_chars, char_offset)
|
||||
|
||||
except httpx.HTTPStatusError as exc:
|
||||
return (
|
||||
|
||||
Reference in New Issue
Block a user