Compare commits
14 Commits
master
..
fd85758e3c
| Author | SHA1 | Date | |
|---|---|---|---|
| fd85758e3c | |||
| 9fb56315c6 | |||
| 6e2b57ed16 | |||
| fd056e3ed0 | |||
| 9b72e20d5a | |||
| 287d20bc56 | |||
| edc8415571 | |||
| 7f66d09d9e | |||
| e38a80532b | |||
| a0c247b152 | |||
| bf13a979c2 | |||
| c0c9278d75 | |||
| 0461c6c7ab | |||
| ac6cff52dd |
@@ -0,0 +1,55 @@
|
|||||||
|
# Web Search Skill
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Enable the agent to search the web using a self-hosted SearXNG instance for privacy-respecting searches.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
- SearXNG Instance: `https://searxng.meeks.freeddns.org`
|
||||||
|
- API Endpoint: `/search`
|
||||||
|
- Supported Formats: `json`, `csv`, `rss`
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Search the Web
|
||||||
|
When you need to look up information, find recent developments, or verify facts:
|
||||||
|
|
||||||
|
```
|
||||||
|
search_web(query="what is the latest version of Python", limit=10, language="en")
|
||||||
|
```
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
- `query` (required): The search query string
|
||||||
|
- `limit` (optional): Number of results to return (default: 10, max: 100)
|
||||||
|
- `language` (optional): Language code (e.g., "en", "sv", "de")
|
||||||
|
- `time_range` (optional): Time filter - "day", "month", or "year"
|
||||||
|
- `categories` (optional): Search category - "general", "images", "news", "videos", "science", "it", "music", "files", "map", "realtime"
|
||||||
|
|
||||||
|
### Example Queries
|
||||||
|
- `search_web(query="uv python package manager tutorial")`
|
||||||
|
- `search_web(query="searxng API documentation", time_range="month")`
|
||||||
|
- `search_web(query="gitea vs github", categories="general")`
|
||||||
|
|
||||||
|
## Implementation Details
|
||||||
|
|
||||||
|
The skill uses the SearXNG JSON API:
|
||||||
|
```
|
||||||
|
GET https://searxng.meeks.freeddns.org/search?q={query}&format=json&limit={limit}&language={language}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Error Handling
|
||||||
|
- If the SearXNG instance is unreachable, log the error and try alternative search methods
|
||||||
|
- If the response format is invalid, parse what's available and report the issue
|
||||||
|
- If rate limited, wait and retry with exponential backoff
|
||||||
|
|
||||||
|
## When to Use
|
||||||
|
- Researching technical issues or solutions
|
||||||
|
- Looking up recent software updates or versions
|
||||||
|
- Finding documentation or tutorials
|
||||||
|
- Verifying facts or current information
|
||||||
|
- Investigating error messages or stack traces
|
||||||
|
- Finding similar projects or libraries
|
||||||
|
|
||||||
|
## When NOT to Use
|
||||||
|
- When you already have the information locally
|
||||||
|
- For simple factual questions that don't require current data
|
||||||
|
- When the user explicitly asks not to search the web
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
MIT License
|
|
||||||
|
|
||||||
Copyright (c) 2026 mi222eh
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
|
||||||
in the Software without restriction, including without limitation the rights
|
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in all
|
|
||||||
copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
||||||
SOFTWARE.
|
|
||||||
@@ -13,9 +13,6 @@ class AgentSettings(BaseSettings):
|
|||||||
gitea_repos_root: str = ""
|
gitea_repos_root: str = ""
|
||||||
agent_model_id: str = "qwen/qwen3.6-35b-a3b"
|
agent_model_id: str = "qwen/qwen3.6-35b-a3b"
|
||||||
agent_max_retries: int = 2
|
agent_max_retries: int = 2
|
||||||
searxng_url: str = ""
|
|
||||||
searxng_username: str = ""
|
|
||||||
searxng_password: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
def get_settings() -> AgentSettings:
|
def get_settings() -> AgentSettings:
|
||||||
@@ -31,15 +28,7 @@ GITEA_TOKEN: str = _agent_settings.gitea_token
|
|||||||
GITEA_REPOS_ROOT: str = _agent_settings.gitea_repos_root
|
GITEA_REPOS_ROOT: str = _agent_settings.gitea_repos_root
|
||||||
AGENT_MODEL_ID: str = _agent_settings.agent_model_id
|
AGENT_MODEL_ID: str = _agent_settings.agent_model_id
|
||||||
AGENT_MAX_RETRIES: int = _agent_settings.agent_max_retries
|
AGENT_MAX_RETRIES: int = _agent_settings.agent_max_retries
|
||||||
SEARXNG_URL: str = _agent_settings.searxng_url
|
|
||||||
SEARXNG_USERNAME: str = _agent_settings.searxng_username
|
|
||||||
SEARXNG_PASSWORD: str = _agent_settings.searxng_password
|
|
||||||
|
|
||||||
import os
|
import os
|
||||||
os.environ["GITEA_SERVER_URL"] = GITEA_URL
|
os.environ["GITEA_SERVER_URL"] = GITEA_URL
|
||||||
os.environ["GITEA_SERVER_TOKEN"] = GITEA_TOKEN
|
os.environ["GITEA_SERVER_TOKEN"] = GITEA_TOKEN
|
||||||
os.environ["SEARXNG_URL"] = SEARXNG_URL
|
|
||||||
os.environ["SEARXNG_USERNAME"] = SEARXNG_USERNAME
|
|
||||||
os.environ["SEARXNG_PASSWORD"] = SEARXNG_PASSWORD
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,13 +20,6 @@ import time
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from duckduckgo_search import DDGS # type: ignore[import-untyped]
|
|
||||||
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
|
|
||||||
DuckDuckGoSearchException,
|
|
||||||
RatelimitException,
|
|
||||||
)
|
|
||||||
|
|
||||||
from gitea.config import SEARXNG_URL, SEARXNG_USERNAME, SEARXNG_PASSWORD
|
|
||||||
|
|
||||||
logger: logging.Logger = logging.getLogger("research-tools")
|
logger: logging.Logger = logging.getLogger("research-tools")
|
||||||
|
|
||||||
@@ -36,12 +29,10 @@ _USER_AGENT: str = (
|
|||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||||
)
|
)
|
||||||
# SearXNG instance
|
# SearXNG instance — override with SEARXNG_URL env var
|
||||||
_SEARXNG_URL: str = SEARXNG_URL
|
_SEARXNG_URL: str = os.getenv(
|
||||||
_SEARXNG_USERNAME: str = SEARXNG_USERNAME
|
"SEARXNG_URL", "https://searxng.meeks.freeddns.org"
|
||||||
_SEARXNG_PASSWORD: str = SEARXNG_PASSWORD
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class ResearchTools:
|
class ResearchTools:
|
||||||
@@ -187,9 +178,6 @@ class ResearchTools:
|
|||||||
Returns formatted results string on success, or None if the instance
|
Returns formatted results string on success, or None if the instance
|
||||||
is unreachable so the caller can fall through to the next backend.
|
is unreachable so the caller can fall through to the next backend.
|
||||||
"""
|
"""
|
||||||
if not _SEARXNG_URL:
|
|
||||||
logger.info("SearXNG URL is not configured; skipping SearXNG search.")
|
|
||||||
return None
|
|
||||||
params: dict[str, Any] = {
|
params: dict[str, Any] = {
|
||||||
"q": query,
|
"q": query,
|
||||||
"format": "json",
|
"format": "json",
|
||||||
@@ -199,13 +187,9 @@ class ResearchTools:
|
|||||||
if time_range:
|
if time_range:
|
||||||
params["time_range"] = time_range
|
params["time_range"] = time_range
|
||||||
|
|
||||||
auth = None
|
|
||||||
if _SEARXNG_USERNAME and _SEARXNG_PASSWORD:
|
|
||||||
auth = (_SEARXNG_USERNAME, _SEARXNG_PASSWORD)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with httpx.Client(
|
with httpx.Client(
|
||||||
timeout=_DEFAULT_TIMEOUT, follow_redirects=True, auth=auth
|
timeout=_DEFAULT_TIMEOUT, follow_redirects=True
|
||||||
) as client:
|
) as client:
|
||||||
response = client.get(
|
response = client.get(
|
||||||
f"{_SEARXNG_URL}/search",
|
f"{_SEARXNG_URL}/search",
|
||||||
@@ -323,6 +307,22 @@ class ResearchTools:
|
|||||||
logger.info("SearXNG unavailable; falling back to DuckDuckGo")
|
logger.info("SearXNG unavailable; falling back to DuckDuckGo")
|
||||||
|
|
||||||
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
|
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
|
||||||
|
try:
|
||||||
|
from duckduckgo_search import DDGS # type: ignore[import-untyped]
|
||||||
|
except ImportError:
|
||||||
|
logger.debug("duckduckgo-search not installed; using httpx fallback")
|
||||||
|
return self._web_search_fallback(query, num_results)
|
||||||
|
|
||||||
|
# Import exception types (package versions differ on exact names)
|
||||||
|
try:
|
||||||
|
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
|
||||||
|
RatelimitException,
|
||||||
|
DuckDuckGoSearchException,
|
||||||
|
)
|
||||||
|
except ImportError:
|
||||||
|
RatelimitException = Exception # type: ignore[assignment,misc]
|
||||||
|
DuckDuckGoSearchException = Exception # type: ignore[assignment,misc]
|
||||||
|
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -191,21 +191,6 @@ class TestSearchSearxng:
|
|||||||
params = call_kwargs[1].get("params", call_kwargs[0][1] if len(call_kwargs[0]) > 1 else {})
|
params = call_kwargs[1].get("params", call_kwargs[0][1] if len(call_kwargs[0]) > 1 else {})
|
||||||
assert params.get("time_range") == "month"
|
assert params.get("time_range") == "month"
|
||||||
|
|
||||||
@patch("gitea.tools.research_tools.httpx.Client")
|
|
||||||
def test_passes_basic_auth_if_configured(
|
|
||||||
self, mock_cls: MagicMock, tools: ResearchTools
|
|
||||||
) -> None:
|
|
||||||
mock_cls.return_value.__enter__.return_value.get.return_value = (
|
|
||||||
self._make_searxng_response([])
|
|
||||||
)
|
|
||||||
with patch("gitea.tools.research_tools._SEARXNG_USERNAME", "user"), \
|
|
||||||
patch("gitea.tools.research_tools._SEARXNG_PASSWORD", "pass"):
|
|
||||||
tools._search_searxng("q", num_results=5)
|
|
||||||
|
|
||||||
mock_cls.assert_called_once()
|
|
||||||
kwargs = mock_cls.call_args[1]
|
|
||||||
assert kwargs.get("auth") == ("user", "pass")
|
|
||||||
|
|
||||||
@patch("gitea.tools.research_tools.httpx.Client")
|
@patch("gitea.tools.research_tools.httpx.Client")
|
||||||
def test_web_search_uses_searxng_first(
|
def test_web_search_uses_searxng_first(
|
||||||
self, mock_cls: MagicMock, tools: ResearchTools
|
self, mock_cls: MagicMock, tools: ResearchTools
|
||||||
@@ -243,8 +228,7 @@ class TestWebSearch:
|
|||||||
self, _mock_sleep: MagicMock, tools: ResearchTools
|
self, _mock_sleep: MagicMock, tools: ResearchTools
|
||||||
) -> None:
|
) -> None:
|
||||||
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
|
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
|
||||||
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
||||||
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
|
||||||
result = tools.web_search("python httpx")
|
result = tools.web_search("python httpx")
|
||||||
assert "[1]" in result
|
assert "[1]" in result
|
||||||
assert "foo.com" in result
|
assert "foo.com" in result
|
||||||
@@ -254,16 +238,14 @@ class TestWebSearch:
|
|||||||
self, _mock_sleep: MagicMock, tools: ResearchTools
|
self, _mock_sleep: MagicMock, tools: ResearchTools
|
||||||
) -> None:
|
) -> None:
|
||||||
mock_cls = MagicMock(return_value=self._mock_ddgs([]))
|
mock_cls = MagicMock(return_value=self._mock_ddgs([]))
|
||||||
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
||||||
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
|
||||||
result = tools.web_search("xyzzy-not-real")
|
result = tools.web_search("xyzzy-not-real")
|
||||||
assert "No results" in result
|
assert "No results" in result
|
||||||
|
|
||||||
def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
|
def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
|
||||||
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
||||||
mock_cls = MagicMock(return_value=ddgs_mock)
|
mock_cls = MagicMock(return_value=ddgs_mock)
|
||||||
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
||||||
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
|
||||||
tools.web_search("q", num_results=999)
|
tools.web_search("q", num_results=999)
|
||||||
# DDGS.text should be called with max_results clamped to 20
|
# DDGS.text should be called with max_results clamped to 20
|
||||||
ddgs_mock.text.assert_called_once()
|
ddgs_mock.text.assert_called_once()
|
||||||
@@ -273,8 +255,7 @@ class TestWebSearch:
|
|||||||
def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
|
def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
|
||||||
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
||||||
mock_cls = MagicMock(return_value=ddgs_mock)
|
mock_cls = MagicMock(return_value=ddgs_mock)
|
||||||
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
||||||
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
|
||||||
tools.web_search("q", num_results=0)
|
tools.web_search("q", num_results=0)
|
||||||
_, kwargs = ddgs_mock.text.call_args
|
_, kwargs = ddgs_mock.text.call_args
|
||||||
assert kwargs.get("max_results", 0) >= 1
|
assert kwargs.get("max_results", 0) >= 1
|
||||||
@@ -290,14 +271,40 @@ class TestWebSearch:
|
|||||||
ddgs_mock.__exit__ = MagicMock(return_value=False)
|
ddgs_mock.__exit__ = MagicMock(return_value=False)
|
||||||
ddgs_mock.text.side_effect = rate_exc
|
ddgs_mock.text.side_effect = rate_exc
|
||||||
|
|
||||||
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
mock_module = MagicMock()
|
||||||
patch("gitea.tools.research_tools.DDGS", return_value=ddgs_mock), \
|
mock_module.DDGS = MagicMock(return_value=ddgs_mock)
|
||||||
patch("gitea.tools.research_tools.RatelimitException", type(rate_exc)), \
|
|
||||||
patch("gitea.tools.research_tools.DuckDuckGoSearchException", ValueError):
|
# Make RatelimitException match our rate_exc type
|
||||||
|
mock_module.exceptions.RatelimitException = type(rate_exc)
|
||||||
|
mock_module.exceptions.DuckDuckGoSearchException = ValueError
|
||||||
|
|
||||||
|
with patch.dict("sys.modules", {
|
||||||
|
"duckduckgo_search": mock_module,
|
||||||
|
"duckduckgo_search.exceptions": mock_module.exceptions,
|
||||||
|
}):
|
||||||
result = tools.web_search("q")
|
result = tools.web_search("q")
|
||||||
|
|
||||||
assert isinstance(result, str) # Returns error string, not raise
|
assert isinstance(result, str) # Returns error string, not raise
|
||||||
|
|
||||||
|
def test_fallback_when_ddgs_not_installed(self, tools: ResearchTools) -> None:
|
||||||
|
"""When duckduckgo-search is not installed, uses httpx HTML fallback."""
|
||||||
|
import builtins
|
||||||
|
real_import = builtins.__import__
|
||||||
|
|
||||||
|
def mock_import(name: str, *args, **kwargs): # type: ignore[no-untyped-def]
|
||||||
|
if name == "duckduckgo_search":
|
||||||
|
raise ImportError("mocked missing")
|
||||||
|
return real_import(name, *args, **kwargs)
|
||||||
|
|
||||||
|
mock_response = MagicMock()
|
||||||
|
mock_response.text = ""
|
||||||
|
mock_response.raise_for_status = MagicMock()
|
||||||
|
|
||||||
|
with patch("builtins.__import__", side_effect=mock_import):
|
||||||
|
with patch("gitea.tools.research_tools.httpx.Client") as mock_cls:
|
||||||
|
mock_cls.return_value.__enter__.return_value.get.return_value = mock_response
|
||||||
|
result = tools.web_search("test")
|
||||||
|
assert isinstance(result, str)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
|
|||||||
Reference in New Issue
Block a user