feat: make SearXNG URL env-configurable and support basic auth #7
@@ -13,6 +13,9 @@ class AgentSettings(BaseSettings):
|
|||||||
gitea_repos_root: str = ""
|
gitea_repos_root: str = ""
|
||||||
agent_model_id: str = "qwen/qwen3.6-35b-a3b"
|
agent_model_id: str = "qwen/qwen3.6-35b-a3b"
|
||||||
agent_max_retries: int = 2
|
agent_max_retries: int = 2
|
||||||
|
searxng_url: str = ""
|
||||||
|
searxng_username: str = ""
|
||||||
|
searxng_password: str = ""
|
||||||
|
|
||||||
|
|
||||||
def get_settings() -> AgentSettings:
|
def get_settings() -> AgentSettings:
|
||||||
@@ -28,7 +31,15 @@ GITEA_TOKEN: str = _agent_settings.gitea_token
|
|||||||
GITEA_REPOS_ROOT: str = _agent_settings.gitea_repos_root
|
GITEA_REPOS_ROOT: str = _agent_settings.gitea_repos_root
|
||||||
AGENT_MODEL_ID: str = _agent_settings.agent_model_id
|
AGENT_MODEL_ID: str = _agent_settings.agent_model_id
|
||||||
AGENT_MAX_RETRIES: int = _agent_settings.agent_max_retries
|
AGENT_MAX_RETRIES: int = _agent_settings.agent_max_retries
|
||||||
|
SEARXNG_URL: str = _agent_settings.searxng_url
|
||||||
|
SEARXNG_USERNAME: str = _agent_settings.searxng_username
|
||||||
|
SEARXNG_PASSWORD: str = _agent_settings.searxng_password
|
||||||
|
|
||||||
import os
|
import os
|
||||||
os.environ["GITEA_SERVER_URL"] = GITEA_URL
|
os.environ["GITEA_SERVER_URL"] = GITEA_URL
|
||||||
os.environ["GITEA_SERVER_TOKEN"] = GITEA_TOKEN
|
os.environ["GITEA_SERVER_TOKEN"] = GITEA_TOKEN
|
||||||
|
os.environ["SEARXNG_URL"] = SEARXNG_URL
|
||||||
|
os.environ["SEARXNG_USERNAME"] = SEARXNG_USERNAME
|
||||||
|
os.environ["SEARXNG_PASSWORD"] = SEARXNG_PASSWORD
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,13 @@ import time
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
from duckduckgo_search import DDGS # type: ignore[import-untyped]
|
||||||
|
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
|
||||||
|
DuckDuckGoSearchException,
|
||||||
|
RatelimitException,
|
||||||
|
)
|
||||||
|
|
||||||
|
from gitea.config import SEARXNG_URL, SEARXNG_USERNAME, SEARXNG_PASSWORD
|
||||||
|
|
||||||
logger: logging.Logger = logging.getLogger("research-tools")
|
logger: logging.Logger = logging.getLogger("research-tools")
|
||||||
|
|
||||||
@@ -29,10 +36,12 @@ _USER_AGENT: str = (
|
|||||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||||
)
|
)
|
||||||
# SearXNG instance — override with SEARXNG_URL env var
|
# SearXNG instance
|
||||||
_SEARXNG_URL: str = os.getenv(
|
_SEARXNG_URL: str = SEARXNG_URL
|
||||||
"SEARXNG_URL", "https://searxng.meeks.freeddns.org"
|
_SEARXNG_USERNAME: str = SEARXNG_USERNAME
|
||||||
)
|
_SEARXNG_PASSWORD: str = SEARXNG_PASSWORD
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class ResearchTools:
|
class ResearchTools:
|
||||||
@@ -178,6 +187,9 @@ class ResearchTools:
|
|||||||
Returns formatted results string on success, or None if the instance
|
Returns formatted results string on success, or None if the instance
|
||||||
is unreachable so the caller can fall through to the next backend.
|
is unreachable so the caller can fall through to the next backend.
|
||||||
"""
|
"""
|
||||||
|
if not _SEARXNG_URL:
|
||||||
|
logger.info("SearXNG URL is not configured; skipping SearXNG search.")
|
||||||
|
return None
|
||||||
params: dict[str, Any] = {
|
params: dict[str, Any] = {
|
||||||
"q": query,
|
"q": query,
|
||||||
"format": "json",
|
"format": "json",
|
||||||
@@ -187,9 +199,13 @@ class ResearchTools:
|
|||||||
if time_range:
|
if time_range:
|
||||||
params["time_range"] = time_range
|
params["time_range"] = time_range
|
||||||
|
|
||||||
|
auth = None
|
||||||
|
if _SEARXNG_USERNAME and _SEARXNG_PASSWORD:
|
||||||
|
auth = (_SEARXNG_USERNAME, _SEARXNG_PASSWORD)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with httpx.Client(
|
with httpx.Client(
|
||||||
timeout=_DEFAULT_TIMEOUT, follow_redirects=True
|
timeout=_DEFAULT_TIMEOUT, follow_redirects=True, auth=auth
|
||||||
) as client:
|
) as client:
|
||||||
response = client.get(
|
response = client.get(
|
||||||
f"{_SEARXNG_URL}/search",
|
f"{_SEARXNG_URL}/search",
|
||||||
@@ -307,22 +323,6 @@ class ResearchTools:
|
|||||||
logger.info("SearXNG unavailable; falling back to DuckDuckGo")
|
logger.info("SearXNG unavailable; falling back to DuckDuckGo")
|
||||||
|
|
||||||
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
|
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
|
||||||
try:
|
|
||||||
from duckduckgo_search import DDGS # type: ignore[import-untyped]
|
|
||||||
except ImportError:
|
|
||||||
logger.debug("duckduckgo-search not installed; using httpx fallback")
|
|
||||||
return self._web_search_fallback(query, num_results)
|
|
||||||
|
|
||||||
# Import exception types (package versions differ on exact names)
|
|
||||||
try:
|
|
||||||
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
|
|
||||||
RatelimitException,
|
|
||||||
DuckDuckGoSearchException,
|
|
||||||
)
|
|
||||||
except ImportError:
|
|
||||||
RatelimitException = Exception # type: ignore[assignment,misc]
|
|
||||||
DuckDuckGoSearchException = Exception # type: ignore[assignment,misc]
|
|
||||||
|
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -191,6 +191,21 @@ class TestSearchSearxng:
|
|||||||
params = call_kwargs[1].get("params", call_kwargs[0][1] if len(call_kwargs[0]) > 1 else {})
|
params = call_kwargs[1].get("params", call_kwargs[0][1] if len(call_kwargs[0]) > 1 else {})
|
||||||
assert params.get("time_range") == "month"
|
assert params.get("time_range") == "month"
|
||||||
|
|
||||||
|
@patch("gitea.tools.research_tools.httpx.Client")
|
||||||
|
def test_passes_basic_auth_if_configured(
|
||||||
|
self, mock_cls: MagicMock, tools: ResearchTools
|
||||||
|
) -> None:
|
||||||
|
mock_cls.return_value.__enter__.return_value.get.return_value = (
|
||||||
|
self._make_searxng_response([])
|
||||||
|
)
|
||||||
|
with patch("gitea.tools.research_tools._SEARXNG_USERNAME", "user"), \
|
||||||
|
patch("gitea.tools.research_tools._SEARXNG_PASSWORD", "pass"):
|
||||||
|
tools._search_searxng("q", num_results=5)
|
||||||
|
|
||||||
|
mock_cls.assert_called_once()
|
||||||
|
kwargs = mock_cls.call_args[1]
|
||||||
|
assert kwargs.get("auth") == ("user", "pass")
|
||||||
|
|
||||||
@patch("gitea.tools.research_tools.httpx.Client")
|
@patch("gitea.tools.research_tools.httpx.Client")
|
||||||
def test_web_search_uses_searxng_first(
|
def test_web_search_uses_searxng_first(
|
||||||
self, mock_cls: MagicMock, tools: ResearchTools
|
self, mock_cls: MagicMock, tools: ResearchTools
|
||||||
@@ -228,7 +243,8 @@ class TestWebSearch:
|
|||||||
self, _mock_sleep: MagicMock, tools: ResearchTools
|
self, _mock_sleep: MagicMock, tools: ResearchTools
|
||||||
) -> None:
|
) -> None:
|
||||||
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
|
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
||||||
|
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
result = tools.web_search("python httpx")
|
result = tools.web_search("python httpx")
|
||||||
assert "[1]" in result
|
assert "[1]" in result
|
||||||
assert "foo.com" in result
|
assert "foo.com" in result
|
||||||
@@ -238,14 +254,16 @@ class TestWebSearch:
|
|||||||
self, _mock_sleep: MagicMock, tools: ResearchTools
|
self, _mock_sleep: MagicMock, tools: ResearchTools
|
||||||
) -> None:
|
) -> None:
|
||||||
mock_cls = MagicMock(return_value=self._mock_ddgs([]))
|
mock_cls = MagicMock(return_value=self._mock_ddgs([]))
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
||||||
|
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
result = tools.web_search("xyzzy-not-real")
|
result = tools.web_search("xyzzy-not-real")
|
||||||
assert "No results" in result
|
assert "No results" in result
|
||||||
|
|
||||||
def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
|
def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
|
||||||
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
||||||
mock_cls = MagicMock(return_value=ddgs_mock)
|
mock_cls = MagicMock(return_value=ddgs_mock)
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
||||||
|
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
tools.web_search("q", num_results=999)
|
tools.web_search("q", num_results=999)
|
||||||
# DDGS.text should be called with max_results clamped to 20
|
# DDGS.text should be called with max_results clamped to 20
|
||||||
ddgs_mock.text.assert_called_once()
|
ddgs_mock.text.assert_called_once()
|
||||||
@@ -255,7 +273,8 @@ class TestWebSearch:
|
|||||||
def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
|
def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
|
||||||
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
||||||
mock_cls = MagicMock(return_value=ddgs_mock)
|
mock_cls = MagicMock(return_value=ddgs_mock)
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
||||||
|
patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
tools.web_search("q", num_results=0)
|
tools.web_search("q", num_results=0)
|
||||||
_, kwargs = ddgs_mock.text.call_args
|
_, kwargs = ddgs_mock.text.call_args
|
||||||
assert kwargs.get("max_results", 0) >= 1
|
assert kwargs.get("max_results", 0) >= 1
|
||||||
@@ -271,40 +290,14 @@ class TestWebSearch:
|
|||||||
ddgs_mock.__exit__ = MagicMock(return_value=False)
|
ddgs_mock.__exit__ = MagicMock(return_value=False)
|
||||||
ddgs_mock.text.side_effect = rate_exc
|
ddgs_mock.text.side_effect = rate_exc
|
||||||
|
|
||||||
mock_module = MagicMock()
|
with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
|
||||||
mock_module.DDGS = MagicMock(return_value=ddgs_mock)
|
patch("gitea.tools.research_tools.DDGS", return_value=ddgs_mock), \
|
||||||
|
patch("gitea.tools.research_tools.RatelimitException", type(rate_exc)), \
|
||||||
# Make RatelimitException match our rate_exc type
|
patch("gitea.tools.research_tools.DuckDuckGoSearchException", ValueError):
|
||||||
mock_module.exceptions.RatelimitException = type(rate_exc)
|
|
||||||
mock_module.exceptions.DuckDuckGoSearchException = ValueError
|
|
||||||
|
|
||||||
with patch.dict("sys.modules", {
|
|
||||||
"duckduckgo_search": mock_module,
|
|
||||||
"duckduckgo_search.exceptions": mock_module.exceptions,
|
|
||||||
}):
|
|
||||||
result = tools.web_search("q")
|
result = tools.web_search("q")
|
||||||
|
|
||||||
assert isinstance(result, str) # Returns error string, not raise
|
assert isinstance(result, str) # Returns error string, not raise
|
||||||
|
|
||||||
def test_fallback_when_ddgs_not_installed(self, tools: ResearchTools) -> None:
|
|
||||||
"""When duckduckgo-search is not installed, uses httpx HTML fallback."""
|
|
||||||
import builtins
|
|
||||||
real_import = builtins.__import__
|
|
||||||
|
|
||||||
def mock_import(name: str, *args, **kwargs): # type: ignore[no-untyped-def]
|
|
||||||
if name == "duckduckgo_search":
|
|
||||||
raise ImportError("mocked missing")
|
|
||||||
return real_import(name, *args, **kwargs)
|
|
||||||
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.text = ""
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
|
|
||||||
with patch("builtins.__import__", side_effect=mock_import):
|
|
||||||
with patch("gitea.tools.research_tools.httpx.Client") as mock_cls:
|
|
||||||
mock_cls.return_value.__enter__.return_value.get.return_value = mock_response
|
|
||||||
result = tools.web_search("test")
|
|
||||||
assert isinstance(result, str)
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
|
|||||||
Reference in New Issue
Block a user