feat: make SearXNG URL env-configurable and support basic auth #7

Merged
michael merged 4 commits from feat/searxng-auth-env-url-v2 into master 2026-06-30 22:48:22 +02:00
3 changed files with 59 additions and 55 deletions
+11
View File
@@ -13,6 +13,9 @@ class AgentSettings(BaseSettings):
gitea_repos_root: str = "" gitea_repos_root: str = ""
agent_model_id: str = "qwen/qwen3.6-35b-a3b" agent_model_id: str = "qwen/qwen3.6-35b-a3b"
agent_max_retries: int = 2 agent_max_retries: int = 2
searxng_url: str = ""
searxng_username: str = ""
searxng_password: str = ""
def get_settings() -> AgentSettings: def get_settings() -> AgentSettings:
@@ -28,7 +31,15 @@ GITEA_TOKEN: str = _agent_settings.gitea_token
GITEA_REPOS_ROOT: str = _agent_settings.gitea_repos_root GITEA_REPOS_ROOT: str = _agent_settings.gitea_repos_root
AGENT_MODEL_ID: str = _agent_settings.agent_model_id AGENT_MODEL_ID: str = _agent_settings.agent_model_id
AGENT_MAX_RETRIES: int = _agent_settings.agent_max_retries AGENT_MAX_RETRIES: int = _agent_settings.agent_max_retries
SEARXNG_URL: str = _agent_settings.searxng_url
SEARXNG_USERNAME: str = _agent_settings.searxng_username
SEARXNG_PASSWORD: str = _agent_settings.searxng_password
import os import os
os.environ["GITEA_SERVER_URL"] = GITEA_URL os.environ["GITEA_SERVER_URL"] = GITEA_URL
os.environ["GITEA_SERVER_TOKEN"] = GITEA_TOKEN os.environ["GITEA_SERVER_TOKEN"] = GITEA_TOKEN
os.environ["SEARXNG_URL"] = SEARXNG_URL
os.environ["SEARXNG_USERNAME"] = SEARXNG_USERNAME
os.environ["SEARXNG_PASSWORD"] = SEARXNG_PASSWORD
+21 -21
View File
@@ -20,6 +20,13 @@ import time
from typing import Any from typing import Any
import httpx import httpx
from duckduckgo_search import DDGS # type: ignore[import-untyped]
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
DuckDuckGoSearchException,
RatelimitException,
)
from gitea.config import SEARXNG_URL, SEARXNG_USERNAME, SEARXNG_PASSWORD
logger: logging.Logger = logging.getLogger("research-tools") logger: logging.Logger = logging.getLogger("research-tools")
@@ -29,10 +36,12 @@ _USER_AGENT: str = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
) )
# SearXNG instance — override with SEARXNG_URL env var # SearXNG instance
_SEARXNG_URL: str = os.getenv( _SEARXNG_URL: str = SEARXNG_URL
"SEARXNG_URL", "https://searxng.meeks.freeddns.org" _SEARXNG_USERNAME: str = SEARXNG_USERNAME
) _SEARXNG_PASSWORD: str = SEARXNG_PASSWORD
class ResearchTools: class ResearchTools:
@@ -178,6 +187,9 @@ class ResearchTools:
Returns formatted results string on success, or None if the instance Returns formatted results string on success, or None if the instance
is unreachable so the caller can fall through to the next backend. is unreachable so the caller can fall through to the next backend.
""" """
if not _SEARXNG_URL:
logger.info("SearXNG URL is not configured; skipping SearXNG search.")
return None
params: dict[str, Any] = { params: dict[str, Any] = {
"q": query, "q": query,
"format": "json", "format": "json",
@@ -187,9 +199,13 @@ class ResearchTools:
if time_range: if time_range:
params["time_range"] = time_range params["time_range"] = time_range
auth = None
if _SEARXNG_USERNAME and _SEARXNG_PASSWORD:
auth = (_SEARXNG_USERNAME, _SEARXNG_PASSWORD)
try: try:
with httpx.Client( with httpx.Client(
timeout=_DEFAULT_TIMEOUT, follow_redirects=True timeout=_DEFAULT_TIMEOUT, follow_redirects=True, auth=auth
) as client: ) as client:
response = client.get( response = client.get(
f"{_SEARXNG_URL}/search", f"{_SEARXNG_URL}/search",
@@ -307,22 +323,6 @@ class ResearchTools:
logger.info("SearXNG unavailable; falling back to DuckDuckGo") logger.info("SearXNG unavailable; falling back to DuckDuckGo")
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) --- # --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
try:
from duckduckgo_search import DDGS # type: ignore[import-untyped]
except ImportError:
logger.debug("duckduckgo-search not installed; using httpx fallback")
return self._web_search_fallback(query, num_results)
# Import exception types (package versions differ on exact names)
try:
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
RatelimitException,
DuckDuckGoSearchException,
)
except ImportError:
RatelimitException = Exception # type: ignore[assignment,misc]
DuckDuckGoSearchException = Exception # type: ignore[assignment,misc]
last_exc: Exception | None = None last_exc: Exception | None = None
for attempt in range(3): for attempt in range(3):
try: try:
+27 -34
View File
@@ -191,6 +191,21 @@ class TestSearchSearxng:
params = call_kwargs[1].get("params", call_kwargs[0][1] if len(call_kwargs[0]) > 1 else {}) params = call_kwargs[1].get("params", call_kwargs[0][1] if len(call_kwargs[0]) > 1 else {})
assert params.get("time_range") == "month" assert params.get("time_range") == "month"
@patch("gitea.tools.research_tools.httpx.Client")
def test_passes_basic_auth_if_configured(
self, mock_cls: MagicMock, tools: ResearchTools
) -> None:
mock_cls.return_value.__enter__.return_value.get.return_value = (
self._make_searxng_response([])
)
with patch("gitea.tools.research_tools._SEARXNG_USERNAME", "user"), \
patch("gitea.tools.research_tools._SEARXNG_PASSWORD", "pass"):
tools._search_searxng("q", num_results=5)
mock_cls.assert_called_once()
kwargs = mock_cls.call_args[1]
assert kwargs.get("auth") == ("user", "pass")
@patch("gitea.tools.research_tools.httpx.Client") @patch("gitea.tools.research_tools.httpx.Client")
def test_web_search_uses_searxng_first( def test_web_search_uses_searxng_first(
self, mock_cls: MagicMock, tools: ResearchTools self, mock_cls: MagicMock, tools: ResearchTools
@@ -228,7 +243,8 @@ class TestWebSearch:
self, _mock_sleep: MagicMock, tools: ResearchTools self, _mock_sleep: MagicMock, tools: ResearchTools
) -> None: ) -> None:
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result())) mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}): with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
patch("gitea.tools.research_tools.DDGS", mock_cls):
result = tools.web_search("python httpx") result = tools.web_search("python httpx")
assert "[1]" in result assert "[1]" in result
assert "foo.com" in result assert "foo.com" in result
@@ -238,14 +254,16 @@ class TestWebSearch:
self, _mock_sleep: MagicMock, tools: ResearchTools self, _mock_sleep: MagicMock, tools: ResearchTools
) -> None: ) -> None:
mock_cls = MagicMock(return_value=self._mock_ddgs([])) mock_cls = MagicMock(return_value=self._mock_ddgs([]))
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}): with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
patch("gitea.tools.research_tools.DDGS", mock_cls):
result = tools.web_search("xyzzy-not-real") result = tools.web_search("xyzzy-not-real")
assert "No results" in result assert "No results" in result
def test_clamps_num_results_max(self, tools: ResearchTools) -> None: def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
ddgs_mock = self._mock_ddgs(self._make_ddgs_result()) ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
mock_cls = MagicMock(return_value=ddgs_mock) mock_cls = MagicMock(return_value=ddgs_mock)
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}): with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
patch("gitea.tools.research_tools.DDGS", mock_cls):
tools.web_search("q", num_results=999) tools.web_search("q", num_results=999)
# DDGS.text should be called with max_results clamped to 20 # DDGS.text should be called with max_results clamped to 20
ddgs_mock.text.assert_called_once() ddgs_mock.text.assert_called_once()
@@ -255,7 +273,8 @@ class TestWebSearch:
def test_clamps_num_results_min(self, tools: ResearchTools) -> None: def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
ddgs_mock = self._mock_ddgs(self._make_ddgs_result()) ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
mock_cls = MagicMock(return_value=ddgs_mock) mock_cls = MagicMock(return_value=ddgs_mock)
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}): with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
patch("gitea.tools.research_tools.DDGS", mock_cls):
tools.web_search("q", num_results=0) tools.web_search("q", num_results=0)
_, kwargs = ddgs_mock.text.call_args _, kwargs = ddgs_mock.text.call_args
assert kwargs.get("max_results", 0) >= 1 assert kwargs.get("max_results", 0) >= 1
@@ -271,40 +290,14 @@ class TestWebSearch:
ddgs_mock.__exit__ = MagicMock(return_value=False) ddgs_mock.__exit__ = MagicMock(return_value=False)
ddgs_mock.text.side_effect = rate_exc ddgs_mock.text.side_effect = rate_exc
mock_module = MagicMock() with patch("gitea.tools.research_tools.ResearchTools._search_searxng", return_value=None), \
mock_module.DDGS = MagicMock(return_value=ddgs_mock) patch("gitea.tools.research_tools.DDGS", return_value=ddgs_mock), \
patch("gitea.tools.research_tools.RatelimitException", type(rate_exc)), \
# Make RatelimitException match our rate_exc type patch("gitea.tools.research_tools.DuckDuckGoSearchException", ValueError):
mock_module.exceptions.RatelimitException = type(rate_exc)
mock_module.exceptions.DuckDuckGoSearchException = ValueError
with patch.dict("sys.modules", {
"duckduckgo_search": mock_module,
"duckduckgo_search.exceptions": mock_module.exceptions,
}):
result = tools.web_search("q") result = tools.web_search("q")
assert isinstance(result, str) # Returns error string, not raise assert isinstance(result, str) # Returns error string, not raise
def test_fallback_when_ddgs_not_installed(self, tools: ResearchTools) -> None:
"""When duckduckgo-search is not installed, uses httpx HTML fallback."""
import builtins
real_import = builtins.__import__
def mock_import(name: str, *args, **kwargs): # type: ignore[no-untyped-def]
if name == "duckduckgo_search":
raise ImportError("mocked missing")
return real_import(name, *args, **kwargs)
mock_response = MagicMock()
mock_response.text = ""
mock_response.raise_for_status = MagicMock()
with patch("builtins.__import__", side_effect=mock_import):
with patch("gitea.tools.research_tools.httpx.Client") as mock_cls:
mock_cls.return_value.__enter__.return_value.get.return_value = mock_response
result = tools.web_search("test")
assert isinstance(result, str)
# ------------------------------------------------------------------ # # ------------------------------------------------------------------ #