feat: make SearXNG URL env-configurable and support basic auth #7
@@ -20,6 +20,11 @@ import time
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
from duckduckgo_search import DDGS # type: ignore[import-untyped]
|
||||||
|
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
|
||||||
|
DuckDuckGoSearchException,
|
||||||
|
RatelimitException,
|
||||||
|
)
|
||||||
|
|
||||||
from gitea.config import SEARXNG_URL
|
from gitea.config import SEARXNG_URL
|
||||||
|
|
||||||
@@ -311,22 +316,6 @@ class ResearchTools:
|
|||||||
logger.info("SearXNG unavailable; falling back to DuckDuckGo")
|
logger.info("SearXNG unavailable; falling back to DuckDuckGo")
|
||||||
|
|
||||||
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
|
# --- Tier 2: DDGS library (handles sessions, cookies, rate limits) ---
|
||||||
try:
|
|
||||||
from duckduckgo_search import DDGS # type: ignore[import-untyped]
|
|
||||||
except ImportError:
|
|
||||||
logger.debug("duckduckgo-search not installed; using httpx fallback")
|
|
||||||
return self._web_search_fallback(query, num_results)
|
|
||||||
|
|
||||||
# Import exception types (package versions differ on exact names)
|
|
||||||
try:
|
|
||||||
from duckduckgo_search.exceptions import ( # type: ignore[import-untyped]
|
|
||||||
RatelimitException,
|
|
||||||
DuckDuckGoSearchException,
|
|
||||||
)
|
|
||||||
except ImportError:
|
|
||||||
RatelimitException = Exception # type: ignore[assignment,misc]
|
|
||||||
DuckDuckGoSearchException = Exception # type: ignore[assignment,misc]
|
|
||||||
|
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -228,7 +228,7 @@ class TestWebSearch:
|
|||||||
self, _mock_sleep: MagicMock, tools: ResearchTools
|
self, _mock_sleep: MagicMock, tools: ResearchTools
|
||||||
) -> None:
|
) -> None:
|
||||||
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
|
mock_cls = MagicMock(return_value=self._mock_ddgs(self._make_ddgs_result()))
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
result = tools.web_search("python httpx")
|
result = tools.web_search("python httpx")
|
||||||
assert "[1]" in result
|
assert "[1]" in result
|
||||||
assert "foo.com" in result
|
assert "foo.com" in result
|
||||||
@@ -238,14 +238,14 @@ class TestWebSearch:
|
|||||||
self, _mock_sleep: MagicMock, tools: ResearchTools
|
self, _mock_sleep: MagicMock, tools: ResearchTools
|
||||||
) -> None:
|
) -> None:
|
||||||
mock_cls = MagicMock(return_value=self._mock_ddgs([]))
|
mock_cls = MagicMock(return_value=self._mock_ddgs([]))
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
result = tools.web_search("xyzzy-not-real")
|
result = tools.web_search("xyzzy-not-real")
|
||||||
assert "No results" in result
|
assert "No results" in result
|
||||||
|
|
||||||
def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
|
def test_clamps_num_results_max(self, tools: ResearchTools) -> None:
|
||||||
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
||||||
mock_cls = MagicMock(return_value=ddgs_mock)
|
mock_cls = MagicMock(return_value=ddgs_mock)
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
tools.web_search("q", num_results=999)
|
tools.web_search("q", num_results=999)
|
||||||
# DDGS.text should be called with max_results clamped to 20
|
# DDGS.text should be called with max_results clamped to 20
|
||||||
ddgs_mock.text.assert_called_once()
|
ddgs_mock.text.assert_called_once()
|
||||||
@@ -255,7 +255,7 @@ class TestWebSearch:
|
|||||||
def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
|
def test_clamps_num_results_min(self, tools: ResearchTools) -> None:
|
||||||
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
ddgs_mock = self._mock_ddgs(self._make_ddgs_result())
|
||||||
mock_cls = MagicMock(return_value=ddgs_mock)
|
mock_cls = MagicMock(return_value=ddgs_mock)
|
||||||
with patch.dict("sys.modules", {"duckduckgo_search": MagicMock(DDGS=mock_cls)}):
|
with patch("gitea.tools.research_tools.DDGS", mock_cls):
|
||||||
tools.web_search("q", num_results=0)
|
tools.web_search("q", num_results=0)
|
||||||
_, kwargs = ddgs_mock.text.call_args
|
_, kwargs = ddgs_mock.text.call_args
|
||||||
assert kwargs.get("max_results", 0) >= 1
|
assert kwargs.get("max_results", 0) >= 1
|
||||||
@@ -271,40 +271,13 @@ class TestWebSearch:
|
|||||||
ddgs_mock.__exit__ = MagicMock(return_value=False)
|
ddgs_mock.__exit__ = MagicMock(return_value=False)
|
||||||
ddgs_mock.text.side_effect = rate_exc
|
ddgs_mock.text.side_effect = rate_exc
|
||||||
|
|
||||||
mock_module = MagicMock()
|
with patch("gitea.tools.research_tools.DDGS", return_value=ddgs_mock), \
|
||||||
mock_module.DDGS = MagicMock(return_value=ddgs_mock)
|
patch("gitea.tools.research_tools.RatelimitException", type(rate_exc)), \
|
||||||
|
patch("gitea.tools.research_tools.DuckDuckGoSearchException", ValueError):
|
||||||
# Make RatelimitException match our rate_exc type
|
|
||||||
mock_module.exceptions.RatelimitException = type(rate_exc)
|
|
||||||
mock_module.exceptions.DuckDuckGoSearchException = ValueError
|
|
||||||
|
|
||||||
with patch.dict("sys.modules", {
|
|
||||||
"duckduckgo_search": mock_module,
|
|
||||||
"duckduckgo_search.exceptions": mock_module.exceptions,
|
|
||||||
}):
|
|
||||||
result = tools.web_search("q")
|
result = tools.web_search("q")
|
||||||
|
|
||||||
assert isinstance(result, str) # Returns error string, not raise
|
assert isinstance(result, str) # Returns error string, not raise
|
||||||
|
|
||||||
def test_fallback_when_ddgs_not_installed(self, tools: ResearchTools) -> None:
|
|
||||||
"""When duckduckgo-search is not installed, uses httpx HTML fallback."""
|
|
||||||
import builtins
|
|
||||||
real_import = builtins.__import__
|
|
||||||
|
|
||||||
def mock_import(name: str, *args, **kwargs): # type: ignore[no-untyped-def]
|
|
||||||
if name == "duckduckgo_search":
|
|
||||||
raise ImportError("mocked missing")
|
|
||||||
return real_import(name, *args, **kwargs)
|
|
||||||
|
|
||||||
mock_response = MagicMock()
|
|
||||||
mock_response.text = ""
|
|
||||||
mock_response.raise_for_status = MagicMock()
|
|
||||||
|
|
||||||
with patch("builtins.__import__", side_effect=mock_import):
|
|
||||||
with patch("gitea.tools.research_tools.httpx.Client") as mock_cls:
|
|
||||||
mock_cls.return_value.__enter__.return_value.get.return_value = mock_response
|
|
||||||
result = tools.web_search("test")
|
|
||||||
assert isinstance(result, str)
|
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ #
|
# ------------------------------------------------------------------ #
|
||||||
|
|||||||
Reference in New Issue
Block a user