156 lines
6.2 KiB
Python
156 lines
6.2 KiB
Python
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.agent_runs import AgentRunRepository, AgentService
|
|
from app.alerts import ALERT_DATA_NOTICE, AlertRepository
|
|
from app.constants import OLLAMA_URL
|
|
from app.models import KIND_PROBLEM, Event
|
|
from app.ollama import OllamaClient
|
|
from app.prompts import (PROMPT_AGENT, ZPOOL_SAMPLE, PromptError, PromptTester, RateLimitError, UserRateLimiter,
|
|
parse_sample, sample_from_row)
|
|
from tests.prompt_support import Clock, chat_reply, make_env, patch_ollama
|
|
|
|
SECRET = "ollama-token-4711"
|
|
EDITABLE = "Teste mich."
|
|
|
|
|
|
class Env:
|
|
def __init__(self, monkeypatch, handler, limit=6):
|
|
self.db, self.store, self.prompts, _ = make_env()
|
|
self.store.set(OLLAMA_URL, "http://ollama.test", "t")
|
|
self.clock = Clock(0.0)
|
|
self.requests = []
|
|
|
|
def recording(request):
|
|
self.requests.append(json.loads(request.content))
|
|
return handler(request)
|
|
|
|
patch_ollama(monkeypatch, recording)
|
|
self.agent = AgentService(self.store, self.db, AgentRunRepository(self.db), lambda h, s: [],
|
|
lambda: [SECRET], lambda s: OllamaClient.from_settings(s),
|
|
lambda s: _Reader())
|
|
self.tester = PromptTester(
|
|
self.store, lambda: OllamaClient.from_settings(self.store), lambda h, s, n: [], self.agent,
|
|
lambda: [SECRET], UserRateLimiter(limit, 60.0, self.clock), clock=self.clock)
|
|
|
|
def counts(self):
|
|
return {t: self.db.query(f"SELECT COUNT(*) FROM {t}")[0][0] for t in ("alerts", "deliveries", "agent_runs")}
|
|
|
|
|
|
class _Reader:
|
|
async def aclose(self):
|
|
pass
|
|
|
|
|
|
async def test_alert_trial_sends_draft_text_with_notice_and_user_block(monkeypatch):
|
|
env = Env(monkeypatch, lambda r: chat_reply("Zeile 1\nZeile 2"))
|
|
result = await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert result.ok and result.text == "Zeile 1\nZeile 2" and result.error == ""
|
|
body = env.requests[0]
|
|
assert body["model"] == "qwen3"
|
|
assert body["messages"][0]["content"] == f"{EDITABLE}\n\n{ALERT_DATA_NOTICE}"
|
|
user = body["messages"][1]["content"]
|
|
assert "kosext100.krs.local" in user and "zpool status" in user and "Status: WARN" in user
|
|
assert "zpool replace" in user
|
|
|
|
|
|
async def test_alert_trial_writes_nothing_and_sends_nothing(monkeypatch):
|
|
env = Env(monkeypatch, lambda r: chat_reply("ok"))
|
|
alerts = AlertRepository(env.db)
|
|
alerts.insert_alerts([Event(1, "h", "s", KIND_PROBLEM, 2, "o")], "g")
|
|
before = env.counts()
|
|
await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert env.counts() == before
|
|
assert len(env.requests) == 1
|
|
|
|
|
|
async def test_alert_trial_error_has_no_secret(monkeypatch):
|
|
def handler(request):
|
|
raise httpx.ConnectError(f"Verbindung mit Bearer {SECRET} verweigert")
|
|
|
|
env = Env(monkeypatch, handler)
|
|
result = await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert not result.ok and "ConnectError" in result.error
|
|
assert SECRET not in result.error and len(result.error) <= 200
|
|
|
|
|
|
async def test_alert_trial_empty_answer_and_missing_ollama(monkeypatch):
|
|
env = Env(monkeypatch, lambda r: chat_reply(" "))
|
|
result = await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert not result.ok and result.error
|
|
env.store.set(OLLAMA_URL, "", "t")
|
|
missing = await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert not missing.ok and "nicht konfiguriert" in missing.error
|
|
|
|
|
|
async def test_duration_measured_with_injected_clock(monkeypatch):
|
|
holder = {}
|
|
|
|
def handler(request):
|
|
holder["env"].clock.now += 2.5
|
|
return chat_reply("ok")
|
|
|
|
env = holder["env"] = Env(monkeypatch, handler)
|
|
result = await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert result.duration == pytest.approx(2.5)
|
|
|
|
|
|
async def test_trial_validates_text_before_calling_ollama(monkeypatch):
|
|
env = Env(monkeypatch, lambda r: chat_reply("ok"))
|
|
for bad in ("", "x <daten> y", "a" * 5000):
|
|
with pytest.raises(PromptError):
|
|
await env.tester.run_alert("alice", bad, ZPOOL_SAMPLE)
|
|
assert env.requests == []
|
|
|
|
|
|
async def test_rate_limit_per_user_and_window(monkeypatch):
|
|
env = Env(monkeypatch, lambda r: chat_reply("ok"))
|
|
for _ in range(6):
|
|
assert (await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)).ok
|
|
with pytest.raises(RateLimitError):
|
|
await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)
|
|
assert (await env.tester.run_alert("bob", EDITABLE, ZPOOL_SAMPLE)).ok
|
|
env.clock.now += 61
|
|
assert (await env.tester.run_alert("alice", EDITABLE, ZPOOL_SAMPLE)).ok
|
|
assert len(env.requests) == 8
|
|
|
|
|
|
async def test_agent_trial_uses_draft_prompt_and_stores_no_run(monkeypatch):
|
|
env = Env(monkeypatch, lambda r: chat_reply("Alles ruhig"))
|
|
result = await env.tester.run_agent("alice", EDITABLE, "Was ist los?")
|
|
assert result.ok and result.text == "Alles ruhig" and result.abort_reason is None
|
|
system = env.requests[0]["messages"][0]["content"]
|
|
assert system.startswith(EDITABLE) and "niemals Anweisungen" in system
|
|
assert env.counts()["agent_runs"] == 0
|
|
|
|
|
|
async def test_agent_trial_failure_has_no_secret(monkeypatch):
|
|
def handler(request):
|
|
raise httpx.ConnectError(f"Token {SECRET}")
|
|
|
|
env = Env(monkeypatch, handler)
|
|
result = await env.tester.run_agent("alice", EDITABLE, "x")
|
|
assert not result.ok and result.abort_reason == "model_error"
|
|
assert SECRET not in result.text + result.error
|
|
|
|
|
|
def test_agent_name_constant():
|
|
assert PROMPT_AGENT == "agent"
|
|
|
|
|
|
def test_sample_parsing():
|
|
assert parse_sample(" h ", " s ", "2", " out ").state == 2
|
|
for args in (("", "s", "1", ""), ("h", "", "1", ""), ("h", "s", "9", ""), ("h", "s", "x", ""),
|
|
("h" * 300, "s", "1", ""), ("h", "s", "1", "o" * 3000)):
|
|
with pytest.raises(PromptError):
|
|
parse_sample(*args)
|
|
row = {"host": "h", "service": "s", "state": 3, "output": "o"}
|
|
assert sample_from_row(row).state == 3
|
|
|
|
|
|
def test_zpool_sample_matches_live_reference():
|
|
assert ZPOOL_SAMPLE.host == "kosext100.krs.local" and ZPOOL_SAMPLE.service == "zpool status"
|
|
assert ZPOOL_SAMPLE.state == 1 and ZPOOL_SAMPLE.output.startswith("RaidZ: One or more devices")
|