33 lines
989 B
Python
33 lines
989 B
Python
import re
|
||
import unicodedata
|
||
|
||
from .constants import NOTE_MAX, TEXT_MAX
|
||
|
||
CONTROL_RE = re.compile("[\x00-\x08\x0b-\x1f\x7f-\x9f-
--]")
|
||
DELIMITER_RE = re.compile(r"<\s*/?\s*daten\s*>", re.IGNORECASE)
|
||
WHITESPACE_RE = re.compile(r"\s+")
|
||
|
||
|
||
class PromptError(Exception):
|
||
pass
|
||
|
||
|
||
def _normalize(text: object) -> str:
|
||
text = unicodedata.normalize("NFKC", str(text)).replace("\r\n", "\n").replace("\r", "\n")
|
||
return CONTROL_RE.sub("", text).strip()
|
||
|
||
|
||
def clean_text(text: object) -> str:
|
||
value = _normalize(text)
|
||
if not value:
|
||
raise PromptError("Der Prompt darf nicht leer sein.")
|
||
if len(value) > TEXT_MAX:
|
||
raise PromptError(f"Der Prompt ist zu lang (höchstens {TEXT_MAX} Zeichen).")
|
||
if DELIMITER_RE.search(value):
|
||
raise PromptError("Die Begrenzer <daten> und </daten> sind im Prompt nicht erlaubt.")
|
||
return value
|
||
|
||
|
||
def clean_note(note: object) -> str:
|
||
return WHITESPACE_RE.sub(" ", _normalize(note or ""))[:NOTE_MAX]
|