nexarch/dms/internal/plaintext/plaintext.go

package plaintext

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"mime"
	"strings"
	"unicode/utf16"
	"unicode/utf8"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"

	dmscrypto "gitea.perlbach24.de/scripte/nexarch/dms/internal/crypto"
	"gitea.perlbach24.de/scripte/nexarch/dms/internal/storage"
)

const (
	DefaultMaxBytes = 5 << 20
	DefaultMaxChars = 1_000_000

	StatusIndexed    = "indexed"
	StatusTruncated  = "truncated"
	StatusTooLarge   = "too_large"
	StatusUnreadable = "unreadable"
)

var supportedTypes = map[string]bool{"text/plain": true, "text/csv": true, "text/markdown": true}

// Supported meldet, ob der MIME-Typ (mit oder ohne Parameter) als Klartext gelesen wird.
func Supported(mimeType string) bool {
	mt, _, err := mime.ParseMediaType(mimeType)
	return err == nil && supportedTypes[strings.ToLower(mt)]
}

// TextStore liefert den Klartext der aktuellen Revision fuer den Suchindex.
// Der Text wird je Revision einmal extrahiert und samt Status abgelegt.
type TextStore struct {
	Pool       *pgxpool.Pool
	Storage    *storage.Service
	Crypto     *dmscrypto.Service
	TenantSlug string
	MaxBytes   int64
	MaxChars   int
}

func (s TextStore) limits() (int64, int) {
	maxBytes, maxChars := s.MaxBytes, s.MaxChars
	if maxBytes <= 0 {
		maxBytes = DefaultMaxBytes
	}
	if maxChars <= 0 {
		maxChars = DefaultMaxChars
	}
	return maxBytes, maxChars
}

func (s TextStore) TextOf(ctx context.Context, documentID string) (string, error) {
	var revisionID, mimeType, storageKey string
	var sizeBytes int64
	var wrappedDEK []byte
	err := s.Pool.QueryRow(ctx, `
		SELECT r.id, r.mime_type, r.storage_key, r.size_bytes, r.wrapped_dek
		FROM documents d JOIN file_revisions r ON r.id = d.current_revision_id
		WHERE d.id = $1
	`, documentID).Scan(&revisionID, &mimeType, &storageKey, &sizeBytes, &wrappedDEK)
	if errors.Is(err, pgx.ErrNoRows) {
		return "", nil
	}
	if err != nil {
		return "", fmt.Errorf("plaintext: revision lesen: %w", err)
	}
	if !Supported(mimeType) {
		return "", nil
	}

	var text string
	err = s.Pool.QueryRow(ctx, `SELECT text FROM revision_text_index WHERE revision_id = $1`, revisionID).Scan(&text)
	if err == nil {
		return text, nil
	}
	if !errors.Is(err, pgx.ErrNoRows) {
		return "", fmt.Errorf("plaintext: ablage lesen: %w", err)
	}

	maxBytes, maxChars := s.limits()
	if sizeBytes > maxBytes {
		return "", s.store(ctx, revisionID, StatusTooLarge, fmt.Sprintf("datei %d bytes groesser als limit %d", sizeBytes, maxBytes), "", sizeBytes, "")
	}
	data, err := s.read(ctx, storageKey, wrappedDEK, maxBytes)
	if err != nil {
		return "", err
	}
	if int64(len(data)) > maxBytes {
		return "", s.store(ctx, revisionID, StatusTooLarge, fmt.Sprintf("datei groesser als limit %d", maxBytes), "", int64(len(data)), "")
	}
	text, charset, ok := Decode(data)
	if !ok {
		return "", s.store(ctx, revisionID, StatusUnreadable, "inhalt ist kein lesbarer text (binaere daten)", charset, sizeBytes, "")
	}
	status, reason := StatusIndexed, ""
	if runes := []rune(text); len(runes) > maxChars {
		text = string(runes[:maxChars])
		status, reason = StatusTruncated, fmt.Sprintf("text auf %d zeichen begrenzt", maxChars)
	}
	return text, s.store(ctx, revisionID, status, reason, charset, sizeBytes, text)
}

func (s TextStore) read(ctx context.Context, storageKey string, wrappedDEK []byte, maxBytes int64) ([]byte, error) {
	cipherReader, err := s.Storage.Get(ctx, storageKey)
	if err != nil {
		return nil, fmt.Errorf("plaintext: original laden: %w", err)
	}
	defer func() { _ = cipherReader.Close() }()
	plain, err := s.Crypto.Open(ctx, s.TenantSlug, wrappedDEK, cipherReader)
	if err != nil {
		return nil, fmt.Errorf("plaintext: original entschluesseln: %w", err)
	}
	data, err := io.ReadAll(io.LimitReader(plain, maxBytes+1))
	if err != nil {
		return nil, fmt.Errorf("plaintext: original lesen: %w", err)
	}
	return data, nil
}

func (s TextStore) store(ctx context.Context, revisionID, status, reason, charset string, sizeBytes int64, text string) error {
	_, err := s.Pool.Exec(ctx, `
		INSERT INTO revision_text_index (revision_id, status, reason, charset, source_bytes, indexed_chars, text)
		VALUES ($1, $2, NULLIF($3, ''), NULLIF($4, ''), $5, $6, $7)
		ON CONFLICT (revision_id) DO NOTHING
	`, revisionID, status, reason, charset, sizeBytes, utf8.RuneCountInString(text), text)
	if err != nil {
		return fmt.Errorf("plaintext: ablage schreiben: %w", err)
	}
	return nil
}

// Decode liest Klartext robust: UTF-8 (mit oder ohne BOM), UTF-16 mit BOM,
// sonst Windows-1252 (Obermenge von ISO-8859-1). Daten mit NUL-Bytes gelten als binaer (ok=false).
func Decode(data []byte) (text, charset string, ok bool) {
	switch {
	case bytes.HasPrefix(data, []byte{0xEF, 0xBB, 0xBF}):
		data, charset = data[3:], "utf-8"
	case bytes.HasPrefix(data, []byte{0xFF, 0xFE}) || bytes.HasPrefix(data, []byte{0xFE, 0xFF}):
		return decodeUTF16(data)
	}
	if bytes.IndexByte(data, 0) >= 0 {
		return "", "", false
	}
	if charset == "" {
		charset = "utf-8"
	}
	if !utf8.Valid(data) {
		runes := make([]rune, len(data))
		for i, b := range data {
			runes[i] = rune(b)
		}
		for i, r := range runes {
			if r >= 0x80 && r <= 0x9F {
				if m := cp1252[r-0x80]; m != 0 {
					runes[i] = m
				}
			}
		}
		return clean(string(runes)), "windows-1252", true
	}
	return clean(string(data)), charset, true
}

var cp1252 = [32]rune{
	0x20AC, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0x017D, 0,
	0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0x017E, 0x0178,
}

func decodeUTF16(data []byte) (string, string, bool) {
	littleEndian := data[0] == 0xFF
	data = data[2:]
	if len(data)%2 != 0 {
		data = data[:len(data)-1]
	}
	units := make([]uint16, len(data)/2)
	for i := range units {
		if littleEndian {
			units[i] = uint16(data[2*i]) | uint16(data[2*i+1])<<8
		} else {
			units[i] = uint16(data[2*i])<<8 | uint16(data[2*i+1])
		}
	}
	return clean(string(utf16.Decode(units))), "utf-16", true
}

func clean(s string) string {
	return strings.Map(func(r rune) rune {
		if r < 0x20 && r != '\n' && r != '\r' && r != '\t' {
			return ' '
		}
		return r
	}, s)
}