docs: vollständige README, PROJ-2 Web-Upload, PROJ-19 Mailpiler-Migration

README.md: - Vollständige Dokumentation aller implementierten Funktionen - Konfigurationsreferenz, Installation, Systemd, REST-API-Übersicht - In-Progress-Features klar gekennzeichnet PROJ-2 (EML/MBOX Web-Upload): - POST /api/admin/upload – Multipart-Upload mit Hintergrund-Job - GET /api/admin/upload/{jobID}/progress – Polling - Admin-Tab "Import" mit Drag-and-Drop, Fortschrittsbalken, Abschlussbericht PROJ-19 (Mailpiler Migration): - archivmail import-piler mit Methoden: pilerexport | direct | auto - Direct: AES-256-CBC + zlib mit defensiven Fallbacks - pilerexport: Wrapper um mailpilers Export-Tool Status-Updates: PROJ-3, PROJ-4, PROJ-6, PROJ-7, PROJ-10, PROJ-11 → Deployed Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 09:23:34 +01:00
parent 31de5ec99c
commit 7c29ee88bd
16 changed files with 1726 additions and 91 deletions
@@ -0,0 +1,426 @@
+package main
+
+import (
+	"bytes"
+	"compress/zlib"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/archivmail/config"
+	"github.com/archivmail/internal/index"
+	"github.com/archivmail/internal/storage"
+)
+
+type pilerImportResult struct {
+	Status      string  `json:"status"`
+	Method      string  `json:"method"`
+	Imported    int     `json:"imported"`
+	Skipped     int     `json:"skipped"`
+	Errors      int     `json:"errors"`
+	DurationSec float64 `json:"duration_sec"`
+}
+
+func runImportPiler(args []string) {
+	fs := flag.NewFlagSet("import-piler", flag.ExitOnError)
+	configPath   := fs.String("config",      "/etc/archivmail/config.yml", "archivmail config path")
+	method       := fs.String("method",      "auto",                       "import method: auto | pilerexport | direct")
+	pilerexpBin  := fs.String("pilerexport", "",                           "path to pilerexport binary (auto-detect)")
+	exportDir    := fs.String("export-dir",  "",                           "output dir for pilerexport (temp dir if empty)")
+	storeDir     := fs.String("store-dir",   "/var/piler/store",           "mailpiler store directory (direct method)")
+	keyFile      := fs.String("key-file",    "/var/piler/store/piler.key", "mailpiler AES key file (direct method)")
+	dateFrom     := fs.String("date-from",   "",                           "export from date YYYY-MM-DD (pilerexport method)")
+	dateTo       := fs.String("date-to",     "",                           "export to date YYYY-MM-DD (pilerexport method)")
+	dryRun       := fs.Bool("dry-run",       false,                        "simulate without saving")
+	jsonOut      := fs.Bool("json",          false,                        "machine-readable JSON output")
+	fs.Usage = func() {
+		fmt.Fprintln(os.Stderr, "Usage: archivmail import-piler [flags]")
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "Flags:")
+		fs.PrintDefaults()
+		fmt.Fprintln(os.Stderr, "")
+		fmt.Fprintln(os.Stderr, "Methods:")
+		fmt.Fprintln(os.Stderr, "  pilerexport  Calls pilerexport, imports resulting EML files (recommended)")
+		fmt.Fprintln(os.Stderr, "  direct       Reads .m files from mailpiler store, decrypts+decompresses")
+		fmt.Fprintln(os.Stderr, "  auto         Tries pilerexport first, falls back to direct")
+	}
+	fs.Parse(args)
+
+	start := time.Now()
+
+	cfg, err := config.Load(*configPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: load config: %v\n", err)
+		os.Exit(1)
+	}
+
+	storeCfg := storage.Config{
+		Dir:     cfg.Storage.StorePath,
+		Keyfile: cfg.Storage.Keyfile,
+		DSN:     cfg.Database.DSN(),
+	}
+	mailStore, err := storage.New(storeCfg)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: storage init: %v\n", err)
+		os.Exit(1)
+	}
+	defer mailStore.Close()
+
+	batchSize := cfg.Index.BatchSize
+	if batchSize <= 0 {
+		batchSize = 100
+	}
+	backend := cfg.Index.Backend
+	if backend == "" {
+		backend = "xapian"
+	}
+	idx, err := index.New(cfg.Index.Path, batchSize, backend)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: index init: %v\n", err)
+		os.Exit(1)
+	}
+	defer idx.Close()
+
+	// Resolve method
+	resolvedMethod := *method
+	if resolvedMethod == "auto" {
+		if bin := resolvePilerexport(*pilerexpBin); bin != "" {
+			resolvedMethod = "pilerexport"
+		} else {
+			resolvedMethod = "direct"
+		}
+	}
+
+	if !*jsonOut {
+		fmt.Printf("Mailpiler → archivmail Migration\n")
+		fmt.Printf("Methode: %s\n\n", resolvedMethod)
+	}
+
+	var imported, skipped, errors int
+
+	switch resolvedMethod {
+	case "pilerexport":
+		imported, skipped, errors = runPilerexportMethod(
+			mailStore, idx,
+			*pilerexpBin, *exportDir, *dateFrom, *dateTo,
+			*dryRun, *jsonOut,
+		)
+	case "direct":
+		imported, skipped, errors = runDirectMethod(
+			mailStore, idx,
+			*storeDir, *keyFile,
+			*dryRun, *jsonOut,
+		)
+	default:
+		fmt.Fprintf(os.Stderr, "error: unknown method %q (use: auto, pilerexport, direct)\n", resolvedMethod)
+		os.Exit(1)
+	}
+
+	result := pilerImportResult{
+		Status:      "done",
+		Method:      resolvedMethod,
+		Imported:    imported,
+		Skipped:     skipped,
+		Errors:      errors,
+		DurationSec: time.Since(start).Seconds(),
+	}
+
+	if *jsonOut {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		enc.Encode(result)
+	} else {
+		fmt.Printf("\n╔══════════════════════════════════════╗\n")
+		fmt.Printf("║  Migration abgeschlossen             ║\n")
+		fmt.Printf("╠══════════════════════════════════════╣\n")
+		fmt.Printf("║  Importiert:   %-22d║\n", imported)
+		fmt.Printf("║  Übersprungen: %-22d║\n", skipped)
+		fmt.Printf("║  Fehler:       %-22d║\n", errors)
+		fmt.Printf("║  Dauer:        %-19.1fs║\n", result.DurationSec)
+		if *dryRun {
+			fmt.Printf("║  [dry-run] Keine Daten gespeichert   ║\n")
+		}
+		fmt.Printf("╚══════════════════════════════════════╝\n")
+	}
+
+	if errors > 0 {
+		os.Exit(1)
+	}
+}
+
+// ── pilerexport method ────────────────────────────────────────────────────
+
+func runPilerexportMethod(mailStore *storage.Store, idx index.Indexer, binPath, exportDir, dateFrom, dateTo string, dryRun, jsonOut bool) (imported, skipped, errors int) {
+	bin := resolvePilerexport(binPath)
+	if bin == "" {
+		fmt.Fprintln(os.Stderr, "error: pilerexport binary not found")
+		fmt.Fprintln(os.Stderr, "  Install mailpiler tools or use --method direct")
+		os.Exit(1)
+	}
+
+	// Create temp dir if needed
+	cleanupDir := false
+	if exportDir == "" {
+		tmp, err := os.MkdirTemp("", "archivmail-piler-*")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error: create temp dir: %v\n", err)
+			os.Exit(1)
+		}
+		exportDir = tmp
+		cleanupDir = true
+	} else {
+		if err := os.MkdirAll(exportDir, 0750); err != nil {
+			fmt.Fprintf(os.Stderr, "error: create export dir %s: %v\n", exportDir, err)
+			os.Exit(1)
+		}
+	}
+
+	if !jsonOut {
+		fmt.Printf("Exportiere aus mailpiler nach %s ...\n", exportDir)
+	}
+
+	// Build pilerexport command
+	cmdArgs := []string{"-D", exportDir}
+	if dateFrom != "" {
+		cmdArgs = append(cmdArgs, "-f", dateFrom)
+	}
+	if dateTo != "" {
+		cmdArgs = append(cmdArgs, "-t", dateTo)
+	}
+
+	cmd := exec.Command(bin, cmdArgs...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: pilerexport failed: %v\n", err)
+		fmt.Fprintln(os.Stderr, "  Ensure pilerexport has access to mailpiler database and store.")
+		os.Exit(1)
+	}
+
+	if !jsonOut {
+		fmt.Println("pilerexport abgeschlossen. Importiere EML-Dateien...")
+	}
+
+	// Import all EML files from the export directory
+	total := 0
+	filepath.WalkDir(exportDir, func(path string, d os.DirEntry, werr error) error {
+		if werr != nil || d.IsDir() {
+			return nil
+		}
+		lower := strings.ToLower(d.Name())
+		if !strings.HasSuffix(lower, ".eml") && !strings.HasSuffix(lower, ".m") {
+			return nil
+		}
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			errors++
+			return nil
+		}
+		total++
+		result := importMessage(mailStore, idx, raw, dryRun)
+		switch result {
+		case "imported":
+			imported++
+		case "skipped":
+			skipped++
+		case "error":
+			errors++
+		}
+		if !jsonOut && total%100 == 0 {
+			fmt.Printf("  Fortschritt: %d (importiert: %d, übersprungen: %d, fehler: %d)\n",
+				total, imported, skipped, errors)
+		}
+		return nil
+	})
+
+	if cleanupDir {
+		os.RemoveAll(exportDir)
+	}
+	return
+}
+
+// ── direct method ─────────────────────────────────────────────────────────
+
+func runDirectMethod(mailStore *storage.Store, idx index.Indexer, storeDir, keyFilePath string, dryRun, jsonOut bool) (imported, skipped, errors int) {
+	if _, err := os.Stat(storeDir); os.IsNotExist(err) {
+		fmt.Fprintf(os.Stderr, "error: store dir not found: %s\n", storeDir)
+		os.Exit(1)
+	}
+
+	// Try to read AES key
+	var aesKey []byte
+	if data, err := os.ReadFile(keyFilePath); err == nil && len(data) >= 32 {
+		aesKey = data[:32]
+		if !jsonOut {
+			fmt.Printf("AES-Schlüssel geladen: %s\n", keyFilePath)
+		}
+	} else {
+		if !jsonOut {
+			fmt.Printf("Hinweis: Kein AES-Schlüssel geladen (%s) – versuche unkomprimiert/unkryptiert\n", keyFilePath)
+		}
+	}
+
+	if !jsonOut {
+		fmt.Printf("Lese .m Dateien aus %s ...\n", storeDir)
+	}
+
+	total := 0
+	filepath.WalkDir(storeDir, func(path string, d os.DirEntry, werr error) error {
+		if werr != nil || d.IsDir() {
+			return nil
+		}
+		if !strings.HasSuffix(d.Name(), ".m") {
+			return nil
+		}
+
+		raw, err := os.ReadFile(path)
+		if err != nil {
+			errors++
+			return nil
+		}
+
+		// Decode the piler file format
+		emailRaw, err := decodePilerFile(raw, aesKey)
+		if err != nil || len(emailRaw) == 0 {
+			errors++
+			return nil
+		}
+
+		total++
+		result := importMessage(mailStore, idx, emailRaw, dryRun)
+		switch result {
+		case "imported":
+			imported++
+		case "skipped":
+			skipped++
+		case "error":
+			errors++
+		}
+		if !jsonOut && total%100 == 0 {
+			fmt.Printf("  Fortschritt: %d (importiert: %d, übersprungen: %d, fehler: %d)\n",
+				total, imported, skipped, errors)
+		}
+		return nil
+	})
+
+	return
+}
+
+// decodePilerFile attempts to decode a mailpiler .m file.
+// mailpiler stores files as: [16-byte IV][AES-256-CBC encrypted zlib data]
+// If no key is provided, zlib decompression alone is attempted.
+func decodePilerFile(data []byte, aesKey []byte) ([]byte, error) {
+	// If key is available, try AES-256-CBC decrypt first, then zlib decompress
+	if len(aesKey) == 32 && len(data) > 16 {
+		decrypted, err := aes256CBCDecrypt(data, aesKey)
+		if err == nil {
+			if decompressed, err := zlibDecompress(decrypted); err == nil {
+				return decompressed, nil
+			}
+			// Maybe the decrypted data is already a raw email (no zlib)
+			if looksLikeEmail(decrypted) {
+				return decrypted, nil
+			}
+		}
+	}
+
+	// Try zlib decompression at various offsets (no encryption, or wrong key)
+	for _, skip := range []int{0, 4, 8, 12, 16} {
+		if skip >= len(data) {
+			break
+		}
+		if out, err := zlibDecompress(data[skip:]); err == nil && looksLikeEmail(out) {
+			return out, nil
+		}
+	}
+
+	// Try raw (uncompressed, unencrypted)
+	if looksLikeEmail(data) {
+		return data, nil
+	}
+
+	return nil, fmt.Errorf("could not decode piler file")
+}
+
+func aes256CBCDecrypt(data, key []byte) ([]byte, error) {
+	if len(data) < aes.BlockSize {
+		return nil, fmt.Errorf("data too short")
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	iv := data[:aes.BlockSize]
+	ciphertext := data[aes.BlockSize:]
+	if len(ciphertext)%aes.BlockSize != 0 {
+		return nil, fmt.Errorf("ciphertext not aligned to block size")
+	}
+	plaintext := make([]byte, len(ciphertext))
+	cipher.NewCBCDecrypter(block, iv).CryptBlocks(plaintext, ciphertext)
+	// Remove PKCS7 padding
+	if len(plaintext) == 0 {
+		return nil, fmt.Errorf("empty plaintext")
+	}
+	pad := int(plaintext[len(plaintext)-1])
+	if pad == 0 || pad > aes.BlockSize {
+		return nil, fmt.Errorf("invalid PKCS7 padding")
+	}
+	return plaintext[:len(plaintext)-pad], nil
+}
+
+func zlibDecompress(data []byte) ([]byte, error) {
+	r, err := zlib.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	out, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// looksLikeEmail returns true if the data resembles an RFC 2822 email.
+func looksLikeEmail(data []byte) bool {
+	if len(data) < 10 {
+		return false
+	}
+	header := strings.ToLower(string(data[:min(512, len(data))]))
+	return strings.Contains(header, "from:") ||
+		strings.Contains(header, "date:") ||
+		strings.Contains(header, "message-id:") ||
+		strings.Contains(header, "subject:")
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// resolvePilerexport returns the path to the pilerexport binary, or "" if not found.
+func resolvePilerexport(hint string) string {
+	candidates := []string{hint, "pilerexport", "/usr/sbin/pilerexport", "/usr/local/sbin/pilerexport", "/opt/piler/bin/pilerexport"}
+	for _, c := range candidates {
+		if c == "" {
+			continue
+		}
+		if path, err := exec.LookPath(c); err == nil {
+			return path
+		}
+		if _, err := os.Stat(c); err == nil {
+			return c
+		}
+	}
+	return ""
+}