archivmail/cmd/archivmail/cmd_import_piler.go

package main

import (
	"bytes"
	"compress/zlib"
	"crypto/aes"
	"crypto/cipher"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/archivmail/config"
	"github.com/archivmail/internal/index"
	"github.com/archivmail/internal/storage"
)

type pilerImportResult struct {
	Status      string  `json:"status"`
	Method      string  `json:"method"`
	Imported    int     `json:"imported"`
	Skipped     int     `json:"skipped"`
	Errors      int     `json:"errors"`
	DurationSec float64 `json:"duration_sec"`
}

func runImportPiler(args []string) {
	fs := flag.NewFlagSet("import-piler", flag.ExitOnError)
	configPath   := fs.String("config",      "/etc/archivmail/config.yml", "archivmail config path")
	method       := fs.String("method",      "auto",                       "import method: auto | pilerexport | direct")
	pilerexpBin  := fs.String("pilerexport", "",                           "path to pilerexport binary (auto-detect)")
	exportDir    := fs.String("export-dir",  "",                           "output dir for pilerexport (temp dir if empty)")
	storeDir     := fs.String("store-dir",   "/var/piler/store",           "mailpiler store directory (direct method)")
	keyFile      := fs.String("key-file",    "/var/piler/store/piler.key", "mailpiler AES key file (direct method)")
	dateFrom     := fs.String("date-from",   "",                           "export from date YYYY-MM-DD (pilerexport method)")
	dateTo       := fs.String("date-to",     "",                           "export to date YYYY-MM-DD (pilerexport method)")
	dryRun       := fs.Bool("dry-run",       false,                        "simulate without saving")
	jsonOut      := fs.Bool("json",          false,                        "machine-readable JSON output")
	fs.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: archivmail import-piler [flags]")
		fmt.Fprintln(os.Stderr, "")
		fmt.Fprintln(os.Stderr, "Flags:")
		fs.PrintDefaults()
		fmt.Fprintln(os.Stderr, "")
		fmt.Fprintln(os.Stderr, "Methods:")
		fmt.Fprintln(os.Stderr, "  pilerexport  Calls pilerexport, imports resulting EML files (recommended)")
		fmt.Fprintln(os.Stderr, "  direct       Reads .m files from mailpiler store, decrypts+decompresses")
		fmt.Fprintln(os.Stderr, "  auto         Tries pilerexport first, falls back to direct")
	}
	fs.Parse(args)

	start := time.Now()

	cfg, err := config.Load(*configPath)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: load config: %v\n", err)
		os.Exit(1)
	}

	storeCfg := storage.Config{
		Dir:     cfg.Storage.StorePath,
		Keyfile: cfg.Storage.Keyfile,
		DSN:     cfg.Database.DSN(),
	}
	mailStore, err := storage.New(storeCfg)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: storage init: %v\n", err)
		os.Exit(1)
	}
	defer mailStore.Close()

	batchSize := cfg.Index.BatchSize
	if batchSize <= 0 {
		batchSize = 100
	}
	backend := cfg.Index.Backend
	if backend == "" {
		backend = "xapian"
	}
	idx, err := index.New(cfg.Index.Path, batchSize, backend)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: index init: %v\n", err)
		os.Exit(1)
	}
	defer idx.Close()

	// Resolve method
	resolvedMethod := *method
	if resolvedMethod == "auto" {
		if bin := resolvePilerexport(*pilerexpBin); bin != "" {
			resolvedMethod = "pilerexport"
		} else {
			resolvedMethod = "direct"
		}
	}

	if !*jsonOut {
		fmt.Printf("Mailpiler → archivmail Migration\n")
		fmt.Printf("Methode: %s\n\n", resolvedMethod)
	}

	var imported, skipped, errors int

	switch resolvedMethod {
	case "pilerexport":
		imported, skipped, errors = runPilerexportMethod(
			mailStore, idx,
			*pilerexpBin, *exportDir, *dateFrom, *dateTo,
			*dryRun, *jsonOut,
		)
	case "direct":
		imported, skipped, errors = runDirectMethod(
			mailStore, idx,
			*storeDir, *keyFile,
			*dryRun, *jsonOut,
		)
	default:
		fmt.Fprintf(os.Stderr, "error: unknown method %q (use: auto, pilerexport, direct)\n", resolvedMethod)
		os.Exit(1)
	}

	result := pilerImportResult{
		Status:      "done",
		Method:      resolvedMethod,
		Imported:    imported,
		Skipped:     skipped,
		Errors:      errors,
		DurationSec: time.Since(start).Seconds(),
	}

	if *jsonOut {
		enc := json.NewEncoder(os.Stdout)
		enc.SetIndent("", "  ")
		enc.Encode(result)
	} else {
		fmt.Printf("\n╔══════════════════════════════════════╗\n")
		fmt.Printf("║  Migration abgeschlossen             ║\n")
		fmt.Printf("╠══════════════════════════════════════╣\n")
		fmt.Printf("║  Importiert:   %-22d║\n", imported)
		fmt.Printf("║  Übersprungen: %-22d║\n", skipped)
		fmt.Printf("║  Fehler:       %-22d║\n", errors)
		fmt.Printf("║  Dauer:        %-19.1fs║\n", result.DurationSec)
		if *dryRun {
			fmt.Printf("║  [dry-run] Keine Daten gespeichert   ║\n")
		}
		fmt.Printf("╚══════════════════════════════════════╝\n")
	}

	if errors > 0 {
		os.Exit(1)
	}
}

// ── pilerexport method ────────────────────────────────────────────────────

func runPilerexportMethod(mailStore *storage.Store, idx index.Indexer, binPath, exportDir, dateFrom, dateTo string, dryRun, jsonOut bool) (imported, skipped, errors int) {
	bin := resolvePilerexport(binPath)
	if bin == "" {
		fmt.Fprintln(os.Stderr, "error: pilerexport binary not found")
		fmt.Fprintln(os.Stderr, "  Install mailpiler tools or use --method direct")
		os.Exit(1)
	}

	// Create temp dir if needed
	cleanupDir := false
	if exportDir == "" {
		tmp, err := os.MkdirTemp("", "archivmail-piler-*")
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: create temp dir: %v\n", err)
			os.Exit(1)
		}
		exportDir = tmp
		cleanupDir = true
	} else {
		if err := os.MkdirAll(exportDir, 0750); err != nil {
			fmt.Fprintf(os.Stderr, "error: create export dir %s: %v\n", exportDir, err)
			os.Exit(1)
		}
	}

	if !jsonOut {
		fmt.Printf("Exportiere aus mailpiler nach %s ...\n", exportDir)
	}

	// Build pilerexport command
	cmdArgs := []string{"-D", exportDir}
	if dateFrom != "" {
		cmdArgs = append(cmdArgs, "-f", dateFrom)
	}
	if dateTo != "" {
		cmdArgs = append(cmdArgs, "-t", dateTo)
	}

	cmd := exec.Command(bin, cmdArgs...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		fmt.Fprintf(os.Stderr, "error: pilerexport failed: %v\n", err)
		fmt.Fprintln(os.Stderr, "  Ensure pilerexport has access to mailpiler database and store.")
		os.Exit(1)
	}

	if !jsonOut {
		fmt.Println("pilerexport abgeschlossen. Importiere EML-Dateien...")
	}

	// Import all EML files from the export directory
	total := 0
	filepath.WalkDir(exportDir, func(path string, d os.DirEntry, werr error) error {
		if werr != nil || d.IsDir() {
			return nil
		}
		lower := strings.ToLower(d.Name())
		if !strings.HasSuffix(lower, ".eml") && !strings.HasSuffix(lower, ".m") {
			return nil
		}
		raw, err := os.ReadFile(path)
		if err != nil {
			errors++
			return nil
		}
		total++
		result := importMessage(mailStore, idx, raw, dryRun)
		switch result {
		case "imported":
			imported++
		case "skipped":
			skipped++
		case "error":
			errors++
		}
		if !jsonOut && total%100 == 0 {
			fmt.Printf("  Fortschritt: %d (importiert: %d, übersprungen: %d, fehler: %d)\n",
				total, imported, skipped, errors)
		}
		return nil
	})

	if cleanupDir {
		os.RemoveAll(exportDir)
	}
	return
}

// ── direct method ─────────────────────────────────────────────────────────

func runDirectMethod(mailStore *storage.Store, idx index.Indexer, storeDir, keyFilePath string, dryRun, jsonOut bool) (imported, skipped, errors int) {
	if _, err := os.Stat(storeDir); os.IsNotExist(err) {
		fmt.Fprintf(os.Stderr, "error: store dir not found: %s\n", storeDir)
		os.Exit(1)
	}

	// Try to read AES key
	var aesKey []byte
	if data, err := os.ReadFile(keyFilePath); err == nil && len(data) >= 32 {
		aesKey = data[:32]
		if !jsonOut {
			fmt.Printf("AES-Schlüssel geladen: %s\n", keyFilePath)
		}
	} else {
		if !jsonOut {
			fmt.Printf("Hinweis: Kein AES-Schlüssel geladen (%s) – versuche unkomprimiert/unkryptiert\n", keyFilePath)
		}
	}

	if !jsonOut {
		fmt.Printf("Lese .m Dateien aus %s ...\n", storeDir)
	}

	total := 0
	filepath.WalkDir(storeDir, func(path string, d os.DirEntry, werr error) error {
		if werr != nil || d.IsDir() {
			return nil
		}
		if !strings.HasSuffix(d.Name(), ".m") {
			return nil
		}

		raw, err := os.ReadFile(path)
		if err != nil {
			errors++
			return nil
		}

		// Decode the piler file format
		emailRaw, err := decodePilerFile(raw, aesKey)
		if err != nil || len(emailRaw) == 0 {
			errors++
			return nil
		}

		total++
		result := importMessage(mailStore, idx, emailRaw, dryRun)
		switch result {
		case "imported":
			imported++
		case "skipped":
			skipped++
		case "error":
			errors++
		}
		if !jsonOut && total%100 == 0 {
			fmt.Printf("  Fortschritt: %d (importiert: %d, übersprungen: %d, fehler: %d)\n",
				total, imported, skipped, errors)
		}
		return nil
	})

	return
}

// decodePilerFile attempts to decode a mailpiler .m file.
// mailpiler stores files as: [16-byte IV][AES-256-CBC encrypted zlib data]
// If no key is provided, zlib decompression alone is attempted.
func decodePilerFile(data []byte, aesKey []byte) ([]byte, error) {
	// If key is available, try AES-256-CBC decrypt first, then zlib decompress
	if len(aesKey) == 32 && len(data) > 16 {
		decrypted, err := aes256CBCDecrypt(data, aesKey)
		if err == nil {
			if decompressed, err := zlibDecompress(decrypted); err == nil {
				return decompressed, nil
			}
			// Maybe the decrypted data is already a raw email (no zlib)
			if looksLikeEmail(decrypted) {
				return decrypted, nil
			}
		}
	}

	// Try zlib decompression at various offsets (no encryption, or wrong key)
	for _, skip := range []int{0, 4, 8, 12, 16} {
		if skip >= len(data) {
			break
		}
		if out, err := zlibDecompress(data[skip:]); err == nil && looksLikeEmail(out) {
			return out, nil
		}
	}

	// Try raw (uncompressed, unencrypted)
	if looksLikeEmail(data) {
		return data, nil
	}

	return nil, fmt.Errorf("could not decode piler file")
}

func aes256CBCDecrypt(data, key []byte) ([]byte, error) {
	if len(data) < aes.BlockSize {
		return nil, fmt.Errorf("data too short")
	}
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	iv := data[:aes.BlockSize]
	ciphertext := data[aes.BlockSize:]
	if len(ciphertext)%aes.BlockSize != 0 {
		return nil, fmt.Errorf("ciphertext not aligned to block size")
	}
	plaintext := make([]byte, len(ciphertext))
	cipher.NewCBCDecrypter(block, iv).CryptBlocks(plaintext, ciphertext)
	// Remove PKCS7 padding
	if len(plaintext) == 0 {
		return nil, fmt.Errorf("empty plaintext")
	}
	pad := int(plaintext[len(plaintext)-1])
	if pad == 0 || pad > aes.BlockSize {
		return nil, fmt.Errorf("invalid PKCS7 padding")
	}
	return plaintext[:len(plaintext)-pad], nil
}

func zlibDecompress(data []byte) ([]byte, error) {
	r, err := zlib.NewReader(bytes.NewReader(data))
	if err != nil {
		return nil, err
	}
	defer r.Close()
	out, err := io.ReadAll(r)
	if err != nil {
		return nil, err
	}
	return out, nil
}

// looksLikeEmail returns true if the data resembles an RFC 2822 email.
func looksLikeEmail(data []byte) bool {
	if len(data) < 10 {
		return false
	}
	header := strings.ToLower(string(data[:min(512, len(data))]))
	return strings.Contains(header, "from:") ||
		strings.Contains(header, "date:") ||
		strings.Contains(header, "message-id:") ||
		strings.Contains(header, "subject:")
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// resolvePilerexport returns the path to the pilerexport binary, or "" if not found.
func resolvePilerexport(hint string) string {
	candidates := []string{hint, "pilerexport", "/usr/sbin/pilerexport", "/usr/local/sbin/pilerexport", "/opt/piler/bin/pilerexport"}
	for _, c := range candidates {
		if c == "" {
			continue
		}
		if path, err := exec.LookPath(c); err == nil {
			return path
		}
		if _, err := os.Stat(c); err == nil {
			return c
		}
	}
	return ""
}