package main import ( "bytes" "compress/zlib" "crypto/aes" "crypto/cipher" "encoding/json" "flag" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "time" "github.com/archivmail/config" "github.com/archivmail/internal/index" "github.com/archivmail/internal/storage" ) type pilerImportResult struct { Status string `json:"status"` Method string `json:"method"` Imported int `json:"imported"` Skipped int `json:"skipped"` Errors int `json:"errors"` DurationSec float64 `json:"duration_sec"` } func runImportPiler(args []string) { fs := flag.NewFlagSet("import-piler", flag.ExitOnError) configPath := fs.String("config", "/etc/archivmail/config.yml", "archivmail config path") method := fs.String("method", "auto", "import method: auto | pilerexport | direct") pilerexpBin := fs.String("pilerexport", "", "path to pilerexport binary (auto-detect)") exportDir := fs.String("export-dir", "", "output dir for pilerexport (temp dir if empty)") storeDir := fs.String("store-dir", "/var/piler/store", "mailpiler store directory (direct method)") keyFile := fs.String("key-file", "/var/piler/store/piler.key", "mailpiler AES key file (direct method)") dateFrom := fs.String("date-from", "", "export from date YYYY-MM-DD (pilerexport method)") dateTo := fs.String("date-to", "", "export to date YYYY-MM-DD (pilerexport method)") dryRun := fs.Bool("dry-run", false, "simulate without saving") jsonOut := fs.Bool("json", false, "machine-readable JSON output") fs.Usage = func() { fmt.Fprintln(os.Stderr, "Usage: archivmail import-piler [flags]") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "Flags:") fs.PrintDefaults() fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "Methods:") fmt.Fprintln(os.Stderr, " pilerexport Calls pilerexport, imports resulting EML files (recommended)") fmt.Fprintln(os.Stderr, " direct Reads .m files from mailpiler store, decrypts+decompresses") fmt.Fprintln(os.Stderr, " auto Tries pilerexport first, falls back to direct") } fs.Parse(args) start := time.Now() cfg, err := config.Load(*configPath) if err != nil { fmt.Fprintf(os.Stderr, "error: load config: %v\n", err) os.Exit(1) } storeCfg := storage.Config{ Dir: cfg.Storage.StorePath, Keyfile: cfg.Storage.Keyfile, DSN: cfg.Database.DSN(), } mailStore, err := storage.New(storeCfg) if err != nil { fmt.Fprintf(os.Stderr, "error: storage init: %v\n", err) os.Exit(1) } defer mailStore.Close() batchSize := cfg.Index.BatchSize if batchSize <= 0 { batchSize = 100 } backend := cfg.Index.Backend if backend == "" { backend = "xapian" } idx, err := index.New(cfg.Index.Path, batchSize, backend) if err != nil { fmt.Fprintf(os.Stderr, "error: index init: %v\n", err) os.Exit(1) } defer idx.Close() // Resolve method resolvedMethod := *method if resolvedMethod == "auto" { if bin := resolvePilerexport(*pilerexpBin); bin != "" { resolvedMethod = "pilerexport" } else { resolvedMethod = "direct" } } if !*jsonOut { fmt.Printf("Mailpiler → archivmail Migration\n") fmt.Printf("Methode: %s\n\n", resolvedMethod) } var imported, skipped, errors int switch resolvedMethod { case "pilerexport": imported, skipped, errors = runPilerexportMethod( mailStore, idx, *pilerexpBin, *exportDir, *dateFrom, *dateTo, *dryRun, *jsonOut, ) case "direct": imported, skipped, errors = runDirectMethod( mailStore, idx, *storeDir, *keyFile, *dryRun, *jsonOut, ) default: fmt.Fprintf(os.Stderr, "error: unknown method %q (use: auto, pilerexport, direct)\n", resolvedMethod) os.Exit(1) } result := pilerImportResult{ Status: "done", Method: resolvedMethod, Imported: imported, Skipped: skipped, Errors: errors, DurationSec: time.Since(start).Seconds(), } if *jsonOut { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") enc.Encode(result) } else { fmt.Printf("\n╔══════════════════════════════════════╗\n") fmt.Printf("║ Migration abgeschlossen ║\n") fmt.Printf("╠══════════════════════════════════════╣\n") fmt.Printf("║ Importiert: %-22d║\n", imported) fmt.Printf("║ Übersprungen: %-22d║\n", skipped) fmt.Printf("║ Fehler: %-22d║\n", errors) fmt.Printf("║ Dauer: %-19.1fs║\n", result.DurationSec) if *dryRun { fmt.Printf("║ [dry-run] Keine Daten gespeichert ║\n") } fmt.Printf("╚══════════════════════════════════════╝\n") } if errors > 0 { os.Exit(1) } } // ── pilerexport method ──────────────────────────────────────────────────── func runPilerexportMethod(mailStore *storage.Store, idx index.Indexer, binPath, exportDir, dateFrom, dateTo string, dryRun, jsonOut bool) (imported, skipped, errors int) { bin := resolvePilerexport(binPath) if bin == "" { fmt.Fprintln(os.Stderr, "error: pilerexport binary not found") fmt.Fprintln(os.Stderr, " Install mailpiler tools or use --method direct") os.Exit(1) } // Create temp dir if needed cleanupDir := false if exportDir == "" { tmp, err := os.MkdirTemp("", "archivmail-piler-*") if err != nil { fmt.Fprintf(os.Stderr, "error: create temp dir: %v\n", err) os.Exit(1) } exportDir = tmp cleanupDir = true } else { if err := os.MkdirAll(exportDir, 0750); err != nil { fmt.Fprintf(os.Stderr, "error: create export dir %s: %v\n", exportDir, err) os.Exit(1) } } if !jsonOut { fmt.Printf("Exportiere aus mailpiler nach %s ...\n", exportDir) } // Build pilerexport command cmdArgs := []string{"-D", exportDir} if dateFrom != "" { cmdArgs = append(cmdArgs, "-f", dateFrom) } if dateTo != "" { cmdArgs = append(cmdArgs, "-t", dateTo) } cmd := exec.Command(bin, cmdArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { fmt.Fprintf(os.Stderr, "error: pilerexport failed: %v\n", err) fmt.Fprintln(os.Stderr, " Ensure pilerexport has access to mailpiler database and store.") os.Exit(1) } if !jsonOut { fmt.Println("pilerexport abgeschlossen. Importiere EML-Dateien...") } // Import all EML files from the export directory total := 0 filepath.WalkDir(exportDir, func(path string, d os.DirEntry, werr error) error { if werr != nil || d.IsDir() { return nil } lower := strings.ToLower(d.Name()) if !strings.HasSuffix(lower, ".eml") && !strings.HasSuffix(lower, ".m") { return nil } raw, err := os.ReadFile(path) if err != nil { errors++ return nil } total++ result := importMessage(mailStore, idx, raw, dryRun) switch result { case "imported": imported++ case "skipped": skipped++ case "error": errors++ } if !jsonOut && total%100 == 0 { fmt.Printf(" Fortschritt: %d (importiert: %d, übersprungen: %d, fehler: %d)\n", total, imported, skipped, errors) } return nil }) if cleanupDir { os.RemoveAll(exportDir) } return } // ── direct method ───────────────────────────────────────────────────────── func runDirectMethod(mailStore *storage.Store, idx index.Indexer, storeDir, keyFilePath string, dryRun, jsonOut bool) (imported, skipped, errors int) { if _, err := os.Stat(storeDir); os.IsNotExist(err) { fmt.Fprintf(os.Stderr, "error: store dir not found: %s\n", storeDir) os.Exit(1) } // Try to read AES key var aesKey []byte if data, err := os.ReadFile(keyFilePath); err == nil && len(data) >= 32 { aesKey = data[:32] if !jsonOut { fmt.Printf("AES-Schlüssel geladen: %s\n", keyFilePath) } } else { if !jsonOut { fmt.Printf("Hinweis: Kein AES-Schlüssel geladen (%s) – versuche unkomprimiert/unkryptiert\n", keyFilePath) } } if !jsonOut { fmt.Printf("Lese .m Dateien aus %s ...\n", storeDir) } total := 0 filepath.WalkDir(storeDir, func(path string, d os.DirEntry, werr error) error { if werr != nil || d.IsDir() { return nil } if !strings.HasSuffix(d.Name(), ".m") { return nil } raw, err := os.ReadFile(path) if err != nil { errors++ return nil } // Decode the piler file format emailRaw, err := decodePilerFile(raw, aesKey) if err != nil || len(emailRaw) == 0 { errors++ return nil } total++ result := importMessage(mailStore, idx, emailRaw, dryRun) switch result { case "imported": imported++ case "skipped": skipped++ case "error": errors++ } if !jsonOut && total%100 == 0 { fmt.Printf(" Fortschritt: %d (importiert: %d, übersprungen: %d, fehler: %d)\n", total, imported, skipped, errors) } return nil }) return } // decodePilerFile attempts to decode a mailpiler .m file. // mailpiler stores files as: [16-byte IV][AES-256-CBC encrypted zlib data] // If no key is provided, zlib decompression alone is attempted. func decodePilerFile(data []byte, aesKey []byte) ([]byte, error) { // If key is available, try AES-256-CBC decrypt first, then zlib decompress if len(aesKey) == 32 && len(data) > 16 { decrypted, err := aes256CBCDecrypt(data, aesKey) if err == nil { if decompressed, err := zlibDecompress(decrypted); err == nil { return decompressed, nil } // Maybe the decrypted data is already a raw email (no zlib) if looksLikeEmail(decrypted) { return decrypted, nil } } } // Try zlib decompression at various offsets (no encryption, or wrong key) for _, skip := range []int{0, 4, 8, 12, 16} { if skip >= len(data) { break } if out, err := zlibDecompress(data[skip:]); err == nil && looksLikeEmail(out) { return out, nil } } // Try raw (uncompressed, unencrypted) if looksLikeEmail(data) { return data, nil } return nil, fmt.Errorf("could not decode piler file") } func aes256CBCDecrypt(data, key []byte) ([]byte, error) { if len(data) < aes.BlockSize { return nil, fmt.Errorf("data too short") } block, err := aes.NewCipher(key) if err != nil { return nil, err } iv := data[:aes.BlockSize] ciphertext := data[aes.BlockSize:] if len(ciphertext)%aes.BlockSize != 0 { return nil, fmt.Errorf("ciphertext not aligned to block size") } plaintext := make([]byte, len(ciphertext)) cipher.NewCBCDecrypter(block, iv).CryptBlocks(plaintext, ciphertext) // Remove PKCS7 padding if len(plaintext) == 0 { return nil, fmt.Errorf("empty plaintext") } pad := int(plaintext[len(plaintext)-1]) if pad == 0 || pad > aes.BlockSize { return nil, fmt.Errorf("invalid PKCS7 padding") } return plaintext[:len(plaintext)-pad], nil } func zlibDecompress(data []byte) ([]byte, error) { r, err := zlib.NewReader(bytes.NewReader(data)) if err != nil { return nil, err } defer r.Close() out, err := io.ReadAll(r) if err != nil { return nil, err } return out, nil } // looksLikeEmail returns true if the data resembles an RFC 2822 email. func looksLikeEmail(data []byte) bool { if len(data) < 10 { return false } header := strings.ToLower(string(data[:min(512, len(data))])) return strings.Contains(header, "from:") || strings.Contains(header, "date:") || strings.Contains(header, "message-id:") || strings.Contains(header, "subject:") } func min(a, b int) int { if a < b { return a } return b } // resolvePilerexport returns the path to the pilerexport binary, or "" if not found. func resolvePilerexport(hint string) string { candidates := []string{hint, "pilerexport", "/usr/sbin/pilerexport", "/usr/local/sbin/pilerexport", "/opt/piler/bin/pilerexport"} for _, c := range candidates { if c == "" { continue } if path, err := exec.LookPath(c); err == nil { return path } if _, err := os.Stat(c); err == nil { return c } } return "" }