docs: vollständige README, PROJ-2 Web-Upload, PROJ-19 Mailpiler-Migration
README.md:
- Vollständige Dokumentation aller implementierten Funktionen
- Konfigurationsreferenz, Installation, Systemd, REST-API-Übersicht
- In-Progress-Features klar gekennzeichnet
PROJ-2 (EML/MBOX Web-Upload):
- POST /api/admin/upload – Multipart-Upload mit Hintergrund-Job
- GET /api/admin/upload/{jobID}/progress – Polling
- Admin-Tab "Import" mit Drag-and-Drop, Fortschrittsbalken, Abschlussbericht
PROJ-19 (Mailpiler Migration):
- archivmail import-piler mit Methoden: pilerexport | direct | auto
- Direct: AES-256-CBC + zlib mit defensiven Fallbacks
- pilerexport: Wrapper um mailpilers Export-Tool
Status-Updates: PROJ-3, PROJ-4, PROJ-6, PROJ-7, PROJ-10, PROJ-11 → Deployed
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
sysops
@@ -13,6 +13,7 @@ import (
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -45,6 +46,7 @@ type Server struct {
|
||||
imapStore *imapstore.Store
|
||||
imapImporter *imapstore.Importer
|
||||
imapScheduler *imapstore.Scheduler
|
||||
uploadJobs sync.Map // jobID → *UploadJob
|
||||
}
|
||||
|
||||
// SetSMTPDaemon wires the SMTP daemon into the API server after construction.
|
||||
@@ -108,6 +110,10 @@ func (s *Server) routes() {
|
||||
s.mux.HandleFunc("GET /api/export/pdf/{id}", s.authMiddleware(s.requireMailAccess(s.handleExportPDF)))
|
||||
s.mux.HandleFunc("POST /api/export/zip", s.authMiddleware(s.requireMailAccess(s.handleExportZIP)))
|
||||
|
||||
// Upload routes (admin only)
|
||||
s.mux.HandleFunc("POST /api/admin/upload", s.authMiddleware(s.requireRole(userstore.RoleAdmin, s.handleUpload)))
|
||||
s.mux.HandleFunc("GET /api/admin/upload/{jobID}/progress", s.authMiddleware(s.requireRole(userstore.RoleAdmin, s.handleUploadProgress)))
|
||||
|
||||
// IMAP routes (accessible to all authenticated users)
|
||||
s.mux.HandleFunc("GET /api/imap", s.authMiddleware(s.handleListImap))
|
||||
s.mux.HandleFunc("POST /api/imap", s.authMiddleware(s.handleCreateImap))
|
||||
|
||||
@@ -0,0 +1,211 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/archivmail/internal/index"
|
||||
"github.com/archivmail/pkg/mailparser"
|
||||
)
|
||||
|
||||
// UploadJob tracks the progress of an EML/MBOX import job.
|
||||
type UploadJob struct {
|
||||
mu sync.Mutex
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"` // "running" | "done" | "error"
|
||||
Total int `json:"total"`
|
||||
Imported int `json:"imported"`
|
||||
Skipped int `json:"skipped"`
|
||||
Errors int `json:"errors"`
|
||||
ErrMsg string `json:"error_msg,omitempty"`
|
||||
}
|
||||
|
||||
func (j *UploadJob) snapshot() uploadJobSnapshot {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return uploadJobSnapshot{
|
||||
ID: j.ID,
|
||||
Status: j.Status,
|
||||
Total: j.Total,
|
||||
Imported: j.Imported,
|
||||
Skipped: j.Skipped,
|
||||
Errors: j.Errors,
|
||||
ErrMsg: j.ErrMsg,
|
||||
}
|
||||
}
|
||||
|
||||
type uploadJobSnapshot struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
Total int `json:"total"`
|
||||
Imported int `json:"imported"`
|
||||
Skipped int `json:"skipped"`
|
||||
Errors int `json:"errors"`
|
||||
ErrMsg string `json:"error_msg,omitempty"`
|
||||
}
|
||||
|
||||
// handleUpload accepts a multipart upload of one or more .eml or .mbox files,
|
||||
// starts a background import job and returns its ID immediately.
|
||||
func (s *Server) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
// 512 MB max total upload
|
||||
if err := r.ParseMultipartForm(512 << 20); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "multipart parse failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
files := r.MultipartForm.File["files"]
|
||||
if len(files) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "no files uploaded")
|
||||
return
|
||||
}
|
||||
|
||||
// Collect all raw messages from uploaded files
|
||||
type rawEntry struct {
|
||||
data []byte
|
||||
isMbox bool
|
||||
}
|
||||
var entries []rawEntry
|
||||
|
||||
for _, fh := range files {
|
||||
f, err := fh.Open()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
buf := make([]byte, fh.Size)
|
||||
f.Read(buf) //nolint
|
||||
f.Close()
|
||||
|
||||
name := strings.ToLower(fh.Filename)
|
||||
isMbox := strings.HasSuffix(name, ".mbox")
|
||||
entries = append(entries, rawEntry{buf, isMbox})
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "no readable files")
|
||||
return
|
||||
}
|
||||
|
||||
// Count total messages upfront
|
||||
var allMessages [][]byte
|
||||
for _, e := range entries {
|
||||
if e.isMbox {
|
||||
msgs := mailparser.SplitMbox(e.data)
|
||||
allMessages = append(allMessages, msgs...)
|
||||
} else {
|
||||
allMessages = append(allMessages, e.data)
|
||||
}
|
||||
}
|
||||
|
||||
jobID := newJobID()
|
||||
job := &UploadJob{
|
||||
ID: jobID,
|
||||
Status: "running",
|
||||
Total: len(allMessages),
|
||||
}
|
||||
s.uploadJobs.Store(jobID, job)
|
||||
|
||||
// Run import in background
|
||||
go s.runUploadJob(job, allMessages)
|
||||
|
||||
writeJSON(w, http.StatusAccepted, map[string]string{"job_id": jobID})
|
||||
}
|
||||
|
||||
// handleUploadProgress returns the current status of an upload job.
|
||||
func (s *Server) handleUploadProgress(w http.ResponseWriter, r *http.Request) {
|
||||
jobID := r.PathValue("jobID")
|
||||
val, ok := s.uploadJobs.Load(jobID)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "job not found")
|
||||
return
|
||||
}
|
||||
job := val.(*UploadJob)
|
||||
writeJSON(w, http.StatusOK, job.snapshot())
|
||||
}
|
||||
|
||||
func (s *Server) runUploadJob(job *UploadJob, messages [][]byte) {
|
||||
ctx := context.Background()
|
||||
|
||||
for _, raw := range messages {
|
||||
result := s.importRawMessage(ctx, raw)
|
||||
job.mu.Lock()
|
||||
switch result {
|
||||
case "imported":
|
||||
job.Imported++
|
||||
case "skipped":
|
||||
job.Skipped++
|
||||
default:
|
||||
job.Errors++
|
||||
}
|
||||
job.mu.Unlock()
|
||||
}
|
||||
|
||||
job.mu.Lock()
|
||||
job.Status = "done"
|
||||
job.mu.Unlock()
|
||||
}
|
||||
|
||||
// importRawMessage stores and indexes a single raw message.
|
||||
// Returns "imported", "skipped", or "error".
|
||||
func (s *Server) importRawMessage(ctx context.Context, raw []byte) string {
|
||||
pm, err := mailparser.Parse(raw)
|
||||
if err != nil {
|
||||
s.logger.Warn("upload: parse failed", "err", err)
|
||||
return "error"
|
||||
}
|
||||
|
||||
id, err := s.store.Save(raw, pm.Date)
|
||||
if err != nil {
|
||||
s.logger.Warn("upload: save failed", "err", err)
|
||||
return "error"
|
||||
}
|
||||
|
||||
// Check dedup: storage.Save returns same id for duplicate content.
|
||||
// If already indexed, skip indexing.
|
||||
if already, _ := s.store.IsIndexed(ctx, id); already {
|
||||
return "skipped"
|
||||
}
|
||||
|
||||
var attachNames []string
|
||||
for _, a := range pm.Attachments {
|
||||
if a.Filename != "" {
|
||||
attachNames = append(attachNames, a.Filename)
|
||||
}
|
||||
}
|
||||
|
||||
doc := index.MailDocument{
|
||||
ID: id,
|
||||
From: pm.From,
|
||||
To: strings.Join(pm.To, " "),
|
||||
Subject: pm.Subject,
|
||||
Body: pm.TextBody + " " + pm.HTMLBody,
|
||||
AttachNames: strings.Join(attachNames, " "),
|
||||
HasAttachment: len(pm.Attachments) > 0,
|
||||
Date: pm.Date,
|
||||
Size: int64(len(raw)),
|
||||
}
|
||||
|
||||
if err := s.idx.IndexSync(doc); err != nil {
|
||||
s.logger.Warn("upload: index failed", "id", id, "err", err)
|
||||
return "error"
|
||||
}
|
||||
|
||||
if err := s.store.SetIndexedAt(ctx, id); err != nil {
|
||||
s.logger.Warn("upload: set indexed_at failed", "id", id, "err", err)
|
||||
}
|
||||
|
||||
if err := s.store.SaveMeta(ctx, id, pm, len(raw)); err != nil {
|
||||
s.logger.Warn("upload: save meta failed", "id", id, "err", err)
|
||||
}
|
||||
|
||||
return "imported"
|
||||
}
|
||||
|
||||
func newJobID() string {
|
||||
b := make([]byte, 8)
|
||||
rand.Read(b) //nolint
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
Reference in New Issue
Block a user