archivmail/cmd/archivmail/cmd_reindex.go

package main

import (
	"context"
	"flag"
	"log/slog"
	"os"
	"strings"

	"archivmail/config"
	"archivmail/internal/index"
	"archivmail/internal/storage"
	"archivmail/pkg/mailparser"
)

// runReindex re-indexes all (or tenant-specific) emails into the configured index backend.
// Usage: archivmail reindex [-config /path/to/config.yml] [-tenant <id>]
func runReindex(args []string) {
	fs := flag.NewFlagSet("reindex", flag.ExitOnError)
	configPath := fs.String("config", "/etc/archivmail/config.yml", "path to config file")
	tenantIDFlag := fs.Int64("tenant", 0, "tenant ID to reindex (0 = all tenants)")
	fs.Parse(args)

	logger := slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))

	cfg, err := config.Load(*configPath)
	if err != nil {
		logger.Error("failed to load config", "err", err)
		os.Exit(1)
	}

	storeCfg := storage.Config{
		Dir:             cfg.Storage.StorePath,
		Keyfile:         cfg.Storage.Keyfile,
		DSN:             cfg.Database.DSN(),
		CompressEnabled: cfg.Storage.Compress,
	}
	mailStore, err := storage.New(storeCfg)
	if err != nil {
		logger.Error("storage init failed", "err", err)
		os.Exit(1)
	}
	defer mailStore.Close()

	indexBackend := cfg.Index.Backend
	if indexBackend == "" {
		indexBackend = "manticore"
	}
	batchSize := cfg.Index.BatchSize
	if batchSize <= 0 {
		batchSize = 100
	}

	var idxMgr index.TenantIndexer
	if indexBackend == "manticore" {
		dsn := cfg.Index.ManticoreDSN
		if dsn == "" {
			dsn = "manticore@tcp(127.0.0.1:9306)/?charset=utf8mb4"
		}
		m, err := index.NewManticoreTenantManager(dsn)
		if err != nil {
			logger.Error("manticore init failed", "err", err)
			os.Exit(1)
		}
		idxMgr = m
	} else {
		m, err := index.NewTenantIndexManager(cfg.Index.Path, batchSize, indexBackend)
		if err != nil {
			logger.Error("index manager init failed", "err", err)
			os.Exit(1)
		}
		idxMgr = m
	}
	defer func() { idxMgr.Close() }()

	ctx := context.Background()

	var ids []string
	if *tenantIDFlag > 0 {
		tid := *tenantIDFlag
		ids, err = mailStore.GetAllIDsByTenant(ctx, &tid)
	} else {
		ids, err = mailStore.GetAllIDs(ctx)
	}
	if err != nil {
		logger.Error("failed to list mail IDs", "err", err)
		os.Exit(1)
	}

	logger.Info("reindex: starting", "backend", indexBackend, "total", len(ids))

	indexed := 0
	errors := 0
	for i, id := range ids {
		raw, err := mailStore.Load(id)
		if err != nil {
			logger.Warn("reindex: load failed", "id", id, "err", err)
			errors++
			continue
		}

		pm, err := mailparser.Parse(raw)
		if err != nil {
			logger.Warn("reindex: parse failed", "id", id, "err", err)
			errors++
			continue
		}

		// email_refs carries tenants that only hold a dedup reference; those
		// mails have no emails.tenant_id but must still land in their tenant
		// index, or they vanish for tenant users after a reindex.
		tenantIDs, err := mailStore.TenantsForMail(ctx, id)
		if err != nil {
			logger.Warn("reindex: tenant lookup failed", "id", id, "err", err)
			errors++
			continue
		}
		var tenantID *int64
		if len(tenantIDs) > 0 {
			tenantID = &tenantIDs[0]
		}
		receivedAt, _ := mailStore.GetReceivedAt(ctx, id)

		var attachNames []string
		for _, a := range pm.Attachments {
			if a.Filename != "" {
				attachNames = append(attachNames, a.Filename)
			}
		}

		doc := index.MailDocument{
			ID:            id,
			From:          pm.From,
			To:            strings.Join(pm.To, ", "),
			CC:            strings.Join(pm.CC, ", "),
			BCC:           strings.Join(pm.BCC, ", "),
			Rcpt:          mailStore.DeliveryRcpt(ctx, id, raw, pm),
			Subject:       pm.Subject,
			Body:          pm.TextBody,
			AttachNames:   strings.Join(attachNames, " "),
			HasAttachment: len(pm.Attachments) > 0,
			Date:          index.EffectiveDate(pm.Date, receivedAt),
			Size:          int64(len(raw)),
			TenantID:      tenantID,
		}

		idx := idxMgr.ForTenant(tenantID)

		// Preserve existing OCR text — IndexSync uses REPLACE which would
		// otherwise wipe attachment_text written by the OCR worker.
		// Only read from the index when the DB confirms OCR has run (ocr_chars>0).
		_, ocrChars, _ := mailStore.GetOCRMeta(ctx, id)
		if ocrChars > 0 {
			if reader, ok := idx.(index.AttachmentTextReader); ok {
				if existing, err := reader.GetAttachmentText(id); err == nil && existing != "" {
					doc.AttachmentText = existing
				}
			}
		}

		if err := idx.IndexSync(doc); err != nil {
			logger.Warn("reindex: index failed", "id", id, "err", err)
			errors++
			continue
		}
		for t := 1; t < len(tenantIDs); t++ {
			extra := tenantIDs[t]
			if err := idxMgr.ForTenant(&extra).IndexSync(doc); err != nil {
				logger.Warn("reindex: tenant index failed", "id", id, "tenant_id", extra, "err", err)
			}
		}
		// Superadmin/global search reads emails_global — mirror every
		// tenant-scoped mail there too, not just untenanted ones.
		if tenantID != nil {
			if err := idxMgr.Global().IndexSync(doc); err != nil {
				logger.Warn("reindex: global index failed", "id", id, "err", err)
			}
		}
		indexed++

		if (i+1)%100 == 0 {
			logger.Info("reindex: progress", "processed", i+1, "indexed", indexed, "errors", errors)
		}
	}

	logger.Info("reindex: complete", "total", len(ids), "indexed", indexed, "errors", errors)
}