nexarch/connect/internal/dms2import/parse.go

// Package dms2import implements DMS2-02: an assistant that reads exports
// from third-party DMS systems, suggests a field mapping onto NEXARCH's
// interoperability format (DMS2-01), and turns each source document into
// a DMS2-01 bundle. paperless-ngx's exporter format (a JSON manifest of
// Django-fixture-style records plus raw document files) is supported
// first, since paperless-ngx is this project's primary reference system
// (see VERGLEICHSANALYSE-QUELLSYSTEME.md).
package dms2import

import (
	"archive/zip"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strings"
)

// FormatPaperlessNGX identifies the source format a bundle was parsed as.
const FormatPaperlessNGX = "paperless_ngx"

var ErrInvalidExport = errors.New("dms2import: invalid third-party export")

// SourceDocument is one document extracted from a third-party export,
// normalized enough to drive field-mapping suggestion and bundle
// construction, but keeping every raw field string so a NEXARCH field can
// be mapped to any of them, not just the ones this package recognizes by
// name.
type SourceDocument struct {
	PK          string
	Title       string
	ContentType string
	Content     []byte
	RawFields   map[string]string
}

// record is one entry of paperless-ngx's exporter manifest.json: a
// Django-fixture-style {model, pk, fields} object. Only the fields this
// package actually uses are declared; unknown fields are preserved via a
// second decode into a generic map (see decodeFieldsAsStrings).
type record struct {
	Model  string          `json:"model"`
	PK     json.Number     `json:"pk"`
	Fields json.RawMessage `json:"fields"`
}

// ParsePaperlessExport reads a paperless-ngx exporter-style ZIP archive:
// manifest.json at the root (an array of {model,pk,fields} records) plus
// original document files under documents/originals/. Only
// "documents.document" records become SourceDocuments; correspondent,
// document type and tag records are resolved by pk to fill in the
// corresponding fields as plain strings, since NEXARCH has no matching
// foreign-key model to preserve them as.
func ParsePaperlessExport(data []byte) ([]SourceDocument, error) {
	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		return nil, fmt.Errorf("%w: not a valid zip: %v", ErrInvalidExport, err)
	}

	var manifestRaw []byte
	files := make(map[string]*zip.File)
	for _, f := range zr.File {
		if f.Name == "manifest.json" {
			manifestRaw, err = readZipEntry(f)
			if err != nil {
				return nil, fmt.Errorf("%w: reading manifest.json: %v", ErrInvalidExport, err)
			}
			continue
		}
		files[f.Name] = f
	}
	if manifestRaw == nil {
		return nil, fmt.Errorf("%w: missing manifest.json", ErrInvalidExport)
	}

	var records []record
	if err := json.Unmarshal(manifestRaw, &records); err != nil {
		return nil, fmt.Errorf("%w: manifest.json is not a valid record array: %v", ErrInvalidExport, err)
	}

	correspondents := map[string]string{}
	documentTypes := map[string]string{}
	tags := map[string]string{}
	var docRecords []record

	for _, rec := range records {
		switch rec.Model {
		case "documents.correspondent":
			correspondents[rec.PK.String()] = nameField(rec.Fields)
		case "documents.documenttype":
			documentTypes[rec.PK.String()] = nameField(rec.Fields)
		case "documents.tag":
			tags[rec.PK.String()] = nameField(rec.Fields)
		case "documents.document":
			docRecords = append(docRecords, rec)
		}
	}
	if len(docRecords) == 0 {
		return nil, fmt.Errorf("%w: no documents.document records in manifest.json", ErrInvalidExport)
	}

	docs := make([]SourceDocument, 0, len(docRecords))
	for _, rec := range docRecords {
		var raw map[string]json.RawMessage
		if err := json.Unmarshal(rec.Fields, &raw); err != nil {
			return nil, fmt.Errorf("%w: document pk %s has invalid fields: %v", ErrInvalidExport, rec.PK, err)
		}

		fields := map[string]string{}
		fields["title"] = jsonString(raw["title"])
		fields["created"] = jsonString(raw["created"])
		fields["content"] = jsonString(raw["content"])
		fields["archive_serial_number"] = jsonString(raw["archive_serial_number"])
		if corrPK := jsonString(raw["correspondent"]); corrPK != "" {
			fields["correspondent"] = correspondents[corrPK]
		}
		if typePK := jsonString(raw["document_type"]); typePK != "" {
			fields["document_type"] = documentTypes[typePK]
		}
		if tagPKs := jsonNumberArray(raw["tags"]); len(tagPKs) > 0 {
			names := make([]string, 0, len(tagPKs))
			for _, pk := range tagPKs {
				if n, ok := tags[pk]; ok {
					names = append(names, n)
				}
			}
			fields["tags"] = strings.Join(names, ",")
		}

		content, contentType, err := findOriginal(files, rec.PK.String())
		if err != nil {
			return nil, fmt.Errorf("%w: document pk %s: %v", ErrInvalidExport, rec.PK, err)
		}

		docs = append(docs, SourceDocument{
			PK:          rec.PK.String(),
			Title:       fields["title"],
			ContentType: contentType,
			Content:     content,
			RawFields:   fields,
		})
	}
	return docs, nil
}

// findOriginal locates the raw document file for pk under
// documents/originals/, matching by a "<pk>" or "<pk>_" filename prefix
// since paperless-ngx names files "<pk>_<slugified-title>.<ext>".
func findOriginal(files map[string]*zip.File, pk string) ([]byte, string, error) {
	const dir = "documents/originals/"
	for name, f := range files {
		if !strings.HasPrefix(name, dir) {
			continue
		}
		base := strings.TrimPrefix(name, dir)
		if base == pk || strings.HasPrefix(base, pk+"_") || strings.HasPrefix(base, pk+".") {
			content, err := readZipEntry(f)
			if err != nil {
				return nil, "", err
			}
			return content, contentTypeForName(base), nil
		}
	}
	return nil, "", fmt.Errorf("original document file not found under %s", dir)
}

func contentTypeForName(name string) string {
	switch {
	case strings.HasSuffix(name, ".pdf"):
		return "application/pdf"
	case strings.HasSuffix(name, ".png"):
		return "image/png"
	case strings.HasSuffix(name, ".jpg"), strings.HasSuffix(name, ".jpeg"):
		return "image/jpeg"
	default:
		return "application/octet-stream"
	}
}

func nameField(fields json.RawMessage) string {
	var raw map[string]json.RawMessage
	if err := json.Unmarshal(fields, &raw); err != nil {
		return ""
	}
	return jsonString(raw["name"])
}

func jsonString(raw json.RawMessage) string {
	if raw == nil {
		return ""
	}
	var s string
	if err := json.Unmarshal(raw, &s); err == nil {
		return s
	}
	var n json.Number
	if err := json.Unmarshal(raw, &n); err == nil {
		return n.String()
	}
	return ""
}

func jsonNumberArray(raw json.RawMessage) []string {
	if raw == nil {
		return nil
	}
	var nums []json.Number
	if err := json.Unmarshal(raw, &nums); err != nil {
		return nil
	}
	out := make([]string, len(nums))
	for i, n := range nums {
		out[i] = n.String()
	}
	return out
}

func readZipEntry(f *zip.File) ([]byte, error) {
	rc, err := f.Open()
	if err != nil {
		return nil, err
	}
	defer rc.Close()
	return io.ReadAll(rc)
}

// DetectFields returns the union of source field names actually present
// across docs, in a stable order, for the mapping-preview UI to list.
func DetectFields(docs []SourceDocument) []string {
	seen := map[string]bool{}
	var order []string
	for _, d := range docs {
		for k, v := range d.RawFields {
			if v == "" || seen[k] {
				continue
			}
			seen[k] = true
			order = append(order, k)
		}
	}
	return order
}