FDN-19: Gesamtlauf-Kommando, Klartext-Befund

scripts/fdn19-gesamtstart.sh

@@ -11,6 +11,7 @@
 #   fdn19-gesamtstart.sh measure <datei>    Ressourcenmessung im Betrieb
 #   fdn19-gesamtstart.sh restart-test       Wiederanlauf-Test (App und Worker hart beenden)
 #   fdn19-gesamtstart.sh reset              Datenbank neu anlegen und migrieren (frischer Lauf)
+#   fdn19-gesamtstart.sh full               Kompletter Lauf (Idle-Messung, Smoke, Wiederanlauf, Token-Modus) mit Ergebnissen unter $BASE/results
 #   fdn19-gesamtstart.sh down               Prozesse stoppen
 #   fdn19-gesamtstart.sh teardown [keep-kosit]  Alles zurueckbauen
 set -uo pipefail
@@ -266,6 +267,34 @@ cmd_restart_test() {
 	python3 "$BASE/src/scripts/fdn19_smoke.py" verify-batch $n
 }
 
+cmd_full() {
+	local res="$BASE/results"
+	mkdir -p "$res"
+	cmd_reset >"$res/00-reset.log" 2>&1 || die "reset"
+	cmd_up caller >"$res/01-up.log" 2>&1 || die "up"
+	sleep 45
+	cmd_measure "$res/10-idle.txt" >/dev/null
+	echo "load-vorher: $(cut -d' ' -f1-3 /proc/loadavg)" >"$res/11-load.txt"
+	rm -f "$BASE/measure/samples.txt"
+	( while true; do echo "$(date +%s) $(free -m | awk 'NR==2{print $3}') $(cut -d' ' -f1 /proc/loadavg)" >>"$BASE/measure/samples.txt"; sleep 1; done ) &
+	local sp=$!
+	cmd_smoke caller >"$res/20-smoke-caller.txt" 2>&1
+	echo "smoke-caller-exit=$?" >>"$res/11-load.txt"
+	cmd_measure "$res/21-nach-smoke.txt" >/dev/null
+	cmd_restart_test >"$res/30-restart.txt" 2>&1
+	echo "restart-exit=$?" >>"$res/11-load.txt"
+	kill "$sp" 2>/dev/null
+	cmd_measure "$res/31-nach-restart.txt" >/dev/null
+	awk '{if($2>m)m=$2; if($3>l)l=$3} END{print "sampler max_used_MB=" m " max_load1=" l " samples=" NR}' "$BASE/measure/samples.txt" >>"$res/11-load.txt"
+	echo "load-nachher: $(cut -d' ' -f1-3 /proc/loadavg)" >>"$res/11-load.txt"
+	cmd_reset >"$res/40-reset2.log" 2>&1 || die "reset2"
+	cmd_up token >"$res/41-up-token.log" 2>&1 || die "up token"
+	cmd_smoke token >"$res/50-smoke-token.txt" 2>&1
+	echo "smoke-token-exit=$?" >>"$res/11-load.txt"
+	cmd_down
+	echo "full fertig"
+}
+
 cmd_teardown() {
 	local keep="${1:-}"
 	cmd_down
@@ -299,7 +328,8 @@ smoke) shift; cmd_smoke "$@" ;;
 measure) shift; cmd_measure "$@" ;;
 restart-test) cmd_restart_test ;;
 reset) cmd_reset ;;
+full) cmd_full ;;
 down) cmd_down ;;
 teardown) shift; cmd_teardown "$@" ;;
-*) sed -n 2,18p "$0"; exit 2 ;;
+*) sed -n 2,19p "$0"; exit 2 ;;
 esac

scripts/fdn19_smoke.py

@@ -105,6 +105,10 @@ class Actor:
         return False
 
 
+def info(msg):
+    print("INFO %s" % msg, flush=True)
+
+
 def check(name, ok, detail=""):
     results.append((name, bool(ok), detail))
     print("%-4s %-62s %s" % ("PASS" if ok else "FAIL", name, detail), flush=True)
@@ -187,6 +191,14 @@ def smoke(mode):
     guard("S08", lambda: einvoice("ubl", "S08 XRechnung UBL (KoSIT-Beispiel)", "rechnung-ubl.xml", "application/xml", "kosit-ubl.xml", True))
     guard("S09", lambda: einvoice("zf", "S09 ZUGFeRD (synthetisch)", "rechnung-zugferd.pdf", "application/pdf", "zugferd.pdf", False))
 
+    def s09b():
+        plain = a.upload("klartext.txt", "text/plain", ("Klartextbeleg Zinnoberrot Wasserturm %d" % time.time_ns()).encode())
+        time.sleep(8)
+        st, body = a.search("Wasserturm")
+        found = st == 200 and any(h["document_id"] == plain for h in body["hits"])
+        info("F5 Volltext von text/plain-Dokumenten im Suchindex: %s (nur Titel wird indexiert, wenn nein)" % ("ja" if found else "nein"))
+    guard("S09b", s09b)
+
     def s10():
         st, body = a.search(TERM, facets="created_by,document_type,tag")
         check("S10 Suche mit Facetten", st == 200 and "facets" in body and body["total"] >= 1, "total=%s" % (body or {}).get("total"))
@@ -318,10 +330,6 @@ def batch(n):
         print(a.upload("batch-%d.pdf" % i, "application/pdf", mini_pdf(lines)))
 
 
-def info(msg):
-    print("INFO %s" % msg, flush=True)
-
-
 def verify_batch(n):
     actor_id = psql("SELECT id FROM users ORDER BY created_at LIMIT 1")
     a = Actor("caller", actor_id)
@@ -335,9 +343,9 @@ def verify_batch(n):
             break
         time.sleep(3)
     waited = int(time.time() - start)
-    docs = psql("SELECT count(*) FROM documents WHERE title LIKE 'batch-%'")
-    dup = psql("SELECT count(*) FROM (SELECT d.id FROM file_revisions fr JOIN documents d ON d.id = fr.document_id WHERE d.title LIKE 'batch-%' GROUP BY d.id HAVING count(*) > 1) x")
-    same = psql("SELECT count(*) FROM (SELECT checksum_sha256 FROM file_revisions fr JOIN documents d ON d.id = fr.document_id WHERE d.title LIKE 'batch-%' GROUP BY checksum_sha256 HAVING count(*) > 1) x")
+    docs = psql("SELECT count(*) FROM documents WHERE title LIKE 'batch-%.pdf'")
+    dup = psql("SELECT count(*) FROM (SELECT d.id FROM file_revisions fr JOIN documents d ON d.id = fr.document_id WHERE d.title LIKE 'batch-%.pdf' GROUP BY d.id HAVING count(*) > 1) x")
+    same = psql("SELECT count(*) FROM (SELECT checksum_sha256 FROM file_revisions fr JOIN documents d ON d.id = fr.document_id WHERE d.title LIKE 'batch-%.pdf' GROUP BY checksum_sha256 HAVING count(*) > 1) x")
     dead = psql("SELECT count(*) FROM processing_jobs WHERE status = 'dead_letter'")
     check("W1 alle %s Dokumente vorhanden, keine Dubletten" % n, docs == str(n) and dup == "0" and same == "0", "Dokumente=%s Dokumente mit mehreren Revisionen=%s gleiche Pruefsummen=%s" % (docs, dup, same))
     check("W2 alle Dokumente nach hartem Neustart im Suchindex", st == 200 and body["total"] >= int(n), "total=%s" % (body or {}).get("total"))