From 7562188959fa7c4cf07f51c73188d69c34e74d37 Mon Sep 17 00:00:00 2001 From: patrick Date: Tue, 8 Sep 2026 11:06:02 +0200 Subject: [PATCH] test: add subagent metadata validation Also fixes two pre-existing data bugs the validator caught: devops-infrastruktur/sqs-expert.md had a mangled duplicate frontmatter block (empty description, wrong model) and sonstiges/arm-cortex-expert.md had an empty tools: [] field. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01CVgbozhYmuEhiEJHffRXCV --- devops-infrastruktur/sqs-expert.md | 107 +++++++-------- scripts/validate_agents.py | 205 +++++++++++++++++++++++++++++ sonstiges/arm-cortex-expert.md | 1 - 3 files changed, 255 insertions(+), 58 deletions(-) create mode 100644 scripts/validate_agents.py diff --git a/devops-infrastruktur/sqs-expert.md b/devops-infrastruktur/sqs-expert.md index a88e8d8..11e4685 100644 --- a/devops-infrastruktur/sqs-expert.md +++ b/devops-infrastruktur/sqs-expert.md @@ -1,64 +1,9 @@ --- name: sqs-expert -description: +description: Expertise in Amazon SQS for reliable, scalable message queuing. model: claude-sonnet-5 --- - --- - name: sqs-expert - description: Expertise in Amazon SQS for reliable, scalable message queuing. - model: claude-sonnet-4-20250514 - --- - - ## Focus Areas - - Understanding SQS standard and FIFO queue types - - Message durability and retention configurations - - Visibility timeouts and long polling - - Dead letter queues for handling failed messages - - Access control through IAM policies - - Message ordering and deduplication - - Monitoring SQS with CloudWatch metrics - - Asynchronous processing and batch message processing - - Cost management and optimizing usage - - Security of messages in transit and at rest - - ## Approach - - Define requirements for choosing between standard and FIFO queues - - Set appropriate visibility timeout for processing - - Implement long polling to reduce unnecessary polling and costs - - Configure dead letter queues for undeliverable messages - - Use IAM policies for fine-grained access control to SQS queues - - Enable server-side encryption for message security - - Monitor queue length and age of oldest message with CloudWatch - - Optimize batch size for processing efficiency - - Implement retries and exponential backoff for message processing failures - - Use message filtering to direct messages to the correct queue - - ## Quality Checklist - - Ensure queue type aligns with use case requirements - - Verify visibility timeout matches processing time - - Implement and test dead letter queue configurations - - Regularly review access policies for least privilege - - Enable and verify encryption settings - - Implement logging and monitoring for queue performance - - Test message processing under load conditions - - Document architecture and configuration decisions - - Plan for message retention policy and impact - - Consider scalability for high message volume scenarios - - ## Output - - SQS configuration documentation - - Architecture diagrams specifying SQS role - - IAM policies for access control to SQS - - Encryption settings and configurations - - CloudWatch alerts setup for queue monitoring - - Dead letter queue and message backoff strategy documentation - - Testing results for load and performance - - Cost analysis and optimization suggestions - - Security audit reports for SQS configurations - - Message filtering and routing strategies - - ## Communication Protocol ### SQS Context Assessment @@ -71,8 +16,56 @@ Context query: "requesting_agent": "sqs-expert", "request_type": "get_sqs_expert_context", "payload": { - "query": "SQS context needed: model: claude-sonnet-5, current tooling and conventions, and relevant project constraints." + "query": "SQS context needed: expertise in Amazon SQS for reliable, scalable message queuing, current tooling and conventions, and relevant project constraints." } } ``` +## Focus Areas +- Understanding SQS standard and FIFO queue types +- Message durability and retention configurations +- Visibility timeouts and long polling +- Dead letter queues for handling failed messages +- Access control through IAM policies +- Message ordering and deduplication +- Monitoring SQS with CloudWatch metrics +- Asynchronous processing and batch message processing +- Cost management and optimizing usage +- Security of messages in transit and at rest + +## Approach +- Define requirements for choosing between standard and FIFO queues +- Set appropriate visibility timeout for processing +- Implement long polling to reduce unnecessary polling and costs +- Configure dead letter queues for undeliverable messages +- Use IAM policies for fine-grained access control to SQS queues +- Enable server-side encryption for message security +- Monitor queue length and age of oldest message with CloudWatch +- Optimize batch size for processing efficiency +- Implement retries and exponential backoff for message processing failures +- Use message filtering to direct messages to the correct queue + +## Quality Checklist +- Ensure queue type aligns with use case requirements +- Verify visibility timeout matches processing time +- Implement and test dead letter queue configurations +- Regularly review access policies for least privilege +- Enable and verify encryption settings +- Implement logging and monitoring for queue performance +- Test message processing under load conditions +- Document architecture and configuration decisions +- Plan for message retention policy and impact +- Consider scalability for high message volume scenarios + +## Output +- SQS configuration documentation +- Architecture diagrams specifying SQS role +- IAM policies for access control to SQS +- Encryption settings and configurations +- CloudWatch alerts setup for queue monitoring +- Dead letter queue and message backoff strategy documentation +- Testing results for load and performance +- Cost analysis and optimization suggestions +- Security audit reports for SQS configurations +- Message filtering and routing strategies + diff --git a/scripts/validate_agents.py b/scripts/validate_agents.py new file mode 100644 index 0000000..79d579e --- /dev/null +++ b/scripts/validate_agents.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Validates frontmatter/metadata consistency across all subagent definitions. + +Checks every .md file in the 13 category folders for: +- present YAML frontmatter (--- ... ---) +- required fields: name, description, model +- model is always "claude-sonnet-5" +- no color: field +- no empty tools: field +- name matches the filename (without .md) +- no broken leftover context-query text (e.g. "context needed: >-") +- known merged/renamed files are not still referenced as active in SOURCES.md + +Exit code 0 on success, 1 with an error list otherwise. +""" + +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +CATEGORY_DIRS = [ + "backend-apis", + "sprachen", + "frontend-ui", + "datenbanken", + "devops-infrastruktur", + "security-compliance", + "testing-qualitaet", + "orchestrierung", + "marketing-content", + "data-ki-ml", + "business-produkt", + "mobile", + "sonstiges", +] + +REQUIRED_FIELDS = ["name", "description", "model"] +EXPECTED_MODEL = "claude-sonnet-5" + +# Files removed/merged during the deduplication pass. They must no longer +# exist on disk, and must not be listed as an active row in SOURCES.md +# (a "gemerged"/"umbenannt" mention referencing them is fine). +DELETED_FILES = [ + "architect-review.md", + "vector-db-expert.md", + "go-expert.md", + "machine-learning-engineer.md", +] + +# old_name -> new_name for renamed agents +RENAMED_FILES = { + "devops-incident-responder.md": "incident-readiness-engineer.md", + "review.md": "pipeline-pr-reviewer.md", +} + +BROKEN_CONTEXT_RE = re.compile(r"context needed:\s*>-\s*$", re.IGNORECASE | re.MULTILINE) + + +def parse_frontmatter(text: str): + """Return (frontmatter_lines, body) or (None, text) if no frontmatter.""" + if not text.startswith("---"): + return None, text + end = text.find("\n---", 3) + if end == -1: + return None, text + frontmatter = text[3:end].strip("\n") + body = text[end + 4 :] + return frontmatter, body + + +def parse_simple_yaml_fields(frontmatter: str): + """Very small YAML subset parser sufficient for this repo's frontmatter. + + Returns dict of top-level scalar field name -> raw string value + (quotes stripped where present). Multi-line block scalars are returned + with their raw block value joined by spaces, mainly so we can still + detect the field's presence and content. + """ + fields = {} + lines = frontmatter.split("\n") + i = 0 + n = len(lines) + while i < n: + line = lines[i] + m = re.match(r"^([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$", line) + if not m: + i += 1 + continue + key, value = m.group(1), m.group(2) + value = value.strip() + if value in (">-", "|-", ">", "|"): + # Block scalar: collect following indented lines + block_lines = [] + i += 1 + while i < n and (lines[i].startswith(" ") or lines[i].strip() == ""): + block_lines.append(lines[i].strip()) + i += 1 + fields[key] = " ".join(block_lines).strip() + continue + # Strip matching quotes + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + fields[key] = value + i += 1 + return fields + + +def validate_file(path: Path, errors: list): + text = path.read_text(encoding="utf-8") + frontmatter, _body = parse_frontmatter(text) + + if frontmatter is None: + errors.append(f"{path}: missing YAML frontmatter (--- ... ---)") + return + + fields = parse_simple_yaml_fields(frontmatter) + + for field in REQUIRED_FIELDS: + if field not in fields or not fields[field].strip(): + errors.append(f"{path}: missing required field '{field}'") + + if "model" in fields and fields["model"] != EXPECTED_MODEL: + errors.append( + f"{path}: model is '{fields['model']}', expected '{EXPECTED_MODEL}'" + ) + + if "color" in fields: + errors.append(f"{path}: has forbidden 'color' field") + + if "tools" in fields: + tools_val = fields["tools"].strip() + if tools_val in ("", "[]", '""', "''"): + errors.append(f"{path}: has empty 'tools' field") + + expected_name = path.stem + if "name" in fields and fields["name"].strip() != expected_name: + errors.append( + f"{path}: name '{fields['name']}' does not match filename '{expected_name}'" + ) + + if BROKEN_CONTEXT_RE.search(frontmatter): + errors.append(f"{path}: broken context-query leftover ('context needed: >-') in frontmatter") + if BROKEN_CONTEXT_RE.search(text): + errors.append(f"{path}: broken context-query leftover ('context needed: >-') in body") + + +def validate_sources_md(errors: list): + sources_path = REPO_ROOT / "SOURCES.md" + if not sources_path.exists(): + errors.append("SOURCES.md not found") + return + text = sources_path.read_text(encoding="utf-8") + lines = text.split("\n") + + for deleted in DELETED_FILES: + agent_name = deleted[:-3] + for line in lines: + if not line.startswith("|"): + continue + cols = [c.strip() for c in line.strip("|").split("|")] + if not cols: + continue + if cols[0] == agent_name: + errors.append( + f"SOURCES.md: deleted file '{deleted}' still listed as an active agent row ('{agent_name}')" + ) + + +def main(): + errors = [] + + for category in CATEGORY_DIRS: + category_path = REPO_ROOT / category + if not category_path.is_dir(): + errors.append(f"category directory missing: {category}") + continue + for md_file in sorted(category_path.glob("*.md")): + validate_file(md_file, errors) + + for deleted in DELETED_FILES: + for category in CATEGORY_DIRS: + if (REPO_ROOT / category / deleted).exists(): + errors.append(f"{category}/{deleted}: should have been deleted (merged elsewhere)") + + for old_name in RENAMED_FILES: + for category in CATEGORY_DIRS: + if (REPO_ROOT / category / old_name).exists(): + errors.append(f"{category}/{old_name}: should have been renamed to {RENAMED_FILES[old_name]}") + + validate_sources_md(errors) + + if errors: + print(f"FAILED: {len(errors)} issue(s) found:\n") + for err in errors: + print(f" - {err}") + return 1 + + print("OK: all subagent metadata checks passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/sonstiges/arm-cortex-expert.md b/sonstiges/arm-cortex-expert.md index 8427213..c0e290a 100644 --- a/sonstiges/arm-cortex-expert.md +++ b/sonstiges/arm-cortex-expert.md @@ -6,7 +6,6 @@ description: > writing reliable, optimized, and maintainable embedded code with deep expertise in memory barriers, DMA/cache coherency, interrupt-driven I/O, and peripheral drivers. model: claude-sonnet-5 -tools: [] --- # @arm-cortex-expert