From 71566e8008947da2161d0b3350365d24fcc9cc2a Mon Sep 17 00:00:00 2001 From: monoxgas Date: Tue, 28 Jul 2026 17:18:40 -0600 Subject: [PATCH] Promote llm-wiki and analysis-ledger --- capabilities/analysis-ledger/README.md | 166 +++++++ .../analysis-ledger/agents/recursive-sast.md | 34 ++ capabilities/analysis-ledger/capability.yaml | 43 ++ capabilities/analysis-ledger/items.py | 357 ++++++++++++++ .../skills/recursive-code-analysis/SKILL.md | 127 +++++ .../agents/openai.yaml | 4 + .../tests/test_analysis_ledger.py | 445 ++++++++++++++++++ .../analysis-ledger/tools/analysis_items.py | 367 +++++++++++++++ capabilities/llm-wiki/README.md | 96 ++++ capabilities/llm-wiki/agents/llm-wiki.md | 45 ++ capabilities/llm-wiki/capability.yaml | 33 ++ capabilities/llm-wiki/items.py | 93 ++++ .../llm-wiki/skills/llm-wiki/SKILL.md | 97 ++++ .../skills/llm-wiki/agents/openai.yaml | 4 + capabilities/llm-wiki/tests/test_llm_wiki.py | 217 +++++++++ capabilities/llm-wiki/tools/wiki_items.py | 156 ++++++ 16 files changed, 2284 insertions(+) create mode 100644 capabilities/analysis-ledger/README.md create mode 100644 capabilities/analysis-ledger/agents/recursive-sast.md create mode 100644 capabilities/analysis-ledger/capability.yaml create mode 100644 capabilities/analysis-ledger/items.py create mode 100644 capabilities/analysis-ledger/skills/recursive-code-analysis/SKILL.md create mode 100644 capabilities/analysis-ledger/skills/recursive-code-analysis/agents/openai.yaml create mode 100644 capabilities/analysis-ledger/tests/test_analysis_ledger.py create mode 100644 capabilities/analysis-ledger/tools/analysis_items.py create mode 100644 capabilities/llm-wiki/README.md create mode 100644 capabilities/llm-wiki/agents/llm-wiki.md create mode 100644 capabilities/llm-wiki/capability.yaml create mode 100644 capabilities/llm-wiki/items.py create mode 100644 capabilities/llm-wiki/skills/llm-wiki/SKILL.md create mode 100644 capabilities/llm-wiki/skills/llm-wiki/agents/openai.yaml create mode 100644 capabilities/llm-wiki/tests/test_llm_wiki.py create mode 100644 capabilities/llm-wiki/tools/wiki_items.py diff --git a/capabilities/analysis-ledger/README.md b/capabilities/analysis-ledger/README.md new file mode 100644 index 0000000..732e213 --- /dev/null +++ b/capabilities/analysis-ledger/README.md @@ -0,0 +1,166 @@ +# Analysis Ledger + +Analysis Ledger supports recursive, single-agent static analysis that can +outlive a model context window. The runtime owns the normal +think/tool/observe loop; typed Agent Output records hold the durable worklist, +evidence, and progress that feed later iterations. + +The design builds on [Andrej Karpathy's LLM Wiki +pattern](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f), +then adds the control state needed for code analysis. Maintained summaries +still compound like wiki pages, while explicit targets and claims make +traversal resumable and bounded. + +## What it demonstrates + +| Need | Implementation | +| --- | --- | +| Bounded autonomous iteration | Native `/auto ` policy | +| Durable run definition | `analysis_run` Item | +| Ranked recursive frontier | `analysis_target` Items + `analysis_next_targets` | +| Evidence and uncertainty | `analysis_claim` Items | +| Compact lookup | Built-in `list_items` / `search_items` + `analysis_progress` | +| Focused rehydration | Built-in `read_item` with links | +| Persistent state transitions | Built-in `update_item` and `link_items` | +| Confirmed security output | Built-in `finding` Item | + +No worker or server modification is involved. Browsing, reading, and full-text +search come from the platform's built-in `list_items`, `read_item`, and +`search_items` tools; writes use the runtime's capability-aware Agent Output +tools. This capability adds only the typed schema and two frontier-control reads +(`analysis_next_targets`, `analysis_progress`) the generic surface cannot +express. + +## Try it + +This requires: + +- Dreadnode SDK 2.0.38 or newer, +- a configured model/provider, +- a platform-connected profile with an active project and effective + `items:read` plus `items:write` grants. + +> **Select a project first.** The ledger's runs, targets, and claims live in +> your profile's active project, and reads/writes require item-read/item-write +> grants. The tools raise `requires ... an active project` mid-run if none is +> selected. Pick one in the TUI project picker (it's shown in the status bar) +> before starting a run. + +```bash +dn capability install dreadnode/analysis-ledger +dn --capability analysis-ledger --agent recursive-sast +``` + +Then enter `/auto 100` and try: + +```text +Analyze authentication bypass risk in /path/to/repo. + +Create a run at the current commit. Seed targets from entrypoints, +authorization checks, and sensitive sinks. Process one target per iteration, +record evidence-bearing claims, and enqueue newly discovered paths. Stop after +30 targets, an empty frontier, or three non-expanding iterations. +``` + +For architecture-oriented exploration: + +```text +Map the request path from HTTP entrypoints to persistence in /path/to/repo. +Maintain module and function targets, record trust-boundary claims with +file:line evidence, and leave a resumable frontier for uncertain paths. +``` + +## How the loop actually runs + +The everyday path is a **single autonomous run**, not manual restarts. You start +one `/auto` session and the runtime keeps the loop going: + +```text +Analyze authentication bypass risk in /path/to/repo. +``` +then `/auto 100`. + +As context fills, the runtime **auto-compacts** (summarizes older messages) and +the agent keeps iterating — it does not stop or wait for you. Each iteration +re-reads the ledger (`analysis_progress` → `analysis_next_targets`) rather than +trusting the conversation. The frontier, claims, and persisted run counters are +durable records in the **active platform project**. That is what lets one run +cover a codebase far larger than a single context window, provided each +completed iteration checkpoints before compaction or interruption. + +Records produced during the run appear under the session's Output tab and on +the project's Agent Output page in the platform web app (the TUI prints a +clickable link beneath each write). They are private to the session owner by +default; promote session visibility to workspace before expecting teammates to +read them. + +**Resuming later is the same mechanism, across a harder boundary.** If a session +dies, or you come back the next day, start a new session and ask it to resume — +**pin the same objective and commit**, because `run_key` is derived from the +canonical repository, immutable revision, and normalized objective as the first +24 lowercase hex characters of their newline-delimited SHA-256 digest: + +```text +Resume the authentication-bypass analysis run for /path/to/repo at . +``` + +The agent calls `search_items` with the exact `run_key` and +`item_type="analysis_run"`, reads the frontier, and resumes an `in-progress` +target before selecting queued work. Completed checkpoints are not repeated. +Keep one live session per run: target claiming remains a read-then-update, not +an atomic lease. + +## Compose with other analysis capabilities + +Analysis Ledger supplies the durable worklist; it does not ship analysis +engines. The composition mechanism is general: the runtime makes every installed +capability's tools globally visible, and the `recursive-sast` agent enables all +of them (`tools: "*"`), so any analysis capability you install alongside becomes +usable inspection tooling while Analysis Ledger stays the owner of the typed +Item schema. Point a target's rationale at those tools and record their output +as claim evidence. + +`sast` is the worked example. Install it alongside Analysis Ledger to add +`filemap`, CodeQL, Semgrep, dangerous-function scanners, and git tooling: + +```bash +dn capability install dreadnode/sast +dn capability install dreadnode/analysis-ledger +dn --capability analysis-ledger --capability sast --agent recursive-sast +``` + +The agent explicitly disables `codesearch` and delegation because `codesearch` +spawns a sub-agent. This capability deliberately exercises the simpler +single-agent path. + +## Operating model + +```text +survey → seed frontier → select target → inspect bounded code neighborhood + ↑ ↓ +progress ← complete/refute ← record claims ← discover child targets +``` + +The local codebase and optional static-analysis engines remain the analysis +plane. Items store only selected work and durable conclusions; they are not a +replacement for an AST, code index, or callgraph database. + +## Current limitations + +- Reads and updates require platform connectivity plus item-read/item-write + grants. Trace-backed writes become queryable only after platform + materialization. +- Item refs are project-unique, so every target and claim ref is run-prefixed. +- Browsing and full-text search are handled server-side by the built-in tools. + Frontier tools narrow candidates with server search on `run_ref`, exact-filter + locally, and scan up to 1,000 records per type to rank custom + `target_state`/`priority` and claim `disposition`. They refuse to select or + stop on a truncated candidate scan. +- Target claiming is a read-then-update operation, not an atomic lease. Use one + session per run. +- Source invalidation is procedural: the run pins a revision and optional + target fingerprints, but no background process marks changed targets stale. +- Compaction is a context pressure valve, not persistence. Items and the + per-iteration run checkpoint remain the source of truth. +- Platform trace-backed materialization runs at session freeze where supported, + but is eventual and cannot satisfy same-iteration reads, updates, or links. diff --git a/capabilities/analysis-ledger/agents/recursive-sast.md b/capabilities/analysis-ledger/agents/recursive-sast.md new file mode 100644 index 0000000..ea22975 --- /dev/null +++ b/capabilities/analysis-ledger/agents/recursive-sast.md @@ -0,0 +1,34 @@ +--- +name: recursive-sast +description: Single-agent static analysis over a durable, evidence-bearing worklist in Agent Output +model: inherit +tools: + "*": true + codesearch: false + spawn_agent: false +skills: [recursive-code-analysis] +--- + +You are a static-analysis researcher operating one bounded target at a time. +Use the `recursive-code-analysis` skill as the controlling procedure. + +Treat Analysis Ledger items as the source of truth for coverage, hypotheses, +and progress. Conversation history, `todo`, and session memory are only working +state. Resume from the ledger instead of reconstructing completed work. + +Prefer deterministic local inspection with `glob`, `grep`, `read`, `bash`, and +`filemap` when available. Use scanners only when their result can prioritize or +verify a ledger target. Never delegate to another agent or use `codesearch`, +which internally spawns one. + +Create built-in `finding` items only for vulnerabilities that survive an +adversarial verification pass. Preserve rejected hypotheses as refuted +`analysis_claim` items so later iterations do not repeat them. + +When the user supplies a repository and objective, begin immediately by +resolving the revision and starting or resuming an analysis run. Otherwise ask +for the repository path and analysis objective. + +If a custom item write says the item type is unknown or its capability/version +is not registered, stop immediately and report the deployment mismatch. Do not +install, sync, or rewrite capability setup from inside an analysis run. diff --git a/capabilities/analysis-ledger/capability.yaml b/capabilities/analysis-ledger/capability.yaml new file mode 100644 index 0000000..ae2c104 --- /dev/null +++ b/capabilities/analysis-ledger/capability.yaml @@ -0,0 +1,43 @@ +schema: 1 +name: analysis-ledger +version: "0.2.1" +description: > + Persistent single-agent code analysis over a durable Agent Output worklist. + The agent records analysis runs, ranked targets, and evidence-bearing claims, + then repeatedly reads, advances, and expands that frontier across context and + session boundaries. + +agents: + - agents/ + +skills: + - skills/ + +tools: + - tools/ + +produces: + values: + - finding + analysis_run: "items:AnalysisRun" + analysis_target: "items:AnalysisTarget" + analysis_claim: "items:AnalysisClaim" + +checks: + - name: git + command: command -v git + - name: ripgrep + command: command -v rg + +author: + name: Dreadnode + url: https://dreadnode.io +license: MIT +repository: https://github.com/dreadnode/capabilities +keywords: + - agent-output + - code-analysis + - recursive-analysis + - single-agent + - static-analysis + - worklist diff --git a/capabilities/analysis-ledger/items.py b/capabilities/analysis-ledger/items.py new file mode 100644 index 0000000..1d923cb --- /dev/null +++ b/capabilities/analysis-ledger/items.py @@ -0,0 +1,357 @@ +"""Typed Agent Output records for persistent recursive code analysis.""" + +import typing as t +from hashlib import sha256 + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints, model_validator + +RunState = t.Literal[ + "planned", + "running", + "paused", + "completed", + "exhausted", + "stale", +] +TargetKind = t.Literal[ + "repository", + "module", + "file", + "class", + "function", + "endpoint", + "data-flow", + "configuration", + "dependency", +] +TargetState = t.Literal[ + "queued", + "in-progress", + "analyzed", + "blocked", + "skipped", + "stale", +] +ClaimCategory = t.Literal[ + "behavior", + "trust-boundary", + "vulnerability", + "assumption", + "coverage-gap", +] +ClaimDisposition = t.Literal[ + "hypothesized", + "verified", + "refuted", + "unresolved", +] +Confidence = t.Literal["high", "medium", "low"] +NonEmptyText = t.Annotated[str, StringConstraints(min_length=1)] +RunKey = t.Annotated[ + str, + StringConstraints( + min_length=24, + max_length=24, + pattern=r"^[0-9a-f]{24}$", + ), +] + + +class AnalysisRun(BaseModel): + """One bounded analysis of a repository revision.""" + + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ + "allOf": [ + { + "if": { + "properties": { + "run_state": { + "enum": ["paused", "completed", "exhausted", "stale"] + } + }, + "required": ["run_state"], + }, + "then": { + "required": ["stop_reason"], + "properties": { + "stop_reason": {"type": "string", "minLength": 1} + }, + }, + } + ] + }, + ) + + title: str = Field(..., min_length=1, max_length=512) + run_key: RunKey = Field( + description=( + "First 24 lowercase hex characters of SHA-256 over canonical " + "repository, immutable revision, and normalized objective separated " + "by newlines." + ), + ) + objective: str = Field( + ..., + min_length=1, + description="The security or code-understanding question this run must answer.", + ) + repository: str = Field( + ..., + min_length=1, + description="Canonical local path or repository URL.", + ) + revision: str = Field( + ..., + min_length=1, + description="Git commit or other immutable source revision.", + ) + run_state: RunState = Field(default="running") + scope: list[str] = Field( + default_factory=list, + description="Included directories, components, or vulnerability classes.", + ) + exclusions: list[str] = Field( + default_factory=list, + description="Explicitly excluded code or concerns.", + ) + max_targets: int = Field( + default=50, + ge=1, + le=1_000, + description="Maximum targets in this run; bounded by the frontier scan contract.", + ) + max_depth: int = Field(default=8, ge=0, le=100) + iterations_completed: int = Field(default=0, ge=0) + non_expanding_iterations: int = Field( + default=0, + ge=0, + description="Consecutive completed targets that added no useful claim or child target.", + ) + targets_discovered: int = Field(default=0, ge=0) + stop_conditions: list[str] = Field( + default_factory=list, + description="Conditions that end the worklist loop.", + ) + stop_reason: str | None = Field( + default=None, + description="Why the run paused or ended.", + ) + + @model_validator(mode="after") + def validate_run_invariants(self) -> "AnalysisRun": + material = f"{self.repository}\n{self.revision}\n{self.objective}".encode() + expected_run_key = sha256(material).hexdigest()[:24] + if self.run_key != expected_run_key: + raise ValueError( + "run_key must be the first 24 lowercase hex characters of SHA-256 " + "over repository, revision, and objective separated by newlines" + ) + if self.run_state in {"paused", "completed", "exhausted", "stale"}: + if not self.stop_reason: + raise ValueError("terminal or paused run_state requires stop_reason") + return self + + +class AnalysisTarget(BaseModel): + """One bounded code unit or path waiting on the analysis frontier.""" + + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ + "allOf": [ + { + "if": { + "properties": { + "target_state": {"enum": ["analyzed", "blocked", "skipped"]} + }, + "required": ["target_state"], + }, + "then": { + "required": ["summary"], + "properties": {"summary": {"type": "string", "minLength": 1}}, + }, + }, + { + "if": { + "properties": {"target_state": {"const": "analyzed"}}, + "required": ["target_state"], + }, + "then": { + "required": ["evidence_refs"], + "properties": { + "evidence_refs": { + "type": "array", + "minItems": 1, + } + }, + }, + }, + ] + }, + ) + + title: str = Field(..., min_length=1, max_length=512) + run_ref: str = Field( + ..., + min_length=1, + description="Stable ref of the analysis_run that owns this target.", + ) + target_key: str = Field( + ..., + min_length=1, + description="Stable identity within the revision, such as path::symbol.", + ) + kind: TargetKind + location: str = Field( + ..., + min_length=1, + description="File, symbol, endpoint, or graph-node location.", + ) + priority: int = Field( + default=50, + ge=0, + le=100, + description="Best-first priority; higher values are processed first.", + ) + rationale: str = Field( + ..., + min_length=1, + description="Why this target could advance the run objective.", + ) + target_state: TargetState = Field(default="queued") + depth: int = Field(default=0, ge=0, le=100) + parent_ref: str | None = Field( + default=None, + description="Ref of the target whose analysis discovered this target.", + ) + summary: str | None = Field( + default=None, + description="Compact conclusion retained after the bounded analysis pass.", + ) + evidence_refs: list[NonEmptyText] = Field( + default_factory=list, + description="Important file:line, tool result, or artifact anchors.", + ) + open_questions: list[NonEmptyText] = Field( + default_factory=list, + description="Questions that may justify new child targets.", + ) + source_fingerprint: str | None = Field( + default=None, + description="Optional content hash used to detect stale conclusions.", + ) + + @model_validator(mode="after") + def require_terminal_summary(self) -> "AnalysisTarget": + if self.target_state in {"analyzed", "blocked", "skipped"} and not self.summary: + raise ValueError("terminal target_state requires summary") + if self.target_state == "analyzed" and not self.evidence_refs: + raise ValueError("analyzed target_state requires evidence_refs") + return self + + +class AnalysisClaim(BaseModel): + """An evidence-bearing hypothesis or conclusion about one analysis target.""" + + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ + "allOf": [ + { + "if": { + "properties": { + "disposition": {"const": "verified"}, + "claim_category": {"const": "vulnerability"}, + }, + "required": ["disposition", "claim_category"], + }, + "then": { + "required": ["weakness", "severity", "impact"], + "properties": { + "weakness": {"type": "string", "minLength": 1}, + "severity": { + "enum": [ + "critical", + "high", + "medium", + "low", + "info", + ] + }, + "impact": {"type": "string", "minLength": 1}, + }, + }, + } + ] + }, + ) + + title: str = Field(..., min_length=1, max_length=512) + run_ref: str = Field( + ..., + min_length=1, + description="Stable ref of the owning analysis_run.", + ) + target_ref: str = Field( + ..., + min_length=1, + description="Stable ref of the analysis_target this claim concerns.", + ) + claim_category: ClaimCategory + statement: str = Field(..., min_length=1) + disposition: ClaimDisposition = Field(default="hypothesized") + confidence: Confidence = Field(default="medium") + weakness: str | None = Field( + default=None, + description=( + "For vulnerability-category claims, the standard taxonomy anchor: a " + "CWE id (e.g. 'CWE-89'), OWASP category, or CVE. Grounds the claim in " + "shared vocabulary and flows into the promoted finding's category." + ), + ) + severity: t.Literal["critical", "high", "medium", "low", "info"] | None = Field( + default=None, + description=( + "CVSS-aligned impact estimate for a vulnerability claim, justified by " + "the evidence. Carries forward to the built-in finding's severity." + ), + ) + evidence_refs: list[NonEmptyText] = Field( + ..., + min_length=1, + description="Concrete file:line, trace, tool result, or artifact anchors.", + ) + counterevidence: list[NonEmptyText] = Field( + default_factory=list, + description="Evidence that weakens or refutes the statement.", + ) + impact: str | None = Field( + default=None, + description="Security or analysis consequence if the claim is true.", + ) + next_steps: list[NonEmptyText] = Field( + default_factory=list, + description="Specific checks that could resolve remaining uncertainty.", + ) + + @model_validator(mode="after") + def require_verified_evidence(self) -> "AnalysisClaim": + if self.disposition == "verified" and not self.evidence_refs: + raise ValueError("verified claim requires evidence_refs") + if self.disposition == "verified" and self.claim_category == "vulnerability": + missing = [ + name + for name, value in ( + ("weakness", self.weakness), + ("severity", self.severity), + ("impact", self.impact), + ) + if not value + ] + if missing: + raise ValueError( + "verified vulnerability claim requires " + ", ".join(missing) + ) + return self diff --git a/capabilities/analysis-ledger/skills/recursive-code-analysis/SKILL.md b/capabilities/analysis-ledger/skills/recursive-code-analysis/SKILL.md new file mode 100644 index 0000000..31d4b99 --- /dev/null +++ b/capabilities/analysis-ledger/skills/recursive-code-analysis/SKILL.md @@ -0,0 +1,127 @@ +--- +name: recursive-code-analysis +description: Use when code analysis, vulnerability research, or architecture mapping needs a durable worklist that can resume across context or session boundaries. +--- + +# Recursive code analysis + +Traverse code through an explicit durable frontier. Store conclusions and +remaining work in Analysis Ledger items; do not rely on conversation history as +the coverage record. + +## Start or resume + +1. Resolve the canonical repository path, normalized objective, and immutable + revision. Use the Git commit when available; otherwise compute and record a + source fingerprint. Derive `run_key` as the first 24 lowercase hex characters + of SHA-256 over `repository + "\n" + revision + "\n" + objective`. +2. Call `search_items` with the computed `run_key` and + `item_type="analysis_run"` before creating one. Read candidates and resume + only an exact + `run_key`/objective/repository/revision match; never mix revisions. +3. Create a run with `report_item(item_type="analysis_run")`. Use a stable, + project-unique ref prefixed with `analysis-`. +4. Survey structure with `glob`, `grep`, and `filemap` when available. Seed + three to ten high-value targets, not every file. Favor entry points, sinks, + trust boundaries, dispatchers, and nodes that gate deeper code. +5. Link each target to the run with `part_of`; link discovered children from + their parent with `expands_to`. + +Prefix target and claim refs with the run ref because refs are project-unique. +Use a stable path-and-symbol key and a deterministic run-prefixed target ref. +Resolve that exact ref before reporting; update an existing target instead of +duplicating it. + +## Advance one target + +Repeat this bounded iteration: + +1. Call `analysis_progress`, then `analysis_next_targets`. Stop with a legible + incomplete-scan error if `scan.complete` or `selection_complete` is false. +2. Read the selected target and relevant claims with the built-in `read_item` + (by ref). If `target_state` is `queued`, set it to `in-progress` before + investigating it. If it is already `in-progress`, resume it; this is + single-agent crash recovery. +3. Inspect only the code neighborhood necessary to answer the target rationale: + the complete definition, direct callers/callees, relevant validation, and + nearby configuration. Keep bulk scanner output on disk and retain paths. +4. Record durable conclusions as `analysis_claim` items with a + `claim_category` and concrete file:line or artifact `evidence_refs`. Link + each claim to its target with `about`. +5. Try to refute security-relevant claims. Set the disposition to `verified`, + `refuted`, or `unresolved`; never erase a refutation. +6. Add a child target only when evidence exposes a meaningful new path or + uncertainty. Respect the run's maximum depth and target budget. +7. Update the target with a compact summary, `evidence_refs`, open questions, + and a terminal `target_state` of `analyzed`, `blocked`, or `skipped`. +8. Update the run counters after the target completes: + `iterations_completed += 1`; set `targets_discovered` to the current target + count; reset `non_expanding_iterations` to zero when the iteration added a + useful claim or child target, otherwise increment it. + +For list-valued fields such as `evidence_refs` and open questions, send the +complete replacement list when updating because item data updates shallow-merge. + +## Ground claims in shared taxonomy + +Vulnerability `claim_category` records must carry a standard anchor so the +ledger speaks the vocabulary practitioners and reports use, not invented terms: + +- Set the claim's `weakness` to the [CWE](https://cwe.mitre.org/) id (e.g. + `CWE-89` for SQL injection) and, where the code is a web surface, the relevant + [OWASP Top 10](https://owasp.org/Top10/) or + [OWASP API Top 10](https://owasp.org/API-Security/editions/2023/en/0x11-t10/) + category. +- Set the claim's `severity` from a [CVSS](https://www.first.org/cvss/)-style + reading of impact and exploitability (`critical`/`high`/`medium`/`low`/`info`), + justified by the recorded evidence rather than asserted. + +## Promote findings + +Create a built-in `finding` only after the vulnerability claim has: + +- an attacker-controlled source or realistic precondition, +- a concrete path to impact, +- file:line evidence for the relevant controls and sink, +- an explicit attempt to disprove the issue, +- a `weakness` (CWE / OWASP) anchor and a CVSS-aligned `severity`, +- deployment assumptions and residual uncertainty. + +The schema rejects a verified claim without `evidence_refs`; a verified +vulnerability also requires `weakness`, `severity`, and `impact`. + +When promoting, carry the claim's `weakness` into the finding's `category` and +its `severity` into the finding's `severity` so the standard taxonomy survives. +Keep weaker observations as claims. Link a promoted finding to the verified +claim with `derived_from` and to the affected target with `affects`. + +## Stop and report + +Stop when any configured condition holds: + +- the frontier is exhausted, +- the maximum target count or depth is reached, +- three consecutive targets add no valuable claim or child target, +- remaining work is blocked on unavailable tooling or user authorization. + +Call `analysis_progress` and stop only when `scan.complete` and `stop_ready` are +both true, or when tooling/user authorization blocks progress. Update +`run_state` and `stop_reason`, then summarize: +revision and scope, target-state counts, verified/refuted/unresolved claims, +promoted findings, and the highest-priority remaining targets. A later session +must be able to continue from those records alone. + +## Boundaries + +- Use Items for the active frontier and durable conclusions, not as a complete + AST or callgraph database. +- This is a single-agent loop. Claiming a target is not atomic; do not run two + sessions against the same run. +- Platform connectivity, an active project, and an item-read grant are required + for durable reads; item-write is also required for durable updates. Browse, + read, and search with the built-in + `list_items`, `read_item`, and `search_items`; use `analysis_next_targets` and + `analysis_progress` for frontier control. +- The frontier tools ask server search to narrow candidates by `run_ref`, then + exact-filter and scan at most 1,000 records of a type. They refuse to select + or stop when that candidate scan is incomplete. diff --git a/capabilities/analysis-ledger/skills/recursive-code-analysis/agents/openai.yaml b/capabilities/analysis-ledger/skills/recursive-code-analysis/agents/openai.yaml new file mode 100644 index 0000000..b4ef694 --- /dev/null +++ b/capabilities/analysis-ledger/skills/recursive-code-analysis/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Recursive Code Analysis" + short_description: "Drive a durable single-agent code analysis loop" + default_prompt: "Use $recursive-code-analysis to inspect a large codebase through a persistent analysis frontier." diff --git a/capabilities/analysis-ledger/tests/test_analysis_ledger.py b/capabilities/analysis-ledger/tests/test_analysis_ledger.py new file mode 100644 index 0000000..464d4a1 --- /dev/null +++ b/capabilities/analysis-ledger/tests/test_analysis_ledger.py @@ -0,0 +1,445 @@ +import asyncio +from hashlib import sha256 +import importlib.util +import sys +import typing as t +from pathlib import Path +from types import SimpleNamespace + +import jsonschema +import pytest +import yaml + +from dreadnode.items.config import selected_builtin_item_types +from dreadnode.packaging.manifest import CapabilityManifest +from dreadnode.tools.report_items import build_capability_report_item + +CAPABILITY_ROOT = Path(__file__).parents[1] + + +def _load_module(name: str, relative_path: str) -> t.Any: + path = CAPABILITY_ROOT / relative_path + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +class FakeApi: + """Stand-in ApiClient serving the built-in list_items surface per item type.""" + + def __init__(self, by_type: dict[str, list[dict[str, t.Any]]]) -> None: + self.by_type = by_type + self.calls: list[dict[str, t.Any]] = [] + + def list_items( + self, + org: str, + workspace: str, + project: str, + **params: t.Any, + ) -> dict[str, t.Any]: + self.calls.append(params) + items = self.by_type.get(params.get("item_type"), []) + if params.get("ref") is not None: + items = [item for item in items if item.get("ref") == params["ref"]] + return {"items": items, "total": len(items), "has_next": False} + + +def _run(ref: str = "run-auth") -> dict[str, t.Any]: + return { + "id": f"id-{ref}", + "ref": ref, + "item_type": "analysis_run", + "title": "Authentication analysis", + "data": { + "title": "Authentication analysis", + "run_key": "0123456789abcdef01234567", + "objective": "Find authentication bypasses", + "repository": "/repo", + "revision": "deadbeef", + "run_state": "running", + "max_targets": 50, + "max_depth": 8, + "iterations_completed": 2, + "non_expanding_iterations": 0, + "targets_discovered": 3, + }, + } + + +def _target( + ref: str, + *, + run_ref: str = "run-auth", + state: str = "queued", + priority: int = 50, + depth: int = 0, +) -> dict[str, t.Any]: + return { + "id": f"id-{ref}", + "ref": ref, + "item_type": "analysis_target", + "title": ref, + "data": { + "title": ref, + "run_ref": run_ref, + "target_key": f"src/auth.py::{ref}", + "kind": "function", + "location": f"src/auth.py::{ref}", + "priority": priority, + "rationale": "Authentication boundary", + "target_state": state, + "depth": depth, + }, + "updated_at": "2026-07-15T12:00:00Z", + } + + +def _claim(run_ref: str, disposition: str) -> dict[str, t.Any]: + return { + "item_type": "analysis_claim", + "data": {"run_ref": run_ref, "disposition": disposition}, + } + + +def test_item_schemas_reject_unknown_fields() -> None: + items = _load_module("analysis_ledger_models", "items.py") + run_material = "/repo\ndeadbeef\nFind authentication bypasses" + run_key = sha256(run_material.encode()).hexdigest()[:24] + run = items.AnalysisRun.model_validate( + { + "title": "Authentication analysis", + "run_key": run_key, + "objective": "Find authentication bypasses", + "repository": "/repo", + "revision": "deadbeef", + } + ) + assert run.run_key == run_key + + with pytest.raises(ValueError, match="run_key must be"): + items.AnalysisRun.model_validate( + { + "title": "Authentication analysis", + "run_key": "0123456789abcdef01234567", + "objective": "Find authentication bypasses", + "repository": "/repo", + "revision": "deadbeef", + } + ) + + target = items.AnalysisTarget.model_validate( + { + "title": "Validate login", + "run_ref": "run-auth", + "target_key": "src/auth.py::login", + "kind": "function", + "location": "src/auth.py:40", + "rationale": "Externally reachable authentication path", + } + ) + assert target.target_state == "queued" + assert target.priority == 50 + + claim = items.AnalysisClaim.model_validate( + { + "title": "SQLi in login", + "run_ref": "run-auth", + "target_ref": "run-auth-target-login", + "claim_category": "vulnerability", + "statement": "Login query concatenates user input.", + "weakness": "CWE-89", + "severity": "high", + "evidence_refs": ["src/auth.py:42"], + } + ) + assert claim.weakness == "CWE-89" + assert claim.severity == "high" + + with pytest.raises(ValueError, match="evidence_refs"): + items.AnalysisClaim.model_validate( + { + "title": "Verified token validation", + "run_ref": "run-auth", + "target_ref": "run-auth-target-login", + "claim_category": "behavior", + "statement": "Tokens are signed.", + "disposition": "verified", + } + ) + + with pytest.raises(ValueError): + items.AnalysisClaim.model_validate( + { + "title": "Token validation", + "run_ref": "run-auth", + "target_ref": "run-auth-target-login", + "claim_category": "behavior", + "statement": "Tokens are signed.", + "unknown": True, + } + ) + + +def test_exported_json_schemas_enforce_terminal_invariants() -> None: + items = _load_module("analysis_ledger_json_schemas", "items.py") + + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate( + { + "title": "Authentication analysis", + "run_key": "0123456789abcdef01234567", + "objective": "Find authentication bypasses", + "repository": "/repo", + "revision": "deadbeef", + "run_state": "completed", + }, + items.AnalysisRun.model_json_schema(), + ) + + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate( + { + "title": "Verified SQL injection", + "run_ref": "run-auth", + "target_ref": "run-auth-target-login", + "claim_category": "vulnerability", + "statement": "User input reaches the query.", + "disposition": "verified", + "evidence_refs": ["src/auth.py:42"], + }, + items.AnalysisClaim.model_json_schema(), + ) + + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate( + { + "title": "Analyze login", + "run_ref": "run-auth", + "target_key": "src/auth.py::login", + "kind": "function", + "location": "src/auth.py:40", + "rationale": "Externally reachable authentication path", + "target_state": "analyzed", + "summary": "No bypass found.", + }, + items.AnalysisTarget.model_json_schema(), + ) + + +def test_manifest_builds_combined_report_item_schema() -> None: + manifest_data = yaml.safe_load((CAPABILITY_ROOT / "capability.yaml").read_text()) + manifest = CapabilityManifest.model_validate(manifest_data) + capability = SimpleNamespace(path=CAPABILITY_ROOT, manifest=manifest) + + report_tool = build_capability_report_item( + capability, + builtin_types=selected_builtin_item_types(manifest), + ) + + assert report_tool is not None + item_types = report_tool.parameters_schema["properties"]["item_type"]["enum"] + assert set(item_types) == { + "finding", + "analysis_run", + "analysis_target", + "analysis_claim", + } + properties = report_tool.parameters_schema["properties"] + assert properties["run_state"]["enum"] == [ + "planned", + "running", + "paused", + "completed", + "exhausted", + "stale", + ] + assert properties["target_state"]["enum"] == [ + "queued", + "in-progress", + "analyzed", + "blocked", + "skipped", + "stale", + ] + assert properties["claim_category"]["enum"] == [ + "behavior", + "trust-boundary", + "vulnerability", + "assumption", + "coverage-gap", + ] + assert properties["evidence_refs"]["type"] == "array" + + +def test_next_targets_uses_stable_best_first_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("analysis_ledger_tools_next", "tools/analysis_items.py") + api = FakeApi( + { + "analysis_run": [_run()], + "analysis_target": [ + _target("target-low", priority=20), + _target("target-deep", priority=90, depth=3), + _target("target-shallow", priority=90, depth=1), + _target("target-done", state="analyzed", priority=100), + ], + } + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.analysis_next_targets.fn(run_ref="run-auth", limit=3)) + + assert [target["ref"] for target in result["targets"]] == [ + "target-shallow", + "target-deep", + "target-low", + ] + assert result["eligible_in_scan"] == 3 + assert result["selection_complete"] is True + assert api.calls[0]["item_type"] == "analysis_run" + assert api.calls[1]["item_type"] == "analysis_target" + assert api.calls[1]["q"] == "run-auth" + + +def test_progress_counts_target_and_claim_states( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("analysis_ledger_tools_progress", "tools/analysis_items.py") + api = FakeApi( + { + "analysis_run": [_run()], + "analysis_target": [ + _target("target-queued", priority=80), + _target("target-done", state="analyzed"), + _target("target-other", run_ref="other-run"), + ], + "analysis_claim": [ + _claim("run-auth", "verified"), + _claim("run-auth", "refuted"), + ], + } + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.analysis_progress.fn(run_ref="run-auth")) + + assert result["targets"]["by_state"] == {"analyzed": 1, "queued": 1} + assert result["targets"]["highest_queued_priority"] == 80 + assert result["claims"]["by_disposition"] == {"refuted": 1, "verified": 1} + assert result["frontier_exhausted"] is False + assert result["scan"]["complete"] is True + assert result["stop_ready"] is False + + +def test_next_targets_recovers_in_progress_before_queued( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("analysis_ledger_tools_recovery", "tools/analysis_items.py") + api = FakeApi( + { + "analysis_run": [_run()], + "analysis_target": [ + _target("target-queued", priority=100), + _target("target-resume", state="in-progress", priority=10), + ], + } + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.analysis_next_targets.fn(run_ref="run-auth")) + + assert [target["ref"] for target in result["targets"]] == ["target-resume"] + assert result["targets"][0]["target_state"] == "in-progress" + assert result["resuming_in_progress"] == 1 + + +def test_missing_run_is_not_reported_as_exhausted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("analysis_ledger_tools_missing_run", "tools/analysis_items.py") + api = FakeApi({}) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + with pytest.raises(ValueError, match="was not found"): + asyncio.run(tools.analysis_progress.fn(run_ref="missing-run")) + + +def test_unseeded_run_is_not_reported_as_exhausted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("analysis_ledger_tools_unseeded", "tools/analysis_items.py") + run = _run() + run["data"]["targets_discovered"] = 0 + api = FakeApi( + { + "analysis_run": [run], + "analysis_target": [], + "analysis_claim": [], + } + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.analysis_progress.fn(run_ref="run-auth")) + + assert result["initialized"] is False + assert result["frontier_exhausted"] is False + assert result["stop_ready"] is False + + +def test_truncated_scan_never_selects_or_stops( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("analysis_ledger_tools_truncated", "tools/analysis_items.py") + monkeypatch.setattr(tools, "_MAX_SCAN", 2) + api = FakeApi( + { + "analysis_run": [_run()], + "analysis_target": [ + _target("target-one", priority=100), + _target("target-two", priority=90), + _target("target-three", priority=80), + ], + "analysis_claim": [], + } + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + selected = asyncio.run(tools.analysis_next_targets.fn(run_ref="run-auth")) + progress = asyncio.run(tools.analysis_progress.fn(run_ref="run-auth")) + + assert selected["targets"] == [] + assert selected["selection_complete"] is False + assert selected["scan_truncated"] is True + assert progress["frontier_exhausted"] is False + assert progress["stop_ready"] is False + assert progress["scan"]["complete"] is False diff --git a/capabilities/analysis-ledger/tools/analysis_items.py b/capabilities/analysis-ledger/tools/analysis_items.py new file mode 100644 index 0000000..7f246f1 --- /dev/null +++ b/capabilities/analysis-ledger/tools/analysis_items.py @@ -0,0 +1,367 @@ +"""Frontier-control reads over the built-in Agent Output read surface. + +Generic browsing, reading, and full-text search are provided by the platform's +built-in `list_items`, `read_item`, and `search_items` tools. This module adds +only the two frontier-control views those cannot express: best-first target +selection (ordered by the run's custom `priority`/`depth` fields) and a run +progress roll-up over target states and claim dispositions. +""" + +import asyncio +import collections +import typing as t + +from dreadnode import get_default_instance +from dreadnode.agents.tools import tool + +_PAGE_SIZE = 100 +_MAX_SCAN = 1_000 + + +def _platform_context() -> tuple[t.Any, str, str, str]: + instance = get_default_instance() + if not instance.can_sync: + raise RuntimeError( + "Analysis Ledger reads require a platform-connected runtime with an active project" + ) + + profile = instance.profile + project = profile.project_key or profile.project_id + if project is None: + raise RuntimeError("Analysis Ledger reads require an active project") + return instance.api, profile.org_key, profile.workspace_key, str(project) + + +def _has_value(value: t.Any) -> bool: + return value is not None and value != "" and value != [] and value != {} + + +def _clean(values: dict[str, t.Any]) -> dict[str, t.Any]: + return {key: value for key, value in values.items() if _has_value(value)} + + +def _data(item: dict[str, t.Any]) -> dict[str, t.Any]: + value = item.get("data") + return value if isinstance(value, dict) else {} + + +def _clip(value: t.Any, limit: int) -> t.Any: + if not isinstance(value, str) or len(value) <= limit: + return value + return value[: limit - 1] + "…" + + +def _target_summary(item: dict[str, t.Any]) -> dict[str, t.Any]: + data = _data(item) + return _clean( + { + "id": item.get("id"), + "ref": _clip(item.get("ref"), 128), + "item_type": item.get("item_type") or "analysis_target", + "title": _clip(item.get("title") or data.get("title"), 512), + "run_ref": _clip(data.get("run_ref"), 128), + "target_key": _clip(data.get("target_key"), 512), + "kind": data.get("kind"), + "location": _clip(data.get("location"), 512), + "priority": data.get("priority"), + "target_state": data.get("target_state"), + "depth": data.get("depth"), + "parent_ref": _clip(data.get("parent_ref"), 128), + "summary": _clip(data.get("summary"), 1_000), + "updated_at": item.get("updated_at"), + } + ) + + +async def _scan_items( + api: t.Any, + org: str, + workspace: str, + project: str, + item_type: str, + *, + query: str, +) -> tuple[list[dict[str, t.Any]], int, int, bool]: + """Page one analysis item type through the built-in list surface. + + Server-side filtering covers `item_type`; the run/state selectors live in the + item ``data`` (not indexed columns). Full-text query narrows candidates by + run ref; callers still apply an exact in-memory check. + """ + records: list[dict[str, t.Any]] = [] + scanned_candidates = 0 + page = 1 + candidate_total = 0 + while scanned_candidates < _MAX_SCAN: + payload = await asyncio.to_thread( + api.list_items, + org, + workspace, + project, + item_type=item_type, + q=query, + page=page, + limit=_PAGE_SIZE, + ) + raw_items = ( + payload.get("items") if isinstance(payload.get("items"), list) else [] + ) + page_items = [item for item in raw_items if isinstance(item, dict)] + candidates = page_items[: _MAX_SCAN - scanned_candidates] + scanned_candidates += len(candidates) + records.extend(candidates) + raw_total = payload.get("total") + candidate_total = ( + raw_total if isinstance(raw_total, int) else scanned_candidates + ) + has_next = payload.get("has_next") + if ( + not page_items + or len(page_items) < _PAGE_SIZE + or (isinstance(has_next, bool) and not has_next) + or page * _PAGE_SIZE >= candidate_total + ): + break + page += 1 + return ( + records, + scanned_candidates, + candidate_total, + candidate_total > scanned_candidates, + ) + + +async def _require_run( + api: t.Any, + org: str, + workspace: str, + project: str, + run_ref: str, +) -> dict[str, t.Any]: + payload = await asyncio.to_thread( + api.list_items, + org, + workspace, + project, + item_type="analysis_run", + ref=run_ref, + page=1, + limit=2, + ) + raw_items = payload.get("items") if isinstance(payload, dict) else None + if not isinstance(raw_items, list): + raw_items = [] + items = [item for item in raw_items if isinstance(item, dict)] + exact = [item for item in items if item.get("ref") == run_ref] + if not exact: + raise ValueError( + f"analysis_run ref '{run_ref}' was not found or is not readable" + ) + if len(exact) > 1: + raise RuntimeError(f"analysis_run ref '{run_ref}' is not unique") + return exact[0] + + +def _target_sort_key(item: dict[str, t.Any]) -> tuple[int, int, int, str]: + data = _data(item) + state_rank = 0 if data.get("target_state") == "in-progress" else 1 + raw_priority = data.get("priority", 0) + raw_depth = data.get("depth", 0) + priority = raw_priority if isinstance(raw_priority, int) else 0 + depth = raw_depth if isinstance(raw_depth, int) else 0 + stable_ref = str(item.get("ref") or data.get("target_key") or item.get("id") or "") + return state_rank, -priority, depth, stable_ref + + +@tool(name="analysis_next_targets", truncate=12000) +async def analysis_next_targets( + run_ref: t.Annotated[str, "Owning analysis_run ref."], + limit: t.Annotated[ + int, + "In-progress recovery target or highest-priority queued targets to return (1-5).", + ] = 1, + min_priority: t.Annotated[int, "Minimum target priority (0-100)."] = 0, +) -> dict[str, t.Any]: + """Resume in-progress work, otherwise select queued targets deterministically. + + `list_items` sorts only by created_at/severity/status, so best-first ordering + over the run's custom priority/depth fields is computed here. In-progress + targets sort first so a fresh single-agent session can recover after a crash. + """ + if limit < 1 or limit > 5: + raise ValueError("limit must be between 1 and 5") + if min_priority < 0 or min_priority > 100: + raise ValueError("min_priority must be between 0 and 100") + + api, org, workspace, project = _platform_context() + await _require_run(api, org, workspace, project, run_ref) + records, scanned_candidates, candidate_total, scan_truncated = await _scan_items( + api, + org, + workspace, + project, + "analysis_target", + query=run_ref, + ) + eligible = [] + for item in records: + data = _data(item) + priority = data.get("priority", 0) + if ( + data.get("run_ref") == run_ref + and data.get("target_state") in {"in-progress", "queued"} + and isinstance(priority, int) + and priority >= min_priority + ): + eligible.append(item) + eligible.sort(key=_target_sort_key) + selected = [] if scan_truncated else eligible[:limit] + return { + "run_ref": run_ref, + "targets": [_target_summary(item) for item in selected], + "resuming_in_progress": sum( + 1 for item in selected if _data(item).get("target_state") == "in-progress" + ), + "eligible_in_scan": len(eligible), + "candidates_scanned": scanned_candidates, + "candidate_platform_total": candidate_total, + "selection_complete": not scan_truncated, + "scan_truncated": scan_truncated, + } + + +@tool(name="analysis_progress", truncate=12000) +async def analysis_progress( + run_ref: t.Annotated[str, "Owning analysis_run ref."], +) -> dict[str, t.Any]: + """Summarize target and claim state for one analysis run. + + Aggregates by the custom `data.target_state` and claim `disposition` fields, which + the built-in facets (severity/status/category/capability/source) do not + expose. + """ + api, org, workspace, project = _platform_context() + run = await _require_run(api, org, workspace, project, run_ref) + run_data = _data(run) + ( + targets, + target_candidates_scanned, + target_candidate_total, + target_truncated, + ) = await _scan_items( + api, + org, + workspace, + project, + "analysis_target", + query=run_ref, + ) + ( + claims, + claim_candidates_scanned, + claim_candidate_total, + claim_truncated, + ) = await _scan_items( + api, + org, + workspace, + project, + "analysis_claim", + query=run_ref, + ) + run_targets = [item for item in targets if _data(item).get("run_ref") == run_ref] + run_claims = [item for item in claims if _data(item).get("run_ref") == run_ref] + target_states = collections.Counter( + str(_data(item).get("target_state", "unknown")) for item in run_targets + ) + claim_states = collections.Counter( + str(_data(item).get("disposition", "unknown")) for item in run_claims + ) + queued_priorities = [ + priority + for item in run_targets + if _data(item).get("target_state") == "queued" + for priority in [_data(item).get("priority")] + if isinstance(priority, int) + ] + observed_depths = [ + depth + for item in run_targets + for depth in [_data(item).get("depth")] + if isinstance(depth, int) + ] + progress_complete = not target_truncated and not claim_truncated + recorded_targets_discovered = run_data.get("targets_discovered", 0) + initialized = bool(run_targets) or ( + isinstance(recorded_targets_discovered, int) and recorded_targets_discovered > 0 + ) + frontier_exhausted = ( + initialized + and progress_complete + and not target_states.get("queued") + and not target_states.get("in-progress") + ) + max_targets = run_data.get("max_targets", 50) + target_budget_reached = ( + progress_complete + and isinstance(max_targets, int) + and len(run_targets) >= max_targets + ) + non_expanding_iterations = run_data.get("non_expanding_iterations", 0) + non_expanding_stop = ( + initialized + and isinstance(non_expanding_iterations, int) + and non_expanding_iterations >= 3 + ) + stop_reasons = [] + if frontier_exhausted: + stop_reasons.append("frontier_exhausted") + if target_budget_reached: + stop_reasons.append("target_budget_reached") + if non_expanding_stop: + stop_reasons.append("non_expanding_limit_reached") + return { + "run_ref": run_ref, + "initialized": initialized, + "run": _clean( + { + "run_key": run_data.get("run_key"), + "run_state": run_data.get("run_state"), + "max_targets": max_targets, + "max_depth": run_data.get("max_depth"), + "iterations_completed": run_data.get("iterations_completed"), + "non_expanding_iterations": non_expanding_iterations, + "targets_discovered": recorded_targets_discovered, + "stop_reason": run_data.get("stop_reason"), + } + ), + "targets": { + "total_in_scan": len(run_targets), + "by_state": dict(sorted(target_states.items())), + **_clean( + { + "highest_queued_priority": max( + queued_priorities, + default=None, + ), + "max_observed_depth": max(observed_depths, default=None), + } + ), + }, + "claims": { + "total_in_scan": len(run_claims), + "by_disposition": dict(sorted(claim_states.items())), + }, + "frontier_exhausted": frontier_exhausted, + "target_budget_reached": target_budget_reached, + "stop_ready": initialized and progress_complete and bool(stop_reasons), + "stop_reasons": stop_reasons, + "scan": { + "target_candidates_scanned": target_candidates_scanned, + "target_candidate_platform_total": target_candidate_total, + "claim_candidates_scanned": claim_candidates_scanned, + "claim_candidate_platform_total": claim_candidate_total, + "complete": progress_complete, + "truncated": target_truncated or claim_truncated, + }, + } diff --git a/capabilities/llm-wiki/README.md b/capabilities/llm-wiki/README.md new file mode 100644 index 0000000..d47744b --- /dev/null +++ b/capabilities/llm-wiki/README.md @@ -0,0 +1,96 @@ +# LLM Wiki + +LLM Wiki adapts [Andrej Karpathy's LLM Wiki +pattern](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) +to Dreadnode Agent Output. Instead of maintaining a directory of Markdown +pages, the agent compiles sources into typed, linked `wiki_page` items that can +be read and revised across turns and sessions. + +## What it demonstrates + +| LLM Wiki operation | Capability implementation | +| --- | --- | +| Index pages | Built-in `list_items` (`item_type="wiki_page"`) | +| Find a page | Built-in `search_items` full-text query | +| Read a page | Built-in `read_item` by ref or UUID | +| Create a page | Typed `report_item` call | +| Revise a page | `update_item` | +| Wikilinks | `link_items` relationships | +| Bounded coverage | `wiki_progress` counts and complete-scan stop signal | +| Ingest, query, lint | `llm-wiki` skill and agent | + +The platform now provides the read surface directly: browsing, reading, and +full-text search are built-in tools (`list_items`, `read_item`, `search_items`) +that every agent in a platform-connected session receives. This capability +supplies the typed `wiki_page` schema, the operating skill/agent, and one +wiki-specific loop signal (`wiki_progress`) the generic tools cannot express. + +## Try it + +This capability requires: + +- Dreadnode SDK 2.0.38 or newer, +- a configured model/provider, +- a platform-connected profile with an active project and effective + `items:read` plus `items:write` grants. + +> **Select a project first.** The built-in reads query, and writes land in, your +> profile's active project, and require item-read and item-write grants. If none +> is selected the tools raise `requires ... an active project` mid-run, not at +> launch. Pick one in the TUI project picker (it's shown in the status bar) +> before starting. + +```bash +dn capability install dreadnode/llm-wiki +dn --capability llm-wiki --agent llm-wiki +``` + +Example prompts: + +```text +Ingest https://example.com/article into the wiki. Use wiki_id example-article +and preserve claims and source evidence. + +Build a structured wiki describing the architecture and trust boundaries in /path/to/repo. + +What does the wiki say about authentication? Cite the evidence stored on relevant pages. + +Lint the wiki for contradictions, unsupported claims, duplicate pages, and open questions. +``` + +Use `/auto 100` for a bounded autonomous ingest or maintenance pass. The agent +polls `wiki_progress(wiki_id=...)` between sweeps and stops only after a complete +scan shows open questions cleared and a sweep makes no repairs. Pages +are found with the built-in `search_items` (full-text) and `list_items`, so +large wikis stay navigable without loading the whole corpus. + +## Where the wiki lives + +The pages are `wiki_page` records in the **active platform project**, not files +on disk — that is what makes them outlive a context window and a session: + +- **In the app.** Pages produced during a session appear under that session's + Output tab and on the project's Agent Output page; the TUI prints a clickable + link beneath each write. This is where the session owner reviews the result + and chooses whether to share it. +- **Across sessions.** A later session owned by the same user reads the same + pages by `ref` and keeps compiling onto them. Another teammate can read them + only after the originating session is promoted from private to workspace + visibility. + +Every page carries a `wiki_id`; refs remain project-unique and should also use a +wiki-specific prefix. + +## Current limitations + +- Reads and direct writes require platform connectivity and item-read/item-write + grants. A trace-backed write is not readable until materialization completes. +- Item refs are project-unique; independent wikis need distinct `wiki_id` values + and ref prefixes. +- `wiki_progress` scans up to 1,000 `wiki_page` records to aggregate kinds and + open questions; very large wikis report the scan as truncated. +- Direct item writes must succeed to be immediately readable. Platform + trace-backed materialization runs at session freeze where supported, but is + eventual and must not be used for same-loop read/link dependencies. +- This is a single-agent knowledge-maintenance pattern, not a concurrent work + queue or deterministic workflow controller. diff --git a/capabilities/llm-wiki/agents/llm-wiki.md b/capabilities/llm-wiki/agents/llm-wiki.md new file mode 100644 index 0000000..551d7d7 --- /dev/null +++ b/capabilities/llm-wiki/agents/llm-wiki.md @@ -0,0 +1,45 @@ +--- +name: llm-wiki +description: Builds, maintains, and queries a persistent, typed, interlinked LLM Wiki in Dreadnode Agent Output +model: inherit +tools: + "*": true + spawn_agent: false +skills: [llm-wiki] +--- + +You maintain an **LLM Wiki**: a persistent knowledge base whose pages are typed +Agent Output records instead of Markdown files. Load the `llm-wiki` skill for +the operating procedure. + +## Operating model + +- Treat source documents, URLs, repositories, and artifacts as immutable evidence. +- Treat `wiki_page` items as the maintained compilation of that evidence. +- Resolve one stable lowercase kebab-case `wiki_id` for the requested corpus and + keep every read, write, progress check, and answer inside that namespace. +- Use the built-in `list_items` (`item_type="wiki_page"`) and `search_items` as + the index, and `read_item` to open only relevant pages; never load the whole + wiki into context without a reason. +- Use `wiki_progress(wiki_id=...)` for a bounded coverage snapshot. Stop an + autonomous maintenance loop only when `complete_scan` is true. +- Create pages with `report_item`, revise them with `update_item`, and connect + them with `link_items`. +- Assign every page a stable lowercase kebab-case `ref`. Read before creating so + the same subject is updated instead of duplicated. +- Create linked pages sequentially. Never issue concurrent `report_item` calls + when one new page links to the other; wait until both refs are readable, then + use `link_items`. +- Keep claims evidence-bearing. Preserve disagreements and uncertainty rather + than forcing false consensus. +- Finish only after the requested ingest, query, or lint operation is complete. + +## First response + +Briefly introduce LLM Wiki and ask the user for one of: + +- a source to ingest, +- a question to answer from the existing wiki, or +- a wiki lint/maintenance pass. + +If the user already supplied one, begin immediately instead of asking again. diff --git a/capabilities/llm-wiki/capability.yaml b/capabilities/llm-wiki/capability.yaml new file mode 100644 index 0000000..3ff727a --- /dev/null +++ b/capabilities/llm-wiki/capability.yaml @@ -0,0 +1,33 @@ +schema: 1 +name: llm-wiki +version: "0.1.0" +description: > + Persistent LLM-maintained wiki built on Dreadnode Agent Output. Agents compile + sources into typed, linked wiki_page items, then browse, read, revise, query, + and lint that accumulated knowledge across turns and sessions. + +agents: + - agents/ + +skills: + - skills/ + +tools: + - tools/ + +produces: + wiki_page: "items:WikiPage" + +author: + name: Dreadnode + url: https://dreadnode.io +license: MIT +repository: https://github.com/dreadnode/capabilities +keywords: + - agent-output + - knowledge-base + - knowledge-management + - llm-wiki + - research + - structured-output + - wiki diff --git a/capabilities/llm-wiki/items.py b/capabilities/llm-wiki/items.py new file mode 100644 index 0000000..e16f141 --- /dev/null +++ b/capabilities/llm-wiki/items.py @@ -0,0 +1,93 @@ +"""Typed Agent Output records for the LLM Wiki capability.""" + +import typing as t + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints + +WikiPageKind = t.Literal[ + "overview", + "source-summary", + "entity", + "concept", + "comparison", + "synthesis", +] +ClaimConfidence = t.Literal["high", "medium", "low"] +WikiId = t.Annotated[ + str, + StringConstraints( + min_length=1, + max_length=64, + pattern=r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$", + ), +] +NonEmptyText = t.Annotated[str, StringConstraints(min_length=1)] +Tag = t.Annotated[ + str, + StringConstraints( + min_length=1, + max_length=64, + pattern=r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$", + ), +] + + +class WikiClaim(BaseModel): + """One evidence-bearing claim compiled into a wiki page.""" + + model_config = ConfigDict(extra="forbid") + + statement: str = Field( + ..., + min_length=1, + description="A concise factual or analytical claim.", + ) + evidence: list[NonEmptyText] = Field( + ..., + min_length=1, + description="Source URLs, file:line locations, quotations, or artifact references.", + ) + confidence: ClaimConfidence = Field( + default="medium", + description="Confidence justified by the available evidence.", + ) + + +class WikiPage(BaseModel): + """A maintained unit of structured knowledge in the LLM Wiki.""" + + model_config = ConfigDict(extra="forbid") + + wiki_id: WikiId = Field( + description=( + "Stable namespace for one independent wiki in a shared project. " + "Use the same lowercase kebab-case value for every page in that wiki." + ) + ) + title: str = Field(..., min_length=1, max_length=512) + kind: WikiPageKind = Field(description="The page's role in the wiki.") + summary: str = Field( + ..., + min_length=1, + description="A compact index description used to decide whether to read this page.", + ) + claims: list[WikiClaim] = Field( + default_factory=list, + description="Evidence-bearing facts and conclusions maintained on this page.", + ) + synthesis: str | None = Field( + default=None, + description="Higher-level interpretation derived from the claims.", + ) + open_questions: list[NonEmptyText] = Field( + default_factory=list, + description="Unresolved questions that should guide later ingestion or research.", + ) + tags: list[Tag] = Field( + default_factory=list, + description="Short discovery labels; use lowercase kebab-case.", + ) + source_refs: list[NonEmptyText] = Field( + default_factory=list, + description="Canonical source URLs, file paths, document IDs, or artifact references.", + ) diff --git a/capabilities/llm-wiki/skills/llm-wiki/SKILL.md b/capabilities/llm-wiki/skills/llm-wiki/SKILL.md new file mode 100644 index 0000000..1aa8c2c --- /dev/null +++ b/capabilities/llm-wiki/skills/llm-wiki/SKILL.md @@ -0,0 +1,97 @@ +--- +name: llm-wiki +description: Use when building, querying, revising, or linting a persistent LLM-maintained wiki from source material in Dreadnode Agent Output. +--- + +# LLM Wiki + +Compile source material into `wiki_page` records that compound across turns. +Operate on the maintained pages instead of repeatedly reconstructing knowledge +from raw sources. + +This instantiates Andrej Karpathy's +[LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) +pattern with typed Agent Output records rather than Markdown files. + +Choose one stable lowercase kebab-case `wiki_id` before reading or writing. Put +that value on every page and pass it to `wiki_progress`; it is the isolation +boundary when a project contains more than one wiki. + +## Choose the operation + +- **Ingest** when the user supplies new source material. +- **Query** when the user asks a question about accumulated knowledge. +- **Lint** when the user asks to check or improve wiki health. + +## Ingest + +1. Call `list_items` with `item_type="wiki_page"` (or `search_items` with a + topic query and `item_type="wiki_page"`) to find existing pages related to + the source. +2. Read only those pages with `read_item`. Reject candidates whose `wiki_id` + differs from the active wiki. +3. Read the raw source and extract claims with concrete evidence references. +4. Update an existing page when it covers the same subject. Create a new page + only for a durable entity, concept, comparison, overview, source summary, or + synthesis that deserves independent maintenance. +5. Use `report_item` with `item_type="wiki_page"`, the active `wiki_id`, and a + stable lowercase kebab-case `ref`. Use `update_item` for revisions. +6. Connect related pages with `link_items`. Prefer specific relationships such + as `supports`, `contradicts`, `summarizes`, `depends_on`, or `related_to`. +7. Re-read changed pages and report what was added, revised, or contradicted. + +Never issue dependent page creates in the same model turn. When one new page +must link to another, create and re-read the target first, create the source +without an inline link, then call `link_items` after both refs are readable. +Trace-backed reconciliation is eventual and cannot satisfy a same-turn link +dependency. + +When updating `claims`, `tags`, `source_refs`, or `open_questions`, send the +complete replacement list for that field because item updates shallow-merge. + +## Query + +1. Find candidate pages with `search_items` (full-text over titles, summaries, + and claims) or browse `list_items`, always with `item_type="wiki_page"`. +2. Read the smallest relevant set of pages with `read_item`, discard pages from + another `wiki_id`, and follow useful links. +3. Answer from their claims and evidence. Distinguish direct evidence from your + synthesis and state unresolved uncertainty. +4. If the answer creates durable new synthesis, offer to file it as a page. Do + not mutate the wiki merely because it was queried. + +## Lint + +Call `wiki_progress(wiki_id=...)` for a bounded coverage snapshot, then page +through `list_items` (`item_type="wiki_page"`) and selectively read suspected +problems with `read_item`. Check for: + +- duplicate pages covering the same subject, +- claims that conflict without a `contradicts` link, +- claims lacking evidence, +- stale summaries after claim updates, +- orphan pages with no useful relationships, +- open questions now answered elsewhere, +- important recurring concepts without their own page. + +Apply unambiguous repairs. Present uncertain merges, deletions, or substantive +reinterpretations to the user before changing them. + +In an autonomous `/auto` maintenance pass, re-check `wiki_progress` between +sweeps and stop only when `complete_scan` and `open_questions_cleared` both hold +and a sweep applies no new repairs. Do not loop on a clean wiki merely because +steps remain in the budget. + +## Boundaries + +- Platform connectivity, an active project, and item-read plus item-write grants + are required; + browsing, reading, and search come from the built-in `list_items`, + `read_item`, and `search_items` tools. +- Treat raw sources as immutable; only maintain the compiled `wiki_page` items. +- Keep `list_items`/`search_items` calls compact and paginated; open full pages + with `read_item` only when needed. Never use full payloads as an index. +- Keep `wiki_id` and ref prefixes stable when maintaining multiple independent + wikis in one project. +- This capability has no atomic work claiming; do not present it as a + concurrent queue. diff --git a/capabilities/llm-wiki/skills/llm-wiki/agents/openai.yaml b/capabilities/llm-wiki/skills/llm-wiki/agents/openai.yaml new file mode 100644 index 0000000..14a2f94 --- /dev/null +++ b/capabilities/llm-wiki/skills/llm-wiki/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "LLM Wiki" + short_description: "Maintain a typed, linked LLM Wiki in Agent Output" + default_prompt: "Build or query a persistent LLM Wiki using structured Agent Output pages." diff --git a/capabilities/llm-wiki/tests/test_llm_wiki.py b/capabilities/llm-wiki/tests/test_llm_wiki.py new file mode 100644 index 0000000..310e8e6 --- /dev/null +++ b/capabilities/llm-wiki/tests/test_llm_wiki.py @@ -0,0 +1,217 @@ +import asyncio +import importlib.util +import sys +import typing as t +from pathlib import Path + +import pytest + +CAPABILITY_ROOT = Path(__file__).parents[1] + + +def _load_module(name: str, relative_path: str) -> t.Any: + path = CAPABILITY_ROOT / relative_path + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +class FakeApi: + """Stand-in for the SDK ApiClient exposing the built-in list_items surface.""" + + def __init__(self, pages: list[dict[str, t.Any]]) -> None: + self.pages = list(pages) + self.calls: list[dict[str, t.Any]] = [] + + def list_items( + self, + org: str, + workspace: str, + project: str, + **params: t.Any, + ) -> dict[str, t.Any]: + self.calls.append(params) + return self.pages.pop(0) + + +def test_wiki_page_schema_rejects_unknown_fields() -> None: + items = _load_module("llm_wiki_models", "items.py") + page = items.WikiPage.model_validate( + { + "wiki_id": "auth-docs", + "title": "Authentication", + "kind": "concept", + "summary": "How users authenticate.", + "claims": [ + { + "statement": "Sessions use signed cookies.", + "evidence": ["src/auth.py:42"], + "confidence": "high", + } + ], + } + ) + assert page.claims[0].confidence == "high" + + with pytest.raises(ValueError): + items.WikiPage.model_validate( + { + "wiki_id": "auth-docs", + "title": "Authentication", + "kind": "concept", + "summary": "How users authenticate.", + "unknown": True, + } + ) + + with pytest.raises(ValueError): + items.WikiPage.model_validate( + { + "wiki_id": "auth-docs", + "title": "Authentication", + "kind": "concept", + "summary": "How users authenticate.", + "claims": [{"statement": "Sessions use signed cookies."}], + } + ) + + +def test_progress_aggregates_kinds_status_and_open_questions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("llm_wiki_tools_progress", "tools/wiki_items.py") + api = FakeApi( + [ + { + "items": [ + { + "data": { + "wiki_id": "auth-docs", + "kind": "concept", + "open_questions": ["why?"], + }, + "disposition": {"status": "open"}, + }, + { + "data": { + "wiki_id": "auth-docs", + "kind": "concept", + "open_questions": [], + }, + "disposition": {"status": "verified"}, + }, + { + "data": {"wiki_id": "other-wiki", "kind": "entity"}, + "effective_status": None, + }, + ], + "page": 1, + "limit": 100, + "total": 3, + "has_next": False, + } + ] + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.wiki_progress.fn(wiki_id="auth-docs")) + + assert result["pages_in_scan"] == 2 + assert result["by_kind"] == {"concept": 2} + assert result["by_status"] == {"open": 1, "verified": 1} + assert result["open_questions"] == 1 + assert result["pages_with_open_questions"] == 1 + assert result["open_questions_cleared"] is False + assert result["complete_scan"] is True + assert result["scan_truncated"] is False + # Scans the wiki_page type through the built-in list surface, not a raw scan. + assert api.calls[0]["item_type"] == "wiki_page" + assert api.calls[0]["q"] == "auth-docs" + + +def test_progress_never_clears_questions_from_a_truncated_scan( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("llm_wiki_tools_truncated", "tools/wiki_items.py") + monkeypatch.setattr(tools, "_MAX_SCAN", 2) + api = FakeApi( + [ + { + "items": [ + { + "data": { + "wiki_id": "auth-docs", + "kind": "concept", + "open_questions": [], + } + }, + { + "data": { + "wiki_id": "auth-docs", + "kind": "entity", + "open_questions": [], + } + }, + { + "data": { + "wiki_id": "auth-docs", + "kind": "overview", + "open_questions": ["unseen"], + } + }, + ], + "page": 1, + "limit": 100, + "total": 3, + "has_next": False, + } + ] + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.wiki_progress.fn(wiki_id="auth-docs")) + + assert result["pages_in_scan"] == 2 + assert result["scan_truncated"] is True + assert result["complete_scan"] is False + assert result["open_questions_cleared"] is False + + +def test_empty_wiki_is_not_a_positive_stop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tools = _load_module("llm_wiki_tools_empty", "tools/wiki_items.py") + api = FakeApi( + [ + { + "items": [], + "page": 1, + "limit": 100, + "total": 0, + "has_next": False, + } + ] + ) + monkeypatch.setattr( + tools, + "_platform_context", + lambda: (api, "acme", "main", "demo"), + ) + + result = asyncio.run(tools.wiki_progress.fn(wiki_id="auth-docs")) + + assert result["initialized"] is False + assert result["complete_scan"] is True + assert result["open_questions_cleared"] is False diff --git a/capabilities/llm-wiki/tools/wiki_items.py b/capabilities/llm-wiki/tools/wiki_items.py new file mode 100644 index 0000000..4b9b726 --- /dev/null +++ b/capabilities/llm-wiki/tools/wiki_items.py @@ -0,0 +1,156 @@ +"""Wiki-specific loop control over the built-in Agent Output read surface. + +Browsing, reading, and full-text search are provided by the platform's built-in +`list_items`, `read_item`, and `search_items` tools, which every agent in a +platform-connected session receives. This module adds only the wiki-specific +coverage signal those generic tools cannot express: a maintenance-loop stop +condition derived from page kinds and open questions. +""" + +import asyncio +import collections +import re +import typing as t + +from dreadnode import get_default_instance +from dreadnode.agents.tools import tool + +_ITEM_TYPE = "wiki_page" +_PAGE_SIZE = 100 +_MAX_SCAN = 1_000 +_WIKI_ID_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") + + +def _platform_context() -> tuple[t.Any, str, str, str]: + instance = get_default_instance() + if not instance.can_sync: + raise RuntimeError( + "LLM Wiki reads require a platform-connected runtime with an active project" + ) + + profile = instance.profile + project = profile.project_key or profile.project_id + if project is None: + raise RuntimeError("LLM Wiki reads require an active project") + return instance.api, profile.org_key, profile.workspace_key, str(project) + + +def _data_of(item: dict[str, t.Any]) -> dict[str, t.Any]: + value = item.get("data") + return value if isinstance(value, dict) else {} + + +def _status_of(item: dict[str, t.Any]) -> str: + disposition = item.get("disposition") + if isinstance(disposition, dict) and disposition.get("status"): + return str(disposition["status"]) + status = item.get("effective_status") + return str(status) if status else "none" + + +async def _scan_wiki_pages( + api: t.Any, + org: str, + workspace: str, + project: str, + wiki_id: str, +) -> tuple[list[dict[str, t.Any]], int, int, bool]: + """Page through wiki_page records via the platform list surface for aggregates.""" + records: list[dict[str, t.Any]] = [] + scanned_candidates = 0 + page = 1 + candidate_total = 0 + while scanned_candidates < _MAX_SCAN: + payload = await asyncio.to_thread( + api.list_items, + org, + workspace, + project, + item_type=_ITEM_TYPE, + q=wiki_id, + page=page, + limit=_PAGE_SIZE, + ) + raw_items = ( + payload.get("items") if isinstance(payload.get("items"), list) else [] + ) + page_items = [item for item in raw_items if isinstance(item, dict)] + candidates = page_items[: _MAX_SCAN - scanned_candidates] + scanned_candidates += len(candidates) + records.extend( + item for item in candidates if _data_of(item).get("wiki_id") == wiki_id + ) + raw_total = payload.get("total") + candidate_total = ( + raw_total if isinstance(raw_total, int) else scanned_candidates + ) + has_next = payload.get("has_next") + if ( + not page_items + or len(page_items) < _PAGE_SIZE + or (isinstance(has_next, bool) and not has_next) + or page * _PAGE_SIZE >= candidate_total + ): + break + page += 1 + return ( + records, + scanned_candidates, + candidate_total, + candidate_total > scanned_candidates, + ) + + +@tool(name="wiki_progress", truncate=8000) +async def wiki_progress( + wiki_id: t.Annotated[ + str, + "Stable lowercase kebab-case namespace of the wiki to summarize.", + ], +) -> dict[str, t.Any]: + """Summarize one wiki's bounded coverage so a maintenance loop knows when to stop. + + Browsing, reading, and search are handled by the built-in `list_items`, + `read_item`, and `search_items` tools; this reports the wiki-specific signal + those do not: page kinds, disposition mix, and outstanding open questions. + """ + if len(wiki_id) > 64 or not _WIKI_ID_PATTERN.fullmatch(wiki_id): + raise ValueError("wiki_id must be 1-64 lowercase kebab-case characters") + + api, org, workspace, project = _platform_context() + ( + records, + scanned_candidates, + candidate_total, + scan_truncated, + ) = await _scan_wiki_pages(api, org, workspace, project, wiki_id) + by_kind = collections.Counter( + str(_data_of(item).get("kind", "unknown")) for item in records + ) + by_status = collections.Counter(str(_status_of(item)) for item in records) + open_questions = 0 + pages_with_open_questions = 0 + for item in records: + questions = _data_of(item).get("open_questions") + count = len(questions) if isinstance(questions, list) else 0 + open_questions += count + if count: + pages_with_open_questions += 1 + initialized = bool(records) + return { + "wiki_id": wiki_id, + "initialized": initialized, + "pages_in_scan": len(records), + "candidates_scanned": scanned_candidates, + "candidate_platform_total": candidate_total, + "by_kind": dict(sorted(by_kind.items())), + "by_status": dict(sorted(by_status.items())), + "open_questions": open_questions, + "pages_with_open_questions": pages_with_open_questions, + # A partial scan must never produce a positive loop stop signal. + "open_questions_cleared": ( + initialized and not scan_truncated and open_questions == 0 + ), + "complete_scan": not scan_truncated, + "scan_truncated": scan_truncated, + }