From 293d0fcb2cbfc0427bbe42a6826f22296ef81fd6 Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 11 Apr 2026 04:33:55 +0530 Subject: [PATCH 1/9] changes --- src/firstops/proxy.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/firstops/proxy.py b/src/firstops/proxy.py index 58940e7..5459fd6 100644 --- a/src/firstops/proxy.py +++ b/src/firstops/proxy.py @@ -3,6 +3,7 @@ import logging import threading from http.server import HTTPServer, BaseHTTPRequestHandler +from socketserver import ThreadingMixIn from urllib.parse import urlparse, urlunparse import httpx @@ -47,7 +48,11 @@ def init( raise ValueError(f"invalid gateway_url: {gateway_url}") handler_class = _make_handler(key, bearer_token, gateway, port) - _server = HTTPServer(("127.0.0.1", port), handler_class) + + class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + _server = ThreadingHTTPServer(("127.0.0.1", port), handler_class) _server_thread = threading.Thread(target=_server.serve_forever, daemon=True) _server_thread.start() logger.info("firstops proxy listening on 127.0.0.1:%d", port) From 640c8cd7305cafa9e25faf46181743544c8b8c28 Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 13 Jun 2026 00:38:54 -0700 Subject: [PATCH 2/9] Release V2 of SDK --- README.md | 207 +++++++++- examples/README.md | 44 +++ examples/_shared.py | 32 ++ examples/claude_sdk_basic.py | 66 ++++ examples/claude_sdk_mcp.py | 70 ++++ examples/customers_openai.txt | 21 + examples/langgraph_basic.py | 66 ++++ examples/langgraph_notion_mcp.py | 92 +++++ examples/openai_agents_basic.py | 74 ++++ examples/openai_agents_mcp.py | 78 ++++ examples/out/customers.txt | 161 ++++++++ src/firstops/__init__.py | 59 ++- src/firstops/_identity.py | 59 +++ src/firstops/_runtime.py | 150 +++++++ src/firstops/channels.py | 38 ++ src/firstops/client.py | 427 ++++++++++++++++++++ src/firstops/coverage.py | 65 ++++ src/firstops/enforcement.py | 73 ++++ src/firstops/events.py | 195 ++++++++++ src/firstops/integrations/__init__.py | 12 + src/firstops/integrations/_common.py | 132 +++++++ src/firstops/integrations/claude.py | 84 ++++ src/firstops/integrations/langgraph.py | 87 +++++ src/firstops/integrations/openai_agents.py | 87 +++++ src/firstops/llm.py | 51 +++ src/firstops/proxy.py | 332 +++++++++++++--- src/firstops/tools.py | 318 +++++++++++++++ tests/test_channels.py | 68 ++++ tests/test_contract_parity.py | 103 +++++ tests/test_enforcement.py | 166 ++++++++ tests/test_enforcement_adversarial.py | 280 ++++++++++++++ tests/test_events.py | 108 ++++++ tests/test_events_adversarial.py | 197 ++++++++++ tests/test_identity_adversarial.py | 164 ++++++++ tests/test_integrations.py | 183 +++++++++ tests/test_integrations_bughunt.py | 429 +++++++++++++++++++++ tests/test_llm_route.py | 155 ++++++++ tests/test_llm_route_adversarial.py | 358 +++++++++++++++++ tests/test_m3_bughunt.py | 328 ++++++++++++++++ tests/test_m3_scrub_coverage.py | 118 ++++++ tests/test_proxy_regression.py | 212 ++++++++++ tests/test_proxy_router_regression.py | 138 +++++++ tests/test_runtime.py | 52 +++ tests/test_runtime_adversarial.py | 206 ++++++++++ tests/test_tools.py | 168 ++++++++ tests/test_tools_adversarial.py | 290 ++++++++++++++ 46 files changed, 6739 insertions(+), 64 deletions(-) create mode 100644 examples/README.md create mode 100644 examples/_shared.py create mode 100644 examples/claude_sdk_basic.py create mode 100644 examples/claude_sdk_mcp.py create mode 100644 examples/customers_openai.txt create mode 100644 examples/langgraph_basic.py create mode 100644 examples/langgraph_notion_mcp.py create mode 100644 examples/openai_agents_basic.py create mode 100644 examples/openai_agents_mcp.py create mode 100644 examples/out/customers.txt create mode 100644 src/firstops/_identity.py create mode 100644 src/firstops/_runtime.py create mode 100644 src/firstops/channels.py create mode 100644 src/firstops/client.py create mode 100644 src/firstops/coverage.py create mode 100644 src/firstops/enforcement.py create mode 100644 src/firstops/events.py create mode 100644 src/firstops/integrations/__init__.py create mode 100644 src/firstops/integrations/_common.py create mode 100644 src/firstops/integrations/claude.py create mode 100644 src/firstops/integrations/langgraph.py create mode 100644 src/firstops/integrations/openai_agents.py create mode 100644 src/firstops/llm.py create mode 100644 src/firstops/tools.py create mode 100644 tests/test_channels.py create mode 100644 tests/test_contract_parity.py create mode 100644 tests/test_enforcement.py create mode 100644 tests/test_enforcement_adversarial.py create mode 100644 tests/test_events.py create mode 100644 tests/test_events_adversarial.py create mode 100644 tests/test_identity_adversarial.py create mode 100644 tests/test_integrations.py create mode 100644 tests/test_integrations_bughunt.py create mode 100644 tests/test_llm_route.py create mode 100644 tests/test_llm_route_adversarial.py create mode 100644 tests/test_m3_bughunt.py create mode 100644 tests/test_m3_scrub_coverage.py create mode 100644 tests/test_proxy_regression.py create mode 100644 tests/test_proxy_router_regression.py create mode 100644 tests/test_runtime.py create mode 100644 tests/test_runtime_adversarial.py create mode 100644 tests/test_tools.py create mode 100644 tests/test_tools_adversarial.py diff --git a/README.md b/README.md index 414c90e..dd30ecb 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # FirstOps Python SDK -Secure MCP proxy sidecar with [DPoP](https://datatracker.ietf.org/doc/html/rfc9449) authentication for AI agents. +The FirstOps SDK has two halves: -FirstOps secures agent-to-tool connections. This SDK runs a lightweight local proxy that transparently adds DPoP-signed authentication headers to every MCP request your agent makes — no changes to your agent code required. +1. **Management client** (`FirstOps`) — programmatically create agents, register MCP connections, and manage their lifecycle from your backend. Authenticates with a tenant-scoped API key. +2. **Runtime proxy** (`firstops.init`) — a lightweight in-process sidecar that transparently signs every MCP request with a [DPoP](https://datatracker.ietf.org/doc/html/rfc9449) proof. Runs inside the agent process. + +The two halves are used at different points in an agent's lifecycle. The management client runs in your **platform code** (the backend that provisions agents). The runtime proxy runs inside the **agent itself** (the process that calls MCP tools). ## Install @@ -10,15 +13,149 @@ FirstOps secures agent-to-tool connections. This SDK runs a lightweight local pr pip install firstops ``` -## Quick Start +## Requirements + +- Python 3.10+ +- Dependencies: `cryptography`, `httpx` + +--- + +## 1. Management Client — Provisioning Agents + +Use this in your platform's backend code to create agents and wire up their MCP connections on demand. + +### Get an API key + +1. Log in to the FirstOps dashboard as an admin. +2. Go to **Settings → API Keys** and create a key with the scopes you need: + - `agents:write` — create and delete agent principals + - `agents:read` — list and get agents + - `connections:write` — register and delete MCP connections + - `connections:read` — list connections +3. Copy the raw key (starts with `fo_key_`). It is shown **once** — store it in your secrets manager. + +### Quick Start + +```python +from firstops import FirstOps + +# Initialize the management client +client = FirstOps(api_key="fo_key_...") + +# 1. Create an agent (returns principal ID, token, and private key) +agent = client.agents.create(name="research-bot-for-alice") +print(f"Agent ID: {agent.id}") +print(f"Agent token: {agent.token}") # fo_agent_ — used in Authorization header +print(f"Private key: {agent.private_key}") # PEM — shown once, save it securely + +# 2. Register MCP connections for the agent +slack_conn = client.connections.register( + principal_id=agent.id, + name="slack", + upstream_url="https://mcp.slack.com/sse", +) + +gdrive_conn = client.connections.register( + principal_id=agent.id, + name="google-drive", + upstream_url="https://mcp.google.com/drive/sse", + auth_type="oauth2", +) + +# 3. List the agent's connections +for conn in client.connections.list(principal_id=agent.id): + print(f"{conn.name} — {conn.status}") + +# 4. Remove a connection when the user no longer needs it +client.connections.delete(slack_conn.id) + +# 5. Delete the agent when the user deletes their instance +client.agents.delete(agent.id) +``` + +### Context Manager + +`FirstOps` is also usable as a context manager for automatic connection cleanup: + +```python +with FirstOps(api_key="fo_key_...") as client: + agents = client.agents.list() +``` + +### API Reference + +#### `FirstOps(api_key, base_url="https://api.firstops.ai", timeout=30.0)` + +The top-level management client. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `api_key` | *required* | Tenant-scoped API key (must start with `fo_key_`) | +| `base_url` | `https://api.firstops.ai` | FirstOps API base URL | +| `timeout` | `30.0` | HTTP timeout in seconds | + +#### `client.agents` + +| Method | Required Scope | Returns | +|--------|----------------|---------| +| `create(name: str)` | `agents:write` | `Agent` (with `private_key`) | +| `list()` | `agents:read` | `list[Agent]` | +| `get(agent_id: str)` | `agents:read` | `Agent` | +| `delete(agent_id: str)` | `agents:write` | `None` | + +**Note:** `agent.private_key` is only populated on `create()`. It is never returned again — store it alongside your agent record at creation time. + +#### `client.connections` + +| Method | Required Scope | Returns | +|--------|----------------|---------| +| `register(principal_id, name, upstream_url, ...)` | `connections:write` | `Connection` | +| `list(principal_id=None)` | `connections:read` | `list[Connection]` | +| `delete(connection_id: str)` | `connections:write` | `None` | + +Full signature for `register`: + +```python +client.connections.register( + principal_id="pr_...", # required — the agent's principal ID + name="slack", # required — display name + upstream_url="https://mcp.slack.com/sse", # required — remote MCP server URL + auth_type="", # optional — "oauth2", "bearer", etc. + transport_type="", # optional — "sse" or empty for auto-detect + upstream_headers=None, # optional — dict of headers to forward + upstream_query_params=None, # optional — dict of query params + source="sdk", # optional — audit label +) +``` + +### Error Handling + +All API errors raise `FirstOpsError`: + +```python +from firstops import FirstOps, FirstOpsError + +try: + client.agents.delete("pr_does_not_exist") +except FirstOpsError as e: + print(f"Error {e.status_code}: {e.message}") +``` + +--- + +## 2. Runtime Proxy — Securing MCP Calls Inside an Agent + +Use this inside the agent process itself. It starts a local HTTP proxy that transparently adds DPoP-signed authentication headers to every MCP request your agent makes — no changes to your agent code required. + +### Quick Start ```python import firstops # Start the proxy sidecar (runs in background thread) firstops.init( - agent_id="your-agent-id", - private_key_pem=open("agent-key.pem").read(), + agent_id="your-agent-id", # from client.agents.create(...).id + private_key_pem=open("agent-key.pem").read(), # from client.agents.create(...).private_key ) # Point your MCP client at localhost:9322 instead of the remote server. @@ -40,15 +177,65 @@ firstops.shutdown() | Parameter | Default | Description | |-----------|---------|-------------| -| `agent_id` | *required* | Your agent's principal ID | -| `private_key_pem` | *required* | EC P-256 private key (PEM format) | +| `agent_id` | *required* | Agent's principal ID (without `fo_agent_` prefix) | +| `private_key_pem` | *required* | EC P-256 private key in PEM format | | `port` | `9322` | Local proxy port | | `gateway_url` | `https://api.firstops.ai` | FirstOps gateway URL | -## Requirements +--- -- Python 3.10+ -- Dependencies: `cryptography`, `httpx` +## End-to-End Example: Dynamic Agent Platform + +Here's how the two halves fit together in a typical "SaaS that offers AI agents" platform: + +```python +# ─── Platform backend (your FastAPI/Django service) ───────────── +from firstops import FirstOps + +firstops = FirstOps(api_key=os.environ["FIRSTOPS_API_KEY"]) + +@app.post("/users/{user_id}/agents") +def create_user_agent(user_id: str, config: dict): + # Create a FirstOps agent identity for this end-user's instance + agent = firstops.agents.create(name=f"research-bot-{user_id}") + + # Store the agent credentials alongside the user's record + db.save_agent( + user_id=user_id, + agent_id=agent.id, + private_key=agent.private_key, # encrypt this at rest + ) + + # Wire up the tools the user selected + for tool in config["selected_tools"]: + firstops.connections.register( + principal_id=agent.id, + name=tool["name"], + upstream_url=tool["url"], + ) + + return {"agent_id": agent.id} + +@app.delete("/users/{user_id}/agents/{agent_id}") +def delete_user_agent(user_id: str, agent_id: str): + firstops.agents.delete(agent_id) # cascades to connections + db.delete_agent(agent_id) + + +# ─── Agent runtime (the worker process that actually runs the agent) ─── +import firstops as fo_runtime + +def run_agent_task(agent_id: str, private_key: str, task: str): + fo_runtime.init(agent_id=agent_id, private_key_pem=private_key) + try: + # Your MCP-using agent logic — point MCP clients at 127.0.0.1:9322 + mcp_client = MCPClient(base_url="http://127.0.0.1:9322") + return mcp_client.run(task) + finally: + fo_runtime.shutdown() +``` + +--- ## Development diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..f07a304 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,44 @@ +# FirstOps SDK — Examples + +Runnable agents that govern every LLM call, tool call, and MCP call through +FirstOps. + +## Setup + +```bash +python -m venv .venv && source .venv/bin/activate +pip install -e .. # the FirstOps SDK (this repo) +pip install "langchain>=1.0" langgraph langchain-openai langchain-mcp-adapters openai +``` + +## Config (env vars) + +| Var | Meaning | +|-----|---------| +| `FO_AGENT_ID` | Agent principal ID (UUID) from `client.agents.create(...)` | +| `FO_PRIVATE_KEY_PATH` | Path to the agent's EC P-256 private-key PEM | +| `FO_GATEWAY` | FirstOps gateway base URL (default `https://api.firstops.dev`) | +| `FO_PORT` | Local sidecar port (default `9322`) | +| `OPENAI_API_KEY` | OpenAI key — passes through the sidecar to OpenAI, never stored | +| `FO_MCP_CONNECTION_ID` | (MCP example) a registered MCP connection ID for the agent | + +## Examples + +- **`langgraph_basic.py`** — a LangGraph agent with two local tools and the LLM + routed through the sidecar chain-link. Exercises tool governance + LLM + governance. +- **`langgraph_notion_mcp.py`** — adds a Notion MCP server (via the sidecar's + MCP proxy) and a local `write_to_file` tool, then asks the agent to fetch + customer info from Notion and write it to a local file. Exercises **MCP + + local tool** governance together. + +```bash +FO_AGENT_ID=... FO_PRIVATE_KEY_PATH=... OPENAI_API_KEY=... \ + python langgraph_basic.py + +FO_AGENT_ID=... FO_PRIVATE_KEY_PATH=... OPENAI_API_KEY=... \ + FO_MCP_CONNECTION_ID=... python langgraph_notion_mcp.py +``` + +Each run prints a `[GOVERN]` line for every governed action (channel, tool, +decision), so you can see exactly what FirstOps evaluated. diff --git a/examples/_shared.py b/examples/_shared.py new file mode 100644 index 0000000..5a61590 --- /dev/null +++ b/examples/_shared.py @@ -0,0 +1,32 @@ +"""Shared boilerplate for the example agents: config + a governance tracer.""" + +import os + + +def load_config() -> dict: + agent_id = os.environ["FO_AGENT_ID"].strip() + key_path = os.environ["FO_PRIVATE_KEY_PATH"] + with open(key_path) as f: + key_pem = f.read() + return { + "agent_id": agent_id, + "key_pem": key_pem, + "gateway": os.environ.get("FO_GATEWAY", "https://api.firstops.dev"), + "port": int(os.environ.get("FO_PORT", "9322")), + } + + +def trace(fo) -> None: + """Wrap the enforcement client so every governed action prints.""" + orig = fo.enforcement.evaluate + + def traced(event): + d = orig(event) + print( + f" [GOVERN] {event.channel:<13} {event.event_type:<14} " + f"{event.tool_name:<32} -> {d.action}" + + (f" (FAILED_OPEN: {d.reason})" if d.failed_open else "") + ) + return d + + fo.enforcement.evaluate = traced diff --git a/examples/claude_sdk_basic.py b/examples/claude_sdk_basic.py new file mode 100644 index 0000000..c496b41 --- /dev/null +++ b/examples/claude_sdk_basic.py @@ -0,0 +1,66 @@ +"""Claude Agent SDK agent governed by FirstOps. + +The Claude Agent SDK runs tools (Bash, Write, Read, MCP) inside the Claude Code +subprocess. FirstOps governs each one via a single PreToolUse hook — block, +rewrite args (updatedInput), or allow — the daemon model, in-process. + +`permission_mode="bypassPermissions"` makes the FirstOps hook the sole gate: +a hook `deny` still blocks; everything else flows. Run with the env vars in +README.md (no OpenAI key needed — Claude uses your local Claude Code auth). +""" + +import asyncio +from pathlib import Path + +import firstops +from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ResultMessage, + TextBlock, + ToolUseBlock, + query, +) +from firstops.integrations.claude import firstops_hooks + +from _shared import load_config, trace + +WORKDIR = Path(__file__).parent / "claude_work" + + +async def main(): + cfg = load_config() + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + WORKDIR.mkdir(exist_ok=True) + try: + options = ClaudeAgentOptions( + hooks=firstops_hooks(fo), # ← every tool call governed by FirstOps + allowed_tools=["Bash", "Write", "Read"], + permission_mode="bypassPermissions", + cwd=str(WORKDIR), + ) + prompt = ( + "Create a file named greeting.txt containing exactly " + "'Hello from a FirstOps-governed Claude agent'. " + "Then run a bash command to print today's date. " + "Finally, read greeting.txt back and report its contents." + ) + print("\n>>> running Claude agent (tools governed via PreToolUse hook)\n") + async for message in query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock) and block.text.strip(): + print(f" [CLAUDE] {block.text.strip()[:160]}") + elif isinstance(block, ToolUseBlock): + print(f" [TOOL-USE] {block.name} {block.input}") + elif isinstance(message, ResultMessage): + print(f"\n>>> result:\n{getattr(message, 'result', message)}") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/claude_sdk_mcp.py b/examples/claude_sdk_mcp.py new file mode 100644 index 0000000..561c4f6 --- /dev/null +++ b/examples/claude_sdk_mcp.py @@ -0,0 +1,70 @@ +"""Claude Agent SDK agent with a Notion MCP server, governed by FirstOps. + +Same daemon-model hook as claude_sdk_basic.py, but now the agent also talks to +a Notion MCP server (through the FirstOps proxy). The single PreToolUse hook +governs BOTH the MCP tool calls (mcp__notion__*) and the built-in Write tool. + +Run with the env vars in README.md, incl. FO_MCP_CONNECTION_ID. Uses your local +Claude Code auth (no OpenAI key needed). +""" + +import asyncio +import os +from pathlib import Path + +import firstops +from claude_agent_sdk import ( + AssistantMessage, + ClaudeAgentOptions, + ResultMessage, + TextBlock, + ToolUseBlock, + query, +) +from firstops.integrations.claude import firstops_hooks + +from _shared import load_config, trace + +WORKDIR = Path(__file__).parent / "claude_work" + + +async def main(): + cfg = load_config() + conn_id = os.environ["FO_MCP_CONNECTION_ID"].strip() + + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + WORKDIR.mkdir(exist_ok=True) + try: + options = ClaudeAgentOptions( + hooks=firstops_hooks(fo), # governs both MCP and built-in tools + mcp_servers={ + "notion": {"type": "http", "url": firstops.mcp_url(conn_id)} + }, + allowed_tools=["Write", "Read", "mcp__notion"], + permission_mode="bypassPermissions", + cwd=str(WORKDIR), + ) + prompt = ( + "Use the Notion tools to find the customer database and fetch the " + "customer records. Then write the customer info to a file named " + "customers_claude.txt." + ) + print("\n>>> running Claude agent (Notion MCP + Write, governed)\n") + async for message in query(prompt=prompt, options=options): + if isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, TextBlock) and block.text.strip(): + print(f" [CLAUDE] {block.text.strip()[:140]}") + elif isinstance(block, ToolUseBlock): + print(f" [TOOL-USE] {block.name}") + elif isinstance(message, ResultMessage): + print(f"\n>>> result:\n{getattr(message, 'result', message)}") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/customers_openai.txt b/examples/customers_openai.txt new file mode 100644 index 0000000..0d66fe1 --- /dev/null +++ b/examples/customers_openai.txt @@ -0,0 +1,21 @@ +# Name Email Phone Company Revenue ($) Plan Status +1 Sarah Chen sarah.chen@acmecorp.io +1-415-555-0142 Acme Corp 248,000 Enterprise Active +2 Marcus Johnson marcus.j@brevity.co +1-212-555-0198 Brevity Inc 85,500 Pro Active +3 Priya Sharma priya@novatech.in +91-98765-43210 NovaTech 412,000 Enterprise Active +4 James O'Brien jobrien@lumendata.com +44-20-7946-0958 Lumen Data 67,200 Starter Churned +5 Mei Lin mei.lin@zephyrcloud.io +65-9123-4567 Zephyr Cloud 195,000 Pro Active +6 Carlos Ruiz cruiz@fintechflow.mx +52-55-5555-0147 FintechFlow 320,000 Enterprise Active +7 Aisha Patel aisha@gridpoint.ai +1-650-555-0173 GridPoint AI 54,800 Starter Trial +8 Tom Eriksson tom.e@nordicsaas.se +46-70-123-4567 Nordic SaaS 128,500 Pro Active +9 Rachel Kim rkim@ocelotlabs.com +1-310-555-0261 Ocelot Labs 91,000 Pro Churned +10 David Müller dmuller@berlinops.de +49-30-5555-0184 BerlinOps 275,000 Enterprise Active +11 Fatima Al-Hassan fatima@clearstack.ae +971-50-555-0139 ClearStack 182,000 Pro Active +12 Liam Cooper lcooper@pulseio.com.au +61-4-5555-0192 Pulse.io 43,200 Starter Trial +13 Yuki Tanaka yuki@kaizenlabs.jp +81-3-5555-0167 Kaizen Labs 356,000 Enterprise Active +14 Nina Petrova nina.p@dataweave.ru +7-495-555-0128 DataWeave 72,400 Pro Churned +15 Ben Adeyemi ben@scalepath.ng +234-801-555-0145 ScalePath 38,900 Starter Active +16 Sophie Dubois sdubois@cloudnine.fr +33-1-5555-0176 CloudNine 210,000 Enterprise Active +17 Raj Menon raj@bytebridge.io +1-512-555-0203 ByteBridge 99,800 Pro Active +18 Emma Walsh ewalsh@peakvector.ie +353-1-555-0154 PeakVector 156,300 Pro Active +19 Alex Novak anovak@synchub.cz +420-222-555-016 SyncHub 61,700 Starter Churned +20 Isabela Costa icosta@flowmetrics.br +55-11-5555-0189 FlowMetrics 287,500 Enterprise Active diff --git a/examples/langgraph_basic.py b/examples/langgraph_basic.py new file mode 100644 index 0000000..754074a --- /dev/null +++ b/examples/langgraph_basic.py @@ -0,0 +1,66 @@ +"""LangGraph agent governed by FirstOps — tool calls + LLM, no MCP. + +Every LLM call routes through the sidecar chain-link; every tool call is +intercepted by FirstOpsMiddleware. Run with the env vars in README.md. +""" + +import os + +import firstops +from firstops.integrations.langgraph import FirstOpsMiddleware +from langchain.agents import create_agent +from langchain_core.tools import tool as lc_tool +from langchain_openai import ChatOpenAI + +from _shared import load_config, trace + + +@lc_tool +def get_weather(city: str) -> str: + """Get the current weather for a city.""" + print(f" [TOOL get_weather] city={city}") + return f"It is 21C and sunny in {city}." + + +@lc_tool +def send_email(to: str, body: str) -> str: + """Send an email to a recipient.""" + print(f" [TOOL send_email] to={to} body={body!r}") + return "email sent" + + +def main(): + cfg = load_config() + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + try: + llm = ChatOpenAI( + model="gpt-4o-mini", + base_url=firstops.llm_base_url("openai"), + api_key=os.environ["OPENAI_API_KEY"], + ) + agent = create_agent( + model=llm, + tools=[get_weather, send_email], + middleware=[FirstOpsMiddleware(fo)], + ) + print("\n>>> invoking agent\n") + result = agent.invoke( + { + "messages": [ + { + "role": "user", + "content": "Check the weather in Paris, then email it to alice@example.com.", + } + ] + } + ) + print(f"\n>>> final answer:\n{result['messages'][-1].content}\n") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + main() diff --git a/examples/langgraph_notion_mcp.py b/examples/langgraph_notion_mcp.py new file mode 100644 index 0000000..cce1751 --- /dev/null +++ b/examples/langgraph_notion_mcp.py @@ -0,0 +1,92 @@ +"""LangGraph agent governed by FirstOps — MCP + local tool, end to end. + +Wires a Notion MCP server (through the sidecar's MCP proxy) alongside a local +`write_to_file` tool, then asks the agent to fetch customer info from Notion and +write it to a local file. Exercises BOTH governance paths in one run: + + - MCP tool calls -> sidecar /mcp/proxy/ -> gateway (server-side + enforcement + credential brokering) AND FirstOpsMiddleware (tool channel) + - the local tool -> FirstOpsMiddleware (system_tools channel) + - the LLM -> sidecar /llm chain-link + +Run with the env vars in README.md (incl. FO_MCP_CONNECTION_ID). +""" + +import asyncio +import os +from pathlib import Path + +import firstops +from firstops.integrations.langgraph import FirstOpsMiddleware +from langchain.agents import create_agent +from langchain_core.tools import tool as lc_tool +from langchain_mcp_adapters.client import MultiServerMCPClient +from langchain_openai import ChatOpenAI + +from _shared import load_config, trace + +OUT_DIR = Path(__file__).parent / "out" + + +@lc_tool +def write_to_file(filename: str, content: str) -> str: + """Write text content to a local file in the output directory.""" + OUT_DIR.mkdir(exist_ok=True) + path = OUT_DIR / Path(filename).name # no path traversal + path.write_text(content) + print(f" [TOOL write_to_file] wrote {len(content)} bytes to {path}") + return f"wrote {len(content)} bytes to {path}" + + +async def main(): + cfg = load_config() + conn_id = os.environ["FO_MCP_CONNECTION_ID"].strip() + + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + try: + # Point the MCP client at the sidecar's MCP proxy for this connection. + # The sidecar DPoP-signs and forwards to the gateway, which brokers the + # Notion credentials — the agent never sees them. + mcp_client = MultiServerMCPClient( + {"notion": {"url": firstops.mcp_url(conn_id), "transport": "streamable_http"}} + ) + mcp_tools = await mcp_client.get_tools() + print(f">>> Notion MCP exposed {len(mcp_tools)} tools: " + f"{[t.name for t in mcp_tools][:8]}{' ...' if len(mcp_tools) > 8 else ''}") + + llm = ChatOpenAI( + model="gpt-4o-mini", + base_url=firstops.llm_base_url("openai"), + api_key=os.environ["OPENAI_API_KEY"], + ) + agent = create_agent( + model=llm, + tools=mcp_tools + [write_to_file], + middleware=[FirstOpsMiddleware(fo)], + ) + + print("\n>>> invoking agent (Notion MCP + local write_to_file)\n") + result = await agent.ainvoke( + { + "messages": [ + { + "role": "user", + "content": ( + "Find the customer database in Notion and fetch the " + "customer records. Print the customer info, then write " + "it to a local file called customers.txt." + ), + } + ] + } + ) + print(f"\n>>> final answer:\n{result['messages'][-1].content}\n") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/openai_agents_basic.py b/examples/openai_agents_basic.py new file mode 100644 index 0000000..e5b51db --- /dev/null +++ b/examples/openai_agents_basic.py @@ -0,0 +1,74 @@ +"""OpenAI Agents SDK agent governed by FirstOps. + +Two wiring points (different from LangGraph/Claude): + - LLM: route the Agents SDK's default OpenAI client at the sidecar chain-link + (chat-completions API; tracing disabled so traces don't bypass governance). + - Tools: attach the FirstOps guardrail PER `@function_tool` + (OpenAI Agents has no agent-level tool guardrail). The guardrail is + block-only — it can deny a tool, but can't rewrite its args. + +Run with the env vars in README.md (needs OPENAI_API_KEY). +""" + +import asyncio +import os + +import firstops +from agents import ( + Agent, + Runner, + function_tool, + set_default_openai_client, + set_tracing_disabled, +) +from firstops.integrations.openai_agents import firstops_tool_input_guardrail +from openai import AsyncOpenAI + +from _shared import load_config, trace + + +async def main(): + cfg = load_config() + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + try: + # LLM calls -> sidecar chain-link (-> OpenAI). Key passes through. + client = AsyncOpenAI( + base_url=firstops.llm_base_url("openai"), + api_key=os.environ["OPENAI_API_KEY"], + ) + set_default_openai_client(client) + # Use the Agents SDK's native Responses API — the sidecar forwards it to + # OpenAI unchanged (the default model is a reasoning model that needs it). + set_tracing_disabled(True) + + guard = firstops_tool_input_guardrail(fo) # one guardrail, attached per tool + + @function_tool(tool_input_guardrails=[guard]) + def get_weather(city: str) -> str: + """Get the current weather for a city.""" + print(f" [TOOL get_weather] city={city}") + return f"It is 21C and sunny in {city}." + + @function_tool(tool_input_guardrails=[guard]) + def send_email(to: str, body: str) -> str: + """Send an email to a recipient.""" + print(f" [TOOL send_email] to={to} body={body!r}") + return "email sent" + + agent = Agent(name="assistant", tools=[get_weather, send_email]) + + print("\n>>> running OpenAI Agents SDK agent\n") + result = await Runner.run( + agent, + "Check the weather in Paris, then email it to alice@example.com.", + ) + print(f"\n>>> final answer:\n{result.final_output}\n") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/openai_agents_mcp.py b/examples/openai_agents_mcp.py new file mode 100644 index 0000000..bf88475 --- /dev/null +++ b/examples/openai_agents_mcp.py @@ -0,0 +1,78 @@ +"""OpenAI Agents SDK agent with a Notion MCP server, governed by FirstOps. + +Governance per surface: + - LLM -> sidecar chain-link (set_default_openai_client at the sidecar) + - local tool -> per-`@function_tool` FirstOps guardrail (block-only) + - MCP server -> sidecar MCP proxy -> gateway (credential brokering + + server-side enforcement; the agent never holds Notion's token) + +Run with the env vars in README.md, incl. FO_MCP_CONNECTION_ID. +""" + +import asyncio +import os + +import firstops +from agents import ( + Agent, + Runner, + function_tool, + set_default_openai_client, + set_tracing_disabled, +) +from agents.mcp import MCPServerStreamableHttp +from firstops.integrations.openai_agents import firstops_tool_input_guardrail +from openai import AsyncOpenAI + +from _shared import load_config, trace + + +async def main(): + cfg = load_config() + conn_id = os.environ["FO_MCP_CONNECTION_ID"].strip() + + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + try: + set_default_openai_client( + AsyncOpenAI( + base_url=firstops.llm_base_url("openai"), + api_key=os.environ["OPENAI_API_KEY"], + ) + ) + set_tracing_disabled(True) + + guard = firstops_tool_input_guardrail(fo) + + @function_tool(tool_input_guardrails=[guard]) + def write_to_file(filename: str, content: str) -> str: + """Write text content to a local file.""" + with open(filename, "w") as f: + f.write(content) + print(f" [TOOL write_to_file] wrote {len(content)} bytes to {filename}") + return f"wrote {len(content)} bytes to {filename}" + + async with MCPServerStreamableHttp( + name="notion", params={"url": firstops.mcp_url(conn_id)} + ) as notion: + agent = Agent( + name="assistant", + instructions="Use Notion to find data, and write files when asked.", + mcp_servers=[notion], + tools=[write_to_file], + ) + print("\n>>> running OpenAI Agents SDK agent (Notion MCP + local tool)\n") + result = await Runner.run( + agent, + "Fetch the customer records from the Notion customer database and " + "write them to customers_openai.txt.", + ) + print(f"\n>>> final answer:\n{result.final_output}\n") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/out/customers.txt b/examples/out/customers.txt new file mode 100644 index 0000000..62c8221 --- /dev/null +++ b/examples/out/customers.txt @@ -0,0 +1,161 @@ +Customer Info: + +1. Name: Sarah Chen + Email: sarah.chen@acmecorp.io + Revenue: $248,000 + Status: Active + Plan: Enterprise + Company: Acme Corp + Phone: +1-415-555-0142 + +2. Name: Marcus Johnson + Email: marcus.j@brevity.co + Revenue: $85,500 + Status: Active + Plan: Pro + Company: Brevity Inc + Phone: +1-212-555-0198 + +3. Name: Priya Sharma + Email: priya@novatech.in + Revenue: $412,000 + Status: Active + Plan: Enterprise + Company: NovaTech + Phone: +91-98765-43210 + +4. Name: James O'Brien + Email: jobrien@lumendata.com + Revenue: $67,200 + Status: Churned + Plan: Starter + Company: Lumen Data + Phone: +44-20-7946-0958 + +5. Name: Mei Lin + Email: mei.lin@zephyrcloud.io + Revenue: $195,000 + Status: Active + Plan: Pro + Company: Zephyr Cloud + Phone: +65-9123-4567 + +6. Name: Carlos Ruiz + Email: cruiz@fintechflow.mx + Revenue: $320,000 + Status: Active + Plan: Enterprise + Company: FintechFlow + Phone: +52-55-5555-0147 + +7. Name: Aisha Patel + Email: aisha@gridpoint.ai + Revenue: $54,800 + Status: Trial + Plan: Starter + Company: GridPoint AI + Phone: +1-650-555-0173 + +8. Name: Tom Eriksson + Email: tom.e@nordicsaas.se + Revenue: $128,500 + Status: Active + Plan: Pro + Company: Nordic SaaS + Phone: +46-70-123-4567 + +9. Name: Rachel Kim + Email: rkim@ocelotlabs.com + Revenue: $91,000 + Status: Churned + Plan: Pro + Company: Ocelot Labs + Phone: +1-310-555-0261 + +10. Name: David Müller + Email: dmuller@berlinops.de + Revenue: $275,000 + Status: Active + Plan: Enterprise + Company: BerlinOps + Phone: +49-30-5555-0184 + +11. Name: Fatima Al-Hassan + Email: fatima@clearstack.ae + Revenue: $182,000 + Status: Active + Plan: Pro + Company: ClearStack + Phone: +971-50-555-0139 + +12. Name: Liam Cooper + Email: lcooper@pulseio.com.au + Revenue: $43,200 + Status: Trial + Plan: Starter + Company: Pulse.io + Phone: +61-4-5555-0192 + +13. Name: Yuki Tanaka + Email: yuki@kaizenlabs.jp + Revenue: $356,000 + Status: Active + Plan: Enterprise + Company: Kaizen Labs + Phone: +81-3-5555-0167 + +14. Name: Nina Petrova + Email: nina.p@dataweave.ru + Revenue: $72,400 + Status: Churned + Plan: Pro + Company: DataWeave + Phone: +7-495-555-0128 + +15. Name: Ben Adeyemi + Email: ben@scalepath.ng + Revenue: $38,900 + Status: Active + Plan: Starter + Company: ScalePath + Phone: +234-801-555-0145 + +16. Name: Sophie Dubois + Email: sdubois@cloudnine.fr + Revenue: $210,000 + Status: Active + Plan: Enterprise + Company: CloudNine + Phone: +33-1-5555-0176 + +17. Name: Raj Menon + Email: raj@bytebridge.io + Revenue: $99,800 + Status: Active + Plan: Pro + Company: ByteBridge + Phone: +1-512-555-0203 + +18. Name: Emma Walsh + Email: ewalsh@peakvector.ie + Revenue: $156,300 + Status: Active + Plan: Pro + Company: PeakVector + Phone: +353-1-555-0154 + +19. Name: Alex Novak + Email: anovak@synchub.cz + Revenue: $61,700 + Status: Churned + Plan: Starter + Company: SyncHub + Phone: +420-222-555-016 + +20. Name: Isabela Costa + Email: icosta@flowmetrics.br + Revenue: $287,500 + Status: Active + Plan: Enterprise + Company: FlowMetrics + Phone: +55-11-5555-0189 diff --git a/src/firstops/__init__.py b/src/firstops/__init__.py index 91b7f35..337a4bb 100644 --- a/src/firstops/__init__.py +++ b/src/firstops/__init__.py @@ -1,5 +1,58 @@ -"""FirstOps SDK — secure MCP proxy sidecar with DPoP authentication.""" +"""FirstOps SDK — secure MCP proxy sidecar with DPoP authentication and management API.""" -from firstops.proxy import init, shutdown +from firstops._runtime import ( + Runtime, + init, + llm_base_url, + mcp_url, + runtime, + shutdown, +) +from firstops.client import ( + Agent, + Connection, + FirstOps, + FirstOpsError, + ParamDefinition, + ServerTemplate, +) +from firstops.coverage import capability, coverage_report, ungoverned_tools +from firstops.enforcement import EnforcementClient +from firstops.events import ActionEvent, Decision +from firstops.llm import anthropic_client, configure_llm_env, openai_client +from firstops.proxy import current_agent_id, is_running +from firstops.tools import FirstOpsPolicyError, tool -__all__ = ["init", "shutdown"] +__all__ = [ + # management client + "FirstOps", + "FirstOpsError", + "Agent", + "Connection", + "ServerTemplate", + "ParamDefinition", + # runtime + "init", + "shutdown", + "runtime", + "Runtime", + "is_running", + "current_agent_id", + # base API — tool governance + "tool", + "FirstOpsPolicyError", + # base API — LLM chain-link + MCP + "llm_base_url", + "mcp_url", + "openai_client", + "anthropic_client", + "configure_llm_env", + # enforcement surface + "EnforcementClient", + "ActionEvent", + "Decision", + # coverage honesty + "capability", + "coverage_report", + "ungoverned_tools", +] diff --git a/src/firstops/_identity.py b/src/firstops/_identity.py new file mode 100644 index 0000000..1e69871 --- /dev/null +++ b/src/firstops/_identity.py @@ -0,0 +1,59 @@ +"""Shared agent identity — the loaded key, DPoP signer, and gateway URL. + +Established once per process and shared by the MCP sidecar proxy and the +enforcement (EvaluateHook) client, so a single agent identity backs every +signed request the SDK makes. Key material never leaves this object. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from urllib.parse import urlparse + +from cryptography.hazmat.primitives.asymmetric import ec + +from firstops.dpop import create_proof, jwk_thumbprint, load_private_key + + +@dataclass +class Identity: + """An agent's signing identity. Construct via :func:`build_identity`.""" + + agent_id: str + gateway_url: str # normalized, no trailing slash + _key: ec.EllipticCurvePrivateKey + + @property + def bearer_token(self) -> str: + return f"fo_agent_{self.agent_id}" + + @property + def jkt(self) -> str: + """RFC 7638 JWK thumbprint of this identity's key.""" + return jwk_thumbprint(self._key) + + def proof(self, method: str, url: str) -> str: + """Create a DPoP proof (RFC 9449) for an HTTP method + target URL. + + The htu claim is the URL with query and fragment stripped, matching the + gateway's **byte-exact** htu binding. We strip via string slicing (not a + urlparse round-trip) so we do NOT alter scheme/host casing — any + normalization the SDK applies that sentinel does not would 401 every + proof, which `evaluate()` silently converts to a fail-open allow. + """ + htu = url.split("#", 1)[0].split("?", 1)[0] + return create_proof(self._key, method, htu) + + +def build_identity(agent_id: str, private_key_pem: str, gateway_url: str) -> Identity: + """Load the key and validate the gateway URL, returning an Identity. + + Raises: + ValueError: if ``gateway_url`` is malformed or the key is not P-256. + """ + gateway = gateway_url.rstrip("/") + parsed = urlparse(gateway) + if not parsed.scheme or not parsed.netloc: + raise ValueError(f"invalid gateway_url: {gateway_url}") + key = load_private_key(private_key_pem) + return Identity(agent_id=agent_id, gateway_url=gateway, _key=key) diff --git a/src/firstops/_runtime.py b/src/firstops/_runtime.py new file mode 100644 index 0000000..cab770f --- /dev/null +++ b/src/firstops/_runtime.py @@ -0,0 +1,150 @@ +"""Process-wide FirstOps runtime — the single entrypoint an agent calls. + +``firstops.init()`` establishes the agent identity, builds the enforcement +client (the EvaluateHook spine), and starts the dual-mode sidecar (MCP terminal ++ LLM chain-link). The returned :class:`Runtime` handle is what tool decorators +and harness adapters use to forward action events; it is also stored +process-globally so ``firstops.shutdown()`` works without a handle. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass + +from firstops import proxy +from firstops._identity import Identity, build_identity +from firstops.enforcement import EnforcementClient + +_DEFAULT_PORT = 9322 +_DEFAULT_GATEWAY = "https://api.firstops.dev" + +# Default LLM upstreams. Override per-provider via ``init(llm_upstreams=...)`` +# to point the chain-link at a customer's existing gateway instead. +_DEFAULT_LLM_UPSTREAMS = { + "openai": "https://api.openai.com", + "anthropic": "https://api.anthropic.com", +} + +_lock = threading.Lock() +_runtime: Runtime | None = None + + +@dataclass +class Runtime: + """Handle to the live FirstOps runtime for this process.""" + + identity: Identity + enforcement: EnforcementClient + port: int + llm_upstreams: dict[str, str] + + +def init( + agent_id: str, + private_key_pem: str, + *, + port: int = _DEFAULT_PORT, + gateway_url: str = _DEFAULT_GATEWAY, + llm_upstreams: dict[str, str] | None = None, +) -> Runtime: + """Establish identity, the enforcement client, and the dual-mode sidecar. + + Idempotent for the same identity: the sidecar refuses to switch agents + mid-flight (raises ``RuntimeError``); re-initializing with the same + parameters returns the existing runtime. + + Args: + llm_upstreams: per-provider upstream base URLs for the LLM chain-link. + Merged over the defaults (openai/anthropic public endpoints). Point + a provider at your own gateway to chain in front of it. + + Returns: + The process :class:`Runtime` handle. + """ + global _runtime + + gateway = gateway_url.rstrip("/") + upstreams = dict(_DEFAULT_LLM_UPSTREAMS) + if llm_upstreams: + upstreams.update(llm_upstreams) + + with _lock: + if _runtime is not None: + # Idempotent re-confirm — proxy.init owns the mismatch guard. + proxy.init( + agent_id=agent_id, + private_key_pem=private_key_pem, + port=port, + gateway_url=gateway, + enforcement=_runtime.enforcement, + llm_upstreams=_runtime.llm_upstreams, + ) + return _runtime + + identity = build_identity(agent_id, private_key_pem, gateway) + enforcement = EnforcementClient(identity) + proxy.init( + agent_id=agent_id, + private_key_pem=private_key_pem, + port=port, + gateway_url=gateway, + enforcement=enforcement, + llm_upstreams=upstreams, + ) + _runtime = Runtime( + identity=identity, + enforcement=enforcement, + port=port, + llm_upstreams=upstreams, + ) + return _runtime + + +def shutdown() -> None: + """Stop the sidecar and tear down the runtime. Idempotent. + + Lifecycle: call at process/agent teardown, **not** concurrently with + in-flight governed calls. ``EnforcementClient.evaluate`` is fully + fail-open, so a racing ``evaluate()`` during shutdown degrades to an + allow (never a crash or hang) — but the supported pattern is init-once, + run, shutdown-once. + """ + global _runtime + proxy.shutdown() + with _lock: + if _runtime is not None: + _runtime.enforcement.close() + _runtime = None + + +def runtime() -> Runtime | None: + """Return the live runtime, or None if init() has not been called.""" + with _lock: + return _runtime + + +def llm_base_url(provider: str = "openai") -> str: + """Return the local sidecar base URL to point an LLM client at. + + Point your client's ``base_url`` (or ``OPENAI_BASE_URL`` / + ``ANTHROPIC_BASE_URL``) here; the sidecar governs the request and forwards + to the configured upstream. + """ + rt = runtime() + if rt is None: + raise RuntimeError("firstops.init() must be called before llm_base_url()") + base = f"http://127.0.0.1:{rt.port}/llm/{provider}" + return base + "/v1" if provider == "openai" else base + + +def mcp_url(connection_id: str) -> str: + """Return the local sidecar URL to point an MCP client at for a connection. + + The sidecar DPoP-signs each request and forwards it to the FirstOps gateway, + which brokers the upstream credentials — the agent never holds them. + """ + rt = runtime() + if rt is None: + raise RuntimeError("firstops.init() must be called before mcp_url()") + return f"http://127.0.0.1:{rt.port}/mcp/proxy/{connection_id}" diff --git a/src/firstops/channels.py b/src/firstops/channels.py new file mode 100644 index 0000000..254a730 --- /dev/null +++ b/src/firstops/channels.py @@ -0,0 +1,38 @@ +"""Channel classification for tool calls. + +Mirrors the daemon's tool_name → channel mapping so sentinel sees SDK tool +calls the same way it sees coding-agent hook events. MCP tools (named +``mcp____``) classify as the MCP channel; everything else a +decorated tool does is ``system_tools``. LLM traffic is stamped ``llm`` +directly by the sidecar's LLM route, not by this classifier. +""" + +from __future__ import annotations + +from firstops.events import CHANNEL_MCP, CHANNEL_SYSTEM_TOOLS, MCPInfo + +_MCP_PREFIX = "mcp__" + + +def classify(tool_name: str) -> str: + """Return the channel a tool call belongs to. + + A name is MCP only if it's a *well-formed* ``mcp____`` (so + classify and :func:`mcp_info` always agree — we never emit a ``channel=mcp`` + event with no mcp metadata). A malformed ``mcp__`` name (e.g. ``mcp__foo``) + is treated as ``system_tools``. + """ + if mcp_info(tool_name) is not None: + return CHANNEL_MCP + return CHANNEL_SYSTEM_TOOLS + + +def mcp_info(tool_name: str) -> MCPInfo | None: + """Parse ``mcp____`` into MCPInfo, or None if not a well-formed MCP name.""" + if not tool_name.startswith(_MCP_PREFIX): + return None + parts = tool_name.split("__") + # Require non-empty server and tool segments. + if len(parts) >= 3 and parts[1] and "__".join(parts[2:]): + return MCPInfo(server=parts[1], tool="__".join(parts[2:])) + return None diff --git a/src/firstops/client.py b/src/firstops/client.py new file mode 100644 index 0000000..a0aa725 --- /dev/null +++ b/src/firstops/client.py @@ -0,0 +1,427 @@ +"""FirstOps management client for programmatic agent and connection CRUD.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import httpx + + +@dataclass +class Agent: + """An agent principal.""" + + id: str + tenant_id: str + name: str + reference_id: str + metadata: dict[str, str] + created_at: int + token: str # fo_agent_ — used in Authorization header + private_key: str | None = None # PEM, only set on creation + + +@dataclass +class Connection: + """A registered MCP connection.""" + + id: str + tenant_id: str + principal_id: str + name: str + upstream_url: str + status: str + created_at: int + + +@dataclass +class ParamDefinition: + """A single parameter required to configure a template connection.""" + + key: str # placeholder key, e.g. "api_token" + display_name: str # human-readable label + description: str # help text + required: bool + secret: bool # if true, treat the value as sensitive + maps_to: str # "HEADER", "QUERY_PARAM", or "URL_PLACEHOLDER" + + +@dataclass +class ServerTemplate: + """A FirstOps catalog entry describing an MCP server integration.""" + + id: str + name: str + description: str + category: str + upstream_url_template: str + required_params: list[ParamDefinition] + tenant_configured: bool + user_connected: bool + existing_connection_id: str + auth_type: str + transport_type: str + oauth_setup_guide: str + requires_org_config: bool + raw: dict[str, Any] # full proto-JSON for fields the SDK doesn't model + + +def _parse_template(entry: dict[str, Any]) -> ServerTemplate: + """Parse a ServerTemplateWithStatus proto-JSON entry. + + The catalog RPC returns ``ServerTemplateWithStatus``, a wrapper with + the actual ``ServerTemplate`` nested under ``template`` plus three + overlay flags (``tenant_configured``, ``user_connected``, + ``existing_connection_id``). We flatten that into a single dataclass. + + When the backend returns a bare ``ServerTemplate`` (no wrapper), we + fall back to reading fields off the entry itself. + """ + inner = ( + entry.get("template") + if isinstance(entry.get("template"), dict) + else entry + ) + + params_raw = inner.get("required_params") or [] + params: list[ParamDefinition] = [] + for p in params_raw: + params.append( + ParamDefinition( + key=p.get("key", ""), + display_name=p.get("display_name", ""), + description=p.get("description", ""), + required=p.get("required", False), + secret=p.get("secret", False), + maps_to=p.get("maps_to", ""), + ) + ) + + # oauth_setup_guide is a nested message — stringify to something usable. + oauth_guide = inner.get("oauth_setup_guide") + if isinstance(oauth_guide, dict): + oauth_guide_str = oauth_guide.get("instructions", "") or "" + elif isinstance(oauth_guide, str): + oauth_guide_str = oauth_guide + else: + oauth_guide_str = "" + + return ServerTemplate( + id=inner.get("id", ""), + name=inner.get("name", ""), + description=inner.get("description", ""), + category=inner.get("category", ""), + upstream_url_template=inner.get("upstream_url", ""), + required_params=params, + tenant_configured=entry.get("tenant_configured", False), + user_connected=entry.get("user_connected", False), + existing_connection_id=entry.get("existing_connection_id", ""), + auth_type=inner.get("auth_method", "") or inner.get("auth_type", ""), + transport_type=inner.get("transport_type", ""), + oauth_setup_guide=oauth_guide_str, + requires_org_config=inner.get("requires_org_config", False), + raw=entry, + ) + + +class FirstOpsError(Exception): + """Raised when the FirstOps API returns an error.""" + + def __init__(self, status_code: int, message: str): + self.status_code = status_code + self.message = message + super().__init__(f"{message} (HTTP {status_code})") + + +class _AgentsResource: + def __init__(self, client: FirstOps): + self._client = client + + def create(self, name: str) -> Agent: + """Create a new agent principal with a DPoP keypair.""" + resp = self._client._request("POST", "/api/v1/sdk/agents", json={"name": name}) + agent_data = resp["agent"] + return Agent( + id=agent_data["id"], + tenant_id=agent_data.get("tenant_id", ""), + name=agent_data.get("name", "") or agent_data.get("metadata", {}).get("name", ""), + reference_id=agent_data.get("reference_id", ""), + metadata=agent_data.get("metadata", {}), + created_at=agent_data.get("created_at", 0), + token=f"fo_agent_{agent_data['id']}", + private_key=resp.get("private_key"), + ) + + def list(self) -> list[Agent]: + """List all agent principals in the tenant.""" + resp = self._client._request("GET", "/api/v1/sdk/agents") + agents = [] + for a in resp.get("agents", []): + agents.append( + Agent( + id=a["id"], + tenant_id=a.get("tenant_id", ""), + name=a.get("name", "") or a.get("metadata", {}).get("name", ""), + reference_id=a.get("reference_id", ""), + metadata=a.get("metadata", {}), + created_at=a.get("created_at", 0), + token=f"fo_agent_{a['id']}", + ) + ) + return agents + + def get(self, agent_id: str) -> Agent: + """Get a single agent by ID.""" + resp = self._client._request("GET", f"/api/v1/sdk/agents/{agent_id}") + a = resp["agent"] + return Agent( + id=a["id"], + tenant_id=a.get("tenant_id", ""), + name=a.get("name", "") or a.get("metadata", {}).get("name", ""), + reference_id=a.get("reference_id", ""), + metadata=a.get("metadata", {}), + created_at=a.get("created_at", 0), + token=f"fo_agent_{a['id']}", + ) + + def delete(self, agent_id: str) -> None: + """Delete an agent principal.""" + self._client._request("DELETE", f"/api/v1/sdk/agents/{agent_id}") + + +class _ConnectionsResource: + def __init__(self, client: FirstOps): + self._client = client + + def register( + self, + *, + principal_id: str, + # Template-based registration (preferred for catalog entries) + template_id: str = "", + user_params: dict[str, str] | None = None, + # Raw registration (when not using a template) + name: str = "", + upstream_url: str = "", + auth_type: str = "", + transport_type: str = "", + upstream_headers: dict[str, str] | None = None, + upstream_query_params: dict[str, str] | None = None, + source: str = "sdk", + ) -> Connection: + """Register an MCP connection for an agent. + + Two modes: + + 1. **Template-based** (preferred): pass `template_id` from the + FirstOps catalog plus `user_params` for the required placeholders. + The backend resolves the upstream URL, pre-fills auth_type and + transport_type from the template, and maps params to headers or + query params per the template's `ParamDefinition.maps_to` rules. + + 2. **Raw**: pass `name` and `upstream_url` directly. Use this only + when registering a custom MCP server that isn't in the catalog. + + Example (template): + conn = client.connections.register( + principal_id=agent.id, + template_id="github-pat", + user_params={"api_token": "ghp_..."}, + ) + + Example (raw): + conn = client.connections.register( + principal_id=agent.id, + name="my-custom-mcp", + upstream_url="https://mcp.internal.corp/sse", + ) + """ + body: dict[str, Any] = { + "principal_id": principal_id, + "source": source, + } + + if template_id: + body["template_id"] = template_id + if user_params: + body["user_params"] = user_params + # Template flow still requires a name for the connection record. + # If the caller doesn't supply one, derive from the template ID. + body["name"] = name or template_id + # upstream_url is resolved server-side from the template. + body["upstream_url"] = upstream_url # empty is fine here + else: + if not name or not upstream_url: + raise ValueError( + "register() requires either template_id or " + "both name and upstream_url" + ) + body["name"] = name + body["upstream_url"] = upstream_url + + if auth_type: + body["auth_type"] = auth_type + if transport_type: + body["transport_type"] = transport_type + if upstream_headers: + body["upstream_headers"] = upstream_headers + if upstream_query_params: + body["upstream_query_params"] = upstream_query_params + + resp = self._client._request( + "POST", "/api/v1/sdk/connections/register", json=body + ) + c = resp["connection"] + return Connection( + id=c["id"], + tenant_id=c.get("tenant_id", ""), + principal_id=c.get("principal_id", ""), + name=c.get("mcp_server_name", "") or c.get("name", ""), + upstream_url=c.get("upstream_url", ""), + status=c.get("status", ""), + created_at=c.get("created_at", 0), + ) + + def list(self, *, principal_id: str | None = None) -> list[Connection]: + """List connections, optionally filtered by principal.""" + params: dict[str, str] = {} + if principal_id: + params["principal_id"] = principal_id + + resp = self._client._request("GET", "/api/v1/sdk/connections", params=params) + connections = [] + for c in resp.get("connections", []): + connections.append( + Connection( + id=c["id"], + tenant_id=c.get("tenant_id", ""), + principal_id=c.get("principal_id", ""), + name=c.get("mcp_server_name", "") or c.get("name", ""), + upstream_url=c.get("upstream_url", ""), + status=c.get("status", ""), + created_at=c.get("created_at", 0), + ) + ) + return connections + + def delete(self, connection_id: str) -> None: + """Delete a connection.""" + self._client._request("DELETE", f"/api/v1/sdk/connections/{connection_id}") + + +class _CatalogResource: + def __init__(self, client: FirstOps): + self._client = client + + def list( + self, + *, + principal_id: str | None = None, + search: str | None = None, + category: str | None = None, + ) -> list[ServerTemplate]: + """List MCP server templates in the FirstOps catalog. + + When `principal_id` is supplied, the response includes + `user_connected` and `existing_connection_id` overlays scoped to + that agent — useful when rendering "Connect" vs "Already connected" + states in a customer's UI. + """ + params: dict[str, str] = {} + if principal_id: + params["principal_id"] = principal_id + if search: + params["search"] = search + if category: + params["category"] = category + + resp = self._client._request( + "GET", "/api/v1/catalog/templates", params=params + ) + return [_parse_template(t) for t in resp.get("templates", [])] + + def get( + self, template_id: str, *, principal_id: str | None = None + ) -> ServerTemplate: + """Get a single catalog template by ID.""" + params: dict[str, str] = {} + if principal_id: + params["principal_id"] = principal_id + + resp = self._client._request( + "GET", f"/api/v1/catalog/templates/{template_id}", params=params + ) + return _parse_template(resp["template"]) + + +class FirstOps: + """Management client for the FirstOps API. + + Usage: + client = FirstOps(api_key="fo_key_...") + agent = client.agents.create(name="my-bot") + client.connections.register( + principal_id=agent.id, + name="slack", + upstream_url="https://mcp.slack.com/sse", + ) + """ + + def __init__( + self, + api_key: str, + base_url: str = "https://api.firstops.dev", + timeout: float = 30.0, + ): + if not api_key or not api_key.startswith("fo_key_"): + raise ValueError("api_key must start with 'fo_key_'") + + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._http = httpx.Client( + timeout=timeout, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + ) + self.agents = _AgentsResource(self) + self.connections = _ConnectionsResource(self) + self.catalog = _CatalogResource(self) + + def _request( + self, + method: str, + path: str, + *, + json: dict[str, Any] | None = None, + params: dict[str, str] | None = None, + ) -> dict[str, Any]: + resp = self._http.request( + method, + f"{self._base_url}{path}", + json=json, + params=params, + ) + if resp.status_code >= 400: + try: + body = resp.json() + msg = body.get("error", f"HTTP {resp.status_code}") + except Exception: + msg = f"HTTP {resp.status_code}" + raise FirstOpsError(resp.status_code, msg) + + if resp.status_code == 204 or not resp.content: + return {} + return resp.json() + + def close(self) -> None: + """Close the underlying HTTP client.""" + self._http.close() + + def __enter__(self) -> FirstOps: + return self + + def __exit__(self, *args: Any) -> None: + self.close() diff --git a/src/firstops/coverage.py b/src/firstops/coverage.py new file mode 100644 index 0000000..3d71bda --- /dev/null +++ b/src/firstops/coverage.py @@ -0,0 +1,65 @@ +"""Coverage honesty — surface what is and isn't governed. + +For a security product, a *silent* coverage gap is the worst failure. This +module makes the gaps loud: reconcile the tools an agent declares against the +tools FirstOps actually governs, and expose the per-adapter capability matrix +so no surface claims more than it delivers. +""" + +from __future__ import annotations + +from collections.abc import Iterable + +from firstops.tools import governed_tool_names + +# Per-surface capability: what each integration can actually enforce. +# block = can stop the call; scrub = can rewrite args/prompt before it runs; +# audit = emits the action to the audit trail. +CAPABILITY_MATRIX: dict[str, dict[str, bool]] = { + "base_decorator": {"block": True, "scrub": True, "audit": True}, + "claude": {"block": True, "scrub": True, "audit": True}, + "langgraph": {"block": True, "scrub": True, "audit": True}, + # OpenAI Agents tool guardrails are read-only: block + audit, but no + # argument scrub (use the base decorator to scrub on OpenAI Agents). + "openai_agents": {"block": True, "scrub": False, "audit": True}, + "llm_chain_link": {"block": True, "scrub": True, "audit": True}, + "mcp_proxy": {"block": True, "scrub": True, "audit": True}, +} + + +def capability(surface: str) -> dict[str, bool]: + """Return the capability dict for a surface, or {} if unknown.""" + return dict(CAPABILITY_MATRIX.get(surface, {})) + + +def ungoverned_tools(declared: Iterable[str]) -> list[str]: + """Return declared tool names NOT governed by ``@firstops.tool`` (a gap). + + Honesty caveats (read before trusting the result): + - This reconciles ONLY tools governed via the base decorator. A tool + governed by a harness adapter (at the framework's execution boundary) + will appear here as "ungoverned" even though it IS governed — adapter + coverage is per-registration, not per-name. + - The governed set is **process-wide**, not per-agent. With one agent per + process (the common case) that equals per-agent; with multiple agents in + one process it's the union, which can under-report a gap. + + Non-string entries are coerced to ``str`` so a malformed registry can't + crash the check. + """ + return sorted({str(d) for d in declared} - governed_tool_names()) + + +def coverage_report(declared: Iterable[str]) -> dict[str, list[str]]: + """Return a ``{governed, ungoverned}`` split of the declared tool names. + + Same caveats as :func:`ungoverned_tools` — ``ungoverned`` here means + "not decorator-governed" and may include adapter-governed tools; the + governed set is process-wide, not per-agent. + """ + declared_set = {str(d) for d in declared} + governed = governed_tool_names() + return { + "governed": sorted(declared_set & governed), + "ungoverned": sorted(declared_set - governed), + } diff --git a/src/firstops/enforcement.py b/src/firstops/enforcement.py new file mode 100644 index 0000000..aff7dfa --- /dev/null +++ b/src/firstops/enforcement.py @@ -0,0 +1,73 @@ +"""The enforcement spine — an in-process client of sentinel's EvaluateHook. + +Every governed action (tool call, LLM call) is forwarded here; sentinel +decides. The SDK performs **no local policy evaluation**. + +Failure semantics (invariant): enforcement **fails open** — on any transport +error, timeout, or non-200, the action is allowed and the failure is audited +locally. Authentication (DPoP) is the only fail-closed surface, and even an +auth failure never blocks the agent: it surfaces as a fail-open allow here. +""" + +from __future__ import annotations + +import logging + +import httpx + +from firstops._identity import Identity +from firstops.events import ActionEvent, Decision + +logger = logging.getLogger("firstops") + +_HOOK_PATH = "/api/v1/daemon/evaluate-hook" +_DEFAULT_TIMEOUT = 5.0 + + +class EnforcementClient: + """Forwards action events to sentinel and returns the Decision.""" + + def __init__( + self, + identity: Identity, + *, + timeout: float = _DEFAULT_TIMEOUT, + http: httpx.Client | None = None, + ): + self._identity = identity + self._url = identity.gateway_url + _HOOK_PATH + self._http = http or httpx.Client(timeout=timeout) + + def evaluate(self, event: ActionEvent) -> Decision: + """Evaluate one action. **Never raises** — fails open on any error. + + Enforcement is fail-open by invariant: a transport error, timeout, + non-200, malformed body, or any unexpected exception must allow the + action (and audit the failure), never block the agent. The only + fail-closed surface is DPoP auth, and even an auth rejection surfaces + here as a fail-open allow rather than a raised exception. + """ + try: + # htu binds to the path only (no query); _url has no query string. + proof = self._identity.proof("POST", self._url) + resp = self._http.post( + self._url, + json=event.to_wire(), + headers={ + "Authorization": f"Bearer {self._identity.bearer_token}", + "DPoP": proof, + "Content-Type": "application/json", + }, + ) + if resp.status_code != 200: + logger.warning( + "firstops enforcement: status %d, failing open", resp.status_code + ) + return Decision.fail_open(f"status {resp.status_code}") + return Decision.from_wire(resp.json()) + except Exception as e: # noqa: BLE001 - fail-open is the whole point + logger.warning("firstops enforcement: failing open: %s", e) + return Decision.fail_open(f"error: {e}") + + def close(self) -> None: + self._http.close() diff --git a/src/firstops/events.py b/src/firstops/events.py new file mode 100644 index 0000000..c8eceea --- /dev/null +++ b/src/firstops/events.py @@ -0,0 +1,195 @@ +"""Action-event model and wire (de)serialization for hook evaluation. + +Mirrors the backend ``hookwire`` JSON contract exactly (see +``backend/shared/lib/hookwire/types.go``): + + request : event_type, agent, tool_name, tool_input(obj), tool_output(obj), + session_id, cwd, channel, mcp{server,tool,url}, cli_version + response: decision, reason, modified_payload(base64 bytes), policy_id + +Principal and tenant are resolved server-side from the DPoP JWK thumbprint and +are never sent in the body. +""" + +from __future__ import annotations + +import base64 +import binascii +from dataclasses import dataclass +from typing import Any + +# --- channel constants (match enforcement/types.go Channel) --- +CHANNEL_MCP = "mcp" +CHANNEL_SYSTEM_TOOLS = "system_tools" +CHANNEL_LLM = "llm" + +# --- event types (match enforcement EventType) --- +EVENT_PRE_TOOL_USE = "pre_tool_use" +EVENT_POST_TOOL_USE = "post_tool_use" + +# --- decisions (match hookwire response decision) --- +DECISION_ALLOW = "allow" +DECISION_DENY = "deny" +DECISION_ASK = "ask" +DECISION_MODIFY = "modify" +_KNOWN_DECISIONS = frozenset( + {DECISION_ALLOW, DECISION_DENY, DECISION_ASK, DECISION_MODIFY} +) + +# Producer label for the `agent` field. The backend normalizer +# (FromHookRequest) maps this to audit Source `SourceSDK` ("sdk") per +# design-doc §7, so SDK actions are attributed correctly in audit. Keep this +# value in sync with the normalizer's `req.Agent == "firstops-sdk"` case. +SDK_AGENT_LABEL = "firstops-sdk" + + +@dataclass +class MCPInfo: + """MCP-specific metadata; populated only when ``channel == "mcp"``.""" + + server: str = "" + tool: str = "" + url: str = "" + + def to_wire(self) -> dict[str, str]: + out: dict[str, str] = {} + if self.server: + out["server"] = self.server + if self.tool: + out["tool"] = self.tool + if self.url: + out["url"] = self.url + return out + + +@dataclass +class ActionEvent: + """A single governable action, serializable to the hookwire request shape.""" + + event_type: str + tool_name: str + channel: str = "" + tool_input: dict[str, Any] | None = None + tool_output: dict[str, Any] | None = None + agent: str = SDK_AGENT_LABEL + session_id: str = "" + cwd: str = "" + mcp: MCPInfo | None = None + cli_version: str = "" + # The producer can apply a request-path modification before the action runs + # (decorator rebind / Claude updatedInput / LangGraph arg rewrite / LLM body + # rewrite). When True, sentinel ships `modify` for outbound scrub instead of + # escalating to deny. Adapters that CAN'T mutate (OpenAI guardrails) leave + # this False. + producer_can_apply_modify: bool = False + # Free-form producer metadata (e.g. {"harness": "langgraph"}). Merged into + # the event metadata server-side and surfaced in audit. + metadata: dict[str, str] | None = None + + def to_wire(self) -> dict[str, Any]: + body: dict[str, Any] = { + "event_type": self.event_type, + "agent": self.agent, + "tool_name": self.tool_name, + } + # tool_input/output are JSON objects (not stringified) so nested + # structure is preserved end-to-end. + if self.tool_input is not None: + body["tool_input"] = self.tool_input + if self.tool_output is not None: + body["tool_output"] = self.tool_output + if self.session_id: + body["session_id"] = self.session_id + if self.cwd: + body["cwd"] = self.cwd + if self.channel: + body["channel"] = self.channel + if self.mcp is not None: + mcp_wire = self.mcp.to_wire() + if mcp_wire: + body["mcp"] = mcp_wire + if self.cli_version: + body["cli_version"] = self.cli_version + if self.producer_can_apply_modify: + body["producer_can_apply_modify"] = True + if self.metadata: + body["metadata"] = self.metadata + return body + + +@dataclass +class Decision: + """The enforcement verdict for an action.""" + + action: str = DECISION_ALLOW + reason: str = "" + modified_payload: bytes | None = None + policy_id: str = "" + failed_open: bool = False # True when we allowed due to an infra failure + + @property + def blocked(self) -> bool: + return self.action == DECISION_DENY + + @property + def modified(self) -> bool: + return self.action == DECISION_MODIFY and self.modified_payload is not None + + @classmethod + def from_wire(cls, data: dict[str, Any]) -> Decision: + action = data.get("decision") or DECISION_ALLOW + reason = data.get("reason", "") or "" + policy_id = data.get("policy_id", "") or "" + + # An unrecognized verdict must not silently behave like a clean allow: + # fail open (never block on a verdict we can't act on) but set the + # failed_open flag so it is visible downstream, not indistinguishable + # from a real allow. + if action not in _KNOWN_DECISIONS: + return cls( + action=DECISION_ALLOW, + reason=f"unknown decision {action!r}: {reason}".rstrip(": "), + policy_id=policy_id, + failed_open=True, + ) + + raw = data.get("modified_payload") + payload: bytes | None = None + if raw: + try: + # Go marshals []byte as a base64 string. validate=True so a + # corrupted payload raises instead of silently decoding to + # wrong bytes that would then be applied to a live call. + payload = ( + base64.b64decode(raw, validate=True) + if isinstance(raw, str) + else bytes(raw) + ) + except (binascii.Error, ValueError): + return cls( + action=DECISION_ALLOW, + reason=f"invalid modified_payload: {reason}".rstrip(": "), + policy_id=policy_id, + failed_open=True, + ) + + # A modify verdict with no usable payload is malformed: fail open + # visibly rather than report a "modify" the caller can't apply. + if action == DECISION_MODIFY and payload is None: + return cls( + action=DECISION_ALLOW, + reason=f"modify without payload: {reason}".rstrip(": "), + policy_id=policy_id, + failed_open=True, + ) + + return cls( + action=action, + reason=reason, + modified_payload=payload, + policy_id=policy_id, + ) + + @classmethod + def fail_open(cls, reason: str) -> Decision: + return cls(action=DECISION_ALLOW, reason=reason, failed_open=True) diff --git a/src/firstops/integrations/__init__.py b/src/firstops/integrations/__init__.py new file mode 100644 index 0000000..5294f86 --- /dev/null +++ b/src/firstops/integrations/__init__.py @@ -0,0 +1,12 @@ +"""Harness adapters — one-touch governance for supported agent frameworks. + +Each adapter wires the FirstOps enforcement spine into a framework's official, +intervention-capable extension point: + + - ``firstops.integrations.claude`` — Claude Agent SDK ``PreToolUse`` hook + - ``firstops.integrations.langgraph`` — LangGraph agent middleware (wrap_tool_call) + - ``firstops.integrations.openai_agents`` — OpenAI Agents SDK tool guardrails + +Adapters import their framework lazily, so importing this package never +requires the frameworks to be installed. +""" diff --git a/src/firstops/integrations/_common.py b/src/firstops/integrations/_common.py new file mode 100644 index 0000000..4f8ab99 --- /dev/null +++ b/src/firstops/integrations/_common.py @@ -0,0 +1,132 @@ +"""Shared governance core for harness adapters. + +The adapters all reduce to: build a pre_tool_use event, ask sentinel, translate +the Decision into the framework's native verb (deny / mutate / allow). That +reduction lives here so every adapter shares one tested implementation. +""" + +from __future__ import annotations + +import json +from typing import Any + +from firstops.channels import classify, mcp_info +from firstops.events import CHANNEL_MCP, EVENT_PRE_TOOL_USE, ActionEvent, Decision + +# A neutral decision vocabulary the adapters translate into framework types. +ACTION_ALLOW = "allow" +ACTION_DENY = "deny" +ACTION_MODIFY = "modify" + +# Harness identifiers stamped into event metadata (audit/filtering). +HARNESS_LANGGRAPH = "langgraph" +HARNESS_CLAUDE = "claude-agent-sdk" +HARNESS_OPENAI_AGENTS = "openai-agents" + + +def _json_safe(value: Any) -> Any: + """Return a JSON-serializable view of ``value`` (str fallback).""" + try: + json.dumps(value) + return value + except (TypeError, ValueError): + return str(value) + + +def coerce_input(tool_input: Any) -> dict[str, Any]: + """Normalize a framework's tool input into a JSON-able dict for the event. + + Never raises and always returns a JSON-serializable dict (bytes decode with + ``errors="replace"``; non-serializable objects are stringified) so a tool + passing binary/exotic args can't crash the agent loop. + """ + if isinstance(tool_input, dict): + return {k: _json_safe(v) for k, v in tool_input.items()} + if isinstance(tool_input, bytes): + tool_input = tool_input.decode("utf-8", errors="replace") + if isinstance(tool_input, str): + try: + parsed = json.loads(tool_input) + if isinstance(parsed, dict): + return {k: _json_safe(v) for k, v in parsed.items()} + except (ValueError, TypeError): + pass + return {"input": tool_input} + if tool_input is None: + return {} + return {"input": _json_safe(tool_input)} + + +def govern_tool( + rt, + tool_name: str, + tool_input: Any, + can_apply_modify: bool = False, + harness: str = "", +) -> Decision: + """Evaluate a tool call via the enforcement spine. Allows if no runtime. + + ``can_apply_modify``: True when the adapter can rewrite the tool args before + execution (Claude updatedInput, LangGraph arg rewrite) — lets sentinel ship + a request-path scrub as ``modify`` instead of escalating to deny. Adapters + that can't mutate (OpenAI guardrails) leave it False. + + ``harness``: the producing framework, stamped into event metadata when known. + + Hardened to never raise into the agent loop: a None/odd tool_name and + non-serializable inputs are coerced rather than propagated. + """ + if rt is None: + return Decision(action="allow") + name = tool_name or "" + channel = classify(name) + event = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, + tool_name=name, + channel=channel, + # coerce_input already returns a fresh dict — no aliasing of the + # caller's live args (which frameworks may mutate after the call). + tool_input=coerce_input(tool_input), + mcp=mcp_info(name) if channel == CHANNEL_MCP else None, + producer_can_apply_modify=can_apply_modify, + metadata={"harness": harness} if harness else None, + ) + return rt.enforcement.evaluate(event) + + +def modified_input(decision: Decision) -> dict[str, Any] | None: + """Return the scrubbed input dict from a modify decision, or None.""" + if decision.modified and decision.modified_payload: + try: + value = json.loads(decision.modified_payload) + if isinstance(value, dict): + return value + except (ValueError, TypeError): + pass + return None + + +def decide( + rt, + tool_name: str, + tool_input: Any, + can_apply_modify: bool = False, + harness: str = "", +) -> tuple[str, Any]: + """Reduce a tool call to ``(action, payload)``: + + - ``("deny", reason)`` — block the call + - ``("modify", new_input_dict)`` — proceed with scrubbed input + - ``("allow", None)`` — proceed unchanged + + ``can_apply_modify`` and ``harness`` are forwarded to the event (see govern_tool). + """ + decision = govern_tool( + rt, tool_name, tool_input, can_apply_modify=can_apply_modify, harness=harness + ) + if decision.blocked: + return ACTION_DENY, decision.reason or "blocked by FirstOps policy" + new_input = modified_input(decision) + if new_input is not None: + return ACTION_MODIFY, new_input + return ACTION_ALLOW, None diff --git a/src/firstops/integrations/claude.py b/src/firstops/integrations/claude.py new file mode 100644 index 0000000..474698e --- /dev/null +++ b/src/firstops/integrations/claude.py @@ -0,0 +1,84 @@ +"""Claude Agent SDK adapter — the daemon model, in-process. + +A single ``PreToolUse`` hook governs every tool the agent calls: built-ins +(``Bash``, ``Write``, ``WebFetch``), MCP tools (``mcp__*``), and in-process +SDK-MCP tools — with both block (deny) and argument rewrite (updatedInput). + +Usage:: + + from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions + from firstops.integrations.claude import firstops_hooks + + options = ClaudeAgentOptions(hooks=firstops_hooks(fo)) + async with ClaudeSDKClient(options=options) as client: + ... + +Targets the Claude Agent SDK PreToolUse hook contract (hookSpecificOutput / +permissionDecision). The governance logic (`_govern_pre_tool_use`) is +framework-free and unit-tested; `firstops_hooks` is the thin lazily-imported +shell. +""" + +from __future__ import annotations + +from typing import Any + +from firstops import _runtime +from firstops.integrations._common import ( + ACTION_DENY, + ACTION_MODIFY, + HARNESS_CLAUDE, + decide, +) + + +def _govern_pre_tool_use(rt, input_data: dict[str, Any]) -> dict[str, Any]: + """Map a PreToolUse hook input to a hook response. Pure / testable. + + Returns ``{}`` (allow) when there's nothing to do, a deny envelope, or an + allow-with-updatedInput envelope for a scrub. + """ + tool_name = input_data.get("tool_name", "") or "" + tool_input = input_data.get("tool_input") + # Claude PreToolUse updatedInput can rewrite args → request-path scrub applies. + action, payload = decide( + rt, tool_name, tool_input, can_apply_modify=True, harness=HARNESS_CLAUDE + ) + if action == ACTION_DENY: + return { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": payload, + } + } + # Only emit an updatedInput envelope when there's a non-empty replacement — + # Claude's updatedInput is a FULL replacement of tool_input, so an empty + # dict would wipe every argument. Empty/odd payload → allow unchanged. + if action == ACTION_MODIFY and isinstance(payload, dict) and payload: + return { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "allow", + "updatedInput": payload, + } + } + return {} + + +def firstops_hooks(fo=None) -> dict[str, Any]: + """Return a Claude Agent SDK ``hooks`` config that governs all tool calls.""" + rt = fo if fo is not None else _runtime.runtime() + try: + from claude_agent_sdk import HookMatcher + except ImportError as e: # pragma: no cover - env-dependent + raise RuntimeError( + "claude-agent-sdk is not installed: pip install claude-agent-sdk" + ) from e + + async def _pre_tool_use(input_data, tool_use_id, context): + return _govern_pre_tool_use(rt, input_data) + + # matcher=None is the match-ALL contract; "*" would match only a tool + # literally named "*" (i.e. nothing) and silently disable governance. + return {"PreToolUse": [HookMatcher(matcher=None, hooks=[_pre_tool_use])]} diff --git a/src/firstops/integrations/langgraph.py b/src/firstops/integrations/langgraph.py new file mode 100644 index 0000000..2b133df --- /dev/null +++ b/src/firstops/integrations/langgraph.py @@ -0,0 +1,87 @@ +"""LangGraph adapter — agent middleware that governs every tool call. + +Built on LangChain v1 agent middleware (`wrap_tool_call`), which can block +(return a ToolMessage instead of running the tool) and mutate (rewrite the +tool args). One middleware governs all tools the agent calls, including +framework built-ins (`ShellTool`, `SQLDatabaseToolkit`, …) the developer never +authored. + +Usage:: + + from langchain.agents import create_agent + from firstops.integrations.langgraph import FirstOpsMiddleware + + agent = create_agent(model, tools=[...], middleware=[FirstOpsMiddleware(fo)]) + +Coverage honesty: middleware attaches per compiled graph and does NOT +auto-propagate into subgraphs. A subgraph built without FirstOps middleware is +ungoverned — wire one per graph. (Detect-and-warn for subgraphs is tracked for +a later milestone.) + +The decision logic is `firstops.integrations._common.decide` (framework-free, +tested); `FirstOpsMiddleware` is the thin lazily-imported shell targeting the +LangChain v1 middleware API. +""" + +from __future__ import annotations + +from firstops import _runtime +from firstops.integrations._common import ( + ACTION_DENY, + ACTION_MODIFY, + HARNESS_LANGGRAPH, + decide, +) + + +def FirstOpsMiddleware(fo=None): + """Return a LangChain agent middleware instance that governs tool calls.""" + rt = fo if fo is not None else _runtime.runtime() + try: + from langchain.agents.middleware import AgentMiddleware + from langchain_core.messages import ToolMessage + except ImportError as e: # pragma: no cover - env-dependent + raise RuntimeError( + "langchain/langgraph not installed: pip install langchain langgraph" + ) from e + + class _FirstOpsMiddleware(AgentMiddleware): + def wrap_tool_call(self, request, handler): + call = getattr(request, "tool_call", None) or {} + name = call.get("name", "") or "" + args = call.get("args", {}) or {} + action, payload = decide( + rt, name, args, can_apply_modify=True, harness=HARNESS_LANGGRAPH + ) + if action == ACTION_DENY: + return ToolMessage( + content=f"blocked by FirstOps policy: {payload}", + tool_call_id=call.get("id", ""), + status="error", + ) + if action == ACTION_MODIFY: + request = request.override( + tool_call={**request.tool_call, "args": payload} + ) + return handler(request) + + async def awrap_tool_call(self, request, handler): + call = getattr(request, "tool_call", None) or {} + name = call.get("name", "") or "" + args = call.get("args", {}) or {} + action, payload = decide( + rt, name, args, can_apply_modify=True, harness=HARNESS_LANGGRAPH + ) + if action == ACTION_DENY: + return ToolMessage( + content=f"blocked by FirstOps policy: {payload}", + tool_call_id=call.get("id", ""), + status="error", + ) + if action == ACTION_MODIFY: + request = request.override( + tool_call={**request.tool_call, "args": payload} + ) + return await handler(request) + + return _FirstOpsMiddleware() diff --git a/src/firstops/integrations/openai_agents.py b/src/firstops/integrations/openai_agents.py new file mode 100644 index 0000000..e1a8cd5 --- /dev/null +++ b/src/firstops/integrations/openai_agents.py @@ -0,0 +1,87 @@ +"""OpenAI Agents SDK adapter — tool input guardrails. + +Uses the SDK's tool input guardrail to block a tool call before it runs. + +**Attachment is per-tool, not per-agent.** OpenAI Agents has no agent-level +tool-input guardrail — `tool_input_guardrails` is a field on `@function_tool` / +`FunctionTool`. So wire the FirstOps guardrail onto each function tool:: + + from agents import function_tool + from firstops.integrations.openai_agents import firstops_tool_input_guardrail + + guard = firstops_tool_input_guardrail(fo) + + @function_tool(tool_input_guardrails=[guard]) + def send_email(to: str, body: str): ... + +Capability note (honest): OpenAI Agents tool guardrails are **read-only** — they +can block but cannot mutate tool arguments. So argument *scrub* is not available +through this adapter; a ``modify`` decision degrades to allow here. To scrub tool +args on OpenAI Agents, wrap the underlying function with ``@firstops.tool`` +instead (the base-API decorator). + +The decision logic (`_decide_tool`) is framework-free and tested; the guardrail +wiring is the thin lazily-imported shell. +""" + +from __future__ import annotations + +from typing import Any + +from firstops import _runtime +from firstops.integrations._common import ( + ACTION_DENY, + ACTION_MODIFY, + HARNESS_OPENAI_AGENTS, + decide, +) + + +def _decide_tool(rt, tool_name: str, tool_input: Any) -> tuple[bool, str]: + """Return ``(blocked, reason)`` for a tool call. + + Guardrails can't mutate args (read-only). We leave ``can_apply_modify`` + False, so sentinel escalates a request-path scrub to deny — but defensively, + if a ``modify`` ever reaches here we **block** (fail closed) rather than let + unscrubbed arguments through. To scrub on OpenAI Agents, use ``@firstops.tool``. + """ + action, payload = decide(rt, tool_name, tool_input, harness=HARNESS_OPENAI_AGENTS) + if action == ACTION_DENY: + return True, str(payload) + if action == ACTION_MODIFY: + return True, ( + "scrub required but tool-arg rewrite is unsupported on OpenAI Agents " + "(use @firstops.tool to scrub)" + ) + return False, "" + + +def firstops_tool_input_guardrail(fo=None): + """Return a single reusable tool-input guardrail to attach per function tool. + + Pass it in each tool's ``tool_input_guardrails=[...]`` list. + """ + rt = fo if fo is not None else _runtime.runtime() + try: + from agents import tool_input_guardrail + from agents.tool_guardrails import ToolGuardrailFunctionOutput + except ImportError as e: # pragma: no cover - env-dependent + raise RuntimeError( + "openai-agents not installed: pip install openai-agents" + ) from e + + @tool_input_guardrail + async def _fo_tool_input_guardrail(data): + # ToolInputGuardrailData carries .context (ToolContext) and .agent. + # tool_arguments is a raw JSON string; coerce_input handles that. + ctx = getattr(data, "context", None) + tool_name = getattr(ctx, "tool_name", "") or "" + tool_args = getattr(ctx, "tool_arguments", None) + blocked, reason = _decide_tool(rt, tool_name, tool_args) + if blocked: + return ToolGuardrailFunctionOutput.reject_content( + message=f"blocked by FirstOps policy: {reason}" + ) + return ToolGuardrailFunctionOutput.allow() + + return _fo_tool_input_guardrail diff --git a/src/firstops/llm.py b/src/firstops/llm.py new file mode 100644 index 0000000..0145875 --- /dev/null +++ b/src/firstops/llm.py @@ -0,0 +1,51 @@ +"""Convenience helpers for pointing LLM clients at the sidecar chain-link. + +These are optional sugar over :func:`firstops.llm_base_url`. The SDK does not +depend on ``openai`` / ``anthropic`` — the client factories import them lazily +and raise a clear error if the package is absent. +""" + +from __future__ import annotations + +import os +from typing import Any + +from firstops._runtime import llm_base_url + + +def openai_client(**kwargs: Any): + """Return an ``openai.OpenAI`` client pointed at the sidecar. + + Pass your real ``api_key`` as usual (or set ``OPENAI_API_KEY``) — it is + forwarded verbatim to the upstream; FirstOps never stores it. + """ + try: + import openai + except ImportError as e: # pragma: no cover - env-dependent + raise RuntimeError("openai is not installed: pip install openai") from e + kwargs.setdefault("base_url", llm_base_url("openai")) + return openai.OpenAI(**kwargs) + + +def anthropic_client(**kwargs: Any): + """Return an ``anthropic.Anthropic`` client pointed at the sidecar.""" + try: + import anthropic + except ImportError as e: # pragma: no cover - env-dependent + raise RuntimeError("anthropic is not installed: pip install anthropic") from e + kwargs.setdefault("base_url", llm_base_url("anthropic")) + return anthropic.Anthropic(**kwargs) + + +def configure_llm_env() -> dict[str, str]: + """Set ``OPENAI_BASE_URL`` / ``ANTHROPIC_BASE_URL`` to the sidecar. + + For frameworks that construct their own LLM client internally and only + honor the env vars. Returns the variables it set. + """ + env = { + "OPENAI_BASE_URL": llm_base_url("openai"), + "ANTHROPIC_BASE_URL": llm_base_url("anthropic"), + } + os.environ.update(env) + return env diff --git a/src/firstops/proxy.py b/src/firstops/proxy.py index 5459fd6..3077f8a 100644 --- a/src/firstops/proxy.py +++ b/src/firstops/proxy.py @@ -1,22 +1,63 @@ -"""Sidecar proxy that adds DPoP auth headers to MCP requests.""" +"""Sidecar proxy — a dual-mode local forward proxy. +Two routes share one local listener: + + ``/mcp/...`` terminal to the FirstOps gateway (DPoP-signed, + credential-brokered) — the original behavior. + ``/llm//...`` inline chain-link: governs the request via + EvaluateHook(channel=llm), then forwards to the + customer-configured upstream (their gateway or the + provider). FirstOps is NOT in the LLM data path; the + agent's own Authorization is passed through verbatim. +""" + +import json import logging import threading from http.server import HTTPServer, BaseHTTPRequestHandler from socketserver import ThreadingMixIn -from urllib.parse import urlparse, urlunparse +from urllib.parse import urlparse import httpx -from firstops.dpop import create_proof, load_private_key +from firstops._identity import Identity, build_identity +from firstops.events import CHANNEL_LLM, EVENT_PRE_TOOL_USE, ActionEvent logger = logging.getLogger("firstops") +# Headers we must not forward verbatim to an LLM upstream: the RFC 7230 +# hop-by-hop set plus the few httpx sets itself. The agent's Authorization +# (its model key) is intentionally NOT here — it passes through verbatim. +_LLM_STRIP_HEADERS = frozenset( + { + "host", + "content-length", + "accept-encoding", + "connection", + "keep-alive", + "transfer-encoding", + "te", + "trailer", + "upgrade", + "proxy-authorization", + "proxy-authenticate", + } +) + +# Hard cap on a forwarded request body (defensive against a huge Content-Length). +_MAX_BODY_BYTES = 100 * 1024 * 1024 + _DEFAULT_PORT = 9322 -_DEFAULT_GATEWAY = "https://api.firstops.ai" +_DEFAULT_GATEWAY = "https://api.firstops.dev" +# Module-global proxy state. Guarded by _lock for thread safety. +_lock = threading.Lock() _server: HTTPServer | None = None _server_thread: threading.Thread | None = None +_current_agent_id: str | None = None +_current_port: int | None = None +_current_gateway: str | None = None +_current_jkt: str | None = None def init( @@ -24,70 +65,264 @@ def init( private_key_pem: str, port: int = _DEFAULT_PORT, gateway_url: str = _DEFAULT_GATEWAY, + enforcement=None, + llm_upstreams: dict[str, str] | None = None, ) -> None: """Start the sidecar proxy on localhost. + This is **idempotent** for the same agent: calling init() twice with the + same ``agent_id``, ``port``, and ``gateway_url`` is a no-op on the second + call. This lets library code call init() defensively without worrying + about coordinating with other callers in the same process. + + Calling init() with a **different** agent, port, or gateway while the + proxy is already running raises ``RuntimeError`` — the sidecar is tied to + one agent's private key and cannot serve multiple identities from a single + instance. Call ``shutdown()`` first if you need to switch agents. + Args: - agent_id: The agent principal ID (without fo_agent_ prefix). + agent_id: The agent principal ID (without ``fo_agent_`` prefix). private_key_pem: EC P-256 private key in PEM format. port: Local port for the proxy (default 9322). gateway_url: FirstOps gateway base URL. - """ - global _server, _server_thread - if _server is not None: - raise RuntimeError("firstops proxy already running") + Raises: + RuntimeError: If a proxy is already running for a different agent, + port, or gateway. + ValueError: If ``gateway_url`` is malformed. + """ + global _server, _server_thread, _current_agent_id, _current_port, _current_gateway + global _current_jkt - key = load_private_key(private_key_pem) - bearer_token = f"fo_agent_{agent_id}" gateway = gateway_url.rstrip("/") - # Validate gateway URL - parsed = urlparse(gateway) - if not parsed.scheme or not parsed.netloc: - raise ValueError(f"invalid gateway_url: {gateway_url}") + # Build identity up front (validates gateway URL + key) so bad input fails + # fast and so we can compare the key thumbprint on the idempotent path. + identity = build_identity(agent_id, private_key_pem, gateway) + jkt = identity.jkt + + with _lock: + if _server is not None: + same_target = ( + _current_agent_id == agent_id + and _current_port == port + and _current_gateway == gateway + ) + if same_target and _current_jkt == jkt: + logger.debug( + "firstops proxy already running for agent %s on port %d — init() is a no-op", + agent_id, + port, + ) + return + if same_target: + # Same agent/port/gateway but a DIFFERENT key — refuse to + # silently keep signing with the stale key (that would 401 and + # fail open invisibly). Force an explicit re-key. + raise RuntimeError( + f"firstops proxy is already running for agent {agent_id!r} " + f"with a different key; call shutdown() before re-keying." + ) + + # Mismatch — refuse to silently replace the running sidecar. + raise RuntimeError( + f"firstops proxy is already running for agent " + f"{_current_agent_id!r} on port {_current_port} " + f"(gateway={_current_gateway!r}); cannot start a second " + f"instance for agent {agent_id!r}. Call shutdown() first " + f"if you want to switch identities." + ) - handler_class = _make_handler(key, bearer_token, gateway, port) + handler_class = _make_handler(identity, port, enforcement, llm_upstreams) - class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - daemon_threads = True + class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True - _server = ThreadingHTTPServer(("127.0.0.1", port), handler_class) - _server_thread = threading.Thread(target=_server.serve_forever, daemon=True) - _server_thread.start() - logger.info("firstops proxy listening on 127.0.0.1:%d", port) + _server = ThreadingHTTPServer(("127.0.0.1", port), handler_class) + _server_thread = threading.Thread( + target=_server.serve_forever, daemon=True + ) + _server_thread.start() + _current_agent_id = agent_id + _current_port = port + _current_gateway = gateway + _current_jkt = jkt + logger.info( + "firstops proxy listening on 127.0.0.1:%d for agent %s", + port, + agent_id, + ) def shutdown() -> None: - """Stop the sidecar proxy.""" - global _server, _server_thread - if _server is not None: - _server.shutdown() - _server = None - _server_thread = None - logger.info("firstops proxy stopped") + """Stop the sidecar proxy. Idempotent — no-op if not running.""" + global _server, _server_thread, _current_agent_id, _current_port, _current_gateway + global _current_jkt + with _lock: + if _server is not None: + _server.shutdown() + _server.server_close() # release the listening socket, not just the loop + _server = None + _server_thread = None + _current_agent_id = None + _current_port = None + _current_gateway = None + _current_jkt = None + logger.info("firstops proxy stopped") + + +def is_running() -> bool: + """Return True if the sidecar proxy is currently running in this process.""" + with _lock: + return _server is not None + + +def current_agent_id() -> str | None: + """Return the agent ID the running proxy is serving, or None if not running.""" + with _lock: + return _current_agent_id + + +def _denial_body(provider: str, decision) -> bytes: + """Build a provider-shaped error envelope for a denied LLM request.""" + msg = f"blocked by FirstOps policy: {decision.reason}" + if provider == "anthropic": + env = { + "type": "error", + "error": {"type": "firstops_policy_violation", "message": msg}, + } + else: # openai-shaped default + env = { + "error": { + "message": msg, + "type": "firstops_policy_violation", + "policy_id": decision.policy_id, + } + } + return json.dumps(env).encode() -def _make_handler(key, bearer_token: str, gateway: str, local_port: int): - """Create a request handler class bound to the given config.""" +def _make_handler(identity: Identity, local_port: int, enforcement, llm_upstreams): + """Create a request handler class bound to the given identity + config.""" + gateway = identity.gateway_url # Pre-create a client for non-streaming requests client = httpx.Client(timeout=httpx.Timeout(120.0, connect=10.0)) class ProxyHandler(BaseHTTPRequestHandler): def do_POST(self): - self._proxy("POST") + self._dispatch("POST") def do_GET(self): - self._proxy("GET") + self._dispatch("GET") def do_DELETE(self): - self._proxy("DELETE") + self._dispatch("DELETE") def log_message(self, fmt, *args): logger.debug(fmt, *args) - def _proxy(self, method: str): + def _dispatch(self, method: str): + # Route by path prefix: LLM chain-link vs MCP terminal. + if self.path.startswith("/llm/"): + self._handle_llm(method) + else: + self._proxy_mcp(method) + + # ---- LLM chain-link route ------------------------------------- + def _handle_llm(self, method: str): + if enforcement is None or llm_upstreams is None: + self.send_error(503, "LLM governance not configured") + return + + # /llm// + rest = self.path[len("/llm/"):] + provider, _, tail = rest.partition("/") + upstream_base = llm_upstreams.get(provider) + if not upstream_base: + self.send_error(502, f"unknown LLM provider: {provider!r}") + return + + path_only, sep, query = tail.partition("?") + # Reject path traversal: the path we govern MUST equal the path we + # forward. Letting httpx normalize `..` would desync the two. + if ".." in path_only.split("/"): + self.send_error(400, "invalid path") + return + + body = self._read_request_body() + if body is None: + self.send_error(400, "invalid or oversized request body") + return + + target = upstream_base.rstrip("/") + "/" + path_only + if sep: + target += "?" + query + + # Pre-request governance (channel=llm). Returns (forward_body, denial). + body, denial = self._govern_llm(provider, path_only, body) + if denial is not None: + self.send_response(403) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(denial) + return + + # Forward everything, strip the few headers we must replace. The + # agent's own Authorization (its model key) passes through verbatim; + # we never DPoP-sign an upstream that isn't the FirstOps gateway. + fwd_headers = { + k: v + for k, v in self.headers.items() + if k.lower() not in _LLM_STRIP_HEADERS + } + self._forward(method, target, fwd_headers, body or None) + + def _read_request_body(self) -> bytes | None: + """Read the request body safely. Returns None on malformed/oversized + framing (caller replies 400). No chunked-request support — a missing + Content-Length is treated as an empty body.""" + raw = self.headers.get("Content-Length") + if raw is None: + return b"" + try: + n = int(raw) + except ValueError: + return None + if n < 0 or n > _MAX_BODY_BYTES: + return None + return self.rfile.read(n) if n > 0 else b"" + + def _govern_llm(self, provider: str, path_only: str, body: bytes): + """Evaluate an LLM request. Returns (body_to_forward, denial_or_None).""" + if not body: + return body, None + try: + parsed = json.loads(body) + except (ValueError, TypeError): + return body, None # non-JSON — can't govern, forward as-is + if not isinstance(parsed, dict): + return body, None + + tool_name = f"{provider}." + path_only.strip("/").replace("/", ".") + event = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, + tool_name=tool_name, + channel=CHANNEL_LLM, + tool_input=parsed, + # The sidecar rewrites the request body before forwarding, so a + # prompt scrub can ship as modify rather than escalate to deny. + producer_can_apply_modify=True, + ) + decision = enforcement.evaluate(event) + if decision.blocked: + return body, _denial_body(provider, decision) + if decision.modified and decision.modified_payload: + return decision.modified_payload, None + return body, None + + # ---- MCP terminal route (original behavior) ------------------- + def _proxy_mcp(self, method: str): path = self.path # e.g. /mcp/proxy/ or /mcp/sse/message?... gateway_url = gateway + path @@ -95,14 +330,12 @@ def _proxy(self, method: str): content_length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(content_length) if content_length > 0 else None - # Create DPoP proof against the gateway URL (path only, no query for htu) - parsed = urlparse(gateway_url) - htu = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", "")) - proof = create_proof(key, method, htu) + # Create DPoP proof; identity.proof canonicalizes htu (strips query). + proof = identity.proof(method, gateway_url) # Build upstream headers upstream_headers = { - "Authorization": f"Bearer {bearer_token}", + "Authorization": f"Bearer {identity.bearer_token}", "DPoP": proof, } # Forward relevant headers @@ -131,23 +364,20 @@ def _forward(self, method: str, url: str, headers: dict, body: bytes | None): method, url, headers=headers, content=body, timeout=httpx.Timeout(120.0, connect=10.0), ) as resp: - content_type = resp.headers.get("content-type", "") - is_sse = "text/event-stream" in content_type - self.send_response(resp.status_code) for k, v in resp.headers.items(): if k.lower() not in ("transfer-encoding", "connection"): self.send_header(k, v) self.end_headers() - if is_sse: - # Stream SSE chunks with flushing so the MCP client - # receives events incrementally instead of timing out. - for chunk in resp.iter_bytes(): - self.wfile.write(chunk) - self.wfile.flush() - else: - self.wfile.write(resp.read()) + # Forward the body RAW: httpx auto-decompresses iter_bytes()/ + # read(), but we keep the upstream Content-Encoding/Length + # headers, so we must pass the original (possibly gzipped) + # bytes through untouched or the client's decode fails. + # Flush per chunk so SSE/streaming responses arrive live. + for chunk in resp.iter_raw(): + self.wfile.write(chunk) + self.wfile.flush() except httpx.HTTPError as e: logger.error("upstream request failed: %s", e) self.send_error(502, "upstream request failed") diff --git a/src/firstops/tools.py b/src/firstops/tools.py new file mode 100644 index 0000000..0926f91 --- /dev/null +++ b/src/firstops/tools.py @@ -0,0 +1,318 @@ +"""The base-API tool decorator — govern any callable. + +``@firstops.tool`` wraps a function so each call is forwarded to sentinel as a +``pre_tool_use`` event (block / modify args) and a ``post_tool_use`` event +(audit). It is the harness-agnostic floor: it audits everywhere and blocks +where the surrounding harness propagates a raised exception (per the design's +Discovery A, raising is the block mechanism for the base layer). + +Coverage is honest: a decorated tool is governed; an un-decorated one is not. +For framework built-ins the developer never authored, use the harness adapter +instead (it hooks the execution boundary). +""" + +from __future__ import annotations + +import functools +import inspect +import json +import logging +from typing import Any, Callable + +from firstops import _runtime +from firstops.channels import classify, mcp_info +from firstops.events import ( + CHANNEL_MCP, + EVENT_POST_TOOL_USE, + EVENT_PRE_TOOL_USE, + ActionEvent, + Decision, +) + +logger = logging.getLogger("firstops") + +# Attributes that mark an already-built framework tool object (LangChain +# StructuredTool, LlamaIndex FunctionTool, etc.). We refuse to wrap these — +# the user must decorate the underlying function before the framework wraps it. +_TOOL_OBJECT_MARKERS = ("invoke", "ainvoke", "args_schema", "_run") + + +# Names of tools governed by @firstops.tool this process — used by +# firstops.coverage to reconcile declared-vs-governed and surface gaps. +_GOVERNED_TOOLS: set[str] = set() + + +def governed_tool_names() -> set[str]: + """Return the set of tool names governed by @firstops.tool.""" + return set(_GOVERNED_TOOLS) + + +class FirstOpsPolicyError(Exception): + """Raised when sentinel denies a governed tool call.""" + + def __init__(self, tool_name: str, reason: str, policy_id: str = ""): + self.tool_name = tool_name + self.reason = reason + self.policy_id = policy_id + super().__init__(f"FirstOps blocked tool {tool_name!r}: {reason}") + + +def tool(fn: Callable | None = None, *, name: str | None = None): + """Decorator that governs a tool function. Usable as ``@tool`` or ``@tool(name=...)``.""" + + def decorator(func: Callable) -> Callable: + _check_wrappable(func) + tool_name = name or getattr(func, "__name__", "tool") + # Register BOTH the governance name and the function's __name__ so a + # coverage check that enumerates tools by either key sees it governed. + _GOVERNED_TOOLS.add(tool_name) + fn_name = getattr(func, "__name__", None) + if fn_name: + _GOVERNED_TOOLS.add(fn_name) + + # Order matters: async-gen and generator are NOT coroutine functions, + # so they must be detected first or they'd fall into the sync wrapper + # and return an un-iterated (async)generator with the body unexecuted. + if inspect.isasyncgenfunction(func): + # Govern EAGERLY at call time (the wrapper is a plain function that + # returns the async generator), so a DENY blocks before any + # iteration rather than on the first __anext__. + @functools.wraps(func) + def agwrapper(*args: Any, **kwargs: Any): + args, kwargs = _govern(func, tool_name, args, kwargs) + return _agen_iterate(func, tool_name, args, kwargs) + + return agwrapper + + if inspect.isgeneratorfunction(func): + + @functools.wraps(func) + def gwrapper(*args: Any, **kwargs: Any): + args, kwargs = _govern(func, tool_name, args, kwargs) + return _gen_iterate(func, tool_name, args, kwargs) + + return gwrapper + + if inspect.iscoroutinefunction(func): + + @functools.wraps(func) + async def awrapper(*args: Any, **kwargs: Any) -> Any: + args, kwargs = _govern(func, tool_name, args, kwargs) + result = None + error: Exception | None = None + try: + result = await func(*args, **kwargs) + return result + except Exception as e: + error = e + raise + finally: + _audit_post(tool_name, result, error) + + return awrapper + + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + args, kwargs = _govern(func, tool_name, args, kwargs) + result = None + error: Exception | None = None + try: + result = func(*args, **kwargs) + return result + except Exception as e: + error = e + raise + finally: + _audit_post(tool_name, result, error) + + return wrapper + + if fn is not None: + return decorator(fn) + return decorator + + +# Sentinel marker for "the result is a stream we didn't materialize". +_STREAM_SENTINEL = object() + + +def _gen_iterate(func, tool_name, args, kwargs): + """Drive a generator tool, auditing once after it's exhausted/aborted.""" + error: Exception | None = None + try: + yield from func(*args, **kwargs) + except Exception as e: + error = e + raise + finally: + _audit_post(tool_name, _STREAM_SENTINEL, error) + + +async def _agen_iterate(func, tool_name, args, kwargs): + """Async analogue of :func:`_gen_iterate`.""" + error: Exception | None = None + try: + async for item in func(*args, **kwargs): + yield item + except Exception as e: + error = e + raise + finally: + _audit_post(tool_name, _STREAM_SENTINEL, error) + + +def _check_wrappable(func: Callable) -> None: + if ( + inspect.isfunction(func) + or inspect.ismethod(func) + or inspect.iscoroutinefunction(func) + or inspect.isgeneratorfunction(func) + or inspect.isasyncgenfunction(func) + ): + return + if callable(func) and any(hasattr(func, m) for m in _TOOL_OBJECT_MARKERS): + raise TypeError( + f"@firstops.tool expects a plain function, not a built framework tool " + f"object ({type(func).__name__}). Decorate the underlying function " + f"before the framework wraps it (put @firstops.tool innermost)." + ) + if not callable(func): + raise TypeError(f"@firstops.tool expects a callable, got {type(func).__name__}") + # Other callables (functools.partial, lambdas) are allowed. + + +def _govern( + func: Callable, tool_name: str, args: tuple, kwargs: dict +) -> tuple[tuple, dict]: + """Run the pre_tool_use evaluation and apply the decision to the call args.""" + tool_input = _bind_inputs(func, args, kwargs) + decision = _evaluate_pre(tool_name, tool_input) + return _apply_pre(func, decision, tool_name, args, kwargs) + + +def _jsonable(value: Any) -> Any: + try: + json.dumps(value) + return value + except (TypeError, ValueError): + return repr(value) + + +def _bind_inputs(func: Callable, args: tuple, kwargs: dict) -> dict[str, Any]: + """Map a call's args/kwargs to a JSON-able {param: value} dict.""" + try: + bound = inspect.signature(func).bind_partial(*args, **kwargs) + bound.apply_defaults() + return {k: _jsonable(v) for k, v in bound.arguments.items()} + except (TypeError, ValueError): + out: dict[str, Any] = {f"arg{i}": _jsonable(a) for i, a in enumerate(args)} + out.update({k: _jsonable(v) for k, v in kwargs.items()}) + return out + + +def _evaluate_pre(tool_name: str, tool_input: dict[str, Any]) -> Decision | None: + rt = _runtime.runtime() + if rt is None: + return None # not initialized — no governance, run normally + channel = classify(tool_name) + event = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, + tool_name=tool_name, + channel=channel, + tool_input=tool_input, + mcp=mcp_info(tool_name) if channel == CHANNEL_MCP else None, + # The decorator rebinds args from a modify payload, so request-path + # scrub is applicable — let sentinel ship modify, not escalate to deny. + producer_can_apply_modify=True, + ) + return rt.enforcement.evaluate(event) + + +def _apply_pre( + func: Callable, decision: Decision | None, tool_name: str, args: tuple, kwargs: dict +) -> tuple[tuple, dict]: + if decision is None: + return args, kwargs + if decision.blocked: + raise FirstOpsPolicyError(tool_name, decision.reason, decision.policy_id) + if decision.modified and decision.modified_payload: + rebound = _rebind(func, decision.modified_payload, args, kwargs) + if rebound is not None: + return rebound + # We signalled producer_can_apply_modify, so sentinel shipped a scrub + # instead of a deny. If it doesn't fit the signature, fail CLOSED — + # running the tool with unscrubbed args is worse than blocking. + logger.warning( + "firstops: could not apply modify to %s; blocking (fail closed)", tool_name + ) + raise FirstOpsPolicyError( + tool_name, + "policy required a modification this tool could not apply", + decision.policy_id, + ) + return args, kwargs + + +def _rebind( + func: Callable, payload: bytes, args: tuple, kwargs: dict +) -> tuple[tuple, dict] | None: + """Overlay sentinel's scrubbed inputs onto the call, respecting param kinds. + + Returns (args, kwargs) honoring positional-only / *args / **kwargs, or None + if the payload can't be applied (caller then falls open to original args). + """ + try: + new_input = json.loads(payload) + except (ValueError, TypeError): + return None + if not isinstance(new_input, dict): + return None + try: + sig = inspect.signature(func) + bound = sig.bind_partial(*args, **kwargs) + bound.apply_defaults() + merged = dict(bound.arguments) + merged.update(new_input) + + out_args: list[Any] = [] + out_kwargs: dict[str, Any] = {} + consumed = set() + for pname, param in sig.parameters.items(): + if param.kind == inspect.Parameter.VAR_POSITIONAL: + out_args.extend(merged.get(pname, ()) or ()) + elif param.kind == inspect.Parameter.VAR_KEYWORD: + out_kwargs.update(merged.get(pname, {}) or {}) + elif pname in merged: + if param.kind == inspect.Parameter.POSITIONAL_ONLY: + out_args.append(merged[pname]) + else: + out_kwargs[pname] = merged[pname] + consumed.add(pname) + # Validate the reconstruction actually binds before returning it. + sig.bind(*out_args, **out_kwargs) + return tuple(out_args), out_kwargs + except (TypeError, ValueError): + return None + + +def _audit_post(tool_name: str, result: Any, error: Exception | None = None) -> None: + """Emit a post_tool_use audit event. Best-effort: never affects the call.""" + rt = _runtime.runtime() + if rt is None: + return + try: + if error is not None: + output: dict[str, Any] = {"error": repr(error)} + elif result is _STREAM_SENTINEL: + output = {"result": ""} + else: + output = {"result": _jsonable(result)} + event = ActionEvent( + event_type=EVENT_POST_TOOL_USE, + tool_name=tool_name, + channel=classify(tool_name), + tool_output=output, + ) + rt.enforcement.evaluate(event) + except Exception as e: # pragma: no cover - defensive + logger.debug("firstops post-eval failed for %s: %s", tool_name, e) diff --git a/tests/test_channels.py b/tests/test_channels.py new file mode 100644 index 0000000..d259ba6 --- /dev/null +++ b/tests/test_channels.py @@ -0,0 +1,68 @@ +"""Channel classification table — classify() and mcp_info(). + +The classifier mirrors the daemon's tool_name -> channel mapping. These are +table-driven and probe the malformed-name boundary: too-few parts, empty tool, +the literal prefix, casing, and unicode. +""" + +from __future__ import annotations + +import pytest + +from firstops.channels import classify, mcp_info +from firstops.events import CHANNEL_MCP, CHANNEL_SYSTEM_TOOLS + + +@pytest.mark.parametrize( + "name,expected", + [ + ("mcp__github__create_issue", CHANNEL_MCP), + ("mcp__s__t", CHANNEL_MCP), + ("mcp__", CHANNEL_SYSTEM_TOOLS), # malformed -> system_tools (agrees w/ mcp_info) + ("mcp__server__", CHANNEL_SYSTEM_TOOLS), # empty tool -> system_tools + ("mcp", CHANNEL_SYSTEM_TOOLS), # not the prefix + ("MCP__x__y", CHANNEL_SYSTEM_TOOLS), # case-sensitive + ("bash", CHANNEL_SYSTEM_TOOLS), + ("read_file", CHANNEL_SYSTEM_TOOLS), + ("", CHANNEL_SYSTEM_TOOLS), + ("mcp__café__tool", CHANNEL_MCP), # unicode server name + ], +) +def test_classify_table(name, expected): + assert classify(name) == expected + + +@pytest.mark.parametrize( + "name,server,tool", + [ + ("mcp__github__create_issue", "github", "create_issue"), + ("mcp__s__a__b", "s", "a__b"), # tool may itself contain __ + ("mcp__café__tool", "café", "tool"), # unicode preserved + ], +) +def test_mcp_info_parses_server_and_tool(name, server, tool): + info = mcp_info(name) + assert info is not None + assert info.server == server + assert info.tool == tool + + +@pytest.mark.parametrize("name", ["mcp__", "mcp", "bash", "", "mcp_single"]) +def test_mcp_info_returns_none_for_unparseable(name): + assert mcp_info(name) is None + + +def test_mcp_info_empty_tool_is_rejected_and_classified_system(): + """FIXED: `mcp__server__` (empty tool) is malformed — mcp_info returns None + and classify falls to system_tools, so the two never disagree (we never emit + a channel=mcp event with no mcp metadata).""" + assert classify("mcp__server__") == CHANNEL_SYSTEM_TOOLS + assert mcp_info("mcp__server__") is None + + +def test_classify_and_mcp_info_fully_agree(): + """FIXED: classify == MCP iff mcp_info parses. No exceptions — the degenerate + literal `mcp__` is system_tools and yields no info.""" + for name in ("mcp__", "mcp__server__", "mcp__github__create_issue", "bash"): + is_mcp = mcp_info(name) is not None + assert (classify(name) == CHANNEL_MCP) == is_mcp diff --git a/tests/test_contract_parity.py b/tests/test_contract_parity.py new file mode 100644 index 0000000..74bf679 --- /dev/null +++ b/tests/test_contract_parity.py @@ -0,0 +1,103 @@ +"""SDK ↔ Go hookwire contract parity (M0.T0.3 / design §I.1, the blocking DoR). + +This is the SDK half of the golden-fixture parity pinned on the Go side in +backend/shared/lib/hookwire/sdk_contract_test.go. The two halves together +guarantee the wire contract cannot drift undetected: + + - Go test: decodes the SDK's request JSON into EvaluateHookRequestJSON and + asserts ToProto preserves nested structure. + - This test: decodes the EXACT bytes Go's encoding/json produces for an + EvaluateHookResponseJSON into Decision.from_wire and asserts the + base64 modified_payload round-trips. + +GO_RESPONSE_JSON below is captured VERBATIM from: + json.Marshal(hookwire.EvaluateHookResponseJSON{ + Decision:"modify", Reason:"PII detected", + ModifiedPayload:[]byte(`{"to":"[REDACTED]"}`), PolicyID:"pii-1"}) + +If the Go response shape changes, regenerate this constant and the Go fixture +in lockstep — a divergence between them is the contract break we're guarding. +""" + +from __future__ import annotations + +import json + +from firstops.events import ( + DECISION_ALLOW, + DECISION_MODIFY, + ActionEvent, + Decision, + MCPInfo, +) + +# Verbatim output of Go's json.Marshal on the response struct. +GO_RESPONSE_JSON = ( + '{"decision":"modify","reason":"PII detected",' + '"modified_payload":"eyJ0byI6IltSRURBQ1RFRF0ifQ==","policy_id":"pii-1"}' +) + + +def test_sdk_decodes_go_response_bytes_exactly(): + """The SDK must decode the real Go-marshaled response: base64 string → + raw bytes, all four fields populated. + """ + d = Decision.from_wire(json.loads(GO_RESPONSE_JSON)) + assert d.action == DECISION_MODIFY + assert d.reason == "PII detected" + assert d.policy_id == "pii-1" + assert d.modified_payload == b'{"to":"[REDACTED]"}' + assert d.modified is True + + +def test_go_allow_response_decodes_clean(): + """A minimal Go allow response (omitempty drops everything but decision).""" + d = Decision.from_wire(json.loads('{"decision":"allow"}')) + assert d.action == DECISION_ALLOW + assert d.modified_payload is None + assert not d.failed_open + + +def test_sdk_request_field_names_match_go_struct_tags(): + """Pin the request JSON keys the SDK emits against the Go struct's json + tags. If the SDK renames a field, sentinel's ShouldBindJSON silently drops + it (omitempty) and governance data is lost. Hard-code the contract. + """ + ev = ActionEvent( + event_type="pre_tool_use", + tool_name="send_email", + channel="system_tools", + tool_input={"x": 1}, + tool_output={"y": 2}, + session_id="s", + cwd="/c", + cli_version="1.0", + mcp=MCPInfo(server="srv", tool="t", url="u"), + ) + wire = ev.to_wire() + # These are the exact json tags on hookwire.EvaluateHookRequestJSON. + go_tags = { + "event_type", + "agent", + "tool_name", + "tool_input", + "tool_output", + "session_id", + "cwd", + "channel", + "mcp", + "cli_version", + } + unknown = set(wire.keys()) - go_tags + assert not unknown, f"SDK emits keys the Go struct ignores: {unknown}" + # And the mcp sub-object keys match hookwire.MCPInfo tags. + assert set(wire["mcp"].keys()) <= {"server", "tool", "url"} + + +def test_agent_label_matches_normalizer_trigger(): + """The SDK's default agent label MUST equal the string the Go normalizer + matches to assign SourceSDK (normalizer.go: req.Agent == "firstops-sdk"). + If these drift, every SDK action mis-attributes to hook_claude_code. + """ + ev = ActionEvent(event_type="pre_tool_use", tool_name="x") + assert ev.to_wire()["agent"] == "firstops-sdk" diff --git a/tests/test_enforcement.py b/tests/test_enforcement.py new file mode 100644 index 0000000..6947a14 --- /dev/null +++ b/tests/test_enforcement.py @@ -0,0 +1,166 @@ +"""Tests for the enforcement client — decision handling + fail-open.""" + +import base64 + +import httpx +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops._identity import build_identity +from firstops.enforcement import EnforcementClient +from firstops.events import ActionEvent + +_GATEWAY = "https://gw.example.com" + + +def _generate_pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +def _client(handler) -> EnforcementClient: + identity = build_identity("agent-1", _generate_pem(), _GATEWAY) + http = httpx.Client(transport=httpx.MockTransport(handler)) + return EnforcementClient(identity, http=http) + + +def _event() -> ActionEvent: + return ActionEvent( + event_type="pre_tool_use", tool_name="send_email", channel="system_tools" + ) + + +def _decode_dpop_claims(proof: str) -> dict: + import base64 + import json + + claims_b64 = proof.split(".")[1] + claims_b64 += "=" * (-len(claims_b64) % 4) # restore padding + return json.loads(base64.urlsafe_b64decode(claims_b64)) + + +def test_evaluate_allow_and_headers(): + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["dpop"] = request.headers.get("DPoP") + captured["auth"] = request.headers.get("Authorization") + captured["path"] = request.url.path + return httpx.Response(200, json={"decision": "allow"}) + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert not d.failed_open + # DPoP-signed, agent bearer present, correct path. + assert captured["dpop"] and len(captured["dpop"].split(".")) == 3 + assert captured["auth"] == "Bearer fo_agent_agent-1" + assert captured["path"] == "/api/v1/daemon/evaluate-hook" + + +def test_dpop_htu_and_htm_are_byte_exact(): + # Regression guard for the gateway-host / htu class of bug: sentinel + # validates htu by exact string match, so the signed htu MUST equal + # gateway + path with no query and no host drift. A wrong default host + # silently fails open — this test makes htu visible. + captured = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["claims"] = _decode_dpop_claims(request.headers["DPoP"]) + return httpx.Response(200, json={"decision": "allow"}) + + _client(handler).evaluate(_event()) + claims = captured["claims"] + assert claims["htu"] == f"{_GATEWAY}/api/v1/daemon/evaluate-hook" + assert claims["htm"] == "POST" + + +def test_unknown_decision_fails_open_visibly(): + # A verdict the SDK can't act on must allow (never block) but be flagged. + def handler(request): + return httpx.Response(200, json={"decision": "quarantine"}) + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert d.failed_open is True + + +def test_evaluate_deny(): + def handler(request): + return httpx.Response(200, json={"decision": "deny", "reason": "policy"}) + + d = _client(handler).evaluate(_event()) + assert d.blocked + assert not d.failed_open + + +def test_evaluate_modify(): + payload = b'{"to":"[REDACTED]"}' + + def handler(request): + return httpx.Response( + 200, + json={ + "decision": "modify", + "modified_payload": base64.b64encode(payload).decode(), + }, + ) + + d = _client(handler).evaluate(_event()) + assert d.action == "modify" + assert d.modified_payload == payload + + +def test_fail_open_on_transport_error(): + def handler(request): + raise httpx.ConnectError("connection refused") + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert d.failed_open is True + + +def test_fail_open_on_5xx(): + def handler(request): + return httpx.Response(500, text="boom") + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert d.failed_open is True + + +def test_fail_open_on_401(): + # Auth failure must NOT block the agent — it surfaces as a fail-open allow. + def handler(request): + return httpx.Response(401, json={"error": "unregistered key"}) + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert d.failed_open is True + + +def test_fail_open_on_malformed_body(): + def handler(request): + return httpx.Response(200, content=b"this is not json") + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert d.failed_open is True + + +def test_evaluate_never_raises(): + # Defensive: even a totally broken transport must not raise into the agent. + def handler(request): + raise RuntimeError("unexpected") + + # RuntimeError is not an httpx.HTTPError — confirm it still doesn't escape. + try: + d = _client(handler).evaluate(_event()) + except Exception as e: # pragma: no cover - this is the assertion + raise AssertionError(f"evaluate() raised into caller: {e!r}") + assert d.failed_open is True diff --git a/tests/test_enforcement_adversarial.py b/tests/test_enforcement_adversarial.py new file mode 100644 index 0000000..9c787a6 --- /dev/null +++ b/tests/test_enforcement_adversarial.py @@ -0,0 +1,280 @@ +"""Adversarial fail-open matrix for enforcement.py (M0.T0.4). + +Invariant #1 (the security spine): evaluate() must NEVER raise into the caller +and NEVER block. Every failure path — transport error, connect vs read timeout, +non-200, 3xx redirect, malformed/empty/non-JSON body, decision:null, unknown +verdict, an exception from event.to_wire(), and any unexpected exception — must +resolve to a fail-open ALLOW (audited). Auth (DPoP) is the only fail-closed +surface and even it surfaces here as a fail-open allow. + +These tests try to make evaluate() block, raise, or silently deny. +""" + +from __future__ import annotations + +import httpx +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops._identity import build_identity +from firstops.enforcement import EnforcementClient +from firstops.events import ActionEvent + +_GATEWAY = "https://gw.example.com" + + +def _pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +def _client(handler) -> EnforcementClient: + ident = build_identity("a", _pem(), _GATEWAY) + return EnforcementClient(ident, http=httpx.Client(transport=httpx.MockTransport(handler))) + + +def _event() -> ActionEvent: + return ActionEvent(event_type="pre_tool_use", tool_name="t", channel="system_tools") + + +def _assert_fail_open(d): + assert d.action == "allow", f"expected fail-open allow, got {d.action!r}" + assert d.failed_open is True + assert not d.blocked + + +# --------------------------------------------------------------------------- +# transport / timeout +# --------------------------------------------------------------------------- + + +def test_connect_timeout_fails_open(): + def handler(req): + raise httpx.ConnectTimeout("connect timed out") + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_read_timeout_fails_open(): + def handler(req): + raise httpx.ReadTimeout("read timed out") + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_pool_timeout_fails_open(): + def handler(req): + raise httpx.PoolTimeout("pool exhausted") + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_connect_error_fails_open(): + def handler(req): + raise httpx.ConnectError("connection refused") + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_unexpected_non_httpx_exception_fails_open(): + """A bug-class exception (not an httpx error) must still be swallowed.""" + def handler(req): + raise RuntimeError("totally unexpected") + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_keyboard_interrupt_is_NOT_swallowed(): + """Control-flow exceptions (KeyboardInterrupt/SystemExit) must propagate. + + `except Exception` does not catch BaseException — verify the agent can + still be Ctrl-C'd mid-evaluate. If someone widens the except to + `BaseException`, this catches the regression. + """ + def handler(req): + raise KeyboardInterrupt() + + with pytest.raises(KeyboardInterrupt): + _client(handler).evaluate(_event()) + + +# --------------------------------------------------------------------------- +# HTTP status codes +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("code", [301, 302, 307, 308]) +def test_redirect_fails_open_and_is_not_followed(code): + """A 3xx must fail open, NOT be followed. + + follow_redirects defaults to False, so a redirect is a non-200 → fail-open. + Following it would risk taking a 'decision' from an attacker-controlled + Location, or SSRF. This pins both: fail-open AND no follow. + """ + hits = {"n": 0} + + def handler(req): + hits["n"] += 1 + return httpx.Response(code, headers={"Location": "https://evil.example.com/x"}) + + _assert_fail_open(_client(handler).evaluate(_event())) + assert hits["n"] == 1, "redirect was followed — must not be" + + +@pytest.mark.parametrize("code", [400, 401, 403, 404, 429, 500, 502, 503, 504]) +def test_error_statuses_fail_open(code): + def handler(req): + return httpx.Response(code, json={"error": "x"}) + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_204_no_content_fails_open(): + """A 200 is required; any other 2xx (e.g. 204 with no body) fails open.""" + def handler(req): + return httpx.Response(204) + + _assert_fail_open(_client(handler).evaluate(_event())) + + +# --------------------------------------------------------------------------- +# 200 with pathological bodies +# --------------------------------------------------------------------------- + + +def test_200_empty_object_is_clean_allow(): + """200 {} → allow, but NOT failed_open (missing decision == allow, #4).""" + def handler(req): + return httpx.Response(200, json={}) + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + assert d.failed_open is False + + +def test_200_non_json_body_fails_open(): + def handler(req): + return httpx.Response(200, content=b"not json") + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_200_json_array_not_object_fails_open(): + """A JSON array (not an object) → .get() raises AttributeError → fail-open.""" + def handler(req): + return httpx.Response(200, json=[1, 2, 3]) + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_200_decision_null_is_allow(): + def handler(req): + return httpx.Response(200, json={"decision": None}) + + d = _client(handler).evaluate(_event()) + assert d.action == "allow" + + +def test_200_unknown_verdict_fails_open_visibly(): + def handler(req): + return httpx.Response(200, json={"decision": "quarantine"}) + + _assert_fail_open(_client(handler).evaluate(_event())) + + +def test_200_modify_with_invalid_base64_fails_open_not_raises(): + """HOLE GUARD: Decision.from_wire raises on bad base64. Inside evaluate() + that raise MUST be caught and converted to fail-open — it must never reach + the agent. (Decision.from_wire itself still raises for direct callers; see + test_events_adversarial.test_from_wire_invalid_base64_payload_raises.) + """ + def handler(req): + return httpx.Response( + 200, json={"decision": "modify", "modified_payload": "!!!bad!!!"} + ) + + # Must not raise; must fail open. + d = _client(handler).evaluate(_event()) + _assert_fail_open(d) + + +def test_200_giant_binary_modified_payload_decodes(): + import base64 + + blob = bytes(range(256)) * 8192 # 2 MiB + + def handler(req): + return httpx.Response( + 200, + json={"decision": "modify", "modified_payload": base64.b64encode(blob).decode()}, + ) + + d = _client(handler).evaluate(_event()) + assert d.action == "modify" + assert d.modified_payload == blob + + +# --------------------------------------------------------------------------- +# event-side failures +# --------------------------------------------------------------------------- + + +def test_exception_in_to_wire_fails_open(): + """If event.to_wire() raises (a malformed event), evaluate must fail open, + not propagate the error into the agent's tool-call path. + """ + class BadEvent(ActionEvent): + def to_wire(self): # type: ignore[override] + raise ValueError("boom in to_wire") + + bad = BadEvent(event_type="pre_tool_use", tool_name="t") + + def handler(req): + return httpx.Response(200, json={"decision": "deny"}) + + _assert_fail_open(_client(handler).evaluate(bad)) + + +# --------------------------------------------------------------------------- +# happy-path decisions still work +# --------------------------------------------------------------------------- + + +def test_deny_is_not_failed_open(): + def handler(req): + return httpx.Response(200, json={"decision": "deny", "reason": "policy"}) + + d = _client(handler).evaluate(_event()) + assert d.blocked + assert d.failed_open is False + + +def test_evaluate_truly_never_raises_across_matrix(): + """Belt-and-suspenders: sweep a battery of broken handlers; none may raise.""" + import base64 + + # '!!!bad!!!' is genuinely invalid base64 (9 chars) → from_wire raises → + # evaluate must catch it and fail open. Use that, not a length-valid blob. + handlers = [ + lambda r: (_ for _ in ()).throw(httpx.ConnectError("x")), + lambda r: httpx.Response(500), + lambda r: httpx.Response(200, content=b"garbage"), + lambda r: httpx.Response(200, json=[1, 2]), + lambda r: httpx.Response(200, json={"decision": "modify", "modified_payload": "!!!bad!!!"}), + lambda r: httpx.Response(302, headers={"Location": "/x"}), + lambda r: httpx.Response(200, json={"decision": None}), + ] + for h in handlers: + try: + d = _client(h).evaluate(_event()) + except Exception as e: # pragma: no cover + raise AssertionError(f"evaluate() raised for handler {h}: {e!r}") + # Every one of these is a failure or a no-decision → must allow. + assert d.action == "allow", f"handler {h} returned {d.action!r}, not allow" diff --git a/tests/test_events.py b/tests/test_events.py new file mode 100644 index 0000000..540cf07 --- /dev/null +++ b/tests/test_events.py @@ -0,0 +1,108 @@ +"""Tests for the action-event model and hookwire (de)serialization.""" + +import base64 + +from firstops.events import ( + CHANNEL_MCP, + CHANNEL_SYSTEM_TOOLS, + DECISION_ALLOW, + DECISION_DENY, + DECISION_MODIFY, + EVENT_PRE_TOOL_USE, + ActionEvent, + Decision, + MCPInfo, +) + + +def test_action_event_wire_shape_minimal(): + ev = ActionEvent(event_type=EVENT_PRE_TOOL_USE, tool_name="send_email") + wire = ev.to_wire() + # Always-present keys. + assert wire["event_type"] == "pre_tool_use" + assert wire["tool_name"] == "send_email" + assert wire["agent"] == "firstops-sdk" + # Optional keys omitted when unset. + assert "tool_input" not in wire + assert "channel" not in wire + assert "mcp" not in wire + + +def test_action_event_preserves_nested_structure(): + # The whole point of sending tool_input as a JSON object (not a + # stringified map) is that nested arrays/objects survive the wire. + nested = {"to": "a@b.com", "body": {"lines": [1, 2, 3], "meta": {"x": True}}} + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, + tool_name="send_email", + channel=CHANNEL_SYSTEM_TOOLS, + tool_input=nested, + ) + wire = ev.to_wire() + assert wire["tool_input"] == nested + assert wire["channel"] == "system_tools" + + +def test_action_event_mcp_info(): + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, + tool_name="search", + channel=CHANNEL_MCP, + mcp=MCPInfo(server="notion", tool="search", url="https://mcp.notion.com/sse"), + ) + assert ev.to_wire()["mcp"] == { + "server": "notion", + "tool": "search", + "url": "https://mcp.notion.com/sse", + } + + +def test_action_event_empty_mcp_omitted(): + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, tool_name="x", channel=CHANNEL_MCP, mcp=MCPInfo() + ) + assert "mcp" not in ev.to_wire() + + +def test_decision_from_wire_allow(): + d = Decision.from_wire({"decision": "allow"}) + assert d.action == DECISION_ALLOW + assert d.modified_payload is None + assert not d.blocked + assert not d.failed_open + + +def test_decision_from_wire_deny(): + d = Decision.from_wire({"decision": "deny", "reason": "blocked", "policy_id": "p1"}) + assert d.action == DECISION_DENY + assert d.blocked + assert d.reason == "blocked" + assert d.policy_id == "p1" + + +def test_decision_from_wire_modify_base64(): + payload = b'{"to":"[REDACTED]"}' + d = Decision.from_wire( + { + "decision": "modify", + "modified_payload": base64.b64encode(payload).decode(), + "policy_id": "pii-1", + } + ) + assert d.action == DECISION_MODIFY + assert d.modified_payload == payload + assert d.modified is True + + +def test_decision_from_wire_empty_defaults_to_allow(): + # A backend that returns {} (or a missing decision) must not crash and + # must default to allow — never silently deny. + d = Decision.from_wire({}) + assert d.action == DECISION_ALLOW + + +def test_decision_fail_open(): + d = Decision.fail_open("sentinel down") + assert d.action == DECISION_ALLOW + assert d.failed_open is True + assert "sentinel down" in d.reason diff --git a/tests/test_events_adversarial.py b/tests/test_events_adversarial.py new file mode 100644 index 0000000..2cbf96e --- /dev/null +++ b/tests/test_events_adversarial.py @@ -0,0 +1,197 @@ +"""Adversarial tests for events.py — wire fidelity + Decision parsing (M0.T0.3). + +Probes the gaps between what the SDK emits/parses and what the backend +hookwire contract expects. The backend reference shapes: + + request : backend/shared/lib/hookwire/types.go EvaluateHookRequestJSON + (tool_input/tool_output are `map[string]any` with `omitempty`) + response: EvaluateHookResponseJSON {decision, reason, modified_payload([]byte), + policy_id}; Go marshals []byte as a base64 string. + +Designed to FIND BUGS, not to pass. +""" + +from __future__ import annotations + +import base64 + +import pytest + +from firstops.events import ( + DECISION_ALLOW, + DECISION_MODIFY, + EVENT_PRE_TOOL_USE, + ActionEvent, + Decision, +) + + +# --------------------------------------------------------------------------- +# tool_input / tool_output fidelity +# --------------------------------------------------------------------------- + + +def test_tool_output_round_trips_nested(): + """tool_output must survive as a JSON object (symmetric with tool_input).""" + out = {"messages": [{"role": "assistant", "content": "hi"}], "n": 3, "ok": True} + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, tool_name="x", tool_output=out + ) + assert ev.to_wire()["tool_output"] == out + + +def test_empty_dict_tool_input_is_emitted_not_omitted(): + """An explicitly-empty tool_input ({}) is DISTINCT from absent (None). + + The SDK emits `tool_input: {}` when the value is {} (only None omits it). + The backend struct has `omitempty` on the map, so Go would *omit* an empty + map on its own marshal — but the SDK is the *producer* here, and sentinel's + ShouldBindJSON accepts an explicit empty object. This test pins the SDK's + behavior so a future 'optimize away empty dicts' change is caught: {} and + None must remain distinguishable on the wire. + """ + ev_empty = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, tool_name="x", tool_input={} + ) + ev_none = ActionEvent(event_type=EVENT_PRE_TOOL_USE, tool_name="x") + assert "tool_input" in ev_empty.to_wire(), "{} must be emitted, not dropped" + assert ev_empty.to_wire()["tool_input"] == {} + assert "tool_input" not in ev_none.to_wire(), "None must be omitted" + + +def test_falsy_string_fields_are_omitted_consistently(): + """session_id='', cwd='', channel='' are omitted (truthiness gate). + + Documents the current behavior: empty strings are omitted via `if self.x:`. + This matches the backend's `omitempty` on those string fields, so it is + correct — but it means you cannot send an *explicit* empty channel to mean + 'match all' distinctly from 'unset'. Pin it so the behavior is intentional. + """ + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, + tool_name="x", + session_id="", + cwd="", + channel="", + ) + wire = ev.to_wire() + assert "session_id" not in wire + assert "cwd" not in wire + assert "channel" not in wire + + +def test_unicode_and_mixed_types_in_tool_input_preserved(): + payload = { + "emoji": "🔥", + "unicode": "café—dash", + "nested_list": [1, 2, [3, {"deep": True}]], + "int": 42, + "float": 3.14, + "bool": False, + "null": None, + } + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, tool_name="x", tool_input=payload + ) + # The SDK must NOT stringify or coerce — it passes the object through. + assert ev.to_wire()["tool_input"] == payload + + +def test_none_value_inside_tool_input_is_kept(): + """A None *inside* the dict is a value (kept), distinct from a None dict.""" + ev = ActionEvent( + event_type=EVENT_PRE_TOOL_USE, tool_name="x", tool_input={"k": None} + ) + wire = ev.to_wire() + assert "tool_input" in wire + assert wire["tool_input"] == {"k": None} + + +# --------------------------------------------------------------------------- +# Decision.from_wire — fail-open & base64 edge cases +# --------------------------------------------------------------------------- + + +def test_from_wire_invalid_base64_payload_fails_open_not_raises(): + """FIXED (was: raised binascii.Error). A `modify` verdict whose + modified_payload is not valid base64 must NOT raise out of from_wire — it + fails open visibly so direct callers (adapters, contract tests) never see + an exception. Invariant #1: never raise into the caller. + """ + d = Decision.from_wire( + {"decision": "modify", "modified_payload": "!!!not-base64!!!"} + ) + assert d.action == DECISION_ALLOW + assert d.failed_open is True + assert d.modified_payload is None + + +def test_from_wire_garbage_base64_is_rejected_not_silently_decoded(): + """FIXED (was: silent coercion to wrong bytes). With validate=True a + corrupted modified_payload is rejected and the decision fails open, rather + than producing silently-wrong scrub bytes applied to a live call. + """ + d = Decision.from_wire({"decision": "modify", "modified_payload": "!!!!"}) + assert d.action == DECISION_ALLOW + assert d.failed_open is True + assert d.modified_payload is None + + +def test_from_wire_modify_with_no_payload_fails_open(): + """FIXED (was: action='modify' with no payload — signals disagreed). A + `modify` verdict with nothing to apply is malformed: fail open visibly + rather than report a modify the caller can't act on. + """ + d = Decision.from_wire({"decision": "modify"}) + assert d.action == DECISION_ALLOW + assert d.modified is False + assert d.modified_payload is None + assert d.failed_open is True + + +def test_from_wire_decision_null_defaults_to_allow(): + """decision: null → allow (not silent deny). Per invariant #4, null≈missing. + + Documents that a null verdict is indistinguishable from a clean allow + (failed_open stays False). If the team wants null flagged as failed_open, + this test must change. + """ + d = Decision.from_wire({"decision": None}) + assert d.action == DECISION_ALLOW + assert d.failed_open is False + + +def test_from_wire_empty_object_is_allow(): + d = Decision.from_wire({}) + assert d.action == DECISION_ALLOW + assert d.failed_open is False + + +def test_from_wire_unknown_verdict_fails_open_visibly(): + d = Decision.from_wire({"decision": "quarantine", "reason": "weird"}) + assert d.action == DECISION_ALLOW + assert d.failed_open is True + assert "quarantine" in d.reason + + +def test_from_wire_empty_string_decision_defaults_to_allow(): + """decision: '' (empty string) → allow via the `or` short-circuit.""" + d = Decision.from_wire({"decision": ""}) + assert d.action == DECISION_ALLOW + assert d.failed_open is False + + +def test_from_wire_large_binary_payload_round_trips(): + """A gigantic / binary modified_payload must decode intact.""" + blob = bytes(range(256)) * 4096 # 1 MiB of all byte values + d = Decision.from_wire( + {"decision": "modify", "modified_payload": base64.b64encode(blob).decode()} + ) + assert d.modified_payload == blob + + +def test_from_wire_bytes_payload_passthrough(): + """If modified_payload arrives as raw bytes (not str), it is used as-is.""" + raw = b"already-bytes" + d = Decision.from_wire({"decision": "modify", "modified_payload": raw}) + assert d.modified_payload == raw diff --git a/tests/test_identity_adversarial.py b/tests/test_identity_adversarial.py new file mode 100644 index 0000000..f3f4789 --- /dev/null +++ b/tests/test_identity_adversarial.py @@ -0,0 +1,164 @@ +"""Adversarial tests for Identity / DPoP htu canonicalization (M0.T0.2). + +The sentinel validates htu by **exact byte-for-byte string compare** (no +normalization): + + backend/shared/lib/dpop/validate.go:132 -> if htu != expectedURL { ... } + expectedURL = h.config.BaseURL + "/api/v1/daemon/evaluate-hook" + +So whatever the SDK signs as htu MUST equal the operator's configured BaseURL ++ path, verbatim. A divergence does NOT error loudly — it 401s, and the +enforcement client converts a 401 into a fail-open *allow*. The agent then runs +completely ungoverned while the dashboard says "protected." That is the +existential silent-fail-open failure mode the design doc calls out, so these +tests probe the canonicalization byte-for-byte. + +These are designed to FAIL where Identity.proof normalizes inconsistently. +""" + +from __future__ import annotations + +import base64 +import json + +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops._identity import build_identity + +_HOOK_PATH = "/api/v1/daemon/evaluate-hook" + + +def _pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +def _htu(proof: str) -> str: + claims_b64 = proof.split(".")[1] + claims_b64 += "=" * (-len(claims_b64) % 4) + return json.loads(base64.urlsafe_b64decode(claims_b64))["htu"] + + +def _htm(proof: str) -> str: + claims_b64 = proof.split(".")[1] + claims_b64 += "=" * (-len(claims_b64) % 4) + return json.loads(base64.urlsafe_b64decode(claims_b64))["htm"] + + +# What sentinel would compute given the same gateway BaseURL the operator set. +def _sentinel_expected_htu(base_url: str) -> str: + # Sentinel does a plain concat: config.BaseURL + path. No normalization. + return base_url.rstrip("/") + _HOOK_PATH + + +def test_htu_strips_query_and_fragment(): + """Query + fragment MUST be stripped (sentinel's BaseURL+path has neither).""" + ident = build_identity("a", _pem(), "https://h.example.com") + url = ident.gateway_url + _HOOK_PATH + "?nonce=abc#frag" + proof = ident.proof("POST", url) + assert _htu(proof) == "https://h.example.com" + _HOOK_PATH + assert "?" not in _htu(proof) + assert "#" not in _htu(proof) + + +def test_htm_is_post_uppercase(): + ident = build_identity("a", _pem(), "https://h.example.com") + proof = ident.proof("POST", ident.gateway_url + _HOOK_PATH) + assert _htm(proof) == "POST" + + +def test_htu_with_trailing_slash_gateway_matches_sentinel(): + """A trailing slash on gateway_url must not produce a double slash in htu.""" + base = "https://h.example.com/" + ident = build_identity("a", _pem(), base) + proof = ident.proof("POST", ident.gateway_url + _HOOK_PATH) + assert _htu(proof) == _sentinel_expected_htu(base) + + +def test_htu_with_port_matches_sentinel(): + base = "https://h.example.com:8443" + ident = build_identity("a", _pem(), base) + proof = ident.proof("POST", ident.gateway_url + _HOOK_PATH) + assert _htu(proof) == _sentinel_expected_htu(base) + + +def test_htu_does_not_alter_host_casing(): + """HOLE PROBE: urlunparse lowercases the *scheme* but NOT the *host*. + + If an operator's SENTINEL_PROXY_BASE_URL is 'https://H.Example.com' (mixed + case — uncommon but legal), sentinel computes + 'https://H.Example.com/api/v1/daemon/evaluate-hook' and compares byte-for- + byte. The SDK must sign the SAME bytes. Since proof() must NOT introduce + normalization sentinel doesn't do, the signed htu host must equal whatever + the caller put in gateway_url, character for character. + + This asserts the *contract*: htu == (operator's base_url, verbatim) + path. + It fails today because urlunparse lowercases the scheme, so for a base_url + with an uppercase scheme the SDK's htu diverges from a plain concat. + """ + base = "HTTPS://H.Example.com" + ident = build_identity("a", _pem(), base) + # The enforcement client signs over identity.gateway_url + path. + signed = _htu(ident.proof("POST", ident.gateway_url + _HOOK_PATH)) + # Sentinel compares against operator BaseURL + path, byte-exact. + expected = _sentinel_expected_htu(base) + assert signed == expected, ( + f"htu divergence => silent 401 => fail-open allow.\n" + f" SDK signed : {signed!r}\n" + f" sentinel : {expected!r}\n" + f" Identity.proof must not normalize what sentinel does not." + ) + + +def test_path_prefixed_gateway_does_not_double_the_prefix(): + """HOLE PROBE: gateway_url with a path prefix (e.g. https://h/api). + + EnforcementClient builds its URL as gateway_url + '/api/v1/daemon/...'. + If gateway_url already ends in '/api', the POST target AND the htu become + '.../api/api/v1/daemon/evaluate-hook' (doubled). Whether that is a bug + depends on the deployment, but the htu the SDK signs must equal the URL it + actually POSTs to AND equal sentinel's expected. This test documents the + behavior so a path-prefixed gateway can't silently 404/401. + """ + base = "https://h.example.com/api" + ident = build_identity("a", _pem(), base) + post_target = ident.gateway_url + _HOOK_PATH + signed = _htu(ident.proof("POST", post_target)) + # The htu must at least match the URL we actually POST to (self-consistency). + assert signed == post_target, ( + f"signed htu {signed!r} != POST target {post_target!r}; " + f"htu/URL self-inconsistency is an automatic 401." + ) + + +def test_proof_htu_matches_enforcement_client_post_url_exactly(): + """End-to-end self-consistency: the htu the client signs equals the URL it + POSTs to. This is the invariant that protects against *any* normalization + drift between EnforcementClient._url and Identity.proof, regardless of how + weird the gateway_url is. + """ + from firstops.enforcement import EnforcementClient + + for base in [ + "https://h.example.com", + "https://h.example.com/", + "https://h.example.com:8443", + "https://h.example.com/api", + ]: + ident = build_identity("a", _pem(), base) + client = EnforcementClient(ident) + # The URL the client will POST to (private, but load-bearing). + post_url = client._url + signed = _htu(ident.proof("POST", post_url)) + assert signed == post_url, ( + f"base={base!r}: signed htu {signed!r} != POST url {post_url!r}" + ) + client.close() diff --git a/tests/test_integrations.py b/tests/test_integrations.py new file mode 100644 index 0000000..689eb62 --- /dev/null +++ b/tests/test_integrations.py @@ -0,0 +1,183 @@ +"""Tests for the harness adapters' framework-free governance core. + +The frameworks themselves aren't installed here, so we test the decision logic +and the exact native outputs the adapters emit, plus that the lazily-imported +shells fail cleanly when the framework is absent. +""" + +import base64 +import json +from types import SimpleNamespace + +import pytest + +from firstops.events import ( + CHANNEL_MCP, + CHANNEL_SYSTEM_TOOLS, + EVENT_PRE_TOOL_USE, + Decision, +) +from firstops.integrations import _common # noqa: F401 (kept for import-sanity) +from firstops.integrations._common import ( + coerce_input, + decide, + govern_tool, + modified_input, +) + + +class _FakeEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + return self.decision + + +def _rt(decision: Decision): + return SimpleNamespace(enforcement=_FakeEnforcement(decision)) + + +def _modify(payload: dict) -> Decision: + raw = base64.b64encode(json.dumps(payload).encode()).decode() + return Decision.from_wire({"decision": "modify", "modified_payload": raw}) + + +# ---- _common --------------------------------------------------------------- + + +def test_govern_tool_builds_system_tools_event(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, "send_email", {"to": "a@b.com"}) + ev = rt.enforcement.events[0] + assert ev.event_type == EVENT_PRE_TOOL_USE + assert ev.tool_name == "send_email" + assert ev.channel == CHANNEL_SYSTEM_TOOLS + assert ev.tool_input == {"to": "a@b.com"} + assert ev.mcp is None + + +def test_govern_tool_builds_mcp_event_with_metadata(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, "mcp__notion__search", {"q": "x"}) + ev = rt.enforcement.events[0] + assert ev.channel == CHANNEL_MCP + assert ev.mcp is not None + assert ev.mcp.server == "notion" + assert ev.mcp.tool == "search" + + +def test_govern_tool_allows_when_no_runtime(): + d = govern_tool(None, "anything", {}) + assert d.action == "allow" + + +@pytest.mark.parametrize( + "raw,expected", + [ + ({"a": 1}, {"a": 1}), + ('{"a": 1}', {"a": 1}), + ("plain", {"input": "plain"}), + (None, {}), + (123, {"input": 123}), + ], +) +def test_coerce_input(raw, expected): + assert coerce_input(raw) == expected + + +def test_modified_input_roundtrip(): + assert modified_input(_modify({"x": 9})) == {"x": 9} + + +def test_modified_input_none_when_not_modify(): + assert modified_input(Decision(action="allow")) is None + + +def test_decide_allow_deny_modify(): + assert decide(_rt(Decision(action="allow")), "t", {}) == ("allow", None) + assert decide(_rt(Decision(action="deny", reason="no")), "t", {}) == ("deny", "no") + assert decide(_rt(_modify({"x": 1})), "t", {}) == ("modify", {"x": 1}) + + +# ---- Claude adapter -------------------------------------------------------- + +from firstops.integrations import claude + + +def test_claude_allow_returns_empty(): + rt = _rt(Decision(action="allow")) + assert ( + claude._govern_pre_tool_use( + rt, {"tool_name": "Bash", "tool_input": {"command": "ls"}} + ) + == {} + ) + + +def test_claude_deny_envelope(): + rt = _rt(Decision(action="deny", reason="destructive command rule")) + out = claude._govern_pre_tool_use( + rt, {"tool_name": "Bash", "tool_input": {"command": "danger"}} + ) + hso = out["hookSpecificOutput"] + assert hso["permissionDecision"] == "deny" + assert "destructive" in hso["permissionDecisionReason"] + + +def test_claude_modify_updates_input(): + rt = _rt(_modify({"command": "ls -la"})) + out = claude._govern_pre_tool_use( + rt, {"tool_name": "Bash", "tool_input": {"command": "ls"}} + ) + hso = out["hookSpecificOutput"] + assert hso["permissionDecision"] == "allow" + assert hso["updatedInput"] == {"command": "ls -la"} + + +def test_claude_hooks_requires_sdk(): + with pytest.raises(RuntimeError, match="claude-agent-sdk"): + claude.firstops_hooks(_rt(Decision(action="allow"))) + + +# ---- OpenAI Agents adapter ------------------------------------------------- + +from firstops.integrations import openai_agents + + +def test_openai_decide_block(): + assert openai_agents._decide_tool( + _rt(Decision(action="deny", reason="x")), "t", {} + ) == (True, "x") + + +def test_openai_decide_allow(): + assert openai_agents._decide_tool(_rt(Decision(action="allow")), "t", {}) == ( + False, + "", + ) + + +def test_openai_modify_blocks_fail_closed(): + # Guardrails can't mutate args, so a scrub-required (modify) decision fails + # CLOSED (block) rather than letting unscrubbed args through. + blocked, reason = openai_agents._decide_tool(_rt(_modify({"x": 1})), "t", {}) + assert blocked is True + assert "scrub" in reason + + +def test_openai_guardrail_requires_sdk(): + with pytest.raises(RuntimeError, match="openai-agents"): + openai_agents.firstops_tool_input_guardrail(_rt(Decision(action="allow"))) + + +# ---- LangGraph adapter ----------------------------------------------------- + +from firstops.integrations import langgraph + + +def test_langgraph_middleware_requires_langchain(): + with pytest.raises(RuntimeError, match="langchain"): + langgraph.FirstOpsMiddleware(_rt(Decision(action="allow"))) diff --git a/tests/test_integrations_bughunt.py b/tests/test_integrations_bughunt.py new file mode 100644 index 0000000..19c2553 --- /dev/null +++ b/tests/test_integrations_bughunt.py @@ -0,0 +1,429 @@ +"""Bug-hunting tests for the M2 harness-adapter governance core. + +These probe the framework-FREE logic the adapters share (``_common``) and the +pure decision functions in each adapter, against the four stated invariants: + + 1. Adapters fail open: no runtime / enforcement weirdness must not raise or + block into the agent loop (except a real DENY). + 2. ``decide`` returns exactly ("allow"|"deny"|"modify", payload). A malformed + modify payload must degrade to allow, never a broken modify. + 3. The Claude hook output must be a valid shape for allow ({}), deny, and + modify — and must NEVER emit a modify envelope with null/empty updatedInput. + 4. OpenAI ``_decide_tool``: modify is not representable → degrade to + (False, "") allow, never block. + +Tests that currently FAIL mark a real defect; each is annotated with +``xfail(strict=True)`` and a BUG-ID so the suite stays green until the code is +fixed, at which point the xfail flips to an unexpected-pass and forces removal +of the marker. Search "BUG-" to find them. +""" + +import base64 +import json +from types import SimpleNamespace + +import pytest + +from firstops.events import ( + CHANNEL_MCP, + CHANNEL_SYSTEM_TOOLS, + ActionEvent, + Decision, +) +from firstops.integrations import claude, openai_agents +from firstops.integrations._common import ( + coerce_input, + decide, + govern_tool, + modified_input, +) + + +# -------------------------------------------------------------------------- +# Test doubles (mirror the baseline fixtures in test_integrations.py) +# -------------------------------------------------------------------------- + + +class _FakeEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + return self.decision + + +def _rt(decision: Decision): + return SimpleNamespace(enforcement=_FakeEnforcement(decision)) + + +def _modify(payload) -> Decision: + """A modify Decision whose base64 payload encodes ``payload`` (any JSON type).""" + raw = base64.b64encode(json.dumps(payload).encode()).decode() + return Decision.from_wire({"decision": "modify", "modified_payload": raw}) + + +# ========================================================================== +# coerce_input — adversarial inputs (Invariant 1: must never raise) +# ========================================================================== + + +def test_coerce_input_utf8_bytes_roundtrips(): + assert coerce_input(b"hello") == {"input": "hello"} + + +def test_coerce_input_json_bytes_parses_to_dict(): + assert coerce_input(b'{"a": 1}') == {"a": 1} + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("[1,2,3]", {"input": "[1,2,3]"}), # JSON list string -> not a dict, wrap raw + ("42", {"input": "42"}), # JSON scalar string -> wrap raw + ("null", {"input": "null"}), + ("true", {"input": "true"}), + ("", {"input": ""}), # empty string + ({}, {}), # empty dict + ("unicode: café ☃", {"input": "unicode: café ☃"}), + ], +) +def test_coerce_input_string_scalars_and_unicode(raw, expected): + assert coerce_input(raw) == expected + + +def test_coerce_input_nested_dict_returned_as_is(): + nested = {"a": {"b": [1, 2, {"c": 3}]}} + assert coerce_input(nested) == nested + + +def test_coerce_input_large_input_does_not_choke(): + big = "x" * 1_000_000 + assert coerce_input(big) == {"input": big} + + +def test_coerce_input_non_utf8_bytes_must_not_raise(): + # Should degrade gracefully, not raise. + out = coerce_input(b"\xff\xfe\x00") + assert isinstance(out, dict) + assert "input" in out + + +def test_coerce_input_produces_json_serializable_value(): + class Weird: + def __str__(self): + return "weird" + + coerced = coerce_input(Weird()) + # The body must be serializable, because to_wire() will json.dumps it. + json.dumps(coerced) # must not raise + + +# ========================================================================== +# modified_input — malformed modify payloads degrade to None (Invariant 2) +# ========================================================================== + + +def test_modified_input_dict_payload_returned(): + assert modified_input(_modify({"x": 9})) == {"x": 9} + + +def test_modified_input_list_payload_is_none(): + # A modify whose payload is a JSON list is not a usable input dict. + d = _modify([1, 2, 3]) + assert modified_input(d) is None + + +@pytest.mark.parametrize("scalar", [42, "string", 3.14, True, None]) +def test_modified_input_scalar_payload_is_none(scalar): + assert modified_input(_modify(scalar)) is None + + +def test_modified_input_empty_dict_is_a_dict(): + # PIN THE CHOICE: an empty-dict {} payload IS a dict, so modified_input + # returns it, and decide() therefore reports ("modify", {}). Whether {} is + # a *meaningful* scrub is the adapter's problem (see BUG-2 for Claude). + assert modified_input(_modify({})) == {} + + +def test_modified_input_none_when_not_modify(): + assert modified_input(Decision(action="allow")) is None + assert modified_input(Decision(action="deny", reason="x")) is None + + +# ========================================================================== +# decide — exact (action, payload) contract (Invariant 2) +# ========================================================================== + + +def test_decide_allow(): + assert decide(_rt(Decision(action="allow")), "t", {}) == ("allow", None) + + +def test_decide_deny_with_reason(): + assert decide(_rt(Decision(action="deny", reason="no")), "t", {}) == ("deny", "no") + + +def test_decide_deny_empty_reason_gets_fallback(): + action, reason = decide(_rt(Decision(action="deny", reason="")), "t", {}) + assert action == "deny" + assert reason == "blocked by FirstOps policy" # never an empty deny reason + + +def test_decide_modify_dict(): + assert decide(_rt(_modify({"x": 1})), "t", {}) == ("modify", {"x": 1}) + + +def test_decide_modify_list_degrades_to_allow(): + # Invariant 2: a malformed modify must NOT become a broken modify. + assert decide(_rt(_modify([1, 2, 3])), "t", {}) == ("allow", None) + + +def test_decide_modify_scalar_degrades_to_allow(): + assert decide(_rt(_modify("just a string")), "t", {}) == ("allow", None) + + +def test_decide_no_runtime_allows(): + assert decide(None, "t", {}) == ("allow", None) + + +# ========================================================================== +# govern_tool — event construction, channel, tool_name edge cases +# ========================================================================== + + +def test_govern_tool_system_tools_channel(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, "send_email", {"to": "a@b.com"}) + ev = rt.enforcement.events[0] + assert ev.channel == CHANNEL_SYSTEM_TOOLS + assert ev.mcp is None + + +def test_govern_tool_mcp_channel_and_metadata(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, "mcp__notion__search", {"q": "x"}) + ev = rt.enforcement.events[0] + assert ev.channel == CHANNEL_MCP + assert ev.mcp.server == "notion" + assert ev.mcp.tool == "search" + + +def test_govern_tool_malformed_mcp_name_is_system_tools(): + # classify() and mcp_info() must agree: a malformed mcp__ name is NOT mcp. + rt = _rt(Decision(action="allow")) + govern_tool(rt, "mcp__foo", {}) + ev = rt.enforcement.events[0] + assert ev.channel == CHANNEL_SYSTEM_TOOLS + assert ev.mcp is None + + +def test_govern_tool_empty_tool_name_does_not_raise(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, "", {}) + assert rt.enforcement.events[0].channel == CHANNEL_SYSTEM_TOOLS + + +def test_govern_tool_unicode_tool_name(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, "send_\U0001f4e7", {}) + assert rt.enforcement.events[0].tool_name == "send_\U0001f4e7" + + +def test_govern_tool_failed_open_allow_flows_through_as_allow(): + # An enforcement that fails open (allow + failed_open) must read as allow. + rt = _rt(Decision.fail_open("sentinel unreachable")) + assert decide(rt, "t", {}) == ("allow", None) + + +def test_govern_tool_none_tool_name_must_not_raise(): + rt = _rt(Decision(action="allow")) + govern_tool(rt, None, {}) # must not raise + assert rt.enforcement.events[0].channel == CHANNEL_SYSTEM_TOOLS + + +def test_govern_tool_does_not_alias_caller_input_dict(): + rt = _rt(Decision(action="allow")) + caller_input = {"to": "a@b.com"} + govern_tool(rt, "send_email", caller_input) + ev = rt.enforcement.events[0] + # Mutating the caller dict after the fact must not change the audited event. + caller_input["to"] = "attacker@evil.com" + assert ev.tool_input == {"to": "a@b.com"} + + +# ========================================================================== +# Claude adapter — exact hook envelope shape (Invariant 3) +# ========================================================================== + + +def test_claude_allow_returns_empty_dict(): + rt = _rt(Decision(action="allow")) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": {"command": "ls"}}) + assert out == {} + + +def test_claude_deny_envelope_exact_shape(): + rt = _rt(Decision(action="deny", reason="destructive command rule")) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": {"command": "wipe"}}) + assert out == { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": "destructive command rule", + } + } + + +def test_claude_deny_empty_reason_never_emits_blank_reason(): + rt = _rt(Decision(action="deny", reason="")) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": {}}) + reason = out["hookSpecificOutput"]["permissionDecisionReason"] + assert reason # not "" and not None + assert reason == "blocked by FirstOps policy" + + +def test_claude_modify_envelope_exact_shape(): + rt = _rt(_modify({"command": "ls -la"})) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": {"command": "ls"}}) + assert out == { + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "allow", + "updatedInput": {"command": "ls -la"}, + } + } + + +def test_claude_modify_list_payload_degrades_to_allow_empty(): + # A modify whose payload is a list is unusable -> allow ({}), not a broken + # modify envelope. + rt = _rt(_modify([1, 2, 3])) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": {}}) + assert out == {} + + +def test_claude_missing_tool_name_allows(): + rt = _rt(Decision(action="allow")) + assert claude._govern_pre_tool_use(rt, {"tool_input": {}}) == {} + + +def test_claude_missing_tool_input_still_governs(): + rt = _rt(Decision(action="deny", reason="r")) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash"}) + assert out["hookSpecificOutput"]["permissionDecision"] == "deny" + + +def test_claude_tool_input_json_string_is_parsed_for_event(): + rt = _rt(Decision(action="allow")) + claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": '{"command": "ls"}'}) + # The event body should carry the parsed object, not the raw string. + assert rt.enforcement.events[0].tool_input == {"command": "ls"} + + +def test_claude_empty_dict_modify_must_not_emit_empty_updated_input(): + rt = _rt(_modify({})) + out = claude._govern_pre_tool_use(rt, {"tool_name": "Bash", "tool_input": {"command": "ls"}}) + hso = out.get("hookSpecificOutput") + if hso is not None: + # If anything is emitted, it must not be a modify with empty input. + assert not (hso.get("permissionDecision") == "allow" and hso.get("updatedInput") == {}) + + +# ========================================================================== +# OpenAI Agents adapter — modify is not representable (Invariant 4) +# ========================================================================== + + +def test_openai_deny_blocks_with_reason(): + assert openai_agents._decide_tool(_rt(Decision(action="deny", reason="x")), "t", {}) == (True, "x") + + +def test_openai_allow(): + assert openai_agents._decide_tool(_rt(Decision(action="allow")), "t", {}) == (False, "") + + +def test_openai_modify_blocks_fail_closed(): + # Guardrails can't mutate args, so a scrub-required (modify) decision must + # FAIL CLOSED (block) rather than let unscrubbed args through. + blocked, reason = openai_agents._decide_tool(_rt(_modify({"x": 1})), "t", {}) + assert blocked is True + assert "scrub" in reason + + +def test_openai_modify_empty_dict_also_blocks(): + blocked, _ = openai_agents._decide_tool(_rt(_modify({})), "t", {}) + assert blocked is True + + +def test_openai_deny_reason_always_string(): + # Even if the upstream reason is a non-string, _decide_tool must return a str. + d = Decision(action="deny") + d.reason = 12345 # type: ignore[assignment] + blocked, reason = openai_agents._decide_tool(_rt(d), "t", {}) + assert blocked is True + assert isinstance(reason, str) + + +def test_openai_deny_empty_reason_gets_fallback_string(): + blocked, reason = openai_agents._decide_tool(_rt(Decision(action="deny", reason="")), "t", {}) + assert blocked is True + assert reason == "blocked by FirstOps policy" + + +# ========================================================================== +# Cross-adapter consistency: the SAME decision must be honored consistently +# ========================================================================== + + +def test_deny_is_honored_by_both_adapters(): + d = Decision(action="deny", reason="pii rule") + claude_out = claude._govern_pre_tool_use(_rt(d), {"tool_name": "t", "tool_input": {}}) + openai_out = openai_agents._decide_tool(_rt(d), "t", {}) + assert claude_out["hookSpecificOutput"]["permissionDecision"] == "deny" + assert openai_out[0] is True + + +def test_modify_is_applied_by_claude_but_blocked_by_openai(): + # Same scrub decision: Claude rewrites the args (updatedInput); OpenAI can't + # mutate, so it fails closed (block) — never a silent unscrubbed allow. + d = _modify({"x": 1}) + claude_out = claude._govern_pre_tool_use(_rt(d), {"tool_name": "t", "tool_input": {}}) + openai_out = openai_agents._decide_tool(_rt(d), "t", {}) + assert claude_out["hookSpecificOutput"].get("updatedInput") == {"x": 1} + assert openai_out[0] is True # OpenAI can't mutate -> block + + +# ========================================================================== +# Double governance — documentation test (decorator + adapter both fire) +# ========================================================================== + + +def test_double_governance_emits_pre_event_twice(monkeypatch): + """A tool that is BOTH @firstops.tool-decorated AND governed by an adapter + is evaluated TWICE: once by the adapter at the harness execution boundary, + and once by the decorator inside the call. This is the documented behavior + (the adapter and the base decorator are independent governance layers). + + Pinning it here so a future de-dup effort is a deliberate, test-visible + change rather than an accidental one. + """ + import firstops + import firstops._runtime as runtime_mod + + shared = _rt(Decision(action="allow")) + monkeypatch.setattr(runtime_mod, "runtime", lambda: shared) + + @firstops.tool + def send_email(to): + return "sent" + + # Adapter governs at the boundary (one pre_tool_use): + decide(shared, "send_email", {"to": "a@b.com"}) + # Then the decorated tool actually runs (decorator governs inside): + send_email("a@b.com") + + pre = [e for e in shared.enforcement.events if e.event_type == "pre_tool_use"] + # Two pre_tool_use evaluations for one logical call: adapter + decorator. + assert len(pre) == 2 diff --git a/tests/test_llm_route.py b/tests/test_llm_route.py new file mode 100644 index 0000000..a4e7eb9 --- /dev/null +++ b/tests/test_llm_route.py @@ -0,0 +1,155 @@ +"""Tests for the LLM chain-link route on the sidecar. + +Spins up the real sidecar (proxy.init) with a stub enforcement and a local +mock 'upstream' so we exercise the actual routing/forwarding code. +""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import httpx +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops import proxy +from firstops.events import CHANNEL_LLM, EVENT_PRE_TOOL_USE, Decision + +_PROXY_PORT = 19940 +_UPSTREAM_PORT = 19941 + + +def _generate_pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +class _StubEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + return self.decision + + +class _MockUpstream(BaseHTTPRequestHandler): + captured: list = [] + + def do_POST(self): + n = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(n) if n else b"" + _MockUpstream.captured.append( + { + "path": self.path, + "auth": self.headers.get("Authorization"), + "body": body, + } + ) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"id": "resp-1", "ok": True}).encode()) + + def log_message(self, *a): + pass + + +@pytest.fixture +def sidecar(): + """Start a mock upstream + the sidecar with a controllable stub enforcement.""" + _MockUpstream.captured = [] + upstream = HTTPServer(("127.0.0.1", _UPSTREAM_PORT), _MockUpstream) + t = threading.Thread(target=upstream.serve_forever, daemon=True) + t.start() + + state = {} + + def start(decision: Decision) -> _StubEnforcement: + stub = _StubEnforcement(decision) + state["stub"] = stub + proxy.init( + agent_id="agent-1", + private_key_pem=_generate_pem(), + port=_PROXY_PORT, + gateway_url="http://127.0.0.1:1", + enforcement=stub, + llm_upstreams={"openai": f"http://127.0.0.1:{_UPSTREAM_PORT}"}, + ) + return stub + + yield start + + proxy.shutdown() + upstream.shutdown() + upstream.server_close() + + +def _post(body: dict, auth: str = "Bearer sk-user-key"): + return httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/chat/completions", + json=body, + headers={"Authorization": auth}, + timeout=5.0, + ) + + +def test_llm_allow_forwards_and_passes_auth_through(sidecar): + stub = sidecar(Decision(action="allow")) + resp = _post({"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}) + + assert resp.status_code == 200 + assert resp.json()["ok"] is True + # Reached the upstream at the right path with the user's key passed through. + assert len(_MockUpstream.captured) == 1 + cap = _MockUpstream.captured[0] + assert cap["path"] == "/v1/chat/completions" + assert cap["auth"] == "Bearer sk-user-key" + # Governed as an LLM-channel pre event. + assert stub.events[0].channel == CHANNEL_LLM + assert stub.events[0].event_type == EVENT_PRE_TOOL_USE + assert stub.events[0].tool_input["model"] == "gpt-4o" + + +def test_llm_deny_returns_403_and_does_not_forward(sidecar): + sidecar(Decision(action="deny", reason="prompt injection", policy_id="pi-1")) + resp = _post({"model": "gpt-4o", "messages": [{"role": "user", "content": "x"}]}) + + assert resp.status_code == 403 + assert "firstops" in resp.json()["error"]["type"] + assert _MockUpstream.captured == [] # never hit the upstream + + +def test_llm_modify_rewrites_body_before_forward(sidecar): + import base64 + + scrubbed = json.dumps( + {"model": "gpt-4o", "messages": [{"role": "user", "content": "[REDACTED]"}]} + ).encode() + dec = Decision.from_wire( + {"decision": "modify", "modified_payload": base64.b64encode(scrubbed).decode()} + ) + sidecar(dec) + _post({"model": "gpt-4o", "messages": [{"role": "user", "content": "my SSN is 123"}]}) + + assert len(_MockUpstream.captured) == 1 + forwarded = json.loads(_MockUpstream.captured[0]["body"]) + assert forwarded["messages"][0]["content"] == "[REDACTED]" + + +def test_llm_unknown_provider_502(sidecar): + sidecar(Decision(action="allow")) + resp = httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}/llm/cohere/v1/chat", + json={"x": 1}, + timeout=5.0, + ) + assert resp.status_code == 502 diff --git a/tests/test_llm_route_adversarial.py b/tests/test_llm_route_adversarial.py new file mode 100644 index 0000000..2c7bb9b --- /dev/null +++ b/tests/test_llm_route_adversarial.py @@ -0,0 +1,358 @@ +"""Adversarial tests for the LLM chain-link route — fresh network code. + +The /llm//... route is new and sits in the request data path. These +tests probe it the way an attacker or a malformed client would: + + - path traversal in the request line (governance sees one path, upstream + receives another) — a path-confusion bug + - non-dict / non-JSON / empty bodies (governed? forwarded? visible?) + - query-string preservation, GET vs POST, missing Authorization + - SSE streaming passthrough + - the load-bearing invariant #2: the agent's model key (Authorization) is + forwarded verbatim and NEVER appears in any log output + +Reuses the _StubEnforcement + mock-upstream pattern from test_llm_route.py. +Bugs are xfail(strict=True); everything else is a positive regression guard. +""" + +from __future__ import annotations + +import io +import json +import logging +import socket +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +import httpx +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops import proxy +from firstops.events import CHANNEL_LLM, Decision + +_PROXY_PORT = 19942 +_UPSTREAM_PORT = 19943 +_SECRET_KEY = "Bearer sk-SUPERSECRETMODELKEY-do-not-log" + + +def _pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +class _StubEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + return self.decision + + +class _MockUpstream(BaseHTTPRequestHandler): + captured: list = [] + sse: bool = False + + def _handle(self, method: str): + n = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(n) if n else b"" + _MockUpstream.captured.append( + { + "method": method, + "path": self.path, + "auth": self.headers.get("Authorization"), + "body": body, + } + ) + if _MockUpstream.sse: + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + for i in range(3): + self.wfile.write(f"data: chunk{i}\n\n".encode()) + self.wfile.flush() + time.sleep(0.02) + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({"ok": True}).encode()) + + def do_POST(self): + self._handle("POST") + + def do_GET(self): + self._handle("GET") + + def log_message(self, *a): + pass + + +@pytest.fixture +def sidecar(): + """Start a mock upstream + sidecar; also capture all 'firstops' log output + so tests can assert the model key never appears in logs.""" + _MockUpstream.captured = [] + _MockUpstream.sse = False + upstream = HTTPServer(("127.0.0.1", _UPSTREAM_PORT), _MockUpstream) + t = threading.Thread(target=upstream.serve_forever, daemon=True) + t.start() + + log_buf = io.StringIO() + handler = logging.StreamHandler(log_buf) + handler.setLevel(logging.DEBUG) + fo_logger = logging.getLogger("firstops") + prior_level = fo_logger.level + fo_logger.setLevel(logging.DEBUG) + fo_logger.addHandler(handler) + + state = {"logs": log_buf} + + def start(decision: Decision) -> _StubEnforcement: + stub = _StubEnforcement(decision) + state["stub"] = stub + proxy.init( + agent_id="agent-1", + private_key_pem=_pem(), + port=_PROXY_PORT, + gateway_url="http://127.0.0.1:1", + enforcement=stub, + llm_upstreams={"openai": f"http://127.0.0.1:{_UPSTREAM_PORT}"}, + ) + return stub + + state["start"] = start + yield state + + fo_logger.removeHandler(handler) + fo_logger.setLevel(prior_level) + proxy.shutdown() + upstream.shutdown() + upstream.server_close() + + +def _post(body, auth: str = _SECRET_KEY, path="/llm/openai/v1/chat/completions", **kw): + return httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}{path}", + headers={"Authorization": auth}, + timeout=5.0, + **({"json": body} if isinstance(body, (dict, list)) else {"content": body}), + **kw, + ) + + +def _raw_request(path: str, body: bytes = b'{"a":1}') -> str: + """Send a request with a literal, un-normalized path (httpx would collapse + `..` client-side, hiding the traversal). Returns the HTTP status line.""" + s = socket.create_connection(("127.0.0.1", _PROXY_PORT), timeout=5) + req = ( + f"POST {path} HTTP/1.1\r\nHost: x\r\n" + f"Content-Length: {len(body)}\r\n" + f"Content-Type: application/json\r\n" + f"Authorization: {_SECRET_KEY}\r\nConnection: close\r\n\r\n" + ).encode() + body + s.sendall(req) + data = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + data += chunk + s.close() + return data.split(b"\r\n", 1)[0].decode() + + +# --------------------------------------------------------------------------- +# INVARIANT #2: the model key is never logged. +# --------------------------------------------------------------------------- + + +def test_model_key_never_appears_in_logs(sidecar): + stub = sidecar["start"](Decision(action="allow")) + _post({"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}) + assert _MockUpstream.captured[-1]["auth"] == _SECRET_KEY # forwarded verbatim + logs = sidecar["logs"].getvalue() + assert "SUPERSECRETMODELKEY" not in logs, "model key leaked into logs" + # And it must not be in any captured pre-event tool_input either. + assert "SUPERSECRETMODELKEY" not in json.dumps(stub.events[0].tool_input) + + +def test_model_key_not_logged_even_on_upstream_failure(sidecar): + """Point the provider at a dead upstream so _forward hits the error path, + which logs. The key must still never appear.""" + sidecar["start"](Decision(action="allow")) + # Re-init with a dead upstream port. + proxy.shutdown() + stub = _StubEnforcement(Decision(action="allow")) + proxy.init( + agent_id="agent-1", + private_key_pem=_pem(), + port=_PROXY_PORT, + gateway_url="http://127.0.0.1:1", + enforcement=stub, + llm_upstreams={"openai": "http://127.0.0.1:1"}, # nothing listening + ) + resp = _post({"model": "gpt-4o", "messages": []}) + assert resp.status_code == 502 + assert "SUPERSECRETMODELKEY" not in sidecar["logs"].getvalue() + + +# --------------------------------------------------------------------------- +# Path confusion / traversal. governance sees /llm/openai/... but the upstream +# receives a different, traversed path. +# --------------------------------------------------------------------------- + + +def test_path_traversal_does_not_desync_governed_and_forwarded_path(sidecar): + """FIXED: a `..` in the LLM path is rejected with 400 before governance or + forwarding, so the governed path can never desync from the forwarded path.""" + stub = sidecar["start"](Decision(action="allow")) + status = _raw_request("/llm/openai/../secret/admin") + # Rejected outright — never governed, never forwarded. + assert "400" in status + assert _MockUpstream.captured == [] + assert stub.events == [] + + +# --------------------------------------------------------------------------- +# Body shape handling: array, non-JSON, empty. +# --------------------------------------------------------------------------- + + +def test_json_array_body_is_forwarded_ungoverned(sidecar): + """A JSON array (not a dict) cannot be governed by the current parser + (`isinstance(parsed, dict)` gate). Documents that it is forwarded WITHOUT a + pre-event — a governance hole. Fail-open is acceptable, silence is not: this + test makes the gap visible so a future change can flag/audit it.""" + stub = sidecar["start"](Decision(action="deny", reason="should-not-apply")) + resp = httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/embeddings", + content=b"[1, 2, 3]", + headers={"Authorization": _SECRET_KEY, "Content-Type": "application/json"}, + timeout=5.0, + ) + # Even though the stub would DENY, an array body bypasses governance and is + # forwarded as-is. + assert resp.status_code == 200 + assert stub.events == [] + assert _MockUpstream.captured[-1]["body"] == b"[1, 2, 3]" + + +def test_non_json_body_forwarded_ungoverned(sidecar): + stub = sidecar["start"](Decision(action="deny", reason="should-not-apply")) + resp = httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/audio", + content=b"\x00\x01binary-not-json", + headers={"Authorization": _SECRET_KEY}, + timeout=5.0, + ) + assert resp.status_code == 200 + assert stub.events == [] # un-governed + assert _MockUpstream.captured[-1]["body"] == b"\x00\x01binary-not-json" + + +def test_empty_body_is_forwarded_and_ungoverned(sidecar): + stub = sidecar["start"](Decision(action="deny", reason="should-not-apply")) + resp = httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/models", + content=b"", + headers={"Authorization": _SECRET_KEY}, + timeout=5.0, + ) + assert resp.status_code == 200 + assert stub.events == [] + assert len(_MockUpstream.captured) == 1 + + +# --------------------------------------------------------------------------- +# Request mechanics: query preservation, GET, missing auth, large body. +# --------------------------------------------------------------------------- + + +def test_query_string_is_preserved_to_upstream(sidecar): + sidecar["start"](Decision(action="allow")) + _post( + {"model": "gpt-4o", "messages": []}, + path="/llm/openai/v1/chat/completions?api-version=2024&beta=true", + ) + assert ( + _MockUpstream.captured[-1]["path"] + == "/v1/chat/completions?api-version=2024&beta=true" + ) + + +def test_get_to_llm_route_forwards_as_get(sidecar): + sidecar["start"](Decision(action="allow")) + resp = httpx.get( + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/models", + headers={"Authorization": _SECRET_KEY}, + timeout=5.0, + ) + assert resp.status_code == 200 + assert _MockUpstream.captured[-1]["method"] == "GET" + + +def test_missing_authorization_still_forwards(sidecar): + """No Authorization header: the route must still forward (fail-open), it + just forwards no auth. The upstream decides.""" + sidecar["start"](Decision(action="allow")) + resp = httpx.post( + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/chat/completions", + json={"model": "gpt-4o", "messages": []}, + timeout=5.0, + ) + assert resp.status_code == 200 + assert _MockUpstream.captured[-1]["auth"] is None + + +def test_large_body_round_trips(sidecar): + sidecar["start"](Decision(action="allow")) + big = {"model": "gpt-4o", "messages": [{"role": "user", "content": "x" * 200_000}]} + resp = _post(big) + assert resp.status_code == 200 + assert len(_MockUpstream.captured[-1]["body"]) > 200_000 + + +def test_sse_response_streams_back(sidecar): + sidecar["start"](Decision(action="allow")) + _MockUpstream.sse = True + chunks = [] + with httpx.stream( + "POST", + f"http://127.0.0.1:{_PROXY_PORT}/llm/openai/v1/chat/completions", + json={"model": "gpt-4o", "stream": True, "messages": []}, + headers={"Authorization": _SECRET_KEY}, + timeout=5.0, + ) as resp: + assert resp.status_code == 200 + assert "text/event-stream" in resp.headers.get("content-type", "") + for c in resp.iter_text(): + chunks.append(c) + assert "data: chunk0" in "".join(chunks) + assert "data: chunk2" in "".join(chunks) + + +def test_deny_response_shape_is_provider_error_envelope(sidecar): + sidecar["start"](Decision(action="deny", reason="pii", policy_id="p-9")) + resp = _post({"model": "gpt-4o", "messages": []}) + assert resp.status_code == 403 + err = resp.json()["error"] + assert err["type"] == "firstops_policy_violation" + assert err["policy_id"] == "p-9" + assert _MockUpstream.captured == [] # never forwarded + + +def test_governed_pre_event_is_llm_channel(sidecar): + stub = sidecar["start"](Decision(action="allow")) + _post({"model": "gpt-4o", "messages": []}) + assert stub.events[-1].channel == CHANNEL_LLM diff --git a/tests/test_m3_bughunt.py b/tests/test_m3_bughunt.py new file mode 100644 index 0000000..15fbb2b --- /dev/null +++ b/tests/test_m3_bughunt.py @@ -0,0 +1,328 @@ +"""M3 bug-hunt — adversarial tests for the request-path scrub capability flag +and coverage honesty. + +Two kinds of tests live here: + +1. Regression locks (passing): behavior that is correct today and must stay + correct — the flag survives every decorator wrapper kind and every decision, + and the OpenAI path never claims a scrub it cannot apply. + +2. Bug pins (``xfail(strict=True)``): real holes found during the hunt. Each is + a *failing* test that documents the defect; when the bug is fixed the test + flips to a hard failure (strict xfail) so the fix can't land without + un-xfailing it. Root cause + severity are in each docstring. + +Reuses the ``_FakeEnforcement`` / ``_rt`` pattern from ``test_m3_scrub_coverage``. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from types import SimpleNamespace + +import pytest + +import firstops.tools as tools_mod +from firstops import coverage +from firstops.events import EVENT_PRE_TOOL_USE, ActionEvent, Decision +from firstops.integrations import openai_agents +from firstops.integrations._common import decide, govern_tool +from firstops.tools import FirstOpsPolicyError, _GOVERNED_TOOLS, governed_tool_names, tool + + +class _FakeEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + if event.event_type != EVENT_PRE_TOOL_USE: + return Decision(action="allow") + return self.decision + + +def _rt(decision: Decision): + return SimpleNamespace(enforcement=_FakeEnforcement(decision)) + + +def _modify(new_input: dict) -> Decision: + raw = base64.b64encode(json.dumps(new_input).encode()).decode() + return Decision(action="modify", modified_payload=base64.b64decode(raw)) + + +def _pre_events(rt): + return [e for e in rt.enforcement.events if e.event_type == EVENT_PRE_TOOL_USE] + + +def _run_decorated_tool(kind: str, decision: Decision): + """Decorate a tool of the given wrapper kind, drive it once under ``rt``, + swallow a policy block, and return the runtime so the caller can inspect + the emitted pre-event. ``kind`` ∈ {sync, async, gen, agen}.""" + rt = _rt(decision) + orig = tools_mod._runtime.runtime + tools_mod._runtime.runtime = lambda: rt + try: + if kind == "sync": + + @tool + def f(x): + return x + + try: + f(1) + except FirstOpsPolicyError: + pass + elif kind == "async": + + @tool + async def f(x): + return x + + try: + asyncio.run(f(1)) + except FirstOpsPolicyError: + pass + elif kind == "gen": + + @tool + def f(x): + yield x + + try: + list(f(1)) + except FirstOpsPolicyError: + pass + elif kind == "agen": + + @tool + async def f(x): + yield x + + async def drive(): + out = [] + async for i in f(1): + out.append(i) + return out + + try: + asyncio.run(drive()) + except FirstOpsPolicyError: + pass + else: # pragma: no cover - test programming error + raise AssertionError(f"unknown kind {kind!r}") + finally: + tools_mod._runtime.runtime = orig + return rt + + +# ========================================================================== +# Regression locks — flag plumbing through every decorator wrapper (Invariant 1) +# ========================================================================== + + +@pytest.mark.parametrize("kind", ["sync", "async", "gen", "agen"]) +@pytest.mark.parametrize("action", ["allow", "deny", "modify"]) +def test_decorator_sets_flag_true_for_every_wrapper_and_decision(kind, action): + """The decorator can always rebind args from a modify payload, so it MUST + emit producer_can_apply_modify=True on the pre-event for *every* wrapper + kind (sync/async/generator/async-generator) and regardless of the verdict + sentinel returns. If a future refactor routes one wrapper around + ``_evaluate_pre``, sentinel would escalate a scrub to deny for that surface + only — a silent, surface-specific honesty regression. Pin all eight cells.""" + decision = _modify({"x": 9}) if action == "modify" else Decision(action=action, reason="r") + rt = _run_decorated_tool(kind, decision) + pre = _pre_events(rt) + assert len(pre) == 1, f"{kind} emitted {len(pre)} pre-events" + assert pre[0].producer_can_apply_modify is True + + +def test_govern_tool_flag_survives_modify_and_deny(): + """The forwarded flag is independent of the decision: a producer that CAN + apply a modify advertises that up front, before sentinel has decided. Lock + that govern_tool keeps the flag set on deny/modify, not just allow.""" + for decision in (Decision(action="deny", reason="r"), _modify({"a": 1})): + rt = _rt(decision) + govern_tool(rt, "t", {"a": 0}, can_apply_modify=True) + assert _pre_events(rt)[0].producer_can_apply_modify is True + + +# ========================================================================== +# Regression lock — OpenAI honesty invariant (Invariant 2, the central one) +# ========================================================================== + + +@pytest.mark.parametrize( + "decision", + [ + Decision(action="allow"), + Decision(action="deny", reason="r"), + pytest.param(None, id="modify"), # built below to avoid shared mutable default + ], +) +def test_openai_never_claims_modify_capability(decision): + """OpenAI guardrails are read-only. If the OpenAI path ever set the flag + True, sentinel would ship a `modify` the guardrail silently drops → + unscrubbed data proceeds. This must hold across allow/deny/modify verdicts, + because the flag is set BEFORE the verdict is known — a verdict-dependent + leak would be the worst kind (only triggers in production on a real scrub).""" + if decision is None: + decision = _modify({"x": 1}) + rt = _rt(decision) + openai_agents._decide_tool(rt, "send_email", {"to": "a@b.com"}) + assert _pre_events(rt)[0].producer_can_apply_modify is False + + +def test_common_decide_default_leaves_flag_false(): + """The _common.decide default (can_apply_modify=False) is what OpenAI relies + on. A change to that default would silently flip OpenAI's honesty.""" + rt = _rt(Decision(action="allow")) + decide(rt, "t", {}) + assert _pre_events(rt)[0].producer_can_apply_modify is False + + +# ========================================================================== +# Regression lock — flag emitted on the wire ONLY when True (omitempty parity) +# ========================================================================== + + +def test_wire_omits_flag_when_false_present_when_true(): + on = ActionEvent(event_type=EVENT_PRE_TOOL_USE, tool_name="t", producer_can_apply_modify=True) + off = ActionEvent(event_type=EVENT_PRE_TOOL_USE, tool_name="t") + assert on.to_wire()["producer_can_apply_modify"] is True + assert "producer_can_apply_modify" not in off.to_wire() + + +def test_capability_returns_defensive_copy(): + """capability() must hand back a copy — a caller mutating the returned dict + must not corrupt the shared CAPABILITY_MATRIX (which feeds every other + coverage answer in the process).""" + d = coverage.capability("openai_agents") + d["scrub"] = True + assert coverage.capability("openai_agents")["scrub"] is False + + +# ========================================================================== +# BUG PINS — coverage honesty holes (Invariant 3: must not overclaim or crash) +# ========================================================================== + + +def test_coverage_does_not_crash_on_non_string_declared_entries(): + """ROOT CAUSE: ungoverned_tools/coverage_report do + ``sorted(set(declared) - governed)``. If ``declared`` contains mixed types + (e.g. a tool registry that yields ints/None for malformed entries), the + set-difference keeps the non-strings and ``sorted`` raises TypeError + comparing int str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +def _decode_claims(dpop: str) -> dict: + c = dpop.split(".")[1] + c += "=" * (-len(c) % 4) + return json.loads(base64.urlsafe_b64decode(c)) + + +class _MockGateway(BaseHTTPRequestHandler): + captured: list = [] + # The gateway base the proxy is pointed at, used for SSE URL-rewrite checks. + base: str = "" + + def _record(self, method, body=b""): + _MockGateway.captured.append( + { + "method": method, + "path": self.path, + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": body, + } + ) + + def do_POST(self): + n = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(n) if n else b"" + self._record("POST", body) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"ok":true}') + + def do_DELETE(self): + self._record("DELETE") + self.send_response(204) + self.end_headers() + + def do_GET(self): + self._record("GET") + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + # Emit an endpoint event referencing the gateway's own message URL so + # the proxy's gateway→localhost rewrite has something to rewrite. + msg_url = _MockGateway.base + "/mcp/sse/message" + self.wfile.write(f"event: endpoint\ndata: {msg_url}\n\n".encode()) + self.wfile.flush() + + def log_message(self, *a): + pass + + +@pytest.fixture +def gateway(): + _MockGateway.captured = [] + srv = HTTPServer(("127.0.0.1", 0), _MockGateway) + _MockGateway.base = f"http://127.0.0.1:{srv.server_address[1]}" + t = threading.Thread(target=srv.serve_forever, daemon=True) + t.start() + try: + yield _MockGateway.base + finally: + srv.shutdown() + + +def _free_local_port() -> int: + import socket + + s = socket.socket() + s.bind(("127.0.0.1", 0)) + p = s.getsockname()[1] + s.close() + return p + + +def test_delete_is_proxied_with_dpop(gateway): + """DELETE (MCP session teardown) must be forwarded with auth headers.""" + local = _free_local_port() + init(agent_id="ag", private_key_pem=_pem(), port=local, gateway_url=gateway) + try: + time.sleep(0.1) + resp = httpx.delete(f"http://127.0.0.1:{local}/mcp/proxy/conn-1") + assert resp.status_code == 204 + assert len(_MockGateway.captured) == 1 + req = _MockGateway.captured[0] + assert req["method"] == "DELETE" + assert req["headers"]["authorization"] == "Bearer fo_agent_ag" + assert "dpop" in req["headers"] + # htm in the proof must match the verb. + claims = _decode_claims(req["headers"]["dpop"]) + assert claims["htm"] == "DELETE" + finally: + shutdown() + + +def test_mcp_htu_strips_query_string(gateway): + """A request path with a query (e.g. /mcp/sse/message?sessionId=x) must + sign an htu WITHOUT the query — Identity.proof owns this canonicalization + after the refactor. A regression here re-introduces htu-mismatch 401s on + every MCP message POST. + """ + local = _free_local_port() + init(agent_id="ag", private_key_pem=_pem(), port=local, gateway_url=gateway) + try: + time.sleep(0.1) + httpx.post( + f"http://127.0.0.1:{local}/mcp/sse/message?sessionId=abc123&seq=2", + json={"jsonrpc": "2.0", "method": "ping", "id": 1}, + headers={"Content-Type": "application/json"}, + ) + req = _MockGateway.captured[-1] + # The upstream path still carries the query (proxy forwards it)... + assert "sessionId=abc123" in req["path"] + # ...but the SIGNED htu must NOT. + claims = _decode_claims(req["headers"]["dpop"]) + assert "?" not in claims["htu"], f"htu leaked query: {claims['htu']!r}" + assert claims["htu"].endswith("/mcp/sse/message") + assert claims["htu"] == gateway + "/mcp/sse/message" + finally: + shutdown() + + +def test_forwards_all_three_headers(gateway): + local = _free_local_port() + init(agent_id="ag", private_key_pem=_pem(), port=local, gateway_url=gateway) + try: + time.sleep(0.1) + httpx.post( + f"http://127.0.0.1:{local}/mcp/proxy/conn-1", + content=b'{"x":1}', + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + "Mcp-Session-Id": "sess-42", + }, + ) + h = _MockGateway.captured[-1]["headers"] + assert h["content-type"] == "application/json" + assert h["accept"] == "application/json" + assert h["mcp-session-id"] == "sess-42" + finally: + shutdown() + + +def test_sse_streams_and_rewrites_gateway_url_to_localhost(gateway): + """An SSE GET must stream through and the gateway message URL must be + rewritten to the localhost sidecar URL (so the MCP client posts back + through the proxy, not directly to the gateway). + """ + local = _free_local_port() + init(agent_id="ag", private_key_pem=_pem(), port=local, gateway_url=gateway) + try: + time.sleep(0.1) + with httpx.stream( + "GET", + f"http://127.0.0.1:{local}/mcp/sse/connect", + headers={"Accept": "text/event-stream"}, + timeout=5.0, + ) as resp: + assert resp.status_code == 200 + assert "text/event-stream" in resp.headers.get("content-type", "") + body = "" + for chunk in resp.iter_text(): + body += chunk + if "data:" in body: + break + # The gateway base must be rewritten to the local sidecar. + local_msg = f"http://127.0.0.1:{local}/mcp/sse/message" + assert local_msg in body, f"SSE URL not rewritten to localhost; got {body!r}" + assert gateway + "/mcp/sse/message" not in body + finally: + shutdown() diff --git a/tests/test_proxy_router_regression.py b/tests/test_proxy_router_regression.py new file mode 100644 index 0000000..1543422 --- /dev/null +++ b/tests/test_proxy_router_regression.py @@ -0,0 +1,138 @@ +"""Router-refactor regression: /mcp/ DPoP-signs, /llm/ never does. + +T1.3 split proxy.py into a router (/mcp/ terminal vs /llm// +chain-link). The one invariant the refactor must never break: the MCP route +still attaches the agent's Bearer + DPoP proof, and the LLM route NEVER +DPoP-signs an upstream (the agent's own Authorization passes through verbatim, +and no `DPoP` header is added). A single mock server backs BOTH an MCP gateway +path and an LLM upstream path so the two routes are compared apples-to-apples +against the same listener. +""" + +from __future__ import annotations + +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +import httpx +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops import proxy +from firstops.events import Decision + + +def _pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +class _Recorder(BaseHTTPRequestHandler): + captured: list = [] + + def do_POST(self): + n = int(self.headers.get("Content-Length", 0)) + self.rfile.read(n) if n else None + _Recorder.captured.append( + { + "path": self.path, + "auth": self.headers.get("Authorization"), + "dpop": self.headers.get("DPoP"), + } + ) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"ok":true}') + + def log_message(self, *a): + pass + + +class _StubEnforcement: + def evaluate(self, event): + return Decision(action="allow") + + +@pytest.fixture +def router(): + _Recorder.captured = [] + server = HTTPServer(("127.0.0.1", 0), _Recorder) + base = f"http://127.0.0.1:{server.server_address[1]}" + threading.Thread(target=server.serve_forever, daemon=True).start() + + local = _free_port() + # The SAME server is both the MCP gateway and the openai LLM upstream. + proxy.init( + agent_id="ag", + private_key_pem=_pem(), + port=local, + gateway_url=base, + enforcement=_StubEnforcement(), + llm_upstreams={"openai": base}, + ) + time.sleep(0.1) + try: + yield local + finally: + proxy.shutdown() + server.shutdown() + + +def _free_port() -> int: + import socket + + s = socket.socket() + s.bind(("127.0.0.1", 0)) + p = s.getsockname()[1] + s.close() + return p + + +def test_mcp_route_dpop_signs(router): + httpx.post( + f"http://127.0.0.1:{router}/mcp/proxy/conn-1", + content=b'{"x":1}', + headers={"Content-Type": "application/json"}, + timeout=5.0, + ) + cap = _Recorder.captured[-1] + assert cap["auth"] == "Bearer fo_agent_ag" # FirstOps brokered credential + assert cap["dpop"] is not None # DPoP proof attached + + +def test_llm_route_never_dpop_signs_and_passes_agent_auth(router): + httpx.post( + f"http://127.0.0.1:{router}/llm/openai/v1/chat/completions", + json={"model": "gpt-4o", "messages": []}, + headers={"Authorization": "Bearer sk-agent-own-key"}, + timeout=5.0, + ) + cap = _Recorder.captured[-1] + # The agent's OWN key is forwarded verbatim — NOT replaced with the + # FirstOps brokered token. + assert cap["auth"] == "Bearer sk-agent-own-key" + # And crucially: no DPoP proof is ever attached to an LLM upstream. + assert cap["dpop"] is None + + +def test_llm_route_does_not_inject_firstops_bearer(router): + """Even when the agent sends NO Authorization, the LLM route must not + fall back to injecting the FirstOps brokered Bearer (that would leak the + brokered credential to an arbitrary upstream).""" + httpx.post( + f"http://127.0.0.1:{router}/llm/openai/v1/models", + json={"model": "gpt-4o"}, + timeout=5.0, + ) + cap = _Recorder.captured[-1] + assert cap["auth"] != "Bearer fo_agent_ag" + assert cap["dpop"] is None diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 0000000..0a74195 --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,52 @@ +"""Tests for the process runtime wiring (init / shutdown / idempotency).""" + +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops import _runtime +from firstops.enforcement import EnforcementClient + +# Unreachable gateway — init() must not connect anywhere; the sidecar only +# binds a local listener. A high port avoids collisions with other suites. +_GATEWAY = "http://127.0.0.1:1" +_PORT = 19911 + + +def _generate_pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +def test_init_builds_runtime_and_shutdown_clears_it(): + pem = _generate_pem() + rt = _runtime.init("agent-1", pem, port=_PORT, gateway_url=_GATEWAY) + try: + assert rt.identity.agent_id == "agent-1" + assert rt.identity.gateway_url == _GATEWAY # normalized, no trailing slash + assert isinstance(rt.enforcement, EnforcementClient) + assert _runtime.runtime() is rt + finally: + _runtime.shutdown() + assert _runtime.runtime() is None + + +def test_init_is_idempotent_for_same_identity(): + pem = _generate_pem() + rt1 = _runtime.init("agent-1", pem, port=_PORT, gateway_url=_GATEWAY) + try: + rt2 = _runtime.init("agent-1", pem, port=_PORT, gateway_url=_GATEWAY) + assert rt1 is rt2 + finally: + _runtime.shutdown() + + +def test_shutdown_is_idempotent(): + # No init() — shutdown() must be a safe no-op. + _runtime.shutdown() + assert _runtime.runtime() is None diff --git a/tests/test_runtime_adversarial.py b/tests/test_runtime_adversarial.py new file mode 100644 index 0000000..9202ef2 --- /dev/null +++ b/tests/test_runtime_adversarial.py @@ -0,0 +1,206 @@ +"""Adversarial lifecycle tests for _runtime.py (M0.T0.5). + +The runtime keeps process-global state (_runtime) and delegates idempotency + +the agent-mismatch guard to proxy.init. These tests probe the seams: +double-init with different params, init→shutdown→init, shutdown-without-init, +port-already-bound, re-key, and module-global leakage between tests. + +A fixture force-resets the global between tests so a leak in one test can't +mask a bug in another (and so these tests stay Isolated). +""" + +from __future__ import annotations + +import socket + +import pytest +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, +) + +from firstops import _runtime, proxy +from firstops.dpop import jwk_thumbprint + +# Unreachable gateway — init binds only a local listener, never connects out. +_GATEWAY = "http://127.0.0.1:1" + + +def _pem() -> str: + key = ec.generate_private_key(ec.SECP256R1()) + return key.private_bytes( + Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption() + ).decode() + + +@pytest.fixture(autouse=True) +def _clean_runtime(): + """Guarantee a clean global before AND after each test (Isolated).""" + _runtime.shutdown() + yield + _runtime.shutdown() + + +def _free_port() -> int: + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +# --------------------------------------------------------------------------- +# basic lifecycle +# --------------------------------------------------------------------------- + + +def test_shutdown_without_init_is_noop(): + # autouse fixture already called shutdown(); call again explicitly. + _runtime.shutdown() + assert _runtime.runtime() is None + assert proxy.is_running() is False + + +def test_init_then_shutdown_then_init_again_is_fresh_runtime(): + port = _free_port() + pem = _pem() + rt1 = _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + _runtime.shutdown() + assert _runtime.runtime() is None + rt2 = _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + assert rt2 is not rt1, "re-init after shutdown must build a fresh Runtime" + assert _runtime.runtime() is rt2 + + +# --------------------------------------------------------------------------- +# mismatch guard — different identity while running +# --------------------------------------------------------------------------- + + +def test_double_init_different_agent_raises_and_leaves_runtime_intact(): + port = _free_port() + pem = _pem() + rt = _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + with pytest.raises(RuntimeError): + _runtime.init("agent-2", pem, port=port, gateway_url=_GATEWAY) + # The original runtime + proxy identity must be untouched by the failed call. + assert _runtime.runtime() is rt + assert proxy.current_agent_id() == "agent-1" + + +def test_double_init_different_gateway_raises(): + port = _free_port() + pem = _pem() + _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + with pytest.raises(RuntimeError): + _runtime.init("agent-1", pem, port=port, gateway_url="http://127.0.0.1:2") + assert proxy.current_agent_id() == "agent-1" + + +def test_double_init_different_port_raises(): + port = _free_port() + pem = _pem() + _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + with pytest.raises(RuntimeError): + _runtime.init("agent-1", pem, port=_free_port(), gateway_url=_GATEWAY) + + +def test_failed_mismatch_init_does_not_leak_a_second_runtime(): + """After a rejected mismatching init, exactly one runtime exists and it is + the original. Guards against 'partial init left _runtime pointing at a new + object whose proxy never started'. + """ + port = _free_port() + pem = _pem() + rt = _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + for bad in [("agent-x", port, _GATEWAY), ("agent-1", port, "http://127.0.0.1:9")]: + with pytest.raises(RuntimeError): + _runtime.init(bad[0], pem, port=bad[1], gateway_url=bad[2]) + assert _runtime.runtime() is rt + + +# --------------------------------------------------------------------------- +# idempotency edge cases +# --------------------------------------------------------------------------- + + +def test_reinit_same_params_is_idempotent_same_object(): + port = _free_port() + pem = _pem() + rt1 = _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + rt2 = _runtime.init("agent-1", pem, port=port, gateway_url=_GATEWAY) + assert rt1 is rt2 + + +def test_reinit_same_params_different_key_raises(): + """FIXED: re-init with same agent/port/gateway but a DIFFERENT private key + now RAISES rather than silently keeping the stale key (which would 401 and + fail open invisibly). A re-key must be explicit: shutdown() then init(). + """ + port = _free_port() + pem1, pem2 = _pem(), _pem() + rt1 = _runtime.init("agent-1", pem1, port=port, gateway_url=_GATEWAY) + first_jkt = jwk_thumbprint(rt1.identity._key) + with pytest.raises(RuntimeError, match="different key"): + _runtime.init("agent-1", pem2, port=port, gateway_url=_GATEWAY) + # Original runtime intact, still the first key. + assert _runtime.runtime() is rt1 + assert jwk_thumbprint(_runtime.runtime().identity._key) == first_jkt + + +def test_runtime_gateway_url_is_normalized_no_trailing_slash(): + port = _free_port() + rt = _runtime.init("agent-1", _pem(), port=port, gateway_url=_GATEWAY + "/") + assert rt.identity.gateway_url == _GATEWAY # trailing slash stripped + + +# --------------------------------------------------------------------------- +# resource / binding failures +# --------------------------------------------------------------------------- + + +def test_init_when_port_already_bound_propagates_and_does_not_set_runtime(): + """If the listener port is already taken by something else, init() must + fail (OSError from bind) and must NOT leave a half-built runtime behind. + """ + port = _free_port() + blocker = socket.socket() + blocker.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + blocker.bind(("127.0.0.1", port)) + blocker.listen(1) + try: + with pytest.raises(OSError): + _runtime.init("agent-1", _pem(), port=port, gateway_url=_GATEWAY) + # The failed bind must not have set the global runtime. + assert _runtime.runtime() is None + assert proxy.is_running() is False + finally: + blocker.close() + + +def test_invalid_gateway_url_raises_value_error_and_no_runtime(): + with pytest.raises(ValueError): + _runtime.init("agent-1", _pem(), port=_free_port(), gateway_url="not-a-url") + assert _runtime.runtime() is None + + +def test_invalid_pem_raises_and_no_runtime(): + with pytest.raises(Exception): + _runtime.init("agent-1", "-----BEGIN nonsense-----", port=_free_port(), gateway_url=_GATEWAY) + assert _runtime.runtime() is None + assert proxy.is_running() is False + + +# --------------------------------------------------------------------------- +# global leakage guard +# --------------------------------------------------------------------------- + + +def test_runtime_global_is_isolated_between_tests(): + """If a previous test leaked the global, this fails — proving the fixture + actually isolates. (Also a canary if someone removes the fixture.) + """ + assert _runtime.runtime() is None + assert proxy.is_running() is False diff --git a/tests/test_tools.py b/tests/test_tools.py new file mode 100644 index 0000000..0511c38 --- /dev/null +++ b/tests/test_tools.py @@ -0,0 +1,168 @@ +"""Tests for the @firstops.tool base decorator.""" + +import asyncio +import base64 +import inspect +import json +from types import SimpleNamespace + +import pytest + +import firstops._runtime as rt_mod +from firstops.events import ( + CHANNEL_SYSTEM_TOOLS, + EVENT_POST_TOOL_USE, + EVENT_PRE_TOOL_USE, + Decision, +) +from firstops.tools import FirstOpsPolicyError, tool + + +class _FakeEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + # Post events are audit-only; always allow them regardless of script. + if event.event_type == EVENT_POST_TOOL_USE: + return Decision(action="allow") + return self.decision + + +@pytest.fixture +def govern(monkeypatch): + """Install a fake runtime returning a scripted decision; yield the fake.""" + + def _install(decision: Decision) -> _FakeEnforcement: + fake = _FakeEnforcement(decision) + monkeypatch.setattr( + rt_mod, "runtime", lambda: SimpleNamespace(enforcement=fake) + ) + return fake + + return _install + + +def test_tool_allows_and_emits_pre_event(govern): + fake = govern(Decision(action="allow")) + + @tool + def add(a, b): + return a + b + + assert add(2, 3) == 5 + pre = [e for e in fake.events if e.event_type == EVENT_PRE_TOOL_USE] + assert len(pre) == 1 + assert pre[0].tool_name == "add" + assert pre[0].channel == CHANNEL_SYSTEM_TOOLS + assert pre[0].tool_input == {"a": 2, "b": 3} + + +def test_tool_emits_post_audit(govern): + fake = govern(Decision(action="allow")) + + @tool + def echo(x): + return x + + echo("hi") + post = [e for e in fake.events if e.event_type == EVENT_POST_TOOL_USE] + assert len(post) == 1 + assert post[0].tool_output == {"result": "hi"} + + +def test_tool_deny_raises_and_does_not_run(govern): + govern(Decision(action="deny", reason="not allowed", policy_id="p1")) + ran = {"called": False} + + @tool + def danger(): + ran["called"] = True + return "ran" + + with pytest.raises(FirstOpsPolicyError) as exc: + danger() + assert exc.value.tool_name == "danger" + assert exc.value.policy_id == "p1" + assert ran["called"] is False # body never executed + + +def test_tool_modify_applies_scrubbed_args(govern): + payload = base64.b64encode(json.dumps({"x": 99}).encode()).decode() + govern(Decision.from_wire({"decision": "modify", "modified_payload": payload})) + + @tool + def keep(x): + return x + + # Called with x=1, but the modify replaces it with x=99. + assert keep(1) == 99 + + +def test_tool_async(govern): + fake = govern(Decision(action="allow")) + + @tool + async def aadd(a, b): + return a + b + + assert asyncio.run(aadd(4, 5)) == 9 + assert any(e.event_type == EVENT_PRE_TOOL_USE for e in fake.events) + + +def test_tool_async_deny_raises(govern): + govern(Decision(action="deny", reason="no")) + + @tool + async def atool(): + return "ran" + + with pytest.raises(FirstOpsPolicyError): + asyncio.run(atool()) + + +def test_tool_no_runtime_runs_ungoverned(monkeypatch): + monkeypatch.setattr(rt_mod, "runtime", lambda: None) + + @tool + def add(a, b): + return a + b + + assert add(1, 1) == 2 # no governance, just runs + + +def test_tool_preserves_signature_and_name(): + @tool + def documented(a: int, b: str = "x") -> str: + """my docstring""" + return f"{a}{b}" + + assert documented.__name__ == "documented" + assert documented.__doc__ == "my docstring" + sig = inspect.signature(documented) + assert list(sig.parameters) == ["a", "b"] + + +def test_tool_refuses_built_framework_tool_object(): + class FakeStructuredTool: + def invoke(self, *a, **k): + return None + + def __call__(self, *a, **k): + return None + + with pytest.raises(TypeError, match="plain function"): + tool(FakeStructuredTool()) + + +def test_tool_custom_name(govern): + fake = govern(Decision(action="allow")) + + @tool(name="renamed") + def original(): + return 1 + + original() + assert fake.events[0].tool_name == "renamed" diff --git a/tests/test_tools_adversarial.py b/tests/test_tools_adversarial.py new file mode 100644 index 0000000..a92844b --- /dev/null +++ b/tests/test_tools_adversarial.py @@ -0,0 +1,290 @@ +"""Adversarial tests for @firstops.tool — designed to FIND BUGS, not to pass. + +These probe the decorator's two riskiest surfaces: + + 1. ``_apply_pre`` reconstructs a *modify*'d call as pure keyword args + (``return (), new_input``). That is wrong for any function whose + parameters are not all bind-as-keyword: positional-only params, + ``*args``, and bound methods (``self``) all explode into a TypeError + that propagates INTO THE CALLER. Per invariant #1 (fail-open everywhere + except a real DENY), a benign ``modify`` must never crash the tool. + + 2. ``post_tool_use`` audit is skipped when the body raises, and for + generator/async-generator tools it fires on the *un-iterated* generator + object — so the audited output is a generator repr, never the yielded + values. Scrub-on-output is structurally impossible for streaming tools, + and failed calls are invisible to governance. + +Bugs are encoded as ``xfail(strict=True)``: they fail today (documenting the +hole) and will turn into a hard failure the moment the bug is fixed, so they +double as regression guards. Non-bug behaviors are plain asserting tests. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from types import SimpleNamespace + +import pytest + +import firstops._runtime as rt_mod +from firstops.events import ( + EVENT_POST_TOOL_USE, + EVENT_PRE_TOOL_USE, + Decision, +) +from firstops.tools import FirstOpsPolicyError, tool + + +class _FakeEnforcement: + def __init__(self, decision: Decision): + self.decision = decision + self.events = [] + + def evaluate(self, event): + self.events.append(event) + if event.event_type == EVENT_POST_TOOL_USE: + return Decision(action="allow") + return self.decision + + +@pytest.fixture +def govern(monkeypatch): + def _install(decision: Decision) -> _FakeEnforcement: + fake = _FakeEnforcement(decision) + monkeypatch.setattr( + rt_mod, "runtime", lambda: SimpleNamespace(enforcement=fake) + ) + return fake + + return _install + + +def _modify(payload: dict) -> Decision: + raw = base64.b64encode(json.dumps(payload).encode()).decode() + return Decision.from_wire({"decision": "modify", "modified_payload": raw}) + + +# --------------------------------------------------------------------------- +# BUG CLASS A: modify reconstructs the call as pure-kwargs and crashes the +# caller for any non-keyword-bindable signature. Root cause: tools._apply_pre +# does ``return (), new_input`` unconditionally. Severity: BLOCKING — turns a +# benign policy decision into a hard crash, violating fail-open invariant #1. +# --------------------------------------------------------------------------- + + +def test_modify_positional_only_param_does_not_crash_caller(govern): + govern(_modify({"x": 99})) + + @tool + def posonly(x, /): + return x + + # A modify rewriting x->99 must apply (or, if unapplicable, fail open to + # the original call). It must NOT raise TypeError into the caller. + assert posonly(1) == 99 + + +def test_modify_varargs_tool_does_not_crash_caller(govern): + govern(_modify({"args": [1, 2, 3]})) + + @tool + def variadic(*args): + return sum(args) + + # Whatever the scrub semantics, the call must not explode. + variadic(10) + + +def test_modify_method_does_not_crash_caller(govern): + govern(_modify({"x": 7})) + + class C: + @tool + def m(self, x): + return x + + assert C().m(1) == 7 + + +def test_modify_mismatched_keys_falls_open_not_crash(govern): + govern(_modify({"totally_unrelated_key": 1})) + + @tool + def strict(a): + return a + + # The scrub key doesn't fit the signature; we should fall back to the + # original args (fail-open), not raise into the caller. + assert strict(5) == 5 + + +def test_modify_matching_keyword_args_still_works(govern): + """Sanity guard: the happy path (kwarg-bindable signature) DOES apply.""" + govern(_modify({"a": 1, "b": 2})) + + @tool + def add(a, b): + return a + b + + assert add(9, 9) == 3 # 1 + 2, scrubbed + + +# --------------------------------------------------------------------------- +# BUG CLASS B: generators. A @tool on a (sync) generator returns the generator +# immediately, so post_tool_use audit fires BEFORE the body runs and audits a +# generator repr, never the yielded values. Root cause: tools.wrapper has no +# generator branch; inspect.iscoroutinefunction is False for gen funcs. +# Severity: CONCERN — output governance/scrub is silently absent for streaming +# tools, and the post event is misleading. +# --------------------------------------------------------------------------- + + +def test_generator_post_audit_sees_yielded_output_not_generator_repr(govern): + fake = govern(Decision(action="allow")) + + @tool + def stream(): + yield "secret-1" + yield "secret-2" + + list(stream()) # drive the generator to completion + post = [e for e in fake.events if e.event_type == EVENT_POST_TOOL_USE] + assert len(post) == 1 + # Output governance can only scrub what it can see. The audited output must + # reflect the yielded values, not "". + assert "generator object" not in json.dumps(post[0].tool_output) + + +def test_generator_governs_eagerly_and_audits_after_iteration(govern): + """FIXED: governance (pre-event) runs EAGERLY at call time, but post-audit + is deferred until the generator is exhausted — so the audit reflects a + completed stream, not an un-iterated generator object.""" + fake = govern(Decision(action="allow")) + side: list[str] = [] + + @tool + def stream(): + side.append("body-ran") + yield 1 + + gen = stream() + # Pre fired at call; post NOT yet (body not iterated). + assert any(e.event_type == EVENT_PRE_TOOL_USE for e in fake.events) + assert not any(e.event_type == EVENT_POST_TOOL_USE for e in fake.events) + assert side == [] + list(gen) + assert side == ["body-ran"] + post = [e for e in fake.events if e.event_type == EVENT_POST_TOOL_USE] + assert len(post) == 1 + + +def test_generator_deny_blocks_before_iteration(govern): + """Positive guard: a DENY on a generator tool still blocks (the pre-event + runs in the wrapper before the generator object is returned).""" + govern(Decision(action="deny", reason="no")) + + @tool + def stream(): + yield 1 + + with pytest.raises(FirstOpsPolicyError): + stream() + + +def test_async_generator_deny_blocks(govern): + """Positive guard: async-generator tools route through the sync wrapper + (iscoroutinefunction is False), but the pre-event still fires before the + async_generator object is returned, so DENY blocks.""" + govern(Decision(action="deny", reason="no")) + + @tool + async def astream(): + yield 1 + + async def drive(): + async for _ in astream(): + pass + + with pytest.raises(FirstOpsPolicyError): + asyncio.run(drive()) + + +# --------------------------------------------------------------------------- +# BUG CLASS C: a tool that RAISES skips post-audit entirely. Root cause: +# tools.wrapper calls func() then _audit_post() sequentially with no +# try/finally; an exception short-circuits the audit. Severity: CONCERN — +# failed tool calls leave no post_tool_use audit trail. +# --------------------------------------------------------------------------- + + +def test_raising_tool_still_emits_post_audit(govern): + fake = govern(Decision(action="allow")) + + @tool + def boom(): + raise ValueError("kaboom") + + with pytest.raises(ValueError): + boom() + + post = [e for e in fake.events if e.event_type == EVENT_POST_TOOL_USE] + assert len(post) == 1, "failed tool call produced no audit event" + + +def test_each_call_costs_exactly_one_pre_and_one_post_roundtrip(govern): + """Latency guard: post-audit is a second synchronous sentinel round-trip + per call. This is by design but must be asserted so a future change that + adds a third round-trip (or makes post async/batched) is a conscious one. + """ + fake = govern(Decision(action="allow")) + + @tool + def add(a, b): + return a + b + + add(1, 2) + assert [e.event_type for e in fake.events] == [ + EVENT_PRE_TOOL_USE, + EVENT_POST_TOOL_USE, + ] + + +# --------------------------------------------------------------------------- +# Non-bug behaviors worth pinning so a regression is caught. +# --------------------------------------------------------------------------- + + +def test_non_jsonable_arg_is_reprd_not_garbage(govern): + """A non-JSON-able argument (an arbitrary object) must reach enforcement as + a repr string, never raise and never leak a non-serializable value into the + wire body.""" + fake = govern(Decision(action="allow")) + + class Obj: + def __repr__(self): + return "" + + @tool + def takes(o): + return "ok" + + assert takes(Obj()) == "ok" + pre = [e for e in fake.events if e.event_type == EVENT_PRE_TOOL_USE][0] + assert pre.tool_input == {"o": ""} + # The wire body must be JSON-serializable. + json.dumps(pre.tool_input) + + +def test_keyword_only_and_default_args_bind_correctly(govern): + fake = govern(Decision(action="allow")) + + @tool + def f(a, b=10, *, c=20): + return a + b + c + + assert f(1) == 31 + pre = [e for e in fake.events if e.event_type == EVENT_PRE_TOOL_USE][0] + assert pre.tool_input == {"a": 1, "b": 10, "c": 20} From 43cd471eef0dee99e1beb8648404e06286842c8b Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 13 Jun 2026 00:50:33 -0700 Subject: [PATCH 3/9] Release prep v0.2.0: per-harness extras, V2 README - pyproject: bump to 0.2.0, add langgraph/claude/openai/all extras, update description+keywords - README: rewrite for V2 (LLM + tool + MCP governance across harnesses) Co-Authored-By: Claude Opus 4.8 --- README.md | 260 ++++++++++++------------------------------------- pyproject.toml | 31 +++++- 2 files changed, 89 insertions(+), 202 deletions(-) diff --git a/README.md b/README.md index dd30ecb..b409ae2 100644 --- a/README.md +++ b/README.md @@ -1,251 +1,113 @@ # FirstOps Python SDK -The FirstOps SDK has two halves: - -1. **Management client** (`FirstOps`) — programmatically create agents, register MCP connections, and manage their lifecycle from your backend. Authenticates with a tenant-scoped API key. -2. **Runtime proxy** (`firstops.init`) — a lightweight in-process sidecar that transparently signs every MCP request with a [DPoP](https://datatracker.ietf.org/doc/html/rfc9449) proof. Runs inside the agent process. - -The two halves are used at different points in an agent's lifecycle. The management client runs in your **platform code** (the backend that provisions agents). The runtime proxy runs inside the **agent itself** (the process that calls MCP tools). - -## Install +Govern what your AI agents do. FirstOps applies identity, policy enforcement, credential brokering, and audit to every **LLM call**, **tool call**, and **MCP call** your agent makes — across LangGraph, the Claude Agent SDK, and the OpenAI Agents SDK, or any custom loop. ```bash -pip install firstops +pip install "firstops[langgraph]" # or [claude], [openai], [all] ``` -## Requirements - -- Python 3.10+ -- Dependencies: `cryptography`, `httpx` +- **Python 3.10+** +- Core deps: `cryptography`, `httpx`. Your agent framework comes in via the extra you pick. --- -## 1. Management Client — Provisioning Agents - -Use this in your platform's backend code to create agents and wire up their MCP connections on demand. +## What FirstOps governs -### Get an API key +| Surface | How it's wired | What you get | +|---|---|---| +| **LLM calls** | point the model `base_url` at the local sidecar | inspect prompts/responses, scrub PII, block, audit | +| **Tool calls** | one adapter (or `@firstops.tool`) | block / rewrite args / audit — including framework built-ins | +| **MCP servers** | point the MCP client at the local proxy | server-side policy + **credential brokering** (the agent never holds the upstream token) | -1. Log in to the FirstOps dashboard as an admin. -2. Go to **Settings → API Keys** and create a key with the scopes you need: - - `agents:write` — create and delete agent principals - - `agents:read` — list and get agents - - `connections:write` — register and delete MCP connections - - `connections:read` — list connections -3. Copy the raw key (starts with `fo_key_`). It is shown **once** — store it in your secrets manager. +Every action is evaluated by FirstOps and returns `allow` / `deny` / `modify` — your agent logic doesn't change. -### Quick Start +## Quick start (LangGraph) ```python -from firstops import FirstOps - -# Initialize the management client -client = FirstOps(api_key="fo_key_...") - -# 1. Create an agent (returns principal ID, token, and private key) -agent = client.agents.create(name="research-bot-for-alice") -print(f"Agent ID: {agent.id}") -print(f"Agent token: {agent.token}") # fo_agent_ — used in Authorization header -print(f"Private key: {agent.private_key}") # PEM — shown once, save it securely - -# 2. Register MCP connections for the agent -slack_conn = client.connections.register( - principal_id=agent.id, - name="slack", - upstream_url="https://mcp.slack.com/sse", -) +import firstops +from firstops.integrations.langgraph import FirstOpsMiddleware +from langchain.agents import create_agent +from langchain_openai import ChatOpenAI -gdrive_conn = client.connections.register( - principal_id=agent.id, - name="google-drive", - upstream_url="https://mcp.google.com/drive/sse", - auth_type="oauth2", +fo = firstops.init( + agent_id="", # from the FirstOps dashboard + private_key_pem=open("agent-key.pem").read(), ) -# 3. List the agent's connections -for conn in client.connections.list(principal_id=agent.id): - print(f"{conn.name} — {conn.status}") +# Route the LLM through FirstOps; wire one middleware to govern every tool call. +llm = ChatOpenAI(model="gpt-4o-mini", base_url=firstops.llm_base_url("openai"), api_key="sk-...") +agent = create_agent(model=llm, tools=[...], middleware=[FirstOpsMiddleware(fo)]) -# 4. Remove a connection when the user no longer needs it -client.connections.delete(slack_conn.id) - -# 5. Delete the agent when the user deletes their instance -client.agents.delete(agent.id) -``` - -### Context Manager - -`FirstOps` is also usable as a context manager for automatic connection cleanup: - -```python -with FirstOps(api_key="fo_key_...") as client: - agents = client.agents.list() +agent.invoke({"messages": [{"role": "user", "content": "..."}]}) ``` -### API Reference +The whole integration is `init()` + a `base_url` swap + one middleware. See [`examples/`](examples/) for runnable agents, including MCP. -#### `FirstOps(api_key, base_url="https://api.firstops.ai", timeout=30.0)` +## Other harnesses -The top-level management client. +**Claude Agent SDK** — one `PreToolUse` hook governs every tool (built-ins, MCP, custom): -| Parameter | Default | Description | -|-----------|---------|-------------| -| `api_key` | *required* | Tenant-scoped API key (must start with `fo_key_`) | -| `base_url` | `https://api.firstops.ai` | FirstOps API base URL | -| `timeout` | `30.0` | HTTP timeout in seconds | - -#### `client.agents` - -| Method | Required Scope | Returns | -|--------|----------------|---------| -| `create(name: str)` | `agents:write` | `Agent` (with `private_key`) | -| `list()` | `agents:read` | `list[Agent]` | -| `get(agent_id: str)` | `agents:read` | `Agent` | -| `delete(agent_id: str)` | `agents:write` | `None` | +```python +from claude_agent_sdk import query, ClaudeAgentOptions +from firstops.integrations.claude import firstops_hooks -**Note:** `agent.private_key` is only populated on `create()`. It is never returned again — store it alongside your agent record at creation time. +options = ClaudeAgentOptions(hooks=firstops_hooks(fo), permission_mode="bypassPermissions") +async for _ in query(prompt="...", options=options): + pass +``` -#### `client.connections` +**OpenAI Agents SDK** — a guardrail per tool + the model routed through the sidecar: -| Method | Required Scope | Returns | -|--------|----------------|---------| -| `register(principal_id, name, upstream_url, ...)` | `connections:write` | `Connection` | -| `list(principal_id=None)` | `connections:read` | `list[Connection]` | -| `delete(connection_id: str)` | `connections:write` | `None` | +```python +from agents import Agent, function_tool, set_default_openai_client +from firstops.integrations.openai_agents import firstops_tool_input_guardrail +from openai import AsyncOpenAI -Full signature for `register`: +set_default_openai_client(AsyncOpenAI(base_url=firstops.llm_base_url("openai"), api_key="sk-...")) +guard = firstops_tool_input_guardrail(fo) -```python -client.connections.register( - principal_id="pr_...", # required — the agent's principal ID - name="slack", # required — display name - upstream_url="https://mcp.slack.com/sse", # required — remote MCP server URL - auth_type="", # optional — "oauth2", "bearer", etc. - transport_type="", # optional — "sse" or empty for auto-detect - upstream_headers=None, # optional — dict of headers to forward - upstream_query_params=None, # optional — dict of query params - source="sdk", # optional — audit label -) +@function_tool(tool_input_guardrails=[guard]) +def send_email(to: str, body: str) -> str: ... ``` -### Error Handling - -All API errors raise `FirstOpsError`: +**Any framework / custom loop** — the base API: ```python -from firstops import FirstOps, FirstOpsError - -try: - client.agents.delete("pr_does_not_exist") -except FirstOpsError as e: - print(f"Error {e.status_code}: {e.message}") +@firstops.tool # govern any callable: block / scrub args / audit +def send_email(to: str, body: str): ... ``` ---- - -## 2. Runtime Proxy — Securing MCP Calls Inside an Agent +## MCP servers -Use this inside the agent process itself. It starts a local HTTP proxy that transparently adds DPoP-signed authentication headers to every MCP request your agent makes — no changes to your agent code required. - -### Quick Start +Point your MCP client at the local proxy; FirstOps brokers the upstream credentials. ```python -import firstops - -# Start the proxy sidecar (runs in background thread) -firstops.init( - agent_id="your-agent-id", # from client.agents.create(...).id - private_key_pem=open("agent-key.pem").read(), # from client.agents.create(...).private_key -) - -# Point your MCP client at localhost:9322 instead of the remote server. -# The proxy handles auth transparently — DPoP proofs, bearer tokens, SSE streaming. +from langchain_mcp_adapters.client import MultiServerMCPClient -# When done: -firstops.shutdown() +mcp = MultiServerMCPClient({"notion": {"url": firstops.mcp_url(""), "transport": "streamable_http"}}) +tools = await mcp.get_tools() ``` -### What happens +## Management client -1. `firstops.init()` starts a local HTTP proxy on `127.0.0.1:9322` -2. Your MCP client sends requests to `localhost:9322/mcp/...` -3. The proxy signs each request with a DPoP proof (RFC 9449, ES256) -4. The signed request is forwarded to the FirstOps gateway -5. SSE streaming responses are proxied back with URLs rewritten to localhost - -### Configuration - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `agent_id` | *required* | Agent's principal ID (without `fo_agent_` prefix) | -| `private_key_pem` | *required* | EC P-256 private key in PEM format | -| `port` | `9322` | Local proxy port | -| `gateway_url` | `https://api.firstops.ai` | FirstOps gateway URL | - ---- - -## End-to-End Example: Dynamic Agent Platform - -Here's how the two halves fit together in a typical "SaaS that offers AI agents" platform: +Provision agents and connections from your backend: ```python -# ─── Platform backend (your FastAPI/Django service) ───────────── from firstops import FirstOps -firstops = FirstOps(api_key=os.environ["FIRSTOPS_API_KEY"]) - -@app.post("/users/{user_id}/agents") -def create_user_agent(user_id: str, config: dict): - # Create a FirstOps agent identity for this end-user's instance - agent = firstops.agents.create(name=f"research-bot-{user_id}") - - # Store the agent credentials alongside the user's record - db.save_agent( - user_id=user_id, - agent_id=agent.id, - private_key=agent.private_key, # encrypt this at rest - ) - - # Wire up the tools the user selected - for tool in config["selected_tools"]: - firstops.connections.register( - principal_id=agent.id, - name=tool["name"], - upstream_url=tool["url"], - ) - - return {"agent_id": agent.id} - -@app.delete("/users/{user_id}/agents/{agent_id}") -def delete_user_agent(user_id: str, agent_id: str): - firstops.agents.delete(agent_id) # cascades to connections - db.delete_agent(agent_id) - - -# ─── Agent runtime (the worker process that actually runs the agent) ─── -import firstops as fo_runtime - -def run_agent_task(agent_id: str, private_key: str, task: str): - fo_runtime.init(agent_id=agent_id, private_key_pem=private_key) - try: - # Your MCP-using agent logic — point MCP clients at 127.0.0.1:9322 - mcp_client = MCPClient(base_url="http://127.0.0.1:9322") - return mcp_client.run(task) - finally: - fo_runtime.shutdown() +admin = FirstOps(api_key="fo_key_...") +agent = admin.agents.create(name="research-bot") # -> id + private_key (shown once) +admin.connections.register(principal_id=agent.id, name="slack", upstream_url="https://mcp.slack.com/sse") ``` ---- +## How it works -## Development +`firstops.init()` starts a local sidecar and establishes the agent's identity (a DPoP-bound principal — RFC 9449). Tool and LLM actions are forwarded to the FirstOps gateway, which evaluates them against your policies and returns the verdict; MCP and LLM traffic flow through the sidecar with credentials brokered. Enforcement fails open on infrastructure errors; authentication fails closed. -```bash -git clone https://github.com/firstops-dev/firstops-python.git -cd firstops-python -python -m venv .venv && source .venv/bin/activate -pip install -e ".[dev]" -pytest -``` +## Documentation + +- Guides: +- Repository: ## License diff --git a/pyproject.toml b/pyproject.toml index 849c681..01276d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "hatchling.build" [project] name = "firstops" -version = "0.1.0" -description = "FirstOps SDK — secure MCP proxy sidecar with DPoP authentication" +version = "0.2.0" +description = "Govern MCP, tool calls, and LLM traffic for AI agents — across LangGraph, Claude Agent SDK, and OpenAI Agents." readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -24,7 +24,10 @@ classifiers = [ "Topic :: Security", "Topic :: Software Development :: Libraries", ] -keywords = ["mcp", "dpop", "proxy", "agent", "security"] +keywords = [ + "mcp", "dpop", "agent", "security", "governance", "llm", + "langgraph", "langchain", "openai-agents", "claude", "guardrails", +] dependencies = [ "cryptography>=42.0", "httpx>=0.27", @@ -37,6 +40,28 @@ Repository = "https://github.com/firstops-dev/firstops-python" Issues = "https://github.com/firstops-dev/firstops-python/issues" [project.optional-dependencies] +# Per-harness extras: install the framework you use, e.g. +# pip install "firstops[langgraph]" +langgraph = [ + "langchain>=1.0", + "langgraph", + "langchain-openai", + "langchain-mcp-adapters", +] +claude = [ + "claude-agent-sdk", +] +openai = [ + "openai-agents", +] +all = [ + "langchain>=1.0", + "langgraph", + "langchain-openai", + "langchain-mcp-adapters", + "claude-agent-sdk", + "openai-agents", +] dev = [ "pytest>=8.0", "pytest-asyncio>=0.23", From 92eaa14e04832896837a772cfa9f9e801674bdff Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 13 Jun 2026 10:22:45 -0700 Subject: [PATCH 4/9] Add Makefile with one-command release target Co-Authored-By: Claude Opus 4.8 --- Makefile | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7bf09f1 --- /dev/null +++ b/Makefile @@ -0,0 +1,61 @@ +# FirstOps SDK — dev + release tasks. +# +# Release: make release VERSION=0.2.1 +# -> bumps pyproject version, runs tests, commits, tags v0.2.1, pushes the +# tag, and the GitHub `publish.yml` workflow builds + publishes to PyPI. +# +# PyPI versions are immutable: every release needs a NEW version. This target +# keeps the pyproject version and the git tag in lockstep so they can't drift. + +.PHONY: help install test build clean release + +PYTHON ?= python3 +VENV ?= .venv +PY = $(VENV)/bin/python + +help: + @echo "make install create venv + install package and dev deps" + @echo "make test run the test suite" + @echo "make build build wheel + sdist into dist/" + @echo "make clean remove build artifacts" + @echo "make release VERSION=X.Y.Z bump version, test, tag, and publish to PyPI" + +$(VENV): + $(PYTHON) -m venv $(VENV) + +install: $(VENV) + $(PY) -m pip install --upgrade pip + $(PY) -m pip install -e ".[dev]" build + +test: + $(PY) -m pytest -q + +build: clean + $(PY) -m build + +clean: + rm -rf dist build src/*.egg-info *.egg-info + +release: + @if [ -z "$(VERSION)" ]; then \ + echo "Usage: make release VERSION=X.Y.Z"; exit 1; fi + @echo "$(VERSION)" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+([abrc][0-9]+|\.post[0-9]+|\.dev[0-9]+)?$$' \ + || { echo "VERSION must look like 1.2.3 (got '$(VERSION)')"; exit 1; } + @if [ -n "$$(git status --porcelain)" ]; then \ + echo "Working tree not clean — commit or stash your changes first."; \ + echo "(A release commit should contain ONLY the version bump.)"; exit 1; fi + @if git rev-parse "v$(VERSION)" >/dev/null 2>&1; then \ + echo "Tag v$(VERSION) already exists. Pick a new version (PyPI is immutable)."; exit 1; fi + @echo ">> bumping pyproject.toml -> $(VERSION)" + @$(PY) -c "import re,pathlib; p=pathlib.Path('pyproject.toml'); s=p.read_text(); s,n=re.subn(r'(?m)^version = \".*\"','version = \"$(VERSION)\"',s,count=1); assert n==1,'version line not found'; p.write_text(s)" + @echo ">> running tests" + @$(PY) -m pytest -q + @echo ">> committing + tagging v$(VERSION)" + @git add pyproject.toml + @git commit -m "Release v$(VERSION)" + @git tag -a "v$(VERSION)" -m "firstops v$(VERSION)" + @echo ">> pushing branch + tag (triggers the PyPI publish workflow)" + @git push + @git push origin "v$(VERSION)" + @echo ">> released v$(VERSION). Watch the publish run:" + @echo " gh run watch \$$(gh run list --workflow=publish.yml -L1 --json databaseId -q '.[0].databaseId') --exit-status" From a3a48e48e457f4337640ad8b4a3e61fabae9add0 Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 13 Jun 2026 10:47:25 -0700 Subject: [PATCH 5/9] Add CODEOWNERS for release machinery Co-Authored-By: Claude Opus 4.8 --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..723a193 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Ownership of the release machinery. Changes to these paths should be +# reviewed by a maintainer (auto-requests review on PRs touching them). +/.github/ @anshal21 +/pyproject.toml @anshal21 +/Makefile @anshal21 From 71a052bea37e94fa5cb4a2aa586f83bd38cb2124 Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 13 Jun 2026 10:53:07 -0700 Subject: [PATCH 6/9] README: clearer install instructions + examples section Co-Authored-By: Claude Opus 4.8 --- README.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b409ae2..1b1e082 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,17 @@ Govern what your AI agents do. FirstOps applies identity, policy enforcement, credential brokering, and audit to every **LLM call**, **tool call**, and **MCP call** your agent makes — across LangGraph, the Claude Agent SDK, and the OpenAI Agents SDK, or any custom loop. +## Install + ```bash -pip install "firstops[langgraph]" # or [claude], [openai], [all] +pip install "firstops[langgraph]" # LangGraph + LangChain +pip install "firstops[claude]" # Claude Agent SDK +pip install "firstops[openai]" # OpenAI Agents SDK +pip install "firstops[all]" # all of the above +pip install firstops # core only (management client / custom loops) ``` -- **Python 3.10+** -- Core deps: `cryptography`, `httpx`. Your agent framework comes in via the extra you pick. +Python 3.10+. Core deps are just `cryptography` and `httpx` — your agent framework comes in via the extra you pick. --- @@ -88,6 +93,16 @@ mcp = MultiServerMCPClient({"notion": {"url": firstops.mcp_url("" tools = await mcp.get_tools() ``` +## Examples + +Runnable agents in [`examples/`](examples/) — each governs the LLM and tool calls; the `*_mcp` variants add a Notion MCP server: + +- LangGraph — [`langgraph_basic.py`](examples/langgraph_basic.py), [`langgraph_notion_mcp.py`](examples/langgraph_notion_mcp.py) +- Claude Agent SDK — [`claude_sdk_basic.py`](examples/claude_sdk_basic.py), [`claude_sdk_mcp.py`](examples/claude_sdk_mcp.py) +- OpenAI Agents SDK — [`openai_agents_basic.py`](examples/openai_agents_basic.py), [`openai_agents_mcp.py`](examples/openai_agents_mcp.py) + +See [`examples/README.md`](examples/README.md) for the env vars to run them. + ## Management client Provision agents and connections from your backend: From 1b61801d8e7c949d959495eb7ad2b5da4be08c55 Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sat, 13 Jun 2026 10:58:50 -0700 Subject: [PATCH 7/9] README: link docs guides + concepts Co-Authored-By: Claude Opus 4.8 --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1b1e082..1464253 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,10 @@ admin.connections.register(principal_id=agent.id, name="slack", upstream_url="ht ## Documentation -- Guides: -- Repository: +- [Docs home](https://firstops.dev/docs) +- Guides: [LangChain / LangGraph](https://firstops.dev/docs/guides/langchain) · [Claude Agent SDK](https://firstops.dev/docs/guides/claude-sdk) · [OpenAI Agents SDK](https://firstops.dev/docs/guides/openai-agents) +- Concepts: [Identity](https://firstops.dev/docs/concepts/identity) · [Enforcement](https://firstops.dev/docs/concepts/enforcement) · [Connections](https://firstops.dev/docs/concepts/connections) +- [Repository](https://github.com/firstops-dev/firstops-python) ## License From d257e805c6fa866d2ad0f4aac7b30d498eac4289 Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sun, 14 Jun 2026 09:37:19 -0700 Subject: [PATCH 8/9] Add Google ADK adapter + sidecar HTTP/1.1 framing fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - google_adk.firstops_before_tool_callback: one agent-level callback governs every tool call — block (non-None return) + scrub (rewrite args in place). - _common: HARNESS_GOOGLE_ADK, stamped into audit metadata. - proxy: HTTP/1.1 + Content-Length framing, strip server/date/transfer-encoding headers, close-on-stream — fixes litellm/aiohttp 'Duplicate Server header' rejection so ADK's LiteLLM transport accepts the chain-link. - adk extra, runnable example, 4 adapter tests, README + description. Co-Authored-By: Claude Opus 4.8 --- README.md | 16 +++++- examples/google_adk_basic.py | 73 +++++++++++++++++++++++++ pyproject.toml | 8 ++- src/firstops/integrations/_common.py | 1 + src/firstops/integrations/google_adk.py | 55 +++++++++++++++++++ src/firstops/proxy.py | 58 ++++++++++++++++---- tests/test_integrations.py | 47 ++++++++++++++++ 7 files changed, 243 insertions(+), 15 deletions(-) create mode 100644 examples/google_adk_basic.py create mode 100644 src/firstops/integrations/google_adk.py diff --git a/README.md b/README.md index 1464253..e8a63b1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FirstOps Python SDK -Govern what your AI agents do. FirstOps applies identity, policy enforcement, credential brokering, and audit to every **LLM call**, **tool call**, and **MCP call** your agent makes — across LangGraph, the Claude Agent SDK, and the OpenAI Agents SDK, or any custom loop. +Govern what your AI agents do. FirstOps applies identity, policy enforcement, credential brokering, and audit to every **LLM call**, **tool call**, and **MCP call** your agent makes — across LangGraph, the Claude Agent SDK, the OpenAI Agents SDK, and Google ADK, or any custom loop. ## Install @@ -8,6 +8,7 @@ Govern what your AI agents do. FirstOps applies identity, policy enforcement, cr pip install "firstops[langgraph]" # LangGraph + LangChain pip install "firstops[claude]" # Claude Agent SDK pip install "firstops[openai]" # OpenAI Agents SDK +pip install "firstops[adk]" # Google ADK pip install "firstops[all]" # all of the above pip install firstops # core only (management client / custom loops) ``` @@ -75,6 +76,16 @@ guard = firstops_tool_input_guardrail(fo) def send_email(to: str, body: str) -> str: ... ``` +**Google ADK** — one `before_tool_callback` governs every tool (block + rewrite args): + +```python +from google.adk.agents import LlmAgent +from firstops.integrations.google_adk import firstops_before_tool_callback + +agent = LlmAgent(name="assistant", model=..., tools=[...], + before_tool_callback=firstops_before_tool_callback(fo)) +``` + **Any framework / custom loop** — the base API: ```python @@ -100,6 +111,7 @@ Runnable agents in [`examples/`](examples/) — each governs the LLM and tool ca - LangGraph — [`langgraph_basic.py`](examples/langgraph_basic.py), [`langgraph_notion_mcp.py`](examples/langgraph_notion_mcp.py) - Claude Agent SDK — [`claude_sdk_basic.py`](examples/claude_sdk_basic.py), [`claude_sdk_mcp.py`](examples/claude_sdk_mcp.py) - OpenAI Agents SDK — [`openai_agents_basic.py`](examples/openai_agents_basic.py), [`openai_agents_mcp.py`](examples/openai_agents_mcp.py) +- Google ADK — [`google_adk_basic.py`](examples/google_adk_basic.py) See [`examples/README.md`](examples/README.md) for the env vars to run them. @@ -122,7 +134,7 @@ admin.connections.register(principal_id=agent.id, name="slack", upstream_url="ht ## Documentation - [Docs home](https://firstops.dev/docs) -- Guides: [LangChain / LangGraph](https://firstops.dev/docs/guides/langchain) · [Claude Agent SDK](https://firstops.dev/docs/guides/claude-sdk) · [OpenAI Agents SDK](https://firstops.dev/docs/guides/openai-agents) +- Guides: [LangChain / LangGraph](https://firstops.dev/docs/guides/langchain) · [Claude Agent SDK](https://firstops.dev/docs/guides/claude-sdk) · [OpenAI Agents SDK](https://firstops.dev/docs/guides/openai-agents) · [Google ADK](https://firstops.dev/docs/guides/google-adk) - Concepts: [Identity](https://firstops.dev/docs/concepts/identity) · [Enforcement](https://firstops.dev/docs/concepts/enforcement) · [Connections](https://firstops.dev/docs/concepts/connections) - [Repository](https://github.com/firstops-dev/firstops-python) diff --git a/examples/google_adk_basic.py b/examples/google_adk_basic.py new file mode 100644 index 0000000..b7158a4 --- /dev/null +++ b/examples/google_adk_basic.py @@ -0,0 +1,73 @@ +"""Google ADK agent governed by FirstOps. + +One `before_tool_callback` governs every tool call (block + rewrite args). The +LLM runs through the sidecar chain-link via LiteLLM pointed at OpenAI. + +Run with the env vars in README.md (needs OPENAI_API_KEY). +""" + +import asyncio +import os + +import firstops +from firstops.integrations.google_adk import firstops_before_tool_callback +from google.adk.agents import LlmAgent +from google.adk.models.lite_llm import LiteLlm +from google.adk.runners import InMemoryRunner +from google.genai import types + +from _shared import load_config, trace + +APP = "firstops-demo" + + +def get_weather(city: str) -> dict: + """Get the current weather for a city.""" + print(f" [TOOL get_weather] city={city}") + return {"weather": f"21C and sunny in {city}"} + + +def send_email(to: str, body: str) -> dict: + """Send an email to a recipient.""" + print(f" [TOOL send_email] to={to} body={body!r}") + return {"status": "sent"} + + +async def main(): + cfg = load_config() + fo = firstops.init( + cfg["agent_id"], cfg["key_pem"], gateway_url=cfg["gateway"], port=cfg["port"] + ) + trace(fo) + try: + agent = LlmAgent( + name="assistant", + model=LiteLlm( + model="openai/gpt-4o-mini", + api_base=firstops.llm_base_url("openai"), # -> sidecar chain-link + api_key=os.environ["OPENAI_API_KEY"], + ), + instruction="You are a helpful assistant.", + tools=[get_weather, send_email], + before_tool_callback=firstops_before_tool_callback(fo), # governs tools + ) + runner = InMemoryRunner(agent=agent, app_name=APP) + session = await runner.session_service.create_session(app_name=APP, user_id="u1") + msg = types.Content( + role="user", + parts=[types.Part(text="Check the weather in Paris, then email it to alice@example.com.")], + ) + print("\n>>> running Google ADK agent\n") + async for event in runner.run_async( + user_id="u1", session_id=session.id, new_message=msg + ): + if event.content and event.content.parts: + for part in event.content.parts: + if getattr(part, "text", None) and part.text.strip(): + print(f" [ADK] {part.text.strip()[:160]}") + finally: + firstops.shutdown() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 01276d1..1bd216c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "firstops" version = "0.2.0" -description = "Govern MCP, tool calls, and LLM traffic for AI agents — across LangGraph, Claude Agent SDK, and OpenAI Agents." +description = "Govern MCP, tool calls, and LLM traffic for AI agents — across LangGraph, Claude Agent SDK, OpenAI Agents, and Google ADK." readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -26,7 +26,7 @@ classifiers = [ ] keywords = [ "mcp", "dpop", "agent", "security", "governance", "llm", - "langgraph", "langchain", "openai-agents", "claude", "guardrails", + "langgraph", "langchain", "openai-agents", "claude", "google-adk", "guardrails", ] dependencies = [ "cryptography>=42.0", @@ -54,6 +54,9 @@ claude = [ openai = [ "openai-agents", ] +adk = [ + "google-adk[extensions]", +] all = [ "langchain>=1.0", "langgraph", @@ -61,6 +64,7 @@ all = [ "langchain-mcp-adapters", "claude-agent-sdk", "openai-agents", + "google-adk[extensions]", ] dev = [ "pytest>=8.0", diff --git a/src/firstops/integrations/_common.py b/src/firstops/integrations/_common.py index 4f8ab99..6b373e1 100644 --- a/src/firstops/integrations/_common.py +++ b/src/firstops/integrations/_common.py @@ -22,6 +22,7 @@ HARNESS_LANGGRAPH = "langgraph" HARNESS_CLAUDE = "claude-agent-sdk" HARNESS_OPENAI_AGENTS = "openai-agents" +HARNESS_GOOGLE_ADK = "google-adk" def _json_safe(value: Any) -> Any: diff --git a/src/firstops/integrations/google_adk.py b/src/firstops/integrations/google_adk.py new file mode 100644 index 0000000..383f605 --- /dev/null +++ b/src/firstops/integrations/google_adk.py @@ -0,0 +1,55 @@ +"""Google ADK adapter — `before_tool_callback` governs every tool call. + +ADK's agent-level `before_tool_callback` can both **block** a tool (return a +result, which short-circuits the call) and **rewrite its args** (mutate the args +dict in place) — so FirstOps gets block and scrub on Google ADK. + +Usage:: + + from google.adk.agents import LlmAgent + from firstops.integrations.google_adk import firstops_before_tool_callback + + agent = LlmAgent( + name="assistant", + model=..., + tools=[...], + before_tool_callback=firstops_before_tool_callback(fo), + ) + +The callback is a plain function (ADK invokes it as +``callback(tool=, args=, tool_context=)``), so this adapter needs no ADK import. +""" + +from __future__ import annotations + +from typing import Any + +from firstops import _runtime +from firstops.integrations._common import ( + ACTION_DENY, + ACTION_MODIFY, + HARNESS_GOOGLE_ADK, + decide, +) + + +def firstops_before_tool_callback(fo=None): + """Return a ``before_tool_callback`` that governs every tool call.""" + rt = fo if fo is not None else _runtime.runtime() + + def _before_tool(tool, args, tool_context) -> dict[str, Any] | None: + tool_name = getattr(tool, "name", "") or "" + action, payload = decide( + rt, tool_name, args, can_apply_modify=True, harness=HARNESS_GOOGLE_ADK + ) + if action == ACTION_DENY: + # A non-None return short-circuits the tool; this becomes the result + # the model sees. + return {"status": "denied", "error": f"blocked by FirstOps policy: {payload}"} + if action == ACTION_MODIFY and isinstance(payload, dict) and isinstance(args, dict): + # Rewrite the call's args in place (full replacement with scrubbed input). + args.clear() + args.update(payload) + return None + + return _before_tool diff --git a/src/firstops/proxy.py b/src/firstops/proxy.py index 3077f8a..e1da990 100644 --- a/src/firstops/proxy.py +++ b/src/firstops/proxy.py @@ -44,6 +44,14 @@ } ) +# Response headers we must not forward back to the client: hop-by-hop, plus the +# ones BaseHTTPRequestHandler sets itself (Server/Date) — forwarding the +# upstream's copies duplicates them, which strict clients (aiohttp/litellm) +# reject with "Duplicate 'Server' header". +_RESP_STRIP_HEADERS = frozenset( + {"transfer-encoding", "connection", "server", "date"} +) + # Hard cap on a forwarded request body (defensive against a huge Content-Length). _MAX_BODY_BYTES = 100 * 1024 * 1024 @@ -210,6 +218,12 @@ def _make_handler(identity: Identity, local_port: int, enforcement, llm_upstream client = httpx.Client(timeout=httpx.Timeout(120.0, connect=10.0)) class ProxyHandler(BaseHTTPRequestHandler): + # HTTP/1.1 so strict clients (litellm/aiohttp, Node MCP) get clean + # keep-alive framing. Non-streaming responses carry an exact + # Content-Length (see _forward); streaming responses close the + # connection explicitly. + protocol_version = "HTTP/1.1" + def do_POST(self): self._dispatch("POST") @@ -264,6 +278,7 @@ def _handle_llm(self, method: str): if denial is not None: self.send_response(403) self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(denial))) self.end_headers() self.wfile.write(denial) return @@ -364,20 +379,39 @@ def _forward(self, method: str, url: str, headers: dict, body: bytes | None): method, url, headers=headers, content=body, timeout=httpx.Timeout(120.0, connect=10.0), ) as resp: + is_stream = "text/event-stream" in resp.headers.get("content-type", "") + + if is_stream: + # Streaming (SSE): pass bytes through raw with flushing so + # events arrive live. No Content-Length, so close the + # connection to delimit the body under HTTP/1.1. + self.close_connection = True + self.send_response(resp.status_code) + for k, v in resp.headers.items(): + if k.lower() not in _RESP_STRIP_HEADERS: + self.send_header(k, v) + self.send_header("Connection", "close") + self.end_headers() + for chunk in resp.iter_raw(): + self.wfile.write(chunk) + self.wfile.flush() + return + + # Non-streaming: buffer the DECODED body and send it with an + # exact Content-Length, dropping Content-Encoding/Length from + # upstream. This gives a cleanly-framed response that ANY + # client reads correctly (httpx, aiohttp, Node) — a + # close-delimited gzipped body trips stricter clients. + payload = resp.read() # httpx auto-decompresses self.send_response(resp.status_code) for k, v in resp.headers.items(): - if k.lower() not in ("transfer-encoding", "connection"): + if k.lower() not in _RESP_STRIP_HEADERS and k.lower() not in ( + "content-encoding", "content-length", + ): self.send_header(k, v) + self.send_header("Content-Length", str(len(payload))) self.end_headers() - - # Forward the body RAW: httpx auto-decompresses iter_bytes()/ - # read(), but we keep the upstream Content-Encoding/Length - # headers, so we must pass the original (possibly gzipped) - # bytes through untouched or the client's decode fails. - # Flush per chunk so SSE/streaming responses arrive live. - for chunk in resp.iter_raw(): - self.wfile.write(chunk) - self.wfile.flush() + self.wfile.write(payload) except httpx.HTTPError as e: logger.error("upstream request failed: %s", e) self.send_error(502, "upstream request failed") @@ -386,10 +420,12 @@ def _stream_sse(self, url: str, headers: dict): """Stream an SSE response, rewriting gateway URLs to localhost.""" try: with httpx.stream("GET", url, headers=headers, timeout=None) as resp: + self.close_connection = True # SSE: no length, close-delimited self.send_response(resp.status_code) for k, v in resp.headers.items(): - if k.lower() not in ("transfer-encoding", "connection"): + if k.lower() not in _RESP_STRIP_HEADERS: self.send_header(k, v) + self.send_header("Connection", "close") self.end_headers() gateway_msg_url = gateway + "/mcp/sse/message" diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 689eb62..d6a5799 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -181,3 +181,50 @@ def test_openai_guardrail_requires_sdk(): def test_langgraph_middleware_requires_langchain(): with pytest.raises(RuntimeError, match="langchain"): langgraph.FirstOpsMiddleware(_rt(Decision(action="allow"))) + + +# ---- Google ADK adapter ---------------------------------------------------- + +from firstops.integrations import google_adk + + +class _AdkTool: + def __init__(self, name): + self.name = name + + +def test_adk_allow_returns_none_and_stamps_harness(): + rt = _rt(Decision(action="allow")) + cb = google_adk.firstops_before_tool_callback(rt) + args = {"city": "Paris"} + assert cb(tool=_AdkTool("get_weather"), args=args, tool_context=None) is None + assert args == {"city": "Paris"} # unchanged + ev = rt.enforcement.events[0] + assert ev.tool_name == "get_weather" + assert ev.metadata == {"harness": "google-adk"} + + +def test_adk_deny_short_circuits_with_result_dict(): + rt = _rt(Decision(action="deny", reason="destructive")) + out = google_adk.firstops_before_tool_callback(rt)( + tool=_AdkTool("run_shell"), args={"cmd": "x"}, tool_context=None + ) + assert out is not None # non-None return blocks the tool + assert out["status"] == "denied" + assert "destructive" in out["error"] + + +def test_adk_modify_rewrites_args_in_place(): + rt = _rt(_modify({"to": "[REDACTED]"})) + args = {"to": "secret@example.com"} + out = google_adk.firstops_before_tool_callback(rt)( + tool=_AdkTool("send_email"), args=args, tool_context=None + ) + assert out is None # proceed + assert args == {"to": "[REDACTED]"} # rewritten in place + + +def test_adk_adapter_needs_no_framework(): + # The callback is a plain function — constructing it must not require ADK. + cb = google_adk.firstops_before_tool_callback(_rt(Decision(action="allow"))) + assert callable(cb) From 4f25f8d6e1abff08c80ac51f5010edc73ede960b Mon Sep 17 00:00:00 2001 From: anshal21 Date: Sun, 14 Jun 2026 18:26:18 -0700 Subject: [PATCH 9/9] Release v0.3.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1bd216c..c23ba95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "firstops" -version = "0.2.0" +version = "0.3.0" description = "Govern MCP, tool calls, and LLM traffic for AI agents — across LangGraph, Claude Agent SDK, OpenAI Agents, and Google ADK." readme = "README.md" requires-python = ">=3.10"