diff --git a/.changeset/heavy-planets-tell.md b/.changeset/heavy-planets-tell.md new file mode 100644 index 000000000..a1b05063b --- /dev/null +++ b/.changeset/heavy-planets-tell.md @@ -0,0 +1,9 @@ +--- +"@executor-js/analytics": patch +"@executor-js/sdk": patch +"@executor-js/api": patch +"@executor-js/host-selfhost": patch +"executor": patch +--- + +Add anonymous product analytics to the local daemon (CLI + desktop) and self-host: execution counts split by MCP/API plane, toolkit usage, integration add/remove, and artifact usage (created/viewed/updated/deleted, attributed to agent tools vs the console UI), filed under a persisted per-install anonymous id. Opt out with DO_NOT_TRACK or EXECUTOR_DISABLE_ANALYTICS. diff --git a/TELEMETRY.md b/TELEMETRY.md new file mode 100644 index 000000000..f84600c06 --- /dev/null +++ b/TELEMETRY.md @@ -0,0 +1,101 @@ +# Telemetry + +Executor's local products (the CLI, the desktop app) and self-hosted +deployments send a small set of anonymous usage events. This document is the +complete account of what is sent, what is deliberately not sent, why the +feature exists, and how to turn it off. + +The hosted cloud product has its own browser-side analytics, disclosed +separately in its terms; this document covers the software that runs on your +machines. + +## Why + +Executor is used mostly outside our infrastructure. Without some signal from +local and self-hosted installs, every product decision about them is a guess: +we cannot tell whether a feature ships broken, whether anyone uses toolkits, +whether executions fail at unusual rates after a release, or whether the +product is growing anywhere except cloud. + +The events exist to answer exactly one kind of question: **how are product +features being used?** They are metadata about the product, not about you. +Anything that would answer "what is this user doing" — which APIs you call, +what your tools are named, what your code does — is out of scope by design, +not by omission. + +## What is sent + +Each event carries its named properties plus, on every event: a random +per-install id, the product surface (`cli`, `desktop`, or `selfhost`), the +release channel, and the app version. + +| Event | Properties | Meaning | +| --------------------- | ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `execution_completed` | `ok`, `plane` (`mcp`/`api`), `toolkit` (boolean) | A code execution finished, split by whether an agent (MCP) or a human-facing API triggered it, and whether a toolkit-scoped endpoint served it. | +| `integration_added` | `plugin_key` | An integration was added, by kind (`openapi`, `mcp`, `graphql`, ...). | +| `integration_removed` | `plugin_key` | An integration was removed, by kind. | +| `artifact_created` | `via` (`agent`/`ui`) | A generative-UI artifact was saved. | +| `artifact_viewed` | `via` | An artifact was opened for its content. | +| `artifact_updated` | `via` | An artifact was overwritten or renamed. | +| `artifact_deleted` | `via` | An artifact was deleted. | + +That table is exhaustive. The typed catalog the code compiles against is +[`packages/core/analytics/src/events.ts`](packages/core/analytics/src/events.ts); +an event that is not in that file cannot be sent. + +## What is never sent + +- No code, tool arguments, tool results, or error messages. +- No secrets, tokens, or credentials. +- No names you typed: no integration slugs, connection names, toolkit slugs, + tool names, or artifact titles. The **kind** of integration (`openapi`, + `mcp`, ...) is a product question; **which** service you connected it to is + your business. +- No identity: no emails, usernames, hostnames, IP-derived location, or org + names. Events are marked so the analytics backend builds no person profile. + +## The anonymous id + +A random UUID is minted the first time the daemon or server starts and stored +as `analytics-id` in the data directory (`~/.executor` for local installs, the +configured data dir for self-host). It exists so that ten events from one +install count as one install, not ten. It is not derived from your machine, +account, or network, and deleting the file resets it. When telemetry is +disabled the file is never created. + +## Opting out + +Set either environment variable to `1`, `true`, or `yes`: + +- `DO_NOT_TRACK` — the [cross-tool convention](https://consoledonottrack.com), + which Executor also honors for its other outbound calls (crash reporting, + the integrations.sh catalog fetch). +- `EXECUTOR_DISABLE_ANALYTICS` — telemetry only, if you want the catalog + fetch and crash reporting to keep working. + +Opting out is total: the analytics service becomes a no-op, nothing is +buffered, nothing is sent, and no id file is written. The CLI's managed +service (launchd/systemd) forwards both variables into the supervised +daemon's environment, so an opted-out install stays opted out. + +## Delivery mechanics + +Events buffer in memory (bounded) and flush in batches on a fixed cadence, +plus once at shutdown. Delivery is best-effort: a failed flush re-queues and +retries later, failures are swallowed, and no user-facing operation ever +waits on — or can be failed by — analytics. The ingest endpoint is PostHog +(`us.i.posthog.com`); the project key in the source identifies the project +and grants no read access. + +## Where the line is enforced + +Structurally, not by convention: + +- The event catalog is a closed, typed interface — adding a property means + editing [`events.ts`](packages/core/analytics/src/events.ts) in a reviewed + change, next to the property rules at the top of that file. +- Attribution (`plane`, `via`) is bound where each serving surface is + composed, so events cannot carry request-controlled labels. +- Observers hang off neutral seams (an engine wrapper, post-commit hooks); the + core SDK carries no analytics vocabulary and hosts that do not opt in — like + every test — compose with no analytics at all. diff --git a/apps/cli/src/service.ts b/apps/cli/src/service.ts index 2bb8c87ae..ade5757fd 100644 --- a/apps/cli/src/service.ts +++ b/apps/cli/src/service.ts @@ -177,6 +177,10 @@ const serviceEnvironment = ( "EXECUTOR_SENTRY_RELEASE", "EXECUTOR_SENTRY_ENVIRONMENT", "EXECUTOR_RUN_ID", + // Analytics opt-out must survive into the supervised unit's minimal env, + // or an opted-out install would silently re-enable analytics under launchd. + "DO_NOT_TRACK", + "EXECUTOR_DISABLE_ANALYTICS", ] as const; const passThrough = Object.fromEntries( passThroughKeys.flatMap((key) => { diff --git a/apps/host-selfhost/package.json b/apps/host-selfhost/package.json index f242a6cc3..0ca7fd912 100644 --- a/apps/host-selfhost/package.json +++ b/apps/host-selfhost/package.json @@ -22,6 +22,7 @@ "@cloudflare/worker-bundler": "0.2.1", "@effect/atom-react": "catalog:", "@effect/platform-bun": "catalog:", + "@executor-js/analytics": "workspace:*", "@executor-js/api": "workspace:*", "@executor-js/app": "workspace:*", "@executor-js/execution": "workspace:*", diff --git a/apps/host-selfhost/src/analytics.ts b/apps/host-selfhost/src/analytics.ts new file mode 100644 index 000000000..7991b97b0 --- /dev/null +++ b/apps/host-selfhost/src/analytics.ts @@ -0,0 +1,83 @@ +import { Effect, Layer, ManagedRuntime } from "effect"; + +import { + Analytics, + defaultLayer as analyticsDefaultLayer, + withExecutionAnalytics, + type AnalyticsService, +} from "@executor-js/analytics"; +import { EngineDecorator } from "@executor-js/api/server"; + +import packageJson from "../package.json" with { type: "json" }; +import { resolveDataDir } from "./config"; + +// --------------------------------------------------------------------------- +// Self-host product analytics: one anonymous per-install service for the +// server lifetime. The anonymous id persists next to the instance's other +// first-boot state (secret.key) under the data dir, so an instance counts +// once across restarts. Module-singleton runtime; dispose flushes at shutdown. +// --------------------------------------------------------------------------- + +// Deployed self-host builds are published releases; the prerelease heuristic +// mirrors apps/local/src/installation.ts. +const VERSION: string = packageJson.version; +const CHANNEL = VERSION.includes("-") ? ("beta" as const) : ("stable" as const); + +let analyticsRuntime: ManagedRuntime.ManagedRuntime | null = null; + +const getAnalyticsRuntime = (): ManagedRuntime.ManagedRuntime => { + if (analyticsRuntime) return analyticsRuntime; + analyticsRuntime = ManagedRuntime.make( + analyticsDefaultLayer({ + surface: "selfhost", + version: VERSION, + channel: CHANNEL, + dataDir: resolveDataDir(), + }), + ); + return analyticsRuntime; +}; + +/** + * Lazy facade over the instance's shared service. `record` forks into the + * module runtime (fire-and-forget) so no request path ever waits on the + * analytics layer; ordering within the runtime preserves event order. + */ +export const selfHostAnalytics: AnalyticsService = { + record: (name, properties) => + Effect.sync(() => { + getAnalyticsRuntime().runFork( + Effect.flatMap(Analytics.asEffect(), (analytics) => analytics.record(name, properties)), + ); + }), + flush: Effect.promise(() => + getAnalyticsRuntime().runPromise( + Effect.flatMap(Analytics.asEffect(), (analytics) => analytics.flush), + ), + ), +}; + +/** Flush and dispose the analytics runtime (server shutdown). */ +export const disposeAnalytics = async (): Promise => { + const runtime = analyticsRuntime; + if (!runtime) return; + analyticsRuntime = null; + await Effect.runPromise(Effect.promise(() => runtime.dispose()).pipe(Effect.ignoreCause())); +}; + +/** + * The execution-analytics `EngineDecorator`: every engine the shared stack + * builds gets wrapped once, with the plane derived from what the stack was + * built to serve — an MCP session carries its `mcpResource`, the HTTP + * executions plane has none. The wrap point IS the plane, so misattribution + * is structurally impossible. + */ +export const SelfHostAnalyticsEngineDecorator: Layer.Layer = Layer.succeed( + EngineDecorator, +)({ + decorate: (engine, _identity, context) => + withExecutionAnalytics(engine, selfHostAnalytics, { + plane: context.mcpResource === undefined ? "api" : "mcp", + toolkit: context.mcpResource?.kind === "toolkit", + }), +}); diff --git a/apps/host-selfhost/src/app.ts b/apps/host-selfhost/src/app.ts index 594d6a69b..4eaf631f5 100644 --- a/apps/host-selfhost/src/app.ts +++ b/apps/host-selfhost/src/app.ts @@ -2,7 +2,12 @@ import { HttpApiSwagger } from "effect/unstable/httpapi"; import { HttpEffect, HttpRouter } from "effect/unstable/http"; import { Effect, Layer } from "effect"; -import { composePluginApi, ExecutorApp, textFailureStrategy } from "@executor-js/api/server"; +import { + ArtifactUsageObserver, + composePluginApi, + ExecutorApp, + textFailureStrategy, +} from "@executor-js/api/server"; import { runSqliteDataMigrations } from "@executor-js/sdk"; @@ -14,6 +19,7 @@ import { makeSelfHostSystemApiLayer } from "./system/handlers"; import { selfHostAccountMiddleware } from "./account"; import { loadConfig, SELF_HOST_NAMESPACE, SELF_HOST_SCHEMA_VERSION } from "./config"; import { createSelfHostDb, SelfHostDb, SelfHostDbProvider } from "./db/self-host-db"; +import { selfHostAnalytics, SelfHostAnalyticsEngineDecorator } from "./analytics"; import { SelfHostCodeExecutorProvider, SelfHostHostConfig, @@ -98,7 +104,12 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { identity: identityLayer, account: selfHostAccountMiddleware(betterAuth), db: SelfHostDbProvider, - engine: { codeExecutor: SelfHostCodeExecutorProvider }, // decorator defaults to no-op (no metering) + engine: { + codeExecutor: SelfHostCodeExecutorProvider, + // Anonymous execution analytics (this seam is the HTTP plane; the MCP + // plane's decorator is wired in mcp/session-store.ts's stack layer). + decorator: SelfHostAnalyticsEngineDecorator, + }, mcp: { auth: mcp.auth, sessions: mcp.sessions, reporter: mcp.reporter }, plugins: { provider: SelfHostPluginsProvider, config: SelfHostHostConfig }, errorCapture: ErrorCaptureLive, @@ -131,8 +142,16 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { config: { mountPrefix: "/api", failure: textFailureStrategy }, // The boot-scoped context provideMerge'd under everything: the long-lived DB // handle (read by the DbProvider seam, Better Auth, and the MCP store) + the - // resolved identity (captured once by the execution middleware + MCP auth). - boot: Layer.merge(Layer.succeed(SelfHostDb)(dbHandle), identityLayer), + // resolved identity (captured once by the execution middleware + MCP auth) + // + the artifact-usage observer (this HTTP plane is the console UI's data + // layer, so operations it serves file as `via: "ui"`). + boot: Layer.mergeAll( + Layer.succeed(SelfHostDb)(dbHandle), + identityLayer, + Layer.succeed(ArtifactUsageObserver)((action) => + selfHostAnalytics.record(`artifact_${action}`, { via: "ui" }), + ), + ), }); return { diff --git a/apps/host-selfhost/src/execution.ts b/apps/host-selfhost/src/execution.ts index c7ffbbf23..270ffc4f8 100644 --- a/apps/host-selfhost/src/execution.ts +++ b/apps/host-selfhost/src/execution.ts @@ -4,13 +4,13 @@ import { CodeExecutorProvider, DbProvider, EngineDecorator, - EngineDecoratorNoop, HostConfig, PluginsProvider, } from "@executor-js/api/server"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; import executorConfig from "../executor.config"; +import { selfHostAnalytics, SelfHostAnalyticsEngineDecorator } from "./analytics"; import { SelfHostDb, SelfHostDbProvider } from "./db/self-host-db"; import { loadConfig } from "./config"; @@ -21,7 +21,7 @@ import { loadConfig } from "./config"; // makeScopedExecutor -> createExecutionEngine -> EngineDecorator.decorate. // Self-host just supplies the five seam Layers it reads from. Differences from // cloud: the QuickJS in-process code substrate (vs the Cloudflare dynamic -// worker) and a NO-OP engine decorator (no usage metering). +// worker) and an analytics engine decorator instead of cloud's usage metering. // // - DbProvider -> SelfHostDbProvider: projects the long-lived // libSQL handle (built once at boot, see db/). The @@ -33,7 +33,8 @@ import { loadConfig } from "./config"; // - HostConfig -> `{ allowLocalNetwork, webBaseUrl }` from // `loadConfig()`. // - CodeExecutorProvider -> `makeQuickJsExecutor()`. -// - EngineDecorator -> no-op (self-host does not meter executions). +// - EngineDecorator -> execution analytics (anonymous per-install +// counters; see ./analytics.ts). // --------------------------------------------------------------------------- export { makeExecutionStack } from "@executor-js/api/server"; @@ -54,6 +55,11 @@ export const SelfHostHostConfig: Layer.Layer = Layer.sync(HostConfig allowLocalNetwork: config.allowLocalNetwork, webBaseUrl: config.webBaseUrl, oauthCallbackPath: "/api/oauth/callback", + onIntegrationChange: (event) => + selfHostAnalytics.record( + event.kind === "added" ? "integration_added" : "integration_removed", + { plugin_key: event.pluginKey }, + ), }; }); @@ -82,4 +88,8 @@ export const SelfHostExecutionStackLayer: Layer.Layer< DbProvider | PluginsProvider | HostConfig | CodeExecutorProvider | EngineDecorator, never, SelfHostDb -> = Layer.mergeAll(SelfHostScopedExecutorSeams, SelfHostCodeExecutorProvider, EngineDecoratorNoop); +> = Layer.mergeAll( + SelfHostScopedExecutorSeams, + SelfHostCodeExecutorProvider, + SelfHostAnalyticsEngineDecorator, +); diff --git a/apps/host-selfhost/src/mcp/session-store.ts b/apps/host-selfhost/src/mcp/session-store.ts index 0f2233be9..c17a8fa4d 100644 --- a/apps/host-selfhost/src/mcp/session-store.ts +++ b/apps/host-selfhost/src/mcp/session-store.ts @@ -8,6 +8,7 @@ import { type InMemoryMcpSessionStore, } from "@executor-js/host-mcp/in-memory-session-store"; +import { selfHostAnalytics } from "../analytics"; import { ErrorCaptureLive } from "../observability"; import { SelfHostDb, type SelfHostDbHandle } from "../db/self-host-db"; import { SelfHostExecutionStackLayer } from "../execution"; @@ -39,7 +40,13 @@ export const makeSelfHostMcpSessionStore = ( makeInMemoryMcpSessionStore( makeMcpBuildServer( SelfHostExecutionStackLayer.pipe(Layer.provide(Layer.succeed(SelfHostDb)(db))), - { loadAppShellHtml: loadMcpAppsShellHtml, smokeRenderArtifact }, + { + loadAppShellHtml: loadMcpAppsShellHtml, + smokeRenderArtifact, + // Artifact operations on the MCP plane come from an agent's tools. + onArtifactUsage: (action) => + selfHostAnalytics.record(`artifact_${action}`, { via: "agent" }), + }, ), { webBaseUrl }, ); diff --git a/apps/host-selfhost/src/serve.ts b/apps/host-selfhost/src/serve.ts index 1803e7bb0..deca78b11 100644 --- a/apps/host-selfhost/src/serve.ts +++ b/apps/host-selfhost/src/serve.ts @@ -26,6 +26,7 @@ import { import { BunFileSystem, BunHttpServer, BunPath, BunRuntime } from "@effect/platform-bun"; import { Effect, Layer } from "effect"; +import { disposeAnalytics } from "./analytics"; import { makeSelfHostApp } from "./app"; import { loadConfig } from "./config"; import type { BetterAuthHandle } from "./auth"; @@ -120,6 +121,11 @@ export const startServer = async (): Promise => { cacheControl: "no-cache", }).pipe(Layer.provide(BunFileSystem.layer), Layer.provide(BunPath.layer)); + // Server-scope finalizer: flush buffered analytics on graceful shutdown. + const AnalyticsFlushLive = Layer.effectDiscard( + Effect.addFinalizer(() => Effect.promise(() => disposeAnalytics())), + ); + const ServerLive = HttpRouter.serve(Layer.mergeAll(AppLayer, AssetsLive, SpaLive), { middleware: selfHostHttpMiddleware(betterAuth), }).pipe( @@ -128,7 +134,7 @@ export const startServer = async (): Promise => { ), ); - await BunRuntime.runMain(Layer.launch(ServerLive)); + await BunRuntime.runMain(Layer.launch(Layer.merge(ServerLive, AnalyticsFlushLive))); }; if (import.meta.main) { diff --git a/apps/local/package.json b/apps/local/package.json index 929de78cb..9a2431749 100644 --- a/apps/local/package.json +++ b/apps/local/package.json @@ -22,6 +22,7 @@ "dependencies": { "@effect/atom-react": "catalog:", "@effect/platform-node": "catalog:", + "@executor-js/analytics": "workspace:*", "@executor-js/api": "workspace:*", "@executor-js/app": "workspace:*", "@executor-js/config": "workspace:*", diff --git a/apps/local/src/analytics.ts b/apps/local/src/analytics.ts new file mode 100644 index 000000000..3f98e2004 --- /dev/null +++ b/apps/local/src/analytics.ts @@ -0,0 +1,68 @@ +import { Effect, ManagedRuntime } from "effect"; + +import { + Analytics, + defaultLayer as analyticsDefaultLayer, + type AnalyticsService, + type AnalyticsSurface, +} from "@executor-js/analytics"; + +import { resolveExecutorDataDir } from "./auth"; +import { CHANNEL, VERSION } from "./installation"; + +// --------------------------------------------------------------------------- +// Local product analytics: one anonymous per-install service for the whole +// daemon lifetime, shared by every surface (HTTP API, in-process MCP, stdio). +// +// The surface is cli|desktop — never "local": the desktop main process marks +// its sidecar via EXECUTOR_CLIENT=desktop (same signal the sidecar sentinels +// key on), everything else hosting apps/local is the CLI. Module-singleton +// runtime mirroring ./integrations.ts, so concurrent surfaces share one +// buffer + one flush loop, and disposeAnalytics() is the shutdown flush. +// --------------------------------------------------------------------------- + +const resolveSurface = (): AnalyticsSurface => + process.env.EXECUTOR_CLIENT === "desktop" ? "desktop" : "cli"; + +let analyticsRuntime: ManagedRuntime.ManagedRuntime | null = null; + +const getAnalyticsRuntime = (): ManagedRuntime.ManagedRuntime => { + if (analyticsRuntime) return analyticsRuntime; + analyticsRuntime = ManagedRuntime.make( + analyticsDefaultLayer({ + surface: resolveSurface(), + version: VERSION, + channel: CHANNEL, + dataDir: resolveExecutorDataDir(), + }), + ); + return analyticsRuntime; +}; + +/** + * Lazy facade over the daemon's shared service, usable from boot paths that + * compose engines outside Effect. `record` forks into the module runtime + * (fire-and-forget, mirroring ./integrations.ts) so callers never wait on the + * layer build; ordering within the runtime preserves event order. + */ +export const localAnalytics: AnalyticsService = { + record: (name, properties) => + Effect.sync(() => { + getAnalyticsRuntime().runFork( + Effect.flatMap(Analytics.asEffect(), (analytics) => analytics.record(name, properties)), + ); + }), + flush: Effect.promise(() => + getAnalyticsRuntime().runPromise( + Effect.flatMap(Analytics.asEffect(), (analytics) => analytics.flush), + ), + ), +}; + +/** Flush and dispose the analytics runtime (daemon shutdown). */ +export const disposeAnalytics = async (): Promise => { + const runtime = analyticsRuntime; + if (!runtime) return; + analyticsRuntime = null; + await Effect.runPromise(Effect.promise(() => runtime.dispose()).pipe(Effect.ignoreCause())); +}; diff --git a/apps/local/src/app.ts b/apps/local/src/app.ts index a9915daf5..c565c14d4 100644 --- a/apps/local/src/app.ts +++ b/apps/local/src/app.ts @@ -2,14 +2,17 @@ import { HttpApiSwagger } from "effect/unstable/httpapi"; import { Layer } from "effect"; import { + ArtifactUsageObserver, composePluginApi, ExecutorApp, FixedExecutionProvider, textFailureStrategy, } from "@executor-js/api/server"; +import { withExecutionAnalytics } from "@executor-js/analytics"; import { createExecutionEngine } from "@executor-js/execution"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; +import { localAnalytics } from "./analytics"; import { getExecutorBundle, type LocalExecutor } from "./executor"; import { makeLocalIdentityLayer } from "./identity"; import { ErrorCaptureLive } from "./observability"; @@ -53,10 +56,16 @@ import { ErrorCaptureLive } from "./observability"; const localFixedExecutionLayer = (executor: LocalExecutor): Layer.Layer => Layer.succeed(FixedExecutionProvider)({ executor, - engine: createExecutionEngine({ - executor, - codeExecutor: makeQuickJsExecutor(), - }), + // This engine serves the HTTP executions API (`executor call`/`resume`, + // the web console) — the wrap binds the "api" plane structurally. + engine: withExecutionAnalytics( + createExecutionEngine({ + executor, + codeExecutor: makeQuickJsExecutor(), + }), + localAnalytics, + { plane: "api", toolkit: false }, + ), // The executor IS its own plugin-extension map (`executor[pluginId]`); the // fixed middleware reads `executor[id]` to satisfy each plugin's // `*ExtensionService` Tag per request — identical binding to the prior @@ -118,8 +127,16 @@ export const makeLocalApiHandler = async (token: string): Promise + localAnalytics.record(`artifact_${action}`, { via: "ui" }), + ), + ), }); const web = toWebHandler(); diff --git a/apps/local/src/executor.ts b/apps/local/src/executor.ts index 2386daf35..dd91838f3 100644 --- a/apps/local/src/executor.ts +++ b/apps/local/src/executor.ts @@ -17,6 +17,7 @@ import { loadPluginsFromJsonc } from "@executor-js/config"; import type { McpPluginExtension } from "@executor-js/plugin-mcp"; import executorConfig from "../executor.config"; +import { localAnalytics } from "./analytics"; import { localDataMigrations } from "./db/data-migrations"; import { openOwnedLocalDatabase } from "./db/owned-database"; @@ -195,6 +196,11 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => { subject: Subject.make(LOCAL_SUBJECT), db: sqlite.db, plugins, + onIntegrationChange: (event) => + localAnalytics.record( + event.kind === "added" ? "integration_added" : "integration_removed", + { plugin_key: event.pluginKey }, + ), onElicitation: "accept-all", oauthEndpointUrlPolicy: { allowHttp: true }, // EXPLICIT OAuth callback — the daemon serves the v2 `/api/oauth/callback` diff --git a/apps/local/src/main.ts b/apps/local/src/main.ts index 4395ef337..26378f58f 100644 --- a/apps/local/src/main.ts +++ b/apps/local/src/main.ts @@ -1,10 +1,12 @@ import { Context, Data, Effect, Layer, ManagedRuntime } from "effect"; +import { withExecutionAnalytics } from "@executor-js/analytics"; import { createExecutionEngine } from "@executor-js/execution"; import { artifactUrlFor } from "@executor-js/host-mcp/create-artifact"; import { loadMcpAppsShellHtml } from "@executor-js/mcp-apps-shell"; import { smokeRenderArtifact } from "@executor-js/mcp-apps-shell/smoke-render"; import { makeQuickJsExecutor } from "@executor-js/runtime-quickjs"; +import { localAnalytics } from "./analytics"; import { makeLocalApiHandler } from "./app"; import { createExecutorHandle, disposeExecutor, getExecutorBundle } from "./executor"; import { createMcpRequestHandler, type McpRequestHandler } from "./mcp"; @@ -85,10 +87,17 @@ export const createServerHandlers = async (token: string): Promise + localAnalytics.record(`artifact_${action}`, { via: "agent" }), }; mcp = createMcpRequestHandler({ defaultConfig: { @@ -127,10 +139,14 @@ export const createServerHandlers = async (token: string): Promise number = () => 200, +): Effect.Effect<{ + readonly layer: Layer.Layer; + readonly batches: Ref.Ref>; +}> => + Effect.gen(function* () { + const batches = yield* Ref.make>([]); + const httpLayer = Layer.succeed(HttpClient.HttpClient)( + HttpClient.make((request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + if (!Predicate.isTagged(request.body, "Uint8Array")) { + return yield* Effect.die("expected a Uint8Array request body"); + } + const body = yield* decodePostHogBatchBody( + new TextDecoder().decode(request.body.body), + ).pipe(Effect.orDie); + yield* Ref.update(batches, (existing) => [...existing, { url: request.url, body }]); + return HttpClientResponse.fromWeb(request, new Response("{}", { status: status() })); + }), + ), + ); + return { layer: httpLayer, batches }; + }); + +const withTempDir = (use: (dir: string) => Effect.Effect): Effect.Effect => + Effect.acquireUseRelease( + Effect.promise(() => mkdtemp(join(tmpdir(), "executor-analytics-"))), + use, + (dir) => Effect.promise(() => rm(dir, { recursive: true, force: true })), + ); + +const testConfig = (dataDir: string) => ({ + surface: "cli" as const, + version: "0.0.0", + channel: "dev" as const, + dataDir, + disabled: false, + posthogHost: "https://posthog.test", + posthogKey: "phc_test", + flushInterval: Duration.hours(1), +}); + +describe("isAnalyticsDisabled", () => { + it("honors DO_NOT_TRACK and EXECUTOR_DISABLE_ANALYTICS", () => { + expect(isAnalyticsDisabled({})).toBe(false); + expect(isAnalyticsDisabled({ DO_NOT_TRACK: "1" })).toBe(true); + expect(isAnalyticsDisabled({ DO_NOT_TRACK: "true" })).toBe(true); + expect(isAnalyticsDisabled({ EXECUTOR_DISABLE_ANALYTICS: "yes" })).toBe(true); + expect(isAnalyticsDisabled({ DO_NOT_TRACK: "0" })).toBe(false); + }); +}); + +describe("Analytics", () => { + it.effect("buffers events and flushes a PostHog batch with shared properties", () => + withTempDir((dir) => + Effect.gen(function* () { + const http = yield* makeRecordingHttpClient(); + yield* Effect.gen(function* () { + const analytics = yield* Analytics; + yield* analytics.record("execution_completed", { + ok: true, + plane: "mcp", + toolkit: false, + }); + yield* analytics.record("integration_added", { plugin_key: "openapi" }); + yield* analytics.flush; + + const batches = yield* Ref.get(http.batches); + expect(batches).toHaveLength(1); + expect(batches[0]?.url).toBe("https://posthog.test/batch/"); + // SAFETY: length asserted to be 1 above. + const body = batches[0]!.body; + expect(body.api_key).toBe("phc_test"); + expect(body.batch.map((event) => event.event)).toEqual([ + "execution_completed", + "integration_added", + ]); + for (const event of body.batch) { + expect(event.properties["$process_person_profile"]).toBe(false); + expect(event.properties["surface"]).toBe("cli"); + expect(event.properties["channel"]).toBe("dev"); + expect(event.properties["app_version"]).toBe("0.0.0"); + expect(event.distinct_id).toBe(body.batch[0]?.distinct_id); + } + }).pipe( + Effect.provide( + layer(testConfig(dir)).pipe( + Layer.provide(http.layer), + Layer.provide(NodeFileSystem.layer), + ), + ), + ); + }), + ), + ); + + it.effect("mints the anonymous id once and reuses it across runtimes", () => + withTempDir((dir) => + Effect.gen(function* () { + const http = yield* makeRecordingHttpClient(); + const runOnce = Effect.gen(function* () { + const analytics = yield* Analytics; + yield* analytics.record("execution_completed", { + ok: true, + plane: "api", + toolkit: false, + }); + yield* analytics.flush; + }).pipe( + Effect.provide( + layer(testConfig(dir)).pipe( + Layer.provide(http.layer), + Layer.provide(NodeFileSystem.layer), + ), + ), + ); + yield* runOnce; + yield* runOnce; + + const persisted = (yield* Effect.promise(() => + readFile(join(dir, "analytics-id"), "utf8"), + )).trim(); + expect(persisted.length).toBeGreaterThan(0); + + const batches = yield* Ref.get(http.batches); + expect(batches).toHaveLength(2); + const ids = batches.map((batch) => batch.body.batch[0]?.distinct_id); + expect(ids[0]).toBe(persisted); + expect(ids[1]).toBe(persisted); + }), + ), + ); + + it.effect("re-queues the batch when delivery fails and never surfaces the error", () => + withTempDir((dir) => + Effect.gen(function* () { + let failing = true; + const http = yield* makeRecordingHttpClient(() => (failing ? 500 : 200)); + yield* Effect.gen(function* () { + const analytics = yield* Analytics; + yield* analytics.record("execution_completed", { + ok: false, + plane: "mcp", + toolkit: true, + }); + yield* analytics.flush; + expect(yield* Ref.get(http.batches)).toHaveLength(1); + + failing = false; + yield* analytics.flush; + const batches = yield* Ref.get(http.batches); + expect(batches).toHaveLength(2); + // SAFETY: length asserted to be 2 above. + const retried = batches[1]!.body; + expect(retried.batch[0]?.event).toBe("execution_completed"); + expect(retried.batch[0]?.properties["toolkit"]).toBe(true); + }).pipe( + Effect.provide( + layer(testConfig(dir)).pipe( + Layer.provide(http.layer), + Layer.provide(NodeFileSystem.layer), + ), + ), + ); + }), + ), + ); + + it.effect("disabled config produces a no-op service that never dials out", () => + withTempDir((dir) => + Effect.gen(function* () { + const http = yield* makeRecordingHttpClient(); + yield* Effect.gen(function* () { + const analytics = yield* Analytics; + yield* analytics.record("execution_completed", { + ok: true, + plane: "api", + toolkit: false, + }); + yield* analytics.flush; + expect(yield* Ref.get(http.batches)).toHaveLength(0); + }).pipe( + Effect.provide( + layer({ ...testConfig(dir), disabled: true }).pipe( + Layer.provide(http.layer), + Layer.provide(NodeFileSystem.layer), + ), + ), + ); + // No id file either: a disabled install leaves no analytics trace. + const persisted = yield* Effect.promise(() => + readFile(join(dir, "analytics-id"), "utf8").then( + () => true, + () => false, + ), + ); + expect(persisted).toBe(false); + }), + ), + ); +}); diff --git a/packages/core/analytics/src/analytics.ts b/packages/core/analytics/src/analytics.ts new file mode 100644 index 000000000..81333d38b --- /dev/null +++ b/packages/core/analytics/src/analytics.ts @@ -0,0 +1,240 @@ +// --------------------------------------------------------------------------- +// Anonymous product analytics for the non-cloud hosts (local daemon, self-host). +// +// Model (mirrors t3code's server telemetry): a per-install anonymous id is +// minted once and persisted under the host's data dir; events buffer in memory +// and flush to PostHog's /batch/ endpoint on a fixed cadence plus a finalizer +// flush at shutdown. Delivery is best-effort — a failed flush re-queues the +// batch and never surfaces to the caller; analytics can never fail or slow a +// user-facing operation. +// +// Opt-out is the established cross-cutting convention: DO_NOT_TRACK (see +// consoledonottrack.com, honored by the desktop diagnostics and the +// integrations-registry fetch) or the feature-specific +// EXECUTOR_DISABLE_ANALYTICS. Disabled -> the layer builds a no-op service, so +// callers never branch. +// --------------------------------------------------------------------------- + +import { Context, DateTime, Duration, Effect, Layer, Ref, Schedule } from "effect"; +import { FileSystem } from "effect"; +import { + FetchHttpClient, + HttpClient, + HttpClientRequest, + HttpClientResponse, +} from "effect/unstable/http"; +import { NodeFileSystem } from "@effect/platform-node"; + +import type { AnalyticsEventName, AnalyticsEvents } from "./events"; + +// --------------------------------------------------------------------------- +// Surface vocabulary +// --------------------------------------------------------------------------- + +/** + * Which product hosts the daemon emitting the event. Always explicit at the + * composition root — there is deliberately no "local" catch-all and no env + * fallback inside this package; a host that cannot say which product it is + * has a wiring bug worth surfacing, not papering over. + */ +export type AnalyticsSurface = "cli" | "desktop" | "selfhost"; + +// --------------------------------------------------------------------------- +// Disable hooks +// --------------------------------------------------------------------------- + +const truthy = (value: string | undefined): boolean => { + if (value === undefined) return false; + const normalized = value.trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; +}; + +export const isAnalyticsDisabled = ( + env: Readonly> = process.env, +): boolean => truthy(env.DO_NOT_TRACK) || truthy(env.EXECUTOR_DISABLE_ANALYTICS); + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +export interface AnalyticsService { + /** Buffer an event for best-effort batched delivery. Never fails. */ + readonly record: ( + name: Name, + properties: AnalyticsEvents[Name], + ) => Effect.Effect; + /** Drain the buffer now (shutdown path; the layer also flushes on a cadence). */ + readonly flush: Effect.Effect; +} + +export class Analytics extends Context.Service()( + "@executor-js/analytics/Analytics", +) {} + +/** No-op service: what the layer builds when analytics is disabled. */ +export const analyticsNoop: AnalyticsService = { + record: () => Effect.void, + flush: Effect.void, +}; + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +// The same PostHog project the cloud browser client reports to (the key is +// public by design — it identifies the project, it grants no read access). +const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"; +const DEFAULT_POSTHOG_KEY = "phc_nNLrNMALpRsfrEkZovUkfMxYbcJvHnsJHeoSPavprgLL"; + +export interface AnalyticsConfig { + /** Which product hosts this daemon. Explicit, decided at the composition root. */ + readonly surface: AnalyticsSurface; + /** Product version (the hosting app's package version). */ + readonly version: string; + /** Release channel, matching the installation user-agent vocabulary. */ + readonly channel: "stable" | "beta" | "dev"; + /** + * Directory the anonymous install id persists under (the host's data dir: + * `~/.executor` for local, the self-host data dir for self-host). The id + * file is `/analytics-id`. + */ + readonly dataDir: string; + /** Disable entirely. Honors DO_NOT_TRACK and EXECUTOR_DISABLE_ANALYTICS if omitted. */ + readonly disabled?: boolean; + /** Override the PostHog ingest origin (tests). */ + readonly posthogHost?: string; + /** Override the PostHog project key (tests). */ + readonly posthogKey?: string; + /** Override the flush cadence. Defaults to 30 seconds. */ + readonly flushInterval?: Duration.Input; +} + +const FLUSH_BATCH_SIZE = 50; +const MAX_BUFFERED_EVENTS = 500; + +// --------------------------------------------------------------------------- +// Anonymous install id +// --------------------------------------------------------------------------- + +const ANALYTICS_ID_FILE = "analytics-id"; + +// Minted once per install, persisted forever (mirrors self-host's +// generate-and-persist secret.key shape, minus the secrecy — this id is not a +// credential, it is the anonymous distinct_id the events file under). +const loadOrMintAnonymousId = ( + dataDir: string, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const idPath = `${dataDir}/${ANALYTICS_ID_FILE}`; + const existing = yield* fs.readFileString(idPath).pipe( + Effect.map((raw) => raw.trim()), + Effect.catch(() => Effect.succeed("")), + ); + if (existing.length > 0) return existing; + const minted = crypto.randomUUID(); + yield* fs.makeDirectory(dataDir, { recursive: true }).pipe(Effect.ignore); + yield* fs.writeFileString(idPath, minted).pipe(Effect.ignore); + return minted; + }); + +// --------------------------------------------------------------------------- +// Layer +// --------------------------------------------------------------------------- + +interface BufferedEvent { + readonly event: AnalyticsEventName; + readonly properties: Record; + readonly timestamp: string; +} + +export const layer = ( + config: AnalyticsConfig, +): Layer.Layer => + Layer.effect(Analytics)( + Effect.gen(function* () { + const disabled = config.disabled ?? isAnalyticsDisabled(); + if (disabled) return analyticsNoop; + + const http = yield* HttpClient.HttpClient; + const distinctId = yield* loadOrMintAnonymousId(config.dataDir); + const buffer = yield* Ref.make>([]); + + const posthogHost = config.posthogHost ?? DEFAULT_POSTHOG_HOST; + const posthogKey = config.posthogKey ?? DEFAULT_POSTHOG_KEY; + + const sendBatch = (events: ReadonlyArray) => + HttpClientRequest.post(`${posthogHost}/batch/`).pipe( + HttpClientRequest.bodyJson({ + api_key: posthogKey, + batch: events.map((entry) => ({ + event: entry.event, + distinct_id: distinctId, + timestamp: entry.timestamp, + properties: { + ...entry.properties, + // Anonymous events only: never materialize a person profile. + $process_person_profile: false, + surface: config.surface, + channel: config.channel, + app_version: config.version, + }, + })), + }), + Effect.flatMap(http.execute), + Effect.flatMap(HttpClientResponse.filterStatusOk), + Effect.asVoid, + Effect.timeout(Duration.seconds(10)), + ); + + const flush: Effect.Effect = Effect.gen(function* () { + while (true) { + const batch = yield* Ref.modify(buffer, (current) => { + const next = current.slice(0, FLUSH_BATCH_SIZE); + return [next, current.slice(next.length)] as const; + }); + if (batch.length === 0) return; + yield* sendBatch(batch).pipe( + // Delivery failed: put the batch back for the next cadence tick and + // stop draining (the endpoint is unreachable, later batches would + // fail the same way). + Effect.tapError(() => Ref.update(buffer, (current) => [...batch, ...current])), + ); + } + }).pipe( + Effect.catchCause((cause) => + Effect.logDebug("Analytics.flush failed").pipe(Effect.annotateLogs("cause", cause)), + ), + ); + + const record: AnalyticsService["record"] = (name, properties) => + Effect.flatMap(DateTime.now, (now) => + Ref.update(buffer, (current) => { + const appended = [ + ...current, + { + event: name, + properties: properties as Record, + timestamp: DateTime.formatIso(now), + }, + ]; + // Bounded buffer: a dead network must not grow memory forever. + return appended.length > MAX_BUFFERED_EVENTS + ? appended.slice(appended.length - MAX_BUFFERED_EVENTS) + : appended; + }), + ); + + const flushEvery = config.flushInterval ?? Duration.seconds(30); + yield* Effect.forkScoped( + flush.pipe(Effect.schedule(Schedule.spaced(flushEvery)), Effect.ignore), + ); + yield* Effect.addFinalizer(() => flush); + + return { record, flush }; + }), + ); + +/** The layer over the default platform services (real HTTP + disk). */ +export const defaultLayer = (config: AnalyticsConfig): Layer.Layer => + layer(config).pipe(Layer.provide(FetchHttpClient.layer), Layer.provide(NodeFileSystem.layer)); diff --git a/packages/core/analytics/src/engine.test.ts b/packages/core/analytics/src/engine.test.ts new file mode 100644 index 000000000..ebca8c527 --- /dev/null +++ b/packages/core/analytics/src/engine.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Data, Effect, Result } from "effect"; + +import { ToolAddress, UrlElicitation, ElicitationId } from "@executor-js/sdk"; +import type { ExecutionEngine, ExecutionResult } from "@executor-js/execution"; + +import type { AnalyticsEvents, AnalyticsEventName } from "./events"; +import { withExecutionAnalytics } from "./engine"; + +class FakeExecutionError extends Data.TaggedError("FakeExecutionError")<{}> {} + +type Recorded = { readonly name: AnalyticsEventName; readonly properties: unknown }; + +const makeRecorder = () => { + const events: Recorded[] = []; + return { + events, + analytics: { + record: (name: Name, properties: AnalyticsEvents[Name]) => + Effect.sync(() => { + events.push({ name, properties }); + }), + flush: Effect.void, + }, + }; +}; + +const completedResult = (error?: string): ExecutionResult => ({ + status: "completed", + result: { result: error ? null : "ok", ...(error ? { error } : {}) }, +}); + +const pausedResult: ExecutionResult = { + status: "paused", + execution: { + id: "exec-1", + elicitationContext: { + address: ToolAddress.make("github.issues.create"), + args: {}, + request: UrlElicitation.make({ + message: "approve", + url: "https://example.test/approve", + elicitationId: ElicitationId.make("elic-1"), + }), + }, + }, +}; + +const makeFakeEngine = ( + outcomes: Partial<{ + execute: Effect.Effect<{ result: unknown; error?: string }, FakeExecutionError>; + executeWithPause: Effect.Effect; + resume: Effect.Effect; + }>, +): ExecutionEngine => ({ + execute: () => outcomes.execute ?? Effect.succeed({ result: "ok" }), + executeWithPause: () => outcomes.executeWithPause ?? Effect.succeed(completedResult()), + resume: () => outcomes.resume ?? Effect.succeed(null), + getPausedExecution: () => Effect.succeed(null), + pausedExecutionCount: () => Effect.succeed(0), + hasPausedExecutions: () => Effect.succeed(false), + getDescription: Effect.succeed("fake"), +}); + +describe("withExecutionAnalytics", () => { + it.effect("records ok on a completed execution", () => + Effect.gen(function* () { + const recorder = makeRecorder(); + const engine = withExecutionAnalytics(makeFakeEngine({}), recorder.analytics, { + plane: "mcp", + toolkit: false, + }); + yield* engine.executeWithPause("code"); + expect(recorder.events).toEqual([ + { + name: "execution_completed", + properties: { ok: true, plane: "mcp", toolkit: false }, + }, + ]); + }), + ); + + it.effect("records not-ok when the sandbox reports an error value", () => + Effect.gen(function* () { + const recorder = makeRecorder(); + const engine = withExecutionAnalytics( + makeFakeEngine({ executeWithPause: Effect.succeed(completedResult("boom")) }), + recorder.analytics, + { plane: "api", toolkit: false }, + ); + yield* engine.executeWithPause("code"); + expect(recorder.events).toEqual([ + { + name: "execution_completed", + properties: { ok: false, plane: "api", toolkit: false }, + }, + ]); + }), + ); + + it.effect("records not-ok when the engine fails on the error channel", () => + Effect.gen(function* () { + const recorder = makeRecorder(); + const engine = withExecutionAnalytics( + makeFakeEngine({ executeWithPause: Effect.fail(new FakeExecutionError()) }), + recorder.analytics, + { plane: "mcp", toolkit: true }, + ); + const result = yield* Effect.result(engine.executeWithPause("code")); + expect(Result.isFailure(result)).toBe(true); + expect(recorder.events).toEqual([ + { + name: "execution_completed", + properties: { ok: false, plane: "mcp", toolkit: true }, + }, + ]); + }), + ); + + it.effect("does not record a pause; records when the resume completes", () => + Effect.gen(function* () { + const recorder = makeRecorder(); + const paused = withExecutionAnalytics( + makeFakeEngine({ executeWithPause: Effect.succeed(pausedResult) }), + recorder.analytics, + { plane: "mcp", toolkit: false }, + ); + yield* paused.executeWithPause("code"); + expect(recorder.events).toEqual([]); + + const resumed = withExecutionAnalytics( + makeFakeEngine({ resume: Effect.succeed(completedResult()) }), + recorder.analytics, + { plane: "mcp", toolkit: false }, + ); + yield* resumed.resume("exec-1", { action: "accept" }); + expect(recorder.events).toEqual([ + { + name: "execution_completed", + properties: { ok: true, plane: "mcp", toolkit: false }, + }, + ]); + }), + ); + + it.effect("resume of an unknown execution records nothing", () => + Effect.gen(function* () { + const recorder = makeRecorder(); + const engine = withExecutionAnalytics(makeFakeEngine({}), recorder.analytics, { + plane: "api", + toolkit: false, + }); + yield* engine.resume("missing", { action: "accept" }); + expect(recorder.events).toEqual([]); + }), + ); +}); diff --git a/packages/core/analytics/src/engine.ts b/packages/core/analytics/src/engine.ts new file mode 100644 index 000000000..70796dfd0 --- /dev/null +++ b/packages/core/analytics/src/engine.ts @@ -0,0 +1,63 @@ +// --------------------------------------------------------------------------- +// Execution-analytics engine wrapper. +// +// The same overlay shape as cloud's `withExecutionUsageTracking` billing +// decorator: wrap the engine once per serving plane, at the composition root. +// Because the wrap point IS the plane, the `plane` label is bound structurally +// — misattribution is impossible the way it would be with a header or flag. +// +// Outcome semantics: one `execution_completed` per execution reaching a +// terminal state. A sandbox-level failure lands on the success channel as +// `ExecuteResult.error`; an infrastructure failure lands on the Effect error +// channel. Both count as `ok: false`. A pause is not terminal — the execution +// is recorded when its resume completes, so approve-then-complete counts once. +// --------------------------------------------------------------------------- + +import { Effect } from "effect"; +import type * as Cause from "effect/Cause"; + +import type { ExecutionEngine, ExecutionResult } from "@executor-js/execution"; + +import type { AnalyticsService } from "./analytics"; +import type { ExecutionPlane } from "./events"; + +export interface ExecutionAnalyticsContext { + readonly plane: ExecutionPlane; + /** Whether this engine serves a toolkit-scoped endpoint. */ + readonly toolkit: boolean; +} + +export const withExecutionAnalytics = ( + engine: ExecutionEngine, + analytics: AnalyticsService, + context: ExecutionAnalyticsContext, +): ExecutionEngine => { + const completed = (ok: boolean): Effect.Effect => + analytics.record("execution_completed", { + ok, + plane: context.plane, + toolkit: context.toolkit, + }); + + const recordOutcome = (result: ExecutionResult | null): Effect.Effect => + result?.status === "completed" ? completed(!result.result.error) : Effect.void; + + return { + ...engine, + execute: (code, options) => + engine.execute(code, options).pipe( + Effect.tap((result) => completed(!result.error)), + Effect.tapError(() => completed(false)), + ), + executeWithPause: (code, options) => + engine.executeWithPause(code, options).pipe( + Effect.tap(recordOutcome), + Effect.tapError(() => completed(false)), + ), + resume: (executionId, response) => + engine.resume(executionId, response).pipe( + Effect.tap(recordOutcome), + Effect.tapError(() => completed(false)), + ), + }; +}; diff --git a/packages/core/analytics/src/events.ts b/packages/core/analytics/src/events.ts new file mode 100644 index 000000000..5113a4e20 --- /dev/null +++ b/packages/core/analytics/src/events.ts @@ -0,0 +1,71 @@ +// --------------------------------------------------------------------------- +// Server-side product-analytics event catalog. +// +// The server analog of the browser catalog in packages/react/src/api/ +// analytics.tsx, which is BROWSER-ONLY by design (its own header says server +// events need their own seam — this is that seam). Cloud keeps its browser +// client; this catalog is for the surfaces that had none: the local daemon +// (CLI + desktop) and self-host. +// +// PROPERTY RULES (same line the browser catalog draws): +// - Metadata about product usage only — the question these answer is "how +// are product features being used", never "what is this user doing". +// - Never secrets, tokens, credentials, code, tool arguments or results. +// - Never user free text: no connection names, no tool addresses, no +// toolkit slugs (user-entered labels — send booleans/counts instead). +// - Plugin keys (`openapi`, `mcp`, `graphql`, ...) ARE allowed: a fixed, +// product-defined vocabulary. Integration slugs are NOT — stricter than +// the browser catalog: these events leave the user's own infrastructure, +// and WHICH integrations someone uses is their business, not ours. The +// kind of integration is the product question; the slug is not. +// - Identity is the anonymous per-install id only; no emails, no hostnames, +// no person or org names. +// --------------------------------------------------------------------------- + +/** + * Which door an execution came through. Bound structurally by the host (each + * serving plane wraps its engine once), never inferred from a header or flag: + * - `mcp` — an agent invoking over the MCP surface. + * - `api` — the HTTP executions API (CLI `call`/`resume`, the web console). + */ +export type ExecutionPlane = "mcp" | "api"; + +/** + * Which door an artifact operation came through: an agent calling the MCP + * artifact tools, or a human in the console UI (whose data layer is the + * artifacts HTTP API). Bound at each door's composition point, mirroring + * `ExecutionPlane`. + */ +export type ArtifactVia = "agent" | "ui"; + +export interface AnalyticsEvents { + /** + * A code execution reached a terminal outcome. Recorded once per execution + * at completion — a paused execution is recorded when its resume completes, + * not when it pauses. `toolkit` marks executions served by a toolkit-scoped + * MCP endpoint (the toolkit's slug is a user label and deliberately absent). + */ + execution_completed: { + readonly ok: boolean; + readonly plane: ExecutionPlane; + readonly toolkit: boolean; + }; + /** An integration row was created (upsert re-registers are not counted). */ + integration_added: { readonly plugin_key: string }; + /** An integration was removed by the user. */ + integration_removed: { readonly plugin_key: string }; + /** A generative-UI artifact was saved as a new row. */ + artifact_created: { readonly via: ArtifactVia }; + /** + * An artifact was opened for its content: the agent's `show-artifact` or + * the console detail page. Internal reads (binding resolution inside an + * execution) deliberately do not count. + */ + artifact_viewed: { readonly via: ArtifactVia }; + /** An existing artifact was overwritten in place, or renamed. */ + artifact_updated: { readonly via: ArtifactVia }; + /** An artifact was deleted. Only the UI offers deletion today. */ + artifact_deleted: { readonly via: ArtifactVia }; +} + +export type AnalyticsEventName = keyof AnalyticsEvents; diff --git a/packages/core/analytics/src/index.ts b/packages/core/analytics/src/index.ts new file mode 100644 index 000000000..f8972a87c --- /dev/null +++ b/packages/core/analytics/src/index.ts @@ -0,0 +1,12 @@ +export { + Analytics, + analyticsNoop, + defaultLayer, + isAnalyticsDisabled, + layer, + type AnalyticsConfig, + type AnalyticsService, + type AnalyticsSurface, +} from "./analytics"; +export { withExecutionAnalytics, type ExecutionAnalyticsContext } from "./engine"; +export type { AnalyticsEventName, AnalyticsEvents, ExecutionPlane } from "./events"; diff --git a/packages/core/analytics/tsconfig.json b/packages/core/analytics/tsconfig.json new file mode 100644 index 000000000..84df9bcf2 --- /dev/null +++ b/packages/core/analytics/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "plugins": [ + { + "name": "@effect/language-service", + "ignoreEffectSuggestionsInTscExitCode": true, + "ignoreEffectWarningsInTscExitCode": true, + "diagnosticSeverity": {} + } + ] + }, + "include": ["src"] +} diff --git a/packages/core/analytics/tsup.config.ts b/packages/core/analytics/tsup.config.ts new file mode 100644 index 000000000..009a9675d --- /dev/null +++ b/packages/core/analytics/tsup.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: { + index: "src/index.ts", + }, + format: ["esm"], + dts: false, + sourcemap: true, + clean: true, + external: [/^@executor-js\//, /^effect/, /^@effect\//, "vitest"], +}); diff --git a/packages/core/analytics/vitest.config.ts b/packages/core/analytics/vitest.config.ts new file mode 100644 index 000000000..ae847ff6d --- /dev/null +++ b/packages/core/analytics/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + }, +}); diff --git a/packages/core/api/src/handlers/artifacts.ts b/packages/core/api/src/handlers/artifacts.ts index 42c5520bf..8647f610a 100644 --- a/packages/core/api/src/handlers/artifacts.ts +++ b/packages/core/api/src/handlers/artifacts.ts @@ -7,9 +7,17 @@ import { } from "@executor-js/sdk"; import { ExecutorApi } from "../api"; -import { ExecutorService } from "../services"; +import { ArtifactUsageObserver, ExecutorService, type ArtifactUsageAction } from "../services"; import { capture } from "@executor-js/api"; +// Best-effort usage observation (see `ArtifactUsageObserver`): notified after +// the operation succeeded, failures swallowed, so analytics can never fail or +// slow an artifact request. +const notifyArtifactUsage = (action: ArtifactUsageAction) => + Effect.flatMap(Effect.service(ArtifactUsageObserver), (observer) => + observer ? observer(action).pipe(Effect.ignoreCause({ log: false })) : Effect.void, + ); + const summaryToResponse = (a: ArtifactSummary) => ({ id: a.id, owner: a.owner, @@ -42,6 +50,7 @@ export const ArtifactsHandlers = HttpApiBuilder.group(ExecutorApi, "artifacts", Effect.gen(function* () { const executor = yield* ExecutorService; const artifact = yield* executor.artifacts.get(path.artifactId); + yield* notifyArtifactUsage("viewed"); return artifactToResponse(artifact); }), ), @@ -57,6 +66,7 @@ export const ArtifactsHandlers = HttpApiBuilder.group(ExecutorApi, "artifacts", code: payload.code, bindings: payload.bindings, }); + yield* notifyArtifactUsage(payload.id === undefined ? "created" : "updated"); return artifactToResponse(saved); }), ), @@ -69,6 +79,7 @@ export const ArtifactsHandlers = HttpApiBuilder.group(ExecutorApi, "artifacts", id: path.artifactId, title: payload.title, }); + yield* notifyArtifactUsage("updated"); return artifactToResponse(renamed); }), ), @@ -78,6 +89,7 @@ export const ArtifactsHandlers = HttpApiBuilder.group(ExecutorApi, "artifacts", Effect.gen(function* () { const executor = yield* ExecutorService; yield* executor.artifacts.remove({ id: path.artifactId }); + yield* notifyArtifactUsage("deleted"); return { removed: true }; }), ), diff --git a/packages/core/api/src/server.ts b/packages/core/api/src/server.ts index 9f8cecfe3..d252e1a74 100644 --- a/packages/core/api/src/server.ts +++ b/packages/core/api/src/server.ts @@ -1,4 +1,9 @@ -export { ExecutorService, ExecutionEngineService } from "./services"; +export { + ArtifactUsageObserver, + ExecutorService, + ExecutionEngineService, + type ArtifactUsageAction, +} from "./services"; export { CoreHandlers, ToolsHandlers, diff --git a/packages/core/api/src/server/execution-stack.ts b/packages/core/api/src/server/execution-stack.ts index 40e9d921d..1ebebf2ce 100644 --- a/packages/core/api/src/server/execution-stack.ts +++ b/packages/core/api/src/server/execution-stack.ts @@ -69,9 +69,20 @@ export interface EngineDecoratorShape { readonly decorate: ( engine: ExecutionEngine, identity: EngineStackIdentity, + context: EngineStackContext, ) => ExecutionEngine; } +/** + * What the stack was built to serve, beyond the identity: the MCP resource + * when the caller is an MCP session (absent on the HTTP plane). Lets a + * decorator distinguish a toolkit-scoped engine without the host threading a + * side channel. + */ +export interface EngineStackContext { + readonly mcpResource?: McpResource; +} + export class EngineDecorator extends Context.Service()( "@executor-js/api/EngineDecorator", ) {} @@ -116,11 +127,15 @@ export const makeExecutionStack = < Effect.withSpan("executor.stack.decorator"), ); const engine = yield* Effect.sync(() => - decorate(createExecutionEngine({ executor, codeExecutor }), { - accountId, - organizationId, - organizationName, - }), + decorate( + createExecutionEngine({ executor, codeExecutor }), + { + accountId, + organizationId, + organizationName, + }, + { mcpResource: options?.mcpResource }, + ), ).pipe(Effect.withSpan("executor.stack.engine.init")); return { executor, engine }; }).pipe(Effect.withSpan("executor.stack.build")); diff --git a/packages/core/api/src/server/mcp-build.ts b/packages/core/api/src/server/mcp-build.ts index 7d447976c..3b9302fac 100644 --- a/packages/core/api/src/server/mcp-build.ts +++ b/packages/core/api/src/server/mcp-build.ts @@ -74,6 +74,7 @@ export const makeMcpBuildServer = ...(hostOptions?.smokeRenderArtifact ? { smokeRenderArtifact: hostOptions.smokeRenderArtifact } : {}), + ...(hostOptions?.onArtifactUsage ? { onArtifactUsage: hostOptions.onArtifactUsage } : {}), // Same org pinning as `RequestOrgSlug` above: self-host serves its // console under `/` (`default` when unconfigured), so the // deep link carries the principal's slug rather than relying on the @@ -101,6 +102,10 @@ export interface McpBuildHostOptions { * the server pass `smokeRenderArtifact` from `@executor-js/mcp-apps-shell`, * which loads it behind a dynamic import; omitting it skips the check. */ readonly smokeRenderArtifact?: (code: string) => Promise; + /** Forwarded to the tool server's `onArtifactUsage`: best-effort observation + * of agent-driven artifact operations, for hosts recording product + * analytics. */ + readonly onArtifactUsage?: (action: "created" | "viewed" | "updated") => Effect.Effect; } /** diff --git a/packages/core/api/src/server/scoped-executor.ts b/packages/core/api/src/server/scoped-executor.ts index 8f884760a..ea0e33ce6 100644 --- a/packages/core/api/src/server/scoped-executor.ts +++ b/packages/core/api/src/server/scoped-executor.ts @@ -40,6 +40,7 @@ import { Tenant, type AnyPlugin, type Executor, + type ExecutorConfig, type StorageFailure, } from "@executor-js/sdk"; import { @@ -90,6 +91,12 @@ export interface HostConfigShape { * detail of credential storage. */ readonly exposeCredentialProviders?: boolean; + /** + * Forwarded to `ExecutorConfig.onIntegrationChange`: best-effort post-commit + * observation of durable integration-catalog changes (see the sdk contract). + * Hosts that record product analytics supply it; omitted -> no observation. + */ + readonly onIntegrationChange?: ExecutorConfig["onIntegrationChange"]; } export class HostConfig extends Context.Service()( @@ -276,6 +283,7 @@ export const makeScopedExecutor = < plugins, httpClientLayer, fetch: hostedFetch, + onIntegrationChange: config.onIntegrationChange, onElicitation: "accept-all", redirectUri, oauthCallbackStateOrgSlug: orgSlug, diff --git a/packages/core/api/src/services.ts b/packages/core/api/src/services.ts index ce8449de6..248b18523 100644 --- a/packages/core/api/src/services.ts +++ b/packages/core/api/src/services.ts @@ -1,4 +1,4 @@ -import { Context } from "effect"; +import { Context, type Effect } from "effect"; import type * as Cause from "effect/Cause"; import type { Executor } from "@executor-js/sdk"; import type { ExecutionEngine } from "@executor-js/execution"; @@ -7,6 +7,23 @@ export class ExecutorService extends Context.Service( "ExecutorService", ) {} +/** A user-meaningful artifact operation on the HTTP plane — the console UI's + * data layer. `viewed` is the detail read (`get`), not `list`; `updated` + * covers both save-overwrite and rename; `setPreview` is a render side + * effect, not a user action, and deliberately absent. */ +export type ArtifactUsageAction = "created" | "viewed" | "updated" | "deleted"; + +/** + * Optional observer for artifact operations served by the artifacts HTTP + * handlers. A `Context.Reference` (default null) rather than a required + * service: hosts that record product analytics provide one at boot; every + * other host — and every test — composes unchanged. Observers are best-effort: + * the handlers swallow their failures. + */ +export const ArtifactUsageObserver = Context.Reference< + ((action: ArtifactUsageAction) => Effect.Effect) | null +>("@executor-js/api/ArtifactUsageObserver", { defaultValue: () => null }); + // Error channel widened to `Cause.YieldableError` so callers that plug // in a runtime-specific tagged error (e.g. // `ExecutionEngine`) assign structurally. diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 664b4d8b9..f5d18cfc8 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -180,6 +180,65 @@ describe("createExecutor", () => { }), ); + it.effect("notifies onIntegrationChange on create and remove, not on re-register", () => + Effect.gen(function* () { + const events: Array<{ kind: string; pluginKey: string; slug: string }> = []; + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + onIntegrationChange: (event) => + Effect.sync(() => { + events.push({ + kind: event.kind, + pluginKey: event.pluginKey, + slug: String(event.slug), + }); + }), + }); + + yield* executor.demo.seed(); + expect(events).toEqual([{ kind: "added", pluginKey: "demo", slug: String(INTEG) }]); + + // Upsert re-register of the same slug is not a durable change. + yield* executor.demo.seed(); + expect(events).toHaveLength(1); + + yield* executor.integrations.remove(INTEG); + expect(events).toEqual([ + { kind: "added", pluginKey: "demo", slug: String(INTEG) }, + { kind: "removed", pluginKey: "demo", slug: String(INTEG) }, + ]); + + // Removing an already-absent slug notifies nothing. + yield* executor.integrations.remove(INTEG); + expect(events).toHaveLength(2); + }), + ); + + it.effect("a failing onIntegrationChange observer never fails the operation", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + onIntegrationChange: () => Effect.die("observer exploded"), + }); + yield* executor.demo.seed(); + const integrations = yield* executor.integrations.list(); + expect(integrations.map((i) => String(i.slug))).toContain(String(INTEG)); + }), + ); + + it.effect("a rolled-back transaction never notifies onIntegrationChange", () => + Effect.gen(function* () { + const events: string[] = []; + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + onIntegrationChange: (event) => Effect.sync(() => void events.push(String(event.slug))), + }); + const result = yield* Effect.result(executor.demo.failAfterPluginAndCoreWrites()); + expect(Result.isFailure(result)).toBe(true); + expect(events).not.toContain("tx-integration"); + }), + ); + it.effect("projects core tools as the built-in Executor integration", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index ff15a4af7..276e92c3d 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -7,6 +7,7 @@ import { schema as fumaSchema, type RelationsMap } from "@executor-js/fumadb/sch import type { AnyColumn } from "@executor-js/fumadb/schema"; import { StorageError, + afterCommit, isStorageFailure, makeFumaClient, type FumaDb, @@ -97,6 +98,7 @@ import { import type { AuthMethodDescriptor, Integration, + IntegrationChangeEvent, IntegrationConfig, IntegrationDisplayDescriptor, RegisterIntegrationInput, @@ -642,6 +644,14 @@ export interface ExecutorConfig Effect.Effect; /** * Opt into the PLATFORM VIEW: a read-only, tenant-wide `executor.admin` * surface that reads across every subject in the tenant (see @@ -2191,6 +2201,15 @@ export const createExecutor = => + config.onIntegrationChange ? afterCommit(config.onIntegrationChange(event)) : Effect.void; + const integrationsRegister = ( pluginId: string, input: RegisterIntegrationInput, @@ -2213,7 +2232,7 @@ export const createExecutor = + created + ? notifyIntegrationChange({ kind: "added", pluginKey: pluginId, slug: input.slug }) + : Effect.void, + ), + Effect.asVoid, ); const integrationsUpdate = ( @@ -2273,7 +2300,7 @@ export const createExecutor = b("slug", "=", String(slug)), }); + return existing.plugin_id; }), + ).pipe( + Effect.tap((removedPluginId) => + removedPluginId !== null + ? notifyIntegrationChange({ kind: "removed", pluginKey: removedPluginId, slug }) + : Effect.void, + ), + Effect.asVoid, ); const integrationsDetect = ( diff --git a/packages/core/sdk/src/fuma-runtime.ts b/packages/core/sdk/src/fuma-runtime.ts index e560b1b09..137c0d01f 100644 --- a/packages/core/sdk/src/fuma-runtime.ts +++ b/packages/core/sdk/src/fuma-runtime.ts @@ -91,6 +91,29 @@ export const activeFumaDbRef = Context.Reference("executor/Active defaultValue: () => null, }); +// Post-commit hooks. `transaction()` nests by pass-through (an inner +// `transaction()` call inside an active transaction just runs its effect), so +// an effect that must observe only DURABLE changes cannot simply run "after +// the transaction" — after an inner pass-through the outer transaction may +// still roll back. `afterCommit` solves this structurally: while a transaction +// is active the effect is queued on the outermost transaction's hook list and +// runs after its commit; with no active transaction it runs immediately. +// Hooks are best-effort observers: failures and defects are swallowed, and a +// rolled-back transaction discards its queue. +const pendingCommitHooksRef = Context.Reference> | null>( + "executor/PendingCommitHooks", + { defaultValue: () => null }, +); + +export const afterCommit = (effect: Effect.Effect): Effect.Effect => + Effect.flatMap(Effect.service(pendingCommitHooksRef), (hooks) => + hooks + ? Effect.sync(() => { + hooks.push(effect); + }) + : effect.pipe(Effect.ignoreCause({ log: false })), + ); + class TransactionEffectFailure { constructor(readonly error: unknown) {} } @@ -158,11 +181,18 @@ export const makeFumaClient = (db: FumaDb, options: MakeFumaClientOptions = {}): Effect.flatMap(Effect.service(activeFumaDbRef), (active) => { if (active) return effect as Effect.Effect; + // The outermost transaction owns the post-commit hook queue; hooks + // queued anywhere inside (including nested pass-through transactions) + // run only after THIS commit, and are discarded on rollback. + const commitHooks: Array> = []; return Effect.tryPromise({ try: () => db.transaction(async (transactionDb) => { const exit = await Effect.runPromiseExit( - effect.pipe(Effect.provideService(activeFumaDbRef, transactionDb)), + effect.pipe( + Effect.provideService(activeFumaDbRef, transactionDb), + Effect.provideService(pendingCommitHooksRef, commitHooks), + ), ); if (Exit.isSuccess(exit)) return exit.value; @@ -179,7 +209,13 @@ export const makeFumaClient = (db: FumaDb, options: MakeFumaClientOptions = {}): } return fumaFailureFromCause("transaction", cause); }, - }); + }).pipe( + Effect.tap(() => + Effect.forEach(commitHooks, (hook) => hook.pipe(Effect.ignoreCause({ log: false })), { + discard: true, + }), + ), + ); }).pipe(Effect.withSpan("fumadb.transaction")) as Effect.Effect; return { use, transaction }; diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index f0a610a19..1a91ccaa4 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -85,6 +85,7 @@ export type { AuthMethodOAuthDescriptor, AuthPlacementDescriptor, Integration, + IntegrationChangeEvent, IntegrationConfig, IntegrationDisplayDescriptor, RegisterIntegrationInput, diff --git a/packages/core/sdk/src/integration.ts b/packages/core/sdk/src/integration.ts index 5d2986386..df1ea5087 100644 --- a/packages/core/sdk/src/integration.ts +++ b/packages/core/sdk/src/integration.ts @@ -152,6 +152,20 @@ export const mergeAuthTemplates = ( return result; }; +/** + * A durable change to the integration catalog, emitted through + * `ExecutorConfig.onIntegrationChange`: a row was created (`added` — upsert + * re-registers of an existing slug do not fire) or removed. The hook is a + * neutral notification seam — hosts decide what to do with it (the non-cloud + * hosts feed product analytics); core carries no analytics vocabulary. + */ +export interface IntegrationChangeEvent { + readonly kind: "added" | "removed"; + /** The owning plugin's id (`openapi`, `mcp`, `graphql`, ...). */ + readonly pluginKey: string; + readonly slug: IntegrationSlug; +} + /** What a plugin's extension method passes to `ctx.core.integrations.register`. * The v2 analog of v1's `SourceInput`, minus the per-source tool list (tools are * produced per-connection now). */ diff --git a/packages/core/sdk/src/test-config.ts b/packages/core/sdk/src/test-config.ts index bce9392b4..9ad7945ff 100644 --- a/packages/core/sdk/src/test-config.ts +++ b/packages/core/sdk/src/test-config.ts @@ -122,6 +122,7 @@ export type TestConfigOptions["onIntegrationChange"]; }; export const makeTestConfig = ( @@ -159,6 +160,7 @@ export const makeTestConfig = { ); }); + it("notifies onArtifactUsage: created on new save, updated on overwrite, viewed on show", async () => { + const store = makeArtifactStore(); + const usage: string[] = []; + await withClient( + makeStubEngine({}), + APPS_CAPS, + async (client) => { + await client.callTool({ + name: "create-artifact", + arguments: { code: COUNTER_CODE, title: "Dashboard" }, + }); + expect(usage).toEqual(["created"]); + + await client.callTool({ + name: "create-artifact", + arguments: { code: COUNTER_CODE, title: "Dashboard v2", artifactId: "art_1" }, + }); + expect(usage).toEqual(["created", "updated"]); + + await client.callTool({ name: "show-artifact", arguments: { id: "art_1" } }); + expect(usage).toEqual(["created", "updated", "viewed"]); + + // A miss is not a view. + await client.callTool({ name: "show-artifact", arguments: { id: "art_missing" } }); + expect(usage).toEqual(["created", "updated", "viewed"]); + + // Listing is not a view either. + await client.callTool({ name: "list-artifacts", arguments: {} }); + expect(usage).toEqual(["created", "updated", "viewed"]); + }, + { + artifacts: store.port, + onArtifactUsage: (action) => Effect.sync(() => void usage.push(action)), + }, + ); + }); + + it("a failing onArtifactUsage observer never fails the tool", async () => { + const store = makeArtifactStore(); + await withClient( + makeStubEngine({}), + APPS_CAPS, + async (client) => { + const result = await client.callTool({ + name: "create-artifact", + arguments: { code: COUNTER_CODE, title: "Dashboard" }, + }); + expect(result.isError).toBeFalsy(); + expect(structuredOf(result)).toEqual({ code: COUNTER_CODE, artifactId: "art_1" }); + }, + { + artifacts: store.port, + onArtifactUsage: () => Effect.die("observer exploded"), + }, + ); + }); + it("delivers a saved artifact as a deep link to clients without apps support", async () => { const store = makeArtifactStore(); await Effect.runPromise( diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 310560e37..a755f68eb 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -202,6 +202,15 @@ type SharedMcpServerConfig = { * has no URL to offer. */ readonly artifactUrl?: (artifactId: string) => string; + /** + * Notified when an agent-facing artifact tool completes a user-meaningful + * operation: `create-artifact` (created, or updated when it overwrote an + * existing id) and `show-artifact` (viewed). Internal artifact reads — + * binding resolution inside `execute-action` — deliberately do not notify. + * Best-effort observation: failures are swallowed and cannot affect the tool + * result. Hosts recording product analytics supply it; core stays agnostic. + */ + readonly onArtifactUsage?: (action: "created" | "viewed" | "updated") => Effect.Effect; }; /** @@ -1501,6 +1510,12 @@ export const createExecutorMcpServer = ( let executeActionTool: { enable: () => void; disable: () => void } | undefined; let executeActionResumeTool: { enable: () => void; disable: () => void } | undefined; + // Best-effort usage observation; a failing observer never affects the tool. + const notifyArtifactUsage = (action: "created" | "viewed" | "updated"): Effect.Effect => + config.onArtifactUsage + ? config.onArtifactUsage(action).pipe(Effect.ignoreCause({ log: false })) + : Effect.void; + const saveAndDeliverArtifact = (input: { readonly code: string; readonly title: string; @@ -1520,6 +1535,7 @@ export const createExecutorMcpServer = ( ...(input.bindings === undefined ? {} : { bindings: input.bindings }), preview: input.preview ?? null, }); + yield* notifyArtifactUsage(input.existingId === undefined ? "created" : "updated"); yield* Effect.annotateCurrentSpan({ "mcp.artifact.id": saved.id, "mcp.artifact.apps_enabled": appsEnabled, @@ -1689,6 +1705,7 @@ export const createExecutorMcpServer = ( .get(id) .pipe(Effect.catchCause(() => Effect.succeed(null))); if (!artifact) return artifactNotFoundResult(id); + yield* notifyArtifactUsage("viewed"); return deliverArtifact({ code: artifact.code, artifactId: artifact.id,