diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 44d4e5284..a3b63e880 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -5,7 +5,6 @@ ### Added - Enabled native deferred tool loading for Fireworks Messages models. Use `ToolSearch` or `tool_search` as the loader name for prompt-prefix deferral ([#9323](https://github.com/earendil-works/pi/issues/9323)). -- Added `SystemMessage.replace` to discard the replayed prompt and tool state and start a new baseline. Providers collapse the transcript into one leading system message when a later system message replaces it. ### Fixed diff --git a/packages/ai/README.md b/packages/ai/README.md index df43d3057..3c8129ae9 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1386,12 +1386,11 @@ interface SystemMessage { sections?: Record; // named prompt sections; later messages patch by name, null removes toolsAdded?: Tool[]; // tools that become available here toolsRemoved?: ToolReference[]; // tools that stop being available here - replace?: boolean; // discard everything replayed so far; this message is the new complete state timestamp: number; } ``` -Sections are opaque text rendered verbatim after `content`, joined by blank lines. Keep each one self-delimiting (a tag, a heading) so the model can relate an update to the original. Replaying every system message in order yields the current prompt and tools; a message with `replace` clears the replayed content, sections, and tools before applying its own. The replay helpers take the message list: +Sections are opaque text rendered verbatim after `content`, joined by blank lines. Keep each one self-delimiting (a tag, a heading) so the model can relate an update to the original. Replaying every system message in order yields the current prompt and tools; the replay helpers take the message list: ```typescript import { getCurrentSystemPrompt, getCurrentTools } from "@earendil-works/pi-ai"; @@ -1407,7 +1406,7 @@ getCurrentTools(messages); // [] A custom `Provider` or `ProviderStreams` implementation reads the prompt and tools the same way from `context.messages`; `context.systemPrompt` and `context.tools` do not exist at that layer. -Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model, and every model when a later system message has `replace` set, receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native `tool_addition`/`tool_removal` blocks: the initial tools stay active at the top level, every later declaration is sent with `defer_loading` (plus a stable deferred placeholder from the first request, which keeps Anthropic's deferred-tool scaffolding in the cached prefix), and removed tools stay declared, so tool changes do not invalidate the prompt cache. That needs at least one initial tool and no same-name redefinition; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` or `supportsToolSearch` anchor additive tool changes at their message; everything else sends the current tool list at the top level. +Models that accept system messages mid-conversation (`supportsMidConvoSystemMessages` in the model's compat settings, set by the generated catalog for verified models) receive each later system message in place, so the cached prefix stays intact; section changes are framed by name for the model. Every other model receives `collapseSystemMessages(transcript)`: the replayed prompt and current tools as the leading system message, with later system messages dropped. Anthropic models that also set `supportsMidConvoToolChanges` send tool changes as native `tool_addition`/`tool_removal` blocks: the initial tools stay active at the top level, every later declaration is sent with `defer_loading` (plus a stable deferred placeholder from the first request, which keeps Anthropic's deferred-tool scaffolding in the cached prefix), and removed tools stay declared, so tool changes do not invalidate the prompt cache. That needs at least one initial tool and no same-name redefinition; otherwise the current tool list is sent at the top level with the system text only. OpenAI Responses models with `supportsAdditionalTools` or `supportsToolSearch` anchor additive tool changes at their message; everything else sends the current tool list at the top level. ## Context Serialization diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 4dcfd08d5..3479b352d 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -431,11 +431,9 @@ export interface DeferredHandle { * The leading system message is the system prompt. Later system messages change it: * `content` adds instructions from that point on, `sections` replace or remove named * prompt sections, and `toolsAdded`/`toolsRemoved` change the tool set. Replaying - * every system message in order yields the current prompt and tools. A message with - * `replace` discards the replayed state first, so it is a complete new baseline. - * Providers that accept system messages mid-conversation send each one in place; other - * providers, and every provider after a replacement, rebuild the leading system message - * from the replayed state. + * every system message in order yields the current prompt and tools. Providers that + * accept system messages mid-conversation send each one in place; other providers + * rebuild the leading system message from the replayed state. */ export interface SystemMessage { role: "system"; @@ -452,11 +450,6 @@ export interface SystemMessage { toolsAdded?: Tool[]; /** Tools that stop being available at this point. */ toolsRemoved?: ToolReference[]; - /** - * Discard every earlier system message before applying this one, so its `content`, - * `sections`, and `toolsAdded` are the complete prompt and tool state from here on. - */ - replace?: boolean; timestamp: number; // Unix timestamp in milliseconds } diff --git a/packages/ai/src/utils/transcript.ts b/packages/ai/src/utils/transcript.ts index caa854499..70bf0e026 100644 --- a/packages/ai/src/utils/transcript.ts +++ b/packages/ai/src/utils/transcript.ts @@ -59,7 +59,6 @@ export function getCurrentTools(messages: TranscriptMessages): Tool[] { const tools = new Map(); for (const message of messages) { if (!isSystemMessage(message)) continue; - if (message.replace) tools.clear(); for (const tool of message.toolsRemoved ?? []) tools.delete(tool.name); for (const tool of message.toolsAdded ?? []) tools.set(tool.name, tool); } @@ -69,8 +68,7 @@ export function getCurrentTools(messages: TranscriptMessages): Tool[] { /** * Replay every system message into one leading system message holding the current * prompt and tools. Later `content` is appended to the base prompt, `sections` are - * patched by name, a `replace` message starts over, and tools are resolved with - * {@link getCurrentTools}. + * patched by name, and tools are resolved with {@link getCurrentTools}. */ export function getCurrentSystemMessage(messages: TranscriptMessages): SystemMessage | undefined { const content: string[] = []; @@ -78,10 +76,6 @@ export function getCurrentSystemMessage(messages: TranscriptMessages): SystemMes let timestamp: number | undefined; for (const message of messages) { if (!isSystemMessage(message)) continue; - if (message.replace) { - content.length = 0; - sections.clear(); - } timestamp ??= message.timestamp; const text = contentText(message.content); if (text.length > 0) content.push(text); @@ -117,19 +111,12 @@ export function collapseSystemMessages(context: TranscriptContext): TranscriptCo return { messages: head ? [head, ...messages] : messages } as TranscriptContext; } -/** - * Keep later system messages in place when the model accepts them; otherwise collapse them. - * A replacement after the leading message always collapses: no provider can retract the - * prompt it already received, so the replayed state must become the leading prompt. - */ +/** Keep later system messages in place when the model accepts them; otherwise collapse them. */ export function resolveTranscript( context: TranscriptContext, supportsMidConvoSystemMessages: boolean | undefined, ): TranscriptContext { - const lateReplacement = context.messages.some( - (message, index) => index > 0 && isSystemMessage(message) && message.replace === true, - ); - return supportsMidConvoSystemMessages && !lateReplacement ? context : collapseSystemMessages(context); + return supportsMidConvoSystemMessages ? context : collapseSystemMessages(context); } /** Strip executable and display-only fields from a tool before transcript comparison or persistence. */ diff --git a/packages/ai/test/system-message-replay.test.ts b/packages/ai/test/system-message-replay.test.ts index 80ede6efb..118af9084 100644 --- a/packages/ai/test/system-message-replay.test.ts +++ b/packages/ai/test/system-message-replay.test.ts @@ -11,7 +11,6 @@ import { hasNonAdditiveToolChanges, hasToolRedefinitions, normalizeContext, - resolveTranscript, } from "../src/utils/transcript.ts"; function tool(name: string, description = `${name} tool`): Tool { @@ -60,29 +59,6 @@ describe("system message replay", () => { expect(collapseSystemMessages(collapsed)).toEqual(collapsed); }); - test("a replacement discards replayed state and collapses even for native providers", () => { - const replacement: Message = { - role: "system", - content: "forced", - toolsAdded: [tool("third")], - replace: true, - timestamp: 15, - }; - const patch: Message = { role: "system", content: "", sections: { d: "1" }, timestamp: 16 }; - const context = { messages: [...transcript.messages, replacement, patch] } as TranscriptContext; - expect(getCurrentSystemMessage(context.messages)).toEqual({ - role: "system", - content: "forced", - sections: { d: "1" }, - toolsAdded: [tool("third")], - timestamp: 10, - }); - expect(resolveTranscript(context, true)).toEqual(collapseSystemMessages(context)); - expect(resolveTranscript(transcript, true)).toBe(transcript); - const leading = { messages: [replacement, patch] } as TranscriptContext; - expect(resolveTranscript(leading, true)).toBe(leading); - }); - test("replay of a transcript without system messages is empty", () => { const context = normalizeContext({ messages: [{ role: "user", content: "hi", timestamp: 1 }] }); expect(getCurrentSystemMessage(context.messages)).toBeUndefined(); diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 22fbe58ed..50695c76a 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -26,7 +26,7 @@ - Fixed premature missing-model errors after login by waiting for catalog discovery. Radius now defaults to `balanced`, falling back to the first available Radius model when needed. - Fixed fullscreen mode reserving a blank row for custom footers that render zero rows ([#8919](https://github.com/earendil-works/pi/issues/8919)). - Fixed extension tools without parameter schemas to be rejected during registration instead of breaking provider requests ([#9300](https://github.com/earendil-works/pi/issues/9300)). -- Fixed `before_agent_start` handlers returning `systemPrompt` (and `forceSystemPrompt`) on models with mid-conversation system messages: the forced prompt is now persisted as a replacing system message and sent as the provider's leading system prompt instead of being appended as a section patch after the original prompt. +- Fixed `before_agent_start` handlers returning `systemPrompt` (and `forceSystemPrompt`) on models with mid-conversation system messages: the forced prompt is now sent as the provider's leading system prompt instead of being appended as a section patch after the original prompt. ## [0.85.1] - 2026-09-05 diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index fad258df4..24241eead 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -563,7 +563,7 @@ pi.on("before_agent_start", async (event, ctx) => { }); ``` -The `systemPromptOptions` field gives extensions access to the same structured data Pi uses to build the system prompt. Collections are mutable. Prefer changing `sections`, `selectedTools`, or `promptGuidelines`: Pi diffs the resulting prompt sections against what the model already has and appends one system message patching only the changed sections. Returning `systemPrompt`, or setting `forceSystemPrompt`, replaces the whole prompt: Pi persists a system message with `replace: true` that discards the prompt the model had, and every provider then receives the forced text as its leading system prompt (a cache miss when it changes). Tool selection changes update both the prompt contributions and executable provider tools; calling `pi.setActiveTools()` inside the handler has the same effect as editing `selectedTools`. Models that accept system messages mid-conversation receive the patch in place and keep their cached prefix; other models get the replayed prompt as their system prompt, which is a cache miss once per change. +The `systemPromptOptions` field gives extensions access to the same structured data Pi uses to build the system prompt. Collections are mutable. Prefer changing `sections`, `selectedTools`, or `promptGuidelines`: Pi diffs the resulting prompt sections against what the model already has and appends one system message patching only the changed sections. Returning `systemPrompt`, or setting `forceSystemPrompt`, replaces the whole prompt for the run: every provider receives the forced text as its leading system prompt (a cache miss when it changes), and the session transcript keeps recording the structured sections. Tool selection changes update both the prompt contributions and executable provider tools; calling `pi.setActiveTools()` inside the handler has the same effect as editing `selectedTools`. Models that accept system messages mid-conversation receive the patch in place and keep their cached prefix; other models get the replayed prompt as their system prompt, which is a cache miss once per change. Inside `before_agent_start`, `event.systemPrompt` and `ctx.getSystemPrompt()` both reflect the chained system prompt as of the current handler. Later `before_agent_start` handlers can still modify it again. diff --git a/packages/coding-agent/docs/session-format.md b/packages/coding-agent/docs/session-format.md index be3d5fea8..070eaffd9 100644 --- a/packages/coding-agent/docs/session-format.md +++ b/packages/coding-agent/docs/session-format.md @@ -82,7 +82,6 @@ interface SystemMessage { content: string | TextContent[]; toolsAdded?: Tool[]; toolsRemoved?: Array<{ name: string }>; - replace?: boolean; // discard earlier system messages; this one is the complete prompt and tool state timestamp: number; // Unix ms } @@ -226,7 +225,7 @@ For sessions with a parent (created via `/fork`, `/clone`, or `newSession({ pare ### SessionMessageEntry -A message in the conversation. The `message` field contains an `AgentMessage`. System messages carry the prompt and tool loadout: the first request of a session persists one with every prompt section and tool declaration, and later changes persist as system messages that patch `sections` by name (`null` removes one) and list `toolsAdded`/`toolsRemoved`. Replaying them in order yields the current prompt and tools; there is no separate prompt state entry. A `before_agent_start` handler that forces the whole prompt persists a system message with `replace: true` holding the forced text in `content` and the full tool set, and leaving the forced prompt persists another one with the structured sections. +A message in the conversation. The `message` field contains an `AgentMessage`. System messages carry the prompt and tool loadout: the first request of a session persists one with every prompt section and tool declaration, and later changes persist as system messages that patch `sections` by name (`null` removes one) and list `toolsAdded`/`toolsRemoved`. Replaying them in order yields the current prompt and tools; there is no separate prompt state entry. ```json {"type":"message","id":"a0b1c2d3","parentId":null,"timestamp":"2024-12-03T14:00:00.000Z","message":{"role":"system","content":"","sections":{"preamble":"You are an expert coding assistant...","tools":"\n- read: ...\n","cwd":"/project"},"toolsAdded":[{"name":"read","description":"...","parameters":{}}],"timestamp":1733234400000}} diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index bde7172e2..9ae0f0e22 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -110,7 +110,7 @@ import type { SlashCommandInfo } from "./slash-commands.ts"; import { createSyntheticSourceInfo, type SourceInfo } from "./source-info.ts"; import { buildSystemPrompt, - buildSystemPromptState, + buildSystemPromptSections, diffSystemPromptSections, type NormalizedBuildSystemPromptOptions, normalizeBuildSystemPromptOptions, @@ -404,6 +404,7 @@ export class AgentSession { this._unsubscribeAgent = this.agent.subscribe(this._handleAgentEvent); this._installAgentToolHooks(); this._installAgentNextTurnRefresh(); + this._installAgentForcedPromptProjection(); this._buildRuntime({ activeToolNames: this._initialActiveToolNames, @@ -1107,9 +1108,9 @@ export class AgentSession { * from `messages`), or undefined when the prompt is unchanged. Tool changes are declared by * the agent loop before the request. * - * A forced prompt is opaque, so entering, changing, or leaving one cannot be expressed as - * a section patch: it returns a `replace` message that discards the replayed state. The - * agent loop fills in the full tool set, since replay through a replacement starts empty. + * A forced prompt does not affect the transcript: the structured sections are still diffed + * and persisted, and the forced text is projected onto the request by + * {@link _installAgentForcedPromptProjection}. */ private _preparePromptAndToolLoadout( options: NormalizedBuildSystemPromptOptions, @@ -1120,20 +1121,40 @@ export class AgentSession { const tool = this._toolRegistry.get(name); return tool ? [tool] : []; }); - const current = getCurrentSystemMessage(messages); - const desired = buildSystemPromptState(options); - const currentIsOpaque = current !== undefined && current.sections === undefined; - if (desired.sections === undefined || currentIsOpaque) { - const unchanged = - currentIsOpaque && - desired.sections === undefined && - contentText(current.content) === contentText(desired.content); - return unchanged ? undefined : { role: "system", ...desired, replace: true, timestamp: Date.now() }; - } - const sections = diffSystemPromptSections(current?.sections ?? {}, desired.sections); + const sections = diffSystemPromptSections( + getCurrentSystemMessage(messages)?.sections ?? {}, + buildSystemPromptSections(options), + ); return sections ? { role: "system", content: "", sections, timestamp: Date.now() } : undefined; } + /** + * Send a forced prompt as the provider's leading system prompt without recording it. + * + * A `before_agent_start` handler that returns `systemPrompt` needs that exact text at the + * head of the request; a mid-conversation system message would leave the original prompt + * in place. The forced text is a rendering of the current prompt, so the transcript keeps + * its structured sections and the request is projected instead: the system messages + * collapse into one head holding the forced text and the current tools. Runs after the + * `context` extension handlers. + */ + private _installAgentForcedPromptProjection(): void { + const previousTransformContext = this.agent.transformContext; + this.agent.transformContext = async (messages, signal) => { + const transformed = previousTransformContext ? await previousTransformContext(messages, signal) : messages; + const forced = this._runSystemPromptOptions?.forceSystemPrompt; + if (forced === undefined) return transformed; + const current = getCurrentSystemMessage(transformed); + const head: SystemMessage = { + role: "system", + content: forced, + ...(current?.toolsAdded ? { toolsAdded: current.toolsAdded } : {}), + timestamp: current?.timestamp ?? Date.now(), + }; + return [head, ...transformed.filter((message) => message.role !== "system")]; + }; + } + /** Restore the active tool loadout declared by the session transcript, if it declares one. */ private _restoreToolsFromTranscript(): void { const current = getCurrentSystemMessage(this.sessionManager.buildSessionContext().messages); diff --git a/packages/coding-agent/test/system-prompt-updates.test.ts b/packages/coding-agent/test/system-prompt-updates.test.ts index 1f317bf02..a55c493d9 100644 --- a/packages/coding-agent/test/system-prompt-updates.test.ts +++ b/packages/coding-agent/test/system-prompt-updates.test.ts @@ -8,7 +8,6 @@ import { getCurrentSystemMessage, getCurrentSystemPrompt, getSystemMessageText, - resolveTranscript, type TranscriptContext, } from "@earendil-works/pi-ai"; import { getModel } from "@earendil-works/pi-ai/compat"; @@ -110,12 +109,13 @@ describe("system prompt updates", () => { ); }); - test("a forced prompt replaces the prompt and tool state and is replayed as the leading prompt", async () => { + test("a forced prompt is sent as the leading prompt for the run and never recorded", async () => { let turn = 0; const extension: ExtensionFactory = (pi) => { - pi.on("before_agent_start", () => - ++turn === 2 || turn === 3 ? { systemPrompt: "Exact prompt." } : undefined, - ); + pi.on("before_agent_start", (event) => { + if (++turn === 3) event.systemPromptOptions.sections.plan_mode = "Plan only."; + return turn === 2 || turn === 3 ? { systemPrompt: "Exact prompt." } : undefined; + }); }; const harness = await createHarness({ extensionFactories: [extension] }); try { @@ -130,19 +130,20 @@ describe("system prompt updates", () => { const systemMessages = requests.map((request) => request.messages.filter((message) => message.role === "system"), ); - expect(systemMessages.map((messages) => messages.length)).toEqual([1, 2, 2, 3]); + // Forced turns collapse to one leading message; the unforced fourth turn passes the + // recorded head and both plan_mode patches through. + expect(systemMessages.map((messages) => messages.length)).toEqual([1, 1, 1, 3]); const forced = systemMessages[1]?.at(-1); expect(forced).toEqual({ role: "system", content: "Exact prompt.", toolsAdded: systemMessages[0]?.[0]?.toolsAdded, - replace: true, - timestamp: expect.any(Number), + timestamp: systemMessages[0]?.[0]?.timestamp, }); + expect(systemMessages[2]?.at(-1)).toEqual(forced); expect(getCurrentSystemPrompt(requests[2]!.messages)).toBe("Exact prompt."); - // Anthropic-style providers keep later system messages in place, but a replacement collapses. - expect(resolveTranscript(requests[2]!, true).messages.map((message) => message.role)).toEqual([ + expect(requests[2]!.messages.map((message) => message.role)).toEqual([ "system", "user", "assistant", @@ -151,9 +152,15 @@ describe("system prompt updates", () => { "user", ]); - const restored = systemMessages[3]?.at(-1); - expect(restored).toMatchObject({ content: "", replace: true, sections: systemMessages[0]?.[0]?.sections }); - expect(restored?.toolsAdded).toEqual(forced?.toolsAdded); + // The transcript only records the structured sections, never the forced text. + const recorded = harness.session.messages.flatMap((message) => + message.role === "system" ? [message.sections] : [], + ); + expect(recorded).toEqual([ + systemMessages[0]?.[0]?.sections, + { plan_mode: "\nPlan only.\n" }, + { plan_mode: null }, + ]); expect(getCurrentSystemPrompt(harness.session.messages)).toBe(harness.session.systemPrompt); } finally { harness.cleanup(); diff --git a/packages/evals/src/harness.ts b/packages/evals/src/harness.ts index f06f10aac..0f8de1810 100644 --- a/packages/evals/src/harness.ts +++ b/packages/evals/src/harness.ts @@ -255,10 +255,9 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi } export function verifySystemPrompt( - messages: AgentSession["messages"], + systemPrompt: string, options: Pick, ): string { - const systemPrompt = getCurrentSystemPrompt(messages); if (options.expectedPiDocumentation === undefined) return systemPrompt; if (!systemPrompt.includes("\n\n")) { throw new Error(`Pi system prompt lost its rules in the ${options.name} eval variant.`); @@ -286,13 +285,17 @@ async function runPiCodingAgent( const isolatedHome = join(root, "home"); const agentDir = join(isolatedHome, ".pi", "agent"); const extensionFactories: InlineExtension[] = []; + let forcedSystemPrompt: string | undefined; if (options.transformSystemPrompt) { const transform = options.transformSystemPrompt; extensionFactories.push({ name: "eval-system-prompt-transform", hidden: true, factory: (pi) => { - pi.on("before_agent_start", ({ systemPrompt }) => ({ systemPrompt: transform(systemPrompt) })); + pi.on("before_agent_start", ({ systemPrompt }) => { + forcedSystemPrompt = transform(systemPrompt); + return { systemPrompt: forcedSystemPrompt }; + }); }, }); } @@ -382,7 +385,9 @@ async function runPiCodingAgent( if (response === undefined) { throw new Error("Pi eval input must include at least one prompt step."); } - const systemPrompt = getCurrentSystemPrompt(session.messages); + // A forced prompt is not recorded in the transcript, so use the one the transform + // extension sent; otherwise the replayed transcript prompt is what the provider received. + const systemPrompt = forcedSystemPrompt ?? getCurrentSystemPrompt(session.messages); const stats = session.getSessionStats(); const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some( ({ input: inputCost, output: outputCost, cacheRead, cacheWrite }) => @@ -405,7 +410,7 @@ async function runPiCodingAgent( }, }, }; - verifySystemPrompt(session.messages, options); + verifySystemPrompt(systemPrompt, options); const output = "output" in options ? await options.output({ response, session, systemPrompt, agentDir }) : response; result = { output, ...runDiagnostics }; diff --git a/packages/evals/test/harness.test.ts b/packages/evals/test/harness.test.ts index bb6ab3033..e98a6bd1b 100644 --- a/packages/evals/test/harness.test.ts +++ b/packages/evals/test/harness.test.ts @@ -1,5 +1,4 @@ import { homedir } from "node:os"; -import type { Message } from "@earendil-works/pi-ai"; import { getDocsPath, getExamplesPath, getReadmePath } from "@earendil-works/pi-coding-agent"; import { describe, expect, it, vi } from "vitest"; import { buildSystemPrompt } from "../../coding-agent/src/core/system-prompt.ts"; @@ -91,18 +90,17 @@ describe("documentation variant", () => { expect(stripped).not.toContain(getExamplesPath()); }); - it("verifies the replayed prompt that was sent before a later reload", () => { + it("verifies the prompt that was sent", () => { const prompt = buildSystemPrompt({ cwd: "/workspace", selectedTools: [...DOCUMENTATION_EVAL_TOOLS], }); const stripped = excludePiDocumentation(prompt); - const messages: Message[] = [ - { role: "system", content: stripped, replace: true, timestamp: 0 }, - { role: "user", content: [{ type: "text", text: "Configure Pi" }], timestamp: 1 }, - ]; - expect(verifySystemPrompt(messages, { name: "without_docs", expectedPiDocumentation: false })).toBe(stripped); + expect(verifySystemPrompt(stripped, { name: "without_docs", expectedPiDocumentation: false })).toBe(stripped); + expect(() => verifySystemPrompt(prompt, { name: "without_docs", expectedPiDocumentation: false })).toThrow( + "does not match", + ); }); it("fails closed when prompt markers are missing", () => {