Compare commits

...
7 changed files with 306 additions and 8 deletions
+60 -2
View File
@@ -1,8 +1,8 @@
{
"version": "7",
"dialect": "sqlite",
"id": "be60f352-8da1-40e1-8d70-dc41121cfbc5",
"prevIds": ["3fb67508-0196-4bae-b2bd-c08ece7583fd"],
"id": "021f81ad-c99d-4405-81a3-284baea2cdf0",
"prevIds": ["be60f352-8da1-40e1-8d70-dc41121cfbc5"],
"ddl": [
{
"name": "account_state",
@@ -1942,6 +1942,64 @@
"entityType": "indexes",
"table": "session_message"
},
{
"columns": [
{
"value": "time_created",
"isExpression": false
},
{
"value": "session_id",
"isExpression": false
},
{
"value": "type",
"isExpression": false
},
{
"value": "json_extract(\"data\", '$.model.providerID')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.model.id')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.model.variant')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.tokens.input')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.tokens.output')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.tokens.reasoning')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.tokens.cache.read')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.tokens.cache.write')",
"isExpression": true
},
{
"value": "json_extract(\"data\", '$.cost')",
"isExpression": true
}
],
"isUnique": false,
"where": null,
"origin": "manual",
"name": "session_message_stats_idx",
"entityType": "indexes",
"table": "session_message"
},
{
"columns": [
{
+2
View File
@@ -45,6 +45,7 @@ import m42 from "./migration/20260812181746_session_inbox.js"
import m43 from "./migration/20260812213948_worktree.js"
import m44 from "./migration/20260819222447_session_viewed_state.js"
import m45 from "./migration/20260823191254_nullable_workspace_binding.js"
import m46 from "./migration/20260905155702_stats-covering-index.js"
export const migrations = [
m00,
@@ -93,4 +94,5 @@ export const migrations = [
m43,
m44,
m45,
m46,
] satisfies DatabaseMigration.Migration[]
@@ -0,0 +1,15 @@
import { Effect } from "effect"
import type { DatabaseMigration } from "../migration.js"
const migration: DatabaseMigration.Migration = {
id: "20260905155702_stats-covering-index",
up(tx) {
return Effect.gen(function* () {
yield* tx.run(
`CREATE INDEX \`session_message_stats_idx\` ON \`session_message\` (\`time_created\`,\`session_id\`,\`type\`,json_extract("data", '$.model.providerID'),json_extract("data", '$.model.id'),json_extract("data", '$.model.variant'),json_extract("data", '$.tokens.input'),json_extract("data", '$.tokens.output'),json_extract("data", '$.tokens.reasoning'),json_extract("data", '$.tokens.cache.read'),json_extract("data", '$.tokens.cache.write'),json_extract("data", '$.cost'));`,
)
})
},
}
export default migration
+3
View File
@@ -259,6 +259,9 @@ const schema: Omit<DatabaseMigration.Migration, "id"> = {
`CREATE INDEX \`session_message_session_time_created_id_idx\` ON \`session_message\` (\`session_id\`,\`time_created\`,\`id\`);`,
)
yield* tx.run(`CREATE INDEX \`session_message_time_created_idx\` ON \`session_message\` (\`time_created\`);`)
yield* tx.run(
`CREATE INDEX \`session_message_stats_idx\` ON \`session_message\` (\`time_created\`,\`session_id\`,\`type\`,json_extract("data", '$.model.providerID'),json_extract("data", '$.model.id'),json_extract("data", '$.model.variant'),json_extract("data", '$.tokens.input'),json_extract("data", '$.tokens.output'),json_extract("data", '$.tokens.reasoning'),json_extract("data", '$.tokens.cache.read'),json_extract("data", '$.tokens.cache.write'),json_extract("data", '$.cost'));`,
)
yield* tx.run(
`CREATE INDEX \`session_pending_session_delivery_seq_idx\` ON \`session_pending\` (\`session_id\`,\`delivery\`,\`admitted_seq\`);`,
)
+15
View File
@@ -94,6 +94,21 @@ export const SessionMessageTable = sqliteTable(
index("session_message_session_type_seq_idx").on(table.session_id, table.type, table.seq),
index("session_message_session_time_created_id_idx").on(table.session_id, table.time_created, table.id),
index("session_message_time_created_idx").on(table.time_created),
// Cover usage statistics without loading message bodies or tool output.
index("session_message_stats_idx").on(
table.time_created,
table.session_id,
table.type,
sql`json_extract(${table.data}, '$.model.providerID')`,
sql`json_extract(${table.data}, '$.model.id')`,
sql`json_extract(${table.data}, '$.model.variant')`,
sql`json_extract(${table.data}, '$.tokens.input')`,
sql`json_extract(${table.data}, '$.tokens.output')`,
sql`json_extract(${table.data}, '$.tokens.reasoning')`,
sql`json_extract(${table.data}, '$.tokens.cache.read')`,
sql`json_extract(${table.data}, '$.tokens.cache.write')`,
sql`json_extract(${table.data}, '$.cost')`,
),
],
)
+16 -6
View File
@@ -119,7 +119,8 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) {
const dateKey = makeDateKey(input.timezone)
yield* Effect.forEach(
ranges,
// Yield between daily message batches; SQLite and the row fold are synchronous.
windows(from, to, 24 * 60 * 60 * 1_000),
(range) =>
db
.all<MessageRow>(
@@ -184,6 +185,7 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) {
})
}),
),
Effect.andThen(Effect.yieldNow),
),
{ concurrency: 1, discard: true },
)
@@ -347,10 +349,10 @@ export const get = Effect.fn("SessionStats.get")(function* (input: Input = {}) {
}
})
function windows(from: number, to: number) {
return Array.from({ length: Math.ceil((to - from) / Window) }, (_, index) => ({
from: from + index * Window,
to: Math.min(to, from + (index + 1) * Window),
function windows(from: number, to: number, size = Window) {
return Array.from({ length: Math.ceil((to - from) / size) }, (_, index) => ({
from: from + index * size,
to: Math.min(to, from + (index + 1) * size),
}))
}
@@ -396,6 +398,8 @@ function addToolStatus(
}
function makeDateKey(timezone = "UTC") {
const zone = DateTime.zoneMakeNamedUnsafe(timezone)
const cached = { from: 0, to: 0, key: "" }
const formatter = new Intl.DateTimeFormat("en-US", {
timeZone: timezone,
year: "numeric",
@@ -403,8 +407,14 @@ function makeDateKey(timezone = "UTC") {
day: "2-digit",
})
return (time: number) => {
if (time >= cached.from && time < cached.to) return cached.key
// Reuse the local date, not a fixed 24-hour bucket: DST changes day length.
const date = DateTime.makeZonedUnsafe(time, { timeZone: zone })
cached.from = DateTime.toEpochMillis(DateTime.startOf(date, "day"))
cached.to = DateTime.toEpochMillis(DateTime.endOf(date, "day")) + 1
const parts = Object.fromEntries(formatter.formatToParts(time).map((part) => [part.type, part.value]))
return `${parts.year}-${parts.month}-${parts.day}`
cached.key = `${parts.year}-${parts.month}-${parts.day}`
return cached.key
}
}
+195
View File
@@ -16,6 +16,8 @@ import { SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql
import { SessionStats } from "@opencode-ai/core/session/stats"
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
import { DateTime, Effect, Schema } from "effect"
import { eq } from "drizzle-orm"
import { Statement } from "effect/unstable/sql"
import { testEffect } from "./lib/effect"
const it = testEffect(AppNodeBuilder.build(Database.node))
@@ -30,6 +32,199 @@ const encodeMessage = Schema.encodeSync(SessionMessage.Info)
const encodeUsage = Schema.encodeSync(SessionEvent.UsageRecorded.data)
describe("SessionStats", () => {
it.effect("covers the production usage query without loading message payloads", () =>
Effect.gen(function* () {
const database = yield* Database.Service
const queries: ReturnType<Statement.Statement<unknown>["compile"]>[] = []
yield* SessionStats.get({ from: 0, to: 2 * 24 * 60 * 60 * 1_000, tools: "none" }).pipe(
Effect.provideService(Statement.CurrentTransformer, (statement) =>
Effect.sync(() => {
queries.push(statement.compile())
return statement
}),
),
)
const messages = queries.filter((query) => query[0].includes("json_extract(message.data, '$.tokens.input')"))
expect(messages).toHaveLength(2)
yield* Effect.forEach(messages, (query) =>
Effect.gen(function* () {
const plan = yield* database.db.$client.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query[0]}`, query[1])
expect(plan.some((row) => row.detail.includes("USING COVERING INDEX session_message_stats_idx"))).toBe(true)
}),
)
}),
)
;[
{
name: "spring DST",
timezone: "America/New_York",
times: [
"2026-03-08T04:59:59.999Z",
"2026-03-08T05:00:00Z",
"2026-03-08T06:59:59Z",
"2026-03-08T07:00:00Z",
"2026-03-09T03:59:59.999Z",
"2026-03-09T04:00:00Z",
],
activity: [
{ date: "2026-03-07", steps: 1 },
{ date: "2026-03-08", steps: 4 },
{ date: "2026-03-09", steps: 1 },
],
},
{
name: "fall DST",
timezone: "America/New_York",
times: [
"2026-11-01T03:59:59.999Z",
"2026-11-01T04:00:00Z",
"2026-11-01T05:30:00Z",
"2026-11-01T06:30:00Z",
"2026-11-02T04:59:59.999Z",
"2026-11-02T05:00:00Z",
],
activity: [
{ date: "2026-10-31", steps: 1 },
{ date: "2026-11-01", steps: 4 },
{ date: "2026-11-02", steps: 1 },
],
},
{
name: "quarter-hour offset",
timezone: "Asia/Kathmandu",
times: ["2026-01-01T18:14:59.999Z", "2026-01-01T18:15:00Z", "2026-01-02T18:14:59.999Z", "2026-01-02T18:15:00Z"],
activity: [
{ date: "2026-01-01", steps: 1 },
{ date: "2026-01-02", steps: 2 },
{ date: "2026-01-03", steps: 1 },
],
},
].forEach((fixture) => {
it.effect(`groups activity by local calendar day across ${fixture.name}`, () =>
Effect.gen(function* () {
const database = yield* Database.Service
yield* database.db
.insert(ProjectTable)
.values({ id: projectID, worktree: AbsolutePath.make("/stats"), sandboxes: [] })
.run()
yield* database.db
.insert(SessionTable)
.values({ id: sessionID, project_id: projectID, slug: "root", directory: "/stats", version: "test" })
.run()
yield* database.db
.insert(SessionMessageTable)
.values(
fixture.times.map((time, index) =>
messageRow(sessionID, index + 1, assistant(`msg_stats_zone_${index}`, Date.parse(time), [])),
),
)
.run()
const stats = yield* SessionStats.get({
from: Date.parse(fixture.times[0]),
to: Date.parse(fixture.times[fixture.times.length - 1]) + 1,
timezone: fixture.timezone,
tools: "none",
})
expect(stats.activity).toEqual(fixture.activity)
expect(stats.activeDays).toBe(3)
expect(stats.streak).toBe(3)
}),
)
})
it.effect("preserves usage across windows and local dates with large message bodies", () =>
Effect.gen(function* () {
const database = yield* Database.Service
const db = database.db
yield* db
.insert(ProjectTable)
.values({ id: projectID, worktree: AbsolutePath.make("/stats"), sandboxes: [] })
.run()
.pipe(Effect.orDie)
yield* db
.insert(SessionTable)
.values({ id: sessionID, project_id: projectID, slug: "root", directory: "/stats", version: "test" })
.run()
.pipe(Effect.orDie)
const from = Date.UTC(2026, 0, 1)
const boundary = from + 31 * 24 * 60 * 60 * 1_000
const to = Date.UTC(2026, 1, 3)
yield* db
.insert(SessionMessageTable)
.values([
messageRow(
sessionID,
1,
SessionMessage.User.make({
id: SessionMessage.ID.make("msg_stats_large_user"),
type: "user",
text: "prompt ".repeat(100_000),
time: { created: DateTime.makeUnsafe(from) },
}),
),
...[boundary - 1, boundary, to].map((created, index) =>
messageRow(
sessionID,
index + 2,
SessionMessage.Assistant.make({
...assistant(`msg_stats_large_${index}`, created, [
SessionMessage.AssistantText.make({ type: "text", text: "content ".repeat(100_000) }),
]),
model: {
providerID: Provider.ID.make("example"),
id: Model.ID.make("model-a"),
variant: Model.VariantID.make("high"),
},
cost: Money.USD.make(0.0123456789),
}),
),
),
])
.run()
.pipe(Effect.orDie)
const stats = yield* SessionStats.get({ from, to, projectID, timezone: "America/New_York", tools: "none" })
expect(stats.sessions).toBe(1)
expect(stats.prompts).toBe(1)
expect(stats.steps).toBe(2)
expect(stats.tokens).toEqual({ input: 20, output: 10, reasoning: 4, cache: { read: 8, write: 2 } })
expect(stats.cost).toBe(Money.USD.make(0.0123456789 * 2))
expect(stats.activity).toEqual([{ date: "2026-01-31", steps: 2 }])
expect(stats.models).toEqual([
{
model: {
providerID: Provider.ID.make("example"),
id: Model.ID.make("model-a"),
variant: Model.VariantID.make("high"),
},
steps: 2,
tokens: stats.tokens,
cost: stats.cost,
},
])
yield* db
.update(SessionMessageTable)
.set({
data: messageRow(sessionID, 3, assistant("msg_stats_large_1", boundary, [], "model-b", 3)).data,
})
.where(eq(SessionMessageTable.id, SessionMessage.ID.make("msg_stats_large_1")))
.run()
const updated = yield* SessionStats.get({ from, to, tools: "none" })
expect(updated.tokens.input).toBe(40)
expect(updated.models[0].model.id).toBe(Model.ID.make("model-b"))
expect(updated.cost).toBe(Money.USD.make(0.0123456789 + 4.5))
yield* db
.delete(SessionMessageTable)
.where(eq(SessionMessageTable.id, SessionMessage.ID.make("msg_stats_large_1")))
.run()
const removed = yield* SessionStats.get({ from, to, tools: "none" })
expect(removed.steps).toBe(1)
expect(removed.tokens.input).toBe(10)
}),
)
it.effect("aggregates activity and tool reliability without reading message payloads outside the range", () =>
Effect.gen(function* () {
const db = (yield* Database.Service).db