mirror of
https://github.com/anomalyco/opencode.git
synced 2026-10-01 21:16:24 +00:00
Compare commits
4
Commits
remove-s3-lake
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
 Adam
|
dc31828006 | ||
|
Adam
|
aa481b8f56 | ||
|
opencode-agent[bot]
|
63cf236140 | ||
|
Adam
|
8bb2ccf829 |
No files matched your search
@@ -880,7 +880,6 @@
|
||||
"name": "@opencode-ai/stats-core",
|
||||
"version": "1.18.34",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-athena": "3.933.0",
|
||||
"@planetscale/database": "1.19.0",
|
||||
"drizzle-orm": "catalog:",
|
||||
"effect": "catalog:",
|
||||
@@ -899,7 +898,6 @@
|
||||
"name": "@opencode-ai/stats-server",
|
||||
"version": "1.18.34",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-firehose": "3.933.0",
|
||||
"@effect/platform-node": "catalog:",
|
||||
"@opencode-ai/stats-core": "workspace:*",
|
||||
"effect": "catalog:",
|
||||
@@ -1268,12 +1266,8 @@
|
||||
|
||||
"@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="],
|
||||
|
||||
"@aws-sdk/client-athena": ["@aws-sdk/client-athena@3.933.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-9eMUCu1Ay3C9ojo+dJcynSdpbxuwDVtZUt/Xhce+c2+mgDsmvRzjww+wfLpZwRNWxBWmeauQQAZk52tCwQgXsQ=="],
|
||||
|
||||
"@aws-sdk/client-cognito-identity": ["@aws-sdk/client-cognito-identity@3.1057.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-node": "^3.972.47", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-5MliYkp2u0+2arTp5fZIaxl+xmm90LEKv/VeSxhfNQW4t0fvWJrNO429/jchWQenNoDRrOGE59VfbuZUfwFujg=="],
|
||||
|
||||
"@aws-sdk/client-firehose": ["@aws-sdk/client-firehose@3.933.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-tDrtgczN2lQsflLDPYu/wdOoyCZLVYtgzmWnYzSEOBWd/cp2AbuQ7D+FemSwUTzyoMTuhhIevyEJKzqsF+QYxA=="],
|
||||
|
||||
"@aws-sdk/client-lambda": ["@aws-sdk/client-lambda@3.1057.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-node": "^3.972.47", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-JoaE3QNvPqOSHJtcAFfza7BRoGUNerIKlSVhsKCBsa1i54Vto1YWUS7itkUELK2rpvS8kNZMPliPxDdi/oV5dw=="],
|
||||
|
||||
"@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.933.0", "", { "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-bucket-endpoint": "3.930.0", "@aws-sdk/middleware-expect-continue": "3.930.0", "@aws-sdk/middleware-flexible-checksums": "3.932.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-location-constraint": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-sdk-s3": "3.932.0", "@aws-sdk/middleware-ssec": "3.930.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/signature-v4-multi-region": "3.932.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/eventstream-serde-browser": "^4.2.5", "@smithy/eventstream-serde-config-resolver": "^4.3.5", "@smithy/eventstream-serde-node": "^4.2.5", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-blob-browser": "^4.2.6", "@smithy/hash-node": "^4.2.5", "@smithy/hash-stream-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/md5-js": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-stream": "^4.5.6", "@smithy/util-utf8": "^4.2.0", "@smithy/util-waiter": "^4.2.5", "tslib": "^2.6.2" } }, "sha512-KxwZvdxdCeWK6o8mpnb+kk7Kgb8V+8AjTwSXUWH1UAD85B0tjdo1cSfE5zoR5fWGol4Ml5RLez12a6LPhsoTqA=="],
|
||||
@@ -5762,16 +5756,12 @@
|
||||
|
||||
"@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="],
|
||||
|
||||
"@aws-sdk/client-athena/@smithy/util-utf8": ["@smithy/util-utf8@4.2.2", "", { "dependencies": { "@smithy/util-buffer-from": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw=="],
|
||||
|
||||
"@aws-sdk/client-cognito-identity/@aws-sdk/core": ["@aws-sdk/core@3.974.15", "", { "dependencies": { "@aws-sdk/types": "^3.973.9", "@aws-sdk/xml-builder": "^3.972.26", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.5", "@smithy/signature-v4": "^5.4.5", "@smithy/types": "^4.14.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-UpA0rTGW/tHGITcCqHisbuuEPraYg9GG+mWmXjY5+RxZBMLGe6aL9oe0ix50LztwAcPIkGZLH0yWdMIkCM10hw=="],
|
||||
|
||||
"@aws-sdk/client-cognito-identity/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.47", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.41", "@aws-sdk/credential-provider-http": "^3.972.43", "@aws-sdk/credential-provider-ini": "^3.972.46", "@aws-sdk/credential-provider-process": "^3.972.41", "@aws-sdk/credential-provider-sso": "^3.972.45", "@aws-sdk/credential-provider-web-identity": "^3.972.45", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/credential-provider-imds": "^4.3.6", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-HrId+C0DWA5qDIyLG64/kjUB2RNtPypxmABnIctK+TA1P1kHlOYoE/Wf5T5tKOMKgb08P7k/zNyhvfJ3lh5Oag=="],
|
||||
|
||||
"@aws-sdk/client-cognito-identity/@aws-sdk/types": ["@aws-sdk/types@3.973.9", "", { "dependencies": { "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg=="],
|
||||
|
||||
"@aws-sdk/client-firehose/@smithy/util-utf8": ["@smithy/util-utf8@4.2.2", "", { "dependencies": { "@smithy/util-buffer-from": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw=="],
|
||||
|
||||
"@aws-sdk/client-lambda/@aws-sdk/core": ["@aws-sdk/core@3.974.15", "", { "dependencies": { "@aws-sdk/types": "^3.973.9", "@aws-sdk/xml-builder": "^3.972.26", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.5", "@smithy/signature-v4": "^5.4.5", "@smithy/types": "^4.14.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-UpA0rTGW/tHGITcCqHisbuuEPraYg9GG+mWmXjY5+RxZBMLGe6aL9oe0ix50LztwAcPIkGZLH0yWdMIkCM10hw=="],
|
||||
|
||||
"@aws-sdk/client-lambda/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.47", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.41", "@aws-sdk/credential-provider-http": "^3.972.43", "@aws-sdk/credential-provider-ini": "^3.972.46", "@aws-sdk/credential-provider-process": "^3.972.41", "@aws-sdk/credential-provider-sso": "^3.972.45", "@aws-sdk/credential-provider-web-identity": "^3.972.45", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/credential-provider-imds": "^4.3.6", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-HrId+C0DWA5qDIyLG64/kjUB2RNtPypxmABnIctK+TA1P1kHlOYoE/Wf5T5tKOMKgb08P7k/zNyhvfJ3lh5Oag=="],
|
||||
|
||||
+2
-4
@@ -1,9 +1,7 @@
|
||||
import { deployAws, domain } from "./stage"
|
||||
import { domain } from "./stage"
|
||||
import { EMAILOCTOPUS_API_KEY } from "./app"
|
||||
import { SECRET } from "./secret"
|
||||
|
||||
const lake = deployAws ? await import("./lake") : undefined
|
||||
|
||||
////////////////
|
||||
// DATABASE
|
||||
////////////////
|
||||
@@ -253,7 +251,7 @@ const SALESFORCE_INSTANCE_URL = new sst.Secret("SALESFORCE_INSTANCE_URL")
|
||||
|
||||
const logProcessor = new sst.cloudflare.Worker("LogProcessor", {
|
||||
handler: "packages/console/function/src/log-processor.ts",
|
||||
link: [SECRET.HoneycombApiKey, ...(lake?.lakeIngest ? [lake.lakeIngest] : [])],
|
||||
link: [SECRET.HoneycombApiKey],
|
||||
})
|
||||
|
||||
new sst.cloudflare.x.SolidStart("Console", {
|
||||
|
||||
+26
-268
@@ -1,4 +1,8 @@
|
||||
import { domain } from "./stage"
|
||||
// Retired in favor of the R2 lake. Nothing writes to it anymore, but the table,
|
||||
// catalog and Athena workgroup stay while the anomalyco/anomaly inference and
|
||||
// analytics dashboards still query its history. Pulumi reads forceDestroy and
|
||||
// retainOnDelete from state at delete time, so keep those settings until the
|
||||
// lake is removed.
|
||||
|
||||
const current = aws.getCallerIdentityOutput({})
|
||||
const partition = aws.getPartitionOutput({})
|
||||
@@ -6,19 +10,14 @@ const region = aws.getRegionOutput({})
|
||||
|
||||
const tableBucketName = `opencode-${$app.stage}-lake`
|
||||
const glueCatalogName = "s3tablescatalog"
|
||||
const glueCatalogArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:catalog`
|
||||
const glueS3TablesCatalogArn = $interpolate`${glueCatalogArn}/${glueCatalogName}`
|
||||
const glueS3TablesChildCatalogArn = $interpolate`${glueS3TablesCatalogArn}/${tableBucketName}`
|
||||
const glueS3TablesDatabaseWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/${glueCatalogName}/${tableBucketName}/*`
|
||||
const glueS3TablesTableWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/${tableBucketName}/*/*`
|
||||
const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*`
|
||||
|
||||
export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", {
|
||||
name: tableBucketName,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
forceDestroy: true,
|
||||
})
|
||||
|
||||
const s3TablesCatalog = new aws.cloudcontrol.Resource(
|
||||
new aws.cloudcontrol.Resource(
|
||||
"LakeS3TablesCatalog",
|
||||
{
|
||||
typeName: "AWS::Glue::Catalog",
|
||||
@@ -51,19 +50,28 @@ const s3TablesCatalog = new aws.cloudcontrol.Resource(
|
||||
{ dependsOn: [tableBucket] },
|
||||
)
|
||||
|
||||
const athenaResultsBucket = new aws.s3.Bucket("LakeAthenaResults", {
|
||||
bucket: `opencode-${$app.stage}-lake-athena-results`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
})
|
||||
const athenaResultsBucket = new aws.s3.Bucket(
|
||||
"LakeAthenaResults",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-athena-results`,
|
||||
forceDestroy: true,
|
||||
},
|
||||
{ retainOnDelete: false },
|
||||
)
|
||||
|
||||
const firehoseErrorBucket = new aws.s3.Bucket("LakeFirehoseErrors", {
|
||||
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
})
|
||||
// Keep the archived delivery failures until their records have been reconciled.
|
||||
new aws.s3.Bucket(
|
||||
"LakeFirehoseErrors",
|
||||
{
|
||||
bucket: `opencode-${$app.stage}-lake-firehose-errors`,
|
||||
forceDestroy: true,
|
||||
},
|
||||
{ retainOnDelete: false },
|
||||
)
|
||||
|
||||
const athenaWorkgroup = new aws.athena.Workgroup("LakeAthenaWorkgroup", {
|
||||
new aws.athena.Workgroup("LakeAthenaWorkgroup", {
|
||||
name: `opencode-${$app.stage}-lake-workgroup`,
|
||||
forceDestroy: $app.stage !== "production",
|
||||
forceDestroy: true,
|
||||
configuration: {
|
||||
enforceWorkgroupConfiguration: true,
|
||||
publishCloudwatchMetricsEnabled: true,
|
||||
@@ -77,255 +85,5 @@ const athenaWorkgroup = new aws.athena.Workgroup("LakeAthenaWorkgroup", {
|
||||
},
|
||||
})
|
||||
|
||||
const firehoseRole = new aws.iam.Role("LakeFirehoseRole", {
|
||||
assumeRolePolicy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["sts:AssumeRole"],
|
||||
principals: [
|
||||
{
|
||||
type: "Service",
|
||||
identifiers: ["firehose.amazonaws.com"],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
})
|
||||
|
||||
const firehosePolicy = new aws.iam.RolePolicy("LakeFirehosePolicy", {
|
||||
role: firehoseRole.id,
|
||||
policy: aws.iam.getPolicyDocumentOutput({
|
||||
statements: [
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"s3tables:ListTableBuckets",
|
||||
"s3tables:GetTableBucket",
|
||||
"s3tables:GetNamespace",
|
||||
"s3tables:GetTable",
|
||||
"s3tables:GetTableData",
|
||||
"s3tables:GetTableMetadataLocation",
|
||||
"s3tables:ListNamespaces",
|
||||
"s3tables:ListTables",
|
||||
"s3tables:PutTableData",
|
||||
"s3tables:UpdateTableMetadataLocation",
|
||||
],
|
||||
resources: ["*"],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"glue:GetCatalog",
|
||||
"glue:GetCatalogs",
|
||||
"glue:GetDatabase",
|
||||
"glue:GetDatabases",
|
||||
"glue:GetTable",
|
||||
"glue:GetTables",
|
||||
"glue:UpdateTable",
|
||||
],
|
||||
resources: [
|
||||
glueCatalogArn,
|
||||
glueS3TablesCatalogArn,
|
||||
$interpolate`${glueS3TablesCatalogArn}/*`,
|
||||
glueS3TablesDatabaseWildcardArn,
|
||||
glueS3TablesTableWildcardArn,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
|
||||
],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: [
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:GetObject",
|
||||
"s3:ListBucket",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:PutObject",
|
||||
],
|
||||
resources: [firehoseErrorBucket.arn, $interpolate`${firehoseErrorBucket.arn}/*`],
|
||||
},
|
||||
{
|
||||
effect: "Allow",
|
||||
actions: ["lakeformation:GetDataAccess"],
|
||||
resources: ["*"],
|
||||
},
|
||||
],
|
||||
}).json,
|
||||
})
|
||||
|
||||
const firehose = new aws.kinesis.FirehoseDeliveryStream(
|
||||
"LakeFirehose",
|
||||
{
|
||||
name: `opencode-${$app.stage}-lake-ingest`,
|
||||
destination: "iceberg",
|
||||
icebergConfiguration: {
|
||||
appendOnly: true,
|
||||
bufferingInterval: 60,
|
||||
bufferingSize: 1,
|
||||
catalogArn: glueS3TablesChildCatalogArn,
|
||||
processingConfiguration: {
|
||||
enabled: true,
|
||||
processors: [
|
||||
{
|
||||
type: "MetadataExtraction",
|
||||
parameters: [
|
||||
{ parameterName: "JsonParsingEngine", parameterValue: "JQ-1.6" },
|
||||
{
|
||||
parameterName: "MetadataExtractionQuery",
|
||||
parameterValue:
|
||||
'{destinationDatabaseName:._lake_database,destinationTableName:._lake_table,operation:(._lake_operation // "insert")}',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
roleArn: firehoseRole.arn,
|
||||
s3BackupMode: "FailedDataOnly",
|
||||
s3Configuration: {
|
||||
roleArn: firehoseRole.arn,
|
||||
bucketArn: firehoseErrorBucket.arn,
|
||||
errorOutputPrefix: "errors/!{firehose:error-output-type}/",
|
||||
},
|
||||
},
|
||||
},
|
||||
{ dependsOn: [s3TablesCatalog, firehosePolicy] },
|
||||
)
|
||||
|
||||
export const lakeVpc = new sst.aws.Vpc("LakeVpc")
|
||||
export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc })
|
||||
export const lakeRegion = region.region
|
||||
export const lakeCatalog = $interpolate`${glueCatalogName}/${tableBucket.name}`
|
||||
export const lakeAthenaWorkgroup = athenaWorkgroup
|
||||
|
||||
const ingestSecret = new random.RandomPassword("LakeIngestSecret", { length: 32 })
|
||||
export const ingestSecretSsm = new aws.ssm.Parameter("LakeIngestSecretSsm", {
|
||||
name: $interpolate`/${$app.name}/${$app.stage}/lake/ingest/secret`,
|
||||
type: "SecureString",
|
||||
value: ingestSecret.result,
|
||||
})
|
||||
|
||||
const ingestConfig = new sst.Linkable("LakeIngestConfig", {
|
||||
properties: {
|
||||
streamName: firehose.name,
|
||||
secret: ingestSecret.result,
|
||||
},
|
||||
})
|
||||
|
||||
const ingestService = new sst.aws.Service("LakeIngestService", {
|
||||
cluster: lakeCluster,
|
||||
architecture: "arm64",
|
||||
cpu: "1 vCPU",
|
||||
memory: "4 GB",
|
||||
image: {
|
||||
context: ".",
|
||||
dockerfile: "packages/stats/server/Dockerfile",
|
||||
},
|
||||
link: [ingestConfig],
|
||||
permissions: [
|
||||
{
|
||||
actions: ["firehose:PutRecord", "firehose:PutRecordBatch"],
|
||||
resources: [firehose.arn],
|
||||
},
|
||||
],
|
||||
scaling: {
|
||||
min: $app.stage === "production" ? 2 : 1,
|
||||
max: $app.stage === "production" ? 32 : 4,
|
||||
cpuUtilization: 60,
|
||||
memoryUtilization: 70,
|
||||
},
|
||||
loadBalancer: {
|
||||
domain: {
|
||||
name: `lake.${domain}`,
|
||||
dns: sst.cloudflare.dns(),
|
||||
},
|
||||
rules: [
|
||||
{ listen: "80/http", redirect: "443/https" },
|
||||
{ listen: "443/https", forward: "3000/http" },
|
||||
],
|
||||
health: {
|
||||
"3000/http": {
|
||||
path: "/ready",
|
||||
successCodes: "200-299",
|
||||
},
|
||||
},
|
||||
},
|
||||
health: {
|
||||
command: [
|
||||
"CMD-SHELL",
|
||||
"bun --eval \"fetch('http://localhost:3000/health').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))\"",
|
||||
],
|
||||
interval: "30 seconds",
|
||||
retries: 3,
|
||||
startPeriod: "30 seconds",
|
||||
timeout: "5 seconds",
|
||||
},
|
||||
dev: {
|
||||
command: "bun run start",
|
||||
directory: "packages/stats/server",
|
||||
url: "http://localhost:3000",
|
||||
},
|
||||
wait: $app.stage === "production",
|
||||
})
|
||||
|
||||
export const lakeIngest = new sst.Linkable("LakeIngest", {
|
||||
properties: {
|
||||
url: ingestService.url,
|
||||
secret: ingestSecret.result,
|
||||
},
|
||||
})
|
||||
|
||||
export const lakeQueryPermissions = [
|
||||
{
|
||||
actions: ["athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResults"],
|
||||
resources: [athenaWorkgroup.arn],
|
||||
},
|
||||
{
|
||||
actions: [
|
||||
"glue:GetCatalog",
|
||||
"glue:GetCatalogs",
|
||||
"glue:GetDatabase",
|
||||
"glue:GetDatabases",
|
||||
"glue:GetTable",
|
||||
"glue:GetTables",
|
||||
"glue:GetPartitions",
|
||||
],
|
||||
resources: [
|
||||
glueCatalogArn,
|
||||
glueS3TablesCatalogArn,
|
||||
$interpolate`${glueS3TablesCatalogArn}/*`,
|
||||
glueS3TablesDatabaseWildcardArn,
|
||||
glueS3TablesTableWildcardArn,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`,
|
||||
$interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`,
|
||||
],
|
||||
},
|
||||
{
|
||||
actions: ["s3:GetBucketLocation", "s3:ListBucket"],
|
||||
resources: [athenaResultsBucket.arn],
|
||||
},
|
||||
{
|
||||
actions: ["s3:GetObject", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads"],
|
||||
resources: [$interpolate`${athenaResultsBucket.arn}/*`],
|
||||
},
|
||||
{
|
||||
actions: [
|
||||
"s3tables:GetTableBucket",
|
||||
"s3tables:GetNamespace",
|
||||
"s3tables:GetTable",
|
||||
"s3tables:GetTableData",
|
||||
"s3tables:GetTableMetadataLocation",
|
||||
"s3tables:ListNamespaces",
|
||||
"s3tables:ListTables",
|
||||
],
|
||||
resources: ["*"],
|
||||
},
|
||||
{
|
||||
actions: ["lakeformation:GetDataAccess"],
|
||||
resources: ["*"],
|
||||
},
|
||||
]
|
||||
+3
-17
@@ -1,4 +1,4 @@
|
||||
import { lakeAthenaWorkgroup, lakeCatalog, lakeCluster, lakeQueryPermissions, lakeRegion, tableBucket } from "./lake"
|
||||
import { lakeCluster, tableBucket } from "./lake"
|
||||
import { EMAILOCTOPUS_API_KEY } from "./app"
|
||||
import { domain } from "./stage"
|
||||
|
||||
@@ -11,7 +11,7 @@ const inferenceNamespace = new aws.s3tables.Namespace("LakeInferenceNamespace",
|
||||
tableBucketArn: tableBucket.arn,
|
||||
})
|
||||
|
||||
const inferenceEventTable = new aws.s3tables.Table(
|
||||
new aws.s3tables.Table(
|
||||
"LakeInferenceEventTable",
|
||||
{
|
||||
name: "event",
|
||||
@@ -89,17 +89,6 @@ const inferenceEventTable = new aws.s3tables.Table(
|
||||
{ deleteBeforeReplace: $app.stage !== "production", ignoreChanges: ["metadata"] },
|
||||
)
|
||||
|
||||
export const inferenceEvent = new sst.Linkable("InferenceEvent", {
|
||||
properties: {
|
||||
region: lakeRegion,
|
||||
catalog: lakeCatalog,
|
||||
database: inferenceNamespace.namespace,
|
||||
table: inferenceEventTable.name,
|
||||
tableBucket: tableBucket.name,
|
||||
workgroup: lakeAthenaWorkgroup.name,
|
||||
},
|
||||
})
|
||||
|
||||
////////////////
|
||||
// DATABASE
|
||||
////////////////
|
||||
@@ -203,10 +192,7 @@ export const statSync = new sst.aws.Service("StatsSyncService", {
|
||||
dockerfile: "packages/stats/server/Dockerfile",
|
||||
},
|
||||
command: ["bun", "src/stat-sync.ts"],
|
||||
// Keep the legacy Athena link and IAM permissions during the first R2-backed
|
||||
// release so reverting the application code remains a one-deploy rollback.
|
||||
link: [database, inferenceEvent, r2Sql, r2SqlAuthToken, statsSyncConfig],
|
||||
permissions: lakeQueryPermissions,
|
||||
link: [database, r2Sql, r2SqlAuthToken, statsSyncConfig],
|
||||
scaling: {
|
||||
min: 1,
|
||||
max: 1,
|
||||
|
||||
+4
-4
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"nodeModules": {
|
||||
"x86_64-linux": "sha256-RIzpvDBR6GOfU2vjrtgP3LJ/j+gLxkMnJ9VE5Y65Pfo=",
|
||||
"aarch64-linux": "sha256-seE7v8AqxWCiyxNARMiLJNB1MZv15OwEf6r21k68xqg=",
|
||||
"aarch64-darwin": "sha256-rbyK+3qz9bGOxg3HlGW6yh3qeazCaxFcuPfdGbXHsIY=",
|
||||
"x86_64-darwin": "sha256-chSE5WtSQ+qdbj5vTvGmr4rn9gk8eVp3pVeow4gVqYI="
|
||||
"x86_64-linux": "sha256-fAqsf8Zgjz9oinwnFmlDbnT8hW61MfP1aJgXDSWFHLg=",
|
||||
"aarch64-linux": "sha256-Ite4U+M3tagfb/0f33XwNdXZRpbQDFD5g1SuKQMU9wo=",
|
||||
"aarch64-darwin": "sha256-53v/BWVbpOFQqP+Wc906FWOih0xKbQgiumGbqmIS1fo=",
|
||||
"x86_64-darwin": "sha256-tjWqIKUiwHg4goIb/aHy3EOEvYcjjK4ft0GHPiPhVbY="
|
||||
}
|
||||
}
|
||||
@@ -37,6 +37,7 @@ export default {
|
||||
"ip.prefix": ipPrefix(ip),
|
||||
}
|
||||
const time = new Date(event.eventTimestamp ?? Date.now()).toISOString()
|
||||
// This also merges each _metric log into data, so keep it while the Honeycomb export below is disabled.
|
||||
const events = [
|
||||
...event.logs.flatMap((log) =>
|
||||
log.message.flatMap((message: string) => {
|
||||
@@ -53,107 +54,20 @@ export default {
|
||||
]
|
||||
console.log(JSON.stringify(data, null, 2))
|
||||
|
||||
const lakeIngest = getLakeIngest()
|
||||
const [lake] = await Promise.all([
|
||||
// fetch("https://api.honeycomb.io/1/batch/zen", {
|
||||
// method: "POST",
|
||||
// headers: {
|
||||
// "Content-Type": "application/json",
|
||||
// "X-Honeycomb-Team": Resource.HONEYCOMB_API_KEY.value,
|
||||
// },
|
||||
// body: JSON.stringify(events),
|
||||
// }),
|
||||
...(lakeIngest
|
||||
? [
|
||||
fetch(lakeIngest.url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${lakeIngest.secret}`,
|
||||
},
|
||||
body: JSON.stringify({ events: events.map((event) => toLakeEvent(event.time, event.data)) }),
|
||||
}),
|
||||
]
|
||||
: []),
|
||||
])
|
||||
// const honeycomb = await fetch("https://api.honeycomb.io/1/batch/zen", {
|
||||
// method: "POST",
|
||||
// headers: {
|
||||
// "Content-Type": "application/json",
|
||||
// "X-Honeycomb-Team": Resource.HONEYCOMB_API_KEY.value,
|
||||
// },
|
||||
// body: JSON.stringify(events),
|
||||
// })
|
||||
// console.log(honeycomb.status)
|
||||
// console.log(await honeycomb.text())
|
||||
if (lake) {
|
||||
console.log(lake.status)
|
||||
console.log(await lake.text())
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
function getLakeIngest(): { url: string; secret: string } | undefined {
|
||||
try {
|
||||
return Resource.LakeIngest
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function toLakeEvent(time: string, data: Record<string, unknown>) {
|
||||
return {
|
||||
_datalake_key: "inference.event",
|
||||
event_timestamp: time,
|
||||
event_date: time.slice(0, 10),
|
||||
event_type: string(data, "event_type"),
|
||||
dataset: "zen",
|
||||
cf_continent: string(data, "cf.continent"),
|
||||
cf_country: string(data, "cf.country"),
|
||||
cf_city: string(data, "cf.city"),
|
||||
cf_region: string(data, "cf.region"),
|
||||
cf_latitude: number(data, "cf.latitude"),
|
||||
cf_longitude: number(data, "cf.longitude"),
|
||||
cf_timezone: string(data, "cf.timezone"),
|
||||
duration: number(data, "duration"),
|
||||
request_length: integer(data, "request_length"),
|
||||
status: integer(data, "status"),
|
||||
ip: string(data, "ip"),
|
||||
ip_prefix: string(data, "ip.prefix"),
|
||||
is_stream: boolean(data, "is_stream"),
|
||||
session: string(data, "session"),
|
||||
request: string(data, "request"),
|
||||
client: string(data, "client"),
|
||||
user_agent: string(data, "user_agent"),
|
||||
model: string(data, "model"),
|
||||
model_tier: string(data, "model.tier"),
|
||||
model_variant: string(data, "model.variant"),
|
||||
source: string(data, "source"),
|
||||
provider: string(data, "provider"),
|
||||
provider_model: string(data, "provider.model"),
|
||||
llm_error_code: integer(data, "llm.error.code"),
|
||||
llm_error_message: string(data, "llm.error.message"),
|
||||
error_response: string(data, "error.response"),
|
||||
error_type: string(data, "error.type"),
|
||||
error_message: string(data, "error.message"),
|
||||
error_cause: string(data, "error.cause"),
|
||||
error_cause2: string(data, "error.cause2"),
|
||||
api_key: string(data, "api_key"),
|
||||
workspace: string(data, "workspace"),
|
||||
user_id: string(data, "user_id"),
|
||||
is_subscription: boolean(data, "isSubscription"), // removed
|
||||
subscription: string(data, "subscription"),
|
||||
response_length: integer(data, "response_length"),
|
||||
time_to_first_byte: integer(data, "time_to_first_byte"),
|
||||
timestamp_first_byte: integer(data, "timestamp.first_byte"),
|
||||
timestamp_last_byte: integer(data, "timestamp.last_byte"),
|
||||
tokens_input: integer(data, "tokens.input"),
|
||||
tokens_output: integer(data, "tokens.output"),
|
||||
tokens_reasoning: integer(data, "tokens.reasoning"),
|
||||
tokens_cache_read: integer(data, "tokens.cache_read"),
|
||||
tokens_cache_write_5m: integer(data, "tokens.cache_write_5m"),
|
||||
tokens_cache_write_1h: integer(data, "tokens.cache_write_1h"),
|
||||
cost_input_microcents: integer(data, "cost.input.microcents"),
|
||||
cost_output_microcents: integer(data, "cost.output.microcents"),
|
||||
cost_cache_read_microcents: integer(data, "cost.cache_read.microcents"),
|
||||
cost_cache_write_microcents: integer(data, "cost.cache_write.microcents"),
|
||||
cost_total_microcents: integer(data, "cost.total.microcents"),
|
||||
}
|
||||
}
|
||||
|
||||
// Returns a stable lookup key for an IP address.
|
||||
// IPv4: full address as /32 (e.g. "203.0.113.45/32").
|
||||
// IPv6: the /64 network prefix (e.g. "2001:db8:abcd:1234::/64"). ISPs commonly
|
||||
@@ -179,33 +93,3 @@ function ipPrefix(ip: string | undefined) {
|
||||
.join(":")
|
||||
return `${prefix}::/64`
|
||||
}
|
||||
|
||||
function string(data: Record<string, unknown>, key: string) {
|
||||
const value = data[key]
|
||||
if (typeof value === "string") return value
|
||||
if (typeof value === "number" || typeof value === "boolean") return String(value)
|
||||
return undefined
|
||||
}
|
||||
|
||||
function boolean(data: Record<string, unknown>, key: string) {
|
||||
const value = data[key]
|
||||
if (typeof value === "boolean") return value
|
||||
if (typeof value === "string") return value === "true" ? true : value === "false" ? false : undefined
|
||||
return undefined
|
||||
}
|
||||
|
||||
function integer(data: Record<string, unknown>, key: string) {
|
||||
const value = number(data, key)
|
||||
if (value === undefined) return undefined
|
||||
return Math.round(value)
|
||||
}
|
||||
|
||||
function number(data: Record<string, unknown>, key: string) {
|
||||
const value = data[key]
|
||||
if (typeof value === "number") return Number.isFinite(value) ? value : undefined
|
||||
if (typeof value === "string") {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) ? parsed : undefined
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -48,6 +48,7 @@ import {
|
||||
} from "../lib/comparison-pages"
|
||||
import { baseUrl } from "../lib/language"
|
||||
import { runStatsEffect } from "../stats-runtime"
|
||||
import { NotFoundMeta } from "./not-found-meta"
|
||||
|
||||
const compareHeaderLinks: readonly HeaderLink[] = [
|
||||
{ href: `${import.meta.env.BASE_URL}#top-models`, label: "Top Models" },
|
||||
@@ -235,20 +236,28 @@ export default function ModelCompareDetailPage(props: ModelCompareDetailPageProp
|
||||
|
||||
return (
|
||||
<main data-page="stats" data-layout="compare-detail" data-theme={themePreference()}>
|
||||
<Show when={catalog() !== undefined}>
|
||||
{/* Server-rendered head tags are never removed, so render them once all data has loaded. */}
|
||||
<Show when={catalog() !== undefined && stats() !== undefined}>
|
||||
<Title>{title()}</Title>
|
||||
<Meta name="description" content={description()} />
|
||||
<Meta name="robots" content={models().length > 2 ? "noindex,follow" : "index,follow"} />
|
||||
<Link rel="canonical" href={canonicalUrl()} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={title()} />
|
||||
<Meta property="og:description" content={description()} />
|
||||
<Meta property="og:url" content={canonicalUrl()} />
|
||||
<Meta name="twitter:card" content="summary" />
|
||||
<Meta name="twitter:title" content={title()} />
|
||||
<Meta name="twitter:description" content={description()} />
|
||||
<script type="application/ld+json">{structuredData()}</script>
|
||||
<Show
|
||||
when={models()
|
||||
.slice(0, 2)
|
||||
.every((model) => model.catalog || model.stats)}
|
||||
fallback={<NotFoundMeta unavailable={catalog()?.models.length === 0} />}
|
||||
>
|
||||
<Meta name="robots" content={models().length > 2 ? "noindex,follow" : "index,follow"} />
|
||||
<Link rel="canonical" href={canonicalUrl()} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={title()} />
|
||||
<Meta property="og:description" content={description()} />
|
||||
<Meta property="og:url" content={canonicalUrl()} />
|
||||
<Meta name="twitter:card" content="summary" />
|
||||
<Meta name="twitter:title" content={title()} />
|
||||
<Meta name="twitter:description" content={description()} />
|
||||
<script type="application/ld+json">{structuredData()}</script>
|
||||
</Show>
|
||||
</Show>
|
||||
<Header
|
||||
githubStars={githubStars() ?? githubLink.fallbackStars}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
import { Meta } from "@solidjs/meta"
|
||||
import { HttpStatusCode } from "@solidjs/start"
|
||||
|
||||
// Use 503 when upstream data failed to load so crawlers retry instead of dropping the URL.
|
||||
export function NotFoundMeta(props: { unavailable?: boolean }) {
|
||||
return (
|
||||
<>
|
||||
<HttpStatusCode code={props.unavailable ? 503 : 404} />
|
||||
<Meta name="robots" content="noindex,follow" />
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Title } from "@solidjs/meta"
|
||||
import { NotFoundMeta } from "../component/not-found-meta"
|
||||
import { useLanguage } from "../context/language"
|
||||
|
||||
export default function NotFound() {
|
||||
const language = useLanguage()
|
||||
return (
|
||||
<main data-page="stats">
|
||||
<Title>Page not found</Title>
|
||||
<NotFoundMeta />
|
||||
<div data-component="empty-state">
|
||||
<strong>Page not found</strong>
|
||||
<p>This data page doesn't exist.</p>
|
||||
<a href={language.route(import.meta.env.BASE_URL)}>Model data</a>
|
||||
</div>
|
||||
</main>
|
||||
)
|
||||
}
|
||||
@@ -14,10 +14,12 @@ import { createAsync, query, useParams } from "@solidjs/router"
|
||||
import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js"
|
||||
import { getRequestEvent } from "solid-js/web"
|
||||
import { LocaleLinks } from "../../component/locale-links"
|
||||
import { NotFoundMeta } from "../../component/not-found-meta"
|
||||
import { useI18n } from "../../context/i18n"
|
||||
import { useLanguage } from "../../context/language"
|
||||
import { localizedUrl } from "../../lib/language"
|
||||
import {
|
||||
catalogModelPath,
|
||||
findModelCatalogEntry,
|
||||
formatCatalogLabName,
|
||||
isKnownCatalogLab,
|
||||
@@ -47,7 +49,7 @@ import {
|
||||
} from "../stats-shell"
|
||||
|
||||
const statsUnfurlPath = "banner.png"
|
||||
const glmFlashCatalogId = "zhipuai/glm-5.3-flash"
|
||||
const glmFlashPath = "/data/zhipuai/glm-5-3-flash"
|
||||
const glmFlashModel = "glm-5.3-flash"
|
||||
const shortMonths = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"] as const
|
||||
|
||||
@@ -110,10 +112,10 @@ export default function StatsModel() {
|
||||
const modelTitle = createMemo(() => i18n.t("model.title", { model: searchModelName() }))
|
||||
const modelDescription = createMemo(() => i18n.t("model.description", { model: searchModelName() }))
|
||||
const modelPath = createMemo(() => {
|
||||
const fallback = formerName()
|
||||
? glmFlashCatalogId
|
||||
: [labParam(), stats()?.slug ?? canonicalModel()].filter((part) => part.length > 0).join("/")
|
||||
return `/data/${catalogEntry()?.id ?? fallback}`
|
||||
const entry = catalogEntry()
|
||||
if (entry) return catalogModelPath(entry)
|
||||
if (formerName()) return glmFlashPath
|
||||
return `/data/${providerSlug(labParam())}/${stats()?.slug ?? providerSlug(canonicalModel())}`
|
||||
})
|
||||
const modelUrl = createMemo(() => localizedUrl(language.locale(), modelPath()))
|
||||
const statsUnfurlUrl = new URL(statsUnfurlPath, localizedUrl("en", "/data/")).toString()
|
||||
@@ -152,24 +154,32 @@ export default function StatsModel() {
|
||||
|
||||
return (
|
||||
<main data-page="stats" data-theme={themePreference()}>
|
||||
<Title>{modelTitle()}</Title>
|
||||
<Meta name="description" content={modelDescription()} />
|
||||
<LocaleLinks path={modelPath()} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={modelTitle()} />
|
||||
<Meta property="og:description" content={modelDescription()} />
|
||||
<Meta property="og:url" content={modelUrl()} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/png" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={modelTitle()} />
|
||||
<Meta name="twitter:description" content={modelDescription()} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
{/* Server-rendered head tags are never removed, so render them once data has loaded. */}
|
||||
<Show when={page()}>
|
||||
<Title>{modelTitle()}</Title>
|
||||
<Meta name="description" content={modelDescription()} />
|
||||
<Show
|
||||
when={catalogEntry() || stats()}
|
||||
fallback={<NotFoundMeta unavailable={page()?.catalog.labs.length === 0} />}
|
||||
>
|
||||
<LocaleLinks path={modelPath()} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={modelTitle()} />
|
||||
<Meta property="og:description" content={modelDescription()} />
|
||||
<Meta property="og:url" content={modelUrl()} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/png" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={modelTitle()} />
|
||||
<Meta name="twitter:description" content={modelDescription()} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
</Show>
|
||||
</Show>
|
||||
<Header
|
||||
githubStars={githubStars() ?? githubLink.fallbackStars}
|
||||
links={modelHeaderLinks()}
|
||||
@@ -325,7 +335,7 @@ function ModelHero(props: {
|
||||
current
|
||||
label={modelName()}
|
||||
options={labModels().map((model) => ({
|
||||
href: language.route(`${import.meta.env.BASE_URL}${model.id}`),
|
||||
href: language.route(catalogModelPath(model)),
|
||||
label: model.name,
|
||||
value: model.id,
|
||||
}))}
|
||||
|
||||
@@ -12,10 +12,12 @@ import { createAsync, query, useParams } from "@solidjs/router"
|
||||
import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js"
|
||||
import { getRequestEvent } from "solid-js/web"
|
||||
import { LocaleLinks } from "../../component/locale-links"
|
||||
import { NotFoundMeta } from "../../component/not-found-meta"
|
||||
import { useI18n } from "../../context/i18n"
|
||||
import { useLanguage } from "../../context/language"
|
||||
import { localizedUrl } from "../../lib/language"
|
||||
import {
|
||||
catalogModelPath,
|
||||
catalogSlug,
|
||||
findModelCatalogLab,
|
||||
formatCatalogLabName,
|
||||
@@ -124,24 +126,29 @@ export default function StatsLab() {
|
||||
|
||||
return (
|
||||
<main data-page="stats" data-theme={themePreference()}>
|
||||
<Title>{labTitle()}</Title>
|
||||
<Meta name="description" content={labDescription()} />
|
||||
<LocaleLinks path={labPath()} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={labTitle()} />
|
||||
<Meta property="og:description" content={labDescription()} />
|
||||
<Meta property="og:url" content={labUrl()} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/png" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={labTitle()} />
|
||||
<Meta name="twitter:description" content={labDescription()} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
{/* Server-rendered head tags are never removed, so render them once data has loaded. */}
|
||||
<Show when={page()}>
|
||||
<Title>{labTitle()}</Title>
|
||||
<Meta name="description" content={labDescription()} />
|
||||
<Show when={lab()} fallback={<NotFoundMeta unavailable={page()?.labs.length === 0} />}>
|
||||
<LocaleLinks path={labPath()} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={labTitle()} />
|
||||
<Meta property="og:description" content={labDescription()} />
|
||||
<Meta property="og:url" content={labUrl()} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/png" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={labTitle()} />
|
||||
<Meta name="twitter:description" content={labDescription()} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
</Show>
|
||||
</Show>
|
||||
<Header
|
||||
githubStars={githubStars() ?? githubLink.fallbackStars}
|
||||
links={labHeaderLinks()}
|
||||
@@ -595,7 +602,7 @@ function LabModelRow(props: {
|
||||
return (
|
||||
<a
|
||||
data-component="lab-model-row"
|
||||
href={language.route(`${import.meta.env.BASE_URL}${props.model.id}`)}
|
||||
href={language.route(catalogModelPath(props.model))}
|
||||
role="row"
|
||||
aria-label={props.model.name}
|
||||
onBlur={() => props.onTooltipChange(undefined)}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { Meta, Title } from "@solidjs/meta"
|
||||
import { Title } from "@solidjs/meta"
|
||||
import { createAsync, useParams } from "@solidjs/router"
|
||||
import { createMemo, Show } from "solid-js"
|
||||
import ModelCompareDetailPage from "../../../component/model-compare-detail"
|
||||
import { NotFoundMeta } from "../../../component/not-found-meta"
|
||||
import { resolveComparisonFamily } from "../../../lib/comparison-pages"
|
||||
import { getModelCatalog } from "../../model-catalog"
|
||||
|
||||
@@ -23,7 +24,7 @@ export default function ModelCompareFamily() {
|
||||
fallback={
|
||||
<Show when={comparison() === null}>
|
||||
<Title>Model comparison not found</Title>
|
||||
<Meta name="robots" content="noindex,follow" />
|
||||
<NotFoundMeta unavailable={catalog()?.models.length === 0} />
|
||||
<main data-page="stats">
|
||||
<div data-component="empty-state">
|
||||
<strong>Comparison not found</strong>
|
||||
|
||||
@@ -108,24 +108,27 @@ export default function ModelCompareIndex() {
|
||||
|
||||
return (
|
||||
<main data-page="stats" data-theme={themePreference()}>
|
||||
<Title>{compareTitle}</Title>
|
||||
<Meta name="description" content={compareDescription} />
|
||||
<LocaleLinks path={comparePath} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={compareTitle} />
|
||||
<Meta property="og:description" content={compareDescription} />
|
||||
<Meta property="og:url" content={compareUrl()} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/png" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={compareTitle} />
|
||||
<Meta name="twitter:description" content={compareDescription} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
{/* Server-rendered head tags are never removed, so render them once data has loaded. */}
|
||||
<Show when={catalog()}>
|
||||
<Title>{compareTitle}</Title>
|
||||
<Meta name="description" content={compareDescription} />
|
||||
<LocaleLinks path={comparePath} />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={compareTitle} />
|
||||
<Meta property="og:description" content={compareDescription} />
|
||||
<Meta property="og:url" content={compareUrl()} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/png" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={compareTitle} />
|
||||
<Meta name="twitter:description" content={compareDescription} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
</Show>
|
||||
<Header
|
||||
githubStars={githubStars() ?? githubLink.fallbackStars}
|
||||
links={compareHeaderLinks()}
|
||||
|
||||
@@ -124,24 +124,27 @@ export default function StatsHome() {
|
||||
|
||||
return (
|
||||
<main data-page="stats" data-theme={themePreference()}>
|
||||
<Title>{i18n.t("app.title")}</Title>
|
||||
<Meta name="description" content={i18n.t("app.description")} />
|
||||
<LocaleLinks path="/data/" />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={i18n.t("app.title")} />
|
||||
<Meta property="og:description" content={i18n.t("app.description")} />
|
||||
<Meta property="og:url" content={statsHomeUrl} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/jpeg" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={i18n.t("app.title")} />
|
||||
<Meta name="twitter:description" content={i18n.t("app.description")} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
{/* Server-rendered head tags are never removed, so render them once data has loaded. */}
|
||||
<Show when={data()}>
|
||||
<Title>{i18n.t("app.title")}</Title>
|
||||
<Meta name="description" content={i18n.t("app.description")} />
|
||||
<LocaleLinks path="/data/" />
|
||||
<Meta property="og:type" content="website" />
|
||||
<Meta property="og:site_name" content="OpenCode" />
|
||||
<Meta property="og:title" content={i18n.t("app.title")} />
|
||||
<Meta property="og:description" content={i18n.t("app.description")} />
|
||||
<Meta property="og:url" content={statsHomeUrl} />
|
||||
<Meta property="og:image" content={statsUnfurlUrl} />
|
||||
<Meta property="og:image:type" content="image/jpeg" />
|
||||
<Meta property="og:image:width" content="1200" />
|
||||
<Meta property="og:image:height" content="630" />
|
||||
<Meta property="og:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
<Meta name="twitter:card" content="summary_large_image" />
|
||||
<Meta name="twitter:title" content={i18n.t("app.title")} />
|
||||
<Meta name="twitter:description" content={i18n.t("app.description")} />
|
||||
<Meta name="twitter:image" content={statsUnfurlUrl} />
|
||||
<Meta name="twitter:image:alt" content={i18n.t("app.unfurlAlt")} />
|
||||
</Show>
|
||||
<Header githubStars={githubStars() ?? githubLink.fallbackStars} />
|
||||
<div data-component="container">
|
||||
<div data-component="content">
|
||||
|
||||
@@ -91,6 +91,10 @@ export function findModelCatalogLab(catalog: ModelCatalog, lab: string) {
|
||||
return catalog.labs.find((entry) => entry.id === id)
|
||||
}
|
||||
|
||||
export function catalogModelPath(entry: Pick<ModelCatalogEntry, "lab" | "slug">) {
|
||||
return `/data/${entry.lab}/${entry.slug}`
|
||||
}
|
||||
|
||||
export function formatCatalogLabName(lab: string) {
|
||||
const known: Record<string, string> = {
|
||||
alibaba: "Alibaba",
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
resolveComparisonFamily,
|
||||
} from "../lib/comparison-pages"
|
||||
import { baseUrl } from "../lib/language"
|
||||
import { loadModelCatalog } from "./model-catalog"
|
||||
import { catalogModelPath, findModelCatalogEntry, loadModelCatalog } from "./model-catalog"
|
||||
|
||||
type SitemapEntry = {
|
||||
path: string
|
||||
@@ -55,7 +55,25 @@ export async function GET() {
|
||||
]
|
||||
}),
|
||||
)
|
||||
const entries = uniqueSitemapEntries([{ path: "/data/compare", lastmod }, ...familyComparisons, ...modelComparisons])
|
||||
const labPages = catalog.labs.map((lab) => ({
|
||||
path: `/data/${lab.id}`,
|
||||
lastmod: sitemapDate(stats?.updatedAt, ...lab.models.map((model) => model.lastUpdated ?? model.releaseDate)),
|
||||
}))
|
||||
// Skip entries whose URL resolves to another model, such as stats aliases.
|
||||
const modelPages = [...catalog.models, ...(catalog.aliases ?? [])]
|
||||
.filter((model) => findModelCatalogEntry(catalog, model.slug, model.lab)?.id === model.id)
|
||||
.map((model) => ({
|
||||
path: catalogModelPath(model),
|
||||
lastmod: sitemapDate(stats?.updatedAt, model.lastUpdated ?? model.releaseDate),
|
||||
}))
|
||||
const entries = uniqueSitemapEntries([
|
||||
{ path: "/data/", lastmod: sitemapDate(stats?.updatedAt) },
|
||||
{ path: "/data/compare", lastmod },
|
||||
...labPages,
|
||||
...modelPages,
|
||||
...familyComparisons,
|
||||
...modelComparisons,
|
||||
])
|
||||
|
||||
return new Response(sitemapXml(entries), {
|
||||
headers: {
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
"license": "MIT",
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./athena": "./src/athena.ts",
|
||||
"./config": "./src/config.ts",
|
||||
"./database": "./src/database.ts",
|
||||
"./database/*": "./src/database/*.ts",
|
||||
@@ -27,7 +26,6 @@
|
||||
"typecheck": "tsgo --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-athena": "3.933.0",
|
||||
"@planetscale/database": "1.19.0",
|
||||
"drizzle-orm": "catalog:",
|
||||
"effect": "catalog:",
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
import {
|
||||
AthenaClient as AwsAthenaClient,
|
||||
GetQueryExecutionCommand,
|
||||
GetQueryResultsCommand,
|
||||
StartQueryExecutionCommand,
|
||||
type Row,
|
||||
} from "@aws-sdk/client-athena"
|
||||
import { Effect, Layer } from "effect"
|
||||
import * as Context from "effect/Context"
|
||||
import { Resource } from "sst/resource"
|
||||
|
||||
const ATHENA_MAX_POLL_ATTEMPTS = 900
|
||||
const ATHENA_PAGE_SIZE = 1000
|
||||
|
||||
export type AthenaData = Record<string, string>
|
||||
|
||||
export class AthenaQueryError extends Error {
|
||||
readonly _tag = "AthenaQueryError"
|
||||
readonly queryExecutionId?: string
|
||||
|
||||
constructor(input: { message: string; queryExecutionId?: string; cause?: unknown }) {
|
||||
super(input.message, { cause: input.cause })
|
||||
this.name = "AthenaQueryError"
|
||||
this.queryExecutionId = input.queryExecutionId
|
||||
}
|
||||
}
|
||||
|
||||
export class AthenaQueryTimeoutError extends Error {
|
||||
readonly _tag = "AthenaQueryTimeoutError"
|
||||
readonly queryExecutionId: string
|
||||
|
||||
constructor(input: { message: string; queryExecutionId: string }) {
|
||||
super(input.message)
|
||||
this.name = "AthenaQueryTimeoutError"
|
||||
this.queryExecutionId = input.queryExecutionId
|
||||
}
|
||||
}
|
||||
|
||||
export declare namespace Athena {
|
||||
export interface Service {
|
||||
readonly query: (query: string) => Effect.Effect<AthenaData[], AthenaQueryError | AthenaQueryTimeoutError>
|
||||
}
|
||||
}
|
||||
|
||||
export class Athena extends Context.Service<Athena, Athena.Service>()("@opencode/stats/Athena") {
|
||||
static readonly layer: Layer.Layer<Athena> = Layer.effect(
|
||||
Athena,
|
||||
Effect.sync(() => {
|
||||
const client = new AwsAthenaClient({ region: Resource.InferenceEvent.region })
|
||||
|
||||
const query = Effect.fn("Athena.query")(function* (query: string) {
|
||||
const started = yield* Effect.tryPromise({
|
||||
try: () =>
|
||||
client.send(
|
||||
new StartQueryExecutionCommand({
|
||||
QueryString: query,
|
||||
WorkGroup: Resource.InferenceEvent.workgroup,
|
||||
QueryExecutionContext: {
|
||||
Catalog: Resource.InferenceEvent.catalog,
|
||||
Database: Resource.InferenceEvent.database,
|
||||
},
|
||||
}),
|
||||
),
|
||||
catch: (cause) => new AthenaQueryError({ message: "Failed to start Athena stats query", cause }),
|
||||
})
|
||||
const queryExecutionId = started.QueryExecutionId
|
||||
if (!queryExecutionId)
|
||||
return yield* Effect.fail(new AthenaQueryError({ message: "Athena did not return a query execution id" }))
|
||||
|
||||
yield* poll(client, queryExecutionId)
|
||||
return yield* results(client, queryExecutionId)
|
||||
})
|
||||
|
||||
return Athena.of({ query })
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const poll: (
|
||||
client: AwsAthenaClient,
|
||||
queryExecutionId: string,
|
||||
attempt?: number,
|
||||
) => Effect.Effect<void, AthenaQueryError | AthenaQueryTimeoutError> = Effect.fn("Athena.poll")(function* (
|
||||
client: AwsAthenaClient,
|
||||
queryExecutionId: string,
|
||||
attempt = 0,
|
||||
) {
|
||||
if (attempt > 0) yield* Effect.sleep("2 seconds")
|
||||
|
||||
const result = yield* Effect.tryPromise({
|
||||
try: () => client.send(new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId })),
|
||||
catch: (cause) => new AthenaQueryError({ message: "Failed to poll Athena stats query", queryExecutionId, cause }),
|
||||
})
|
||||
const status = result.QueryExecution?.Status
|
||||
|
||||
if (status?.State === "SUCCEEDED") return
|
||||
if (status?.State === "FAILED" || status?.State === "CANCELLED")
|
||||
return yield* Effect.fail(
|
||||
new AthenaQueryError({
|
||||
message: `Athena stats query ${status.State.toLowerCase()}: ${status.StateChangeReason ?? "unknown reason"}`,
|
||||
queryExecutionId,
|
||||
}),
|
||||
)
|
||||
|
||||
if (attempt >= ATHENA_MAX_POLL_ATTEMPTS - 1)
|
||||
return yield* Effect.fail(
|
||||
new AthenaQueryTimeoutError({
|
||||
message: `Athena stats query ${queryExecutionId} did not complete`,
|
||||
queryExecutionId,
|
||||
}),
|
||||
)
|
||||
|
||||
return yield* poll(client, queryExecutionId, attempt + 1)
|
||||
})
|
||||
|
||||
const results: (client: AwsAthenaClient, queryExecutionId: string) => Effect.Effect<AthenaData[], AthenaQueryError> =
|
||||
Effect.fn("Athena.results")(function* (client: AwsAthenaClient, queryExecutionId: string) {
|
||||
// Accumulate pages iteratively; recursive spreads copied every previously
|
||||
// fetched row per page and blew up memory on large result sets.
|
||||
const rows: AthenaData[] = []
|
||||
let nextToken: string | undefined
|
||||
while (true) {
|
||||
const result = yield* Effect.tryPromise({
|
||||
try: () =>
|
||||
client.send(
|
||||
new GetQueryResultsCommand({
|
||||
QueryExecutionId: queryExecutionId,
|
||||
NextToken: nextToken,
|
||||
MaxResults: ATHENA_PAGE_SIZE,
|
||||
}),
|
||||
),
|
||||
catch: (cause) =>
|
||||
new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }),
|
||||
})
|
||||
const columns = result.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((item) => item.Name ?? "") ?? []
|
||||
// The first page starts with the header row.
|
||||
for (const row of (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1)) rows.push(rowData(columns, row))
|
||||
if (!result.NextToken) return rows
|
||||
nextToken = result.NextToken
|
||||
}
|
||||
})
|
||||
|
||||
function rowData(columns: string[], row: Row): AthenaData {
|
||||
return Object.fromEntries(
|
||||
columns.flatMap((column, index) => {
|
||||
const value = row.Data?.[index]?.VarCharValue
|
||||
if (!column || value === undefined) return []
|
||||
return [[column, value]]
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
export * as Athena from "./athena"
|
||||
export * as AppConfig from "./config"
|
||||
export * as Database from "./database"
|
||||
export * as GeoStat from "./domain/geo"
|
||||
|
||||
-9
@@ -2,15 +2,6 @@ import "sst/resource"
|
||||
|
||||
declare module "sst/resource" {
|
||||
export interface Resource {
|
||||
InferenceEvent: {
|
||||
catalog: string
|
||||
database: string
|
||||
region: string
|
||||
table: string
|
||||
tableBucket: string
|
||||
type: "sst.sst.Linkable"
|
||||
workgroup: string
|
||||
}
|
||||
R2Sql: {
|
||||
accountId: string
|
||||
bucket: string
|
||||
|
||||
@@ -29,6 +29,4 @@ COPY --from=pruner /app/out/full/ ./
|
||||
|
||||
WORKDIR /app/packages/stats/server
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
CMD ["bun", "src/server.ts"]
|
||||
CMD ["bun", "src/stat-sync.ts"]
|
||||
@@ -5,16 +5,15 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"main": "./src/server.ts",
|
||||
"main": "./src/stat-sync.ts",
|
||||
"exports": {
|
||||
".": "./src/server.ts"
|
||||
".": "./src/stat-sync.ts"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "bun src/server.ts",
|
||||
"start": "bun src/stat-sync.ts",
|
||||
"typecheck": "tsgo --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-firehose": "3.933.0",
|
||||
"@effect/platform-node": "catalog:",
|
||||
"@opencode-ai/stats-core": "workspace:*",
|
||||
"effect": "catalog:",
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
import { Buffer } from "node:buffer"
|
||||
import { FirehoseClient, PutRecordBatchCommand } from "@aws-sdk/client-firehose"
|
||||
import { Effect, Layer } from "effect"
|
||||
import * as Context from "effect/Context"
|
||||
import { Resource } from "sst/resource"
|
||||
|
||||
const MAX_FIREHOSE_BATCH_SIZE = 500
|
||||
const MAX_FIREHOSE_ATTEMPTS = 3
|
||||
const LAKE_TYPE = /^([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)$/
|
||||
|
||||
type IngestEvent = Record<string, unknown>
|
||||
type LakeRoute = { database: string; table: string }
|
||||
type FirehoseRecord = { Data: Uint8Array }
|
||||
|
||||
export class IngestError extends Error {
|
||||
readonly _tag = "IngestError"
|
||||
readonly failed: number
|
||||
|
||||
constructor(input: { message: string; failed: number; cause?: unknown }) {
|
||||
super(input.message, { cause: input.cause })
|
||||
this.name = "IngestError"
|
||||
this.failed = input.failed
|
||||
}
|
||||
}
|
||||
|
||||
export declare namespace Ingest {
|
||||
export interface Service {
|
||||
readonly write: (events: unknown[]) => Effect.Effect<{ records: number }, IngestError>
|
||||
}
|
||||
}
|
||||
|
||||
export class Ingest extends Context.Service<Ingest, Ingest.Service>()("@opencode/stats/Ingest") {
|
||||
static readonly layer: Layer.Layer<Ingest> = Layer.effect(
|
||||
Ingest,
|
||||
Effect.sync(() => {
|
||||
const client = new FirehoseClient({})
|
||||
|
||||
const write = Effect.fn("Ingest.write")(function* (events: unknown[]) {
|
||||
if (events.length === 0) return { records: 0 }
|
||||
const counts = countRoutedEvents(events)
|
||||
if (counts.unsupported > 0) {
|
||||
yield* Effect.logWarning(
|
||||
`lake ingest rejected ${JSON.stringify({ records: counts.records, unsupported: counts.unsupported })}`,
|
||||
)
|
||||
return yield* Effect.fail(
|
||||
new IngestError({
|
||||
message: "Unsupported lake event type",
|
||||
failed: counts.unsupported,
|
||||
}),
|
||||
)
|
||||
}
|
||||
if (counts.records === 0) return { records: 0 }
|
||||
|
||||
let batch: FirehoseRecord[] = []
|
||||
let batches = 0
|
||||
let failed = 0
|
||||
|
||||
for (const event of events) {
|
||||
if (!isRecord(event)) continue
|
||||
const route = routeEvent(event)
|
||||
if (!route) continue
|
||||
batch.push(toFirehoseRecord(event, route))
|
||||
if (batch.length < MAX_FIREHOSE_BATCH_SIZE) continue
|
||||
failed += yield* putRecords(client, Resource.LakeIngestConfig.streamName, batch)
|
||||
batches++
|
||||
batch = []
|
||||
}
|
||||
|
||||
if (batch.length > 0) {
|
||||
failed += yield* putRecords(client, Resource.LakeIngestConfig.streamName, batch)
|
||||
batches++
|
||||
}
|
||||
|
||||
if (failed > 0) {
|
||||
yield* Effect.logWarning(`lake ingest incomplete ${JSON.stringify({ records: counts.records, failed })}`)
|
||||
return yield* Effect.fail(new IngestError({ message: "Failed to ingest all lake records", failed }))
|
||||
}
|
||||
|
||||
yield* Effect.logInfo(`lake ingest complete ${JSON.stringify({ records: counts.records, batches })}`)
|
||||
return { records: counts.records }
|
||||
})
|
||||
|
||||
return Ingest.of({ write })
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
const putRecords: (
|
||||
client: FirehoseClient,
|
||||
streamName: string,
|
||||
records: FirehoseRecord[],
|
||||
attempt?: number,
|
||||
) => Effect.Effect<number, IngestError> = Effect.fn("Ingest.putRecords")(function* (
|
||||
client,
|
||||
streamName,
|
||||
records,
|
||||
attempt = 1,
|
||||
) {
|
||||
const result = yield* Effect.tryPromise({
|
||||
try: () => client.send(new PutRecordBatchCommand({ DeliveryStreamName: streamName, Records: records })),
|
||||
catch: (cause) =>
|
||||
new IngestError({ message: "Failed to write lake records to Firehose", failed: records.length, cause }),
|
||||
}).pipe(
|
||||
Effect.tapError(() =>
|
||||
Effect.logWarning(`firehose batch write failed ${JSON.stringify({ records: records.length, attempt })}`),
|
||||
),
|
||||
)
|
||||
const failed =
|
||||
result.RequestResponses?.flatMap((item, index) => {
|
||||
const record = records[index]
|
||||
if (!item.ErrorCode || !record) return []
|
||||
return [record]
|
||||
}) ?? []
|
||||
|
||||
if (failed.length === 0) return 0
|
||||
if (attempt >= MAX_FIREHOSE_ATTEMPTS) {
|
||||
yield* Effect.logWarning(
|
||||
`firehose batch failed ${JSON.stringify({ records: failed.length, attempts: MAX_FIREHOSE_ATTEMPTS })}`,
|
||||
)
|
||||
return failed.length
|
||||
}
|
||||
|
||||
yield* Effect.logWarning(
|
||||
`firehose batch retrying ${JSON.stringify({ records: failed.length, attempt: attempt + 1 })}`,
|
||||
)
|
||||
yield* Effect.sleep(`${250 * 2 ** (attempt - 1)} millis`)
|
||||
return yield* putRecords(client, streamName, failed, attempt + 1)
|
||||
})
|
||||
|
||||
function countRoutedEvents(events: unknown[]) {
|
||||
let records = 0
|
||||
let unsupported = 0
|
||||
for (const event of events) {
|
||||
if (!isRecord(event)) continue
|
||||
if (routeEvent(event)) records++
|
||||
else unsupported++
|
||||
}
|
||||
return { records, unsupported }
|
||||
}
|
||||
|
||||
function isRecord(item: unknown): item is IngestEvent {
|
||||
return Boolean(item) && typeof item === "object" && !Array.isArray(item)
|
||||
}
|
||||
|
||||
function routeEvent(event: IngestEvent): LakeRoute | undefined {
|
||||
if (typeof event._datalake_key !== "string") return
|
||||
const match = event._datalake_key.match(LAKE_TYPE)
|
||||
if (!match?.[1] || !match[2]) return
|
||||
return {
|
||||
database: match[1],
|
||||
table: match[2],
|
||||
}
|
||||
}
|
||||
|
||||
function toFirehoseRecord(event: IngestEvent, route: LakeRoute): FirehoseRecord {
|
||||
return {
|
||||
Data: Buffer.from(
|
||||
JSON.stringify({
|
||||
...Object.fromEntries(Object.entries(event).filter(([key]) => key !== "_datalake_key")),
|
||||
_lake_database: route.database,
|
||||
_lake_table: route.table,
|
||||
_lake_operation: "insert" as const,
|
||||
}),
|
||||
),
|
||||
}
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
import "sst/resource"
|
||||
|
||||
declare module "sst/resource" {
|
||||
export interface Resource {
|
||||
LakeIngestConfig: {
|
||||
secret: string
|
||||
streamName: string
|
||||
type: "sst.sst.Linkable"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
import { Buffer } from "node:buffer"
|
||||
import { timingSafeEqual } from "node:crypto"
|
||||
import { Effect, Schema } from "effect"
|
||||
import * as Semaphore from "effect/Semaphore"
|
||||
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"
|
||||
import { Resource } from "sst/resource"
|
||||
import { Ingest } from "./ingest"
|
||||
import { isShuttingDown } from "./shutdown"
|
||||
|
||||
const MAX_CONCURRENT_INGEST_REQUESTS = 8
|
||||
|
||||
const IngestPayload = Schema.Struct({
|
||||
events: Schema.optional(Schema.Unknown),
|
||||
})
|
||||
|
||||
export const Routes = HttpRouter.use((router) =>
|
||||
Effect.gen(function* () {
|
||||
const ingestService = yield* Ingest
|
||||
const ingestRequests = yield* Semaphore.make(MAX_CONCURRENT_INGEST_REQUESTS)
|
||||
|
||||
yield* Effect.all(
|
||||
[
|
||||
router.add("GET", "/health", () => json(200, { ok: true })),
|
||||
router.add("GET", "/ready", () => json(isShuttingDown() ? 503 : 200, { ok: !isShuttingDown() })),
|
||||
router.add("POST", "/", ingestRequests.withPermit(ingest(ingestService))),
|
||||
],
|
||||
{ discard: true },
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
const ingest = (ingestService: Ingest.Service) =>
|
||||
Effect.gen(function* () {
|
||||
const request = yield* HttpServerRequest.HttpServerRequest
|
||||
if (!isAuthorized(request.headers)) return yield* json(401, { ok: false, error: "Unauthorized" })
|
||||
|
||||
const payload = yield* HttpServerRequest.schemaBodyJson(IngestPayload).pipe(
|
||||
Effect.match({
|
||||
onFailure: () => undefined,
|
||||
onSuccess: (value) => value,
|
||||
}),
|
||||
)
|
||||
if (!payload) return yield* json(400, { ok: false, error: "Invalid JSON body" })
|
||||
|
||||
const events = Array.isArray(payload.events) ? payload.events : []
|
||||
if (events.length === 0) return yield* json(202, { ok: true, records: 0 })
|
||||
|
||||
return yield* ingestService.write(events).pipe(
|
||||
Effect.flatMap((result) => json(202, { ok: true, records: result.records })),
|
||||
Effect.catchTag("IngestError", (error) =>
|
||||
json(502, { ok: false, records: countRecords(events), failed: error.failed }),
|
||||
),
|
||||
)
|
||||
})
|
||||
|
||||
function isAuthorized(headers: Record<string, string | undefined>) {
|
||||
const actual = Buffer.from(headers.authorization ?? headers.Authorization ?? "")
|
||||
const expected = Buffer.from(`Bearer ${Resource.LakeIngestConfig.secret}`)
|
||||
if (actual.length !== expected.length) return false
|
||||
return timingSafeEqual(actual, expected)
|
||||
}
|
||||
|
||||
function countRecords(items: unknown[]) {
|
||||
let records = 0
|
||||
for (const item of items) {
|
||||
if (Boolean(item) && typeof item === "object" && !Array.isArray(item)) records++
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
function json(status: number, body: Record<string, unknown>) {
|
||||
return HttpServerResponse.json(body, { status }).pipe(Effect.orDie)
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer"
|
||||
import * as NodeRuntime from "@effect/platform-node/NodeRuntime"
|
||||
import { Config, Layer } from "effect"
|
||||
import { HttpRouter } from "effect/unstable/http"
|
||||
import { createServer } from "node:http"
|
||||
import { Ingest } from "./ingest"
|
||||
import { Routes } from "./router"
|
||||
import { registerShutdownSignalHandlers } from "./shutdown"
|
||||
|
||||
registerShutdownSignalHandlers()
|
||||
|
||||
const ServerLive = NodeHttpServer.layerConfig(
|
||||
() => createServer(),
|
||||
Config.all({
|
||||
port: Config.number("PORT").pipe(Config.withDefault(3000)),
|
||||
host: Config.string("HOST").pipe(Config.withDefault("0.0.0.0")),
|
||||
}),
|
||||
)
|
||||
|
||||
const runtimeLayer = Ingest.layer
|
||||
const programLayer = Routes.pipe(Layer.provide(runtimeLayer))
|
||||
const main = Layer.launch(
|
||||
HttpRouter.serve(programLayer, {
|
||||
disableLogger: true,
|
||||
}).pipe(Layer.provideMerge(ServerLive)),
|
||||
)
|
||||
|
||||
NodeRuntime.runMain(main, { disableErrorReporting: true })
|
||||
@@ -1,17 +0,0 @@
|
||||
let shuttingDown = false
|
||||
let signalHandlersRegistered = false
|
||||
|
||||
export function isShuttingDown() {
|
||||
return shuttingDown
|
||||
}
|
||||
|
||||
export function registerShutdownSignalHandlers() {
|
||||
if (signalHandlersRegistered) return
|
||||
signalHandlersRegistered = true
|
||||
process.once("SIGTERM", markShuttingDown)
|
||||
process.once("SIGINT", markShuttingDown)
|
||||
}
|
||||
|
||||
function markShuttingDown() {
|
||||
shuttingDown = true
|
||||
}
|
||||
@@ -30,7 +30,6 @@ export default $config({
|
||||
async run() {
|
||||
const stage = await import("./infra/stage.js")
|
||||
await import("./infra/app.js")
|
||||
const lake = stage.deployAws ? await import("./infra/lake.js") : undefined
|
||||
const stats = stage.deployAws ? await import("./infra/stats.js") : undefined
|
||||
const { stat } = await import("./infra/console.js")
|
||||
await import("./infra/enterprise.js")
|
||||
@@ -41,12 +40,6 @@ export default $config({
|
||||
return {
|
||||
StatWorkerUrl: stat.url,
|
||||
...(stats ? { StatsUrl: stats.app.url } : {}),
|
||||
...(lake
|
||||
? {
|
||||
LakeUrl: lake.lakeIngest.properties.url,
|
||||
LakeSecretSsm: lake.ingestSecretSsm.name,
|
||||
}
|
||||
: {}),
|
||||
AwsStage: stage.awsStage,
|
||||
}
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user