mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
test(qa): record Matrix route state inventory (#100039)
This commit is contained in:
Dallin Romney
+14
-12
@@ -23,12 +23,13 @@ Plain `pnpm openclaw qa matrix` runs `--profile all` and does not stop on first
|
||||
|
||||
## What the lane does
|
||||
|
||||
1. Provisions a disposable Tuwunel homeserver in Docker (default image `ghcr.io/matrix-construct/tuwunel:v1.5.1`, server name `matrix-qa.test`, port `28008`).
|
||||
1. Provisions a disposable Tuwunel homeserver in Docker (default image `ghcr.io/matrix-construct/tuwunel:v1.5.1`, server name `matrix-qa.test`, port `28008`) behind a bounded redacting request/response recorder.
|
||||
2. Registers three temporary users - `driver` (sends inbound traffic), `sut` (the OpenClaw Matrix account under test), `observer` (third-party traffic capture).
|
||||
3. Seeds rooms required by the selected scenarios (main, threading, media, restart, secondary, allowlist, E2EE, verification DM, etc.).
|
||||
4. Starts a child OpenClaw gateway with the real Matrix plugin scoped to the SUT account; `qa-channel` is not loaded in the child.
|
||||
5. Runs scenarios in sequence, observing events through the driver/observer Matrix clients.
|
||||
6. Tears down the homeserver, writes report and summary artifacts, then exits.
|
||||
4. Runs the substrate-neutral `matrix-qa-v1` protocol probe against the recorded Tuwunel boundary. Unit tests prove the probe contract with the Matrix protocol fixture; the canonical QA transport adapter host in [#99707](https://github.com/openclaw/openclaw/pull/99707) owns real Crabline target wiring.
|
||||
5. Starts a child OpenClaw gateway with the real Matrix plugin scoped to the SUT account; `qa-channel` is not loaded in the child.
|
||||
6. Runs scenarios in sequence, observing events through the driver/observer Matrix clients and deriving route/state expectations from the recorded traffic.
|
||||
7. Tears down the homeserver, writes report and evidence artifacts, then exits.
|
||||
|
||||
## CLI
|
||||
|
||||
@@ -38,14 +39,14 @@ pnpm openclaw qa matrix [options]
|
||||
|
||||
### Common flags
|
||||
|
||||
| Flag | Default | Description |
|
||||
| --------------------- | --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--profile <profile>` | `all` | Scenario profile. See [Profiles](#profiles). |
|
||||
| `--fail-fast` | off | Stop after the first failed check or scenario. |
|
||||
| `--scenario <id>` | - | Run only this scenario. Repeatable. See [Scenarios](#scenarios). |
|
||||
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/matrix-<timestamp>` | Where reports, summary, observed events, and the output log are written. Relative paths resolve against `--repo-root`. |
|
||||
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral working directory. |
|
||||
| `--sut-account <id>` | `sut` | Matrix account id inside the QA gateway config. |
|
||||
| Flag | Default | Description |
|
||||
| --------------------- | --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `--profile <profile>` | `all` | Scenario profile. See [Profiles](#profiles). |
|
||||
| `--fail-fast` | off | Stop after the first failed check or scenario. |
|
||||
| `--scenario <id>` | - | Run only this scenario. Repeatable. See [Scenarios](#scenarios). |
|
||||
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/matrix-<timestamp>` | Where reports, summary, route/state inventory, observed events, and the output log are written. Relative paths resolve against `--repo-root`. |
|
||||
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral working directory. |
|
||||
| `--sut-account <id>` | `sut` | Matrix account id inside the QA gateway config. |
|
||||
|
||||
### Provider flags
|
||||
|
||||
@@ -114,6 +115,7 @@ Written to `--output-dir`:
|
||||
|
||||
- `matrix-qa-report.md` - Markdown protocol report (what passed, failed, was skipped, and why).
|
||||
- `matrix-qa-summary.json` - Structured summary suitable for CI parsing and dashboards.
|
||||
- `matrix-qa-route-state-manifest.json` - Dynamic `matrix-qa-v1` inventory keyed by scenario id. It records redacted route/body shapes, request ordering, observed retries, errors, sync-token continuity, and device/key/media/backup state families observed during that run. This is executable evidence, not a checked-in baseline.
|
||||
- `matrix-qa-observed-events.json` - Observed Matrix events from the driver and observer clients. Bodies are redacted unless `OPENCLAW_QA_MATRIX_CAPTURE_CONTENT=1`; approval metadata is summarized with selected safe fields and truncated command preview.
|
||||
- `matrix-qa-output.log` - Combined stdout/stderr from the run. If `OPENCLAW_RUN_NODE_OUTPUT_LOG` is set, the outer launcher's log is reused instead.
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ describe("matrix qa cli runtime", () => {
|
||||
reportPath: "/tmp/matrix-report.md",
|
||||
summaryPath: "/tmp/matrix-summary.json",
|
||||
observedEventsPath: "/tmp/matrix-events.json",
|
||||
routeStateManifestPath: "/tmp/matrix-route-state.json",
|
||||
});
|
||||
const originalStdoutWrite = process.stdout["write"];
|
||||
process.stdout.write = (() => true) as typeof process.stdout.write;
|
||||
@@ -104,6 +105,7 @@ describe("matrix qa cli runtime", () => {
|
||||
reportPath: "/tmp/matrix-report.md",
|
||||
summaryPath: "/tmp/matrix-summary.json",
|
||||
observedEventsPath: "/tmp/matrix-events.json",
|
||||
routeStateManifestPath: "/tmp/matrix-route-state.json",
|
||||
});
|
||||
const originalStdoutWrite = process.stdout["write"];
|
||||
process.stdout.write = vi.fn(() => true) as unknown as typeof process.stdout.write;
|
||||
|
||||
@@ -72,6 +72,7 @@ export async function runQaMatrixCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const result = await runMatrixQaLive(checkedRunOptions);
|
||||
printLiveTransportQaArtifacts("Matrix QA", {
|
||||
report: result.reportPath,
|
||||
"route/state manifest": result.routeStateManifestPath,
|
||||
summary: result.summaryPath,
|
||||
"observed events": result.observedEventsPath,
|
||||
});
|
||||
|
||||
@@ -37,6 +37,7 @@ function buildMatrixQaSummaryInput(
|
||||
artifactPaths: {
|
||||
observedEvents: "/tmp/observed.json",
|
||||
report: "/tmp/report.md",
|
||||
routeStateManifest: "/tmp/route-state.json",
|
||||
summary: "/tmp/summary.json",
|
||||
},
|
||||
checks: [{ name: "Matrix harness ready", status: "pass" }],
|
||||
@@ -53,6 +54,11 @@ function buildMatrixQaSummaryInput(
|
||||
}),
|
||||
scenarios: [],
|
||||
},
|
||||
differentialProbe: {
|
||||
profile: "matrix-qa-v1",
|
||||
steps: [],
|
||||
sync: { continuity: true, incrementalStatus: 200, initialStatus: 200 },
|
||||
},
|
||||
finishedAt: "2026-04-10T10:05:00.000Z",
|
||||
harness: {
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
@@ -78,6 +84,25 @@ function buildMatrixQaSummaryInput(
|
||||
}
|
||||
|
||||
describe("matrix live qa runtime", () => {
|
||||
it("preserves a failed differential probe check without a probe payload", () => {
|
||||
const summary = liveTesting.buildMatrixQaSummary(
|
||||
buildMatrixQaSummaryInput({
|
||||
checks: [
|
||||
{ name: "Matrix harness ready", status: "pass" },
|
||||
{
|
||||
details: "missing-state response did not return M_NOT_FOUND",
|
||||
name: "Matrix differential probe",
|
||||
status: "fail",
|
||||
},
|
||||
],
|
||||
differentialProbe: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(summary.differentialProbe).toBeUndefined();
|
||||
expect(summary.counts).toEqual({ failed: 1, passed: 1, total: 2 });
|
||||
});
|
||||
|
||||
it("uses unique default artifact directories", () => {
|
||||
const repoRoot = "/repo";
|
||||
const firstOutputDir = liveTesting.resolveMatrixQaOutputDir({ repoRoot });
|
||||
@@ -86,9 +111,9 @@ describe("matrix live qa runtime", () => {
|
||||
expect(path.dirname(firstOutputDir)).toBe(path.join(repoRoot, ".artifacts", "qa-e2e"));
|
||||
expect(path.basename(firstOutputDir)).toMatch(/^matrix-[a-z0-9]+-[a-f0-9]{8}$/u);
|
||||
expect(secondOutputDir).not.toBe(firstOutputDir);
|
||||
expect(
|
||||
liveTesting.resolveMatrixQaOutputDir({ outputDir: ".artifacts/custom", repoRoot }),
|
||||
).toBe(".artifacts/custom");
|
||||
expect(liveTesting.resolveMatrixQaOutputDir({ outputDir: ".artifacts/custom", repoRoot })).toBe(
|
||||
".artifacts/custom",
|
||||
);
|
||||
});
|
||||
|
||||
it("prints Matrix QA progress by default for non-interactive runs", () => {
|
||||
@@ -388,6 +413,7 @@ describe("matrix live qa runtime", () => {
|
||||
artifactPaths: {
|
||||
observedEvents: "/tmp/observed.json",
|
||||
report: "/tmp/report.md",
|
||||
routeStateManifest: "/tmp/route-state.json",
|
||||
summary: "/tmp/summary.json",
|
||||
},
|
||||
checks: [{ name: "Matrix harness ready", status: "pass" }],
|
||||
@@ -450,6 +476,11 @@ describe("matrix live qa runtime", () => {
|
||||
},
|
||||
],
|
||||
},
|
||||
differentialProbe: {
|
||||
profile: "matrix-qa-v1",
|
||||
steps: [],
|
||||
sync: { continuity: true, incrementalStatus: 200, initialStatus: 200 },
|
||||
},
|
||||
finishedAt: "2026-04-10T10:05:00.000Z",
|
||||
harness: {
|
||||
baseUrl: "http://127.0.0.1:28008/",
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
type QaReportCheck,
|
||||
} from "openclaw/plugin-sdk/qa-runtime";
|
||||
import { normalizeQaProviderMode, type QaProviderModeInput } from "../../run-config.js";
|
||||
import { createLiveTransportQaRunId } from "../../shared/live-transport-artifacts.js";
|
||||
import { buildMatrixQaObservedEventsArtifact } from "../../substrate/artifacts.js";
|
||||
import { provisionMatrixQaRoom, type MatrixQaProvisionResult } from "../../substrate/client.js";
|
||||
import {
|
||||
@@ -27,9 +28,12 @@ import {
|
||||
type MatrixQaConfigOverrides,
|
||||
type MatrixQaConfigSnapshot,
|
||||
} from "../../substrate/config.js";
|
||||
import {
|
||||
runMatrixQaDifferentialProbe,
|
||||
type MatrixQaDifferentialProbeResult,
|
||||
} from "../../substrate/differential-probe.js";
|
||||
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
|
||||
import { startMatrixQaHarness } from "../../substrate/harness.runtime.js";
|
||||
import { createLiveTransportQaRunId } from "../../shared/live-transport-artifacts.js";
|
||||
import { resolveMatrixQaModels, type ResolvedMatrixQaModels } from "./model-selection.js";
|
||||
import type { MatrixQaSyncStreams } from "./scenario-runtime-shared.js";
|
||||
import {
|
||||
@@ -135,9 +139,11 @@ type MatrixQaSummary = {
|
||||
serverName: string;
|
||||
};
|
||||
canary?: MatrixQaCanaryArtifact;
|
||||
differentialProbe?: MatrixQaDifferentialProbeResult;
|
||||
observedEventCount: number;
|
||||
observedEventsPath: string;
|
||||
reportPath: string;
|
||||
routeStateManifestPath: string;
|
||||
scenarios: MatrixQaScenarioResult[];
|
||||
startedAt: string;
|
||||
summaryPath: string;
|
||||
@@ -153,6 +159,7 @@ type MatrixQaSummary = {
|
||||
type MatrixQaArtifactPaths = {
|
||||
observedEvents: string;
|
||||
report: string;
|
||||
routeStateManifest: string;
|
||||
summary: string;
|
||||
};
|
||||
|
||||
@@ -468,6 +475,7 @@ type MatrixQaRunResult = {
|
||||
observedEventsPath: string;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
routeStateManifestPath: string;
|
||||
scenarios: MatrixQaScenarioResult[];
|
||||
summaryPath: string;
|
||||
};
|
||||
@@ -477,6 +485,7 @@ function buildMatrixQaSummary(params: {
|
||||
canary?: MatrixQaCanaryArtifact;
|
||||
checks: QaReportCheck[];
|
||||
config: MatrixQaSummary["config"];
|
||||
differentialProbe?: MatrixQaDifferentialProbeResult;
|
||||
finishedAt: string;
|
||||
harness: MatrixQaSummary["harness"];
|
||||
observedEventCount: number;
|
||||
@@ -500,9 +509,11 @@ function buildMatrixQaSummary(params: {
|
||||
finishedAt: params.finishedAt,
|
||||
harness: params.harness,
|
||||
canary: params.canary,
|
||||
differentialProbe: params.differentialProbe,
|
||||
observedEventCount: params.observedEventCount,
|
||||
observedEventsPath: params.artifactPaths.observedEvents,
|
||||
reportPath: params.artifactPaths.report,
|
||||
routeStateManifestPath: params.artifactPaths.routeStateManifest,
|
||||
scenarios: params.scenarios,
|
||||
startedAt: params.startedAt,
|
||||
summaryPath: params.artifactPaths.summary,
|
||||
@@ -729,7 +740,6 @@ export async function runMatrixQaLive(params: {
|
||||
writeMatrixQaProgress(
|
||||
`topology ready ${formatMatrixQaDurationMs(provisioningMs)} rooms=${provisioning.topology.rooms.length}`,
|
||||
);
|
||||
|
||||
const checks: QaReportCheck[] = [
|
||||
{
|
||||
name: "Matrix harness ready",
|
||||
@@ -743,6 +753,28 @@ export async function runMatrixQaLive(params: {
|
||||
].join("\n"),
|
||||
},
|
||||
];
|
||||
harness.recording.setScenarioId("matrix-qa-v1-probe");
|
||||
let differentialProbe: MatrixQaDifferentialProbeResult | undefined;
|
||||
try {
|
||||
differentialProbe = await withMatrixQaRunDeadline(
|
||||
runDeadline,
|
||||
"Matrix differential probe",
|
||||
() =>
|
||||
runMatrixQaDifferentialProbe({
|
||||
accessToken: provisioning.driver.accessToken,
|
||||
baseUrl: harness.baseUrl,
|
||||
roomId: provisioning.roomId,
|
||||
userId: provisioning.driver.userId,
|
||||
}),
|
||||
);
|
||||
checks.push({ name: "Matrix differential probe", status: "pass" });
|
||||
} catch (error) {
|
||||
checks.push({
|
||||
details: error instanceof Error ? error.message : String(error),
|
||||
name: "Matrix differential probe",
|
||||
status: "fail",
|
||||
});
|
||||
}
|
||||
const scenarioResults: Array<MatrixQaScenarioResult | undefined> = Array.from({
|
||||
length: scenarios.length,
|
||||
});
|
||||
@@ -755,7 +787,7 @@ export async function runMatrixQaLive(params: {
|
||||
const syncState: { driver?: string; observer?: string } = {};
|
||||
const syncStreams: MatrixQaSyncStreams = {};
|
||||
let canaryMs: number | undefined;
|
||||
let initialGatewayBootMs;
|
||||
let initialGatewayBootMs = 0;
|
||||
let scenarioGatewayBootMs = 0;
|
||||
let scenarioRestartGatewayMs = 0;
|
||||
let scenarioTransportInterruptMs = 0;
|
||||
@@ -777,7 +809,12 @@ export async function runMatrixQaLive(params: {
|
||||
|
||||
const scheduledScenarios = scheduleMatrixQaScenariosInCatalogOrder(scenarios);
|
||||
|
||||
try {
|
||||
matrixQaExecution: try {
|
||||
if (params.failFast && !differentialProbe) {
|
||||
writeMatrixQaProgress("fail-fast stop");
|
||||
break matrixQaExecution;
|
||||
}
|
||||
harness.recording.setScenarioId("canary");
|
||||
const ensureGatewayHarness = async (selection: MatrixQaGatewaySelection = {}) => {
|
||||
const models = resolveMatrixQaGatewayModels({
|
||||
defaultModels,
|
||||
@@ -890,6 +927,7 @@ export async function runMatrixQaLive(params: {
|
||||
|
||||
if (!canaryFailed) {
|
||||
for (const { scenario, originalIndex } of scheduledScenarios) {
|
||||
harness.recording.setScenarioId(scenario.id);
|
||||
const { entry: scenarioConfigEntry, summary: scenarioConfigSummary } =
|
||||
buildMatrixQaScenarioConfigEntry({
|
||||
gatewayConfigParams,
|
||||
@@ -916,6 +954,8 @@ export async function runMatrixQaLive(params: {
|
||||
driverDeviceId: provisioning.driver.deviceId,
|
||||
driverPassword: provisioning.driver.password,
|
||||
driverUserId: provisioning.driver.userId,
|
||||
faultProxyObserver: harness.recording,
|
||||
faultProxyTargetBaseUrl: harness.upstreamBaseUrl,
|
||||
interruptTransport: async () => {
|
||||
writeMatrixQaProgress(`transport interrupt start ${scenario.id}`);
|
||||
const measuredInterrupt = await measureMatrixQaStep(async () => {
|
||||
@@ -1079,6 +1119,7 @@ export async function runMatrixQaLive(params: {
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
harness.recording.setScenarioId("cleanup");
|
||||
if (gatewayHarness) {
|
||||
try {
|
||||
const shouldPreserveGatewayDebugArtifacts =
|
||||
@@ -1137,11 +1178,22 @@ export async function runMatrixQaLive(params: {
|
||||
const reportPath = path.join(outputDir, "matrix-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "matrix-qa-summary.json");
|
||||
const observedEventsPath = path.join(outputDir, "matrix-qa-observed-events.json");
|
||||
const routeStateManifestPath = path.join(outputDir, "matrix-qa-route-state-manifest.json");
|
||||
const artifactPaths = {
|
||||
observedEvents: observedEventsPath,
|
||||
report: reportPath,
|
||||
routeStateManifest: routeStateManifestPath,
|
||||
summary: summaryPath,
|
||||
} satisfies MatrixQaArtifactPaths;
|
||||
const routeStateManifest = harness.recording.buildManifest({
|
||||
generatedAt: finishedAt,
|
||||
requestedProfile: params.profile?.trim() || "all",
|
||||
scenarioIds: completedScenarioResults.map((scenario) => scenario.id),
|
||||
substrate: {
|
||||
id: "tuwunel",
|
||||
version: harness.image,
|
||||
},
|
||||
});
|
||||
const report = renderQaMarkdownReport({
|
||||
title: "Matrix QA Report",
|
||||
startedAt: startedAtDate,
|
||||
@@ -1173,6 +1225,7 @@ export async function runMatrixQaLive(params: {
|
||||
default: defaultConfigSnapshot,
|
||||
scenarios: scenarioConfigSnapshots,
|
||||
},
|
||||
differentialProbe,
|
||||
finishedAt,
|
||||
harness: {
|
||||
baseUrl: harness.baseUrl,
|
||||
@@ -1221,6 +1274,10 @@ export async function runMatrixQaLive(params: {
|
||||
)}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
await fs.writeFile(routeStateManifestPath, `${JSON.stringify(routeStateManifest, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
summary.timings.artifactWriteMs = Date.now() - artifactWriteStartedAtMs;
|
||||
summary.timings.totalMs = Date.now() - runStartedAtMs;
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
|
||||
@@ -1263,6 +1320,7 @@ export async function runMatrixQaLive(params: {
|
||||
observedEventsPath,
|
||||
outputDir,
|
||||
reportPath,
|
||||
routeStateManifestPath,
|
||||
scenarios: completedScenarioResults,
|
||||
summaryPath,
|
||||
};
|
||||
|
||||
@@ -1017,7 +1017,8 @@ async function runMatrixQaFaultedE2eeBootstrap(context: MatrixQaScenarioContext)
|
||||
result: MatrixQaE2eeBootstrapResult;
|
||||
}> {
|
||||
const proxy = await startMatrixQaFaultProxy({
|
||||
targetBaseUrl: context.baseUrl,
|
||||
targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl,
|
||||
...context.faultProxyObserver,
|
||||
rules: [buildRoomKeyBackupUnavailableFaultRule(context.driverAccessToken)],
|
||||
});
|
||||
try {
|
||||
@@ -1053,7 +1054,8 @@ async function runMatrixQaFaultedRecoveryOwnerVerification(params: {
|
||||
verification: Awaited<ReturnType<MatrixQaE2eeScenarioClient["verifyWithRecoveryKey"]>>;
|
||||
}> {
|
||||
const proxy = await startMatrixQaFaultProxy({
|
||||
targetBaseUrl: params.context.baseUrl,
|
||||
targetBaseUrl: params.context.faultProxyTargetBaseUrl ?? params.context.baseUrl,
|
||||
...params.context.faultProxyObserver,
|
||||
rules: [buildOwnerSignatureUploadBlockedFaultRule(params.accessToken)],
|
||||
});
|
||||
const recoveryClient = await createMatrixQaE2eeScenarioClient({
|
||||
@@ -1386,7 +1388,8 @@ export async function runMatrixQaE2eeStateAfterMissingEncryptionScenario(
|
||||
configPath,
|
||||
});
|
||||
const proxy = await startMatrixQaFaultProxy({
|
||||
targetBaseUrl: context.baseUrl,
|
||||
targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl,
|
||||
...context.faultProxyObserver,
|
||||
rules: [buildSyncStateAfterMissingEncryptionFaultRule(context.sutAccessToken)],
|
||||
});
|
||||
let gatewayPatched = false;
|
||||
@@ -2236,7 +2239,8 @@ export async function runMatrixQaE2eeCliEncryptionSetupBootstrapFailureScenario(
|
||||
throw new Error("Matrix E2EE CLI bootstrap-failure login did not return a device id");
|
||||
}
|
||||
const proxy = await startMatrixQaFaultProxy({
|
||||
targetBaseUrl: context.baseUrl,
|
||||
targetBaseUrl: context.faultProxyTargetBaseUrl ?? context.baseUrl,
|
||||
...context.faultProxyObserver,
|
||||
rules: [buildRoomKeyBackupUnavailableFaultRule(cliDevice.accessToken)],
|
||||
});
|
||||
const cli = await createMatrixQaCliE2eeSetupRuntime({
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createMatrixQaClient, type MatrixQaRoomObserver } from "../../substrate/client.js";
|
||||
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
|
||||
import type { MatrixQaFaultProxyObserver } from "../../substrate/fault-proxy.js";
|
||||
import { createMatrixQaRoomObserver } from "../../substrate/sync.js";
|
||||
import type { MatrixQaProvisionedTopology } from "../../substrate/topology.js";
|
||||
import { resolveMatrixQaScenarioRoomId } from "./scenario-catalog.js";
|
||||
@@ -23,6 +24,8 @@ export type MatrixQaScenarioContext = {
|
||||
driverDeviceId?: string;
|
||||
driverPassword?: string;
|
||||
driverUserId: string;
|
||||
faultProxyObserver?: MatrixQaFaultProxyObserver;
|
||||
faultProxyTargetBaseUrl?: string;
|
||||
observedEvents: MatrixQaObservedEvent[];
|
||||
observerAccessToken: string;
|
||||
observerDeviceId?: string;
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
// Qa Matrix tests prove one differential probe against the Matrix substrate contract.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runMatrixQaDifferentialProbe } from "./differential-probe.js";
|
||||
|
||||
function createProbeFetch(params?: {
|
||||
missingStateErrcode?: string;
|
||||
userId?: string;
|
||||
}): typeof fetch {
|
||||
return async (input) => {
|
||||
const url = new URL(input instanceof Request ? input.url : input.toString());
|
||||
const json = (body: unknown, status = 200) =>
|
||||
new Response(JSON.stringify(body), {
|
||||
headers: { "content-type": "application/json" },
|
||||
status,
|
||||
});
|
||||
if (url.pathname.endsWith("/versions")) {
|
||||
return json({ versions: ["v1.11"] });
|
||||
}
|
||||
if (url.pathname.endsWith("/account/whoami")) {
|
||||
return json({ user_id: params?.userId ?? "@probe:matrix.test" });
|
||||
}
|
||||
if (url.pathname.endsWith("/sync")) {
|
||||
return json({ next_batch: url.searchParams.has("since") ? "sync-2" : "sync-1" });
|
||||
}
|
||||
return json({ errcode: params?.missingStateErrcode ?? "M_NOT_FOUND" }, 404);
|
||||
};
|
||||
}
|
||||
|
||||
describe("Matrix QA differential probe", () => {
|
||||
it("runs unchanged against the Matrix substrate contract", async () => {
|
||||
const result = await runMatrixQaDifferentialProbe({
|
||||
accessToken: "token",
|
||||
baseUrl: "http://matrix.test",
|
||||
fetchImpl: createProbeFetch(),
|
||||
roomId: "!probe:matrix.test",
|
||||
userId: "@probe:matrix.test",
|
||||
});
|
||||
|
||||
expect(result.profile).toBe("matrix-qa-v1");
|
||||
expect(result.sync).toEqual({
|
||||
continuity: true,
|
||||
incrementalStatus: 200,
|
||||
initialStatus: 200,
|
||||
});
|
||||
expect(result.steps.map((step) => [step.id, step.status])).toEqual([
|
||||
["versions", 200],
|
||||
["whoami", 200],
|
||||
["sync-initial", 200],
|
||||
["sync-incremental", 200],
|
||||
["missing-state", 404],
|
||||
]);
|
||||
expect(result.steps.at(-1)?.errcode).toBe("M_NOT_FOUND");
|
||||
});
|
||||
|
||||
it("rejects a mismatched whoami identity", async () => {
|
||||
await expect(
|
||||
runMatrixQaDifferentialProbe({
|
||||
accessToken: "token",
|
||||
baseUrl: "http://matrix.test",
|
||||
fetchImpl: createProbeFetch({ userId: "@other:matrix.test" }),
|
||||
roomId: "!probe:matrix.test",
|
||||
userId: "@probe:matrix.test",
|
||||
}),
|
||||
).rejects.toThrow("unexpected user_id");
|
||||
});
|
||||
|
||||
it("rejects a missing-state response with the wrong Matrix errcode", async () => {
|
||||
await expect(
|
||||
runMatrixQaDifferentialProbe({
|
||||
accessToken: "token",
|
||||
baseUrl: "http://matrix.test",
|
||||
fetchImpl: createProbeFetch({ missingStateErrcode: "M_FORBIDDEN" }),
|
||||
roomId: "!probe:matrix.test",
|
||||
userId: "@probe:matrix.test",
|
||||
}),
|
||||
).rejects.toThrow("did not return M_NOT_FOUND");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { normalizeMatrixQaRoute } from "./recording-proxy.js";
|
||||
// Qa Matrix plugin module probes Matrix substrates through one protocol contract.
|
||||
import { requestMatrixJson, type MatrixQaFetchLike } from "./request.js";
|
||||
|
||||
const MATRIX_QA_DIFFERENTIAL_PROFILE = "matrix-qa-v1";
|
||||
|
||||
type MatrixQaProbeStep = {
|
||||
errcode?: string;
|
||||
id: string;
|
||||
method: "GET";
|
||||
responseFields: string[];
|
||||
route: string;
|
||||
status: number;
|
||||
};
|
||||
|
||||
export type MatrixQaDifferentialProbeResult = {
|
||||
profile: typeof MATRIX_QA_DIFFERENTIAL_PROFILE;
|
||||
steps: MatrixQaProbeStep[];
|
||||
sync: {
|
||||
continuity: boolean;
|
||||
incrementalStatus: number;
|
||||
initialStatus: number;
|
||||
};
|
||||
};
|
||||
|
||||
function topLevelFields(value: unknown) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? Object.keys(value).toSorted()
|
||||
: [];
|
||||
}
|
||||
|
||||
function errcode(value: unknown) {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
const candidate = (value as { errcode?: unknown }).errcode;
|
||||
return typeof candidate === "string" ? candidate : undefined;
|
||||
}
|
||||
|
||||
function nextBatch(value: unknown) {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
const candidate = (value as { next_batch?: unknown }).next_batch;
|
||||
return typeof candidate === "string" ? candidate : undefined;
|
||||
}
|
||||
|
||||
function userId(value: unknown) {
|
||||
if (typeof value !== "object" || value === null) {
|
||||
return undefined;
|
||||
}
|
||||
const candidate = (value as { user_id?: unknown }).user_id;
|
||||
return typeof candidate === "string" ? candidate : undefined;
|
||||
}
|
||||
|
||||
function buildStep(params: {
|
||||
body: unknown;
|
||||
endpoint: string;
|
||||
id: string;
|
||||
status: number;
|
||||
}): MatrixQaProbeStep {
|
||||
const code = errcode(params.body);
|
||||
return {
|
||||
...(code ? { errcode: code } : {}),
|
||||
id: params.id,
|
||||
method: "GET",
|
||||
responseFields: topLevelFields(params.body),
|
||||
route: normalizeMatrixQaRoute(new URL(params.endpoint, "http://matrix.test").pathname),
|
||||
status: params.status,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runMatrixQaDifferentialProbe(params: {
|
||||
accessToken: string;
|
||||
baseUrl: string;
|
||||
fetchImpl?: MatrixQaFetchLike;
|
||||
roomId: string;
|
||||
userId: string;
|
||||
}): Promise<MatrixQaDifferentialProbeResult> {
|
||||
const fetchImpl = params.fetchImpl ?? fetch;
|
||||
const versions = await requestMatrixJson<Record<string, unknown>>({
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: "/_matrix/client/versions",
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
});
|
||||
const whoami = await requestMatrixJson<Record<string, unknown>>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: "/_matrix/client/v3/account/whoami",
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
});
|
||||
if (userId(whoami.body) !== params.userId) {
|
||||
throw new Error("Matrix differential probe whoami returned an unexpected user_id");
|
||||
}
|
||||
const initialSync = await requestMatrixJson<Record<string, unknown>>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: "/_matrix/client/v3/sync",
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
query: { timeout: 0 },
|
||||
});
|
||||
const initialToken = nextBatch(initialSync.body);
|
||||
if (!initialToken) {
|
||||
throw new Error("Matrix differential probe initial sync did not return next_batch");
|
||||
}
|
||||
const incrementalSync = await requestMatrixJson<Record<string, unknown>>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: "/_matrix/client/v3/sync",
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
query: { since: initialToken, timeout: 0 },
|
||||
});
|
||||
const incrementalToken = nextBatch(incrementalSync.body);
|
||||
if (!incrementalToken) {
|
||||
throw new Error("Matrix differential probe incremental sync did not return next_batch");
|
||||
}
|
||||
const missingStateEndpoint = `/_matrix/client/v3/rooms/${encodeURIComponent(params.roomId)}/state/org.openclaw.qa.missing/${encodeURIComponent(params.userId)}`;
|
||||
const missingState = await requestMatrixJson<Record<string, unknown>>({
|
||||
accessToken: params.accessToken,
|
||||
baseUrl: params.baseUrl,
|
||||
endpoint: missingStateEndpoint,
|
||||
fetchImpl,
|
||||
method: "GET",
|
||||
okStatuses: [404],
|
||||
});
|
||||
if (errcode(missingState.body) !== "M_NOT_FOUND") {
|
||||
throw new Error("Matrix differential probe missing state did not return M_NOT_FOUND");
|
||||
}
|
||||
|
||||
return {
|
||||
profile: MATRIX_QA_DIFFERENTIAL_PROFILE,
|
||||
steps: [
|
||||
buildStep({
|
||||
body: versions.body,
|
||||
endpoint: "/_matrix/client/versions",
|
||||
id: "versions",
|
||||
status: versions.status,
|
||||
}),
|
||||
buildStep({
|
||||
body: whoami.body,
|
||||
endpoint: "/_matrix/client/v3/account/whoami",
|
||||
id: "whoami",
|
||||
status: whoami.status,
|
||||
}),
|
||||
buildStep({
|
||||
body: initialSync.body,
|
||||
endpoint: "/_matrix/client/v3/sync",
|
||||
id: "sync-initial",
|
||||
status: initialSync.status,
|
||||
}),
|
||||
buildStep({
|
||||
body: incrementalSync.body,
|
||||
endpoint: "/_matrix/client/v3/sync",
|
||||
id: "sync-incremental",
|
||||
status: incrementalSync.status,
|
||||
}),
|
||||
buildStep({
|
||||
body: missingState.body,
|
||||
endpoint: missingStateEndpoint,
|
||||
id: "missing-state",
|
||||
status: missingState.status,
|
||||
}),
|
||||
],
|
||||
sync: {
|
||||
continuity: Boolean(initialToken && incrementalToken),
|
||||
incrementalStatus: incrementalSync.status,
|
||||
initialStatus: initialSync.status,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,11 +1,15 @@
|
||||
// Qa Matrix tests cover fault proxy plugin behavior.
|
||||
import { createServer } from "node:http";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { startMatrixQaFaultProxy, type MatrixQaFaultProxy } from "./fault-proxy.js";
|
||||
|
||||
const servers: Array<{ close(): Promise<void> }> = [];
|
||||
|
||||
async function startTargetServer(params?: { responseBody?: string }) {
|
||||
async function startTargetServer(params?: {
|
||||
responseBody?: Buffer | string;
|
||||
responseHeaders?: Record<string, string>;
|
||||
}) {
|
||||
const requests: Array<{
|
||||
authorization?: string;
|
||||
body: string;
|
||||
@@ -24,7 +28,7 @@ async function startTargetServer(params?: { responseBody?: string }) {
|
||||
method: req.method ?? "GET",
|
||||
url: req.url ?? "/",
|
||||
});
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.writeHead(200, { "content-type": "application/json", ...params?.responseHeaders });
|
||||
res.end(params?.responseBody ?? JSON.stringify({ forwarded: true }));
|
||||
})();
|
||||
});
|
||||
@@ -128,6 +132,24 @@ describe("Matrix QA fault proxy", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("strips stale content-encoding after buffering decoded bodies", async () => {
|
||||
const body = Buffer.from(JSON.stringify({ forwarded: true }));
|
||||
const target = await startTargetServer({
|
||||
responseBody: gzipSync(body),
|
||||
responseHeaders: {
|
||||
"content-encoding": "gzip",
|
||||
"content-length": String(gzipSync(body).byteLength),
|
||||
},
|
||||
});
|
||||
proxy = await startMatrixQaFaultProxy({ targetBaseUrl: target.baseUrl, rules: [] });
|
||||
|
||||
const response = await fetch(`${proxy.baseUrl}/encoded`);
|
||||
|
||||
expect(response.headers.get("content-encoding")).toBeNull();
|
||||
expect(response.headers.get("content-length")).toBeNull();
|
||||
await expect(response.json()).resolves.toEqual({ forwarded: true });
|
||||
});
|
||||
|
||||
it("mutates matching forwarded Matrix responses", async () => {
|
||||
const target = await startTargetServer();
|
||||
proxy = await startMatrixQaFaultProxy({
|
||||
@@ -187,6 +209,7 @@ describe("Matrix QA fault proxy", () => {
|
||||
});
|
||||
|
||||
expect(rejected.status).toBe(413);
|
||||
expect(rejected.headers.get("connection")).toBe("close");
|
||||
await expect(rejected.json()).resolves.toMatchObject({
|
||||
errcode: "MATRIX_QA_FAULT_PROXY_REQUEST_TOO_LARGE",
|
||||
});
|
||||
|
||||
@@ -8,8 +8,8 @@ import {
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
|
||||
const DEFAULT_FAULT_PROXY_REQUEST_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const DEFAULT_FAULT_PROXY_RESPONSE_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const DEFAULT_FAULT_PROXY_REQUEST_MAX_BYTES = 20 * 1024 * 1024;
|
||||
const DEFAULT_FAULT_PROXY_RESPONSE_MAX_BYTES = 20 * 1024 * 1024;
|
||||
|
||||
const HOP_BY_HOP_HEADERS = new Set([
|
||||
"connection",
|
||||
@@ -23,8 +23,9 @@ const HOP_BY_HOP_HEADERS = new Set([
|
||||
"upgrade",
|
||||
]);
|
||||
|
||||
type MatrixQaFaultProxyRequest = {
|
||||
export type MatrixQaFaultProxyRequest = {
|
||||
bearerToken?: string;
|
||||
body: Buffer;
|
||||
headers: IncomingHttpHeaders;
|
||||
method: string;
|
||||
path: string;
|
||||
@@ -37,12 +38,23 @@ type MatrixQaFaultProxyResponse = {
|
||||
status: number;
|
||||
};
|
||||
|
||||
type MatrixQaFaultProxyForwardedResponse = {
|
||||
export type MatrixQaFaultProxyForwardedResponse = {
|
||||
body: Buffer;
|
||||
headers: Headers;
|
||||
status: number;
|
||||
};
|
||||
|
||||
export type MatrixQaFaultProxyExchange = {
|
||||
context?: unknown;
|
||||
request: MatrixQaFaultProxyRequest;
|
||||
response: MatrixQaFaultProxyForwardedResponse;
|
||||
};
|
||||
|
||||
export type MatrixQaFaultProxyObserver = {
|
||||
createExchangeContext?: (request: MatrixQaFaultProxyRequest) => unknown;
|
||||
onExchange?: (exchange: MatrixQaFaultProxyExchange) => Promise<void> | void;
|
||||
};
|
||||
|
||||
class MatrixQaFaultProxyHttpError extends Error {
|
||||
constructor(
|
||||
readonly status: number,
|
||||
@@ -228,13 +240,19 @@ function bufferToArrayBuffer(buffer: Buffer) {
|
||||
) as ArrayBuffer;
|
||||
}
|
||||
|
||||
function writeJsonResponse(res: ServerResponse, response: MatrixQaFaultProxyResponse) {
|
||||
const body = response.body === undefined ? "" : JSON.stringify(response.body);
|
||||
res.writeHead(response.status, {
|
||||
"content-type": "application/json",
|
||||
...response.headers,
|
||||
});
|
||||
res.end(body);
|
||||
function normalizeJsonResponse(
|
||||
response: MatrixQaFaultProxyResponse,
|
||||
): MatrixQaFaultProxyForwardedResponse {
|
||||
const body =
|
||||
response.body === undefined ? Buffer.alloc(0) : Buffer.from(JSON.stringify(response.body));
|
||||
return {
|
||||
body,
|
||||
headers: new Headers({
|
||||
"content-type": "application/json",
|
||||
...response.headers,
|
||||
}),
|
||||
status: response.status,
|
||||
};
|
||||
}
|
||||
|
||||
async function forwardMatrixQaFaultProxyRequest(params: {
|
||||
@@ -279,10 +297,18 @@ async function forwardMatrixQaFaultProxyRequest(params: {
|
||||
function writeForwardedResponse(
|
||||
res: ServerResponse,
|
||||
response: MatrixQaFaultProxyForwardedResponse,
|
||||
options: { preserveConnectionClose?: boolean } = {},
|
||||
) {
|
||||
const headers: Record<string, string> = {};
|
||||
for (const [key, value] of response.headers) {
|
||||
if (!HOP_BY_HOP_HEADERS.has(key.toLowerCase())) {
|
||||
const normalizedKey = key.toLowerCase();
|
||||
const isIntentionalConnectionClose =
|
||||
options.preserveConnectionClose && normalizedKey === "connection" && value === "close";
|
||||
if (
|
||||
(!HOP_BY_HOP_HEADERS.has(normalizedKey) || isIntentionalConnectionClose) &&
|
||||
normalizedKey !== "content-encoding" &&
|
||||
normalizedKey !== "content-length"
|
||||
) {
|
||||
headers[key] = value;
|
||||
}
|
||||
}
|
||||
@@ -290,30 +316,38 @@ function writeForwardedResponse(
|
||||
res.end(response.body);
|
||||
}
|
||||
|
||||
export async function startMatrixQaFaultProxy(params: {
|
||||
maxRequestBytes?: number;
|
||||
maxResponseBytes?: number;
|
||||
rules: MatrixQaFaultProxyRule[];
|
||||
targetBaseUrl: string;
|
||||
}): Promise<MatrixQaFaultProxy> {
|
||||
export async function startMatrixQaFaultProxy(
|
||||
params: MatrixQaFaultProxyObserver & {
|
||||
maxRequestBytes?: number;
|
||||
maxResponseBytes?: number;
|
||||
rules: MatrixQaFaultProxyRule[];
|
||||
targetBaseUrl: string;
|
||||
},
|
||||
): Promise<MatrixQaFaultProxy> {
|
||||
const targetBaseUrl = new URL(params.targetBaseUrl);
|
||||
const maxRequestBytes = params.maxRequestBytes ?? DEFAULT_FAULT_PROXY_REQUEST_MAX_BYTES;
|
||||
const maxResponseBytes = params.maxResponseBytes ?? DEFAULT_FAULT_PROXY_RESPONSE_MAX_BYTES;
|
||||
const hits: MatrixQaFaultProxyHit[] = [];
|
||||
const server = createServer((req, res) => {
|
||||
void (async () => {
|
||||
let observedRequest: MatrixQaFaultProxyRequest | undefined;
|
||||
let observedContext: unknown;
|
||||
try {
|
||||
const requestUrl = new URL(req.url ?? "/", targetBaseUrl);
|
||||
const path = requestUrl.pathname;
|
||||
const bearerToken = extractBearerToken(req.headers);
|
||||
const body = await readRequestBody(req, maxRequestBytes);
|
||||
const request: MatrixQaFaultProxyRequest = {
|
||||
...(bearerToken ? { bearerToken } : {}),
|
||||
body,
|
||||
headers: req.headers,
|
||||
method: req.method ?? "GET",
|
||||
path,
|
||||
search: requestUrl.search,
|
||||
};
|
||||
const body = await readRequestBody(req, maxRequestBytes);
|
||||
observedRequest = request;
|
||||
const context = params.createExchangeContext?.(request);
|
||||
observedContext = context;
|
||||
const rule = params.rules.find((candidate) => candidate.match(request));
|
||||
if (rule) {
|
||||
hits.push({
|
||||
@@ -322,7 +356,13 @@ export async function startMatrixQaFaultProxy(params: {
|
||||
ruleId: rule.id,
|
||||
});
|
||||
if (rule.response) {
|
||||
writeJsonResponse(res, rule.response(request));
|
||||
const response = normalizeJsonResponse(rule.response(request));
|
||||
await params.onExchange?.({
|
||||
...(context !== undefined ? { context } : {}),
|
||||
request,
|
||||
response,
|
||||
});
|
||||
writeForwardedResponse(res, response);
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -339,25 +379,41 @@ export async function startMatrixQaFaultProxy(params: {
|
||||
response: forwarded,
|
||||
})
|
||||
: forwarded;
|
||||
await params.onExchange?.({
|
||||
...(context !== undefined ? { context } : {}),
|
||||
request,
|
||||
response,
|
||||
});
|
||||
writeForwardedResponse(res, response);
|
||||
} catch (error) {
|
||||
if (error instanceof MatrixQaFaultProxyHttpError) {
|
||||
writeJsonResponse(res, {
|
||||
body: {
|
||||
errcode: error.code,
|
||||
error: error.message,
|
||||
},
|
||||
...(error.status === 413 ? { headers: { connection: "close" } } : {}),
|
||||
status: error.status,
|
||||
const failure =
|
||||
error instanceof MatrixQaFaultProxyHttpError
|
||||
? {
|
||||
body: {
|
||||
errcode: error.code,
|
||||
error: error.message,
|
||||
},
|
||||
...(error.status === 413 ? { headers: { connection: "close" } } : {}),
|
||||
status: error.status,
|
||||
}
|
||||
: {
|
||||
body: {
|
||||
errcode: "MATRIX_QA_FAULT_PROXY_ERROR",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
status: 502,
|
||||
};
|
||||
const response = normalizeJsonResponse(failure);
|
||||
if (observedRequest) {
|
||||
await params.onExchange?.({
|
||||
...(observedContext !== undefined ? { context: observedContext } : {}),
|
||||
request: observedRequest,
|
||||
response,
|
||||
});
|
||||
return;
|
||||
}
|
||||
writeJsonResponse(res, {
|
||||
body: {
|
||||
errcode: "MATRIX_QA_FAULT_PROXY_ERROR",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
status: 502,
|
||||
writeForwardedResponse(res, response, {
|
||||
preserveConnectionClose:
|
||||
error instanceof MatrixQaFaultProxyHttpError && error.status === 413,
|
||||
});
|
||||
}
|
||||
})();
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { withTempDir } from "openclaw/plugin-sdk/test-env";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { testing, startMatrixQaHarness, writeMatrixQaHarnessFiles } from "./harness.runtime.js";
|
||||
import type { MatrixQaRecordingProxy } from "./recording-proxy.js";
|
||||
|
||||
type MatrixQaHarnessDeps = Parameters<typeof startMatrixQaHarness>[1];
|
||||
type MatrixQaHarnessResult = Awaited<ReturnType<typeof startMatrixQaHarness>>;
|
||||
@@ -15,13 +17,23 @@ async function withStartedMatrixHarness(
|
||||
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-harness-"));
|
||||
|
||||
try {
|
||||
const startRecordingProxyImpl =
|
||||
deps?.startRecordingProxyImpl ??
|
||||
(async ({ targetBaseUrl }: { targetBaseUrl: string }) =>
|
||||
({
|
||||
baseUrl: targetBaseUrl,
|
||||
buildManifest: vi.fn(),
|
||||
records: () => [],
|
||||
setScenarioId: vi.fn(),
|
||||
stop: vi.fn(async () => {}),
|
||||
}) as unknown as MatrixQaRecordingProxy);
|
||||
const result = await startMatrixQaHarness(
|
||||
{
|
||||
outputDir,
|
||||
repoRoot: "/repo/openclaw",
|
||||
homeserverPort: 28008,
|
||||
},
|
||||
deps,
|
||||
{ ...deps, startRecordingProxyImpl },
|
||||
);
|
||||
await verify({ outputDir, result });
|
||||
} finally {
|
||||
@@ -137,6 +149,33 @@ describe("matrix harness runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("stops Tuwunel when recorder startup fails", async () => {
|
||||
const calls: string[] = [];
|
||||
await withTempDir("matrix-qa-harness-", async (outputDir) => {
|
||||
await expect(
|
||||
startMatrixQaHarness(
|
||||
{ outputDir, repoRoot: "/repo/openclaw" },
|
||||
{
|
||||
async runCommand(command, args, cwd) {
|
||||
calls.push([command, ...args, `@${cwd}`].join(" "));
|
||||
if (args.join(" ").includes("ps --format json")) {
|
||||
return { stdout: '[{"State":"running"}]\n', stderr: "" };
|
||||
}
|
||||
return { stdout: "", stderr: "" };
|
||||
},
|
||||
fetchImpl: vi.fn(async () => ({ ok: true })),
|
||||
sleepImpl: vi.fn(async () => {}),
|
||||
resolveHostPortImpl: vi.fn(async (port: number) => port),
|
||||
startRecordingProxyImpl: vi.fn(async () => {
|
||||
throw new Error("recorder startup failed");
|
||||
}),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("recorder startup failed");
|
||||
expect(calls.filter((call) => call.includes("down --remove-orphans"))).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
it("treats empty Docker health fields as a fallback to running state", async () => {
|
||||
await withStartedMatrixHarness(
|
||||
{
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
type FetchLike,
|
||||
type RunCommand,
|
||||
} from "../docker-runtime.js";
|
||||
import { startMatrixQaRecordingProxy, type MatrixQaRecordingProxy } from "./recording-proxy.js";
|
||||
|
||||
const MATRIX_QA_DEFAULT_IMAGE = "ghcr.io/matrix-construct/tuwunel:v1.5.1";
|
||||
const MATRIX_QA_DEFAULT_SERVER_NAME = "matrix-qa.test";
|
||||
@@ -41,9 +42,11 @@ type MatrixQaHarnessFiles = {
|
||||
|
||||
type MatrixQaHarness = MatrixQaHarnessFiles & {
|
||||
baseUrl: string;
|
||||
recording: MatrixQaRecordingProxy;
|
||||
restartService(): Promise<void>;
|
||||
stopCommand: string;
|
||||
stop(): Promise<void>;
|
||||
upstreamBaseUrl: string;
|
||||
};
|
||||
|
||||
function buildVersionsUrl(baseUrl: string) {
|
||||
@@ -216,6 +219,7 @@ export async function startMatrixQaHarness(
|
||||
runCommand?: RunCommand;
|
||||
sleepImpl?: (ms: number) => Promise<unknown>;
|
||||
resolveHostPortImpl?: typeof resolveHostPort;
|
||||
startRecordingProxyImpl?: typeof startMatrixQaRecordingProxy;
|
||||
},
|
||||
): Promise<MatrixQaHarness> {
|
||||
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
|
||||
@@ -223,6 +227,7 @@ export async function startMatrixQaHarness(
|
||||
const runCommand = deps?.runCommand ?? execCommand;
|
||||
const fetchImpl = deps?.fetchImpl ?? fetchHealthUrl;
|
||||
const sleepImpl = deps?.sleepImpl ?? sleep;
|
||||
const startRecordingProxyImpl = deps?.startRecordingProxyImpl ?? startMatrixQaRecordingProxy;
|
||||
const homeserverPort = await resolveHostPortImpl(
|
||||
params.homeserverPort ?? MATRIX_QA_DEFAULT_PORT,
|
||||
params.homeserverPort != null,
|
||||
@@ -255,7 +260,7 @@ export async function startMatrixQaHarness(
|
||||
);
|
||||
|
||||
const hostBaseUrl = `http://127.0.0.1:${homeserverPort}/`;
|
||||
let baseUrl = hostBaseUrl;
|
||||
let upstreamBaseUrl = hostBaseUrl;
|
||||
const hostReachable = await isMatrixVersionsReachable(hostBaseUrl, fetchImpl);
|
||||
if (!hostReachable) {
|
||||
const containerBaseUrl = await resolveComposeServiceUrl(
|
||||
@@ -265,7 +270,7 @@ export async function startMatrixQaHarness(
|
||||
repoRoot,
|
||||
runCommand,
|
||||
);
|
||||
baseUrl = await waitForReachableMatrixBaseUrl({
|
||||
upstreamBaseUrl = await waitForReachableMatrixBaseUrl({
|
||||
composeFile: files.composeFile,
|
||||
containerBaseUrl,
|
||||
fetchImpl,
|
||||
@@ -274,12 +279,27 @@ export async function startMatrixQaHarness(
|
||||
});
|
||||
}
|
||||
|
||||
await waitForHealth(buildVersionsUrl(baseUrl), {
|
||||
await waitForHealth(buildVersionsUrl(upstreamBaseUrl), {
|
||||
label: "Matrix homeserver",
|
||||
composeFile: files.composeFile,
|
||||
fetchImpl,
|
||||
sleepImpl,
|
||||
});
|
||||
let recording: MatrixQaRecordingProxy;
|
||||
try {
|
||||
recording = await startRecordingProxyImpl({ targetBaseUrl: upstreamBaseUrl });
|
||||
} catch (error) {
|
||||
await withMatrixQaHarnessTimeout(
|
||||
"Matrix homeserver cleanup after recorder startup failure",
|
||||
MATRIX_QA_CLEANUP_TIMEOUT_MS,
|
||||
runCommand(
|
||||
"docker",
|
||||
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
|
||||
repoRoot,
|
||||
),
|
||||
).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
|
||||
const waitForReady = async () => {
|
||||
await sleepImpl(1_000);
|
||||
@@ -290,7 +310,7 @@ export async function startMatrixQaHarness(
|
||||
runCommand,
|
||||
sleepImpl,
|
||||
);
|
||||
await waitForHealth(buildVersionsUrl(baseUrl), {
|
||||
await waitForHealth(buildVersionsUrl(upstreamBaseUrl), {
|
||||
label: "Matrix homeserver",
|
||||
composeFile: files.composeFile,
|
||||
fetchImpl,
|
||||
@@ -300,7 +320,8 @@ export async function startMatrixQaHarness(
|
||||
|
||||
return {
|
||||
...files,
|
||||
baseUrl,
|
||||
baseUrl: recording.baseUrl,
|
||||
recording,
|
||||
async restartService() {
|
||||
await runCommand(
|
||||
"docker",
|
||||
@@ -311,16 +332,26 @@ export async function startMatrixQaHarness(
|
||||
},
|
||||
stopCommand: `docker compose -f ${files.composeFile} down --remove-orphans`,
|
||||
async stop() {
|
||||
await withMatrixQaHarnessTimeout(
|
||||
"Matrix homeserver cleanup",
|
||||
MATRIX_QA_CLEANUP_TIMEOUT_MS,
|
||||
runCommand(
|
||||
"docker",
|
||||
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
|
||||
repoRoot,
|
||||
const results = await Promise.allSettled([
|
||||
recording.stop(),
|
||||
withMatrixQaHarnessTimeout(
|
||||
"Matrix homeserver cleanup",
|
||||
MATRIX_QA_CLEANUP_TIMEOUT_MS,
|
||||
runCommand(
|
||||
"docker",
|
||||
["compose", "-f", files.composeFile, "down", "--remove-orphans"],
|
||||
repoRoot,
|
||||
),
|
||||
),
|
||||
]);
|
||||
const failures = results.flatMap((result) =>
|
||||
result.status === "rejected" ? [result.reason] : [],
|
||||
);
|
||||
if (failures.length > 0) {
|
||||
throw new AggregateError(failures, "Matrix QA harness cleanup failed");
|
||||
}
|
||||
},
|
||||
upstreamBaseUrl,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,714 @@
|
||||
// Qa Matrix tests cover redacted protocol recording and manifest derivation.
|
||||
import { createServer } from "node:http";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { startMatrixQaFaultProxy } from "./fault-proxy.js";
|
||||
import { normalizeMatrixQaRoute, startMatrixQaRecordingProxy } from "./recording-proxy.js";
|
||||
|
||||
const closeCallbacks: Array<() => Promise<void>> = [];
|
||||
|
||||
afterEach(async () => {
|
||||
while (closeCallbacks.length > 0) {
|
||||
await closeCallbacks.pop()?.();
|
||||
}
|
||||
});
|
||||
|
||||
async function startRecordingTarget(options?: { alwaysFailState?: boolean }) {
|
||||
let syncCount = 0;
|
||||
let stateCount = 0;
|
||||
const server = createServer((req, res) => {
|
||||
const url = new URL(req.url ?? "/", "http://matrix.test");
|
||||
if (url.pathname.endsWith("/sync")) {
|
||||
syncCount += 1;
|
||||
const since = url.searchParams.get("since");
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
account_data: {
|
||||
events: [
|
||||
{
|
||||
content: {
|
||||
encrypted: {
|
||||
SECRET_NESTED_STORAGE_KEY: { ciphertext: "secret-nested-ciphertext" },
|
||||
},
|
||||
unsigned: {
|
||||
"secret-custom-field-name": "secret-custom-field-value",
|
||||
},
|
||||
url: "mxc://matrix.test/secret-media",
|
||||
info: { mimetype: "image/png" },
|
||||
},
|
||||
type: "m.cross_signing.master",
|
||||
},
|
||||
],
|
||||
},
|
||||
device_keys: {
|
||||
"@secret-user:matrix.test": {
|
||||
SECRET_DEVICE: {
|
||||
keys: {
|
||||
"ed25519:SECRET_DEVICE": "secret-signing-key",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
device_one_time_keys_count: { signed_curve25519: 12 },
|
||||
device_unused_fallback_key_types: ["signed_curve25519"],
|
||||
next_batch: since === "echoed-unknown" ? since : `secret-sync-${syncCount}`,
|
||||
device_lists: { changed: ["secret-device"] },
|
||||
rooms: {
|
||||
join: {
|
||||
"!secret-room:matrix.test": {
|
||||
timeline: {
|
||||
events: [
|
||||
{ type: "m.room.message" },
|
||||
{ sender: "@secret-sender:matrix.test", type: "m.room.encrypted" },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (url.pathname.includes("/room_keys/keys")) {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
rooms: {
|
||||
"!secret-backup-room:matrix.test": {
|
||||
sessions: {
|
||||
"secret-backup-session": { first_message_index: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (url.pathname.endsWith("/keys/upload")) {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ one_time_key_counts: {} }));
|
||||
return;
|
||||
}
|
||||
if (url.pathname.endsWith("/keys/query")) {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
failures: {
|
||||
"secret-server.example": { errcode: "M_UNAVAILABLE" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (url.pathname.includes("/account_data/")) {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({}));
|
||||
return;
|
||||
}
|
||||
if (url.pathname.includes("/sendToDevice/")) {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({}));
|
||||
return;
|
||||
}
|
||||
if (url.pathname.includes("/state/")) {
|
||||
stateCount += 1;
|
||||
const status = options?.alwaysFailState || stateCount === 1 ? 401 : 200;
|
||||
res.writeHead(status, { "content-type": "application/json" });
|
||||
res.end(
|
||||
status === 401
|
||||
? JSON.stringify({ errcode: "M_UNKNOWN_TOKEN", error: "secret-response" })
|
||||
: JSON.stringify({ name: "secret-state-name" }),
|
||||
);
|
||||
return;
|
||||
}
|
||||
res.writeHead(401, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ errcode: "M_UNKNOWN_TOKEN", error: "secret-response" }));
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.once("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
server.off("error", reject);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("recording target did not bind");
|
||||
}
|
||||
closeCallbacks.push(
|
||||
async () =>
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
}),
|
||||
);
|
||||
return `http://127.0.0.1:${address.port}`;
|
||||
}
|
||||
|
||||
describe("Matrix QA recording proxy", () => {
|
||||
it("records only redacted shapes and derives scenario expectations", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const proxy = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => proxy.stop());
|
||||
proxy.setScenarioId("matrix-recording-test");
|
||||
|
||||
const firstSync = await fetch(
|
||||
`${proxy.baseUrl}/_matrix/client/v3/sync?timeout=0&access_token=secret-query`,
|
||||
{ headers: { authorization: "Bearer secret-header" } },
|
||||
);
|
||||
const firstBody = (await firstSync.json()) as { next_batch: string };
|
||||
await fetch(
|
||||
`${proxy.baseUrl}/_matrix/client/v3/sync?timeout=0&since=${encodeURIComponent(firstBody.next_batch)}`,
|
||||
{ headers: { authorization: "Bearer secret-header" } },
|
||||
);
|
||||
const postState = async () => {
|
||||
await fetch(
|
||||
`${proxy.baseUrl}/_matrix/client/v3/rooms/!secret:matrix.test/state/m.room.name`,
|
||||
{
|
||||
body: JSON.stringify({ password: "secret-password", body: "secret-message" }),
|
||||
headers: {
|
||||
authorization: "Bearer secret-header",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
method: "POST",
|
||||
},
|
||||
);
|
||||
};
|
||||
await postState();
|
||||
await fetch(`${proxy.baseUrl}/_matrix/client/v3/room_keys/keys`, {
|
||||
headers: {
|
||||
authorization: "Bearer secret-header",
|
||||
},
|
||||
});
|
||||
await fetch(`${proxy.baseUrl}/_matrix/client/v3/keys/upload`, {
|
||||
body: JSON.stringify({
|
||||
device_keys: {
|
||||
algorithms: ["m.olm.v1.curve25519-aes-sha2"],
|
||||
device_id: "SECRET_UPLOAD_DEVICE",
|
||||
keys: { "ed25519:SECRET_UPLOAD_DEVICE": "secret-upload-key" },
|
||||
user_id: "@secret-upload:matrix.test",
|
||||
},
|
||||
one_time_keys: {
|
||||
"signed_curve25519:SECRET_ONE_TIME": { key: "secret-one-time-key" },
|
||||
},
|
||||
}),
|
||||
headers: {
|
||||
authorization: "Bearer secret-header",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
method: "POST",
|
||||
});
|
||||
await fetch(`${proxy.baseUrl}/_matrix/client/v3/keys/query`, {
|
||||
body: JSON.stringify({ device_keys: {} }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
await fetch(
|
||||
`${proxy.baseUrl}/_matrix/client/v3/user/@secret-upload%3Amatrix.test/account_data/m.cross_signing.master`,
|
||||
{
|
||||
body: JSON.stringify({
|
||||
encrypted: {
|
||||
SECRET_STORAGE_KEY: { ciphertext: "secret-ciphertext" },
|
||||
},
|
||||
}),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "PUT",
|
||||
},
|
||||
);
|
||||
await fetch(
|
||||
`${proxy.baseUrl}/_matrix/client/v3/user/@secret-upload%3Amatrix.test/account_data/m.megolm_backup.v1`,
|
||||
{
|
||||
body: JSON.stringify({ version: "1" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "PUT",
|
||||
},
|
||||
);
|
||||
await fetch(`${proxy.baseUrl}/_matrix/client/v3/sendToDevice/m.room.encrypted/transaction-42`, {
|
||||
body: JSON.stringify({
|
||||
messages: {
|
||||
"@secret-recipient:matrix.test": {
|
||||
SECRET_RECIPIENT_DEVICE: { content: "secret-device-message" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "PUT",
|
||||
});
|
||||
await postState();
|
||||
|
||||
const records = proxy.records();
|
||||
const serialized = JSON.stringify(records);
|
||||
expect(serialized).not.toContain("secret-query");
|
||||
expect(serialized).not.toContain("secret-header");
|
||||
expect(serialized).not.toContain("secret-password");
|
||||
expect(serialized).not.toContain("secret-message");
|
||||
expect(serialized).not.toContain("secret-sync");
|
||||
expect(serialized).not.toContain("secret-response");
|
||||
expect(serialized).not.toContain("secret-room");
|
||||
expect(serialized).not.toContain("secret-backup-room");
|
||||
expect(serialized).not.toContain("secret-backup-session");
|
||||
expect(serialized).not.toContain("secret-state-name");
|
||||
expect(serialized).not.toContain("secret-user");
|
||||
expect(serialized).not.toContain("SECRET_DEVICE");
|
||||
expect(serialized).not.toContain("SECRET_UPLOAD_DEVICE");
|
||||
expect(serialized).not.toContain("SECRET_ONE_TIME");
|
||||
expect(serialized).not.toContain("SECRET_RECIPIENT_DEVICE");
|
||||
expect(serialized).not.toContain("secret-server.example");
|
||||
expect(serialized).not.toContain("SECRET_STORAGE_KEY");
|
||||
expect(serialized).not.toContain("SECRET_NESTED_STORAGE_KEY");
|
||||
expect(serialized).not.toContain("secret-custom-field-name");
|
||||
const keyUpload = records.find((record) => record.request.route.endsWith("/keys/upload"));
|
||||
expect(keyUpload?.request.body).toEqual({
|
||||
kind: "json",
|
||||
fields: [
|
||||
"device_keys.algorithms",
|
||||
"device_keys.device_id",
|
||||
"device_keys.keys.{keyId}",
|
||||
"device_keys.user_id",
|
||||
"one_time_keys.{keyId}.key",
|
||||
],
|
||||
});
|
||||
const sendToDevice = records.find((record) => record.request.route.includes("sendToDevice"));
|
||||
expect(sendToDevice?.request.body).toEqual({
|
||||
kind: "json",
|
||||
fields: ["messages.{userId}.{deviceId}.content"],
|
||||
});
|
||||
const keyQuery = records.find((record) => record.request.route.endsWith("/keys/query"));
|
||||
expect(keyQuery?.response.body).toEqual({
|
||||
kind: "json",
|
||||
fields: ["failures.{serverName}.errcode"],
|
||||
});
|
||||
const secretStorage = records.find((record) =>
|
||||
record.request.route.endsWith("/account_data/m.cross_signing.master"),
|
||||
);
|
||||
expect(secretStorage?.request.body).toEqual({
|
||||
kind: "json",
|
||||
fields: ["encrypted.{keyId}.ciphertext"],
|
||||
});
|
||||
expect(records[1]?.sync).toMatchObject({ continuity: true, since: "sync-1" });
|
||||
|
||||
const manifest = proxy.buildManifest({
|
||||
generatedAt: "2026-07-03T00:00:00.000Z",
|
||||
requestedProfile: "all",
|
||||
scenarioIds: ["matrix-recording-test"],
|
||||
substrate: { id: "tuwunel", version: "v1.5.1" },
|
||||
});
|
||||
const expectation = manifest.scenarios["matrix-recording-test"];
|
||||
expect(manifest.profile).toEqual({
|
||||
derivedFrom: "observed-request-response-traffic",
|
||||
id: "matrix-qa-v1",
|
||||
});
|
||||
expect(expectation?.syncTokens).toEqual({
|
||||
continuityObserved: true,
|
||||
incrementalRequests: 1,
|
||||
initialRequests: 1,
|
||||
responseTokens: 2,
|
||||
});
|
||||
expect(expectation?.state.device).toEqual([
|
||||
"/_matrix/client/v3/keys/upload",
|
||||
"/_matrix/client/v3/sync",
|
||||
]);
|
||||
expect(expectation?.state.key).toContain(
|
||||
"/_matrix/client/v3/user/{userId}/account_data/m.cross_signing.master",
|
||||
);
|
||||
expect(expectation?.state.key).toContain("/_matrix/client/v3/sync");
|
||||
expect(expectation?.state.backup).toContain(
|
||||
"/_matrix/client/v3/user/{userId}/account_data/m.megolm_backup.v1",
|
||||
);
|
||||
expect(expectation?.state.media).toContain("/_matrix/client/v3/sync");
|
||||
expect(expectation?.ordering[0]).toMatchObject({
|
||||
requestBody: { kind: "empty" },
|
||||
responseBody: {
|
||||
kind: "json",
|
||||
fields: expect.arrayContaining([
|
||||
"rooms.join.{roomId}.timeline.events[].sender",
|
||||
"rooms.join.{roomId}.timeline.events[].type",
|
||||
]),
|
||||
},
|
||||
});
|
||||
expect(expectation?.retries).toEqual([]);
|
||||
expect(expectation?.errors).toEqual([
|
||||
{
|
||||
errcode: "M_UNKNOWN_TOKEN",
|
||||
method: "POST",
|
||||
route: "/_matrix/client/v3/rooms/{roomId}/state/m.room.name",
|
||||
status: 401,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("normalizes substrate-specific Matrix identifiers from routes", () => {
|
||||
expect(
|
||||
normalizeMatrixQaRoute(
|
||||
"/_matrix/client/v3/rooms/!room%3Amatrix.test/send/m.room.message/txn-123",
|
||||
),
|
||||
).toBe("/_matrix/client/v3/rooms/{roomId}/send/m.room.message/{transactionId}");
|
||||
expect(
|
||||
normalizeMatrixQaRoute(
|
||||
"/_matrix/client/v3/rooms/!room%3Amatrix.test/redact/$event%3Amatrix.test/txn-123",
|
||||
),
|
||||
).toBe("/_matrix/client/v3/rooms/{roomId}/redact/{eventId}/{transactionId}");
|
||||
expect(normalizeMatrixQaRoute("/_matrix/media/v3/download/matrix.test/secret-media-id")).toBe(
|
||||
"/_matrix/media/v3/download/{serverName}/{mediaId}",
|
||||
);
|
||||
expect(
|
||||
normalizeMatrixQaRoute(
|
||||
"/_matrix/media/v3/download/matrix.test/secret-media-id/private-report.pdf",
|
||||
),
|
||||
).toBe("/_matrix/media/v3/download/{serverName}/{mediaId}/{filename}");
|
||||
expect(
|
||||
normalizeMatrixQaRoute("/_matrix/client/v1/media/thumbnail/matrix.test/secret-media-id"),
|
||||
).toBe("/_matrix/client/v1/media/thumbnail/{serverName}/{mediaId}");
|
||||
expect(
|
||||
normalizeMatrixQaRoute("/_matrix/client/v3/user/@alice%3Amatrix.test/filter/filter-42"),
|
||||
).toBe("/_matrix/client/v3/user/{userId}/filter/{filterId}");
|
||||
expect(
|
||||
normalizeMatrixQaRoute(
|
||||
"/_matrix/client/v3/user/@alice%3Amatrix.test/account_data/m.secret_storage.key.secret-key-id",
|
||||
),
|
||||
).toBe("/_matrix/client/v3/user/{userId}/account_data/m.secret_storage.key.{keyId}");
|
||||
expect(
|
||||
normalizeMatrixQaRoute("/_matrix/client/v3/room_keys/keys/!room%3Amatrix.test/session-42"),
|
||||
).toBe("/_matrix/client/v3/room_keys/keys/{roomId}/{sessionId}");
|
||||
});
|
||||
|
||||
it("records the response observed after scenario-local fault injection", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-fault-recording-test");
|
||||
const faultProxy = await startMatrixQaFaultProxy({
|
||||
targetBaseUrl,
|
||||
...recording,
|
||||
rules: [
|
||||
{
|
||||
id: "backup-unavailable",
|
||||
match: (request) => request.path.endsWith("/room_keys/version"),
|
||||
response: () => ({
|
||||
body: { errcode: "M_NOT_FOUND", error: "secret-fault-message" },
|
||||
status: 404,
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
closeCallbacks.push(() => faultProxy.stop());
|
||||
|
||||
await fetch(`${faultProxy.baseUrl}/_matrix/client/v3/room_keys/version`);
|
||||
|
||||
const record = recording
|
||||
.records()
|
||||
.find((entry) => entry.scenarioId === "matrix-fault-recording-test");
|
||||
expect(record?.response).toMatchObject({ errcode: "M_NOT_FOUND", status: 404 });
|
||||
expect(JSON.stringify(record)).not.toContain("secret-fault-message");
|
||||
});
|
||||
|
||||
it("records proxy-generated upstream failures", async () => {
|
||||
const recording = await startMatrixQaRecordingProxy({
|
||||
targetBaseUrl: "http://127.0.0.1:1",
|
||||
});
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-upstream-failure-test");
|
||||
|
||||
const response = await fetch(`${recording.baseUrl}/_matrix/client/v3/sync?timeout=0`);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
expect(recording.records().at(-1)?.response).toMatchObject({
|
||||
errcode: "MATRIX_QA_FAULT_PROXY_ERROR",
|
||||
status: 502,
|
||||
});
|
||||
});
|
||||
|
||||
it("redacts signature upload device and cross-signing key identifiers", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-signatures-recording-test");
|
||||
|
||||
await fetch(`${recording.baseUrl}/_matrix/client/v3/keys/signatures/upload`, {
|
||||
body: JSON.stringify({
|
||||
"@secret-signing-user:matrix.test": {
|
||||
SECRET_CROSS_SIGNING_KEY: { signatures: {} },
|
||||
},
|
||||
}),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
const record = recording.records().at(-1);
|
||||
expect(record?.request.body).toEqual({
|
||||
kind: "json",
|
||||
fields: ["{userId}.{deviceOrKeyId}.signatures"],
|
||||
});
|
||||
expect(JSON.stringify(record)).not.toContain("secret-signing-user");
|
||||
expect(JSON.stringify(record)).not.toContain("SECRET_CROSS_SIGNING_KEY");
|
||||
});
|
||||
|
||||
it("reports sync continuity only when every incremental token is recognized", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-sync-discontinuity-test");
|
||||
|
||||
await fetch(`${recording.baseUrl}/_matrix/client/v3/sync?timeout=0&since=unknown-token`);
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-sync-discontinuity-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-sync-discontinuity-test"]?.syncTokens).toMatchObject({
|
||||
continuityObserved: false,
|
||||
incrementalRequests: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not learn an unknown request token from the same sync response", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-echoed-sync-token-test");
|
||||
|
||||
await fetch(`${recording.baseUrl}/_matrix/client/v3/sync?timeout=0&since=echoed-unknown`);
|
||||
|
||||
expect(recording.records().at(-1)?.sync).toMatchObject({
|
||||
continuity: false,
|
||||
since: "sync-unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not conflate distinct same-shape operations as retries", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-distinct-operation-test");
|
||||
|
||||
await fetch(
|
||||
`${recording.baseUrl}/_matrix/client/v3/rooms/!first:matrix.test/state/m.room.name`,
|
||||
{
|
||||
body: JSON.stringify({ name: "first" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
},
|
||||
);
|
||||
await fetch(
|
||||
`${recording.baseUrl}/_matrix/client/v3/rooms/!second:matrix.test/state/m.room.name`,
|
||||
{
|
||||
body: JSON.stringify({ name: "second" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
},
|
||||
);
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-distinct-operation-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-distinct-operation-test"]?.retries).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not conflate identical operations from different principals", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-principal-retry-test");
|
||||
const endpoint = `${recording.baseUrl}/_matrix/client/v3/rooms/!same:matrix.test/state/m.room.name`;
|
||||
|
||||
await fetch(endpoint, {
|
||||
body: JSON.stringify({ name: "same" }),
|
||||
headers: { authorization: "Bearer first", "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
await fetch(endpoint, {
|
||||
body: JSON.stringify({ name: "same" }),
|
||||
headers: { authorization: "Bearer second", "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-principal-retry-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-principal-retry-test"]?.retries).toEqual([]);
|
||||
});
|
||||
|
||||
it("attributes sync completion to the active scenario", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("previous-scenario");
|
||||
const context = recording.createExchangeContext?.({
|
||||
body: Buffer.alloc(0),
|
||||
headers: {},
|
||||
method: "GET",
|
||||
path: "/_matrix/client/v3/sync",
|
||||
search: "?timeout=0",
|
||||
});
|
||||
recording.setScenarioId("active-scenario");
|
||||
await recording.onExchange?.({
|
||||
context,
|
||||
request: {
|
||||
body: Buffer.alloc(0),
|
||||
headers: {},
|
||||
method: "GET",
|
||||
path: "/_matrix/client/v3/sync",
|
||||
search: "?timeout=0",
|
||||
},
|
||||
response: {
|
||||
body: Buffer.from(JSON.stringify({ next_batch: "sync-complete" })),
|
||||
headers: new Headers({ "content-type": "application/json" }),
|
||||
status: 200,
|
||||
},
|
||||
});
|
||||
|
||||
expect(recording.records().at(-1)?.scenarioId).toBe("active-scenario");
|
||||
});
|
||||
|
||||
it("does not share sync-token continuity across principals", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-principal-sync-test");
|
||||
const exchange = async (bearerToken: string, search: string, nextBatch: string) => {
|
||||
const request = {
|
||||
bearerToken,
|
||||
body: Buffer.alloc(0),
|
||||
headers: {},
|
||||
method: "GET",
|
||||
path: "/_matrix/client/v3/sync",
|
||||
search,
|
||||
};
|
||||
await recording.onExchange?.({
|
||||
context: recording.createExchangeContext?.(request),
|
||||
request,
|
||||
response: {
|
||||
body: Buffer.from(JSON.stringify({ next_batch: nextBatch })),
|
||||
headers: new Headers({ "content-type": "application/json" }),
|
||||
status: 200,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
await exchange("first", "?timeout=0", "shared-token");
|
||||
await exchange("second", "?timeout=0&since=shared-token", "second-token");
|
||||
|
||||
expect(recording.records().at(-1)?.sync).toMatchObject({
|
||||
continuity: false,
|
||||
since: "sync-unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("ends a retry chain at the first successful recovery", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-retry-boundary-test");
|
||||
const endpoint = `${recording.baseUrl}/_matrix/client/v3/rooms/!same:matrix.test/state/m.room.name`;
|
||||
const request = () =>
|
||||
fetch(endpoint, {
|
||||
body: JSON.stringify({ name: "same" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
await request();
|
||||
await request();
|
||||
await request();
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-retry-boundary-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-retry-boundary-test"]?.retries).toEqual([
|
||||
expect.objectContaining({ attempts: 2, statuses: [401, 200] }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("records exhausted retry chains without recovery", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget({ alwaysFailState: true });
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-exhausted-retry-test");
|
||||
const endpoint = `${recording.baseUrl}/_matrix/client/v3/rooms/!exhausted:matrix.test/state/m.room.name`;
|
||||
|
||||
await fetch(endpoint, { method: "POST" });
|
||||
await fetch(endpoint, { method: "POST" });
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-exhausted-retry-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-exhausted-retry-test"]?.retries).toEqual([
|
||||
expect.objectContaining({ attempts: 2, statuses: [401, 401] }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not infer retries across intervening operations", async () => {
|
||||
const targetBaseUrl = await startRecordingTarget();
|
||||
const recording = await startMatrixQaRecordingProxy({ targetBaseUrl });
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-independent-operation-test");
|
||||
const stateEndpoint = `${recording.baseUrl}/_matrix/client/v3/rooms/!same:matrix.test/state/m.room.name`;
|
||||
|
||||
await fetch(stateEndpoint, { method: "POST" });
|
||||
await fetch(`${recording.baseUrl}/_matrix/client/versions`);
|
||||
await fetch(stateEndpoint, { method: "POST" });
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-independent-operation-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-independent-operation-test"]?.retries).toEqual([]);
|
||||
});
|
||||
|
||||
it("records repeated retry chains for the same operation", async () => {
|
||||
let requestCount = 0;
|
||||
const server = createServer((_req, res) => {
|
||||
requestCount += 1;
|
||||
const status = requestCount % 2 === 1 ? 503 : 200;
|
||||
res.writeHead(status, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify(status === 200 ? {} : { errcode: "M_UNAVAILABLE" }));
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", () => resolve());
|
||||
});
|
||||
closeCallbacks.push(
|
||||
() =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
}),
|
||||
);
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("retry target did not bind");
|
||||
}
|
||||
const recording = await startMatrixQaRecordingProxy({
|
||||
targetBaseUrl: `http://127.0.0.1:${address.port}`,
|
||||
});
|
||||
closeCallbacks.push(() => recording.stop());
|
||||
recording.setScenarioId("matrix-repeated-retry-test");
|
||||
const endpoint = `${recording.baseUrl}/_matrix/client/v3/rooms/!same:matrix.test/state/m.room.name`;
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
await fetch(endpoint, { method: "POST" });
|
||||
}
|
||||
|
||||
const manifest = recording.buildManifest({
|
||||
requestedProfile: "test",
|
||||
scenarioIds: ["matrix-repeated-retry-test"],
|
||||
substrate: { id: "tuwunel", version: "test" },
|
||||
});
|
||||
expect(manifest.scenarios["matrix-repeated-retry-test"]?.retries).toEqual([
|
||||
expect.objectContaining({ attempts: 2, statuses: [503, 200] }),
|
||||
expect.objectContaining({ attempts: 2, statuses: [503, 200] }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,732 @@
|
||||
// Qa Matrix plugin module records redacted Matrix protocol behavior.
|
||||
import { createHash } from "node:crypto";
|
||||
import type { IncomingHttpHeaders } from "node:http";
|
||||
import {
|
||||
startMatrixQaFaultProxy,
|
||||
type MatrixQaFaultProxyExchange,
|
||||
type MatrixQaFaultProxyObserver,
|
||||
} from "./fault-proxy.js";
|
||||
|
||||
const MATRIX_QA_RECORDING_PROFILE = "matrix-qa-v1";
|
||||
const REDACTED_QUERY_VALUE = "[redacted]";
|
||||
|
||||
type MatrixQaStateFamily = "backup" | "device" | "key" | "media" | "sync-token";
|
||||
|
||||
type MatrixQaBodyShape =
|
||||
| { kind: "binary" }
|
||||
| { kind: "empty" }
|
||||
| { kind: "json"; fields: string[] }
|
||||
| { kind: "text" };
|
||||
|
||||
export type MatrixQaRecordedExchange = {
|
||||
categories: MatrixQaStateFamily[];
|
||||
request: {
|
||||
body: MatrixQaBodyShape;
|
||||
method: string;
|
||||
query: Record<string, string>;
|
||||
route: string;
|
||||
};
|
||||
response: {
|
||||
body: MatrixQaBodyShape;
|
||||
errcode?: string;
|
||||
status: number;
|
||||
};
|
||||
scenarioId: string;
|
||||
sequence: number;
|
||||
sync?: {
|
||||
continuity?: boolean;
|
||||
nextBatch?: string;
|
||||
since?: string;
|
||||
};
|
||||
};
|
||||
|
||||
type MatrixQaInternalRecordedExchange = MatrixQaRecordedExchange & {
|
||||
operationFingerprint: string;
|
||||
};
|
||||
|
||||
type MatrixQaRouteExpectation = {
|
||||
count: number;
|
||||
method: string;
|
||||
route: string;
|
||||
statuses: number[];
|
||||
};
|
||||
|
||||
type MatrixQaOrderingExpectation = {
|
||||
categories: MatrixQaStateFamily[];
|
||||
count: number;
|
||||
method: string;
|
||||
requestBody: MatrixQaBodyShape;
|
||||
responseBody: MatrixQaBodyShape;
|
||||
route: string;
|
||||
status: number;
|
||||
};
|
||||
|
||||
type MatrixQaScenarioRouteStateExpectation = {
|
||||
errors: Array<{
|
||||
errcode?: string;
|
||||
method: string;
|
||||
route: string;
|
||||
status: number;
|
||||
}>;
|
||||
ordering: MatrixQaOrderingExpectation[];
|
||||
retries: Array<{
|
||||
attempts: number;
|
||||
kind: "retry";
|
||||
method: string;
|
||||
route: string;
|
||||
statuses: number[];
|
||||
}>;
|
||||
routes: MatrixQaRouteExpectation[];
|
||||
state: Record<MatrixQaStateFamily, string[]>;
|
||||
syncTokens: {
|
||||
continuityObserved: boolean;
|
||||
incrementalRequests: number;
|
||||
initialRequests: number;
|
||||
responseTokens: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type MatrixQaRouteStateManifest = {
|
||||
generatedAt: string;
|
||||
phases: Record<string, MatrixQaScenarioRouteStateExpectation>;
|
||||
profile: {
|
||||
derivedFrom: "observed-request-response-traffic";
|
||||
id: typeof MATRIX_QA_RECORDING_PROFILE;
|
||||
};
|
||||
requestedProfile: string;
|
||||
scenarios: Record<string, MatrixQaScenarioRouteStateExpectation>;
|
||||
substrate: {
|
||||
id: string;
|
||||
version: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type MatrixQaRecordingProxy = MatrixQaFaultProxyObserver & {
|
||||
baseUrl: string;
|
||||
buildManifest(params: {
|
||||
generatedAt?: string;
|
||||
requestedProfile: string;
|
||||
scenarioIds: string[];
|
||||
substrate: MatrixQaRouteStateManifest["substrate"];
|
||||
}): MatrixQaRouteStateManifest;
|
||||
records(): MatrixQaRecordedExchange[];
|
||||
setScenarioId(scenarioId: string): void;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
function normalizeHeaderValue(value: string | string[] | undefined) {
|
||||
return Array.isArray(value) ? value.join(", ") : value;
|
||||
}
|
||||
|
||||
function hasJsonContentType(headers: Headers | IncomingHttpHeaders) {
|
||||
const raw = headers instanceof Headers ? headers.get("content-type") : headers["content-type"];
|
||||
return (
|
||||
normalizeHeaderValue(raw ?? undefined)
|
||||
?.toLowerCase()
|
||||
.includes("json") === true
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeJsonMapKey(key: string, prefix: string, route: string) {
|
||||
if (route.endsWith("/sync") && prefix === "rooms") {
|
||||
return key;
|
||||
}
|
||||
if (key.startsWith("!")) {
|
||||
return "{roomId}";
|
||||
}
|
||||
if (key.startsWith("@")) {
|
||||
return "{userId}";
|
||||
}
|
||||
if (key.startsWith("$")) {
|
||||
return "{eventId}";
|
||||
}
|
||||
if (route.endsWith("/keys/signatures/upload") && prefix === "") {
|
||||
return "{userId}";
|
||||
}
|
||||
if (route.endsWith("/keys/signatures/upload") && prefix === "{userId}") {
|
||||
return "{deviceOrKeyId}";
|
||||
}
|
||||
if (route.endsWith("/keys/upload") && prefix === "one_time_keys") {
|
||||
return "{keyId}";
|
||||
}
|
||||
if (/(?:^|\.)encrypted$/u.test(prefix)) {
|
||||
return "{keyId}";
|
||||
}
|
||||
if (prefix === "failures" && /\/keys\/(?:claim|query)$/u.test(route)) {
|
||||
return "{serverName}";
|
||||
}
|
||||
if (
|
||||
!route.endsWith("/keys/upload") &&
|
||||
/^(?:device_keys|master_keys|self_signing_keys|user_signing_keys)$/u.test(prefix)
|
||||
) {
|
||||
return "{userId}";
|
||||
}
|
||||
if (/^(?:device_keys|devices|messages|one_time_keys)\.\{userId\}$/u.test(prefix)) {
|
||||
return "{deviceId}";
|
||||
}
|
||||
if (!route.endsWith("/keys/upload") && prefix === "one_time_keys") {
|
||||
return "{userId}";
|
||||
}
|
||||
if (
|
||||
/^(?:device_keys\.)?\{userId\}\.\{deviceId\}\.(?:fallback_keys|keys|one_time_keys)$/u.test(
|
||||
prefix,
|
||||
) ||
|
||||
/^one_time_keys\.\{userId\}\.\{deviceId\}$/u.test(prefix) ||
|
||||
/^(?:ed25519|curve25519|signed_curve25519):/u.test(key)
|
||||
) {
|
||||
return "{keyId}";
|
||||
}
|
||||
if (/(?:^|\.)rooms$/u.test(prefix)) {
|
||||
return "{roomId}";
|
||||
}
|
||||
if (/(?:^|\.)rooms\.\{roomId\}\.sessions$/u.test(prefix)) {
|
||||
return "{sessionId}";
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
function collectJsonFields(value: unknown, route: string, prefix = "", depth = 0): string[] {
|
||||
if (depth >= 8 || value === null || typeof value !== "object") {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (value.length === 0) {
|
||||
return prefix ? [`${prefix}[]`] : [];
|
||||
}
|
||||
const arrayPrefix = prefix ? `${prefix}[]` : "[]";
|
||||
return [
|
||||
...new Set(value.flatMap((entry) => collectJsonFields(entry, route, arrayPrefix, depth + 1))),
|
||||
].toSorted();
|
||||
}
|
||||
const fixedDeviceKeysObject =
|
||||
prefix === "device_keys" &&
|
||||
["algorithms", "device_id", "keys", "signatures", "user_id"].some((key) => key in value);
|
||||
return Object.entries(value)
|
||||
.flatMap(([key, child]) => {
|
||||
const safeKey = fixedDeviceKeysObject ? key : normalizeJsonMapKey(key, prefix, route);
|
||||
const field = prefix ? `${prefix}.${safeKey}` : safeKey;
|
||||
if (
|
||||
/^(?:access_token|auth|body|ciphertext|content|file|formatted_body|password|recovery_key|session_data|token|unsigned)$/iu.test(
|
||||
key,
|
||||
)
|
||||
) {
|
||||
return [field];
|
||||
}
|
||||
const nested = collectJsonFields(child, route, field, depth + 1);
|
||||
return nested.length > 0 ? nested : [field];
|
||||
})
|
||||
.toSorted();
|
||||
}
|
||||
|
||||
function parseJsonBody(body: Buffer, headers: Headers | IncomingHttpHeaders): unknown {
|
||||
if (body.byteLength === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const text = body.toString("utf8");
|
||||
const firstNonWhitespace = text.trimStart()[0];
|
||||
if (!hasJsonContentType(headers) && firstNonWhitespace !== "[" && firstNonWhitespace !== "{") {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(text) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const MATRIX_QA_STATE_FIELD_MARKERS = new Set([
|
||||
"backup",
|
||||
"content_uri",
|
||||
"device_id",
|
||||
"device_keys",
|
||||
"device_lists",
|
||||
"device_one_time_keys_count",
|
||||
"device_unused_fallback_key_types",
|
||||
"mimetype",
|
||||
"one_time_key",
|
||||
"session_data",
|
||||
]);
|
||||
|
||||
function collectStateFieldMarkers(value: unknown, depth = 0): string[] {
|
||||
if (depth >= 8 || value === null || typeof value !== "object") {
|
||||
return [];
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return [...new Set(value.flatMap((entry) => collectStateFieldMarkers(entry, depth + 1)))];
|
||||
}
|
||||
const markers = new Set<string>();
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
const normalizedKey = key.toLowerCase();
|
||||
if (MATRIX_QA_STATE_FIELD_MARKERS.has(normalizedKey)) {
|
||||
markers.add(normalizedKey);
|
||||
}
|
||||
for (const marker of collectStateFieldMarkers(child, depth + 1)) {
|
||||
markers.add(marker);
|
||||
}
|
||||
}
|
||||
return [...markers];
|
||||
}
|
||||
|
||||
function extractStateFieldMarkers(body: Buffer, headers: Headers | IncomingHttpHeaders) {
|
||||
const parsed = parseJsonBody(body, headers);
|
||||
return parsed === undefined ? [] : collectStateFieldMarkers(parsed);
|
||||
}
|
||||
|
||||
function buildBodyShape(
|
||||
body: Buffer,
|
||||
headers: Headers | IncomingHttpHeaders,
|
||||
route: string,
|
||||
): MatrixQaBodyShape {
|
||||
if (body.byteLength === 0) {
|
||||
return { kind: "empty" };
|
||||
}
|
||||
const parsed = parseJsonBody(body, headers);
|
||||
if (parsed !== undefined) {
|
||||
return { kind: "json", fields: collectJsonFields(parsed, route) };
|
||||
}
|
||||
const contentType =
|
||||
headers instanceof Headers
|
||||
? headers.get("content-type")
|
||||
: normalizeHeaderValue(headers["content-type"]);
|
||||
return contentType?.toLowerCase().startsWith("text/") ? { kind: "text" } : { kind: "binary" };
|
||||
}
|
||||
|
||||
function normalizeMatrixIdSegment(segment: string) {
|
||||
let decoded: string;
|
||||
try {
|
||||
decoded = decodeURIComponent(segment);
|
||||
} catch {
|
||||
return segment;
|
||||
}
|
||||
if (decoded.startsWith("!")) {
|
||||
return "{roomId}";
|
||||
}
|
||||
if (decoded.startsWith("@")) {
|
||||
return "{userId}";
|
||||
}
|
||||
if (decoded.startsWith("$")) {
|
||||
return "{eventId}";
|
||||
}
|
||||
if (decoded.startsWith("#")) {
|
||||
return "{roomAlias}";
|
||||
}
|
||||
return segment;
|
||||
}
|
||||
|
||||
export function normalizeMatrixQaRoute(pathname: string) {
|
||||
const segments = pathname.split("/");
|
||||
for (let index = 0; index < segments.length; index += 1) {
|
||||
const previous = segments[index - 1];
|
||||
const beforePrevious = segments[index - 2];
|
||||
if (previous === "rooms") {
|
||||
segments[index] = "{roomId}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "profile" || previous === "user") {
|
||||
segments[index] = "{userId}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "filter") {
|
||||
segments[index] = "{filterId}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "join") {
|
||||
segments[index] = normalizeMatrixIdSegment(segments[index] ?? "");
|
||||
continue;
|
||||
}
|
||||
if (previous === "devices") {
|
||||
segments[index] = "{deviceId}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "redact") {
|
||||
segments[index] = "{eventId}";
|
||||
continue;
|
||||
}
|
||||
if (beforePrevious === "send" || beforePrevious === "redact") {
|
||||
segments[index] = "{transactionId}";
|
||||
continue;
|
||||
}
|
||||
if (beforePrevious === "sendToDevice") {
|
||||
segments[index] = "{transactionId}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "version" && beforePrevious === "room_keys") {
|
||||
segments[index] = "{backupVersion}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "keys" && beforePrevious === "room_keys") {
|
||||
segments[index] = "{roomId}";
|
||||
continue;
|
||||
}
|
||||
if (segments[index - 2] === "keys" && segments[index - 3] === "room_keys") {
|
||||
segments[index] = "{sessionId}";
|
||||
continue;
|
||||
}
|
||||
if (segments[index - 2] === "state" && segments[index] !== "") {
|
||||
segments[index] = "{stateKey}";
|
||||
continue;
|
||||
}
|
||||
if (previous === "account_data" && segments[index]?.startsWith("m.secret_storage.key.")) {
|
||||
segments[index] = "m.secret_storage.key.{keyId}";
|
||||
continue;
|
||||
}
|
||||
const mediaActionIndex = segments.findIndex(
|
||||
(segment) => segment === "download" || segment === "thumbnail",
|
||||
);
|
||||
if (mediaActionIndex >= 0 && index === mediaActionIndex + 2) {
|
||||
segments[index] = "{mediaId}";
|
||||
segments[index - 1] = "{serverName}";
|
||||
continue;
|
||||
}
|
||||
if (mediaActionIndex >= 0 && index === mediaActionIndex + 3) {
|
||||
segments[index] = "{filename}";
|
||||
continue;
|
||||
}
|
||||
segments[index] = normalizeMatrixIdSegment(segments[index] ?? "");
|
||||
}
|
||||
return segments.join("/");
|
||||
}
|
||||
|
||||
function buildRedactedQuery(search: string, syncTokens: Map<string, string>) {
|
||||
const result: Record<string, string> = {};
|
||||
for (const [key, value] of new URLSearchParams(search)) {
|
||||
if (key === "since") {
|
||||
result[key] = syncTokens.get(value) ?? "sync-unknown";
|
||||
} else if (key === "timeout" || key === "full_state" || key === "set_presence") {
|
||||
result[key] = value;
|
||||
} else {
|
||||
result[key] = REDACTED_QUERY_VALUE;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function resolveStateFamilies(params: {
|
||||
requestFields: string[];
|
||||
responseFields: string[];
|
||||
route: string;
|
||||
}) {
|
||||
const fields = [...params.requestFields, ...params.responseFields].join(" ").toLowerCase();
|
||||
const route = params.route.toLowerCase();
|
||||
const families = new Set<MatrixQaStateFamily>();
|
||||
if (route.includes("/sync")) {
|
||||
families.add("sync-token");
|
||||
}
|
||||
if (route.includes("/room_keys/") || fields.includes("backup")) {
|
||||
families.add("backup");
|
||||
}
|
||||
if (route.includes("/account_data/m.megolm_backup.")) {
|
||||
families.add("backup");
|
||||
}
|
||||
if (
|
||||
route.includes("/keys/") ||
|
||||
route.includes("/sendtodevice/") ||
|
||||
route.includes("/account_data/m.cross_signing.") ||
|
||||
route.includes("/account_data/m.secret_storage.") ||
|
||||
fields.includes("one_time_key") ||
|
||||
fields.includes("device_one_time_keys_count") ||
|
||||
fields.includes("device_unused_fallback_key_types") ||
|
||||
fields.includes("device_keys") ||
|
||||
fields.includes("session_data")
|
||||
) {
|
||||
families.add("key");
|
||||
}
|
||||
if (
|
||||
route.includes("/devices") ||
|
||||
fields.includes("device_id") ||
|
||||
fields.includes("device_lists")
|
||||
) {
|
||||
families.add("device");
|
||||
}
|
||||
if (route.includes("/media/") || fields.includes("content_uri") || fields.includes("mimetype")) {
|
||||
families.add("media");
|
||||
}
|
||||
return [...families].toSorted();
|
||||
}
|
||||
|
||||
function buildExpectation(
|
||||
records: MatrixQaInternalRecordedExchange[],
|
||||
): MatrixQaScenarioRouteStateExpectation {
|
||||
const orderedRecords = records.toSorted((left, right) => left.sequence - right.sequence);
|
||||
const routeGroups = new Map<string, MatrixQaRouteExpectation>();
|
||||
const ordering: MatrixQaOrderingExpectation[] = [];
|
||||
const state = {
|
||||
backup: new Set<string>(),
|
||||
device: new Set<string>(),
|
||||
key: new Set<string>(),
|
||||
media: new Set<string>(),
|
||||
"sync-token": new Set<string>(),
|
||||
} satisfies Record<MatrixQaStateFamily, Set<string>>;
|
||||
|
||||
for (const record of orderedRecords) {
|
||||
const routeKey = `${record.request.method} ${record.request.route}`;
|
||||
const route = routeGroups.get(routeKey) ?? {
|
||||
count: 0,
|
||||
method: record.request.method,
|
||||
route: record.request.route,
|
||||
statuses: [],
|
||||
};
|
||||
route.count += 1;
|
||||
if (!route.statuses.includes(record.response.status)) {
|
||||
route.statuses.push(record.response.status);
|
||||
route.statuses.sort((left, right) => left - right);
|
||||
}
|
||||
routeGroups.set(routeKey, route);
|
||||
for (const category of record.categories) {
|
||||
state[category].add(record.request.route);
|
||||
}
|
||||
const previous = ordering.at(-1);
|
||||
if (
|
||||
previous?.method === record.request.method &&
|
||||
previous.route === record.request.route &&
|
||||
previous.status === record.response.status &&
|
||||
JSON.stringify(previous.requestBody) === JSON.stringify(record.request.body) &&
|
||||
JSON.stringify(previous.responseBody) === JSON.stringify(record.response.body) &&
|
||||
previous.categories.join("\0") === record.categories.join("\0")
|
||||
) {
|
||||
previous.count += 1;
|
||||
} else {
|
||||
ordering.push({
|
||||
categories: record.categories,
|
||||
count: 1,
|
||||
method: record.request.method,
|
||||
requestBody: record.request.body,
|
||||
responseBody: record.response.body,
|
||||
route: record.request.route,
|
||||
status: record.response.status,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const routes = [...routeGroups.values()].toSorted((left, right) =>
|
||||
`${left.method} ${left.route}`.localeCompare(`${right.method} ${right.route}`),
|
||||
);
|
||||
const retries: MatrixQaScenarioRouteStateExpectation["retries"] = [];
|
||||
const adjacentOperationRuns: MatrixQaInternalRecordedExchange[][] = [];
|
||||
for (const record of orderedRecords) {
|
||||
const currentRun = adjacentOperationRuns.at(-1);
|
||||
if (currentRun?.[0]?.operationFingerprint === record.operationFingerprint) {
|
||||
currentRun.push(record);
|
||||
} else {
|
||||
adjacentOperationRuns.push([record]);
|
||||
}
|
||||
}
|
||||
for (const attempts of adjacentOperationRuns) {
|
||||
if (attempts[0]?.request.route.endsWith("/sync")) {
|
||||
continue;
|
||||
}
|
||||
for (let index = 0; index < attempts.length; index += 1) {
|
||||
const first = attempts[index];
|
||||
if (!first || first.response.status < 400) {
|
||||
continue;
|
||||
}
|
||||
const recoveryOffset = attempts
|
||||
.slice(index + 1)
|
||||
.findIndex((attempt) => attempt.response.status < 400);
|
||||
const retryEndIndex = recoveryOffset < 0 ? attempts.length : index + recoveryOffset + 2;
|
||||
const retryAttempts = attempts.slice(index, retryEndIndex);
|
||||
if (retryAttempts.length < 2) {
|
||||
continue;
|
||||
}
|
||||
retries.push({
|
||||
attempts: retryAttempts.length,
|
||||
kind: "retry",
|
||||
method: first.request.method,
|
||||
route: first.request.route,
|
||||
statuses: retryAttempts.map((attempt) => attempt.response.status),
|
||||
});
|
||||
index = retryEndIndex - 1;
|
||||
}
|
||||
}
|
||||
const incrementalSyncRecords = orderedRecords.filter(
|
||||
(record) => record.sync?.since !== undefined,
|
||||
);
|
||||
return {
|
||||
errors: orderedRecords
|
||||
.filter((record) => record.response.status >= 400)
|
||||
.map((record) => {
|
||||
const error: MatrixQaScenarioRouteStateExpectation["errors"][number] = {
|
||||
method: record.request.method,
|
||||
route: record.request.route,
|
||||
status: record.response.status,
|
||||
};
|
||||
if (record.response.errcode) {
|
||||
error.errcode = record.response.errcode;
|
||||
}
|
||||
return error;
|
||||
}),
|
||||
ordering,
|
||||
retries,
|
||||
routes,
|
||||
state: Object.fromEntries(
|
||||
Object.entries(state).map(([key, values]) => [key, [...values].toSorted()]),
|
||||
) as Record<MatrixQaStateFamily, string[]>,
|
||||
syncTokens: {
|
||||
continuityObserved:
|
||||
incrementalSyncRecords.length > 0 &&
|
||||
incrementalSyncRecords.every((record) => record.sync?.continuity === true),
|
||||
incrementalRequests: incrementalSyncRecords.length,
|
||||
initialRequests: orderedRecords.filter(
|
||||
(record) => record.categories.includes("sync-token") && record.sync?.since === undefined,
|
||||
).length,
|
||||
responseTokens: orderedRecords.filter((record) => record.sync?.nextBatch !== undefined)
|
||||
.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function extractErrcode(body: Buffer, headers: Headers) {
|
||||
const parsed = parseJsonBody(body, headers);
|
||||
if (typeof parsed !== "object" || parsed === null) {
|
||||
return undefined;
|
||||
}
|
||||
const errcode = (parsed as { errcode?: unknown }).errcode;
|
||||
return typeof errcode === "string" ? errcode : undefined;
|
||||
}
|
||||
|
||||
function extractNextBatch(body: Buffer, headers: Headers) {
|
||||
const parsed = parseJsonBody(body, headers);
|
||||
if (typeof parsed !== "object" || parsed === null) {
|
||||
return undefined;
|
||||
}
|
||||
const nextBatch = (parsed as { next_batch?: unknown }).next_batch;
|
||||
return typeof nextBatch === "string" ? nextBatch : undefined;
|
||||
}
|
||||
|
||||
export async function startMatrixQaRecordingProxy(params: {
|
||||
targetBaseUrl: string;
|
||||
}): Promise<MatrixQaRecordingProxy> {
|
||||
let scenarioId = "setup";
|
||||
let sequence = 0;
|
||||
const records: MatrixQaInternalRecordedExchange[] = [];
|
||||
const syncTokensByPrincipal = new Map<string, Map<string, string>>();
|
||||
const observer: Required<MatrixQaFaultProxyObserver> = {
|
||||
createExchangeContext: () => ({ scenarioId, sequence: ++sequence }),
|
||||
onExchange(exchange: MatrixQaFaultProxyExchange) {
|
||||
recordExchange(exchange);
|
||||
},
|
||||
};
|
||||
const recordExchange = (exchange: MatrixQaFaultProxyExchange) => {
|
||||
const context =
|
||||
typeof exchange.context === "object" && exchange.context !== null
|
||||
? (exchange.context as { scenarioId?: unknown; sequence?: unknown })
|
||||
: undefined;
|
||||
const exchangeSequence = typeof context?.sequence === "number" ? context.sequence : ++sequence;
|
||||
const route = normalizeMatrixQaRoute(exchange.request.path);
|
||||
const exchangeScenarioId = route.endsWith("/sync")
|
||||
? scenarioId
|
||||
: typeof context?.scenarioId === "string"
|
||||
? context.scenarioId
|
||||
: "unattributed";
|
||||
const requestBody = buildBodyShape(exchange.request.body, exchange.request.headers, route);
|
||||
const responseBody = buildBodyShape(exchange.response.body, exchange.response.headers, route);
|
||||
const requestFields = extractStateFieldMarkers(exchange.request.body, exchange.request.headers);
|
||||
const responseFields = extractStateFieldMarkers(
|
||||
exchange.response.body,
|
||||
exchange.response.headers,
|
||||
);
|
||||
const syncPrincipal = exchange.request.bearerToken ?? "anonymous";
|
||||
const syncTokens = syncTokensByPrincipal.get(syncPrincipal) ?? new Map<string, string>();
|
||||
syncTokensByPrincipal.set(syncPrincipal, syncTokens);
|
||||
const sinceRaw = new URLSearchParams(exchange.request.search).get("since") ?? undefined;
|
||||
const since = sinceRaw ? (syncTokens.get(sinceRaw) ?? "sync-unknown") : undefined;
|
||||
const nextBatch = extractNextBatch(exchange.response.body, exchange.response.headers);
|
||||
if (nextBatch && !syncTokens.has(nextBatch)) {
|
||||
syncTokens.set(nextBatch, `sync-${syncTokens.size + 1}`);
|
||||
}
|
||||
const nextBatchAlias = nextBatch ? syncTokens.get(nextBatch) : undefined;
|
||||
const responseErrcode = extractErrcode(exchange.response.body, exchange.response.headers);
|
||||
const operationFingerprint = createHash("sha256")
|
||||
.update(exchange.request.method)
|
||||
.update("\0")
|
||||
.update(exchange.request.path)
|
||||
.update("\0")
|
||||
.update(exchange.request.search)
|
||||
.update("\0")
|
||||
.update(exchange.request.body)
|
||||
.update("\0")
|
||||
.update(exchange.request.bearerToken ?? "anonymous")
|
||||
.digest("hex");
|
||||
records.push({
|
||||
categories: resolveStateFamilies({ requestFields, responseFields, route }),
|
||||
request: {
|
||||
body: requestBody,
|
||||
method: exchange.request.method,
|
||||
query: buildRedactedQuery(exchange.request.search, syncTokens),
|
||||
route,
|
||||
},
|
||||
response: {
|
||||
body: responseBody,
|
||||
...(responseErrcode ? { errcode: responseErrcode } : {}),
|
||||
status: exchange.response.status,
|
||||
},
|
||||
scenarioId: exchangeScenarioId,
|
||||
sequence: exchangeSequence,
|
||||
operationFingerprint,
|
||||
...(route.endsWith("/sync")
|
||||
? {
|
||||
sync: {
|
||||
...(since ? { since } : {}),
|
||||
...(nextBatchAlias ? { nextBatch: nextBatchAlias } : {}),
|
||||
...(since && nextBatchAlias ? { continuity: since !== "sync-unknown" } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
};
|
||||
const proxy = await startMatrixQaFaultProxy({
|
||||
targetBaseUrl: params.targetBaseUrl,
|
||||
rules: [],
|
||||
...observer,
|
||||
});
|
||||
|
||||
return {
|
||||
baseUrl: proxy.baseUrl,
|
||||
...observer,
|
||||
buildManifest({ generatedAt, requestedProfile, scenarioIds, substrate }) {
|
||||
const selectedIds = new Set(scenarioIds);
|
||||
const byScenario = new Map<string, MatrixQaInternalRecordedExchange[]>();
|
||||
for (const record of records) {
|
||||
const entries = byScenario.get(record.scenarioId) ?? [];
|
||||
entries.push(record);
|
||||
byScenario.set(record.scenarioId, entries);
|
||||
}
|
||||
const scenarios = Object.fromEntries(
|
||||
scenarioIds.map((id) => [id, buildExpectation(byScenario.get(id) ?? [])]),
|
||||
);
|
||||
const phases = Object.fromEntries(
|
||||
[...byScenario.entries()]
|
||||
.filter(([id]) => !selectedIds.has(id))
|
||||
.toSorted(([left], [right]) => left.localeCompare(right))
|
||||
.map(([id, entries]) => [id, buildExpectation(entries)]),
|
||||
);
|
||||
return {
|
||||
generatedAt: generatedAt ?? new Date().toISOString(),
|
||||
phases,
|
||||
profile: {
|
||||
derivedFrom: "observed-request-response-traffic",
|
||||
id: MATRIX_QA_RECORDING_PROFILE,
|
||||
},
|
||||
requestedProfile,
|
||||
scenarios,
|
||||
substrate,
|
||||
};
|
||||
},
|
||||
records: () =>
|
||||
structuredClone(
|
||||
records
|
||||
.toSorted((left, right) => left.sequence - right.sequence)
|
||||
.map(({ operationFingerprint: _operationFingerprint, ...record }) => record),
|
||||
),
|
||||
setScenarioId(nextScenarioId) {
|
||||
scenarioId = nextScenarioId;
|
||||
},
|
||||
stop: () => proxy.stop(),
|
||||
};
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
buildExpectation,
|
||||
normalizeMatrixQaRoute,
|
||||
};
|
||||
export { testing as __testing };
|
||||
Reference in New Issue
Block a user